package services

import (
	"encoding/hex"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"

	"wa-assistant/backend/database"
	"wa-assistant/backend/models"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// Konstanta crawler. Disengaja konservatif: hemat resource VPS & sopan ke situs target.
const (
	crawlUA         = "ChatLoopBot/1.0 (+https://chatloop.id; pelatihan AI customer service)"
	crawlTimeout    = 15 * time.Second
	crawlDelay      = 400 * time.Millisecond // jeda antar-halaman (politeness)
	maxPageBytes    = 3 << 20                // batas 3MB/halaman agar tidak boros memori
	maxSitemapDepth = 2                      // sitemap index boleh bersarang sampai 2 level
	chunkSize       = 800                    // ukuran chunk (rune) saat melatih
	chunkOverlap    = 100                    // tumpang-tindih antar-chunk agar konteks tak terpotong
	minContentChars = 200                    // halaman di bawah ini dianggap "tipis" (tak direkomendasi)
)

// shouldSkipURL: URL yang tak pernah berguna untuk training (aset & halaman fungsional).
// Halaman ini tidak di-fetch sama sekali agar daftar bersih & hemat waktu.
func shouldSkipURL(rawurl string) bool {
	u, err := url.Parse(rawurl)
	if err != nil {
		return true
	}
	p := strings.ToLower(u.Path)
	for _, route := range []string{"/chat", "/notifications", "/wallet", "/orders", "/topup", "/withdraw"} {
		if p == route || strings.HasPrefix(p, route+"/") {
			return true
		}
	}
	for _, ext := range []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico",
		".css", ".js", ".pdf", ".zip", ".rar", ".mp4", ".mp3", ".woff", ".woff2", ".ttf", ".xml", ".json"} {
		if strings.HasSuffix(p, ext) {
			return true
		}
	}
	for _, kw := range []string{
		"/cart", "/keranjang", "/checkout", "/login", "/signin", "/masuk", "/logout",
		"/register", "/signup", "/daftar-akun", "/my-account", "/akun", "/account",
		"/wp-admin", "/wp-login", "/xmlrpc", "/feed", "/rss", "/wishlist", "/compare",
	} {
		if strings.Contains(p, kw) {
			return true
		}
	}
	return false
}

// isLowValueURL: halaman yang biasanya tak penting untuk CS (tetap di-list, tapi tak direkomendasi).
func isLowValueURL(rawurl string) bool {
	u, err := url.Parse(rawurl)
	if err != nil {
		return true
	}
	p := strings.ToLower(u.Path)
	// Hanya halaman yang HAMPIR PASTI tak berguna untuk CS. Catatan: /category /tag TIDAK
	// dimasukkan—pada situs e-commerce itu justru halaman katalog produk (berisi harga).
	// Biar kekayaan konten (char_count) + AI saat training yang memutuskan halaman archive.
	for _, kw := range []string{
		"privacy", "privasi",
		"/author/", "/search", "/cari", "/page/", "/404", "/disclaimer",
	} {
		if strings.Contains(p, kw) {
			return true
		}
	}
	return false
}

// isListingHubURL: halaman hub/indeks daftar (mis. /category, /shop, /blog) TANPA item spesifik.
// Biasanya cuma navigasi tanpa info CS, jadi jangan auto-direkomendasi (tetap bisa dipilih manual).
// /category/buket (ada slug) TIDAK kena—itu halaman katalog berisi produk.
func isListingHubURL(rawurl string) bool {
	u, err := url.Parse(rawurl)
	if err != nil {
		return false
	}
	seg := strings.Split(strings.Trim(strings.ToLower(u.Path), "/"), "/")
	if len(seg) != 1 || seg[0] == "" {
		return false
	}
	switch seg[0] {
	case "category", "categories", "kategori", "tag", "tags", "blog", "artikel", "articles",
		"shop", "store", "toko", "product", "products", "produk", "koleksi", "collection", "collections":
		return true
	}
	return false
}

var crawlClient = newCrawlHTTPClient()

// RunCrawl seeds a bounded Colly crawl from the root and sitemap.
// Each page keeps its own content, provenance, recommendation, and fetch errors.
func RunCrawl(jobID uint, maxPages int) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("Crawl job #%d panic: %v", jobID, r)
			finishCrawl(jobID, 0, fmt.Sprintf("panic: %v", r))
		}
	}()

	var job models.CrawlJob
	if database.DB.First(&job, jobID).Error != nil {
		return
	}
	database.DB.Model(&job).Update("status", "crawling")

	base, err := url.Parse(strings.TrimSpace(job.RootURL))
	if err != nil || (base.Scheme != "http" && base.Scheme != "https") || base.Host == "" || base.User != nil {
		finishCrawl(jobID, 0, "URL tidak valid (harus diawali http:// atau https://)")
		return
	}
	host := canonicalHost(base.Host)
	database.DB.Model(&job).Update("domain", host)

	if maxPages <= 0 {
		maxPages = 50
	}

	pages, readable := 0, 0
	savePage := func(pageURL, title, text string, ferr error) {
		p := models.CrawlPage{JobID: jobID, AgentID: job.AgentID, URL: pageURL, Title: title}
		if ferr != nil {
			p.Status, p.Error = "failed", ferr.Error()
		} else {
			readable++
			p.Status, p.Content, p.CharCount = "crawled", text, len([]rune(text))
			sc := ScorePageForCSTraining(pageURL, title, text)
			p.Recommended = sc.Recommended
			p.RecommendScore = sc.Score
			p.RecommendTier = sc.Tier
			p.RecommendReason = strings.Join(sc.Reasons, " · ")
		}
		database.DB.Create(&p)
		pages++
	}

	// Sitemap supplies extra seeds; following links still finds pages missing from it.
	seeds := append([]string{job.RootURL}, fetchSitemapURLs(base, host)...)
	if err := crawlWebsite(job.RootURL, seeds, maxPages, crawlClient, crawlDelay, savePage); err != nil {
		finishCrawl(jobID, pages, err.Error())
		return
	}

	refineJobRecommendations(jobID)
	if readable == 0 {
		finishCrawl(jobID, pages, "Belum ada isi halaman yang berhasil dibaca. Periksa keterangan pada halaman, lalu ambil ulang website.")
		return
	}
	finishCrawl(jobID, pages, "")
}

// refineJobRecommendations: re-rank relatif dalam satu job (promosi top relatif,
// cap top 30) agar rekomendasi tidak kosong di situs tipis / tidak banjir di situs besar.
func refineJobRecommendations(jobID uint) {
	var pages []models.CrawlPage
	database.DB.Where("job_id = ? AND status = ?", jobID, "crawled").Find(&pages)
	if len(pages) == 0 {
		return
	}
	scores := make([]PageTrainScore, len(pages))
	for i := range pages {
		sc := ScorePageForCSTraining(pages[i].URL, pages[i].Title, pages[i].Content)
		if pages[i].Content == "" && pages[i].RecommendScore > 0 {
			reasons := []string{}
			if pages[i].RecommendReason != "" {
				reasons = strings.Split(pages[i].RecommendReason, " · ")
			}
			sc = PageTrainScore{
				Score: pages[i].RecommendScore, Recommended: pages[i].Recommended,
				Tier: pages[i].RecommendTier, Reasons: reasons,
			}
		}
		scores[i] = sc
	}
	final := RankAndSelectRecommended(scores)
	for i, sc := range final {
		database.DB.Model(&pages[i]).Updates(map[string]any{
			"recommended":      sc.Recommended,
			"recommend_score":  sc.Score,
			"recommend_tier":   sc.Tier,
			"recommend_reason": strings.Join(sc.Reasons, " · "),
		})
	}
}

func finishCrawl(jobID uint, pages int, errMsg string) {
	now := time.Now()
	status := "done"
	if errMsg != "" {
		status = "failed"
	}
	database.DB.Model(&models.CrawlJob{}).Where("id = ?", jobID).Updates(map[string]any{
		"status": status, "pages_found": pages, "error": errMsg, "finished_at": &now,
	})
	var rec int64
	database.DB.Model(&models.CrawlPage{}).Where("job_id = ? AND recommended = ?", jobID, true).Count(&rec)
	log.Printf("Crawl job #%d selesai: %d halaman (%d direkomendasi), status=%s %s", jobID, pages, rec, status, errMsg)
}

// extractTitleText uses goquery selectors, preserving paragraphs and list boundaries.
func extractTitleText(root *html.Node) (title, text string) {
	doc := goquery.NewDocumentFromNode(root)
	title = collapseSpaces(doc.Find("title").First().Text())
	if title == "" {
		title = collapseSpaces(doc.Find("h1").First().Text())
	}
	doc.Find(`script, style, noscript, svg, iframe, nav, aside, body > header, body > footer, [hidden], [aria-hidden="true"], [role="navigation"], [role="banner"], [role="contentinfo"], .cookie-banner, .cookie-notice, .breadcrumbs`).Remove()
	source := doc.Find("main, [role=main]").First()
	if source.Length() == 0 {
		candidates := doc.Find("article, .entry-content, .post-content, .article-content, .post-body")
		best := 0
		candidates.Each(func(_ int, candidate *goquery.Selection) {
			if n := len(candidate.Text()); n > best {
				source, best = candidate, n
			}
		})
	}
	if source.Length() == 0 {
		source = doc.Find("body")
	}
	if source.Length() == 0 {
		source = doc.Selection
	}
	var out strings.Builder
	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode {
			// Proteksi email Cloudflare: alamat aslinya ada di atribut data-cfemail, sedangkan
			// teks elemennya hanya placeholder "[email protected]". Pakai alamat aslinya supaya
			// pengetahuan tidak memuat alamat palsu.
			if alamat, ok := cloudflareEmailAddress(n); ok {
				out.WriteString(alamat)
				out.WriteByte(' ')
				return // anak elemen ini hanya placeholder, jangan ditelusuri
			}
		}
		if n.Type == html.TextNode {
			// Spasi non-breaking (&#160;) dari templat ikut terbawa HTML; normalkan supaya
			// teks hasil crawl bersih dan placeholder email bisa dikenali.
			teks := strings.ReplaceAll(n.Data, "\u00a0", " ")
			for _, placeholder := range []string{"[email protected]", "[email protected]"} {
				if strings.Contains(teks, placeholder) {
					// Placeholder tanpa data-cfemail: jangan diteruskan ke teks hasil crawl.
					teks = strings.ReplaceAll(teks, placeholder, " ")
				}
			}
			out.WriteString(teks)
			out.WriteByte(' ')
		}
		block := n.Type == html.ElementNode && strings.Contains(" p div br li h1 h2 h3 h4 tr section article dt dd ", " "+n.Data+" ")
		if block {
			out.WriteByte('\n')
		}
		for child := n.FirstChild; child != nil; child = child.NextSibling {
			walk(child)
		}
		if block {
			out.WriteByte('\n')
		}
	}
	for _, node := range source.Nodes {
		walk(node)
	}
	var lines []string
	for _, line := range strings.Split(out.String(), "\n") {
		if line = collapseSpaces(line); line != "" {
			lines = append(lines, line)
		}
	}
	return title, strings.Join(lines, "\n")
}

// cloudflareEmailAddress mengembalikan alamat email asli dari atribut data-cfemail milik proteksi
// email Cloudflare. Formatnya heksadesimal: byte pertama kunci, byte sisanya XOR dengan kunci itu.
func cloudflareEmailAddress(n *html.Node) (string, bool) {
	for _, attr := range n.Attr {
		if attr.Key != "data-cfemail" || attr.Val == "" {
			continue
		}
		raw, err := hex.DecodeString(attr.Val)
		if err != nil || len(raw) < 2 {
			return "", false
		}
		kunci := raw[0]
		var b strings.Builder
		for _, c := range raw[1:] {
			b.WriteByte(c ^ kunci)
		}
		alamat := strings.TrimSpace(b.String())
		if strings.Contains(alamat, "@") && !strings.ContainsAny(alamat, " \t\r\n") {
			return alamat, true
		}
	}
	return "", false
}

func extractLinks(root *html.Node, base *url.URL) []string {
	var out []string
	var walk func(n *html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "a" {
			for _, a := range n.Attr {
				if a.Key == "href" {
					if u, err := base.Parse(strings.TrimSpace(a.Val)); err == nil {
						out = append(out, u.String())
					}
				}
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(root)
	return out
}

// --- Sitemap ---

type sitemapDoc struct {
	URLs     []sitemapLoc `xml:"url"`
	Sitemaps []sitemapLoc `xml:"sitemap"`
}
type sitemapLoc struct {
	Loc string `xml:"loc"`
}

// fetchSitemapURLs membaca /sitemap.xml (mendukung sitemap index bersarang) & mengembalikan
// daftar URL same-domain (sudah dedup). Kosong bila tidak ada sitemap.
func fetchSitemapURLs(base *url.URL, host string) []string {
	smURL := base.Scheme + "://" + base.Host + "/sitemap.xml"
	seen := map[string]bool{}
	var out []string
	collect := func(u string) {
		n := normalizeURL(u)
		if n != "" && canonicalHost(hostOf(n)) == host && !seen[n] {
			seen[n] = true
			out = append(out, n)
		}
	}
	fetchSitemapRecursive(smURL, collect, 0)
	return out
}

func fetchSitemapRecursive(smURL string, collect func(string), depth int) {
	remaining := 20
	fetchSitemapBounded(smURL, collect, depth, &remaining, map[string]bool{})
}

func fetchSitemapBounded(smURL string, collect func(string), depth int, remaining *int, seen map[string]bool) {
	if *remaining <= 0 || seen[smURL] {
		return
	}
	*remaining--
	seen[smURL] = true
	if depth > maxSitemapDepth {
		return
	}
	req, err := http.NewRequest(http.MethodGet, smURL, nil)
	if err != nil {
		return
	}
	req.Header.Set("User-Agent", crawlUA)
	resp, err := crawlClient.Do(req)
	if err != nil || resp.StatusCode != http.StatusOK {
		if resp != nil {
			resp.Body.Close()
		}
		return
	}
	defer resp.Body.Close()
	var doc sitemapDoc
	if xml.NewDecoder(io.LimitReader(resp.Body, maxPageBytes)).Decode(&doc) != nil {
		return
	}
	for _, u := range doc.URLs {
		if loc := strings.TrimSpace(u.Loc); loc != "" {
			collect(loc)
		}
	}
	for i, s := range doc.Sitemaps {
		if i >= 20 {
			break
		}
		if loc := strings.TrimSpace(s.Loc); loc != "" && canonicalHost(hostOf(loc)) == canonicalHost(hostOf(smURL)) {
			fetchSitemapBounded(loc, collect, depth+1, remaining, seen)
		}
	}
}

// --- helper URL & teks ---

// normalizeURL removes fragments and tracking parameters but preserves content IDs and language.
func normalizeURL(raw string) string {
	u, err := url.Parse(strings.TrimSpace(raw))
	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
		return ""
	}
	u.Fragment = ""
	query := u.Query()
	for key := range query {
		if strings.HasPrefix(strings.ToLower(key), "utm_") || key == "fbclid" || key == "gclid" {
			query.Del(key)
		}
	}
	u.RawQuery = query.Encode()
	s := u.String()
	return strings.TrimSuffix(s, "/")
}

func hostOf(raw string) string {
	u, err := url.Parse(raw)
	if err != nil {
		return ""
	}
	return u.Host
}

// canonicalHost menyeragamkan host (buang "www." dan port) supaya www & non-www dianggap sama.
func canonicalHost(h string) string {
	h = strings.ToLower(h)
	if i := strings.IndexByte(h, ':'); i >= 0 {
		h = h[:i]
	}
	return strings.TrimPrefix(h, "www.")
}

// collapseSpaces merapatkan semua whitespace beruntun jadi satu spasi.
func collapseSpaces(s string) string {
	return strings.Join(strings.Fields(s), " ")
}

// ChunkText memecah teks jadi potongan ~chunkSize rune, diakhiri di batas kalimat
// (titik + spasi, baris baru) agar tidak terpotong di tengah kata.
func ChunkText(text string) []string {
	text = strings.TrimSpace(text)
	if text == "" {
		return nil
	}
	runes := []rune(text)
	if len(runes) <= chunkSize {
		return []string{text}
	}
	var chunks []string
	start := 0
	for start < len(runes) {
		end := start + chunkSize
		if end >= len(runes) {
			end = len(runes)
		} else {
			// Mundur cari batas kalimat: ". ", "? ", "! ", atau "\n" dalam 200 karakter terakhir
			searchStart := end - 200
			if searchStart < start {
				searchStart = start
			}
			segment := string(runes[searchStart:end])
			best := -1 // posisi terbaik untuk memotong
			for _, sep := range []string{". ", "? ", "! ", ".\n", "\n"} {
				if idx := strings.LastIndex(segment, sep); idx >= 0 {
					cut := searchStart + idx + len(sep) - 1 // setelah separator
					if cut > best && cut >= start+chunkSize/2 {
						best = cut
					}
				}
			}
			if best >= 0 {
				end = best
			} else {
				// Fallback: potong di spasi terakhir
				if idx := strings.LastIndex(segment, " "); idx >= 0 {
					cut := searchStart + idx
					if cut >= start+chunkSize/2 {
						end = cut
					}
				}
			}
		}
		chunk := strings.TrimSpace(string(runes[start:end]))
		if chunk != "" {
			chunks = append(chunks, chunk)
		}
		if end >= len(runes) {
			break
		}
		start = end - chunkOverlap
		if start < 0 {
			start = 0
		}
	}
	return chunks
}