"""VNEWS v2 Entry Point - with fast bongda proxy + rewrite endpoints + multilingual TTS"""
import sys, os
from main import app, HEADERS, BONGDA_HEADERS, fetch_bongda_api, HL_LEAGUES

try:
    import ai_ext
except Exception as e:
    print(f"[WARN] ai_ext import failed: {e}")

try:
    import ai_patch
except Exception as e:
    print(f"[WARN] ai_patch import failed: {e}")

try:
    import ai_short_v2  # Short AI v2: TikTok music, uploaded audio/video/img, recreate from slides
except Exception as e:
    print(f"[WARN] ai_short_v2 import failed: {e}")

try:
    import pid_gen
    pid_gen.patch_app(app)
except Exception as e:
    print(f"[WARN] pid_gen import failed: {e}")

from fastapi.responses import HTMLResponse, JSONResponse, FileResponse, Response, StreamingResponse
from fastapi.staticfiles import StaticFiles
from starlette.routing import Mount
from fastapi import Query, Request, UploadFile, File, Form
import requests as req
from bs4 import BeautifulSoup
import re, html as html_lib, json, threading, time, uuid
from concurrent.futures import ThreadPoolExecutor, as_completed
from urllib.parse import quote, urljoin
import asyncio

HL_LEAGUES['friendly'] = {"path": "giai-khac/friendly", "name": "Giao hữu", "emoji": "🤝"}

STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static')
SPACE = "https://bep40-vnews.hf.space"  # SEO URL base for share links
app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)=='/' and hasattr(r,'methods') and 'GET' in getattr(r,'methods',set()))]
app.routes[:]=[r for r in app.routes if not isinstance(r, Mount)]
app.router.routes=[r for r in app.router.routes if not isinstance(r, Mount)]

def _clean(s): return re.sub(r"\s+"," ",html_lib.unescape(str(s or""))).strip()

def _ensure_sentence_complete(text):
    """Ensure text ends with complete sentence (ends with . ! or ?). Trim if cut mid-sentence."""
    text = _clean(text)
    if not text:
        return text
    # Find last sentence ending
    for end_char in ['.', '!', '?']:
        last_pos = text.rfind(end_char)
        if last_pos > len(text) * 0.5:  # Keep if ending is in latter half
            return text[:last_pos + 1].strip()
    # If no ending found, try to find last complete sentence
    sentences = re.split(r'(?<=[.!?])\s+', text)
    complete = [s.strip() for s in sentences if s.strip() and len(s.strip()) > 20]
    if complete[:-1]:  # Return all but last incomplete
        return ' '.join(complete[:-1])
    return text[:150] + '.' if len(text) > 150 else text

# Cache for match details (5 min TTL)
_match_cache = {}

# === FAST BONGDA PROXY ENDPOINT ===
def _get_match_detail(event_id, slug=None):
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "Accept": "text/html", "Referer": "https://bongda.com.vn/"}
    if slug:
        url = f"https://bongda.com.vn/tran-dau/{event_id}/centre/{slug}"
    else:
        url = f"https://bongda.com.vn/tran-dau/{event_id}"
    resp = req.get(url, headers=headers, timeout=15, allow_redirects=True)
    if resp.status_code != 200:
        return None
    soup = BeautifulSoup(resp.text, 'html.parser')
    result = {"event_id": event_id, "found": False, "sections": []}
    info = {}
    tel = soup.select_one('.teams')
    if tel:
        he = tel.select_one('.team.home')
        if he:
            p_tags = [p for p in he.select('p') if not p.get('class') or 'logo' not in p.get('class', [])]
            if p_tags: info['home_team'] = _clean(p_tags[0].get_text())
            lo = he.select_one('img')
            if lo: info['home_logo'] = lo.get('src', '')
        ae = tel.select_one('.team.away')
        if ae:
            p_tags = ae.select('p')
            team_ps = [p for p in p_tags if not p.get('class') or 'logo' not in p.get('class', [])]
            if team_ps: info['away_team'] = _clean(team_ps[-1].get_text())
            lo = ae.select_one('img')
            if lo: info['away_logo'] = lo.get('src', '')
        sc = tel.select_one('.score')
        if sc:
            parts = [_clean(p.get_text()) for p in sc.select('p')]
            if len(parts) >= 2: info['score'] = f"{parts[0]} - {parts[1]}"
            lb = sc.select_one('.label')
            if lb: info['status_label'] = _clean(lb.get_text())
    if info.get('home_team') and info.get('away_team'):
        result['info'] = info
        result['found'] = True
        result['sections'].append('info')
    events = []
    for ev in soup.select('.events .period .event'):
        ev_cls = ' '.join(ev.get('class', []))
        ev_data = {'team': 'home' if 'home' in ev_cls else 'away', 'period': '', 'type': 'unknown', 'time': '', 'players': ''}
        parent = ev.parent
        if parent:
            h2 = parent.find('h2')
            if h2: ev_data['period'] = _clean(h2.get_text())
        if ev.select_one('[class*="goal"]'): ev_data['type'] = 'goal'
        elif ev.select_one('[class*="redcard"]'): ev_data['type'] = 'redcard'
        elif ev.select_one('[class*="yellowcard"]'): ev_data['type'] = 'yellowcard'
        elif ev.select_one('[class*="substitution"]'): ev_data['type'] = 'substitution'
        players_el = ev.select_one('.players')
        if players_el:
            pl_text = _clean(players_el.get_text(' ', strip=True))
            m = re.match(r"(\d+)'(.*)", pl_text)
            if m:
                ev_data['time'] = f"{m.group(1)}'"
                ev_data['players'] = m.group(2)
            else:
                ev_data['players'] = pl_text
        events.append(ev_data)
    if events:
        result['events'] = events
        result['sections'].append('events')
    pred = soup.select_one('.prediction-card')
    if pred:
        team_info = pred.select_one('.team-info')
        if team_info:
            teams = team_info.select('.team')
            pred_data = {}
            if len(teams) >= 2:
                pred_data['home_name'] = _clean(teams[0].select_one('.team-name').get_text()) if teams[0].select_one('.team-name') else ''
                pred_data['away_name'] = _clean(teams[1].select_one('.team-name').get_text()) if teams[1].select_one('.team-name') else ''
                divider = team_info.select_one('.divider')
                if divider: pred_data['result'] = _clean(divider.get_text())
                vc = pred.select_one('.vote-count')
                if vc: pred_data['vote_count'] = _clean(vc.get_text())
            result['prediction'] = pred_data
    recent = []
    ml = soup.select_one('.matches-list')
    if ml:
        for item in ml.select('.match-detail, .match-item, li'):
            de = item.select_one('.date, .time')
            le = item.select_one('.league')
            he_item = item.select_one('.home, .team-home')
            ae_item = item.select_one('.away, .team-away')
            se = item.select_one('.score, .result')
            if he_item or ae_item:
                recent.append({'date': _clean(de.get_text()) if de else '', 'league': _clean(le.get_text()) if le else '', 'home': _clean(he_item.get_text()) if he_item else '', 'away': _clean(ae_item.get_text()) if ae_item else '', 'score': _clean(se.get_text()) if se else 'vs'})
    if recent:
        result['recent_matches'] = recent
        result['sections'].append('recent')
    try:
        api_h = {"User-Agent": "Mozilla/5.0", "Accept": "application/json", "X-Requested-With": "XMLHttpRequest", "Referer": "https://bongda.com.vn/"}
        ar = req.get(f"https://bongda.com.vn/api/fixtures/h2h-stats?event_id={event_id}", headers=api_h, timeout=10)
        if ar.status_code == 200:
            ad = ar.json()
            if ad.get('status') == 'success' and ad.get('html'):
                asp = BeautifulSoup(ad['html'], 'html.parser')
                ast = {}
                for row in asp.select('li, tr'):
                    cells = row.select('td, span, p')
                    if len(cells) >= 3:
                        lb = _clean(cells[0].get_text())
                        if lb: ast[lb] = {'home': _clean(cells[1].get_text()), 'away': _clean(cells[2].get_text())}
                if ast:
                    result['h2h_stats_parsed'] = ast
                    result['sections'].append('h2h_stats')
    except: pass
    return result

@app.get('/api/proxy/bongda')
def proxy_bongda(event_id: int = Query(default=None), slug: str = Query(default=None)):
    if event_id is None:
        return JSONResponse({'error': 'event_id required'}, status_code=400)
    cache_key = f"{event_id}_{slug}"
    now = time.time()
    cached = _match_cache.get(cache_key)
    if cached and now - cached.get('_ts', 0) < 300:
        return JSONResponse(cached)
    try:
        result = _get_match_detail(event_id, slug)
        if result:
            result['_ts'] = now
            _match_cache[cache_key] = result
            return JSONResponse(result)
    except Exception as e:
        err = {"event_id": event_id, "found": False, "error": str(e), "_ts": now}
        _match_cache[cache_key] = err
        return JSONResponse(err)
    return JSONResponse({"event_id": event_id, "found": False})

@app.get('/api/match/{event_id}/detail')
def api_match_detail(event_id: int, url: str = Query(default=None)):
    slug = None
    if url:
        m = re.match(r'.+/tran-dau/\d+/(?:centre|preview)/(.+)', url)
        if m:
            slug = m.group(1)
    cache_key = f"{event_id}_{slug or ''}"
    now = time.time()
    cached = _match_cache.get(cache_key)
    if cached and now - cached.get('_ts', 0) < 300:
        return JSONResponse(cached)
    try:
        if not slug:
            try:
                home_r = req.get("https://bongda.com.vn/", headers={"User-Agent": "Mozilla/5.0"}, timeout=10)
                if home_r.status_code == 200:
                    home_soup = BeautifulSoup(home_r.text, 'html.parser')
                    for a in home_soup.select(f'a[href*="/tran-dau/{event_id}/"]'):
                        href = a.get('href', '')
                        m = re.match(r'/tran-dau/\d+/(?:centre|preview)/(.+)', href)
                        if m:
                            slug = m.group(1)
                            cache_key = f"{event_id}_{slug}"
                            break
            except: pass
        result = _get_match_detail(event_id, slug)
        if result:
            result['_ts'] = now
            _match_cache[cache_key] = result
            return JSONResponse(result)
    except Exception as e:
        err = {"event_id": event_id, "found": False, "error": str(e), "_ts": now}
        _match_cache[cache_key] = err
        return JSONResponse(err)
    return JSONResponse({"event_id": event_id, "found": False})

_STOP=set('và của các những một được trong với cho tại sau trước khi không người việt nam hôm nay mới nhất nóng tin tức cập nhật theo từ đến là có thì này đã để'.split())

def _has_kw(topic,title):
    tl=topic.lower();tt=(title or'').lower()
    if tl in tt:return True
    words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',tl) if len(w)>2 and w.lower() not in _STOP]
    if not words:return True
    return any(w in tt for w in words)

def _s_vnexpress(topic,limit=8):
    items=[]
    try:
        r=req.get(f"https://timkiem.vnexpress.net/?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for art in soup.select('article.item-news')[:limit]:
            a=art.select_one('h2 a, h3 a')
            if a and a.get('href'):
                t=_clean(a.get('title','') or a.get_text(strip=True))
                if _has_kw(topic,t):items.append({'title':t,'url':a['href'],'via':'VnExpress'})
    except:pass
    return items

def _s_dantri(topic,limit=8):
    items=[]
    try:
        r=req.get(f"https://dantri.com.vn/tim-kiem/{quote(topic)}.htm",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .article-title a[href]')[:limit*2]:
            t=_clean(a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://dantri.com.vn'+href
                items.append({'title':t,'url':href,'via':'Dân Trí'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_vietnamnet(topic,limit=6):
    items=[]
    try:
        r=req.get(f"https://vietnamnet.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .vnn-title a')[:limit*2]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://vietnamnet.vn'+href
                items.append({'title':t,'url':href,'via':'VietNamNet'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_bongda(topic,limit=5):
    items=[]
    try:
        r=req.get(f"https://bongda.com.vn/tim-kiem.html?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .title a[href]')[:limit*2]:
            t=_clean(a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://bongda.com.vn'+href
                items.append({'title':t,'url':href,'via':'Bóng Đá'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_genk(topic,limit=5):
    items=[]
    try:
        r=req.get(f"https://genk.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('a[href$=".chn"]')[:limit*3]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if href.startswith('/'):href='https://genk.vn'+href
                items.append({'title':t,'url':href,'via':'GenK'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_thanhnien(topic,limit=6):
    items=[]
    try:
        r=req.get(f"https://thanhnien.vn/tim-kiem?q={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .box-title a')[:limit*2]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://thanhnien.vn'+href
                items.append({'title':t,'url':href,'via':'Thanh Niên'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_tuoitre(topic,limit=6):
    items=[]
    try:
        r=req.get(f"https://tuoitre.vn/tim-kiem.htm?keywords={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=10);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .box-title-text a')[:limit*2]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://tuoitre.vn'+href
                items.append({'title':t,'url':href,'via':'Tuổi Trẻ'})
            if len(items)>=limit:break
    except:pass
    return items

def _s_thethaovanhoa(topic,limit=5):
    items=[]
    try:
        r=req.get(f"https://thethaovanhoa.vn/tim-kiem.htm?keyword={quote(topic)}",headers={'User-Agent':'Mozilla/5.0'},timeout=8);soup=BeautifulSoup(r.text,'lxml')
        for a in soup.select('h3 a[href], .title a[href]')[:limit*2]:
            t=_clean(a.get('title','') or a.get_text(strip=True));href=a.get('href','')
            if t and len(t)>15 and _has_kw(topic,t):
                if not href.startswith('http'):href='https://thethaovanhoa.vn'+href
                items.append({'title':t,'url':href,'via':'TT&VH'})
            if len(items)>=limit:break
    except:pass
    return items

def _search_all(topic,limit=36):
    results={}
    with ThreadPoolExecutor(8) as ex:
        futs={ex.submit(_s_vnexpress,topic,8):'vne',ex.submit(_s_dantri,topic,8):'dt',ex.submit(_s_vietnamnet,topic,6):'vnn',ex.submit(_s_bongda,topic,5):'bd',ex.submit(_s_genk,topic,5):'gk',ex.submit(_s_thanhnien,topic,6):'tn',ex.submit(_s_tuoitre,topic,6):'tt',ex.submit(_s_thethaovanhoa,topic,5):'tvh'}
        for f in as_completed(futs,timeout=14):
            try:results[futs[f]]=f.result()
            except:results[futs[f]]=[]
    srcs=list(results.values());out=[];seen=set()
    for i in range(max((len(s) for s in srcs),default=0)):
        for s in srcs:
            if i<len(s) and s[i].get('url') and s[i]['url'] not in seen:seen.add(s[i]['url']);out.append(s[i])
    return out[:limit]

for _path in ['/api/article', '/api/hot_topics', '/api/categories', '/api/storage_status']:
    app.router.routes=[r for r in app.router.routes if not(getattr(r,'path',None)==_path and 'GET' in getattr(r,'methods',set()))]

_article_cache = {}
_article_cache_ttl = 1800

_art_session = None
_art_lock = threading.Lock()
def _get_art_session():
    global _art_session
    if _art_session is None:
        with _art_lock:
            if _art_session is None:
                _art_session = req.Session()
                _art_session.headers.update({
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
                    "Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8",
                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                })
    return _art_session

def _scrape_article_fast(url):
    from urllib.parse import urlparse
    domain = urlparse(url).netloc
    sess = _get_art_session()
    uas = [
        {"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"},
        {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"},
    ]
    for ua in uas:
        try:
            r = sess.get(url, headers=ua, timeout=6, allow_redirects=True)
            if not r or r.status_code != 200:
                continue
            r.encoding = 'utf-8'
            soup = BeautifulSoup(r.text, 'lxml')
            for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','iframe','.ads','.ad','.banner-ads','.fb-comments','.fb-root','.social-share','.related-news','.tag','.breadcrumb']):
                tag.decompose()
            title = summary = og_img = ""
            ogt = soup.find('meta', property='og:title')
            if ogt: title = ogt.get('content', '')
            ogd = soup.find('meta', property='og:description') or soup.find('meta', attrs={'name': 'description'})
            if ogd: summary = ogd.get('content', '')[:500]
            ogi = soup.find('meta', property='og:image')
            if ogi:
                og_img = ogi.get('content', '')
                if og_img.startswith('//'): og_img = 'https:' + og_img
            h1 = soup.find('h1')
            if not title and h1: title = h1.get_text(strip=True)[:200]
            body = []
            selectors = [
                '.fck_detail', '.sidebar-1',
                '.singular-content', '.dt__content', '.article-content', '.content-detail', '#divNewsContent',
                '.content-detail', '.main-content-detail', '.box-content',
                '.knc-content', '.article-body', '.detail-body',
                '.article-detail', '.detail-content',
                'article', 'main', '.cms-body', '.article__body', '.post-content',
                '.entry-content', '#content', '.article-text', '.story-body',
            ]
            for sel in selectors:
                el = soup.select_one(sel)
                if el and len(el.find_all('p')) >= 2:
                    seen_imgs = set()
                    for child in el.find_all(['p','h2','h3','figure','img'], recursive=True):
                        if child.name == 'p':
                            t = child.get_text(strip=True)
                            if t and len(t) > 15:
                                body.append({'type': 'p', 'text': t})
                        elif child.name in ('h2','h3'):
                            t = child.get_text(strip=True)
                            if t:
                                body.append({'type': 'heading', 'text': t})
                        elif child.name in ('figure','img'):
                            im = child if child.name == 'img' else child.find('img')
                            if im:
                                src = im.get('data-src') or im.get('src') or im.get('data-lazy') or ''
                                if src and 'base64' not in src and src not in seen_imgs:
                                    seen_imgs.add(src)
                                    if src.startswith('//'): src = 'https:' + src
                                    body.append({'type': 'img', 'src': src})
                        if child.name == 'figure':
                            cap = child.find('figcaption')
                            if cap:
                                ct = cap.get_text(strip=True)
                                if ct: body.append({'type': 'p', 'text': ct})
                    if len(body) >= 2:
                        return {'title': _clean(title), 'summary': _clean(summary), 'og_image': og_img,
                                'body': body[:50], 'source': domain, 'url': url}
            if title and (summary or og_img):
                fallback = []
                if og_img: fallback.append({'type': 'img', 'src': og_img})
                if summary: fallback.append({'type': 'p', 'text': summary})
                if fallback:
                    return {'title': _clean(title), 'summary': _clean(summary), 'og_image': og_img,
                            'body': fallback, 'source': domain, 'url': url, 'fallback': True}
            if title:
                return {'title': _clean(title), 'summary': '', 'og_image': '',
                        'body': [{'type': 'p', 'text': 'Nội dung đang được tải...'}],
                        'source': domain, 'url': url, 'fallback': True}
            break
        except Exception:
            continue
    return None

@app.get('/api/article')
def api_article_v2(url: str = Query(...)):
    from urllib.parse import unquote
    safe_url = unquote(url)
    try:
        now = time.time()
        cached = _article_cache.get(safe_url)
        if cached and now - cached['t'] < _article_cache_ttl:
            resp = JSONResponse(cached['d'])
            resp.headers["Cache-Control"] = "public, max-age=1800"
            return resp
        data = _scrape_article_fast(safe_url)
        if data and data.get('body'):
            _article_cache[safe_url] = {'d': data, 't': now}
            resp = JSONResponse(data)
            resp.headers["Cache-Control"] = "public, max-age=1800"
            return resp
        result = {'error': 'Không đọc được', 'url': safe_url}
        resp = JSONResponse(result)
        resp.headers["Cache-Control"] = "public, max-age=60"
        return resp
    except Exception as e:
        return JSONResponse({'error': f'Server error: {str(e)[:100]}', 'url': safe_url}, status_code=200)

_hot_cache={'t':0,'d':[]}
def _parse_feed(feed_url):
    try:
        r=req.get(feed_url,headers={'User-Agent':'Mozilla/5.0'},timeout=4)
        r.encoding='utf-8';soup=BeautifulSoup(r.text,'xml')
        out=[]
        for item in soup.find_all('item')[:12]:
            t=item.find('title')
            title=_clean(t.get_text() if t else '')
            if title: out.append(title)
        return out
    except Exception:
        return []
def _get_hot_topics():
    now=time.time()
    if _hot_cache['d'] and now-_hot_cache['t']<600:return _hot_cache['d']
    feeds=['https://vnexpress.net/rss/tin-moi-nhat.rss','https://dantri.com.vn/rss/home.rss','https://vietnamnet.vn/rss/tin-moi-nhat.rss','https://thanhnien.vn/rss/home.rss','https://tuoitre.vn/rss/tin-moi-nhat.rss','https://genk.vn/rss','https://vnexpress.net/rss/the-thao.rss','https://thethaovanhoa.vn/rss/tin-nong.rss']
    titles=[]
    with ThreadPoolExecutor(8) as ex:
        futs=[ex.submit(_parse_feed,f) for f in feeds]
        for f in as_completed(futs,timeout=5):
            try:
                for t in f.result(): titles.append(t)
            except Exception: pass
    freq={};display={}
    for title in titles:
        if not title: continue
        title=re.sub(r'\s*[-|].*$','',title)
        words=[w for w in re.findall(r'[A-Za-zÀ-ỹ0-9]+',title) if len(w)>2 and w.lower() not in _STOP]
        if len(words)<2:continue
        for n in(3,4,2):
            for i in range(max(0,len(words)-n+1)):
                phrase=' '.join(words[i:i+n])
                if 8<=len(phrase)<=45:key=phrase.lower();freq[key]=freq.get(key,0)+1;display[key]=phrase
    ranked=sorted(freq.items(),key=lambda x:x[1],reverse=True);topics=[];seen=set()
    for key,count in ranked:
        is_dup=any(len(set(e.split())&set(key.split()))/max(len(set(e.split())),len(set(key.split())),1)>0.6 for e in seen)
        if is_dup:continue
        seen.add(key);topics.append({'label':'#'+re.sub(r'\s+','',display[key].title()),'topic':display[key],'count':count})
        if len(topics)>=20:break
    for kw in['World Cup 2026','Kinh tế Việt Nam','Bóng đá châu Âu','Công nghệ AI','Giá vàng','Thời tiết']:
        if len(topics)>=24:break
        if not any(kw.lower() in s for s in seen):topics.append({'label':'#'+re.sub(r'\s+','',kw.title()),'topic':kw,'count':0})
    _hot_cache.update({'t':now,'d':topics[:24]});return topics[:24]

@app.get('/api/hot_topics')
def api_hot_topics():
    resp = JSONResponse({'topics':_get_hot_topics()})
    resp.headers["Cache-Control"] = "public, max-age=120"
    return resp
@app.get('/')
async def serve_index():
    p=os.path.join(STATIC_DIR,'index_v2.html')
    if os.path.exists(p):return FileResponse(p,media_type='text/html')
    return HTMLResponse('<h1>VNEWS</h1>')
@app.get('/api/hashtag/sources')
def _ht(topic:str=Query(...),page:int=Query(default=0)):
    items=_search_all(topic,36);per_page=8;start=page*per_page;end=start+per_page
    return JSONResponse({'sources':items[start:end],'topic':topic,'page':page,'has_more':end<len(items),'total':len(items)})
@app.get('/api/categories')
def _cat():
    try:
        from main import VNE_CATS
        cats=[{"id":k,"name":n,"source":"vne"} for k,(u,n) in VNE_CATS.items()]
    except Exception:
        cats = [{"id":"the-thao","name":"Thể Thao","source":"vne"}]
    # Prepend the M3U (live sports streams) category at the front of the bar
    cats.insert(0, {"id":"m3u","name":"⚽ Trực Tiếp","source":"m3u"})
    resp=JSONResponse(cats)
    resp.headers["Cache-Control"]="public, max-age=600"
    return resp
@app.get('/api/storage_status')
def _st():return JSONResponse({'persistent':os.path.isdir('/data') and os.access('/data',os.W_OK)})
# ===== SHARE HELPERS: render content pages for shared links =====
# Default image + description used when a shared link has no image/description
# (e.g. M3U channel share links built from the player).
_M3U_SHARE_DESC = "Xem trực tiếp bóng đá và sự kiện thể thao playlist m3u với trình xem Tiktok player!"
_M3U_SHARE_IMG = f"{SPACE}/static/tv_default.png"

def _m3u_og_parts(safe_img, safe_title):
    """Return (og_image, description) for share pages. If no image given, use a
    default TV icon so shared links always show an og:image preview."""
    img = _clean(safe_img) if safe_img else _M3U_SHARE_IMG
    desc = _M3U_SHARE_DESC if not safe_img else _clean(safe_title or '')
    return img, desc

def _render_slides_page(post, safe_title, safe_img, safe_url):
    slides = post.get('slides', [])
    # Get image from post.img or first slide's image
    if not safe_img and slides and slides[0].get('image'):
        safe_img = slides[0].get('image', '')
    og_img, description = _m3u_og_parts(safe_img, safe_title)
    # Use text for description if available
    if safe_img:
        description = _clean((post.get('text') or '')[:200]) or "Tin tức tóm tắt, AI rewrite, World Cup 2026"
    
    # Build canonical URL preserving original query format if url was provided
    if safe_url and safe_url != '/':
        canonical_url = f"{SPACE}/s?url={quote(safe_url)}&title={quote(safe_title[:100])}"
    else:
        canonical_url = f"{SPACE}/s?post_id={post.get('id') or ''}"
    
    h = f'''<!DOCTYPE html>
<html lang="vi">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>{_clean(safe_title)}</title>
<meta property="og:title" content="{_clean(safe_title)}">
<meta property="og:image" content="{og_img}">
<meta property="og:description" content="{description}">
<meta property="og:url" content="{canonical_url}">
<link rel="canonical" href="{canonical_url}">
<style>
*{{box-sizing:border-box;margin:0;padding:0}}body{{background:#111;color:#eee;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;padding:12px}}
.slide-card{{background:#1a1a1a;border:1px solid #2a2a2a;border-radius:12px;padding:16px;margin-bottom:12px;max-width:600px;margin-left:auto;margin-right:auto}}
.slide-num{{color:#5cb87a;font-size:12px;font-weight:700;margin-bottom:6px}}
.slide-img{{width:100%;max-height:300px;object-fit:cover;border-radius:8px;margin-bottom:8px}}
.slide-text{{color:#ddd;font-size:14px;line-height:1.6;margin:0}}
</style>
</head>
<body>'''
    for s in slides:
        img_src = s.get('image', '')
        if img_src and ('cdnphoto.dantri' in img_src or 'refooty' in img_src or 'vnexpress' in img_src or 'vcdn' in img_src):
            img_tag = f'<img src="/api/proxy/img?url={quote(img_src, safe="")}" class="slide-img" loading="lazy" onerror="this.style.display=\'none\'">'
        else:
            img_tag = f'<img src="{_clean(img_src)}" class="slide-img" loading="lazy" onerror="this.style.display=\'none\'">' if img_src else ''
        h += f'<div class="slide-card"><div class="slide-num">Slide {s.get("index",1)}/{len(slides)}</div>{img_tag}<p class="slide-text">{_clean(s.get("text",""))}</p></div>'
    h += '</body></html>'
    return HTMLResponse(h)

def _render_video_page(post, safe_title, safe_img, safe_url):
    video_url = post.get('video', '')
    og_img, _desc = _m3u_og_parts(safe_img, safe_title)
    # Use text for description if available
    description = _clean((post.get('text') or '')[:200]) or "Tin tức tóm tắt, AI rewrite, World Cup 2026" if safe_img else _M3U_SHARE_DESC
    
    # Build canonical URL preserving original query format if url was provided
    if safe_url and safe_url != '/':
        canonical_url = f"{SPACE}/s?url={quote(safe_url)}&title={quote(safe_title[:100])}"
    else:
        canonical_url = f"{SPACE}/s?post_id={post.get('id') or ''}"
    
    h = f'''<!DOCTYPE html>
<html lang="vi">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no">
<title>{_clean(safe_title)}</title>
<meta property="og:title" content="{_clean(safe_title)}">
<meta property="og:image" content="{og_img}">
<meta property="og:description" content="{description}">
<meta property="og:url" content="{canonical_url}">
<link rel="canonical" href="{canonical_url}">
<meta name="twitter:card" content="player">
<meta name="twitter:player" content="{video_url}">
<style>
*{{box-sizing:border-box;margin:0;padding:0}}body{{background:#111;color:#eee;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;padding:0;overflow:hidden}}
.video-container{{width:100vw;height:100vh;display:flex;flex-direction:column;align-items:center;justify-content:center;background:#000}}
video{{width:100%;height:100%;max-height:100vh;object-fit:contain;background:#000}}
.title-bar{{position:fixed;bottom:0;left:0;right:0;background:linear-gradient(transparent,rgba(0,0,0,.8));padding:40px 16px 16px;text-align:center}}
.title-text{{color:#fff;font-size:13px;line-height:1.4;max-width:600px;margin:0 auto}}
</style>
</head>
<body>
<div class="video-container">
<video src="{_clean(video_url)}" controls autoplay playsinline loop></video>
<div class="title-bar"><div class="title-text">{_clean(safe_title)}</div></div>
</div>
</body></html>'''
    return HTMLResponse(h)

@app.get('/s/{slug}')
async def _sh_slug(slug: str, request: Request, url: str = '', title: str = '', img: str = ''):
    """SEO-friendly share endpoint with slug in URL path.
    Shows slide content when slug matches a wall post ID, otherwise redirects.
    """
    safe_title = _clean(title) if title else 'VNEWS - Tin tức'
    safe_img = _clean(img) if img else ''
    safe_url = _clean(url) if url else '/'
    
    # Try to find post by slug (post ID)
    post = None
    try:
        if slug and len(slug) > 5:  # Likely a post ID
            posts = _load_wall_posts()
            for p in posts:
                if p.get('id') == slug:
                    post = p
                    safe_title = p.get('title', safe_title) or safe_title
                    safe_img = p.get('img', safe_img) or safe_img
                    safe_url = p.get('url', safe_url) or safe_url
                    break
    except:
        pass
    
    if post and post.get('slides'):
        return _render_slides_page(post, safe_title, safe_img, safe_url)
    
    if post and post.get('video'):
        return _render_video_page(post, safe_title, safe_img, safe_url)
    
    # Otherwise redirect
    _fimg, _fdesc = _m3u_og_parts(safe_img, safe_title)
    return HTMLResponse(f'''<!DOCTYPE html>
<html lang="vi">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>{_clean(safe_title)}</title>
<meta property="og:title" content="{_clean(safe_title)}">
<meta property="og:image" content="{_fimg}">
<meta property="og:description" content="{_fdesc}">
<meta property="og:url" content="{SPACE}/s/{slug}">
<link rel="canonical" href="{SPACE}/s/{slug}">
<meta http-equiv="refresh" content="0;url={safe_url}">
</head><body></body></html>''')

@app.get('/s')
async def _sh(url:str='',title:str='',img:str='',post_id:str=''):
    safe_title = _clean(title) if title else 'VNEWS - Tin tức'
    safe_img = _clean(img) if img else ''
    safe_url = _clean(url) if url else '/'
    
    # Try to find wall post by post_id or URL (prioritize posts with slides/video)
    post = None
    try:
        posts = _load_wall_posts()
        if post_id:
            for p in posts:
                if p.get('id') == post_id:
                    post = p
                    safe_title = p.get('title', safe_title) or safe_title
                    safe_img = p.get('img', safe_img) or safe_img
                    safe_url = p.get('url', safe_url) or safe_url
                    break
        elif url:
            # Find matching URL - prioritize posts with slides or video
            for p in posts:
                if p.get('url') == url and p.get('slides'):
                    post = p
                    safe_title = p.get('title', safe_title) or safe_title
                    safe_img = p.get('img', safe_img) or safe_img
                    safe_url = p.get('url', safe_url) or safe_url
                    break
            if not post:
                # Fallback: find any matching URL
                for p in posts:
                    if p.get('url') == url:
                        post = p
                        safe_title = p.get('title', safe_title) or safe_title
                        safe_img = p.get('img', safe_img) or safe_img
                        safe_url = p.get('url', safe_url) or safe_url
                        break
    except:
        pass
    
    if post and post.get('slides'):
        return _render_slides_page(post, safe_title, safe_img, safe_url)
    
    if post and post.get('video'):
        return _render_video_page(post, safe_title, safe_img, safe_url)
    
    # Fallback: redirect to original URL
    _fimg, _fdesc = _m3u_og_parts(safe_img, safe_title)
    return HTMLResponse(f'''<!DOCTYPE html>
<html lang="vi">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>{safe_title}</title>
<meta property="og:title" content="{safe_title}">
<meta property="og:image" content="{_fimg}">
<meta property="og:description" content="{_fdesc}">
<meta property="og:url" content="{SPACE}/s?url={quote(safe_url)}">
<link rel="canonical" href="{SPACE}/s?url={quote(safe_url)}">
<meta http-equiv="refresh" content="0;url={safe_url}">
</head><body></body></html>''')

from wc2026_scraper import scrape_summary,scrape_fixtures,scrape_standings,scrape_stats,scrape_wc_news,scrape_road_to_wc,get_wc2026_all,scrape_history,scrape_h2h,scrape_lineups,scrape_match_detail

_xlb_cache = {}
_xlb_lock = threading.Lock()

def _xlb_scrape(path):
    url = f"https://xemlaibongda.top/{path}"
    r = req.get(url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}, timeout=15, allow_redirects=True)
    if r.status_code != 200:
        return []
    soup = BeautifulSoup(r.text, 'lxml')
    vids = []
    seen = set()
    for a in soup.select('a[href*="/video/"]'):
        href = a.get('href', '')
        if not href or href in seen:
            continue
        seen.add(href)
        if not href.startswith('http'):
            href = 'https://xemlaibongda.top' + href
        img = a.select_one('img')
        p = a.parent
        for _ in range(4):
            if img:
                break
            if p:
                img = p.select_one('img')
                p = p.parent
        img_src = ''
        if img:
            img_src = img.get('data-src','') or img.get('src','') or img.get('data-lazy','') or img.get('data-original','')
            if img_src.startswith('//'):
                img_src = 'https:' + img_src
            elif img_src.startswith('/'):
                img_src = 'https://xemlaibongda.top' + img_src
        title = ''
        for sel in ['.title', 'h3', 'h2', '.name', '.post-title', '.entry-title', '.video-title']:
            t = a.select_one(sel)
            if t:
                title = _clean(t.get_text())
                break
        if not title:
            title = _clean(a.get('title',''))
        if not title:
            img_alt = a.select_one('img')
            if img_alt:
                title = _clean(img_alt.get('alt',''))
        if not title:
            parent = a.parent
            if parent:
                pt = _clean(parent.get_text(' ',strip=True))
                if 5 < len(pt) < 120:
                    title = pt
        if not title or len(title) < 3:
            continue
        vids.append({"link": href, "img": img_src, "title": title})
        if len(vids) >= 30:
            break
    return vids

@app.get('/api/proxy/xlb')
def proxy_xlb(path: str = Query(default="")):
    now = time.time()
    cache_key = f"xlb:{path}"
    with _xlb_lock:
        cached = _xlb_cache.get(cache_key)
        if cached and now - cached['t'] < 120:
            return JSONResponse(cached['d'])
    try:
        vids = _xlb_scrape(path)
        result = {"videos": vids, "count": len(vids)}
        with _xlb_lock:
            _xlb_cache[cache_key] = {'t': now, 'd': result}
        return JSONResponse(result)
    except Exception as e:
        return JSONResponse({"videos": [], "count": 0, "error": str(e)}, status_code=500)

@app.get('/api/wc2026')
def _w():return JSONResponse(get_wc2026_all())
@app.get('/api/wc2026/fixtures')
def _wf():return JSONResponse(scrape_fixtures())
@app.get('/api/wc2026/standings')
def _ws():return JSONResponse(scrape_standings())
@app.get('/api/wc2026/stats')
def _wst():return JSONResponse(scrape_stats())
@app.get('/api/wc2026/history')
def _whi():return JSONResponse(scrape_history())
@app.get('/api/wc2026/news')
def _wn():return JSONResponse(scrape_wc_news())
@app.get('/api/wc2026/road')
def _wr():return JSONResponse(scrape_road_to_wc())
@app.get('/api/wc2026/h2h/{eid}')
def _wh2(eid:int):return JSONResponse(scrape_h2h(eid))
@app.get('/api/wc2026/lineups/{eid}')
def _wl(eid:int):return JSONResponse(scrape_lineups(eid))
@app.get('/api/wc2026/match/{eid}')
def _wm(eid:int):return JSONResponse(scrape_match_detail(eid))

DATA_DIR='/data' if os.path.isdir('/data') else os.path.join(os.path.dirname(os.path.abspath(__file__)),'data')
os.makedirs(DATA_DIR,exist_ok=True)
IF=os.path.join(DATA_DIR,'interactions_v2.json')
CF=os.path.join(DATA_DIR,'comments_v2.json')
WALL_FILE=os.path.join(DATA_DIR,'wall_posts.json')
WALL_VIDEO_DIR=os.path.join(DATA_DIR,'wall_videos')
os.makedirs(WALL_VIDEO_DIR,exist_ok=True)

_il=threading.Lock();_cl=threading.Lock();_wl_lock=threading.Lock()
def _lj(p):
    try:
        if os.path.exists(p):return json.load(open(p,'r',encoding='utf-8'))
    except:pass
    return{}
def _sj(p,d):
    try:open(p+'.tmp','w',encoding='utf-8').write(json.dumps(d,ensure_ascii=False));os.replace(p+'.tmp',p)
    except:pass

@app.post('/api/v2/interact')
async def _int(request:Request):
    b=await request.json();v=str(b.get('id','')).strip();t=str(b.get('type','')).strip()
    if not v or t not in('view','like'):return JSONResponse({'error':'x'},status_code=400)
    with _il:db=_lj(IF);db.setdefault(v,{'views':0,'likes':0,'comments':0});db[v][t+'s']+=1;_sj(IF,db);return JSONResponse(db[v])

@app.get('/api/v2/interactions')
def _gi(id:str=Query(...)):
    with _il:return JSONResponse(_lj(IF).get(id.strip(),{'views':0,'likes':0,'comments':0}))

@app.get('/api/v2/comments')
def _gc(id:str=Query(...)):
    with _cl:return JSONResponse({'comments':_lj(CF).get(id.strip(),[])})

@app.post('/api/v2/comment')
async def _pc(request:Request):
    b=await request.json();v=str(b.get('id','')).strip();tx=str(b.get('text','')).strip()[:500]
    if not v or not tx:return JSONResponse({'error':'x'},status_code=400)
    c={'text':tx,'time':time.strftime('%H:%M %d/%m',time.localtime()),'ts':int(time.time())}
    with _cl:db=_lj(CF);db.setdefault(v,[]);db[v].append(c);db[v]=db[v][-200:];_sj(CF,db);cms=db[v]
    with _il:idb=_lj(IF);idb.setdefault(v,{'views':0,'likes':0,'comments':0});idb[v]['comments']=len(cms);_sj(IF,idb)
    return JSONResponse({'comments':cms})

def _load_wall_posts():
    with _wl_lock:
        posts = _lj(WALL_FILE)
    if not isinstance(posts, list):
        posts = []
    return posts

def _save_wall_posts(posts):
    with _wl_lock:
        _sj(WALL_FILE, posts)

def _post_has_valid_image(p):
    """A wall post is considered to have a 'valid' (non-placeholder) image if
    at least one of the following is a real, non-placeholder URL:
      - p.img        (main thumbnail)
      - p.short_thumb (generated by Short AI / oEmbed fallback)
      - p.images[]    (fallback image list)
      - p.slides[].image (designed slide images)

    Auto posts that failed metadata enrichment — e.g. pollinations.ai
    AI-generated placeholder images, vnexpress logo placeholders, or empty/None
    fields — are flagged as having NO valid image so they can be filtered out
    of the wall (they show the wrong picture vs the original article)."""
    PLACEHOLDER_MARKERS = (
        "pollinations.ai",          # AI-generated editorial illustration placeholder
        "logo_default.jpg",        # vnexpress placeholder logo
        "logo.png",
        "/logo",
        "placeholder",
        "no-image",
        "blank",
    )

    def _is_placeholder(url):
        if not url or not isinstance(url, str):
            return True
        u = url.strip().lower()
        if not u or u.startswith(("about:blank", "data:")):
            return True
        return any(m in u for m in PLACEHOLDER_MARKERS)

    def _is_real(url):
        return bool(url) and not _is_placeholder(url)

    img = p.get("img")
    if _is_real(img):
        return True
    thumb = p.get("short_thumb")
    if _is_real(thumb):
        return True
    images = p.get("images") or []
    if any(_is_real(x) for x in images):
        return True
    slides = p.get("slides") or []
    if any(_is_real(s.get("image")) for s in slides if isinstance(s, dict)):
        return True
    return False

def _post_has_slide_info(p):
    """True if the post has usable slide/rewrite AI content:
      - designed slides with text+image, OR
      - a generated short video (/api/ai/short-file/) with a real thumbnail, OR
      - an oEmbed iframe (YouTube/TikTok/Facebook embed) with a real thumbnail
        (Short HOT fallback case — embeddable, viewable as a slide)."""
    slides = p.get("slides") or []
    if any((s.get("image") or "").strip() and (s.get("text") or "").strip() for s in slides if isinstance(s, dict)):
        return True
    # Generated short video with a real thumbnail
    vid = (p.get("video") or "").strip()
    if vid and "api/ai/short-file/" in vid and (p.get("short_thumb") or "").strip():
        return True
    # oEmbed iframe (YouTube/TikTok/FB) with a valid image -> viewable embed slide
    if vid and ("youtube.com/embed" in vid or "tiktok.com/embed" in vid or
                "facebook.com/plugins" in vid or "instagram.com" in vid or
                "twitter.com" in vid or "x.com" in vid or "player.vimeo" in vid):
        if _post_has_valid_image(p):
            return True
    return False

def _is_auto_post(p):
    """Auto-generated wall posts: Short HOT, oEmbed fallback, FPT auto,
    slide-rewrite AI (auto_rewrite), topic auto-summary (topic_article), or
    posts with no meaningful source / created by the AI pipeline."""
    kind = (p.get("kind") or "").strip().lower()
    source = (p.get("source") or "").strip().lower()
    return kind in ("hot_short", "slide_summary", "auto_rewrite", "topic_article", "summary") or \
           source in ("hot_short", "fptplay", "ai", "auto_rewrite", "topic_article")

def _post_has_content(p):
    """True when the post has meaningful content: non-empty text (>= 20 chars)
    OR at least one slide with non-empty text OR a video. Rewrite posts that
    failed to generate content (empty text+slides) get dropped from the feed."""
    if not isinstance(p, dict):
        return False
    text = (p.get("text") or "").strip()
    if len(text) >= 20:
        return True
    slides = p.get("slides") or []
    if len(slides) >= 1:
        for s in slides:
            if isinstance(s, dict) and len((s.get("text") or "").strip()) >= 8:
                return True
    if (p.get("video") or "").strip():
        return True
    return False

def _created_ts(p):
    """Best-effort creation timestamp NORMALIZED TO SECONDS. `created` is
    stored as Unix milliseconds by some paths while `ts` is Unix seconds —
    normalize both so newest-first ordering mixes them correctly. Falls back
    to id-derived Unix ms (converted to seconds), then 0."""
    for field in ("created", "ts"):
        v = p.get(field)
        if v is not None:
            try:
                v = int(v)
            except (ValueError, TypeError):
                pass
            if v > 100000000000:  # milliseconds -> seconds
                v = v // 1000
            return v
    # try to derive from id (numeric prefix = Unix ms timestamp)
    pid = str(p.get("id", "")).strip()
    if pid.isdigit() and len(pid) >= 12:
        try:
            v = int(pid)
            if v > 100000000000:
                v = v // 1000
            return v
        except (ValueError, TypeError):
            pass
    return 0

def _wall_posts_for_view():
    """Apply the user-facing wall filtering + sorting (VIEW-ONLY — never
    touches the persistent store):
      1) Drop posts with NO meaningful content (empty text AND empty slides
         AND no video) — 'bài đăng rewrite fără nội dung';
      2) Hide auto-generated posts that have a wrong/placeholder image (no
         valid image at all — pollinations.ai, vnexpress logo, empty) or have
         no slide-AI content (no slides / no short-video / no embed);
      3) Sort newest-first by creation timestamp, mixing all sources."""
    try:
        from ai_ext import wall_posts_for_view as _shared_view
        return _shared_view(limit=200)
    except Exception:
        pass
    posts = _load_wall_posts()
    if not posts:
        return []
    filtered = []
    for p in posts:
        if not isinstance(p, dict):
            continue
        if not _post_has_content(p):
            continue
        if _is_auto_post(p) and not _post_has_valid_image(p) and not _post_has_slide_info(p):
            # auto post with a wrong/placeholder/missing image AND no slide AI
            # content -> hide from the wall (non-destructive)
            continue
        filtered.append(p)
    filtered.sort(key=_created_ts, reverse=True)
    return filtered

@app.get("/api/ai/wall")
def api_ai_wall():
    return JSONResponse({"posts": _wall_posts_for_view()})


# ===== YOUTUBE RSS FEED PROXY (avoids browser CORS) =====
_YT_FEED_CACHE = {}
_YT_FEED_TTL = 600  # 10 minutes


def _fetch_youtube_rss_feed(channel_id: str = "UC4LvrpNXujjbGOS4RDvr41g"):
    """Fetch & parse the YouTube RSS feed server-side.
    
    The browser cannot reach youtube.com directly (CORS / network block).
    We try two sources in order:
      1) Direct feed XML (works if the server has outbound YouTube access)
      2) rss2json.com proxy (always works — it fetches the feed server-side)
    Cached for _YT_FEED_TTL seconds."""
    now = time.time()
    cached = _YT_FEED_CACHE.get(channel_id)
    if cached and now - cached.get('_ts', 0) < _YT_FEED_TTL:
        return cached.get('videos', [])
    out = []
    try:
        # Attempt 1: direct YouTube RSS
        feed_url = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
        try:
            r = req.get(feed_url, headers=HEADERS, timeout=10)
            if r.status_code == 200 and r.text.strip():
                soup = BeautifulSoup(r.text, 'xml')
                for e in soup.find_all('entry'):
                    out.extend(_parse_yt_entry(e))
        except Exception as ex2:
            print(f"[yt-feed] direct fetch failed, trying rss2json: {ex2}")
        # Attempt 2: rss2json proxy (fallback / always)
        if not out:
            rss2json_url = f"https://api.rss2json.com/v1/api.json?rss_url={feed_url}"
            r2 = req.get(rss2json_url, headers=HEADERS, timeout=15)
            if r2.status_code == 200:
                data = r2.json()
                if data.get('status') == 'ok':
                    for item in data.get('items', []):
                        vid_m = re.search(r'([a-zA-Z0-9_-]{11})', item.get('id', ''))
                        if not vid_m:
                            continue
                        vid = vid_m.group(1)
                        title = item.get('title', 'Video') or 'Video'
                        # prefer the youtube watch URL, fallback to item link
                        link = item.get('link', '')
                        if '/watch' not in link:
                            link = f'https://www.youtube.com/watch?v={vid}'
                        # thumbnail: prefer media:thumbnail, fallback to youtube hqdefault
                        thumb = item.get('thumbnail', f'https://i.ytimg.com/vi/{vid}/hqdefault.jpg')
                        published = item.get('pubDate', '')
                        out.append({
                            'id': 'yt-' + vid,
                            'videoId': vid,
                            'title': title.strip()[:200],
                            'link': link,
                            'img': thumb,
                            'published': published,
                            'source': 'yt-feed'
                        })
        if out:
            _YT_FEED_CACHE[channel_id] = {'_ts': now, 'videos': out}
        return out
    except Exception as ex:
        print(f"[yt-feed] server error: {ex}")
        return []


def _parse_yt_entry(e):
    """Parse a single YouTube RSS <entry> element into a video dict."""
    out = []
    try:
        id_el = e.find('id')
        if not id_el:
            return out
        vid_m = re.search(r'([a-zA-Z0-9_-]{11})', id_el.get_text(''))
        if not vid_m:
            return out
        vid = vid_m.group(1)
        title_el = e.find('title')
        title = title_el.get_text('') if title_el else 'Video'
        link_el = e.find('link', {'rel': 'alternate'})
        link = link_el.get('href') if link_el else f'https://www.youtube.com/watch?v={vid}'
        pub_el = e.find('published')
        published = pub_el.get_text('') if pub_el else ''
        ns = {'media': 'http://search.yahoo.com/mrss/'}
        thumb_el = e.find('media:thumbnail', ns)
        thumb_url = thumb_el.get('url') if thumb_el else f'https://i.ytimg.com/vi/{vid}/hqdefault.jpg'
        out.append({
            'id': 'yt-' + vid,
            'videoId': vid,
            'title': title.strip()[:200],
            'link': link,
            'img': thumb_url,
            'published': published,
            'source': 'yt-feed'
        })
    except Exception as ex:
        print(f"[yt-feed] parse entry error: {ex}")
    return out


@app.get("/api/yt/feed")
def api_yt_feed(channel_id: str = Query(default="UC4LvrpNXujjbGOS4RDvr41g")):
    """Server-side YouTube RSS feed fetcher — avoids browser CORS.
    Returns parsed feed videos as JSON for the Tường AI wall."""
    videos = _fetch_youtube_rss_feed(channel_id)
    return JSONResponse({"videos": videos, "count": len(videos)})
@app.post("/api/ai/short/{post_id}")
def api_ai_short(post_id: str):
    """Generate a short video for a wall post.
    Returns clear JSON error when video not available yet.
    """
    post_id_s = str(post_id)
    posts = _load_wall_posts()
    if not isinstance(posts, list):
        posts = []
    post = None
    for p in posts:
        pid = str(p.get("id", ""))
        if pid == post_id_s or pid.startswith(post_id_s):
            post = p
            break
    if not post:
        return JSONResponse({"error": "Không tìm thấy bài viết", "post_id": post_id}, status_code=404)
    if post.get("video"):
        return JSONResponse({"video": post["video"], "post": post})
    return JSONResponse({"error": "Chưa có video cho bài này. Vui lòng upload video trước."}, status_code=409)

@app.get('/api/wall')
def api_wall():
    posts = _wall_posts_for_view()
    if not posts:
        return JSONResponse({"posts": []})
    return JSONResponse({"posts": posts})

@app.post('/api/wall')
async def api_wall_post(request: Request):
    content_type = request.headers.get('content-type', '')
    if 'multipart/form-data' in content_type:
        try:
            form = await request.form()
        except Exception as e:
            return JSONResponse({"error": f"Form parse error: {str(e)}"}, status_code=400)
        title = form.get('title', 'Video mới') or 'Video mới'
        text = form.get('text', '') or ''
        source = form.get('source', 'vtv_recorder') or 'vtv_recorder'
        video_file = form.get('video')
        post_id = str(uuid.uuid4())[:12]
        video_url = None
        if video_file and hasattr(video_file, 'filename') and video_file.filename:
            fname = video_file.filename.lower()
            if fname.endswith('.mp4'):
                ext = '.mp4'
            elif fname.endswith('.webm'):
                ext = '.webm'
            else:
                ext = '.webm'
            video_filename = f"wall_{post_id}{ext}"
            video_path = os.path.join(WALL_VIDEO_DIR, video_filename)
            try:
                content = await video_file.read()
                if not content:
                    return JSONResponse({"error": "Empty video file"}, status_code=400)
                with open(video_path, 'wb') as f:
                    f.write(content)
                file_size_mb = len(content) / 1024 / 1024
                if file_size_mb > 50:
                    os.remove(video_path)
                    return JSONResponse({"error": f"Video quá lớn ({file_size_mb:.1f}MB). Tối đa 50MB."}, status_code=400)
                video_url = f"/api/wall/video/{video_filename}"
            except Exception as e:
                return JSONResponse({"error": f"Lỗi lưu video: {str(e)}"}, status_code=500)
        post = {
            "id": post_id,
            "title": title[:200],
            "text": text[:2000],
            "source": source,
            "video": video_url,
            "img": None,
            "images": [],
            "created": int(time.time()),
            "created_str": time.strftime('%H:%M %d/%m/%Y', time.localtime()),
        }
        posts = _load_wall_posts()
        if not isinstance(posts, list):
            posts = []
        posts.insert(0, post)
        posts = posts[:200]
        _save_wall_posts(posts)
        return JSONResponse({"post": post, "ok": True})
    try:
        body = await request.json()
    except:
        body = {}
    title = body.get('title', 'Bài mới') or 'Bài mới'
    text = body.get('text', '') or ''
    img = body.get('img', None)
    source = body.get('source', 'user') or 'user'
    incoming_id = body.get('id') or ''  # allow client to (re)publish an existing post by id
    post_id = str(uuid.uuid4())[:12]
    # Preserve slide-design post fields (slides, kind, url, images, video, voice, etc.)
    # so that "Thiết kế ảnh" -> "Đăng lên Tường AI" keeps the selected slides.
    slides = body.get('slides')
    kind = body.get('kind') or 'user'
    post_url = body.get('url') or ''
    images = body.get('images') or []
    video = body.get('video')
    voice = body.get('voice') or ''
    emotion = body.get('emotion') or ''
    language = body.get('language') or ''
    post = {
        "id": post_id,
        "title": title[:200],
        "text": text[:2000],
        "source": source,
        "video": video,
        "img": img,
        "images": images[:10],
        "url": post_url,
        "kind": kind,
        "slides": slides if slides is not None else None,
        "voice": voice,
        "emotion": emotion,
        "language": language,
        "created": int(time.time()),
        "created_str": time.strftime('%H:%M %d/%m/%Y', time.localtime()),
    }
    # Remove keys with None so the JSON stays lean (but keep structure)
    post = {k: v for k, v in post.items() if v is not None}
    posts = _load_wall_posts()
    if not isinstance(posts, list):
        posts = []
    # If the client re-publishes (e.g. designer "Lưu & Đăng lên Tường AI" sends
    # the original post id), update that post in place instead of creating a
    # duplicate. This keeps the homepage wall clean (no duplicate slide posts).
    updated = False
    if incoming_id:
        for _p in posts:
            if str(_p.get('id')) == str(incoming_id):
                # Merge new fields into the existing post, but PRESERVE the
                # original id (and created timestamp) so the post identity is
                # stable and no duplicate is created.
                preserve_id = _p.get('id')
                preserve_created = _p.get('created')
                preserve_created_str = _p.get('created_str')
                _p.update(post)
                _p['id'] = preserve_id
                if preserve_created is not None: _p['created'] = preserve_created
                if preserve_created_str is not None: _p['created_str'] = preserve_created_str
                updated = True
                post = _p  # return the merged post (with stable id) to the client
                break
    if not updated:
        posts.insert(0, post)
    else:
        # Move the updated post to the top so it re-appears at the top of the wall.
        posts = [p for p in posts if str(p.get('id')) != str(incoming_id)]
        posts.insert(0, post)
    posts = posts[:200]
    _save_wall_posts(posts)
    return JSONResponse({"post": post, "ok": True})

@app.get('/api/wall/video/{filename}')
def api_wall_video(filename: str):
    if '..' in filename or '/' in filename:
        return Response(status_code=403)
    video_path = os.path.join(WALL_VIDEO_DIR, filename)
    if not os.path.exists(video_path):
        return Response(status_code=404)
    ext = os.path.splitext(filename)[1].lower()
    media_type = 'video/mp4' if ext == '.mp4' else 'video/webm'
    return FileResponse(video_path, media_type=media_type)

WALL_IMG_DIR = os.path.join(DATA_DIR, 'wall_imgs')
os.makedirs(WALL_IMG_DIR, exist_ok=True)

@app.post('/api/wall/img')
async def api_wall_img(request: Request):
    """Upload a designed slide image (PNG) and return a served URL."""
    global WALL_IMG_DIR
    try:
        form = await request.form()
        f = form.get('file')
        if not f or not hasattr(f, 'filename') or not f.filename:
            return JSONResponse({"error": "Thiếu file ảnh"}, status_code=400)
        ext = os.path.splitext(f.filename)[1].lower()
        if ext not in ('.png', '.jpg', '.jpeg', '.webp'):
            ext = '.png'
        img_id = str(uuid.uuid4())[:12]
        fname = f"wallimg_{img_id}{ext}"
        fpath = os.path.join(WALL_IMG_DIR, fname)
        content = await f.read()
        if not content:
            return JSONResponse({"error": "File rỗng"}, status_code=400)
        if len(content) > 15 * 1024 * 1024:
            return JSONResponse({"error": "Ảnh quá lớn (>15MB)"}, status_code=400)
        with open(fpath, 'wb') as fh:
            fh.write(content)
        post_id = (form.get('post_id') or '').strip()
        if post_id:
            posts = _load_wall_posts()
            if isinstance(posts, list):
                for p in posts:
                    if str(p.get('id')) == str(post_id):
                        p['img'] = f"/api/wall/img/{fname}"
                        break
                _save_wall_posts(posts)
        url = f"/api/wall/img/{fname}"
        return JSONResponse({"ok": True, "url": url, "img_id": img_id})
    except Exception as e:
        return JSONResponse({"error": f"Lỗi upload ảnh: {str(e)[:150]}"}, status_code=500)


@app.get('/api/wall/img/{fname}')
def api_wall_img_file(fname: str):
    if '..' in fname or '/' in fname:
        return Response(status_code=403)
    img_path = os.path.join(WALL_IMG_DIR, fname)
    if not os.path.exists(img_path):
        return Response(status_code=404)
    ext = os.path.splitext(fname)[1].lower()
    media_type = 'image/png' if ext == '.png' else ('image/jpeg' if ext in ('.jpg', '.jpeg') else 'image/webp')
    return FileResponse(img_path, media_type=media_type)

@app.delete('/api/wall/{post_id}')
def api_wall_delete(post_id: str):
    posts = _load_wall_posts()
    if not isinstance(posts, list):
        return JSONResponse({"error": "No posts"}, status_code=404)
    for i, p in enumerate(posts):
        if p.get('id') == post_id:
            if p.get('video'):
                video_name = p['video'].split('/')[-1]
                video_path = os.path.join(WALL_VIDEO_DIR, video_name)
                if os.path.exists(video_path):
                    os.remove(video_path)
            posts.pop(i)
            _save_wall_posts(posts)
            return JSONResponse({"ok": True})
    return JSONResponse({"error": "Post not found"}, status_code=404)

@app.post('/api/wall/cleanup')
def api_wall_cleanup():
    """Permanently remove auto-generated wall posts that are completely
    unrecoverable (no valid image, no video, no slides, no embed — i.e. a
    post that can never be displayed), and sort the remaining posts
    newest-first. Returns the number of removed posts and the new count.

    NOTE: Conservative — auto posts that HAVE a valid image or a video/embed
    are preserved (they remain visible at view time via
    _wall_posts_for_view). Only use this to purge genuine garbage."""
    posts = _load_wall_posts()
    if not isinstance(posts, list):
        return JSONResponse({"error": "No posts"}, status_code=404)
    before = len(posts)
    kept = []
    removed = []
    for p in posts:
        if not isinstance(p, dict):
            removed.append(str(p))
            continue
        if _is_auto_post(p) and not _post_has_valid_image(p) and not _post_has_slide_info(p) and not (p.get("video") or "").strip():
            removed.append(p.get('id', '?'))
        else:
            kept.append(p)
    # sort newest-first
    kept.sort(key=_created_ts, reverse=True)
    _save_wall_posts(kept)
    return JSONResponse({
        "ok": True,
        "before": before,
        "removed_count": before - len(kept),
        "removed_ids": removed,
        "after": len(kept),
    })

# ===== LANGUAGE & EMOTION DETECTION =====
import random as _random2
from urllib.parse import quote as _quote2

_UA_RW = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', 'Accept-Language': 'vi-VN,vi;q=0.9'}

# Unique character markers for language detection
_UNIQUE_CHARS = {
    'vietnamese': set('đăâêôơưàảãạáằẳẵặắầẩẫậấèẻẽẹéềễểệếìỉĩịíòỏõọóồổỗộốờởỡợớùủũụúừửữựứỳỷỹỵý'),
    'spanish': set('ñáéíóúü¿¡'),
    'portuguese': set('ãõçáéíóúâêôà'),
}

_STOPWORDS = {
    'english': {'the', 'is', 'at', 'which', 'on', 'a', 'an', 'and', 'or', 'but', 'in', 'with', 'to', 'for', 'of', 'not', 'no', 'can', 'had', 'have', 'has', 'was', 'were', 'are', 'be', 'been', 'this', 'that', 'it', 'he', 'she', 'they', 'his', 'her', 'my', 'your', 'our', 'we', 'you', 'i'},
    'vietnamese': {'là', 'của', 'và', 'có', 'được', 'cho', 'không', 'với', 'này', 'đó', 'từ', 'trong', 'đã', 'sẽ', 'một', 'các', 'những', 'về', 'tại', 'người', 'năm', 'đến', 'ra', 'lại', 'như', 'khi', 'để', 'rất', 'cũng', 'mà', 'nếu', 'sau', 'trên', 'theo', 'vì', 'do', 'nên', 'thì', 'mình', 'tôi', 'bạn', 'anh', 'chị', 'em'},
    'portuguese': {'de', 'um', 'que', 'e', 'do', 'da', 'em', 'para', 'com', 'não', 'uma', 'os', 'no', 'se', 'na', 'por', 'mais', 'as', 'dos', 'como', 'mas', 'ao', 'ele', 'das', 'tem', 'seu', 'sua', 'ou', 'quando', 'muito', 'nos', 'já', 'eu', 'também', 'só', 'pelo', 'pela', 'até', 'isso', 'ela', 'entre', 'depois', 'sem', 'mesmo', 'aos', 'são', 'está', 'ter', 'ser', 'foi', 'era', 'há', 'estão', 'você', 'nós', 'eles', 'elas'},
    'spanish': {'de', 'que', 'el', 'en', 'y', 'a', 'los', 'del', 'se', 'las', 'por', 'un', 'para', 'con', 'no', 'una', 'su', 'al', 'es', 'lo', 'como', 'más', 'pero', 'sus', 'le', 'ya', 'o', 'fue', 'este', 'ha', 'si', 'porque', 'esta', 'son', 'entre', 'está', 'cuando', 'muy', 'sin', 'sobre', 'ser', 'también', 'me', 'hasta', 'hay', 'donde', 'han', 'quien', 'están', 'desde', 'todo', 'nos', 'durante', 'todos', 'uno', 'les', 'ni', 'contra', 'otros', 'fueron', 'ese', 'eso', 'ante', 'ellos', 'yo', 'tú', 'él', 'ella', 'nosotros', 'usted', 'ustedes'},
}

def detect_language(text):
    """Detect language from text content using stopword + character analysis."""
    if not text:
        return 'vietnamese'
    text_lower = text.lower()
    text_chars = set(text_lower)
    
    # Strong signal: Vietnamese unique characters
    vn_chars = len(text_chars & _UNIQUE_CHARS['vietnamese'])
    if vn_chars >= 2:
        return 'vietnamese'
    
    # Spanish unique chars (ñ, ¿, ¡)
    es_chars = len(text_chars & _UNIQUE_CHARS['spanish'])
    pt_chars = len(text_chars & _UNIQUE_CHARS['portuguese'])
    
    # Stopword scoring
    words = set(re.findall(r'\b\w+\b', text_lower))
    scores = {}
    for lang, stops in _STOPWORDS.items():
        scores[lang] = len(words & stops) / max(len(stops), 1)
    
    # Disambiguate Portuguese vs Spanish
    pt_markers = {'não', 'pelo', 'pela', 'isso', 'há', 'estão', 'num', 'numa', 'tenho', 'posso', 'você', 'nós', 'eles', 'elas', 'também', 'muito', 'já', 'só', 'até', 'entre', 'depois', 'sem', 'mesmo', 'aos', 'serão'}
    es_markers = {'pero', 'está', 'están', 'porque', 'también', 'hasta', 'donde', 'quien', 'fue', 'son', 'fueron', 'ese', 'eso', 'ante', 'ellos', 'ella', 'nosotros', 'usted', 'ustedes', 'tú', 'él', 'desde', 'todo', 'durante', 'todos', 'uno', 'les', 'ni', 'contra', 'otros', 'fueron'}
    
    pt_overlap = len(words & pt_markers)
    es_overlap = len(words & es_markers)
    
    if scores.get('portuguese', 0) > 0 and pt_overlap > es_overlap:
        return 'portuguese'
    if scores.get('spanish', 0) > 0 and es_overlap > pt_overlap:
        return 'spanish'
    if scores.get('english', 0) > 0.15:
        return 'english'
    
    best = max(scores, key=scores.get)
    return best if scores[best] > 0.05 else 'vietnamese'

# Emotion keyword-based detection
_EMOTION_KEYWORDS = {
    'happy': {
        'en': ['happy', 'joy', 'wonderful', 'great', 'amazing', 'fantastic', 'love', 'excellent', 'beautiful', 'glad', 'delighted', 'pleased', 'cheerful', 'celebrate', 'victory', 'win', 'success'],
        'pt': ['feliz', 'alegria', 'maravilhoso', 'ótimo', 'incrível', 'fantástico', 'amor', 'excelente', 'lindo', 'contente', 'encantado', 'vitória', 'sucesso'],
        'es': ['feliz', 'alegria', 'maravilloso', 'genial', 'increíble', 'fantástico', 'amor', 'excelente', 'hermoso', 'contento', 'encantado', 'victoria', 'éxito'],
        'vi': ['vui', 'hạnh phúc', 'tuyệt vời', 'tuyệt', 'ý nghĩa', 'đẹp', 'thích', 'yêu', 'vui vẻ', 'hân hoan', 'phấn khích', 'chiến thắng', 'thành công'],
    },
    'sad': {
        'en': ['sad', 'unhappy', 'terrible', 'awful', 'horrible', 'miserable', 'depressed', 'grief', 'sorrow', 'tragic', 'unfortunate', 'painful', 'death', 'die', 'kill'],
        'pt': ['triste', 'infeliz', 'terrível', 'horrível', 'miserável', 'deprimido', 'dor', 'trágico', 'infelizmente', 'penoso', 'morte', 'morrer'],
        'es': ['triste', 'infeliz', 'terrible', 'horrible', 'miserable', 'deprimido', 'dolor', 'trágico', 'desafortunado', 'penoso', 'muerte', 'morir'],
        'vi': ['buồn', 'không vui', 'tồi tệ', 'kinh khủng', 'đau khổ', 'đau buồn', 'bi thương', 'khốn nạn', 'đau đớn', 'thảm họa', 'chết', 'mất'],
    },
    'excited': {
        'en': ['excited', 'thrilling', 'amazing', 'wow', 'incredible', 'unbelievable', 'awesome', 'exhilarating', 'electrifying', 'breathtaking', 'breakthrough', 'record'],
        'pt': ['animado', 'emocionante', 'incrível', 'impressionante', 'sensacional', 'eletrizante', 'empolgante', 'recorde'],
        'es': ['emocionante', 'increíble', 'impresionante', 'sensacional', 'electrizante', 'emocionado', 'entusiasmado', 'récord'],
        'vi': ['hào hứng', 'phấn khích', 'thú vị', 'tuyệt cú mèo', 'đỉnh cao', 'ngoạn mục', 'sục sôi', 'kỷ lục', 'đột phá'],
    },
    'humorous': {
        'en': ['funny', 'hilarious', 'joke', 'laugh', 'comedy', 'humor', 'amusing', 'witty', 'sarcastic', 'ironic', 'ridiculous', 'absurd', 'lol', 'haha'],
        'pt': ['engraçado', 'hilário', 'piada', 'rir', 'comédia', 'humor', 'divertido', 'irônico', 'ridículo', 'absurdo', 'kkk'],
        'es': ['gracioso', 'hilarante', 'broma', 'risa', 'comedia', 'humor', 'divertido', 'irónico', 'ridículo', 'absurdo', 'jaja'],
        'vi': ['hài hước', 'buồn cười', 'đùa', 'cười', 'hài', 'vui nhộn', 'hóm hỉnh', 'mỉa mai', 'lố bịch', 'vô lý', 'haha'],
    },
    'serious': {
        'en': ['serious', 'critical', 'important', 'urgent', 'severe', 'grave', 'significant', 'crucial', 'vital', 'essential', 'alarming', 'concerning', 'crisis', 'war', 'conflict'],
        'pt': ['sério', 'crítico', 'importante', 'urgente', 'grave', 'significativo', 'crucial', 'vital', 'essencial', 'preocupante', 'crise', 'guerra', 'conflito'],
        'es': ['serio', 'crítico', 'importante', 'urgente', 'grave', 'significativo', 'crucial', 'vital', 'esencial', 'preocupante', 'crisis', 'guerra', 'conflicto'],
        'vi': ['nghiêm trọng', 'quan trọng', 'khẩn cấp', 'nghiêm túc', 'đáng kể', 'thiết yếu', 'cần thiết', 'báo động', 'lo ngại', 'khủng hoảng', 'chiến tranh', 'xung đột'],
    },
}

def detect_emotion(text, language='vietnamese'):
    """Detect emotion from text using keyword matching."""
    if not text:
        return 'neutral'
    text_lower = text.lower()
    
    scores = {}
    for emotion, lang_keywords in _EMOTION_KEYWORDS.items():
        keywords = lang_keywords.get(language, lang_keywords.get('en', []))
        score = sum(1 for kw in keywords if kw in text_lower)
        scores[emotion] = score
    
    if max(scores.values()) == 0:
        return 'neutral'
    
    return max(scores, key=scores.get)

def detect_language_and_emotion(title, text):
    """Detect both language and emotion from article content."""
    combined = f"{title} {text}"
    lang = detect_language(combined)
    emotion = detect_emotion(combined, lang)
    return lang, emotion

# Voice selection based on language and emotion (using MultilingualNeural voices)
VOICE_BY_LANG_EMOTION = {
    'vietnamese': {
        'happy': ('vi-VN-HoaiMyNeural', 'vui'),
        'sad': ('vi-VN-NamMinhNeural', 'buồn'),
        'excited': ('vi-VN-HoaiMyNeural', 'hào hứng'),
        'humorous': ('vi-VN-HoaiMyNeural', 'vui'),
        'serious': ('vi-VN-NamMinhNeural', 'nghiêm túc'),
        'neutral': ('vi-VN-HoaiMyNeural', 'trung_tinh'),
    },
    'portuguese': {
        'happy': ('pt-BR-ThalitaMultilingualNeural', 'feliz'),
        'sad': ('pt-BR-ThalitaMultilingualNeural', 'triste'),
        'excited': ('pt-BR-ThalitaMultilingualNeural', 'animado'),
        'humorous': ('pt-BR-ThalitaMultilingualNeural', 'engraçado'),
        'serious': ('pt-BR-ThalitaMultilingualNeural', 'sério'),
        'neutral': ('pt-BR-ThalitaMultilingualNeural', 'neutro'),
    },
    'english': {
        'happy': ('en-US-AndrewMultilingualNeural', 'happy'),
        'sad': ('en-AU-WilliamMultilingualNeural', 'sad'),
        'excited': ('en-US-AndrewMultilingualNeural', 'excited'),
        'humorous': ('en-US-AndrewMultilingualNeural', 'funny'),
        'serious': ('en-AU-WilliamMultilingualNeural', 'serious'),
        'neutral': ('en-US-AndrewMultilingualNeural', 'neutral'),
    },
    'french': {
        'happy': ('fr-FR-VivienneMultilingualNeural', 'heureux'),
        'sad': ('fr-FR-RemyMultilingualNeural', 'triste'),
        'excited': ('fr-FR-VivienneMultilingualNeural', 'excité'),
        'humorous': ('fr-FR-VivienneMultilingualNeural', 'drôle'),
        'serious': ('fr-FR-RemyMultilingualNeural', 'sérieux'),
        'neutral': ('fr-FR-VivienneMultilingualNeural', 'neutre'),
    },
    'german': {
        'happy': ('de-DE-SeraphinaMultilingualNeural', 'glücklich'),
        'sad': ('de-DE-FlorianMultilingualNeural', 'traurig'),
        'excited': ('de-DE-SeraphinaMultilingualNeural', 'aufgeregt'),
        'humorous': ('de-DE-SeraphinaMultilingualNeural', 'lustig'),
        'serious': ('de-DE-FlorianMultilingualNeural', 'ernst'),
        'neutral': ('de-DE-SeraphinaMultilingualNeural', 'neutral'),
    },
    'korean': {
        'happy': ('ko-KR-HyunsuMultilingualNeural', '행복'),
        'sad': ('ko-KR-HyunsuMultilingualNeural', '슬픔'),
        'excited': ('ko-KR-HyunsuMultilingualNeural', '흥분'),
        'humorous': ('ko-KR-HyunsuMultilingualNeural', '유쾌'),
        'serious': ('ko-KR-HyunsuMultilingualNeural', '진지'),
        'neutral': ('ko-KR-HyunsuMultilingualNeural', '중립'),
    },
    'italian': {
        'happy': ('it-IT-GiuseppeMultilingualNeural', 'felice'),
        'sad': ('it-IT-GiuseppeMultilingualNeural', 'triste'),
        'excited': ('it-IT-GiuseppeMultilingualNeural', 'emozionato'),
        'humorous': ('it-IT-GiuseppeMultilingualNeural', 'divertente'),
        'serious': ('it-IT-GiuseppeMultilingualNeural', 'serio'),
        'neutral': ('it-IT-GiuseppeMultilingualNeural', 'neutro'),
    },
}

# All valid voice IDs (new MultilingualNeural format)
VALID_VOICES = {
    'vi-VN-HoaiMyNeural', 'vi-VN-NamMinhNeural',
    'en-US-AndrewMultilingualNeural', 'en-AU-WilliamMultilingualNeural',
    'pt-BR-ThalitaMultilingualNeural',
    'fr-FR-VivienneMultilingualNeural', 'fr-FR-RemyMultilingualNeural',
    'de-DE-SeraphinaMultilingualNeural', 'de-DE-FlorianMultilingualNeural',
    'ko-KR-HyunsuMultilingualNeural',
    'it-IT-GiuseppeMultilingualNeural',
}

def get_voice_for_content(title, text, preferred_voice=None):
    """Get appropriate voice based on content language and emotion."""
    # Accept the new MultilingualNeural voices directly
    if preferred_voice and preferred_voice in VALID_VOICES:
        return preferred_voice
    
    # Also accept old shorthand voice IDs and map them to new format
    old_voice_map = {
        'hoaimy': 'vi-VN-HoaiMyNeural',
        'namminh': 'vi-VN-NamMinhNeural',
        'andrew': 'en-US-AndrewMultilingualNeural',
        'jenny': 'en-US-AndrewMultilingualNeural',
        'thalita': 'pt-BR-ThalitaMultilingualNeural',
        'pt_thalita': 'pt-BR-ThalitaMultilingualNeural',
        'pt_francisco': 'pt-BR-ThalitaMultilingualNeural',
        'ela': 'en-US-AndrewMultilingualNeural',
        'es_carlos': 'en-US-AndrewMultilingualNeural',
        'denise': 'fr-FR-VivienneMultilingualNeural',
        'katja': 'de-DE-SeraphinaMultilingualNeural',
        'nanami': 'en-US-AndrewMultilingualNeural',
        'sunhee': 'ko-KR-HyunsuMultilingualNeural',
        'xiaochen': 'en-US-AndrewMultilingualNeural',
    }
    if preferred_voice and preferred_voice in old_voice_map:
        return old_voice_map[preferred_voice]
    
    lang, emotion = detect_language_and_emotion(title, text)
    lang_map = VOICE_BY_LANG_EMOTION.get(lang, VOICE_BY_LANG_EMOTION['vietnamese'])
    voice, _ = lang_map.get(emotion, lang_map['neutral'])
    return voice


def _is_relevant_image(img_url, title, text):
    """Check if an image is relevant to the article content."""
    if not img_url:
        return False
    skip_patterns = ['pixel', 'analytics', 'tracking', '1x1.gif', 'spacer.gif',
                     'logo', 'icon', 'avatar', 'emoji', 'smiley', 'sprite',
                     'advertisement', 'ad-banner', 'sponsored', 'banner-ads']
    img_lower = img_url.lower()
    for p in skip_patterns:
        if p in img_lower:
            return False
    if not any(img_lower.endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.webp', '.gif']):
        return False
    return True


def _filter_relevant_images(images, title, text, max_images=8):
    """Filter and rank images by relevance to article content."""
    if not images:
        return []
    seen = set()
    relevant = []
    for img in images:
        if img in seen:
            continue
        seen.add(img)
        if _is_relevant_image(img, title, text):
            relevant.append(img)
    return relevant[:max_images]


def _scrape_article_for_rewrite(url):
    """Scrape article: extract title, paragraphs, RELEVANT images, OG image."""
    try:
        r = req.get(url, headers=_UA_RW, timeout=15, allow_redirects=True)
        r.encoding = 'utf-8'
        soup = BeautifulSoup(r.text, 'lxml')
        for tag in soup.find_all(['script', 'style', 'nav', 'footer', 'aside', 'form']):
            tag.decompose()
        h1 = soup.find('h1')
        ogt = soup.find('meta', property='og:title')
        title = (h1.get_text(strip=True) if h1 else '') or (ogt.get('content', '') if ogt else '')
        ogi = soup.find('meta', property='og:image')
        og_img = ogi.get('content', '') if ogi else ''
        if og_img and og_img.startswith('//'):
            og_img = 'https:' + og_img
        block = None
        for sel in ['article', '.singular-content', '.detail-content', '.fck_detail', '.content-detail', '.knc-content', 'main', '.cms-body', '.article__body']:
            el = soup.select_one(sel)
            if el and len(el.find_all('p')) >= 2:
                block = el
                break
        if not block:
            block = soup.body or soup
        paragraphs = []
        all_images = []
        seen_imgs = set()
        if og_img and og_img not in seen_imgs:
            all_images.append(og_img)
            seen_imgs.add(og_img)
        for el in block.find_all(['p', 'h2', 'h3', 'figure', 'img'], recursive=True):
            if el.name == 'p':
                t = _clean(el.get_text(strip=True))
                if t and len(t) > 40:
                    paragraphs.append(t)
            elif el.name in ('figure', 'img'):
                im = el if el.name == 'img' else el.find('img')
                if im:
                    src = im.get('data-src') or im.get('src') or im.get('data-original') or ''
                    if src and 'base64' not in src:
                        if src.startswith('//'):
                            src = 'https:' + src
                        if src not in seen_imgs:
                            all_images.append(src)
                            seen_imgs.add(src)
        # Filter to relevant images only
        relevant_images = _filter_relevant_images(all_images, title, ' '.join(paragraphs[:5]))
        return {'title': _clean(title), 'paragraphs': paragraphs, 'images': relevant_images, 'og_img': og_img}
    except Exception:
        return None


def _extract_key_points_rw(paragraphs, max_points=5):
    r"""Extract key points from paragraphs - extracts ALL sentences, not just first one.
    
    Fixes: Original regex `^(.+?[.!?])\s` only captured first sentence per paragraph.
    Now splits on all sentence boundaries and takes valid sentences until max_points.
    """
    points = []
    
    for p in paragraphs:
        if len(points) >= max_points:
            break
        
        p = _clean(p)
        if not p:
            continue
        
        # Split paragraph into sentences using Vietnamese + English punctuation
        sentences = re.split(r'(?<=[.!?])\s+(?=[A-ZÀ-Ỹ0-9])', p)
        sentences = [s.strip() for s in sentences if s.strip()]
        
        for sentence in sentences:
            if len(points) >= max_points:
                break
            
            # Clean sentence - remove extra whitespace
            sentence = _clean(sentence)
            
            if len(sentence) < 30:
                continue
            
            # Check for duplicates
            if any(sentence[:60] in existing for existing in points):
                continue
            
            # Ensure sentence ends with punctuation
            if not sentence.endswith(('.', '!', '?')):
                sentence = sentence + '.'
            
            points.append(sentence)
    
    # If no valid sentences found, take chunks from raw text
    if not points:
        raw = '\n'.join(paragraphs)
        for i in range(0, min(len(raw), max_points * 300), 280):
            chunk = _clean(raw[i:i+280])
            if len(chunk) >= 30 and chunk not in points:
                points.append(chunk + ('.' if not chunk.endswith('.') else ''))
            if len(points) >= max_points:
                break
    
    return points


@app.post("/api/rewrite_slide")
async def api_rewrite_slide(request: Request):
    """Fast rewrite as SLIDES - no AI needed, instant response."""
    body = await request.json()
    url = _clean(body.get("url", ""))
    context = body.get("context", "")
    preferred_voice = body.get("voice", "")  # Accept custom voice selection
    if not url and not context:
        return JSONResponse({"error": "Cần URL hoặc nội dung"}, status_code=400)
    data = None
    if url and url.startswith("http"):
        data = _scrape_article_for_rewrite(url)
    if not data and context:
        paragraphs = [_clean(p) for p in context.split('\n') if len(_clean(p)) > 40]
        data = {'title': paragraphs[0][:80] if paragraphs else 'Bài viết', 'paragraphs': paragraphs, 'images': [], 'og_img': ''}
    if not data or not data.get('paragraphs'):
        return JSONResponse({"error": "Không đọc được bài viết"}, status_code=422)
    points = _extract_key_points_rw(data['paragraphs'], max_points=12)
    if not points:
        return JSONResponse({"error": "Không tìm được ý chính"}, status_code=422)
    images = data.get('images', [])
    slides = []
    for i, point in enumerate(points):
        img = images[i] if i < len(images) else (images[-1] if images else '')
        if img and 'cdnphoto.dantri' in img:
            img = '/api/proxy/img?url=' + _quote2(img, safe='')
        slides.append({'text': point, 'image': img, 'index': i + 1})
    summary_text = '\n\n'.join([f"• {s['text']}" for s in slides])
    
    # Auto-detect language and emotion
    lang, emotion = detect_language_and_emotion(data['title'], summary_text)
    # Use preferred voice if provided, otherwise auto-detect
    voice = preferred_voice if preferred_voice else get_voice_for_content(data['title'], summary_text)
    
    post = {
        "id": str(int(time.time() * 1000)) + str(_random2.randint(100, 999)),
        "title": data['title'],
        "text": summary_text,
        "img": images[0] if images else '',
        "url": url,
        "kind": "slide_summary",
        "slides": slides,
        "images": images[:10],
        "video": "",
        "voice": voice,
        "emotion": emotion,
        "language": lang,
        "ts": int(time.time())
    }
    posts = _load_wall_posts()
    posts.insert(0, post)
    _save_wall_posts(posts)
    return JSONResponse({"post": post, "slides": slides})


@app.post("/api/rewrite_share")
async def api_rewrite_share(request: Request):
    """Rewrite article and post to Tường AI with SLIDES + AI text."""
    body = await request.json()
    url = _clean(body.get("url", ""))
    ctx = _clean(body.get("context", ""))
    preferred_voice = body.get("voice", "")  # Accept custom voice selection
    if not url and not ctx:
        return JSONResponse({"error": "Cần URL hoặc nội dung"}, status_code=400)
    data = None
    if url and url.startswith("http"):
        data = _scrape_article_for_rewrite(url)
    if not data and ctx:
        paragraphs = [_clean(p) for p in ctx.split('\n') if len(_clean(p)) > 40]
        data = {'title': paragraphs[0][:80] if paragraphs else 'Bài viết', 'paragraphs': paragraphs, 'images': [], 'og_img': ''}
    if not data or not data.get('paragraphs'):
        return JSONResponse({"error": "Không đọc được bài viết"}, status_code=422)
    raw_text = '\n'.join(data['paragraphs'])
    if len(raw_text) < 50:
        raw_text = ctx[:14000]
    if len(raw_text) < 50:
        return JSONResponse({"error": "Bài viết quá ngắn"}, status_code=422)
    domain = ''
    try:
        from urllib.parse import urlparse
        domain = urlparse(url).netloc.replace('www.', '')
    except:
        pass
    
    # Generate AI summary text
    ai_text = None
    try:
        import ai_ext
        if hasattr(ai_ext, 'qwen_generate'):
            prompt = f'Tóm tắt đăng Tường AI:\nTiêu đề: {data["title"]}\n{raw_text[:14000]}\n\n4-6 ý chính. Cuối ghi nguồn.'
            ai_text = await ai_ext.qwen_generate(prompt, max_tokens=1000)
    except Exception:
        pass
    if not ai_text or len(ai_text) < 80:
        key_pts = _extract_key_points_rw(data['paragraphs'], max_points=12)
        if key_pts:
            ai_text = '\n\n'.join([f"• {p}" for p in key_pts])
        else:
            ai_text = f"Tóm tắt: {data['title']}\n\n{raw_text[:1200]}\n\nNguồn: {domain}"
    
    # Build slides from key points (FIX: include slides in rewrite_share too!)
    points = _extract_key_points_rw(data['paragraphs'], max_points=12)
    images = data.get('images', [])
    slides = []
    for i, point in enumerate(points):
        img = images[i] if i < len(images) else (images[-1] if images else '')
        if img and 'cdnphoto.dantri' in img:
            img = '/api/proxy/img?url=' + _quote2(img, safe='')
        slides.append({'text': point, 'image': img, 'index': i + 1})
    
    # Auto-detect language and emotion
    lang, emotion = detect_language_and_emotion(data['title'], ai_text)
    # Use preferred voice if provided, otherwise auto-detect
    voice = preferred_voice if preferred_voice else get_voice_for_content(data['title'], ai_text)
    
    post = {
        "id": str(int(time.time() * 1000)) + str(_random2.randint(100, 999)),
        "title": data['title'],
        "text": ai_text,
        "img": images[0] if images else '',
        "url": url,
        "kind": "rewrite",
        "slides": slides,
        "images": images[:10],
        "video": "",
        "voice": voice,
        "emotion": emotion,
        "language": lang,
        "ts": int(time.time())
    }
    posts = _load_wall_posts()
    posts.insert(0, post)
    _save_wall_posts(posts)
    return JSONResponse({"post": post, "slides": slides})


@app.post("/api/url_wall")
async def api_url_wall(request: Request):
    """Submit URL to add to Tường AI."""
    body = await request.json()
    url = _clean(body.get("url", ""))
    if not url or not url.startswith('http'):
        return JSONResponse({"error": "URL không hợp lệ"}, status_code=400)
    # Reuse rewrite_share logic
    req._body = json.dumps({"url": url}).encode()
    return await api_rewrite_share(request)


# ===== PERSONAL OPINION POST v2: AI tổng hợp bài viết từ quan điểm + nguồn tin HOT =====

# ===== KEYWORD EXTRACTION FROM OPINION =====
_STOP_WORDS_EX = set("""
và của các những một được trong với cho tại sau trước khi không người 
việt nam hôm nay mới nhất nóng tin tức cập nhật theo từ đến là có thì 
này đã để về lại nên cũng rất như vì do nếu sẽ nếu thế nhưng mà vẫn 
đang vào ra hơn đây đó nào cả cùng đã từng hãy còn chỉ cũng đều khiến 
được đã bị bởi qua những lúc cái gì cô chú bác anh chị em bạn tôi mình 
ông bà thầy vì vậy chính phải ấy đấy đâu đó thôi nhé đấy ạ nhỉ 
ngày tháng năm giờ phút giây tuần tháng quý
""".strip().split())

def _extract_keywords_from_opinion(text, max_keywords=5):
    """Extract meaningful keywords from user's opinion for news search."""
    if not text:
        return []
    text = text.lower()
    text = re.sub(r'https?://\S+', '', text)
    text = re.sub(r'[^\w\sÀ-ỹ]', ' ', text)
    text = re.sub(r'\s+', ' ', text).strip()
    words = [w for w in text.split() if len(w) > 2 and w not in _STOP_WORDS_EX]
    word_scores = {}
    for w in words:
        word_scores[w] = word_scores.get(w, 0) + 1
    sorted_words = sorted(word_scores.items(), key=lambda x: -x[1])
    top_words = [w for w, s in sorted_words[:max_keywords]]
    phrases = []
    for i in range(len(words) - 1):
        phrase = words[i] + ' ' + words[i + 1]
        if len(phrase) > 5:
            phrases.append(phrase)
    phrase_scores = {}
    for p in phrases:
        phrase_scores[p] = phrase_scores.get(p, 0) + 1
    sorted_phrases = sorted(phrase_scores.items(), key=lambda x: -x[1])
    top_phrases = [p for p, s in sorted_phrases[:3]]
    result = []
    for p in top_phrases:
        if p not in result:
            result.append(p)
    for w in top_words:
        if w not in result:
            result.append(w)
    return result[:max_keywords]


@app.post("/api/personal_post/preview")
async def api_personal_post_preview(request: Request):
    """Preview personal post: fetch full articles, let AI compose logical article with images."""
    body = await request.json()
    opinion = _clean(body.get("opinion", ""))
    selected_topics = body.get("selected_topics", []) or []
    selected_sources = body.get("selected_sources", []) or []
    
    if not opinion or len(opinion) < 10:
        return JSONResponse({"error": "Quan điểm cá nhân quá ngắn (cần ít nhất 10 ký tự)"}, status_code=400)
    
    # Lấy keywords từ QUAN ĐIỂM CÁ NHÂN để tìm nguồn tin chính xác
    keywords = _extract_keywords_from_opinion(opinion, max_keywords=5)
    if keywords:
        selected_topics = keywords[:3]
    else:
        # Fallback: hot topics
        hot = _get_hot_topics()
        selected_topics = [t.get("topic", "") for t in hot[:3] if t.get("topic")]
    
    # Tìm nguồn tin
    all_sources = []
    seen_urls = set()
    for topic in selected_topics[:3]:
        sources = _search_all(topic, limit=5)
        for s in sources:
            if s.get("url") and s["url"] not in seen_urls:
                seen_urls.add(s["url"])
                all_sources.append(s)
                if len(all_sources) >= 6:
                    break
        if len(all_sources) >= 6:
            break
    
    for src in selected_sources:
        if src.get("url") and src["url"] not in seen_urls:
            all_sources.insert(0, src)
    
    # Scrape nội dung đầy đủ từng nguồn (paragraphs + images)
    source_details = []
    source_images = []
    for src in all_sources[:5]:
        url = src.get("url", "")
        if not url:
            continue
        try:
            art = _scrape_article_for_rewrite(url)
            if art:
                src_detail = {
                    "title": art.get("title", src.get("title", "")),
                    "url": url,
                    "via": src.get("via", ""),
                    "paragraphs": art.get("paragraphs", [])[:8],
                    "images": art.get("images", [])[:3],
                    "og_image": art.get("og_img", "")
                }
                source_details.append(src_detail)
                # Collect images for proxy
                for img in art.get("images", [])[:2]:
                    if any(x in img for x in ["cdnphoto.dantri", "vnexpress", "vcdn", "refooty"]):
                        img = "/api/proxy/img?url=" + _quote2(img, safe="")
                    source_images.append(img)
        except:
            pass
        if len(source_details) >= 5:
            break
    
    # Tạo title từ opinion
    opinion_words = re.findall(r"[A-Za-zÀ-ỹ0-9]+", opinion)
    title_words = opinion_words[:8] if len(opinion_words) >= 8 else opinion_words[:4]
    title = " ".join([w[0].upper() + w[1:] for w in title_words]) if title_words else "Quan điểm cá nhân"
    title = title[:80]
    
    # AI sinh bài viết hoàn chỉnh
    ai_text = None
    try:
        import ai_ext
        if hasattr(ai_ext, 'qwen_generate'):
            # Build detailed context from source articles
            source_context = ""
            for i, sd in enumerate(source_details[:5]):
                src_title = sd.get("title", "")
                src_via = sd.get("via", "")
                src_paras = sd.get("paragraphs", [])
                source_context += f"\n=== Nguồn {i+1}: {src_title} ({src_via}) ===\n"
                for j, p in enumerate(src_paras[:4]):
                    source_context += f"  - {p[:300]}\n"
            
            prompt = (
                "QUAN ĐIỂM: " + opinion[:500] + "\nNGUỒN: " + source_context[:1000] + "\n\n"
                "=== NGUỒN TIN THAM KHẢO ===\n" + source_context + "\n\n"
                "=== YÊU CẦU VIẾT BÀI THEO SLIDE ===\n"
                "Viết bài thành 5-6 ĐOẠN VĂN NGẮN, mỗi đoạn là 1 SLIDE.\n"
                "\n"
                "QUAN TRỌNG NHẤT: MỗI SLIDE PHẢI KẾT HỢP QUAN ĐIỂM CÁ NHÂN + NỘI DUNG NGUỒN TIN, KHÔNG PHẢI CHỈ NÓI VỀ NGUỒN TIN.\n"
                "\n"
                "SLIDE 1 - MỞ ĐẦU:\n"
                "- NHIỆN HỮU QUAN ĐIỂM CÁ NHÂN LÊN ĐẦU\n"
                "- Giới thiệu chủ đề, nêu rõ quan điểm của bạn (dựa vào QUAN ĐIỂM CÁ NHÂN ở trên)\n"
                "- 2-4 câu hoàn chỉnh\n"
                "\n"
                "SLIDE 2-3-4-5 - PHÂN TÍCH:\n"
                "- Mỗi slide: B�Commencer bằng QUAN ĐIỂM CÁ NHÂN, sau đó dẫn chứng từ 1 nguồn tin\n"
                "- Ví dụ: \"Theo quan điểm của tôi, đây là vấn đề cần lưu ý. Theo VnExpress...\"\n"
                "- Dẫn chứng từ nguồn (ghi rõ tên báo: Theo VnExpress, Theo Thanh Niên...)\n"
                "- 2-4 câu hoàn chỉnh mỗi slide\n"
                "\n"
                "SLIDE 6 - KẾT LUẬN:\n"
                "- Tổng kết quan điểm cá nhân, đưa ra nhận định cuối cùng\n"
                "- 2-3 câu hoàn chỉnh\n"
                "\n"
                "Định dạng đầu ra:\n"
                "---SLIDE 1---\n"
                "[nội dung đoạn văn slide 1]\n"
                "---SLIDE 2---\n"
                "[nội dung đoạn văn slide 2]\n"
                "...v.v...\n"
                "\n"
                "QUAN TRỌNG:\n"
                "- Mỗi slide là 1 đoạn văn HOÀN CHỈNH, 2-4 câu\n"
                "- PHẢI KẾT THÚC BẰNG DẤU CHẤM (.) HOẢN TOÀN\n"
                "- Kết hợp QUAN ĐIỂM CÁ NHÂN với NỘI DUNG NGUỒN TIN\n"
                "- Không gạch đầu dòng, không bullet points\n"
                "- Viết liền mạch tự nhiên, giọng văn báo chí\n"
                "- Mỗi slide phải khác nhau, không lặp ý\n"
                "- Độ dài: 300-600 từ"
            )
            ai_text = None  # Không dùng AI, để code tự kết hợp opinion + source
    except:
        pass
    
    if not ai_text or len(ai_text) < 100:
        # Fallback: build article manually
        ai_text = "## " + title + "\n\n"
        ai_text += opinion + "\n\n"
        for i, sd in enumerate(source_details[:5]):
            ai_text += "### " + sd.get("title", f"Nguồn {i+1}") + "\n"
            for p in sd.get("paragraphs", [])[:3]:
                ai_text += p[:250] + "\n"
            ai_text += "*Nguồn: " + sd.get("via", "") + "*\n\n"
        ai_text += "\n---\n*Bài viết tổng hợp từ quan điểm cá nhân và các nguồn tin liên quan*"
    
    # Parse slides từ AI output (format: ---SLIDE N--- content)
    slides = []
    if ai_text:
        # Try to parse the ---SLIDE--- format
        pattern = r'---SLIDE\s*(\d+)---\s*\n(.*?)(?=---SLIDE|\Z)'
        matches = re.findall(pattern, ai_text, re.DOTALL)
        
        if matches:
            for idx, (num, content) in enumerate(matches):
                # Normalize: ensure complete sentences
                text = _ensure_sentence_complete(content)
                if len(text) > 40:
                    img = source_images[idx] if idx < len(source_images) else ""
                    slides.append({"text": text, "image": img, "index": idx + 1})
        
        # If we have parsed slides, ensure minimum 3
        if len(slides) < 3:
            # Use parsed slides as base, fill remaining from AI text
            used_indices = set()
            for s in slides:
                used_indices.add(s['index'] - 1)
            
            # Split remaining AI text into more slides
            sentences = re.split(r'(?<=[.!?])\s+', ai_text)
            current_chunk = ""
            next_idx = len(slides)
            
            for sent in sentences:
                sent = _ensure_sentence_complete(sent)
                if len(sent) < 20:
                    continue
                
                # Skip if this sentence is already in parsed slides
                found = False
                for slide in slides:
                    if sent[:50] in slide['text']:
                        found = True
                        break
                
                if found:
                    continue
                    
                if current_chunk and len(current_chunk + " " + sent) <= 380:
                    current_chunk += " " + sent
                else:
                    if len(current_chunk) > 50:
                        img = source_images[next_idx] if next_idx < len(source_images) else ""
                        slides.append({"text": current_chunk, "image": img, "index": next_idx + 1})
                    current_chunk = sent
                    next_idx += 1
            
            # Add final chunk
            if len(current_chunk) > 50 and next_idx < 6:
                img = source_images[next_idx] if next_idx < len(source_images) else ""
                slides.append({"text": current_chunk, "image": img, "index": next_idx + 1})
    
    # Ultimate fallback: create slides from opinion + source
    if len(slides) < 2:
        slides = []
        # Slide 1: opinion
        if opinion and len(opinion) > 20:
            slides.append({"text": opinion[:450], "image": source_images[0] if source_images else "", "index": 1})
        
        # Slide 2-6: from AI text or sources
        if ai_text:
            sentences = re.split(r'(?<=[.!?])\s+', ai_text)
            for i, sent in enumerate(sentences[:5]):
                text = _ensure_sentence_complete(_clean(sent))
                if len(text) > 60:
                    if len(slides) < 6:
                        img = source_images[len(slides)] if len(slides) < len(source_images) else ""
                        slides.append({"text": text, "image": img, "index": len(slides) + 1})
        
        # Fill remaining with key points from sources - KẾT HỢP VỚI QUAN ĐIỂM CÁ NHÂN
        src_idx = len(slides)
        while len(slides) < 4 and src_idx < len(source_details):
            paragraphs = source_details[src_idx].get("paragraphs", [])
            src_title = source_details[src_idx].get("title", "")
            src_via = source_details[src_idx].get("via", "")
            for p in paragraphs[:2]:
                if len(p) > 60 and len(slides) < 6:
                    # Kết hợp opinion với nội dung source
                    combined = f"Theo góc nhìn của tôi, {opinion[:100]}... Theo {src_via}: {p[:250]}"
                    img = source_images[len(slides)] if len(slides) < len(source_images) else ""
                    slides.append({"text": _ensure_sentence_complete(combined), "image": img, "index": len(slides) + 1})
                    break  # Mỗi nguồn 1 slide
            src_idx += 1
        
        # Final fallback: ensure at least 2-3 slides
        while len(slides) < 3:
            idx = len(slides)
            if idx == 0 and opinion:
                slides.append({"text": opinion[:400], "image": "", "index": 1})
            elif ai_text:
                slides.append({"text": ai_text[idx*300:(idx+1)*300], "image": "", "index": idx + 1})
            else:
                slides.append({"text": f"Nguồn tham khảo {idx + 1}", "image": "", "index": idx + 1})
    
    preview = {
        "title": title,
        "text": ai_text,
        "opinion": opinion,
        "images": source_images[:10],
        "sources": source_details[:5],
        "slides": slides[:6]  # Max 6 slides
    }
    
    return JSONResponse({"preview": preview})


@app.post("/api/personal_post")
async def api_personal_post(request: Request):
    """Create and save personal opinion post."""
    body = await request.json()
    opinion = _clean(body.get("opinion", ""))
    selected_topics = body.get("selected_topics", []) or []
    selected_sources = body.get("selected_sources", []) or []
    custom_title = body.get("custom_title", "")
    custom_slides = body.get("custom_slides", [])
    
    if not opinion or len(opinion) < 10:
        return JSONResponse({"error": "Quan điểm cá nhân quá ngắn (cần ít nhất 10 ký tự)"}, status_code=400)
    
    if not selected_topics:
        # Lấy keywords từ QUAN ĐIỂM CÁ NHÂN để tìm nguồn tin chính xác
        keywords = _extract_keywords_from_opinion(opinion, max_keywords=5)
        if keywords:
            selected_topics = keywords[:3]
        else:
            hot = _get_hot_topics()
            selected_topics = [t.get("topic", "") for t in hot[:3] if t.get("topic")]
    
    all_sources = []
    seen_urls = set()
    for topic in selected_topics[:3]:
        sources = _search_all(topic, limit=5)
        for s in sources:
            if s.get("url") and s["url"] not in seen_urls:
                seen_urls.add(s["url"])
                all_sources.append(s)
                if len(all_sources) >= 6:
                    break
        if len(all_sources) >= 6:
            break
    
    for src in selected_sources:
        if src.get("url") and src["url"] not in seen_urls:
            all_sources.insert(0, src)
    
    source_details = []
    source_images = []
    for src in all_sources[:5]:
        url = src.get("url", "")
        if not url:
            continue
        try:
            art = _scrape_article_for_rewrite(url)
            if art:
                src_detail = {
                    "title": art.get("title", src.get("title", "")),
                    "url": url,
                    "via": src.get("via", ""),
                    "paragraphs": art.get("paragraphs", [])[:6],
                    "images": art.get("images", [])[:2],
                    "og_image": art.get("og_img", "")
                }
                source_details.append(src_detail)
                for img in art.get("images", [])[:2]:
                    if any(x in img for x in ["cdnphoto.dantri", "vnexpress", "vcdn", "refooty"]):
                        img = "/api/proxy/img?url=" + _quote2(img, safe="")
                    source_images.append(img)
        except:
            pass
    
    # Title
    if custom_title:
        title = custom_title[:80]
    else:
        opinion_words = re.findall(r"[A-Za-zÀ-ỹ0-9]+", opinion)
        title_words = opinion_words[:8] if len(opinion_words) >= 8 else opinion_words[:4]
        title = " ".join([w[0].upper() + w[1:] for w in title_words]) if title_words else "Quan điểm cá nhân"
        title = title[:80]
    
    # AI sinh bài
    ai_text = None
    try:
        import ai_ext
        if hasattr(ai_ext, 'qwen_generate'):
            source_context = ""
            for i, sd in enumerate(source_details[:5]):
                src_title = sd.get("title", "")
                src_via = sd.get("via", "")
                src_paras = sd.get("paragraphs", [])
                source_context += f"\nNguồn {i+1}: {src_title} ({src_via})\n"
                for j, p in enumerate(src_paras[:3]):
                    source_context += f"  - {p[:300]}\n"
            prompt = (
                "QUAN ĐIỂM: " + opinion[:500] + "\nNGUỒN: " + source_context[:1000] + "\n\n"
                "=== NGUỒN TIN ===\n" + source_context + "\n\n"
                "=== YÊU CẦU VIẾT BÀI THEO SLIDE ===\n"
                "Viết bài thành 5-6 ĐOẠN VĂN NGẮN, mỗi đoạn là 1 SLIDE.\n"
                "\n"
                "SLIDE 1 - MỞ ĐẦU: Giới thiệu chủ đề, nêu quan điểm cá nhân (2-4 câu hoàn chỉnh)\n"
                "SLIDE 2-3-4-5 - PHÂN TÍCH: Mỗi slide dùng 1 nguồn tin cụ thể, kết hợp quan điểm cá nhân, ghi rõ nguồn (Theo VnExpress...), 2-4 câu hoàn chỉnh, thành 1 đoạn văn hoàn chỉnh\n"
                "SLIDE 6 - KẾT LUẬN: Tổng kết quan điểm, nhận định cuối cùng (2-3 câu hoàn chỉnh)\n"
                "\n"
                "Định dạng:\n"
                "---SLIDE 1---\n[đoạn văn hoàn chỉnh kết thúc bằng dấu chấm]\n---SLIDE 2---\n[đoạn văn hoàn chỉnh kết thúc bằng dấu chấm]\n...\n"
                "\n"
                "QUAN TRỌNG: Mỗi slide là 1 đoạn văn HOÀN CHỈNH, 2-4 câu, PHẢI KẾT THÚC BẰNG DẤU CHẤM (.). Kết hợp QUAN ĐIỂM + NGUỒN TIN. Không gạch đầu dòng. Viết liền mạch. 300-600 từ."
            )
            ai_text = None  # Không dùng AI, để code tự kết hợp opinion + source
    except:
        pass
    
    if not ai_text or len(ai_text) < 100:
        ai_text = "## " + title + "\n\n" + opinion + "\n\n"
        for i, sd in enumerate(source_details[:5]):
            ai_text += "### " + sd.get("title", "") + "\n"
            for p in sd.get("paragraphs", [])[:2]:
                ai_text += p[:250] + "\n"
            ai_text += "\n---\n*Nguồn: " + sd.get("via", "") + "*\n\n"
    
    # Tạo slides
    if custom_slides and len(custom_slides) > 0:
        slides = []
        for i, slide in enumerate(custom_slides):
            slides.append({
                "text": slide.get("text", ""),
                "image": slide.get("image", ""),
                "index": i + 1
            })
    else:
        slides = []
        # Parse từ AI output (format: ---SLIDE N---)
        if ai_text:
            pattern = r'---SLIDE\s*(\d+)---\s*\n(.*?)(?=---SLIDE|\Z)'
            matches = re.findall(pattern, ai_text, re.DOTALL)
            if matches:
                for idx, (num, content) in enumerate(matches):
                    # Normalize: ensure complete sentences
                    text = _ensure_sentence_complete(content)
                    if len(text) > 30:
                        img = source_images[idx] if idx < len(source_images) else ""
                        slides.append({"text": text, "image": img, "index": idx + 1})
            
            # Fallback: split by paragraphs
            if len(slides) < 3:
                paragraphs = [p.strip() for p in re.split(r'\n\n+', ai_text) if p.strip()]
                slides = []
                para_count = 0
                for p in paragraphs:
                    p = p.strip()
                    if p.startswith('#') or p.startswith('---') or p.startswith('*Nguồn'):
                        continue
                    # Normalize: ensure complete sentences
                    p_normalized = _ensure_sentence_complete(p)
                    if len(p_normalized) > 50:
                        img = source_images[para_count] if para_count < len(source_images) else ""
                        slides.append({"text": p_normalized, "image": img, "index": para_count + 1})
                        para_count += 1
                        if para_count >= 6:
                            break
        
        if len(slides) < 2:
            slides = []
            # Slide 1: QUAN ĐIỂM CÁ NHÂN (BẮT BUỘC)
            slides.append({"text": f"Theo quan điểm cá nhân: {opinion[:400]}", "image": source_images[0] if source_images else "", "index": 1})
            
            # Slide 2-6: KẾT HỢP QUAN ĐIỂM + SOURCE
            for i in range(min(5, len(source_details))):
                if len(slides) >= 6:
                    break
                src = source_details[i]
                src_via = src.get("via", "")
                src_paras = src.get("paragraphs", [])
                
                src_text = ""
                for p in src_paras[:2]:
                    p = p.strip()[:280]
                    if len(p) > 50:
                        src_text = p
                        break
                
                if src_text:
                    combined = f"Theo góc nhìn cá nhân, {opinion[:60]}. Theo {src_via}: {src_text}"
                    img = source_images[len(slides)] if len(slides) < len(source_images) else (source_images[-1] if source_images else "")
                    slides.append({"text": _ensure_sentence_complete(combined), "image": img, "index": len(slides) + 1})
    
    lang, emotion = detect_language_and_emotion(title, ai_text)
    voice = get_voice_for_content(title, ai_text)
    
    post = {
        "id": str(int(time.time() * 1000)) + str(_random2.randint(100, 999)),
        "title": title,
        "text": ai_text,
        "img": source_images[0] if source_images else "",
        "url": "",
        "kind": "personal_opinion",
        "slides": slides,
        "images": source_images[:10],
        "video": "",
        "voice": voice,
        "emotion": emotion,
        "language": lang,
        "ts": int(time.time()),
        "sources": source_details[:5]
    }
    
    posts = _load_wall_posts()
    posts.insert(0, post)
    _save_wall_posts(posts)
    
    return JSONResponse({"post": post, "slides": slides})


# ===== END PERSONAL OPINION POST v2 =====

def _bg():
    time.sleep(15)
    while True:
        try:get_wc2026_all()
        except:pass
        time.sleep(90)
threading.Thread(target=_bg,daemon=True).start()

# ===== AUTO SCHEDULER: rewrite AI + short at 7/13/19 VN time =====
_AUTO_SCHEDULE_TIMES = [(7, '07:00'), (13, '13:00'), (19, '19:00')]
_AUTO_LOG = os.path.join(DATA_DIR, 'auto_rewrite_log.json')

def _load_auto_log():
    try:
        if os.path.exists(_AUTO_LOG):
            with open(_AUTO_LOG, 'r') as f:
                return json.load(f)
    except: pass
    return {}

def _save_auto_log(log):
    try:
        tmp = _AUTO_LOG + '.tmp'
        with open(tmp, 'w') as f:
            json.dump(log, f)
        os.replace(tmp, _AUTO_LOG)
    except: pass

async def _auto_fetch_short(post_id):
    """Try to auto-generate a short for a post."""
    try:
        import httpx
        async with httpx.AsyncClient(timeout=180) as cl:
            r = await cl.post(
                f"http://localhost:7860/api/ai/short/{post_id}",
                json={"voice":"vi-VN-HoaiMyNeural","emotion":"neutral","speed":1.2},
                headers={"Content-Type":"application/json"}
            )
            if r.status_code < 300:
                sj = r.json()
                if sj.get('video'):
                    posts = _load_wall_posts()
                    for p in posts:
                        if p.get('id') == post_id:
                            p['video'] = sj['video']
                            break
                    _save_wall_posts(posts)
                    return True
    except: pass
    return False

async def _auto_rewrite_one(topic, slot_label, used_urls=None, post_index=0):
    """Rewrite one topic: find articles, summarize, post to wall, trigger short.
    used_urls: shared set to avoid duplicate articles across topics.
    post_index: 0-based index to create multiple posts per topic (0,1,2 = up to 3 posts)."""
    from urllib.parse import quote as _q
    # Get MORE items to support 1-3 posts per topic
    items = _search_all(topic, limit=12)
    # Skip URLs already used by another topic
    if used_urls is not None:
        filtered = [it for it in items if it.get('url') not in used_urls]
        if filtered:
            items = filtered
    if not items or post_index >= len(items):
        return False
    
    # Get article at post_index (0,1,2 for multiple posts)
    item = items[post_index]  # post_index allows multiple articles per topic
    url = item.get('url', '')
    title = item.get('title', topic)
    if url and used_urls is not None:
        used_urls.add(url)
    if not url.startswith('http'):
        return False
    
    data = _scrape_article_for_rewrite(url)
    if not data or not data.get('paragraphs'):
        return False
    
    raw_text = '\n'.join(data['paragraphs'])
    ai_text = None
    
    # Try AI generation
    try:
        import ai_ext
        prompt = f"Tóm tắt tin tức (tự động {slot_label}):\nTiêu đề: {data['title']}\n{raw_text[:10000]}\n\n4-6 ý chính dạng bullet. Cuối ghi nguồn."
        ai_text = await ai_ext.qwen_generate(prompt, max_tokens=1000)
    except: pass
    
    if not ai_text or len(ai_text) < 80:
        pts = data['paragraphs'][:6]
        ai_text = '\n\n'.join([f"• {p[:300]}" for p in pts])
        via = item.get('via', '') or urlparse(url).netloc.replace('www.', '')
        ai_text += f"\n\nNguồn tham khảo: {via}"
    
    # Build slides
    images = data.get('images', [])
    pts = data['paragraphs'][:10]
    slides = []
    for i, p in enumerate(pts[:8]):
        img = images[i] if i < len(images) else (images[-1] if images else data.get('og_img', ''))
        slides.append({'text': p[:300], 'image': img, 'index': i + 1})
    
    post_id = str(int(time.time() * 1000)) + str(_random2.randint(100, 999))
    post = {
        "id": post_id, "title": data.get('title', title)[:200],
        "text": ai_text, "img": images[0] if images else data.get('og_img', ''),
        "url": url, "kind": "auto_rewrite", "slides": slides,
        "images": images[:10], "video": "",
        "voice": "vi-VN-HoaiMyNeural", "emotion": "neutral",
        "language": "vietnamese", "ts": int(time.time()),
        "auto_scheduled": True, "slot": slot_label,
    }
    
    posts = _load_wall_posts()
    posts.insert(0, post)
    _save_wall_posts(posts)
    
    # Trigger short generation async
    threading.Thread(target=lambda: asyncio.run(_auto_fetch_short(post_id)), daemon=True).start()
    return True

async def _do_scheduled_run(slot_label):
    """Main scheduled run: 1-3 posts from 3 different HOT topics (3-9 total), no duplicates."""
    print(f"[auto] Starting scheduled rewrite for {slot_label}")
    
    # Get top hot topics, skip duplicates
    all_topics = _get_hot_topics()
    seen_topics = set()
    unique_topics = []
    for t in all_topics:
        kw = t.get('topic', '').lower().strip()
        if kw and len(kw) > 5 and kw not in seen_topics:
            is_dup = False
            for s in seen_topics:
                # Check if one topic is substring of another
                if kw in s or s in kw:
                    is_dup = True
                    break
            if not is_dup:
                seen_topics.add(kw)
                unique_topics.append(t)
        if len(unique_topics) >= 3:
            break
    
    job_topics = [t['topic'] for t in unique_topics[:3] if t.get('topic')]
    if not job_topics:
        print(f"[auto] No hot topics found, skipping")
        return
    
    print(f"[auto] Running 3 topics: {job_topics}")
    
    # Track used URLs to avoid cross-topic duplicates
    _used_urls = set()
    results = []
    
    # Process each topic, create 1-3 posts per topic
    for jt in job_topics:
        for post_idx in range(3):  # Try up to 3 posts per topic
            try:
                ok = await asyncio.wait_for(_auto_rewrite_one(jt, slot_label, _used_urls, post_idx), timeout=120)
                if ok:
                    results.append((jt, post_idx, True))
                    print(f"[auto] Created post {post_idx+1} for '{jt}'")
                else:
                    # No more articles for this topic
                    break
            except Exception as e:
                print(f"[auto] Error on '{jt}' post {post_idx}: {e}")
                results.append((jt, post_idx, False))
            await asyncio.sleep(1)  # Small delay between posts
    
    # Ensure at least 3 posts total (fallback if needed)
    successful_posts = sum(1 for _, _, ok in results if ok)
    print(f"[auto] Done {slot_label}: {successful_posts} posts created")
    
    # Log
    from datetime import datetime, timezone, timedelta
    VN_TZ_SCHED = timezone(timedelta(hours=7))
    today_str = datetime.now(VN_TZ_SCHED).strftime('%Y-%m-%d')
    log = _load_auto_log()
    if today_str not in log: log[today_str] = {}
    log[today_str][slot_label] = {
        'time': datetime.now(VN_TZ_SCHED).strftime('%H:%M:%S'),
        'count': successful_posts,
        'total': len(job_topics),
    }
    _save_auto_log(log)

def _scheduler_loop():
    """Check every 60s; trigger at 7:00, 13:00, 19:00 VN time.
    On startup, check for any missed slots today and run them immediately."""
    time.sleep(35)
    from datetime import datetime, timezone, timedelta
    VN_TZ_SCHED = timezone(timedelta(hours=7))
    
    _last_run_date = ""
    _last_run_slots = set()
    
    # On startup: check log for missed slots today
    try:
        start_now = datetime.now(VN_TZ_SCHED)
        today_str = start_now.strftime('%Y-%m-%d')
        current_hour = start_now.hour
        current_minute = start_now.minute
        log = _load_auto_log()
        today_log = log.get(today_str, {})
        for h, label in _AUTO_SCHEDULE_TIMES:
            # Run if slot is past (either strictly earlier hour, or same hour but window has passed)
            should_run = False
            if h < current_hour:
                should_run = True
            elif h == current_hour and current_minute > 10:
                should_run = True
            if should_run and label not in today_log:
                print(f"[auto] Detected missed slot {label} (h={h} < now={current_hour}:{current_minute}), running catch-up now")
                _run_scheduled_sync(label)
                _last_run_slots.add(label)
    except Exception as e:
        print(f"[auto] Catch-up check error: {e}")
    
    while True:
        try:
            now = datetime.now(VN_TZ_SCHED)
            today = now.strftime('%Y-%m-%d')
            hour = now.hour
            minute = now.minute
            
            if today != _last_run_date:
                _last_run_date = today
                _last_run_slots = set()
            
            slot = None
            for h, label in _AUTO_SCHEDULE_TIMES:
                if hour == h and 0 <= minute < 5:
                    slot = label
                    break
            
            if slot and slot not in _last_run_slots:
                _last_run_slots.add(slot)
                _run_scheduled_sync(slot)
        except Exception as e:
            print(f"[auto] Loop error: {e}")
        
        time.sleep(60)

threading.Thread(target=_scheduler_loop, daemon=True, name='auto-rewrite-scheduler').start()

def _wall_cleanup_once():
    """One-time startup cleanup: sort the persistent wall store newest-first
    and remove truly-garbage auto posts (no valid image at ALL, no video, no
    slides, no embed URL — i.e. a post that can never be displayed).

    NOTE: This is intentionally conservative. Posts with a valid image or a
    valid embed/video are NEVER deleted here — they are only hidden at view
    time by _wall_posts_for_view() when the image is a placeholder. This
    prevents accidental data loss of FPT Short AI posts whose image is valid
    but may temporarily lack a short_thumb."""
    try:
        time.sleep(8)
        posts = _load_wall_posts()
        if not isinstance(posts, list):
            return
        kept = []
        removed = 0
        removed_ids = []
        needs_sort = False
        for p in posts:
            if not isinstance(p, dict):
                removed += 1
                continue
            if _is_auto_post(p):
                has_img = _post_has_valid_image(p)
                has_content = _post_has_slide_info(p)
                vid = (p.get("video") or "").strip()
                # Only delete auto posts that are completely unrecoverable:
                # no valid image AND no video/slides/embed at all.
                if not has_img and not has_content and not vid:
                    removed += 1
                    removed_ids.append(p.get('id', '?'))
                    continue
            kept.append(p)
        # Sort if any post lacks proper ordering
        prev_ts = None
        is_sorted = True
        for p in kept:
            ts = _created_ts(p)
            if prev_ts is not None and ts > prev_ts:
                is_sorted = False
                break
            prev_ts = ts
        if removed > 0 or not is_sorted:
            kept.sort(key=_created_ts, reverse=True)
            _save_wall_posts(kept)
            print(f"[wall] startup cleanup: removed {removed} garbage posts ({removed_ids[:10]}), kept {len(kept)}, sorted newest-first")
    except Exception as e:
        print(f"[wall] startup cleanup error: {e}")

threading.Thread(target=_wall_cleanup_once, daemon=True, name='wall-cleanup').start()

@app.get('/api/debug/auto_schedule')
async def debug_auto_schedule(slot: str = '07:00'):
    """Manually trigger auto scheduler for debugging."""
    try:
        # Check if we can access the data directory
        log = _load_auto_log()
        topics = _get_hot_topics()[:3]
        job_topics = [t['topic'] for t in topics if t.get('topic')]
        return JSONResponse({
            "slot": slot,
            "log": log,
            "hot_topics": job_topics,
            "wall_posts_count": len(_load_wall_posts()),
            "data_dir_writable": os.access(DATA_DIR, os.W_OK) if os.path.isdir(DATA_DIR) else False,
            "data_dir_exists": os.path.isdir(DATA_DIR),
        })
    except Exception as e:
        return JSONResponse({"error": str(e)}, status_code=500)

def _run_scheduled_sync(slot):
    """Run _do_scheduled_run in a separate event loop (for background thread)."""
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    try:
        loop.run_until_complete(_do_scheduled_run(slot))
    except Exception as e:
        print(f"[auto] Background run error: {e}")
    finally:
        loop.close()

@app.get('/api/debug/trigger_auto')
async def debug_trigger_auto(slot: str = '19:00'):
    """Trigger _do_scheduled_run in background thread (non-blocking)."""
    threading.Thread(target=_run_scheduled_sync, args=(slot,), daemon=True).start()
    return JSONResponse({"status": "started", "slot": slot})

# ===== SHORTS RSS PROXY ENDPOINT =====
@app.get("/api/shorts/rss")
def shorts_rss():
    """Get shorts from YouTube RSS feeds server-side"""
    import xml.etree.ElementTree as ET
    import html as html_lib2
    import re as re2
    
    YOUTUBE_CHANNELS = {
        "baodantri7941": "UC_x5TKhOgd6GhYvv5z4I3jg",
        "baosuckhoedoisongboyte": "UCBsY5fXTQLkF_JnH9kLkL4g",
    }
    
    shorts = []
    seen = set()
    
    for handle, channel_id in YOUTUBE_CHANNELS.items():
        try:
            rss_url = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
            r = req.get(rss_url, headers=HEADERS, timeout=15)
            if r.status_code != 200:
                continue
            
            root = ET.fromstring(r.text)
            ns = {
                'atom': 'http://www.w3.org/2005/Atom',
                'yt': 'http://www.youtube.com/xml/schemas/2015',
                'media': 'http://search.yahoo.com/mrss/'
            }
            
            for entry in root.findall('atom:entry', ns)[:30]:
                title_el = entry.find('atom:title', ns)
                title = html_lib2.unescape(title_el.text) if title_el is not None and title_el.text else ''
                
                link_el = entry.find('atom:link', ns)
                link = link_el.get('href', '') if link_el is not None else ''
                
                vid_el = entry.find('yt:videoId', ns)
                vid = vid_el.text if vid_el is not None else ''
                
                if not vid or vid in seen:
                    continue
                
                # Check if it's a short
                is_short = '#shorts' in title.lower() or '#short' in title.lower() or '/shorts/' in link
                
                if not is_short:
                    desc_el = entry.find('media:description', ns)
                    if desc_el is not None and desc_el.text:
                        if '#shorts' in desc_el.text.lower():
                            is_short = True
                
                if not is_short:
                    continue
                
                seen.add(vid)
                
                # Get thumbnail
                thumb = f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg"
                media_group = entry.find('media:group', ns)
                if media_group is not None:
                    thumb_el = media_group.find('media:thumbnail', ns)
                    if thumb_el is not None:
                        thumb = thumb_el.get('url', thumb)
                
                shorts.append({
                    'id': vid,
                    'title': title.replace('#shorts', '').replace('#short', '').strip()[:120],
                    'img': thumb,
                    'link': f'https://www.youtube.com/shorts/{vid}',
                    'channel': handle,
                    'source': 'yt'
                })
                
                if len(shorts) >= 40:
                    break
                    
        except Exception as e:
            print(f"RSS error for {handle}: {e}")
            continue
    
    return {"shorts": shorts, "count": len(shorts)}

# ===== M3U IPTV Channels (vmttv list + existing bundled playlist) =====
_M3U_URL = "https://raw.githubusercontent.com/vuminhthanh12/vuminhthanh12/main/vmttv"
# Secondary source: the existing curated playlist bundled in the repo. We keep
# it as a second source so channels that work on the Space IP are preserved
# while vmttv adds new ones. Both are validated on the Space egress IP, dead
# ones are dropped, duplicates by URL are merged.
_M3U_URL_EXISTING = "https://huggingface.co/spaces/bep40/vnews/resolve/main/static/playlist.m3u"
# Event-only groups: these channels only have a working stream while an event
# / match is live (movie premieres, TV360/VTVPrime event slots, BLV co-commentator
# streams). When no event is running their URLs are dead -> they would be
# validated out anyway. We KEEP them (no longer dropped) so that live football /
# event streams are detected and grouped into the EVENT section at the top of
# the 📺 Trực tiếp tab. Dead event streams are still removed by the validator,
# so a group with no live event simply never appears.
_M3U_EVENT_GROUPS = {
    "LIVE EVENTS 🔴",
    "Sự Kiện TV360",
    "Sự Kiện VTVPrime",
    "🔴 ⚽ COLA TV",
    "🔴 ⚽ 🏀 COLA TV SV2",
    "🔴 ⚽ 🏐 PHÁO HOA TV",
}
_M3U_FOOTBALL_GROUPS = {
    "🔴 ⚽ COLA TV",
    "🔴 ⚽ 🏀 COLA TV SV2",
    "🔴 ⚽ 🏐 PHÁO HOA TV",
    "⚽| Thể thao quốc tế",
    "THỂ THAO QUỐC TẾ",
}
# Title patterns that identify a live football match / event channel vs a
# regular 24/7 sports channel (e.g. "Bayern vs Dortmund", "01:45 02/09 Liverpool ...").
# NOTE: bare " - " / " – " separators are NOT used — too many normal channel
# names contain them (e.g. "On Movies - You TV"). Only strong match signals.
_M3U_MATCH_TITLE_RE = re.compile(
    r'\bvs\b|\d{1,2}:\d{2}\s*\d{1,2}/\d{1,2}|\bđấu\b|\bgặp\b|trực tiếp',
    re.IGNORECASE)
_M3U_FOOTBALL_TITLE_RE = re.compile(
    r'\bđấu\b|\bgặp\b|trực tiếp bóng đá|\bvs\b|bóng đá|champions|premier|v-league|vleague|nha\b|liga|bundesliga|serie a|world cup',
    re.IGNORECASE)

_M3U_EVENT_SUFFIXES = ("TV360+", "VTVPrime", "⚽ COLA", "COLA TV", "PHÁO HOA")
# Sports-cluster groups: when international football is LIVE (per bongda
# livescore), ALL channels in these groups are surfaced together with the
# EVENT slide on the 📺 Trực tiếp tab (the user asked: "khi có livescore bóng
# đá Quốc Tế cho tất cả kênh từ thể thao và thể thao quốc tế").
_M3U_SPORTS_CLUSTER_GROUPS = {
    "Thể Thao",
    "THỂ THAO QUỐC TẾ",
    "⚽| Thể thao quốc tế",
}
# Bongda livescore status labels: only matches that are actually in progress
# count as "live football". KT/FT/PEN etc = finished, VS = not started yet.
_M3U_LS_DONE_LABEL_RE = re.compile(r'^(KT|FT|ET|AET|PEN|VS|Hủy|Hoãn|Reported|Hủy bỏ)$', re.IGNORECASE)
# Vietnamese domestic competitions are NOT "bóng đá quốc tế" — exclude them.
_M3U_LS_VN_DOMESTIC_RE = re.compile(r'v[- ]?league|vleague|hạng nhất|hạng nhì|cúp quốc gia|siêu cúp|siêu cup|hạng ba|hạng tư', re.IGNORECASE)
_m3u_ls_cache = {"t": 0, "d": None}
_M3U_LS_TTL = 60

def _m3u_livescore_football_state(force=False):
    """Return whether international football is LIVE right now, based on the
    bongda.com.vn live fixtures feed. Cached for 60s (matches' status can
    change, so a short TTL is important).
      -> {"live": bool, "leagues": [..], "matches_count": int, "live_matches": [{..}]}
    """
    now = time.time()
    if not force and _m3u_ls_cache["d"] is not None and now - _m3u_ls_cache["t"] < _M3U_LS_TTL:
        return _m3u_ls_cache["d"]
    state = {"live": False, "leagues": [], "matches_count": 0, "live_matches": []}
    try:
        html_txt = fetch_bongda_api("/api/fixtures/live")
        if html_txt:
            soup = BeautifulSoup(html_txt, "lxml")
            for li in soup.select("li.match-detail"):
                status = li.select_one(".status")
                if not status:
                    continue
                label_el = status.select_one(".label")
                label = label_el.get_text(" ", strip=True) if label_el else ""
                if not label or _M3U_LS_DONE_LABEL_RE.match(label):
                    continue  # KT / VS / FT -> finished or not started
                league_el = li.select_one(".league")
                league = league_el.get_text(" ", strip=True) if league_el else ""
                if _M3U_LS_VN_DOMESTIC_RE.search(league):
                    continue  # V-League etc is domestic, not "quốc tế"
                home_el = li.select_one(".home-team .name")
                away_el = li.select_one(".away-team .name")
                state["live"] = True
                state["matches_count"] += 1
                state["live_matches"].append({
                    "home": home_el.get_text(" ", strip=True) if home_el else "",
                    "away": away_el.get_text(" ", strip=True) if away_el else "",
                    "league": league,
                    "label": label,
                })
                if league and league not in state["leagues"]:
                    state["leagues"].append(league)
    except Exception:
        pass
    _m3u_ls_cache["t"] = now
    _m3u_ls_cache["d"] = state
    return state

_m3u_cache = {"t": 0, "d": []}
_M3U_CACHE_FILE = os.path.join("/data" if os.path.isdir('/data') else "/app/data", 'm3u_cache.json')

def _load_m3u_disk():
    """Load M3U cache from disk (if available)."""
    try:
        if os.path.exists(_M3U_CACHE_FILE):
            with open(_M3U_CACHE_FILE, 'r', encoding='utf-8') as f:
                d = json.load(f)
                if d and isinstance(d.get('channels'), list):
                    return d
    except Exception:
        pass
    return None

def _save_m3u_disk(channels, ts):
    """Save M3U cache to disk."""
    try:
        os.makedirs(os.path.dirname(_M3U_CACHE_FILE), exist_ok=True)
        with open(_M3U_CACHE_FILE, 'w', encoding='utf-8') as f:
            json.dump({"channels": channels, "t": ts}, f, ensure_ascii=False)
    except Exception:
        pass

def _check_m3u_stream(c, timeout=9):
    """Probe a single M3U channel URL. Returns True only if it actually
    delivers STREAM data — raw MPEG-TS bytes (0x47 sync) OR an HLS playlist
    (#EXTM3U). Rejects HTTP error pages: some servers return HTTP 200 for
    HTML/JSON error responses, and those used to pass the old >=2-bytes check
    (e.g. CANAL+ portals returning <!DOCTYPE html>, 403 Forbidden pages).
    Applies VLC http-user-agent/http-cookie when present."""
    try:
        h = dict(HEADERS)
        vopts = c.get("vlcopts", {}) or {}
        if vopts.get("http-user-agent"):
            h["User-Agent"] = vopts["http-user-agent"]
        if vopts.get("http-cookie"):
            h["Cookie"] = vopts["http-cookie"]
        if vopts.get("http-header"):
            hdr = vopts.get("http-header")
            if ":" in hdr:
                kk, vv = hdr.split(":", 1)
                h[kk.strip()] = vv.strip()
        rr = req.get(c["url"], headers=h, timeout=timeout, stream=True, allow_redirects=True)
        if rr.status_code != 200:
            try: rr.close()
            except Exception: pass
            return False
        try:
            chunk = next(rr.iter_content(4096))
            rr.close()
        except Exception:
            try: rr.close()
            except Exception: pass
            return False
        if not chunk or len(chunk) < 2:
            return False
        head = chunk[:512].lstrip()
        # Reject HTML/XML/JSON error pages (200-with-error-body servers).
        if head[:1] in (b'<', b'{', b'['):
            return False
        # Accept real stream payloads: MPEG-TS sync bytes or an HLS playlist.
        if head.startswith(b'#EXTM3U') or b'#EXT-X-' in head[:256]:
            # For a MASTER playlist (#EXT-X-STREAM-INF) the URL alone being
            # live is not enough — the variant(s) it points to must also be
            # reachable. Follow variant chains recursively (up to 4 levels)
            # until we hit a media playlist with #EXTINF or a stream; drop
            # chains that 404 / return HTML / loop forever (e.g. gr.seetv.cc
            # serves self-referencing nested master playlists whose depth-2
            # variant is 404).
            try:
                cur = c["url"]
                depth = 0
                media_found = False
                while depth < 4:
                    is_master = b'#EXT-X-STREAM-INF' in chunk
                    if not is_master:
                        media_found = True
                        break
                    body = ''
                    try:
                        fr = req.get(cur, headers=h, timeout=timeout)
                        if fr.status_code == 200:
                            body = fr.text
                    except Exception:
                        pass
                    variant = None
                    lines = body.split('\n')
                    for i, ln in enumerate(lines):
                        if ln.startswith('#EXT-X-STREAM-INF') and i + 1 < len(lines):
                            cand = lines[i + 1].strip()
                            if cand and not cand.startswith('#'):
                                variant = cand if cand.startswith('http') else urljoin(cur, cand)
                                break
                    if not variant:
                        return False
                    if variant == cur:
                        return False  # self-referencing master -> dead CDN
                    try:
                        vr = req.get(variant, headers=h, timeout=timeout, stream=True, allow_redirects=True)
                        st2 = vr.status_code
                        try:
                            chunk = next(vr.iter_content(8192))
                            vr.close()
                        except Exception:
                            vr.close()
                            chunk = b''
                    except Exception:
                        return False
                    if st2 != 200 or not chunk:
                        return False
                    cur = variant
                    vhead = chunk[:512].lstrip()
                    if vhead[:1] in (b'<', b'{', b'['):
                        return False
                    depth += 1
                # media_found: we have a media playlist (#EXTINF, not master).
                # The segment(s) inside are what Hls.js actually loads — probe
                # the FIRST segment through the same URL-resolution the proxy
                # uses and require real media bytes (TS or another playlist).
                # This drops channels whose playlist is alive but whose
                # segments 404 (e.g. Canal Showsport lumpentv: playlist 200,
                # segment -> upstream 404 on the Space egress IP).
                if media_found:
                    body = ''
                    try:
                        fr = req.get(cur, headers=h, timeout=timeout)
                        if fr.status_code == 200:
                            body = fr.text
                    except Exception:
                        pass
                    first_seg = None
                    for ln in body.split('\n'):
                        s = ln.strip()
                        if s and not s.startswith('#') and not s.startswith('http'):
                            first_seg = urljoin(cur, s)
                            break
                        elif s and s.startswith('http'):
                            first_seg = s
                            break
                    if first_seg:
                        try:
                            sr = req.get(first_seg, headers=h, timeout=timeout, stream=True, allow_redirects=True)
                            st3 = sr.status_code
                            try:
                                schunk = next(sr.iter_content(2048))
                                sr.close()
                            except Exception:
                                sr.close()
                                schunk = b''
                            if st3 != 200 or not schunk:
                                return False
                            shead = schunk[:256].lstrip()
                            if shead[:1] in (b'<', b'{', b'['):
                                return False
                            # Playlist or TS/binary media both OK (seg proxy
                            # handles playlist-in-seg via rewrite.)
                            return True
                        except Exception:
                            return False
                return True
            except Exception:
                pass
            return True
        if head[0] == 0x47:
            # MPEG-TS: verify the 188-byte sync pattern if we have enough data
            if len(head) > 188:
                return head[188] == 0x47
            return True
        # Unknown binary but not HTML — accept (some servers send raw flv/ps)
        return True
    except Exception:
        return False

def _parse_m3u():
    """Fetch & parse the M3U playlists (vmttv remote + existing bundled list).
    Validates EVERY channel (all groups) with a real byte-read so dead
    streams are removed, then groups channels by NAME (title) so the API
    returns one entry per channel name with its working mirror streams:
      [{name, group, logo, streams:[{url, vlcopts}, ...]}, ...]
    Unreachable / broken channels and event-only groups are filtered out."""
    out = []
    seen_raw = set()
    for src_url in (_M3U_URL, _M3U_URL_EXISTING):
        try:
            r = req.get(src_url, headers={"User-Agent": "Mozilla/5.0"}, timeout=20)
            if r.status_code != 200:
                continue
            lines = r.text.splitlines()
            cur = {}
            for line in lines:
                line = line.strip()
                if line.startswith("#EXTINF"):
                    t_match = re.search(r',(.+)$', line)
                    cur = {
                        "title": t_match.group(1).strip() if t_match else "Channel",
                        "id": "",
                        "logo": "",
                        "group": "",
                        "url": "",
                        "vlcopts": {},
                    }
                    for attr_match in re.finditer(r'([a-zA-Z0-9-]+)="([^"]*)"', line):
                        k, v = attr_match.group(1), attr_match.group(2)
                        if k == "tvg-id":
                            cur["id"] = v
                        elif k == "tvg-logo":
                            cur["logo"] = v
                        elif k == "group-title":
                            cur["group"] = v
                elif line.startswith("#EXTVLCOPT:"):
                    key_part = line[len("#EXTVLCOPT:"):]
                    if "=" in key_part:
                        k, v = key_part.split("=", 1)
                        cur.setdefault("vlcopts", {})[k.strip()] = v.strip()
                elif line and not line.startswith("#") and cur:
                    cur["url"] = line
                    if cur["url"] not in seen_raw:
                        seen_raw.add(cur["url"])
                        out.append(cur)
                    cur = {}
        except Exception:
            continue
    # Event channels are KEPT in the list (not pre-dropped): the validator
    # below already removes streams that are dead, so a group with no live
    # event simply never appears. Each channel is tagged below so the frontend
    # can (a) place live events in the featured slide and (b) list event
    # channels on top of the Trực tiếp tab.
    def _event_kind(c):
        g = (c.get("group") or "").strip()
        t = (c.get("title") or "").strip()
        if g in _M3U_EVENT_GROUPS or any(t.startswith(s) for s in _M3U_EVENT_SUFFIXES):
            is_football = g in _M3U_FOOTBALL_GROUPS or bool(_M3U_FOOTBALL_TITLE_RE.search(t))
            return "football" if is_football else "event"
        if _M3U_MATCH_TITLE_RE.search(t):
            return "football"
        return None

    for c in out:
        k = _event_kind(c)
        if k:
            c["is_event"] = True
            c["event_kind"] = k
    # Deduplicate by URL (safety net — the seen_raw pass already dedupes)
    seen = set()
    deduped = []
    for c in out:
        if c["url"] not in seen:
            seen.add(c["url"])
            deduped.append(c)
    # Validate ALL groups (not just Thể Thao) with a real byte-read.
    valid = []
    with ThreadPoolExecutor(max_workers=24) as ex:
        futures = {ex.submit(_check_m3u_stream, c): c for c in deduped}
        for f in as_completed(futures):
            if f.result():
                valid.append(futures[f])
    valid_urls = {c["url"] for c in valid}
    good = [c for c in deduped if c["url"] in valid_urls]
    # Group by channel NAME (title): each entry is one channel name with the
    # list of working mirror streams so users can find a channel easily and
    # the player can auto-failover between mirrors.
    grouped = {}
    order = []
    for c in good:
        key = (c.get("group") or "Khác", c.get("title") or "Kênh")
        if key not in grouped:
            grouped[key] = {
                "name": c.get("title") or "Kênh",
                "group": c.get("group") or "Khác",
                "logo": c.get("logo") or "",
                "streams": [],
            }
            order.append(key)
        if _M3U_EVENT_GROUPS and (c.get("group") or "").strip() in _M3U_EVENT_GROUPS:
            # Keep one "live event" flag per channel (merged mirrors)
            grouped[key]["is_event"] = True
            grouped[key]["event_kind"] = c.get("event_kind") or "event"
        elif c.get("is_event"):
            grouped[key]["is_event"] = True
            grouped[key]["event_kind"] = c.get("event_kind") or "event"
        grouped[key]["streams"].append({
            "url": c["url"],
            "vlcopts": c.get("vlcopts") or {},
        })
    # Sort: live events (football first, then movie/TV events) on top, then
    # everything else in original order. Grouped dicts keep insertion order.
    def _sort_key(entry):
        if entry.get("is_event"):
            return (0, 0) if entry.get("event_kind") == "football" else (0, 1)
        return (1, 0)
    entries = [grouped[k] for k in order]
    entries.sort(key=_sort_key)
    return entries

def _m3u_refresh():
    """Refresh M3U cache in background."""
    n = int(time.time())
    try:
        channels = _parse_m3u()
        _m3u_cache["t"] = n
        _m3u_cache["d"] = channels
        _save_m3u_disk(channels, n)
    except Exception:
        pass

def _m3u_daily_updater():
    """Auto-refresh the M3U list once per day. Re-fetches the remote list
    (so channels added/removed upstream are picked up within 24h), re-validates
    every channel (dead ones are dropped, tokens refreshed) and persists to
    disk. Runs on a daemon thread."""
    while True:
        try:
            time.sleep(24 * 3600)  # 24h
            _m3u_refresh()
            print(f"[M3U] daily auto-update complete @ {time.strftime('%Y-%m-%d %H:%M:%S')}")
        except Exception as e:
            print(f"[M3U] daily auto-update error: {e}")
            try:
                time.sleep(3600)
            except Exception:
                pass

def _m3u_load():
    """Load M3U: return disk/in-memory cache immediately, refresh in background if stale."""
    global _m3u_cache
    n = int(time.time())
    # Try in-memory cache first
    if _m3u_cache["d"] and n - _m3u_cache["t"] < 180:
        return _m3u_cache["d"], _m3u_cache["t"]
    # Try disk cache
    disk = _load_m3u_disk()
    if disk and disk.get("channels"):
        _m3u_cache = {"t": disk.get("t", 0), "d": disk["channels"]}
        # Refresh in background if older than 3 min (Stalker play_tokens expire ~3-7 min)
        if n - _m3u_cache["t"] >= 180:
            threading.Thread(target=_m3u_refresh, daemon=True).start()
        return _m3u_cache["d"], _m3u_cache["t"]
    # No cache at all — fetch synchronously (first run)
    _m3u_refresh()
    return _m3u_cache["d"], _m3u_cache["t"]

@app.get('/api/m3u/channels')
def api_m3u_channels(refresh: int = Query(default=0)):
    if refresh:
        # FIX: sync refresh — re-parse the M3U so stalker play_tokens are fresh
        # before the frontend builds the /api/proxy/m3u_hls URL. The old code
        # only kicked off a background thread and returned the STALE cache,
        # so clients played with expired tokens -> black screen / 502.
        try:
            _m3u_refresh()
        except Exception as e:
            print(f"[M3U] sync refresh error: {e}")
        channels = _m3u_cache["d"]
        updated = _m3u_cache["t"]
    else:
        channels, updated = _m3u_load()
    # LIVE international football (bongda livescore): surface ALL sports-cluster
    # channels (Thể Thao / THỂ THAO QUỐC TẾ / ⚽| Thể thao quốc tế) together with
    # the EVENT slide. This is computed per-request (NOT persisted to the disk
    # cache) so when the last match ends the flag disappears automatically.
    ls = _m3u_livescore_football_state()
    if ls.get("live") and channels:
        channels = list(channels)
        for i, c in enumerate(channels):
            if (c.get("group") or "").strip() in _M3U_SPORTS_CLUSTER_GROUPS:
                channels[i] = dict(c)
                channels[i]["is_event"] = True
                channels[i]["event_kind"] = "football"
                channels[i]["live_football_sports"] = True
    return JSONResponse({
        "channels": channels,
        "updated": updated,
        "live_football": ls,
    })

# Pre-warm M3U cache on startup + start the daily auto-update loop.
threading.Thread(target=lambda: _m3u_load(), daemon=True).start()
threading.Thread(target=_m3u_daily_updater, daemon=True).start()

def _seg_url(base, ua=None, cookie=None, auth=None):
    """Build an /api/proxy/seg URL carrying the stalker headers."""
    q = urlencode({'url': base})
    if ua: q += '&ua=' + quote(ua, safe="")
    if cookie: q += '&cookie=' + quote(cookie, safe="")
    if auth: q += '&auth=' + quote(auth, safe="")
    return "/api/proxy/seg?" + q

def _rewrite_media_playlist(text, base_url, ua=None, cookie=None, auth=None):
    """Rewrite a MEDIA (not master) HLS playlist so every data line is proxied."""
    out = []
    for line in text.splitlines():
        s = line.strip()
        if not s or s.startswith('#'):
            out.append(line)
            continue
        resolved = s if s.startswith('http') else urljoin(base_url, s)
        out.append(_seg_url(resolved, ua, cookie, auth))
    return '\n'.join(out)

def _probe_upstream(url, headers):
    """Best-effort HEAD/GET probe. Returns (is_playlist: bool|None, content, ctype, final_url).
    None means 'unknown' (could not decide) -> caller treats as raw stream.
    We NEVER block long: connect 10s, read 12s. A .ts stream that does not
    respond in ~12s is treated as raw TS (wrap it). Only content that begins
    with #EXTM3U / #EXT-X- or has an HLS content-type is treated as a playlist."""
    try:
        r = req.get(url, headers=headers, timeout=(10, 12), stream=True, allow_redirects=True)
        final = r.url
        if r.status_code != 200:
            try: r.close()
            except Exception: pass
            return None, "", "", final
        ct = r.headers.get("Content-Type", "").lower()
        lead = b""
        try:
            lead = next(r.iter_content(4096))
        except Exception:
            pass
        try: r.close()
        except Exception: pass
        if not lead:
            # Empty 200 — ambiguous. Content-Type decides.
            return ('m3u8' in ct or 'mpegurl' in ct), "", ct, final
        head = lead[:512].lstrip()
        is_pl = head.startswith(b'#EXTM3U') or b'#EXT-X-' in head[:256]
        if is_pl:
            # Re-fetch full body (small text) for rewriting.
            body = ""
            try:
                rr = req.get(url, headers=headers, timeout=(10, 12), allow_redirects=True)
                if rr.status_code == 200:
                    body = rr.text
            except Exception:
                pass
            return True, body, ct, final
        return False, lead, ct, final
    except Exception:
        return None, "", "", url

def _master_to_media(url, headers):
    """Follow a MASTER playlist to its first playable MEDIA playlist.
    Returns (media_url, media_text) or (None, None) if it can't resolve."""
    # Fetch up to 3 levels (rare nested masters).
    cur = url
    for _ in range(3):
        try:
            r = req.get(cur, headers=headers, timeout=(10, 12), allow_redirects=True)
            if r.status_code != 200:
                return None, None
            body = r.text
        except Exception:
            return None, None
        if '#EXT-X-STREAM-INF' not in body:
            # Already a media playlist (or not a playlist at all — caller
            # validates before calling).
            return cur, body
        variant = None
        for i, ln in enumerate(body.splitlines()):
            if ln.startswith('#EXT-X-STREAM-INF') and i + 1 < len(body.splitlines()):
                cand = body.splitlines()[i + 1].strip()
                if cand and not cand.startswith('#'):
                    variant = cand if cand.startswith('http') else urljoin(cur, cand)
                    break
        if not variant or variant == cur:
            return None, None
        cur = variant
    return cur, None

@app.get('/api/proxy/m3u_hls')
def proxy_m3u_hls(url: str = Query(...), ua: str = Query(default=None), cookie: str = Query(default=None), auth: str = Query(default=None)):
    """Convert ANY stream URL (MPEG-TS, Stalker .ts, .m3u8/.m3u/.m3u8 playlists,
    MPD, raw) into an HLS live playlist that Hls.js can play in the browser.

    The old heuristic only probed URLs whose PATH contained '.m3u8'. That broke
    the common case where a '.ts' / mp2t stream URL actually serves an .m3u8
    manifest (many Stalker portals redirect/serve a playlist under a .ts path)
    AND the case where a plain '.m3u' URL is an HLS playlist rather than raw
    TS. Those URL-suffix mismatches caused Hls.js to receive playlist TEXT as
    video/mp2t media, stalling playback.

    New behaviour: content-sniff probe on EVERY URL. If the upstream bytes start
    with #EXTM3U / #EXT-X- we rewrite it into a proxied HLS playlist (resolving
    masters to their first media variant). Otherwise we wrap the URL in a live
    open-ended HLS manifest whose segments stream raw through /api/proxy/seg.
    A fast probe failure (timeout / 5xx) is treated as a raw stream and wrapped,
    so Stalker servers that never answer on port 80 from the egress IP still
    play via the TS wrapper."""
    if not url or not url.startswith('http'):
        return Response(status_code=400, content="Invalid URL")
    from urllib.parse import urlencode
    headers = dict(HEADERS)
    if ua: headers["User-Agent"] = ua
    if cookie: headers["Cookie"] = cookie
    if auth: headers["Authorization"] = auth
    # Content-sniff probe (short timeout; failure -> assume raw TS & wrap).
    is_pl, body, ct, final_url = _probe_upstream(url, headers)
    if is_pl:
        # Decide master vs media.
        if '#EXT-X-STREAM-INF' in body:
            media_url, media_text = _master_to_media(url, headers)
            if media_url and media_text and '#EXT-X-STREAM-INF' not in media_text:
                body = media_text
                # Rewrite against the resolved media URL.
                rewritten = _rewrite_media_playlist(body, media_url, ua, cookie, auth)
                return Response(content=rewritten.encode('utf-8'),
                                media_type="application/vnd.apple.mpegurl",
                                headers={"Access-Control-Allow-Origin":"*", "Cache-Control":"no-cache, no-store, must-revalidate"})
            elif media_url:
                # media_text None but media_url resolved to itself (media playlist) -> rewrite.
                rewritten = _rewrite_media_playlist(media_text or body, media_url, ua, cookie, auth)
                return Response(content=rewritten.encode('utf-8'),
                                media_type="application/vnd.apple.mpegurl",
                                headers={"Access-Control-Allow-Origin":"*", "Cache-Control":"no-cache, no-store, must-revalidate"})
            return Response(status_code=502, content="master playlist without playable variant")
        # Media playlist (or a plain .m3u/.m3u8 that is itself a media list).
        rewritten = _rewrite_media_playlist(body, final_url, ua, cookie, auth)
        return Response(content=rewritten.encode('utf-8'),
                        media_type="application/vnd.apple.mpegurl",
                        headers={"Access-Control-Allow-Origin":"*", "Cache-Control":"no-cache, no-store, must-revalidate"})
    # Raw MPEG-TS / Stalker / unknown: emit a live open-ended HLS wrapper whose
    # segments stream raw bytes through /api/proxy/seg. Multiple repeating
    # segments keep Hls.js buffer alive and polling for the next window, which
    # fixes the stall observed with single-segment manifests on short TS bursts.
    _seg = _seg_url(url, ua, cookie, auth)
    manifest = (
        "#EXTM3U\n"
        "#EXT-X-VERSION:6\n"
        "#EXT-X-PLAYLIST-TYPE:EVENT\n"
        "#EXT-X-TARGETDURATION:8\n"
        "#EXT-X-MEDIA-SEQUENCE:0\n"
        "#EXT-X-START:TIME-OFFSET=0,PRECISE=YES\n"
        "#EXTINF:8.0,\n"
        f"{_seg}\n"
        "#EXTINF:8.0,\n"
        f"{_seg}\n"
        "#EXTINF:8.0,\n"
        f"{_seg}\n"
        "#EXT-X-DISCONTINUITY\n"
    )
    return Response(content=manifest.encode('utf-8'),
                    media_type="application/vnd.apple.mpegurl",
                    headers={"Access-Control-Allow-Origin":"*", "Cache-Control":"no-cache, no-store, must-revalidate"})


@app.get('/api/proxy/m3u')
def proxy_m3u_stream(url: str = Query(...), ua: str = Query(default=None), cookie: str = Query(default=None), auth: str = Query(default=None)):
    """Proxy MPEG-TS / HLS streams from stalker portals with custom headers.

    For MPEG-TS (.ts) / mp2t and other non-HLS URLs: redirects to the HLS
    playlist generator (/api/proxy/m3u_hls) which content-sniffs the upstream
    and emits a playable HLS manifest. For .m3u8 / .m3u / playlist URLs: proxies
    segments through /api/proxy/seg with the stalker headers forwarded."""
    if not url or not url.startswith('http'):
        return Response(status_code=400, content="Invalid URL")
    from urllib.parse import urlencode
    low = url.lower()
    # URLs that are HLS playlists (m3u8 or plain m3u): stream through seg proxy.
    if low.endswith(('.m3u8', '.m3u')) or 'm3u8' in low or '/m3u8' in low:
        headers = dict(HEADERS)
        if ua: headers["User-Agent"] = ua
        if cookie: headers["Cookie"] = cookie
        if auth: headers["Authorization"] = auth
        try:
            r = req.get(url, headers=headers, timeout=15, stream=True)
            if r.status_code != 200:
                return Response(status_code=502, content="upstream error")
            ct = r.headers.get("Content-Type", "")
            # If a .m3u/.m3u8 URL actually serves raw TS (rare), fall back to the
            # HLS wrapper instead of streaming mp2t bytes to Hls.js.
            lead = b""
            try:
                lead = next(r.iter_content(4096))
            except Exception:
                pass
            head = lead[:512].lstrip()
            if head.startswith(b'#EXTM3U') or b'#EXT-X-' in head[:256] or 'mpegurl' in ct.lower():
                # HLS playlist: stream it through /api/proxy/seg so the rewritten
                # manifest (produced lazily by seg) reaches the browser CORS-free.
                def _stream2(src=r):
                    if lead:
                        yield lead
                    for chunk in src.iter_content(chunk_size=65536):
                        yield chunk
                return StreamingResponse(
                    _stream2(),
                    media_type="application/vnd.apple.mpegurl",
                    headers={"Access-Control-Allow-Origin":"*", "Cache-Control":"no-cache", "Accept-Ranges":"bytes"})
            # Not actually a playlist — serve as TS through the seg proxy instead.
            r.close()
        except Exception:
            pass
        # Fall through to the HLS wrapper path below.
    # MPEG-TS / stalker / non-playlist URL: redirect to the HLS playlist generator
    # (content-sniffs upstream and wraps raw streams, or rewrites real playlists).
    from fastapi.responses import RedirectResponse
    params = {'url': url}
    if ua: params['ua'] = ua
    if cookie: params['cookie'] = cookie
    if auth: params['auth'] = auth
    return RedirectResponse(url="/api/proxy/m3u_hls?" + urlencode(params), status_code=302)


# ===== M3U LOGO FALLBACK =====
@app.get('/api/m3u/logo_fallback')
def m3u_logo_fallback(group: str = Query(default="")):
    """Generate a fallback SVG logo for M3U channels when tvg-logo fails to load.
    Uses the group-title to pick an appropriate emoji/badge."""
    import html as _html
    g = _html.escape((group or '').strip())
    # Determine emoji based on group content
    g_lower = g.lower()
    if 'premier' in g_lower or 'premier league' in g_lower:
        emoji, bg = '⚽', '#1e3a8a'
    elif 'liga' in g_lower and 'la' in g_lower:
        emoji, bg = '⚽', '#0f172a'
    elif 'champions' in g_lower:
        emoji, bg = '⭐', '#7c2d12'
    elif 'bundes' in g_lower:
        emoji, bg = '⚽', '#047857'
    elif 'serie' in g_lower:
        emoji, bg = '⚽', '#713f12'
    elif 'ligue' in g_lower:
        emoji, bg = '⚽', '#7e22ce'
    elif 'world cup' in g_lower:
        emoji, bg = '🌍', '#075985'
    elif 'fpt' in g_lower:
        emoji, bg = '📺', '#be123c'
    else:
        emoji, bg = '📺', '#374151'
    svg = f'''<svg xmlns="http://www.w3.org/2000/svg" width="120" height="120" viewBox="0 0 120 120">
<rect width="120" height="120" rx="16" fill="{bg}"/>
<text x="60" y="72" font-size="42" text-anchor="middle" fill="white" font-family="Arial, sans-serif">{emoji}</text>
</svg>'''
    return Response(content=svg.encode('utf-8'), media_type='image/svg+xml',
                    headers={"Access-Control-Allow-Origin":"*"})


app.mount('/static',StaticFiles(directory=STATIC_DIR),name='vnews_static')