"""
extractor.py — the pipeline.

  Stage 1: render_pdf_pages()      PDF -> high-DPI PNGs
  Stage 2: detect_regions()        page PNG -> regions.json
  Stage 3: group_into_articles()   regions.json -> articles.json
              Primary:  Claude API (spatial reasoning over regions)
              Fallback: column-aware + dateline-anchored Python rules
  Stage 4: crop_articles()         articles.json -> article PNGs
  Stage 5: ocr_articles()          article PNGs -> article TXTs
"""
import os
import re
import json
import base64
import shutil
import threading
from io import BytesIO
from pathlib import Path

try:
    from generate_pdf import generate_political_pdf
except ImportError:
    generate_political_pdf = None

# Load .env file if it exists, to populate environment variables like ANTHROPIC_API_KEY
if not os.environ.get("ANTHROPIC_API_KEY"):
    env_path = Path(__file__).resolve().parent / ".env"
    if env_path.exists():
        try:
            for line in env_path.read_text(encoding="utf-8").splitlines():
                line = line.strip()
                if line and not line.startswith("#") and "=" in line:
                    key, val = line.split("=", 1)
                    if key.strip() == "ANTHROPIC_API_KEY":
                        os.environ["ANTHROPIC_API_KEY"] = val.strip().strip('"').strip("'")
        except Exception as e:
            print(f"Error loading .env file: {e}")

import fitz  # PyMuPDF
from PIL import Image

JOBS = {}
JOB_LOCK = threading.Lock()

# Minimum article bounding-box thresholds — filters out layout artifacts
# (tiny title fragments, page decorations) before they waste OCR compute.
MIN_ARTICLE_AREA = 10000   # pixels²  (e.g. 100×100 px minimum)
MIN_ARTICLE_DIM  = 60      # pixels    either dimension below this → skip

_LAYOUT = None
_LAYOUT_LOCK = threading.Lock()


def _set_status(job_id, msg):
    with JOB_LOCK:
        if job_id in JOBS:
            JOBS[job_id]["message"] = msg


# ---------------------------------------------------------------------------
# Stage 1 — render PDF pages
# ---------------------------------------------------------------------------
def render_pdf_pages(pdf_path, out_dir, target_w, target_h, max_pages=None, pages_to_process=None):
    out_dir = Path(out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)

    doc = fitz.open(pdf_path)
    pages = []
    
    # Determine page selection target set
    targets = set(pages_to_process) if pages_to_process is not None else None

    for i, page in enumerate(doc, start=1):
        if max_pages and i > max_pages:
            break
        if targets is not None and i not in targets:
            continue
        rect = page.rect
        scale = min(target_w / rect.width, target_h / rect.height)
        mat = fitz.Matrix(scale, scale)
        pix = page.get_pixmap(matrix=mat, alpha=False)
        out_path = out_dir / f"page_{i:03d}.png"
        pix.save(str(out_path))
        pages.append({"page": i, "path": str(out_path),
                      "width": pix.width, "height": pix.height})
    doc.close()
    return pages


# ---------------------------------------------------------------------------
# Stage 2 — detect regions with Surya Layout
# ---------------------------------------------------------------------------
_SURYA_LAYOUT_MODEL = None
_SURYA_LAYOUT_PROCESSOR = None

def _load_layout_model():
    global _SURYA_LAYOUT_MODEL, _SURYA_LAYOUT_PROCESSOR
    if _SURYA_LAYOUT_MODEL is not None:
        return _SURYA_LAYOUT_MODEL, _SURYA_LAYOUT_PROCESSOR
    with _LAYOUT_LOCK:
        if _SURYA_LAYOUT_MODEL is not None:
            return _SURYA_LAYOUT_MODEL, _SURYA_LAYOUT_PROCESSOR
        import surya.settings
        from surya.model.detection.model import load_model, load_processor
        
        checkpoint = surya.settings.settings.LAYOUT_MODEL_CHECKPOINT
        _SURYA_LAYOUT_MODEL = load_model(checkpoint=checkpoint)
        _SURYA_LAYOUT_PROCESSOR = load_processor(checkpoint=checkpoint)
        return _SURYA_LAYOUT_MODEL, _SURYA_LAYOUT_PROCESSOR


def detect_regions(page_png_path):
    model, processor = _load_layout_model()
    from surya.layout import batch_layout_detection
    img = Image.open(page_png_path).convert("RGB")

    layout_predictions = batch_layout_detection([img], model, processor)

    regions = []

    # Label mapping from Surya to our expected types
    label_map = {
        "Picture": "image",
        "Section-header": "paragraph_title",
        "Title": "doc_title",
        "Text": "text",
        "Caption": "text",
        "Table": "table",
        "Page-header": "header",
        "Page-footer": "footer",
        "List-item": "list"
    }

    if layout_predictions:
        layout = layout_predictions[0]
        if hasattr(layout, 'bboxes'):
            for rid, box in enumerate(layout.bboxes, start=1):
                bbox = [int(v) for v in box.bbox]
                raw_label = box.label
                label = label_map.get(raw_label, "text").lower()
                # Surya doesn't expose confidence in this API level easily, default to 1.0
                regions.append({"id": rid, "type": label, "bbox": bbox, "confidence": 1.0})

    # Detailed logging
    print(f"\n{'='*70}")
    print(f"LAYOUT DETECTION RESULTS: {len(regions)} regions found")
    print(f"{'='*70}")

    # Count by type
    type_counts = {}
    for r in regions:
        t = r["type"]
        type_counts[t] = type_counts.get(t, 0) + 1
    print(f"Region types: {dict(sorted(type_counts.items(), key=lambda x: -x[1]))}")
    print()

    # Print each region
    for r in regions:
        x1, y1, x2, y2 = r["bbox"]
        w = x2 - x1
        h = y2 - y1
        print(f"  Region {r['id']:3d}: type={r['type']:15s} bbox=[{x1:5d},{y1:5d},{x2:5d},{y2:5d}] size={w:5d}x{h:5d}px conf={r['confidence']:.2f}")
    print(f"{'='*70}\n")

    # Draw bounding boxes on page image and save as debug image
    try:
        from PIL import ImageDraw, ImageFont
        debug_img = Image.open(page_png_path).copy()
        draw = ImageDraw.Draw(debug_img)

        # Color map for different region types
        colors = {
            "text": "blue", "image": "green", "title": "red", "doc_title": "red",
            "header": "gray", "footer": "gray", "paragraph_title": "orange",
            "table": "purple", "figure": "green", "list": "cyan",
        }

        for r in regions:
            x1, y1, x2, y2 = r["bbox"]
            color = colors.get(r["type"], "yellow")
            # Draw rectangle
            draw.rectangle([x1, y1, x2, y2], outline=color, width=3)
            # Draw label
            label_text = f"{r['id']}:{r['type']}"
            draw.text((x1 + 5, y1 + 5), label_text, fill=color)

        # Save debug image next to the page image
        debug_path = page_png_path.replace(".png", "_regions_debug.png")
        debug_img.save(debug_path)
        print(f"DEBUG IMAGE saved: {debug_path}")
        print(f"  Open this image to see all {len(regions)} bounding boxes drawn on the page")
        print(f"  Colors: text=blue, title/doc_title=red, image/figure=green, header/footer=gray, paragraph_title=orange")
        print()
    except Exception as e:
        print(f"  Could not create debug image: {e}")

    return regions


# ---------------------------------------------------------------------------
# Stage 3 — group regions into articles
# ---------------------------------------------------------------------------
def group_into_articles(page_png_path, regions, page_width, page_img=None):
    """Geometric grouping. Returns (articles, enriched_regions)."""
    # Filter out page-level chrome to prevent unneccessary top/bottom content
    valid_regions = [r for r in regions if r["type"] not in ("header", "footer", "page_number")]
    
    print("[running geometric column grouping]")
    import geometric_grouper
    articles = geometric_grouper.group_surya_regions_geometrically(valid_regions, page_png_path)
    return articles, valid_regions


# ---------------------------------------------------------------------------
# Stage 4 — crop each article
# ---------------------------------------------------------------------------
def crop_headlines(page_png_path, articles, regions, out_dir, page_num):
    # How many pixels below the title region to include in the headline crop.
    # Including a few lines of body text stabilises Telugu OCR output and gives
    # Claude's triage enough context to make a reliable keep/reject decision.
    HEADLINE_BODY_EXTENSION = 300

    out_dir = Path(out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)
    img = Image.open(page_png_path)
    
    region_by_id = {r["id"]: r for r in regions}
    results = []
    
    for art in articles:
        # Find the headline box
        title_id = art.get("title_region")
        if title_id and title_id in region_by_id:
            l, t, r, b = region_by_id[title_id]["bbox"]
            # Extend downward into the body to give OCR more context.
            # A bare 68px title strip in Telugu is too short for stable OCR;
            # including the first few lines of body text below it anchors the
            # reading and reduces non-determinism in Claude's triage input.
            art_bottom = art["bbox"][3]
            b = min(art_bottom, b + HEADLINE_BODY_EXTENSION)
        else:
            # Fallback: top 20% of the article bbox when no title region exists
            l, t, r, b = art["bbox"]
            b = min(b, int(t + (b - t) * 0.20))
            
        # Pad slightly
        l = max(0, l - 10)
        t = max(0, t - 10)
        r = min(img.width, r + 10)
        b = min(img.height, b + 10)
        
        crop = img.crop((l, t, r, b))
        art_name = f"p{page_num:03d}_a{art['article_id']:03d}"
        art_dir = out_dir / art_name
        art_dir.mkdir(parents=True, exist_ok=True)
        img_path = art_dir / "headline.png"
        crop.save(img_path)
        
        results.append({
            "article_id": art["article_id"],
            "name": art_name,
            "dir": str(art_dir),
            "headline_img_path": str(img_path),
            "bbox": art["bbox"],
            "title_region": title_id,
            "member_regions": art.get("member_regions", [])
        })
    return results


def crop_full_articles(page_png_path, selected_ids, articles, out_dir, page_num):
    out_dir = Path(out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)
    img = Image.open(page_png_path)

    results = []
    for art in articles:
        art_name = f"p{page_num:03d}_a{art['article_id']:03d}"
        if art_name not in selected_ids:
            continue
            
        l, t, r, b = art["bbox"]
        
        # Dynamic Padding: expand by up to 25px, but don't intersect other articles
        pad_l = pad_t = pad_r = pad_b = 25
        
        for other in articles:
            if other["article_id"] == art["article_id"]:
                continue
            ol, ot, or_, ob = other["bbox"]
            
            # Check vertical overlap
            if max(t, ot) < min(b, ob):
                if ol >= r:  # Other is to the right
                    pad_r = min(pad_r, max(0, (ol - r) // 2))
                elif or_ <= l: # Other is to the left
                    pad_l = min(pad_l, max(0, (l - or_) // 2))
                    
            # Check horizontal overlap
            if max(l, ol) < min(r, or_):
                if ot >= b:  # Other is below
                    pad_b = min(pad_b, max(0, (ot - b) // 2))
                elif ob <= t:  # Other is above
                    pad_t = min(pad_t, max(0, (t - ob) // 2))

        l = max(0, int(l - pad_l))
        t = max(0, int(t - pad_t))
        r = min(img.width, int(r + pad_r))
        b = min(img.height, int(b + pad_b))

        crop = img.crop((l, t, r, b))
        name = f"p{page_num:03d}_a{art['article_id']:03d}"
        art_dir = out_dir / name
        art_dir.mkdir(parents=True, exist_ok=True)
        crop.save(art_dir / "article.png")

        info = {
            "page": page_num,
            "article_id": art["article_id"],
            "bbox": [l, t, r, b],
            "member_regions": art["member_regions"],
            "title_region": art.get("title_region"),
            "dateline": art.get("dateline"),
            "headline_hint": art.get("headline_hint"),
            "grouped_by": art.get("grouped_by", "rules"),
        }
        (art_dir / "info.json").write_text(json.dumps(info, indent=2, ensure_ascii=False), encoding="utf-8")
        results.append({"name": name, "dir": str(art_dir), **info})
    return results


# ---------------------------------------------------------------------------
# Stage 5 — OCR each cropped article (Telugu)
# ---------------------------------------------------------------------------
_paddle_ocr_instance = None

def _ocr_with_paddleocr(img_path):
    """Use PaddleOCR for better reading-order-aware Telugu OCR."""
    global _paddle_ocr_instance
    temp_path = None
    try:
        from PIL import Image
        with Image.open(img_path) as im:
            px = im.width * im.height
            if px > 1000000:
                # Instead of skipping, resize the article image to prevent PaddleOCR deadlock or crash
                ratio = (900000 / px) ** 0.5
                new_w = int(im.width * ratio)
                new_h = int(im.height * ratio)
                print(f"Article image {img_path.name} is too large ({im.width}x{im.height} = {px}px). Resizing to {new_w}x{new_h} for safe PaddleOCR.")
                resized_im = im.resize((new_w, new_h), Image.LANCZOS)
                temp_path = img_path.parent / f"temp_resized_{img_path.name}"
                resized_im.save(temp_path)
        
        import paddleocr
        if _paddle_ocr_instance is None:
            _paddle_ocr_instance = paddleocr.PaddleOCR(lang='te')
        
        path_to_ocr = str(temp_path) if temp_path else str(img_path)
        result = _paddle_ocr_instance.predict(path_to_ocr)

        lines = []
        if isinstance(result, list) and len(result) > 0:
            det_result = result[0]
            # PaddleOCR v3.5 returns DetResult with 'rec_texts' or nested structure
            rec_texts = None
            if hasattr(det_result, 'get'):
                rec_texts = det_result.get('rec_texts', None)
            if rec_texts:
                lines = list(rec_texts)
            else:
                # Try to extract from boxes/text pairs
                boxes = det_result.get('dt_polys', []) if hasattr(det_result, 'get') else []
                texts = det_result.get('rec_texts', []) if hasattr(det_result, 'get') else []
                if texts:
                    # Sort by y-coordinate (top to bottom), then x (left to right)
                    # to get proper reading order
                    pairs = []
                    for i, txt in enumerate(texts):
                        if i < len(boxes):
                            box = boxes[i]
                            if hasattr(box, 'tolist'):
                                box = box.tolist()
                            # Get top-left y coordinate for sorting
                            if isinstance(box, (list, tuple)) and len(box) >= 4:
                                y = min(box[j] for j in range(1, len(box), 2)) if len(box) >= 8 else box[1]
                                x = min(box[j] for j in range(0, len(box), 2)) if len(box) >= 8 else box[0]
                            else:
                                y, x = 0, 0
                            pairs.append((y, x, txt))
                        else:
                            pairs.append((0, 0, txt))

                    # Sort: primarily by y (row), then x (column within row)
                    # Group into rows based on y proximity
                    if pairs:
                        pairs.sort(key=lambda p: (p[0], p[1]))
                        row_threshold = 20  # pixels
                        rows = []
                        current_row = [pairs[0]]
                        for p in pairs[1:]:
                            if abs(p[0] - current_row[0][0]) < row_threshold:
                                current_row.append(p)
                            else:
                                current_row.sort(key=lambda p: p[1])  # sort by x within row
                                rows.append(current_row)
                                current_row = [p]
                        current_row.sort(key=lambda p: p[1])
                        rows.append(current_row)

                        for row in rows:
                            lines.append(" ".join(p[2] for p in row))
                else:
                    # No text found via any path — return None cleanly.
                    # (Caller will substitute "[No text extracted by OCR]".)
                    # NOTE: do NOT fall back to str(det_result) here — that
                    # returns the raw PaddleOCR result dict as a string,
                    # which poisons Claude's triage with file-path garbage.
                    pass

        return "\n".join(lines) if lines else None
    except Exception as e:
        print(f"PaddleOCR text extraction failed: {e}")
        return None
    finally:
        if temp_path and temp_path.exists():
            try:
                temp_path.unlink()
            except:
                pass


def ocr_headlines(headline_records):
    for rec in headline_records:
        img_path = Path(rec["headline_img_path"])
        if not img_path.exists():
            rec["headline_text"] = ""
            continue
        text = _ocr_with_paddleocr(img_path)
        rec["headline_text"] = (text or "").strip()
        (img_path.parent / "headline.txt").write_text(rec["headline_text"], encoding="utf-8")

def ocr_articles(article_records):
    for rec in article_records:
        art_dir = Path(rec["dir"])
        img_path = art_dir / "article.png"
        if not img_path.exists():
            continue

        text = _ocr_with_paddleocr(img_path)
        (art_dir / "article.txt").write_text(text or "", encoding="utf-8")

        info_path = art_dir / "info.json"
        if info_path.exists():
            info = json.loads(info_path.read_text(encoding="utf-8"))
            info_path.write_text(json.dumps(info, indent=2, ensure_ascii=False), encoding="utf-8")


# ---------------------------------------------------------------------------
# Political Filter — crop first, then filter with Claude
# ---------------------------------------------------------------------------
def _triage_headlines_with_claude(headline_records, page_num, job_dir=None):
    """Send HEADLINES ONLY to Claude to select political articles, saving 90% compute."""
    api_key = os.environ.get("ANTHROPIC_API_KEY")
    if not api_key:
        # If no key, select all to avoid losing data
        return [r["name"] for r in headline_records]

    # Build the list of text articles
    article_list_data = []
    for rec in headline_records:
        txt = rec.get("headline_text", "").strip()
        if not txt:
            txt = "[No text extracted by OCR]"
        
        article_list_data.append({
            "id": rec["name"],
            "headline_text": txt
        })

    if not article_list_data:
        return []

    try:
        import anthropic
        client = anthropic.Anthropic(api_key=api_key)

        articles_json_str = json.dumps(article_list_data, indent=2, ensure_ascii=False)

        prompt = f"""You are a senior opposition party strategist in Telangana state, India.
I have extracted the HEADLINES from page {page_num} of a Telugu newspaper. Here is the list:

{articles_json_str}

Your job is a BROAD FIRST PASS — keep anything that COULD be politically useful. You are NOT the final judge; a deeper analysis step will follow.

KEEP articles about ANY of these:
- Government failures, broken promises, delays, inaction
- Price hikes, inflation, fuel costs, essential goods
- Farmer distress, crop losses, water/power issues for farming
- Crime, law & order failures, police negligence
- Infrastructure problems (roads, hospitals, schools in bad shape)
- Corruption, scams, misuse of funds by officials or politicians
- Protests, strikes, public grievances against authorities
- Health emergencies, hospital negligence, shortage of medicines
- Job losses, unemployment, labour disputes
- Education failures (school closures, NEET, exam problems)
- Accidents or disasters with possible government negligence angle
- Any ruling party internal conflict or embarrassment

DO NOT KEEP (filter these out only if you are certain):
- Pure sports scores or match results
- Pure entertainment / cinema news with zero political angle
- Horoscopes, puzzles, crosswords
- Pure classified advertisements or product promotions
- International news with zero Telangana/India relevance
- Completely blank or unreadable OCR regions

When in doubt, KEEP IT. It is far better to pass 10 borderline articles to the next stage than to miss one important story.

Return ONLY a valid JSON list of IDs. Example: ["p001_a002", "p001_a005"]. No prose."""

        response = client.messages.create(
            model="claude-sonnet-4-6",
            max_tokens=1000,
            messages=[{"role": "user", "content": prompt}],
        )

        text = response.content[0].text.strip()
        if text.startswith("```"):
            text = text.split("\n", 1)[1]
            if text.endswith("```"):
                text = text[:-3]

        selected_ids = json.loads(text.strip())
        print(f"  Page {page_num}: Triage kept {len(selected_ids)}/{len(headline_records)} articles based on headlines.")
        return selected_ids

    except Exception as e:
        print(f"Headline triage failed: {e}")
        return [r["name"] for r in headline_records]


def _analyze_articles_with_claude(records, page_num, job_dir=None):
    """Deep analysis on the full OCR text of the selected articles."""
    api_key = os.environ.get("ANTHROPIC_API_KEY")
    if not api_key:
        return None

    article_list_data = []
    for rec in records:
        art_dir = Path(rec["dir"])
        txt_path = art_dir / "article.txt"
        txt_content = ""
        if txt_path.exists():
            txt_content = txt_path.read_text(encoding="utf-8").strip()
        if not txt_content:
            txt_content = "[No text extracted by OCR]"
            
        article_list_data.append({
            "id": rec["name"],
            "ocr_text": txt_content[:3000]
        })

    if not article_list_data:
        return []

    try:
        import anthropic
        client = anthropic.Anthropic(api_key=api_key)

        # Convert list to JSON string for the prompt
        articles_json_str = json.dumps(article_list_data, indent=2, ensure_ascii=False)

        prompt = f"""You are a senior opposition party strategist in Telangana state, India.

I have run OCR on the cropped articles of page {page_num} of a Telugu newspaper. Here is the list of article IDs and their Telugu OCR text contents:

{articles_json_str}

For EACH article, ask yourself these TWO questions:

QUESTION 1: "Is this article POLITICALLY SIGNIFICANT for an opposition party in Telangana?"
- Political significance includes: MLA/MLC/MP statements, press conferences, padayatras, political protests, and party activities.
- It ALSO includes government scheme announcements tied to politicians, or Collector administrative directives.
- It ALSO includes government failures, corruption, price hikes, farmer distress, infrastructure failures, and public grievances.
- If it is just sports, cinema, horoscopes, advertisements, or purely neutral non-political news -> answer is NO.
- If YES -> proceed to Question 2
- If NO -> REJECT this article immediately

QUESTION 2: "Can the opposition party USE this article in any way (e.g. to attack the government, to track ruling party activities, or to monitor scheme implementations)?"
- If the article is politically relevant and useful for an opposition war room to track -> answer is YES.
- If it is completely irrelevant to state politics -> answer is NO.

BOTH answers must be YES to select. If either answer is NO, reject.

DEDUPLICATION: If two or more articles cover the SAME issue or event, only SELECT the ONE with the most complete coverage.

NOW go through EACH article ID I gave you. Ask Q1 and Q2 for each. Only select if BOTH are YES.

IMPORTANT: This application is for TELANGANA STATE politics.

DO NOT SELECT — automatically REJECT:
- International news (wars, foreign affairs, immigration)
- Newspaper masthead, headers, advertisements, classifieds
- Sports, entertainment, cinema, horoscopes
- General statistics without political context
- News about other states completely unrelated to Telangana

Return a JSON object with TWO lists:

1. "reasoning": for EVERY article (selected or not), explain your thinking (KEEP ALL REASONS EXTREMELY BRIEF, MAX 10 WORDS EACH TO PREVENT TOKEN CUTOFF):
  - "id": the article ID
  - "headline_english": brief English headline (max 8 words)
  - "q1": "YES" or "NO"
  - "q1_reason": short reason (max 10 words)
  - "q2": "YES" or "NO" (skip if q1 is NO)
  - "q2_reason": short reason (max 10 words, skip if q1 is NO)
  - "decision": "SELECT" or "REJECT"

2. "selected": only the articles where both Q1 and Q2 are YES:
  - "id": the article ID
  - "headline_english": English translation of the headline
  - "q1_answer": answer to Question 1 (1 short sentence)
  - "q2_answer": answer to Question 2 (1 short sentence)
  - "attack_angle": what opposition should say (1 sentence)
  - "priority": "high" or "medium"
  - "category": one of corruption, governance_failure, public_grievance, law_order, health, education, infrastructure, ruling_party_crisis, farmer_issues

Return ONLY valid JSON. No markdown. If no articles pass both questions, return {{"reasoning": [...], "selected": []}}"""

        response = client.messages.create(
            model="claude-sonnet-4-6",
            max_tokens=8192,
            messages=[{"role": "user", "content": prompt}],
        )

        text = response.content[0].text.strip()
        if text.startswith("```"):
            text = text.split("\n", 1)[1]
            if text.endswith("```"):
                text = text[:-3]

        result = json.loads(text.strip())

        # Log full reasoning to file and terminal
        reasoning = result.get("reasoning", [])
        if reasoning:
            print(f"\n=== POLITICAL FILTER REASONING (Page {page_num}) ===")
            for r in reasoning:
                status = "✅ SELECT" if r.get("decision") == "SELECT" else "❌ REJECT"
                print(f"  {r.get('id','?')}: {r.get('headline_english','')[:50]}")
                print(f"    Q1: {r.get('q1','?')} — {r.get('q1_reason','')}")
                if r.get('q2'):
                    print(f"    Q2: {r.get('q2','?')} — {r.get('q2_reason','')}")
                print(f"    -> {status}")
            print(f"=== END REASONING ===\n")

        # Save reasoning log to job directory
        log_path = Path(job_dir) / f"filter_reasoning_page{page_num}.json" if job_dir else None
        if log_path:
            log_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")

        return result.get("selected", [])

    except Exception as e:
        print(f"Political filter failed: {e}")
        import traceback; traceback.print_exc()
        return None


def _generate_opposition_brief(selected_articles, job_dir):
    """Generate a readable opposition daily brief text file."""
    high = [a for a in selected_articles if a.get("priority") == "high"]
    medium = [a for a in selected_articles if a.get("priority") == "medium"]

    brief_path = Path(job_dir) / "opposition_daily_brief.txt"
    with open(brief_path, "w", encoding="utf-8") as f:
        f.write("=" * 70 + "\n")
        f.write("  OPPOSITION DAILY BRIEF\n")
        f.write("=" * 70 + "\n\n")
        f.write(f"Total significant articles: {len(selected_articles)}\n")
        f.write(f"HIGH: {len(high)} | MEDIUM: {len(medium)}\n")
        f.write("-" * 70 + "\n\n")

        for label, group in [("HIGH PRIORITY — IMMEDIATE ACTION", high),
                              ("MEDIUM PRIORITY", medium)]:
            if group:
                f.write(f">>> {label} <<<\n\n")
                for i, a in enumerate(group, 1):
                    f.write(f"{i}. [{a.get('id', '?')}] {a.get('headline_english', 'N/A')}\n")
                    f.write(f"   Category: {a.get('category', '?')}\n")
                    f.write(f"   WHY: {a.get('political_significance', 'N/A')}\n")
                    f.write(f"   ATTACK: {a.get('attack_angle', 'N/A')}\n\n")
                f.write("-" * 70 + "\n")

        f.write("END OF BRIEF\n")


# ---------------------------------------------------------------------------
# Orchestrator
# ---------------------------------------------------------------------------
def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, pages_to_process=None):
    job_dir = Path(job_dir)
    pages_dir = job_dir / "pages"
    articles_root = job_dir / "articles"
    pages_dir.mkdir(parents=True, exist_ok=True)
    articles_root.mkdir(parents=True, exist_ok=True)

    # Step 1: Render PDF pages
    _set_status(job_id, "Rendering PDF pages...")
    pages = render_pdf_pages(pdf_path, pages_dir, target_w, target_h, max_pages=max_pages, pages_to_process=pages_to_process)

    all_records = []
    all_selected = []

    for p in pages:
        n = p["page"]
        page_img_path = p["path"]

        # Step 2: PaddleOCR layout detection — precise bounding boxes
        _set_status(job_id, f"Page {n}: detecting layout regions...")
        regions = detect_regions(page_img_path)
        (pages_dir / f"page_{n:03d}.regions.json").write_text(
            json.dumps({"page": n, "regions": regions}, indent=2, ensure_ascii=False), encoding="utf-8"
        )
        print(f"  Page {n}: {len(regions)} layout regions detected")

        # Step 3: Group regions into articles
        _set_status(job_id, f"Page {n}: grouping articles...")
        page_img = Image.open(page_img_path)
        articles, enriched = group_into_articles(page_img_path, regions, p["width"], page_img=page_img)

        (pages_dir / f"page_{n:03d}.articles.json").write_text(
            json.dumps({"page": n, "articles": articles, "enriched_regions": enriched},
                       indent=2, ensure_ascii=False), encoding="utf-8"
        )

        # --- DETAILED GROUPING LOG ---
        print(f"\n{'='*70}")
        print(f"GROUPING RESULTS — PAGE {n}")
        print(f"{'='*70}")
        print(f"  Input: {len(regions)} regions -> Output: {len(articles)} articles")
        print()

        # Build region lookup
        region_map = {r["id"]: r for r in regions}

        for art in articles:
            aid = art.get("article_id", "?")
            member_ids = art.get("member_region_ids", art.get("member_regions", []))
            title_id = art.get("title_region_id", art.get("title_region"))
            hint = (art.get("headline_hint") or "")[:60]
            dateline = art.get("dateline", "")
            grouped_by = art.get("grouped_by", "")
            bbox = art.get("bbox", [])

            print(f"  ARTICLE {aid}: \"{hint}\"")
            if dateline:
                print(f"    Dateline: {dateline}")
            if title_id:
                title_r = region_map.get(title_id, {})
                print(f"    Title region: ID {title_id} (type={title_r.get('type','?')}, bbox={title_r.get('bbox',[])})")
            print(f"    Member regions ({len(member_ids)}): {member_ids}")

            # Show each member region's details
            if member_ids:
                min_x, min_y, max_x, max_y = 99999, 99999, 0, 0
                for rid in member_ids:
                    r = region_map.get(rid, {})
                    rb = r.get("bbox", [0,0,0,0])
                    print(f"      Region {rid:3d}: type={r.get('type','?'):15s} bbox=[{rb[0]:5d},{rb[1]:5d},{rb[2]:5d},{rb[3]:5d}] size={rb[2]-rb[0]:5d}x{rb[3]-rb[1]:5d}px")
                    min_x = min(min_x, rb[0])
                    min_y = min(min_y, rb[1])
                    max_x = max(max_x, rb[2])
                    max_y = max(max_y, rb[3])
                print(f"    Merged bbox: [{min_x},{min_y},{max_x},{max_y}] size={max_x-min_x}x{max_y-min_y}px")
            if bbox:
                print(f"    Article bbox: {bbox}")
            print()

        # Summary stats
        region_counts = [len(art.get("member_region_ids", art.get("member_regions", []))) for art in articles]
        if region_counts:
            print(f"  STATS:")
            print(f"    Articles: {len(articles)}")
            print(f"    Regions per article: min={min(region_counts)}, max={max(region_counts)}, avg={sum(region_counts)/len(region_counts):.1f}")
            large = [i+1 for i,c in enumerate(region_counts) if c > 6]
            if large:
                print(f"    ⚠️ Large articles (>6 regions): {large} — may have merged multiple stories")
        print(f"{'='*70}\n")

        # Draw grouped articles on debug image with different colors per article
        try:
            from PIL import ImageDraw
            debug_img = Image.open(page_img_path).copy()
            draw = ImageDraw.Draw(debug_img)

            # Dark distinct colors for each article
            article_colors = [
                (255, 0, 0),       # red
                (0, 0, 255),       # blue
                (0, 150, 0),       # dark green
                (180, 0, 180),     # purple
                (200, 100, 0),     # dark orange
                (0, 150, 150),     # teal
                (150, 0, 0),       # dark red
                (0, 0, 150),       # dark blue
                (100, 100, 0),     # olive
                (200, 0, 100),     # magenta
                (0, 100, 200),     # steel blue
                (150, 75, 0),      # brown
                (100, 0, 150),     # dark purple
                (0, 130, 80),      # dark teal
                (180, 50, 50),     # brick red
                (50, 50, 180),     # indigo
                (0, 180, 0),       # green
                (180, 0, 0),       # crimson
                (0, 0, 180),       # navy
                (150, 150, 0),     # dark yellow
                (200, 50, 150),    # pink
                (50, 150, 50),     # forest green
                (100, 50, 200),    # violet
                (200, 150, 50),    # gold
                (50, 100, 150),    # slate
            ]

            for idx, art in enumerate(articles):
                color = article_colors[idx % len(article_colors)]
                aid = art.get("article_id", idx+1)
                member_ids = art.get("member_region_ids", art.get("member_regions", []))

                for rid in member_ids:
                    r = region_map.get(rid, {})
                    rb = r.get("bbox", [0,0,0,0])
                    # Draw thick border with article color
                    draw.rectangle([rb[0], rb[1], rb[2], rb[3]], outline=color, width=5)
                    # Label with article ID
                    draw.text((rb[0]+8, rb[1]+8), f"A{aid}", fill=color)

                # Draw merged article bbox as thick dark outer border
                if member_ids:
                    all_bboxes = [region_map.get(rid, {}).get("bbox", [0,0,0,0]) for rid in member_ids]
                    mx1 = min(b[0] for b in all_bboxes)
                    my1 = min(b[1] for b in all_bboxes)
                    mx2 = max(b[2] for b in all_bboxes)
                    my2 = max(b[3] for b in all_bboxes)
                    # Thick dark outer border around the whole article
                    draw.rectangle([mx1-15, my1-15, mx2+15, my2+15], outline=color, width=15)
                    # Dark background label
                    label = f"Article {aid}: {(art.get('headline_hint') or '')[:30]}"
                    draw.rectangle([mx1-10, my1-40, mx1 + len(label)*12, my1-10], fill=color)
                    draw.text((mx1-5, my1-38), label, fill="white")

            debug_path = page_img_path.replace(".png", "_articles_debug.png")
            debug_img.save(debug_path)
            print(f"ARTICLES DEBUG IMAGE saved: {debug_path}")
            print(f"  Each article has a unique color. Regions with same color = same article.")
            print()
        except Exception as e:
            print(f"  Could not create articles debug image: {e}")
        # Step 4: Filter out garbage/tiny article regions before cropping.
        # Surya sometimes produces title fragments (e.g. 29×33 px) that aren't
        # real articles — they waste OCR compute and pollute triage results.
        articles_before = len(articles)
        articles = [
            a for a in articles
            if (lambda b: (b[2]-b[0]) * (b[3]-b[1]) >= MIN_ARTICLE_AREA and (b[2]-b[0]) >= MIN_ARTICLE_DIM and (b[3]-b[1]) >= MIN_ARTICLE_DIM)(a["bbox"])
        ]
        filtered_count = articles_before - len(articles)
        if filtered_count:
            print(f"  Page {n}: filtered out {filtered_count} tiny/garbage articles "
                  f"(< {MIN_ARTICLE_DIM}px or < {MIN_ARTICLE_AREA}px\u00b2), "
                  f"{len(articles)} remain for cropping.")

        # Step 4b: Crop and OCR HEADLINES only
        _set_status(job_id, f"Page {n}: cropping {len(articles)} headlines...")
        headline_records = crop_headlines(page_img_path, articles, regions, articles_root, n)
        
        _set_status(job_id, f"Page {n}: running fast OCR on {len(headline_records)} headlines...")
        ocr_headlines(headline_records)

        # Step 5: Triage with Claude Haiku based on headlines
        _set_status(job_id, f"Page {n}: triaging headlines with Claude...")
        selected_ids = _triage_headlines_with_claude(headline_records, n, job_dir=str(job_dir))

        # --- TRIAGE LOG: save all decisions for auditing ---
        rejected_dir = job_dir / "rejected"
        rejected_dir.mkdir(exist_ok=True)
        triage_log = {
            "page": n,
            "total": len(headline_records),
            "selected_count": len(selected_ids),
            "rejected_count": len(headline_records) - len(selected_ids),
            "articles": []
        }
        for rec in headline_records:
            triage_log["articles"].append({
                "id": rec["name"],
                "headline_text": rec.get("headline_text", "")[:200],
                "decision": "KEEP" if rec["name"] in selected_ids else "REJECT",
            })
        (pages_dir / f"page_{n:03d}.triage_log.json").write_text(
            json.dumps(triage_log, indent=2, ensure_ascii=False), encoding="utf-8"
        )

        # Move (not delete) rejected dirs to rejected/ so they can be audited
        for rec in headline_records:
            if rec["name"] not in selected_ids:
                src = Path(rec["dir"])
                dst = rejected_dir / src.name
                if src.exists() and not dst.exists():
                    shutil.move(str(src), str(dst))

        if not selected_ids:
            print(f"  Page {n}: 0 political articles found during triage.")
            continue

        # Step 6: Crop FULL articles for the selected ones
        _set_status(job_id, f"Page {n}: cropping {len(selected_ids)} FULL articles...")
        records = crop_full_articles(page_img_path, selected_ids, articles, articles_root, n)
        print(f"  Page {n}: cropped {len(records)} full articles")

        # Step 7: OCR full articles
        _set_status(job_id, f"Page {n}: running heavy OCR on {len(records)} full articles...")
        ocr_articles(records)

        # Step 8: Deep Analysis with Claude Opus
        _set_status(job_id, f"Page {n}: deep political analysis with Claude...")
        selected = _analyze_articles_with_claude(records, n, job_dir=str(job_dir))

        if selected:
            final_ids = {s["id"] for s in selected}
            print(f"  Page {n}: {len(selected)} politically significant articles: {sorted(final_ids)}")

            # Keep selected, move rejected to rejected/ for audit (never permanently delete)
            for rec in records:
                art_dir = Path(rec["dir"])
                art_name = art_dir.name
                if art_name in final_ids:
                    # Update info.json with political analysis
                    sel_info = next((s for s in selected if s["id"] == art_name), {})
                    info_path = art_dir / "info.json"
                    if info_path.exists():
                        info = json.loads(info_path.read_text(encoding="utf-8"))
                        info["headline_english"] = sel_info.get("headline_english", "")
                        info["political_significance"] = sel_info.get("political_significance", "")
                        info["attack_angle"] = sel_info.get("attack_angle", "")
                        info["priority"] = sel_info.get("priority", "")
                        info["category"] = sel_info.get("category", "")
                        info["grouped_by"] = "paddleocr + political_filter"
                        info_path.write_text(json.dumps(info, indent=2, ensure_ascii=False), encoding="utf-8")
                    all_records.append(rec)
                    sel_info["image_file"] = f"{art_name}.png"
                    all_selected.append(sel_info)
                else:
                    # Move to rejected/ for audit instead of deleting
                    dst = rejected_dir / art_name
                    if art_dir.exists() and not dst.exists():
                        shutil.move(str(art_dir), str(dst))
        else:
            # No API key or no political articles — keep all cropped articles
            print(f"  Page {n}: no final political filter applied, keeping all {len(records)} articles")
            all_records.extend(records)

    # --- SUMMARY ACROSS ALL PAGES ---
    print(f"\n{'='*70}")
    print(f"LAYOUT DETECTION SUMMARY — ALL PAGES")
    print(f"{'='*70}")

    # Reload all regions from saved JSON
    all_page_summaries = []
    grand_total = 0
    grand_type_counts = {}

    for p in pages:
        n = p["page"]
        regions_file = pages_dir / f"page_{n:03d}.regions.json"
        if regions_file.exists():
            data = json.loads(regions_file.read_text(encoding="utf-8"))
            page_regions = data.get("regions", [])
            grand_total += len(page_regions)

            # Count by type
            type_counts = {}
            for r in page_regions:
                t = r["type"]
                type_counts[t] = type_counts.get(t, 0) + 1
                grand_type_counts[t] = grand_type_counts.get(t, 0) + 1

            # Estimate articles: doc_title + paragraph_title = potential article starts
            doc_titles = type_counts.get("doc_title", 0)
            para_titles = type_counts.get("paragraph_title", 0)

            print(f"\n  PAGE {n}:")
            print(f"    Total regions: {len(page_regions)}")
            print(f"    Types: {dict(sorted(type_counts.items(), key=lambda x: -x[1]))}")
            print(f"    Potential articles (doc_title): {doc_titles}")
            print(f"    Potential sub-articles (paragraph_title): {para_titles}")
            print(f"    Estimated article range: {doc_titles} (min) to {doc_titles + para_titles} (max)")

            all_page_summaries.append({
                "page": n,
                "total_regions": len(page_regions),
                "type_counts": type_counts,
                "doc_titles": doc_titles,
                "paragraph_titles": para_titles,
            })

    print(f"\n  GRAND TOTAL:")
    print(f"    Pages: {len(pages)}")
    print(f"    Total regions: {grand_total}")
    print(f"    All types: {dict(sorted(grand_type_counts.items(), key=lambda x: -x[1]))}")
    total_doc = grand_type_counts.get("doc_title", 0)
    total_para = grand_type_counts.get("paragraph_title", 0)
    print(f"    Estimated articles across all pages: {total_doc} (min) to {total_doc + total_para} (max)")
    print(f"{'='*70}\n")

    # Save summary JSON
    summary_path = job_dir / "layout_summary.json"
    summary_path.write_text(json.dumps({
        "total_pages": len(pages),
        "total_regions": grand_total,
        "grand_type_counts": grand_type_counts,
        "estimated_articles_min": total_doc,
        "estimated_articles_max": total_doc + total_para,
        "pages": all_page_summaries,
    }, indent=2, ensure_ascii=False), encoding="utf-8")
    print(f"Summary saved: {summary_path}")

    # Save political filter results
    if all_selected:
        (job_dir / "political_articles.json").write_text(
            json.dumps(all_selected, indent=2, ensure_ascii=False), encoding="utf-8"
        )
        _generate_opposition_brief(all_selected, job_dir)

    # Copy all kept article images into one folder
    if all_records:
        all_images_dir = job_dir / "all_political_images"
        all_images_dir.mkdir(parents=True, exist_ok=True)
        for rec in all_records:
            art_dir = Path(rec["dir"])
            img_path = art_dir / "article.png"
            if img_path.exists():
                art_name = art_dir.name
                shutil.copy2(str(img_path), str(all_images_dir / f"{art_name}.png"))
        print(f"  Copied {len(all_records)} images to {all_images_dir}")

    # Generate the combined PDF with AI insights and images
    if all_selected and generate_political_pdf:
        generate_political_pdf(all_selected, job_dir)

    _set_status(job_id, "Done!")
    return {"pages": len(pages), "articles": len(all_records)}