From 817931fdbf9a5a0f011160910dcbf2f04e969497 Mon Sep 17 00:00:00 2001 From: Deep Koluguri Date: Fri, 12 Jun 2026 21:56:11 -0400 Subject: [PATCH] feat: user code changes merged --- _lines.py | 6 +- _partition_debug.py | 2 +- _replay_political.py | 2 +- app.py | 11 +- app_py_utf8.txt | 1 + extractor.py | 884 +++++++++++++++++++------------------------ fix_pdf.py | 11 + generate_pdf.py | 71 ++++ geometric_grouper.py | 170 +++++++++ original_app_py.txt | Bin 0 -> 14 bytes run_pipeline.py | 8 +- templates/index.html | 48 ++- templates/job.html | 55 +-- test_fpdf_real.py | 24 ++ test_pdf.py | 1 + xy_cut.py | 167 ++++++++ 16 files changed, 906 insertions(+), 555 deletions(-) create mode 100644 app_py_utf8.txt create mode 100644 fix_pdf.py create mode 100644 geometric_grouper.py create mode 100644 original_app_py.txt create mode 100644 test_fpdf_real.py create mode 100644 xy_cut.py diff --git a/_lines.py b/_lines.py index ea4ad7d..024b1dc 100644 --- a/_lines.py +++ b/_lines.py @@ -312,7 +312,7 @@ def separator_barriers(png_path, regions, near=80, min_overlap=0.3, # Column width (median text-region width). A horizontal line counts as an article # separator only when it spans most of a column. NOTE: a strict ≥1.0× column floor is # too aggressive — a SINGLE-column article's own boundary rule is itself <1 column wide, - # so ≥1.0× drops real separators and merges articles (measured: NT pg4 R121 → 2 datelines). + # so ≥1.0× drops real separators and merges articles (measured: NT pg4 R121 -> 2 datelines). # 0.6× column is the safe floor: it removes only the short caption / decorative underlines # and merges NO articles on AJ / NT / JG. _twid = sorted((r["bbox"][2] - r["bbox"][0]) for r in regions if r.get("type") == "text") @@ -363,7 +363,7 @@ def separator_barriers(png_path, regions, near=80, min_overlap=0.3, continue b = r["bbox"] if (b[2] - b[0]) < multicol_frac * W: - continue # single-column header → skip + continue # single-column header -> skip yb = _faint_rule_above(gray, b) if yb is not None and not _lead_photo_above(yb, b[0], b[2]): bars.append({"y": yb, "x1": b[0], "x2": b[2], @@ -377,7 +377,7 @@ def separator_barriers(png_path, regions, near=80, min_overlap=0.3, continue b = r["bbox"] if (b[2] - b[0]) >= multicol_frac * W: - continue # multi-column → handled above + continue # multi-column -> handled above yb = _faint_grey_band_above(gray, b) if yb is not None and not _lead_photo_above(yb, b[0], b[2]): bars.append({"y": yb, "x1": b[0], "x2": b[2]}) diff --git a/_partition_debug.py b/_partition_debug.py index e36aee5..35c35fb 100644 --- a/_partition_debug.py +++ b/_partition_debug.py @@ -58,7 +58,7 @@ def run(pdir, pg, paper, tag): and max(0, min(y2, w["y2"]) - max(y1, w["y1"])) >= 0.4 * bh] right = min(rcand) if rcand else cright right = max(right, x2) # never shrink - # topmost dead-end: top-row article → up to masthead bottom + # topmost dead-end: top-row article -> up to masthead bottom top = mh if (y1 <= top_row_cut and mh > 0) else y1 top = min(top, y1) # never shrink downward return [x1, top, right, y2] diff --git a/_replay_political.py b/_replay_political.py index 4720e03..b5fc71a 100644 --- a/_replay_political.py +++ b/_replay_political.py @@ -45,7 +45,7 @@ for paper, RUN, pages in RUNS: ds = se.find_article_starts_by_dateline(regs, png, paper) recs = se.crop_political_articles(png, political, regs, OUT, pg, dateline_starts=ds, sep_lines=bars) - print(f"=== {RUN.name} page {pg}: {len(political)} political → {len(recs)} crops") + print(f"=== {RUN.name} page {pg}: {len(political)} political -> {len(recs)} crops") for l in buf.getvalue().splitlines(): if any(k in l for k in ("Block-snap", "Cropped:", "Clipped", "Floored", "Grew", "Banner", "block-snap unavailable")): diff --git a/app.py b/app.py index 9b6033f..98e2906 100644 --- a/app.py +++ b/app.py @@ -33,7 +33,16 @@ from flask import ( send_from_directory, send_file, abort, jsonify, ) -from extractor import process_pdf, JOBS, JOB_LOCK +import sys +if hasattr(sys.stdout, 'reconfigure'): + sys.stdout.reconfigure(encoding='utf-8') +if hasattr(sys.stderr, 'reconfigure'): + sys.stderr.reconfigure(encoding='utf-8') + +from extractor import process_pdf + +JOBS = {} +JOB_LOCK = threading.Lock() ROOT = Path(__file__).parent UPLOAD_DIR = ROOT / "uploads" diff --git a/app_py_utf8.txt b/app_py_utf8.txt new file mode 100644 index 0000000..40a629e --- /dev/null +++ b/app_py_utf8.txt @@ -0,0 +1 @@ +None diff --git a/extractor.py b/extractor.py index b1fa034..fd0b3a8 100644 --- a/extractor.py +++ b/extractor.py @@ -13,10 +13,30 @@ import os import re import json import base64 +import shutil import threading from io import BytesIO from pathlib import Path +try: + from generate_pdf import generate_political_pdf +except ImportError: + generate_political_pdf = None + +# Load .env file if it exists, to populate environment variables like ANTHROPIC_API_KEY +if not os.environ.get("ANTHROPIC_API_KEY"): + env_path = Path(__file__).resolve().parent / ".env" + if env_path.exists(): + try: + for line in env_path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + key, val = line.split("=", 1) + if key.strip() == "ANTHROPIC_API_KEY": + os.environ["ANTHROPIC_API_KEY"] = val.strip().strip('"').strip("'") + except Exception as e: + print(f"Error loading .env file: {e}") + import fitz # PyMuPDF from PIL import Image @@ -36,13 +56,21 @@ def _set_status(job_id, msg): # --------------------------------------------------------------------------- # Stage 1 — render PDF pages # --------------------------------------------------------------------------- -def render_pdf_pages(pdf_path, out_dir, target_w, target_h): +def render_pdf_pages(pdf_path, out_dir, target_w, target_h, max_pages=None, pages_to_process=None): out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) doc = fitz.open(pdf_path) pages = [] + + # Determine page selection target set + targets = set(pages_to_process) if pages_to_process is not None else None + for i, page in enumerate(doc, start=1): + if max_pages and i > max_pages: + break + if targets is not None and i not in targets: + continue rect = page.rect scale = min(target_w / rect.width, target_h / rect.height) mat = fitz.Matrix(scale, scale) @@ -56,88 +84,58 @@ def render_pdf_pages(pdf_path, out_dir, target_w, target_h): # --------------------------------------------------------------------------- -# Stage 2 — detect regions with PaddleOCR PP-Structure +# Stage 2 — detect regions with Surya Layout # --------------------------------------------------------------------------- +_SURYA_LAYOUT_MODEL = None +_SURYA_LAYOUT_PROCESSOR = None + def _load_layout_model(): - global _LAYOUT - if _LAYOUT is not None: - return _LAYOUT + global _SURYA_LAYOUT_MODEL, _SURYA_LAYOUT_PROCESSOR + if _SURYA_LAYOUT_MODEL is not None: + return _SURYA_LAYOUT_MODEL, _SURYA_LAYOUT_PROCESSOR with _LAYOUT_LOCK: - if _LAYOUT is not None: - return _LAYOUT - import paddleocr - # PaddleOCR v3.5.0 — use LayoutDetection for layout analysis. - # Stock ~0.5 confidence cutoff (no threshold override) — this is what - # Claude grouping is tuned to see. - try: - _LAYOUT = paddleocr.LayoutDetection() - except Exception: - try: - _LAYOUT = paddleocr.PPStructureV3() - except Exception: - _LAYOUT = paddleocr.PaddleOCR(lang='te') - return _LAYOUT + if _SURYA_LAYOUT_MODEL is not None: + return _SURYA_LAYOUT_MODEL, _SURYA_LAYOUT_PROCESSOR + import surya.settings + from surya.model.detection.model import load_model, load_processor + + checkpoint = surya.settings.settings.LAYOUT_MODEL_CHECKPOINT + _SURYA_LAYOUT_MODEL = load_model(checkpoint=checkpoint) + _SURYA_LAYOUT_PROCESSOR = load_processor(checkpoint=checkpoint) + return _SURYA_LAYOUT_MODEL, _SURYA_LAYOUT_PROCESSOR def detect_regions(page_png_path): - layout = _load_layout_model() - import numpy as np - img = np.array(Image.open(page_png_path).convert("RGB")) + model, processor = _load_layout_model() + from surya.layout import batch_layout_detection + img = Image.open(page_png_path).convert("RGB") - # PaddleOCR v3.5.0 uses .predict() instead of direct call - if hasattr(layout, 'predict'): - raw = layout.predict(page_png_path) - elif callable(layout): - raw = layout(img) - else: - raw = layout.predict(img) + layout_predictions = batch_layout_detection([img], model, processor) regions = [] - # PaddleOCR v3.5.0: raw is a list of DetResult objects (dict-like). - # Each DetResult has 'boxes' key containing list of detected regions. - # Each box has: 'cls_id', 'label', 'score', 'coordinate' (4 floats: x1,y1,x2,y2) - if isinstance(raw, list) and len(raw) > 0: - for det_result in raw: - # Get boxes from the DetResult - boxes = None - if hasattr(det_result, 'get'): - boxes = det_result.get('boxes', None) - elif hasattr(det_result, 'boxes'): - boxes = det_result.boxes - elif isinstance(det_result, dict) and 'boxes' in det_result: - boxes = det_result['boxes'] + # Label mapping from Surya to our expected types + label_map = { + "Picture": "image", + "Section-header": "paragraph_title", + "Title": "doc_title", + "Text": "text", + "Caption": "text", + "Table": "table", + "Page-header": "header", + "Page-footer": "footer", + "List-item": "list" + } - if boxes is not None: - for box in boxes: - rid = len(regions) + 1 - # Extract coordinate - coord = None - if isinstance(box, dict): - coord = box.get('coordinate', box.get('bbox', None)) - label = box.get('label', box.get('type', 'unknown')).lower() - score = float(box.get('score', box.get('confidence', 0))) - elif hasattr(box, 'coordinate'): - coord = box.coordinate - label = getattr(box, 'label', 'unknown').lower() - score = float(getattr(box, 'score', 0)) - else: - label = 'unknown' - score = 0.0 - - bbox = [0, 0, 0, 0] - if coord is not None: - if hasattr(coord, 'tolist'): - coord = coord.tolist() - coord = [float(v) for v in coord] - if len(coord) == 4: - bbox = [int(v) for v in coord] - elif len(coord) >= 8: - xs = [coord[j] for j in range(0, len(coord), 2)] - ys = [coord[j] for j in range(1, len(coord), 2)] - bbox = [int(min(xs)), int(min(ys)), int(max(xs)), int(max(ys))] - - regions.append({"id": rid, "type": label, "bbox": bbox, "confidence": score}) + if layout_predictions: + layout = layout_predictions[0] + if hasattr(layout, 'bboxes'): + for rid, box in enumerate(layout.bboxes, start=1): + bbox = [int(v) for v in box.bbox] + raw_label = box.label + label = label_map.get(raw_label, "text").lower() + # Surya doesn't expose confidence in this API level easily, default to 1.0 + regions.append({"id": rid, "type": label, "bbox": bbox, "confidence": 1.0}) # Detailed logging print(f"\n{'='*70}") @@ -197,282 +195,103 @@ def detect_regions(page_png_path): # --------------------------------------------------------------------------- # Stage 3 — group regions into articles -# -# Primary: Claude API (set ANTHROPIC_API_KEY env var). -# Fallback: column-aware + dateline Python rules. # --------------------------------------------------------------------------- - -DATELINE_RE = re.compile( - r"^[\u0C00-\u0C7F\u0964\u0965A-Za-z0-9\s.\-]{1,40}?," - r"\s*[\u0C00-\u0C7F]{1,8}\s*\d{1,2}" - r"[^:]{0,30}\(ఆంధ్రజ్యోతి\)[\s\u0C00-\u0C7F]{0,4}:", - re.UNICODE, -) - - -def looks_like_dateline(text): - if not text: - return None - snippet = text.strip()[:120] - m = DATELINE_RE.match(snippet) - if m: - return snippet[:m.end()] - if "ఆంధ్రజ్యోతి" in snippet[:100] and ":" in snippet[:120]: - return snippet[:snippet.find(":") + 1] - return None - - -def _bbox_union(a, b): - return [min(a[0], b[0]), min(a[1], b[1]), max(a[2], b[2]), max(a[3], b[3])] - - -def _make_topband_ocr(page_img): - try: - import pytesseract - except ImportError: - return None - - def topband(reg): - l, t, r, b = reg["bbox"] - strip_h = min(140, b - t) - if strip_h <= 0: - return "" - strip = page_img.crop((l, t, r, t + strip_h)) - try: - return pytesseract.image_to_string(strip, lang="tel") - except Exception: - return "" - return topband - - -def _enrich_regions_with_text(regions, topband_ocr): - enriched = [] - for r in regions: - rec = dict(r) - if topband_ocr and r["type"] in ("text", "paragraph", "list"): - try: - txt = topband_ocr(r) or "" - except Exception: - txt = "" - rec["top_text"] = txt.strip()[:160] - d = looks_like_dateline(rec["top_text"]) - rec["dateline"] = d if d else None - else: - rec["top_text"] = "" - rec["dateline"] = None - enriched.append(rec) - return enriched - - -# ---------- Claude-based grouping ---------- -def _make_page_thumbnail(page_png_path, max_dim=900): - img = Image.open(page_png_path) - img.thumbnail((max_dim, max_dim), Image.LANCZOS) - buf = BytesIO() - img.save(buf, format="JPEG", quality=80) - return base64.standard_b64encode(buf.getvalue()).decode("ascii") - - -CLAUDE_GROUPING_PROMPT = """You group newspaper page regions into INDIVIDUAL articles. Each news story is a SEPARATE article. - -You receive: -1. A thumbnail of the full page (for spatial context). -2. A JSON list of regions on the page. Each has: - - id (int) - - type ("title", "text", "figure", "table", etc.) - - bbox [left, top, right, bottom] in pixels (top-left origin) - - top_text: a Tesseract OCR snippet from the top of text regions - - dateline: if non-null, the region starts an article body (strong signal) - -CRITICAL RULES — BE CONSERVATIVE, SPLIT MORE: -1. A `title` or `doc_title` region ALWAYS starts a NEW article. Every title = separate article. -2. A region with a non-null `dateline` ALWAYS starts a new article body. -3. Text regions attach to the NEAREST title above them IN THE SAME COLUMN ONLY. -4. Newspapers have COLUMNS. A region in column 1 NEVER belongs to an article in column 2, even if vertically close. -5. When in doubt, SPLIT into separate articles rather than merge. Too many small articles is better than one giant article containing multiple stories. -6. An article should typically have 1-5 regions. If an article has more than 6 regions, you are probably merging multiple stories — split them. -7. A banner title spanning multiple columns owns ONLY the text directly below it, not the entire page. -8. Types "header", "footer", "page_number" — make these their own separate article. -9. Figures/images attach to the nearest title above them in THEIR column only. -10. NEVER merge two articles that have different headlines or topics into one. - -Return ONLY valid JSON, no prose, no markdown fences: -{ - "articles": [ - { - "article_id": 1, - "title_region_id": , - "member_region_ids": [, ...], - "headline_hint": "", - "dateline": "" - } - ] -} - -Order articles top-to-bottom, then left-to-right by topmost member region. -Every region (except headers/footers) appears in exactly one article. -AIM FOR 15-25 articles per page for a typical Telugu newspaper front page.""" - - -def _group_with_claude(page_png_path, regions, api_key, model="claude-opus-4-8"): - try: - import anthropic - except ImportError: - raise RuntimeError("anthropic package not installed — pip install anthropic") - - client = anthropic.Anthropic(api_key=api_key) - thumbnail_b64 = _make_page_thumbnail(page_png_path) - - compact = [ - { - "id": r["id"], - "type": r["type"], - "bbox": r["bbox"], - "top_text": r.get("top_text", "")[:160], - "dateline": r.get("dateline"), - } - for r in regions - ] - - user_content = [ - {"type": "image", - "source": {"type": "base64", "media_type": "image/jpeg", "data": thumbnail_b64}}, - {"type": "text", - "text": "Regions:\n" + json.dumps(compact, ensure_ascii=False, indent=2)}, - ] - - msg = client.messages.create( - model=model, - max_tokens=4000, - system=CLAUDE_GROUPING_PROMPT, - messages=[{"role": "user", "content": user_content}], - ) - - raw = "".join(b.text for b in msg.content if hasattr(b, "text")).strip() - if raw.startswith("```"): - raw = raw.split("\n", 1)[1] if "\n" in raw else raw - if raw.endswith("```"): - raw = raw[:raw.rfind("```")] - data = json.loads(raw) - - region_by_id = {r["id"]: r for r in regions} - articles = [] - for art in data.get("articles", []): - ids = art.get("member_region_ids", []) - if not ids: - continue - bbox = None - for rid in ids: - r = region_by_id.get(rid) - if not r: - continue - bbox = list(r["bbox"]) if bbox is None else _bbox_union(bbox, r["bbox"]) - if bbox is None: - continue - articles.append({ - "article_id": art.get("article_id", len(articles) + 1), - "title_region": art.get("title_region_id"), - "member_regions": ids, - "bbox": bbox, - "dateline": art.get("dateline"), - "headline_hint": art.get("headline_hint"), - "grouped_by": "claude", - }) - return articles - - -# ---------- Fallback Python heuristic ---------- -def _assign_columns(regions, page_width): - if not regions: - return {0: regions} - threshold = page_width * 0.04 - lefts = sorted({r["bbox"][0] for r in regions}) - column_starts = [lefts[0]] - for prev, curr in zip(lefts, lefts[1:]): - if curr - prev > threshold: - column_starts.append(curr) - columns = {i: [] for i in range(len(column_starts))} - for r in regions: - left = r["bbox"][0] - col_idx = 0 - for i, cs in enumerate(column_starts): - if left >= cs - threshold: - col_idx = i - columns[col_idx].append(r) - return columns - - -def _group_with_rules(regions, page_width): - columns = _assign_columns(regions, page_width) - articles = [] - next_id = 1 - for col_idx in sorted(columns.keys()): - col_regs = sorted(columns[col_idx], key=lambda r: r["bbox"][1]) - current = None - for reg in col_regs: - rtype = reg["type"] - dateline = reg.get("dateline") - starts = (rtype in ("title", "header")) or (dateline is not None) - if starts: - if current is not None and current["member_regions"]: - articles.append(current) - current = { - "article_id": next_id, - "title_region": reg["id"] if rtype in ("title", "header") else None, - "member_regions": [reg["id"]], - "bbox": list(reg["bbox"]), - "dateline": dateline, - "grouped_by": "rules", - } - next_id += 1 - else: - if current is None: - current = { - "article_id": next_id, "title_region": None, - "member_regions": [reg["id"]], "bbox": list(reg["bbox"]), - "dateline": None, "grouped_by": "rules", - } - next_id += 1 - else: - current["member_regions"].append(reg["id"]) - current["bbox"] = _bbox_union(current["bbox"], reg["bbox"]) - if current is not None and current["member_regions"]: - articles.append(current) - return articles - - def group_into_articles(page_png_path, regions, page_width, page_img=None): - """Claude first; rules fallback. Returns (articles, enriched_regions).""" - topband_ocr = _make_topband_ocr(page_img) if page_img else None - enriched = _enrich_regions_with_text(regions, topband_ocr) - - api_key = os.environ.get("ANTHROPIC_API_KEY") - if api_key: - try: - articles = _group_with_claude(page_png_path, enriched, api_key) - if articles: - return articles, enriched - except Exception as e: - print(f"[claude grouping failed, falling back to rules]: {e}") - - articles = _group_with_rules(enriched, page_width) - return articles, enriched + """Geometric grouping. Returns (articles, enriched_regions).""" + # Filter out page-level chrome to prevent unneccessary top/bottom content + valid_regions = [r for r in regions if r["type"] not in ("header", "footer", "page_number")] + + print("[running geometric column grouping]") + import geometric_grouper + articles = geometric_grouper.group_surya_regions_geometrically(valid_regions) + return articles, valid_regions # --------------------------------------------------------------------------- # Stage 4 — crop each article # --------------------------------------------------------------------------- -def crop_articles(page_png_path, articles, out_dir, page_num): +def crop_headlines(page_png_path, articles, regions, out_dir, page_num): + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + img = Image.open(page_png_path) + + region_by_id = {r["id"]: r for r in regions} + results = [] + + for art in articles: + # Find the headline box + title_id = art.get("title_region") + if title_id and title_id in region_by_id: + l, t, r, b = region_by_id[title_id]["bbox"] + else: + # Fallback to the top 15% of the article if no title region + l, t, r, b = art["bbox"] + b = min(b, int(t + (b - t) * 0.15)) + + # Pad slightly + l = max(0, l - 10) + t = max(0, t - 10) + r = min(img.width, r + 10) + b = min(img.height, b + 10) + + crop = img.crop((l, t, r, b)) + art_name = f"p{page_num:03d}_a{art['article_id']:03d}" + art_dir = out_dir / art_name + art_dir.mkdir(parents=True, exist_ok=True) + img_path = art_dir / "headline.png" + crop.save(img_path) + + results.append({ + "article_id": art["article_id"], + "name": art_name, + "dir": str(art_dir), + "headline_img_path": str(img_path), + "bbox": art["bbox"], + "title_region": title_id, + "member_regions": art.get("member_regions", []) + }) + return results + + +def crop_full_articles(page_png_path, selected_ids, articles, out_dir, page_num): out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) img = Image.open(page_png_path) results = [] for art in articles: + art_name = f"p{page_num:03d}_a{art['article_id']:03d}" + if art_name not in selected_ids: + continue + l, t, r, b = art["bbox"] - margin = 6 - l = max(0, l - margin); t = max(0, t - margin) - r = min(img.width, r + margin); b = min(img.height, b + margin) + + # Dynamic Padding: expand by up to 25px, but don't intersect other articles + pad_l = pad_t = pad_r = pad_b = 25 + + for other in articles: + if other["article_id"] == art["article_id"]: + continue + ol, ot, or_, ob = other["bbox"] + + # Check vertical overlap + if max(t, ot) < min(b, ob): + if ol >= r: # Other is to the right + pad_r = min(pad_r, max(0, (ol - r) // 2)) + elif or_ <= l: # Other is to the left + pad_l = min(pad_l, max(0, (l - or_) // 2)) + + # Check horizontal overlap + if max(l, ol) < min(r, or_): + if ot >= b: # Other is below + pad_b = min(pad_b, max(0, (ot - b) // 2)) + elif ob <= t: # Other is above + pad_t = min(pad_t, max(0, (t - ob) // 2)) + + l = max(0, int(l - pad_l)) + t = max(0, int(t - pad_t)) + r = min(img.width, int(r + pad_r)) + b = min(img.height, int(b + pad_b)) crop = img.crop((l, t, r, b)) name = f"p{page_num:03d}_a{art['article_id']:03d}" @@ -490,7 +309,7 @@ def crop_articles(page_png_path, articles, out_dir, page_num): "headline_hint": art.get("headline_hint"), "grouped_by": art.get("grouped_by", "rules"), } - (art_dir / "info.json").write_text(json.dumps(info, indent=2, ensure_ascii=False)) + (art_dir / "info.json").write_text(json.dumps(info, indent=2, ensure_ascii=False), encoding="utf-8") results.append({"name": name, "dir": str(art_dir), **info}) return results @@ -498,12 +317,32 @@ def crop_articles(page_png_path, articles, out_dir, page_num): # --------------------------------------------------------------------------- # Stage 5 — OCR each cropped article (Telugu) # --------------------------------------------------------------------------- +_paddle_ocr_instance = None + def _ocr_with_paddleocr(img_path): """Use PaddleOCR for better reading-order-aware Telugu OCR.""" + global _paddle_ocr_instance + temp_path = None try: + from PIL import Image + with Image.open(img_path) as im: + px = im.width * im.height + if px > 4000000: + # Instead of skipping, resize the article image to prevent PaddleOCR deadlock + ratio = (3800000 / px) ** 0.5 + new_w = int(im.width * ratio) + new_h = int(im.height * ratio) + print(f"Article image {img_path.name} is too large ({im.width}x{im.height} = {px}px). Resizing to {new_w}x{new_h} for safe PaddleOCR.") + resized_im = im.resize((new_w, new_h), Image.LANCZOS) + temp_path = img_path.parent / f"temp_resized_{img_path.name}" + resized_im.save(temp_path) + import paddleocr - ocr = paddleocr.PaddleOCR(lang='te') - result = ocr.predict(str(img_path)) + if _paddle_ocr_instance is None: + _paddle_ocr_instance = paddleocr.PaddleOCR(lang='te') + + path_to_ocr = str(temp_path) if temp_path else str(img_path) + result = _paddle_ocr_instance.predict(path_to_ocr) lines = [] if isinstance(result, list) and len(result) > 0: @@ -569,24 +408,23 @@ def _ocr_with_paddleocr(img_path): except Exception as e: print(f"PaddleOCR text extraction failed: {e}") return None + finally: + if temp_path and temp_path.exists(): + try: + temp_path.unlink() + except: + pass -def _ocr_with_tesseract(img_path): - """Fallback: use Tesseract with column-aware PSM for Telugu OCR.""" - try: - import pytesseract - # PSM 1 = Automatic page segmentation with OSD (better for multi-column) - # PSM 3 = Fully automatic (default, often jumbles columns) - config = '--psm 1 --oem 3' - text = pytesseract.image_to_string(Image.open(img_path), lang="tel", config=config) - if not text.strip(): - # Fallback to PSM 4 (single column) if PSM 1 gives nothing - config = '--psm 4 --oem 3' - text = pytesseract.image_to_string(Image.open(img_path), lang="tel", config=config) - return text - except Exception as e: - return f"[OCR error: {e}]" - +def ocr_headlines(headline_records): + for rec in headline_records: + img_path = Path(rec["headline_img_path"]) + if not img_path.exists(): + rec["headline_text"] = "" + continue + text = _ocr_with_paddleocr(img_path) + rec["headline_text"] = (text or "").strip() + (img_path.parent / "headline.txt").write_text(rec["headline_text"], encoding="utf-8") def ocr_articles(article_records): for rec in article_records: @@ -595,57 +433,114 @@ def ocr_articles(article_records): if not img_path.exists(): continue - # Try PaddleOCR first (better reading order), fall back to Tesseract text = _ocr_with_paddleocr(img_path) - if not text or len(text.strip()) < 10: - text = _ocr_with_tesseract(img_path) - (art_dir / "article.txt").write_text(text or "", encoding="utf-8") info_path = art_dir / "info.json" if info_path.exists(): - info = json.loads(info_path.read_text()) - first_line = next((ln.strip() for ln in text.splitlines() if ln.strip()), "") - info["headline_preview"] = first_line[:120] - info_path.write_text(json.dumps(info, indent=2, ensure_ascii=False)) + info = json.loads(info_path.read_text(encoding="utf-8")) + info_path.write_text(json.dumps(info, indent=2, ensure_ascii=False), encoding="utf-8") # --------------------------------------------------------------------------- # Political Filter — crop first, then filter with Claude # --------------------------------------------------------------------------- -def _filter_articles_with_claude(page_img_path, article_ids_with_locations, page_num, job_dir=None): - """Send page image + article IDs to Claude, get back only politically significant IDs.""" - import base64 - from io import BytesIO +def _triage_headlines_with_claude(headline_records, page_num, job_dir=None): + """Send HEADLINES ONLY to Claude to select political articles, saving 90% compute.""" api_key = os.environ.get("ANTHROPIC_API_KEY") if not api_key: - return None + # If no key, select all to avoid losing data + return [r["name"] for r in headline_records] + + # Build the list of text articles + article_list_data = [] + for rec in headline_records: + txt = rec.get("headline_text", "").strip() + if not txt: + txt = "[No text extracted by OCR]" + + article_list_data.append({ + "id": rec["name"], + "headline_text": txt + }) + + if not article_list_data: + return [] try: import anthropic client = anthropic.Anthropic(api_key=api_key) - # Resize page image for API - img = Image.open(page_img_path) - max_w = 1500 - if img.width > max_w: - ratio = max_w / img.width - img = img.resize((max_w, int(img.height * ratio)), Image.LANCZOS) + articles_json_str = json.dumps(article_list_data, indent=2, ensure_ascii=False) - buf = BytesIO() - img.convert("RGB").save(buf, format="JPEG", quality=80) - img_data = base64.standard_b64encode(buf.getvalue()).decode("utf-8") + prompt = f"""You are a senior opposition party strategist in Telangana state, India. +I have extracted the HEADLINES from page {page_num} of a Telugu newspaper. Here is the list: - # Build article list text - article_list = "\n".join( - f"- {aid}: located at {loc}" for aid, loc in article_ids_with_locations +{articles_json_str} + +Return the IDs of articles that MIGHT describe DAMAGE, LOSS, CORRUPTION, or SUFFERING happening to people in Telangana, where the government could be blamed. +Also return the IDs of any articles about farmer protests, NEET paper leaks, or protests against the ruling party. +If you are unsure, INCLUDE it. We just want to filter out obvious ads, sports, and unrelated filler. + +Return ONLY a valid JSON list of IDs. Example: ["p001_a002", "p001_a005"]. No prose.""" + + response = client.messages.create( + model="claude-sonnet-4-6", + max_tokens=1000, + messages=[{"role": "user", "content": prompt}], ) + text = response.content[0].text.strip() + if text.startswith("```"): + text = text.split("\n", 1)[1] + if text.endswith("```"): + text = text[:-3] + + selected_ids = json.loads(text.strip()) + print(f" Page {page_num}: Triage kept {len(selected_ids)}/{len(headline_records)} articles based on headlines.") + return selected_ids + + except Exception as e: + print(f"Headline triage failed: {e}") + return [r["name"] for r in headline_records] + + +def _analyze_articles_with_claude(records, page_num, job_dir=None): + """Deep analysis on the full OCR text of the selected articles.""" + api_key = os.environ.get("ANTHROPIC_API_KEY") + if not api_key: + return None + + article_list_data = [] + for rec in records: + art_dir = Path(rec["dir"]) + txt_path = art_dir / "article.txt" + txt_content = "" + if txt_path.exists(): + txt_content = txt_path.read_text(encoding="utf-8").strip() + if not txt_content: + txt_content = "[No text extracted by OCR]" + + article_list_data.append({ + "id": rec["name"], + "ocr_text": txt_content[:3000] + }) + + if not article_list_data: + return [] + + try: + import anthropic + client = anthropic.Anthropic(api_key=api_key) + + # Convert list to JSON string for the prompt + articles_json_str = json.dumps(article_list_data, indent=2, ensure_ascii=False) + prompt = f"""You are a senior opposition party strategist in Telangana state, India. -I have already cropped the following articles from this newspaper page: +I have run OCR on the cropped articles of page {page_num} of a Telugu newspaper. Here is the list of article IDs and their Telugu OCR text contents: -{article_list} +{articles_json_str} For EACH article, ask yourself these TWO questions: @@ -654,16 +549,16 @@ QUESTION 1: "Does this article describe actual DAMAGE, LOSS, DEATH, CORRUPTION, - There must be a VICTIM — someone who died, lost money, was cheated, is suffering, or was harmed. - The government (Telangana state OR central/BJP) can be blamed — for causing it, failing to prevent it, or failing to respond to it. - Central government failures also qualify IF the article describes actual harm to Telangana citizens (e.g., NEET paper leak harming Telangana students, central policy causing job losses in Telangana). -- If the article describes something POSITIVE (new scheme, development, achievement) → answer is NO. -- If the article describes a PLAN or ANNOUNCEMENT (not something bad that happened) → answer is NO. -- If there is NO victim and NO damage described → answer is NO. +- If the article describes something POSITIVE (new scheme, development, achievement) -> answer is NO. +- If the article describes a PLAN, PETITION, or ANNOUNCEMENT (e.g. MLA/civilians submitting a representation requesting funds, work commencement, inspections, or temple matters) -> answer is NO. It must describe an active issue/damage event, not requests for future work. +- If there is NO victim and NO damage described -> answer is NO. - RESPONSIBILITY CHECK: Ask "WHO caused this damage?" If the damage was caused by nature (fire, flood, lightning), individual negligence (reckless driving, personal accident), criminals (theft, murder), or private parties — and the government had NO role in causing or preventing it — answer is NO. Only answer YES if the government is CLEARLY responsible through its policy, negligence, corruption, or failure to act. -- If YES → proceed to Question 2 -- If NO → REJECT this article immediately +- If YES -> proceed to Question 2 +- If NO -> REJECT this article immediately QUESTION 2: "Can the opposition party DIRECTLY USE this article to attack the Telangana government in a press conference, assembly session, or public rally?" -- If YES → SELECT this article -- If NO → REJECT this article +- If YES -> SELECT this article +- If NO -> REJECT this article BOTH answers must be YES to select. If either answer is NO, reject. @@ -686,32 +581,30 @@ DO NOT SELECT — automatically REJECT: Return a JSON object with TWO lists: -1. "reasoning": for EVERY article (selected or not), explain your thinking: +1. "reasoning": for EVERY article (selected or not), explain your thinking (KEEP ALL REASONS EXTREMELY BRIEF, MAX 10 WORDS EACH TO PREVENT TOKEN CUTOFF): - "id": the article ID - - "headline_english": English translation + - "headline_english": brief English headline (max 8 words) - "q1": "YES" or "NO" - - "q1_reason": why (1 sentence) + - "q1_reason": short reason (max 10 words) - "q2": "YES" or "NO" (skip if q1 is NO) - - "q2_reason": why (1 sentence, skip if q1 is NO) + - "q2_reason": short reason (max 10 words, skip if q1 is NO) - "decision": "SELECT" or "REJECT" 2. "selected": only the articles where both Q1 and Q2 are YES: - "id": the article ID - "headline_english": English translation of the headline - - "q1_answer": your answer to Question 1 (1 sentence) - - "q2_answer": your answer to Question 2 (1 sentence) - - "attack_angle": what opposition should say in a press conference (1 sentence) + - "q1_answer": answer to Question 1 (1 short sentence) + - "q2_answer": answer to Question 2 (1 short sentence) + - "attack_angle": what opposition should say (1 sentence) - "priority": "high" or "medium" - "category": one of corruption, governance_failure, public_grievance, law_order, health, education, infrastructure, ruling_party_crisis, farmer_issues Return ONLY valid JSON. No markdown. If no articles pass both questions, return {{"reasoning": [...], "selected": []}}""" response = client.messages.create( - model="claude-opus-4-8", max_tokens=4096, - messages=[{"role": "user", "content": [ - {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": img_data}}, - {"type": "text", "text": prompt}, - ]}], + model="claude-sonnet-4-6", + max_tokens=8192, + messages=[{"role": "user", "content": prompt}], ) text = response.content[0].text.strip() @@ -732,11 +625,10 @@ Return ONLY valid JSON. No markdown. If no articles pass both questions, return print(f" Q1: {r.get('q1','?')} — {r.get('q1_reason','')}") if r.get('q2'): print(f" Q2: {r.get('q2','?')} — {r.get('q2_reason','')}") - print(f" → {status}") + print(f" -> {status}") print(f"=== END REASONING ===\n") # Save reasoning log to job directory - from pathlib import Path log_path = Path(job_dir) / f"filter_reasoning_page{page_num}.json" if job_dir else None if log_path: log_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") @@ -780,7 +672,7 @@ def _generate_opposition_brief(selected_articles, job_dir): # --------------------------------------------------------------------------- # Orchestrator # --------------------------------------------------------------------------- -def process_pdf(pdf_path, job_dir, target_w, target_h, job_id): +def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, pages_to_process=None): job_dir = Path(job_dir) pages_dir = job_dir / "pages" articles_root = job_dir / "articles" @@ -789,7 +681,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id): # Step 1: Render PDF pages _set_status(job_id, "Rendering PDF pages...") - pages = render_pdf_pages(pdf_path, pages_dir, target_w, target_h) + pages = render_pdf_pages(pdf_path, pages_dir, target_w, target_h, max_pages=max_pages, pages_to_process=pages_to_process) all_records = [] all_selected = [] @@ -802,7 +694,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id): _set_status(job_id, f"Page {n}: detecting layout regions...") regions = detect_regions(page_img_path) (pages_dir / f"page_{n:03d}.regions.json").write_text( - json.dumps({"page": n, "regions": regions}, indent=2, ensure_ascii=False) + json.dumps({"page": n, "regions": regions}, indent=2, ensure_ascii=False), encoding="utf-8" ) print(f" Page {n}: {len(regions)} layout regions detected") @@ -813,14 +705,14 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id): (pages_dir / f"page_{n:03d}.articles.json").write_text( json.dumps({"page": n, "articles": articles, "enriched_regions": enriched}, - indent=2, ensure_ascii=False) + indent=2, ensure_ascii=False), encoding="utf-8" ) # --- DETAILED GROUPING LOG --- print(f"\n{'='*70}") print(f"GROUPING RESULTS — PAGE {n}") print(f"{'='*70}") - print(f" Input: {len(regions)} regions → Output: {len(articles)} articles") + print(f" Input: {len(regions)} regions -> Output: {len(articles)} articles") print() # Build region lookup @@ -939,70 +831,69 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id): print() except Exception as e: print(f" Could not create articles debug image: {e}") - # - # # Step 4: Crop ALL articles with precise PaddleOCR boundaries - # _set_status(job_id, f"Page {n}: cropping {len(articles)} articles...") - # records = crop_articles(page_img_path, articles, articles_root, n) - # print(f" Page {n}: cropped {len(records)} articles") - # - # # Build article ID + location + headline list for Claude - # article_ids_with_locations = [] - # for rec in records: - # art_dir = Path(rec["dir"]) - # art_name = art_dir.name - # info_path = art_dir / "info.json" - # if info_path.exists(): - # info = json.loads(info_path.read_text()) - # bbox = info.get("bbox", [0, 0, 0, 0]) - # hint = info.get("headline_hint", "") - # pw, ph = page_img.size - # loc = f"headline: \"{hint}\", position: x:{bbox[0]}-{bbox[2]}, y:{bbox[1]}-{bbox[3]}" - # else: - # loc = "unknown" - # article_ids_with_locations.append((art_name, loc)) - # - # # Step 5: Political filter — send page image + article IDs to Claude - # if article_ids_with_locations: - # _set_status(job_id, f"Page {n}: filtering {len(records)} articles for political significance...") - # selected = _filter_articles_with_claude(page_img_path, article_ids_with_locations, n, job_dir=str(job_dir)) - # - # if selected: - # selected_ids = {s["id"] for s in selected} - # print(f" Page {n}: {len(selected)} politically significant articles: {selected_ids}") - # - # # Step 6: Keep only selected articles, remove the rest - # for rec in records: - # art_dir = Path(rec["dir"]) - # art_name = art_dir.name - # if art_name in selected_ids: - # # Update info.json with political analysis - # sel_info = next((s for s in selected if s["id"] == art_name), {}) - # info_path = art_dir / "info.json" - # if info_path.exists(): - # info = json.loads(info_path.read_text()) - # info["headline_english"] = sel_info.get("headline_english", "") - # info["political_significance"] = sel_info.get("political_significance", "") - # info["attack_angle"] = sel_info.get("attack_angle", "") - # info["priority"] = sel_info.get("priority", "") - # info["category"] = sel_info.get("category", "") - # info["grouped_by"] = "paddleocr + political_filter" - # info_path.write_text(json.dumps(info, indent=2, ensure_ascii=False)) - # all_records.append(rec) - # all_selected.append(sel_info) - # else: - # # Delete non-political article - # import shutil - # shutil.rmtree(str(art_dir), ignore_errors=True) - # else: - # # No political articles on this page — delete all cropped articles - # print(f" Page {n}: no political articles found, removing all {len(records)} articles") - # import shutil - # for rec in records: - # shutil.rmtree(str(Path(rec["dir"])), ignore_errors=True) - # else: - # all_records.extend(records) + # Step 4: Crop and OCR HEADLINES only + _set_status(job_id, f"Page {n}: cropping {len(articles)} headlines...") + headline_records = crop_headlines(page_img_path, articles, regions, articles_root, n) + + _set_status(job_id, f"Page {n}: running fast OCR on {len(headline_records)} headlines...") + ocr_headlines(headline_records) - print(f" Page {n}: Steps 3-6 skipped (commented out). Check regions JSON.") + # Step 5: Triage with Claude Haiku based on headlines + _set_status(job_id, f"Page {n}: triaging headlines with Claude...") + selected_ids = _triage_headlines_with_claude(headline_records, n, job_dir=str(job_dir)) + + # Cleanup non-selected directories + for rec in headline_records: + if rec["name"] not in selected_ids: + shutil.rmtree(rec["dir"], ignore_errors=True) + + if not selected_ids: + print(f" Page {n}: 0 political articles found during triage.") + continue + + # Step 6: Crop FULL articles for the selected ones + _set_status(job_id, f"Page {n}: cropping {len(selected_ids)} FULL articles...") + records = crop_full_articles(page_img_path, selected_ids, articles, articles_root, n) + print(f" Page {n}: cropped {len(records)} full articles") + + # Step 7: OCR full articles + _set_status(job_id, f"Page {n}: running heavy OCR on {len(records)} full articles...") + ocr_articles(records) + + # Step 8: Deep Analysis with Claude Opus + _set_status(job_id, f"Page {n}: deep political analysis with Claude...") + selected = _analyze_articles_with_claude(records, n, job_dir=str(job_dir)) + + if selected: + final_ids = {s["id"] for s in selected} + print(f" Page {n}: {len(selected)} politically significant articles: {sorted(final_ids)}") + + # Keep only selected articles, remove the rest + for rec in records: + art_dir = Path(rec["dir"]) + art_name = art_dir.name + if art_name in final_ids: + # Update info.json with political analysis + sel_info = next((s for s in selected if s["id"] == art_name), {}) + info_path = art_dir / "info.json" + if info_path.exists(): + info = json.loads(info_path.read_text(encoding="utf-8")) + info["headline_english"] = sel_info.get("headline_english", "") + info["political_significance"] = sel_info.get("political_significance", "") + info["attack_angle"] = sel_info.get("attack_angle", "") + info["priority"] = sel_info.get("priority", "") + info["category"] = sel_info.get("category", "") + info["grouped_by"] = "paddleocr + political_filter" + info_path.write_text(json.dumps(info, indent=2, ensure_ascii=False), encoding="utf-8") + all_records.append(rec) + sel_info["image_file"] = f"{art_name}.png" + all_selected.append(sel_info) + else: + shutil.rmtree(str(art_dir), ignore_errors=True) + else: + # No API key or no political articles — keep all cropped articles + print(f" Page {n}: no final political filter applied, keeping all {len(records)} articles") + all_records.extend(records) # --- SUMMARY ACROSS ALL PAGES --- print(f"\n{'='*70}") @@ -1018,7 +909,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id): n = p["page"] regions_file = pages_dir / f"page_{n:03d}.regions.json" if regions_file.exists(): - data = json.loads(regions_file.read_text()) + data = json.loads(regions_file.read_text(encoding="utf-8")) page_regions = data.get("regions", []) grand_total += len(page_regions) @@ -1069,37 +960,28 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id): }, indent=2, ensure_ascii=False), encoding="utf-8") print(f"Summary saved: {summary_path}") - # # Save political filter results - # if all_selected: - # (job_dir / "political_articles.json").write_text( - # json.dumps(all_selected, indent=2, ensure_ascii=False), encoding="utf-8" - # ) - # _generate_opposition_brief(all_selected, job_dir) + # Save political filter results + if all_selected: + (job_dir / "political_articles.json").write_text( + json.dumps(all_selected, indent=2, ensure_ascii=False), encoding="utf-8" + ) + _generate_opposition_brief(all_selected, job_dir) - # # Copy all kept article images into one folder - # all_images_dir = job_dir / "all_political_images" - # all_images_dir.mkdir(parents=True, exist_ok=True) - # import shutil - # for rec in all_records: - # art_dir = Path(rec["dir"]) - # img_path = art_dir / "article.png" - # if img_path.exists(): - # art_name = art_dir.name - # shutil.copy2(str(img_path), str(all_images_dir / f"{art_name}.png")) - # print(f" Copied {len(all_records)} images to {all_images_dir}") + # Copy all kept article images into one folder + if all_records: + all_images_dir = job_dir / "all_political_images" + all_images_dir.mkdir(parents=True, exist_ok=True) + for rec in all_records: + art_dir = Path(rec["dir"]) + img_path = art_dir / "article.png" + if img_path.exists(): + art_name = art_dir.name + shutil.copy2(str(img_path), str(all_images_dir / f"{art_name}.png")) + print(f" Copied {len(all_records)} images to {all_images_dir}") - # # Create a single PDF with all political article images - # if all_records: - # pdf_images = [] - # for rec in sorted(all_records, key=lambda r: r.get("article_id", 0)): - # art_dir = Path(rec["dir"]) - # img_path = art_dir / "article.png" - # if img_path.exists(): - # pdf_images.append(Image.open(img_path).convert("RGB")) - # if pdf_images: - # pdf_path_out = job_dir / "political_articles.pdf" - # pdf_images[0].save(str(pdf_path_out), save_all=True, append_images=pdf_images[1:]) - # print(f" Created PDF with {len(pdf_images)} articles: {pdf_path_out}") + # Generate the combined PDF with AI insights and images + if all_selected and generate_political_pdf: + generate_political_pdf(all_selected, job_dir) - _set_status(job_id, f"Done! Steps 1-2 complete. {len(regions)} regions detected.") - return {"pages": len(pages), "articles": 0} + _set_status(job_id, "Done!") + return {"pages": len(pages), "articles": len(all_records)} diff --git a/fix_pdf.py b/fix_pdf.py new file mode 100644 index 0000000..835cae5 --- /dev/null +++ b/fix_pdf.py @@ -0,0 +1,11 @@ +import json +from pathlib import Path +from generate_pdf import generate_political_pdf + +job_dir = Path(r'c:\Users\sunde\proxmox\news-scan\output\20260612_131909') +articles = json.loads((job_dir / 'political_articles.json').read_text(encoding='utf-8')) +for a in articles: + a['image_file'] = f"{a['id']}.png" + +generate_political_pdf(articles, job_dir) +print("Regenerated PDF with images!") diff --git a/generate_pdf.py b/generate_pdf.py index e69de29..43d4dd9 100644 --- a/generate_pdf.py +++ b/generate_pdf.py @@ -0,0 +1,71 @@ +import json +from pathlib import Path +from fpdf import FPDF +import unicodedata + +def clean_text(text): + if not text: + return "N/A" + # Replace unicode quotes and dashes with ascii equivalents to avoid FPDF errors with Latin-1 + text = str(text) + text = text.replace('"', '"').replace('"', '"').replace('"', "'").replace('"', "'") + text = text.replace('—', '-').replace('–', '-') + # Normalize to nearest ascii character + res = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('ascii') + res = res.strip() + if not res: + return "N/A" + # Break up extremely long words that crash FPDF + res = " ".join([word[:50] for word in res.split()]) + return res + +def generate_political_pdf(selected_articles, job_dir): + job_dir = Path(job_dir) + images_dir = job_dir / "all_political_images" + + if not images_dir.exists() or not selected_articles: + return + + pdf = FPDF() + pdf.set_auto_page_break(auto=True, margin=15) + + # Sort articles: high priority first + high = [a for a in selected_articles if a.get("priority") == "high"] + medium = [a for a in selected_articles if a.get("priority") == "medium"] + sorted_articles = high + medium + + for i, art in enumerate(sorted_articles, 1): + pdf.add_page() + + # Add Header + pdf.set_font("helvetica", "B", 16) + priority = str(art.get("priority", "N/A")).upper() + pdf.cell(0, 10, f"Article {i} - {priority} PRIORITY", new_x="LMARGIN", new_y="NEXT", align="C") + pdf.ln(5) + + # Details + pdf.set_font("helvetica", "B", 12) + pdf.multi_cell(0, 8, f"Headline: {clean_text(art.get('headline_english'))}", new_x="LMARGIN", new_y="NEXT") + pdf.multi_cell(0, 8, f"Category: {clean_text(art.get('category')).upper()}", new_x="LMARGIN", new_y="NEXT") + + pdf.set_font("helvetica", "", 12) + pdf.multi_cell(0, 8, f"Why: {clean_text(art.get('reasoning'))}", new_x="LMARGIN", new_y="NEXT") + pdf.ln(5) + + # Image + img_name = art.get("image_file") + if img_name: + img_path = images_dir / img_name + if img_path.exists(): + # Get image dimensions to scale it properly if needed + try: + pdf.image(str(img_path), w=180) + except Exception as e: + print(f"Error adding image to PDF: {e}") + + output_path = job_dir / "political_articles_insights.pdf" + try: + pdf.output(str(output_path)) + print(f"Successfully generated PDF report at {output_path}") + except Exception as e: + print(f"Failed to generate PDF: {e}") diff --git a/geometric_grouper.py b/geometric_grouper.py new file mode 100644 index 0000000..d57a8b9 --- /dev/null +++ b/geometric_grouper.py @@ -0,0 +1,170 @@ +import math + +def get_horizontal_overlap(bbox1, bbox2): + x_left = max(bbox1[0], bbox2[0]) + x_right = min(bbox1[2], bbox2[2]) + return max(0, x_right - x_left) + +def group_surya_regions_geometrically(regions): + """ + Groups regions spatially. + Instead of a linear reading order, this associates text and images + with the headline that is directly above them in the same column. + """ + if not regions: + return [] + + titles = [r for r in regions if r['type'] in ('doc_title', 'paragraph_title')] + non_titles = [r for r in regions if r['type'] not in ('doc_title', 'paragraph_title')] + + # If no titles exist, return everything as one article + if not titles: + member_ids = [r['id'] for r in regions] + min_x = min(r['bbox'][0] for r in regions) + min_y = min(r['bbox'][1] for r in regions) + max_x = max(r['bbox'][2] for r in regions) + max_y = max(r['bbox'][3] for r in regions) + return [{ + "article_id": 1, + "title_region": None, + "member_regions": member_ids, + "bbox": [min_x, min_y, max_x, max_y], + "grouped_by": "geometric_fallback" + }] + + article_groups = {t['id']: [t] for t in titles} + + for item in non_titles: + item_bbox = item['bbox'] + item_cy = (item_bbox[1] + item_bbox[3]) / 2 + item_cx = (item_bbox[0] + item_bbox[2]) / 2 + item_width = item_bbox[2] - item_bbox[0] + + best_title = None + + # ------------------------------------------------------------- + # Rule 1: Find titles ABOVE the item that share the same column + # ------------------------------------------------------------- + candidates_above = [] + for t in titles: + t_bbox = t['bbox'] + if t_bbox[1] <= item_cy: + overlap = get_horizontal_overlap(item_bbox, t_bbox) + t_width = t_bbox[2] - t_bbox[0] + t_cx = (t_bbox[0] + t_bbox[2]) / 2 + + if overlap > (item_width * 0.1) or overlap > (t_width * 0.1) or abs(item_cx - t_cx) < 150: + distance = item_bbox[1] - t_bbox[3] + if distance < 1500: # MAX DISTANCE ABOVE + candidates_above.append(t) + + if candidates_above: + best_title = min(candidates_above, key=lambda t: abs(item_bbox[1] - t['bbox'][3])) + else: + # ------------------------------------------------------------- + # Rule 2: If no title is above, find titles BELOW the item + # ------------------------------------------------------------- + candidates_below = [] + for t in titles: + t_bbox = t['bbox'] + if t_bbox[1] > item_cy: + overlap = get_horizontal_overlap(item_bbox, t_bbox) + t_width = t_bbox[2] - t_bbox[0] + t_cx = (t_bbox[0] + t_bbox[2]) / 2 + + if overlap > (item_width * 0.1) or overlap > (t_width * 0.1) or abs(item_cx - t_cx) < 150: + distance = t_bbox[1] - item_bbox[3] + if distance < 800: # MAX DISTANCE BELOW (stricter because text usually flows down, not up) + candidates_below.append(t) + + if candidates_below: + best_title = min(candidates_below, key=lambda t: abs(t['bbox'][1] - item_bbox[3])) + else: + # ------------------------------------------------------------- + # Rule 3: Absolute fallback. No column overlap at all. + # Find the absolute closest title via Euclidean distance. + # ------------------------------------------------------------- + closest_t = min(titles, key=lambda t: math.hypot( + item_cx - ((t['bbox'][0] + t['bbox'][2]) / 2), + item_cy - ((t['bbox'][1] + t['bbox'][3]) / 2) + )) + # Only attach if it's reasonably close, else leave as orphan + dist = math.hypot(item_cx - ((closest_t['bbox'][0] + closest_t['bbox'][2]) / 2), + item_cy - ((closest_t['bbox'][1] + closest_t['bbox'][3]) / 2)) + if dist < 1500: + best_title = closest_t + + if best_title: + article_groups[best_title['id']].append(item) + else: + # Treat as an orphan (create a new article group for it, or group orphans together) + # For simplicity, we'll assign it a virtual title ID based on its own ID so it becomes a standalone article + virtual_id = f"orphan_{item['id']}" + article_groups[virtual_id] = [item] + + # Format output + articles = [] + # Sort groups top-to-bottom, left-to-right based on title position + sorted_titles = sorted(titles, key=lambda t: (t['bbox'][1], t['bbox'][0])) + + for idx, (title_id, group_regions) in enumerate(article_groups.items(), start=1): + if not group_regions: + continue + + min_x = min(r['bbox'][0] for r in group_regions) + min_y = min(r['bbox'][1] for r in group_regions) + max_x = max(r['bbox'][2] for r in group_regions) + max_y = max(r['bbox'][3] for r in group_regions) + + # Determine if it's an orphan group + is_orphan = str(title_id).startswith("orphan_") + + articles.append({ + "article_id": idx, + "title_region": None if is_orphan else title_id, + "member_regions": [r['id'] for r in group_regions], + "bbox": [min_x, min_y, max_x, max_y], + "grouped_by": "geometric_orphan" if is_orphan else "geometric" + }) + + # We should merge orphans that are vertically adjacent into the same orphan group to prevent shattering + # Simple vertical merge for orphans + orphan_articles = [a for a in articles if a['grouped_by'] == 'geometric_orphan'] + valid_articles = [a for a in articles if a['grouped_by'] == 'geometric'] + + # Sort orphans top-to-bottom + orphan_articles.sort(key=lambda a: a['bbox'][1]) + + merged_orphans = [] + current_orphan = None + + for o in orphan_articles: + if not current_orphan: + current_orphan = o + continue + + # Check if they overlap horizontally and are close vertically + overlap = get_horizontal_overlap(current_orphan['bbox'], o['bbox']) + vertical_dist = o['bbox'][1] - current_orphan['bbox'][3] + + if overlap > 0 and vertical_dist < 400: + # Merge + current_orphan['member_regions'].extend(o['member_regions']) + current_orphan['bbox'][0] = min(current_orphan['bbox'][0], o['bbox'][0]) + current_orphan['bbox'][1] = min(current_orphan['bbox'][1], o['bbox'][1]) + current_orphan['bbox'][2] = max(current_orphan['bbox'][2], o['bbox'][2]) + current_orphan['bbox'][3] = max(current_orphan['bbox'][3], o['bbox'][3]) + else: + merged_orphans.append(current_orphan) + current_orphan = o + + if current_orphan: + merged_orphans.append(current_orphan) + + final_articles = valid_articles + merged_orphans + + # Reassign IDs + for idx, a in enumerate(final_articles, start=1): + a['article_id'] = idx + + return final_articles diff --git a/original_app_py.txt b/original_app_py.txt new file mode 100644 index 0000000000000000000000000000000000000000..af0e4f4ad319ac2a57dbdc1205414434ef9e6117 GIT binary patch literal 14 VcmezW&yOLWA&()IftP`c0RSla1El}} literal 0 HcmV?d00001 diff --git a/run_pipeline.py b/run_pipeline.py index 46d30fe..de88c65 100644 --- a/run_pipeline.py +++ b/run_pipeline.py @@ -1,4 +1,10 @@ import os +import sys +if hasattr(sys.stdout, 'reconfigure'): + sys.stdout.reconfigure(encoding='utf-8') +if hasattr(sys.stderr, 'reconfigure'): + sys.stderr.reconfigure(encoding='utf-8') + import shutil import argparse from pathlib import Path @@ -87,7 +93,7 @@ def main(): timestamp = datetime.now().strftime("%H%M%S") dest_path = processed_dir / f"{pdf_path.stem}_{timestamp}{pdf_path.suffix}" - shutil.move(str(pdf_path), str(dest_path)) + shutil.move(str(pdf_path.absolute()), str(dest_path.absolute())) print(f"\nMoved processed file to: {dest_path}") # Check if the PDF report was generated diff --git a/templates/index.html b/templates/index.html index 8fa527a..45419f7 100644 --- a/templates/index.html +++ b/templates/index.html @@ -3,7 +3,8 @@

Upload a PDF

-

Each page will be rendered to fit inside the pixel box below, then split into per-article images and text.

+

Each page will be rendered to fit inside the pixel box below, then split into per-article images and + text.