refactor: optimize article extraction with size filtering, improved OCR context, and non-destructive audit logging for rejected content

extractor.py

@@ -43,6 +43,11 @@ from PIL import Image
 JOBS = {}
 JOB_LOCK = threading.Lock()
 
+# Minimum article bounding-box thresholds — filters out layout artifacts
+# (tiny title fragments, page decorations) before they waste OCR compute.
+MIN_ARTICLE_AREA = 10000   # pixels²  (e.g. 100×100 px minimum)
+MIN_ARTICLE_DIM  = 60      # pixels    either dimension below this → skip
+
 _LAYOUT = None
 _LAYOUT_LOCK = threading.Lock()
 
@@ -203,7 +208,7 @@ def group_into_articles(page_png_path, regions, page_width, page_img=None):
     
     print("[running geometric column grouping]")
     import geometric_grouper
-    articles = geometric_grouper.group_surya_regions_geometrically(valid_regions)
+    articles = geometric_grouper.group_surya_regions_geometrically(valid_regions, page_png_path)
     return articles, valid_regions
 
 
@@ -211,6 +216,11 @@ def group_into_articles(page_png_path, regions, page_width, page_img=None):
 # Stage 4 — crop each article
 # ---------------------------------------------------------------------------
 def crop_headlines(page_png_path, articles, regions, out_dir, page_num):
+    # How many pixels below the title region to include in the headline crop.
+    # Including a few lines of body text stabilises Telugu OCR output and gives
+    # Claude's triage enough context to make a reliable keep/reject decision.
+    HEADLINE_BODY_EXTENSION = 300
+
     out_dir = Path(out_dir)
     out_dir.mkdir(parents=True, exist_ok=True)
     img = Image.open(page_png_path)
@@ -223,10 +233,16 @@ def crop_headlines(page_png_path, articles, regions, out_dir, page_num):
         title_id = art.get("title_region")
         if title_id and title_id in region_by_id:
             l, t, r, b = region_by_id[title_id]["bbox"]
+            # Extend downward into the body to give OCR more context.
+            # A bare 68px title strip in Telugu is too short for stable OCR;
+            # including the first few lines of body text below it anchors the
+            # reading and reduces non-determinism in Claude's triage input.
+            art_bottom = art["bbox"][3]
+            b = min(art_bottom, b + HEADLINE_BODY_EXTENSION)
         else:
-            # Fallback to the top 15% of the article if no title region
+            # Fallback: top 20% of the article bbox when no title region exists
             l, t, r, b = art["bbox"]
-            b = min(b, int(t + (b - t) * 0.15))
+            b = min(b, int(t + (b - t) * 0.20))
             
         # Pad slightly
         l = max(0, l - 10)
@@ -396,13 +412,12 @@ def _ocr_with_paddleocr(img_path):
                         for row in rows:
                             lines.append(" ".join(p[2] for p in row))
                 else:
-                    # Last resort: try str representation
+                    # No text found via any path — return None cleanly.
-                    try:
+                    # (Caller will substitute "[No text extracted by OCR]".)
-                        s = str(det_result)
+                    # NOTE: do NOT fall back to str(det_result) here — that
-                        if len(s) > 10:
+                    # returns the raw PaddleOCR result dict as a string,
-                            return s
+                    # which poisons Claude's triage with file-path garbage.
-                    except:
+                    pass
-                        pass
 
         return "\n".join(lines) if lines else None
     except Exception as e:
@@ -478,9 +493,31 @@ I have extracted the HEADLINES from page {page_num} of a Telugu newspaper. Here
 
 {articles_json_str}
 
-Return the IDs of articles that MIGHT describe DAMAGE, LOSS, CORRUPTION, or SUFFERING happening to people in Telangana, where the government could be blamed.
+Your job is a BROAD FIRST PASS — keep anything that COULD be politically useful. You are NOT the final judge; a deeper analysis step will follow.
-Also return the IDs of any articles about farmer protests, NEET paper leaks, or protests against the ruling party.
+
-If you are unsure, INCLUDE it. We just want to filter out obvious ads, sports, and unrelated filler.
+KEEP articles about ANY of these:
+- Government failures, broken promises, delays, inaction
+- Price hikes, inflation, fuel costs, essential goods
+- Farmer distress, crop losses, water/power issues for farming
+- Crime, law & order failures, police negligence
+- Infrastructure problems (roads, hospitals, schools in bad shape)
+- Corruption, scams, misuse of funds by officials or politicians
+- Protests, strikes, public grievances against authorities
+- Health emergencies, hospital negligence, shortage of medicines
+- Job losses, unemployment, labour disputes
+- Education failures (school closures, NEET, exam problems)
+- Accidents or disasters with possible government negligence angle
+- Any ruling party internal conflict or embarrassment
+
+DO NOT KEEP (filter these out only if you are certain):
+- Pure sports scores or match results
+- Pure entertainment / cinema news with zero political angle
+- Horoscopes, puzzles, crosswords
+- Pure classified advertisements or product promotions
+- International news with zero Telangana/India relevance
+- Completely blank or unreadable OCR regions
+
+When in doubt, KEEP IT. It is far better to pass 10 borderline articles to the next stage than to miss one important story.
 
 Return ONLY a valid JSON list of IDs. Example: ["p001_a002", "p001_a005"]. No prose."""
 
@@ -831,7 +868,21 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
             print()
         except Exception as e:
             print(f"  Could not create articles debug image: {e}")
-        # Step 4: Crop and OCR HEADLINES only
+        # Step 4: Filter out garbage/tiny article regions before cropping.
+        # Surya sometimes produces title fragments (e.g. 29×33 px) that aren't
+        # real articles — they waste OCR compute and pollute triage results.
+        articles_before = len(articles)
+        articles = [
+            a for a in articles
+            if (lambda b: (b[2]-b[0]) * (b[3]-b[1]) >= MIN_ARTICLE_AREA and (b[2]-b[0]) >= MIN_ARTICLE_DIM and (b[3]-b[1]) >= MIN_ARTICLE_DIM)(a["bbox"])
+        ]
+        filtered_count = articles_before - len(articles)
+        if filtered_count:
+            print(f"  Page {n}: filtered out {filtered_count} tiny/garbage articles "
+                  f"(< {MIN_ARTICLE_DIM}px or < {MIN_ARTICLE_AREA}px\u00b2), "
+                  f"{len(articles)} remain for cropping.")
+
+        # Step 4b: Crop and OCR HEADLINES only
         _set_status(job_id, f"Page {n}: cropping {len(articles)} headlines...")
         headline_records = crop_headlines(page_img_path, articles, regions, articles_root, n)
         
@@ -842,10 +893,33 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
         _set_status(job_id, f"Page {n}: triaging headlines with Claude...")
         selected_ids = _triage_headlines_with_claude(headline_records, n, job_dir=str(job_dir))
 
-        # Cleanup non-selected directories
+        # --- TRIAGE LOG: save all decisions for auditing ---
+        rejected_dir = job_dir / "rejected"
+        rejected_dir.mkdir(exist_ok=True)
+        triage_log = {
+            "page": n,
+            "total": len(headline_records),
+            "selected_count": len(selected_ids),
+            "rejected_count": len(headline_records) - len(selected_ids),
+            "articles": []
+        }
+        for rec in headline_records:
+            triage_log["articles"].append({
+                "id": rec["name"],
+                "headline_text": rec.get("headline_text", "")[:200],
+                "decision": "KEEP" if rec["name"] in selected_ids else "REJECT",
+            })
+        (pages_dir / f"page_{n:03d}.triage_log.json").write_text(
+            json.dumps(triage_log, indent=2, ensure_ascii=False), encoding="utf-8"
+        )
+
+        # Move (not delete) rejected dirs to rejected/ so they can be audited
         for rec in headline_records:
             if rec["name"] not in selected_ids:
-                shutil.rmtree(rec["dir"], ignore_errors=True)
+                src = Path(rec["dir"])
+                dst = rejected_dir / src.name
+                if src.exists() and not dst.exists():
+                    shutil.move(str(src), str(dst))
 
         if not selected_ids:
             print(f"  Page {n}: 0 political articles found during triage.")
@@ -868,7 +942,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
             final_ids = {s["id"] for s in selected}
             print(f"  Page {n}: {len(selected)} politically significant articles: {sorted(final_ids)}")
 
-            # Keep only selected articles, remove the rest
+            # Keep selected, move rejected to rejected/ for audit (never permanently delete)
             for rec in records:
                 art_dir = Path(rec["dir"])
                 art_name = art_dir.name
@@ -889,7 +963,10 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
                     sel_info["image_file"] = f"{art_name}.png"
                     all_selected.append(sel_info)
                 else:
-                    shutil.rmtree(str(art_dir), ignore_errors=True)
+                    # Move to rejected/ for audit instead of deleting
+                    dst = rejected_dir / art_name
+                    if art_dir.exists() and not dst.exists():
+                        shutil.move(str(art_dir), str(dst))
         else:
             # No API key or no political articles — keep all cropped articles
             print(f"  Page {n}: no final political filter applied, keeping all {len(records)} articles")

generate_pdf.py

@@ -49,11 +49,13 @@ def generate_political_pdf(selected_articles, job_dir):
         pdf.multi_cell(0, 8, f"Category: {clean_text(art.get('category')).upper()}", new_x="LMARGIN", new_y="NEXT")
         
         pdf.set_font("helvetica", "", 12)
-        pdf.multi_cell(0, 8, f"Why: {clean_text(art.get('reasoning'))}", new_x="LMARGIN", new_y="NEXT")
+        pdf.multi_cell(0, 8, f"Why: {clean_text(art.get('political_significance'))}", new_x="LMARGIN", new_y="NEXT")
+        pdf.multi_cell(0, 8, f"Attack Angle: {clean_text(art.get('attack_angle'))}", new_x="LMARGIN", new_y="NEXT")
         pdf.ln(5)
         
         # Image
-        img_name = art.get("image_file")
+        art_id = art.get("id")
+        img_name = f"{art_id}.png" if art_id else None
         if img_name:
             img_path = images_dir / img_name
             if img_path.exists():

geometric_grouper.py

@@ -1,23 +1,103 @@
 import math
+import cv2
+import numpy as np
 
 def get_horizontal_overlap(bbox1, bbox2):
     x_left = max(bbox1[0], bbox2[0])
     x_right = min(bbox1[2], bbox2[2])
     return max(0, x_right - x_left)
 
-def group_surya_regions_geometrically(regions):
+def extract_barriers(img_path):
+    """Extracts horizontal and vertical barrier lines using OpenCV."""
+    if not img_path:
+        return [], []
+        
+    img = cv2.imread(str(img_path))
+    if img is None:
+        return [], []
+        
+    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    _, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)
+    
+    line_length = 300
+    h_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (line_length, 1))
+    h_lines_raw = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, h_kernel, iterations=2)
+    
+    v_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, line_length))
+    v_lines_raw = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, v_kernel, iterations=2)
+    
+    h_contours_raw, _ = cv2.findContours(h_lines_raw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    v_contours_raw, _ = cv2.findContours(v_lines_raw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    
+    h_lines = []
+    for cnt in h_contours_raw:
+        x, y, w, h = cv2.boundingRect(cnt)
+        if h <= 15:  # Thin lines only
+            h_lines.append((x, y, x + w, y + h))
+            
+    v_lines = []
+    for cnt in v_contours_raw:
+        x, y, w, h = cv2.boundingRect(cnt)
+        if w <= 15:  # Thin lines only
+            v_lines.append((x, y, x + w, y + h))
+            
+    return h_lines, v_lines
+
+def is_blocked_by_horizontal_barrier(top_y, bottom_y, item_left, item_right, h_lines):
     """
-    Groups regions spatially.
+    Checks if a horizontal line sits between top_y and bottom_y and spans across the item.
-    Instead of a linear reading order, this associates text and images 
-    with the headline that is directly above them in the same column.
     """
+    item_width = item_right - item_left
+    tolerance = item_width * 0.2
+    
+    for (lx1, ly1, lx2, ly2) in h_lines:
+        line_y = ly1
+        if top_y < line_y < bottom_y:
+            # Does it span across the item?
+            if lx1 <= (item_left + tolerance) and lx2 >= (item_right - tolerance):
+                return True
+    return False
+
+def is_blocked_by_vertical_barrier(t_bbox, item_bbox, v_lines):
+    """
+    Checks if a vertical line sits between the centers of two regions.
+    """
+    t_cx = (t_bbox[0] + t_bbox[2]) / 2
+    item_cx = (item_bbox[0] + item_bbox[2]) / 2
+    
+    left_x = min(t_cx, item_cx)
+    right_x = max(t_cx, item_cx)
+    
+    # Only check if they are in different columns
+    if right_x - left_x < 100:
+        return False
+        
+    y_overlap_top = max(t_bbox[1], item_bbox[1])
+    y_overlap_bottom = min(t_bbox[3], item_bbox[3])
+    
+    for (vx1, vy1, vx2, vy2) in v_lines:
+        line_x = vx1
+        if left_x < line_x < right_x:
+            # Does the line overlap them vertically?
+            # A true vertical column separator will be very tall
+            if vy1 < y_overlap_bottom and vy2 > y_overlap_top:
+                return True
+    return False
+
+def group_surya_regions_geometrically(regions, img_path=None):
     if not regions:
         return []
         
+    # Extract barriers using OpenCV
+    h_lines, v_lines = extract_barriers(img_path)
+    if img_path:
+        print(f"  [grouper] Extracted {len(h_lines)} horizontal and {len(v_lines)} vertical OpenCV barriers")
+
     titles = [r for r in regions if r['type'] in ('doc_title', 'paragraph_title')]
     non_titles = [r for r in regions if r['type'] not in ('doc_title', 'paragraph_title')]
+
+    region_by_id = {r['id']: r for r in regions}
     
-    # If no titles exist, return everything as one article
     if not titles:
         member_ids = [r['id'] for r in regions]
         min_x = min(r['bbox'][0] for r in regions)
@@ -42,69 +122,86 @@ def group_surya_regions_geometrically(regions):
         
         best_title = None
         
-        # -------------------------------------------------------------
+        max_dist_above = 2500 if item['type'] == 'image' else 1500
-        # Rule 1: Find titles ABOVE the item that share the same column
-        # -------------------------------------------------------------
         candidates_above = []
         for t in titles:
             t_bbox = t['bbox']
             if t_bbox[1] <= item_cy:
+                # Barrier Check
+                if is_blocked_by_horizontal_barrier(t_bbox[3], item_bbox[1], item_bbox[0], item_bbox[2], h_lines):
+                    continue
+                if is_blocked_by_vertical_barrier(t_bbox, item_bbox, v_lines):
+                    continue
+                    
                 overlap = get_horizontal_overlap(item_bbox, t_bbox)
                 t_width = t_bbox[2] - t_bbox[0]
                 t_cx = (t_bbox[0] + t_bbox[2]) / 2
                 
                 if overlap > (item_width * 0.1) or overlap > (t_width * 0.1) or abs(item_cx - t_cx) < 150:
                     distance = item_bbox[1] - t_bbox[3]
-                    if distance < 1500:  # MAX DISTANCE ABOVE
+                    if distance < max_dist_above:
                         candidates_above.append(t)
                     
         if candidates_above:
-            best_title = min(candidates_above, key=lambda t: abs(item_bbox[1] - t['bbox'][3]))
+            def _above_score(t):
+                vert  = item_bbox[1] - t['bbox'][3]
+                horiz = abs(item_cx - (t['bbox'][0] + t['bbox'][2]) / 2)
+                return vert + horiz * 0.8
+            best_title = min(candidates_above, key=_above_score)
         else:
-            # -------------------------------------------------------------
-            # Rule 2: If no title is above, find titles BELOW the item
-            # -------------------------------------------------------------
             candidates_below = []
             for t in titles:
                 t_bbox = t['bbox']
                 if t_bbox[1] > item_cy:
+                    # Barrier Check
+                    if is_blocked_by_horizontal_barrier(item_bbox[3], t_bbox[1], item_bbox[0], item_bbox[2], h_lines):
+                        continue
+                    if is_blocked_by_vertical_barrier(t_bbox, item_bbox, v_lines):
+                        continue
+
                     overlap = get_horizontal_overlap(item_bbox, t_bbox)
                     t_width = t_bbox[2] - t_bbox[0]
                     t_cx = (t_bbox[0] + t_bbox[2]) / 2
                     
                     if overlap > (item_width * 0.1) or overlap > (t_width * 0.1) or abs(item_cx - t_cx) < 150:
                         distance = t_bbox[1] - item_bbox[3]
-                        if distance < 800:  # MAX DISTANCE BELOW (stricter because text usually flows down, not up)
+                        if distance < 800:
                             candidates_below.append(t)
             
             if candidates_below:
-                best_title = min(candidates_below, key=lambda t: abs(t['bbox'][1] - item_bbox[3]))
+                def _below_score(t):
+                    vert  = t['bbox'][1] - item_bbox[3]
+                    horiz = abs(item_cx - (t['bbox'][0] + t['bbox'][2]) / 2)
+                    return vert + horiz * 0.8
+                best_title = min(candidates_below, key=_below_score)
             else:
-                # -------------------------------------------------------------
+                # Absolute fallback
-                # Rule 3: Absolute fallback. No column overlap at all.
+                valid_closest = []
-                # Find the absolute closest title via Euclidean distance.
+                for t in titles:
-                # -------------------------------------------------------------
+                    if is_blocked_by_horizontal_barrier(min(item_bbox[3], t['bbox'][3]), max(item_bbox[1], t['bbox'][1]), item_bbox[0], item_bbox[2], h_lines):
-                closest_t = min(titles, key=lambda t: math.hypot(
+                        continue
-                    item_cx - ((t['bbox'][0] + t['bbox'][2]) / 2),
+                    if is_blocked_by_vertical_barrier(t['bbox'], item_bbox, v_lines):
-                    item_cy - ((t['bbox'][1] + t['bbox'][3]) / 2)
+                        continue
-                ))
+                    valid_closest.append(t)
-                # Only attach if it's reasonably close, else leave as orphan
+                
-                dist = math.hypot(item_cx - ((closest_t['bbox'][0] + closest_t['bbox'][2]) / 2),
+                if valid_closest:
-                                  item_cy - ((closest_t['bbox'][1] + closest_t['bbox'][3]) / 2))
+                    closest_t = min(valid_closest, key=lambda t: math.hypot(
-                if dist < 1500:
+                        item_cx - ((t['bbox'][0] + t['bbox'][2]) / 2),
-                    best_title = closest_t
+                        item_cy - ((t['bbox'][1] + t['bbox'][3]) / 2)
+                    ))
+                    dist = math.hypot(item_cx - ((closest_t['bbox'][0] + closest_t['bbox'][2]) / 2),
+                                      item_cy - ((closest_t['bbox'][1] + closest_t['bbox'][3]) / 2))
+                    if dist < 1500:
+                        best_title = closest_t
                 
         if best_title:
             article_groups[best_title['id']].append(item)
         else:
-            # Treat as an orphan (create a new article group for it, or group orphans together)
-            # For simplicity, we'll assign it a virtual title ID based on its own ID so it becomes a standalone article
             virtual_id = f"orphan_{item['id']}"
             article_groups[virtual_id] = [item]
             
     # Format output
     articles = []
-    # Sort groups top-to-bottom, left-to-right based on title position
     sorted_titles = sorted(titles, key=lambda t: (t['bbox'][1], t['bbox'][0]))
     
     for idx, (title_id, group_regions) in enumerate(article_groups.items(), start=1):
@@ -116,7 +213,6 @@ def group_surya_regions_geometrically(regions):
         max_x = max(r['bbox'][2] for r in group_regions)
         max_y = max(r['bbox'][3] for r in group_regions)
         
-        # Determine if it's an orphan group
         is_orphan = str(title_id).startswith("orphan_")
         
         articles.append({
@@ -127,12 +223,9 @@ def group_surya_regions_geometrically(regions):
             "grouped_by": "geometric_orphan" if is_orphan else "geometric"
         })
         
-    # We should merge orphans that are vertically adjacent into the same orphan group to prevent shattering
-    # Simple vertical merge for orphans
     orphan_articles = [a for a in articles if a['grouped_by'] == 'geometric_orphan']
     valid_articles = [a for a in articles if a['grouped_by'] == 'geometric']
     
-    # Sort orphans top-to-bottom
     orphan_articles.sort(key=lambda a: a['bbox'][1])
     
     merged_orphans = []
@@ -143,28 +236,101 @@ def group_surya_regions_geometrically(regions):
             current_orphan = o
             continue
             
-        # Check if they overlap horizontally and are close vertically
         overlap = get_horizontal_overlap(current_orphan['bbox'], o['bbox'])
         vertical_dist = o['bbox'][1] - current_orphan['bbox'][3]
         
         if overlap > 0 and vertical_dist < 400:
-            # Merge
+            # Check barrier before merging orphans
-            current_orphan['member_regions'].extend(o['member_regions'])
+            if not is_blocked_by_horizontal_barrier(current_orphan['bbox'][3], o['bbox'][1], current_orphan['bbox'][0], current_orphan['bbox'][2], h_lines):
-            current_orphan['bbox'][0] = min(current_orphan['bbox'][0], o['bbox'][0])
+                current_orphan['member_regions'].extend(o['member_regions'])
-            current_orphan['bbox'][1] = min(current_orphan['bbox'][1], o['bbox'][1])
+                current_orphan['bbox'][0] = min(current_orphan['bbox'][0], o['bbox'][0])
-            current_orphan['bbox'][2] = max(current_orphan['bbox'][2], o['bbox'][2])
+                current_orphan['bbox'][1] = min(current_orphan['bbox'][1], o['bbox'][1])
-            current_orphan['bbox'][3] = max(current_orphan['bbox'][3], o['bbox'][3])
+                current_orphan['bbox'][2] = max(current_orphan['bbox'][2], o['bbox'][2])
-        else:
+                current_orphan['bbox'][3] = max(current_orphan['bbox'][3], o['bbox'][3])
-            merged_orphans.append(current_orphan)
+                continue
-            current_orphan = o
+                
+        merged_orphans.append(current_orphan)
+        current_orphan = o
             
     if current_orphan:
         merged_orphans.append(current_orphan)
         
     final_articles = valid_articles + merged_orphans
-    
+
-    # Reassign IDs
+    MAX_SPLIT_HEIGHT  = 2500
-    for idx, a in enumerate(final_articles, start=1):
+    MIN_MEMBERS_SPLIT = 5
+    MIN_SPLIT_GAP     = 30
+    MIN_LOWER_HEIGHT  = 400
+
+    split_result = []
+    for art in final_articles:
+        _, art_t, _, art_b = art['bbox']
+        if (art_b - art_t) <= MAX_SPLIT_HEIGHT or len(art['member_regions']) < MIN_MEMBERS_SPLIT:
+            split_result.append(art)
+            continue
+
+        members = [region_by_id[rid] for rid in art['member_regions'] if rid in region_by_id]
+        members.sort(key=lambda reg: reg['bbox'][1])
+
+        best_gap  = MIN_SPLIT_GAP
+        split_idx = -1
+        max_bottom_so_far = 0
+        for i in range(len(members) - 1):
+            cur_bottom = members[i]['bbox'][3]
+            max_bottom_so_far = max(max_bottom_so_far, cur_bottom)
+            
+            next_top = members[i + 1]['bbox'][1]
+            gap = next_top - max_bottom_so_far
+            
+            # If there is a barrier line crossing this gap, artificially inflate the gap score to force a split here
+            is_barrier = is_blocked_by_horizontal_barrier(max_bottom_so_far, next_top, members[i]['bbox'][0], members[i]['bbox'][2], h_lines)
+            effective_gap = gap + 10000 if is_barrier else gap
+            
+            if effective_gap > best_gap:
+                lower_group = members[i + 1:]
+                if not any(reg['type'] == 'image' for reg in lower_group):
+                    best_gap  = effective_gap
+                    split_idx = i
+
+        if split_idx < 0:
+            split_result.append(art)
+            continue
+
+        upper = members[:split_idx + 1]
+        lower = members[split_idx + 1:]
+
+        lower_bottom = max(reg['bbox'][3] for reg in lower)
+        if lower_bottom - lower[0]['bbox'][1] < MIN_LOWER_HEIGHT:
+            split_result.append(art)
+            continue
+
+        def _bbox(regs):
+            return [min(reg['bbox'][0] for reg in regs), min(reg['bbox'][1] for reg in regs),
+                    max(reg['bbox'][2] for reg in regs), max(reg['bbox'][3] for reg in regs)]
+
+        real_gap = members[split_idx+1]['bbox'][1] - max(r['bbox'][3] for r in upper)
+        print(f"  [split] Article {art['article_id']}: split at gap={real_gap}px after region "
+              f"{upper[-1]['id']} (upper {len(upper)} regions, lower {len(lower)} regions)")
+
+        upper_ids = [reg['id'] for reg in upper]
+        lower_ids = [reg['id'] for reg in lower]
+
+        split_result.append({
+            'article_id':     art['article_id'],
+            'title_region':   art['title_region'] if art['title_region'] in upper_ids else None,
+            'member_regions': upper_ids,
+            'bbox':           _bbox(upper),
+            'grouped_by':     art['grouped_by'],
+        })
+        split_result.append({
+            'article_id':     None,
+            'title_region':   art['title_region'] if art['title_region'] in lower_ids else None,
+            'member_regions': lower_ids,
+            'bbox':           _bbox(lower),
+            'grouped_by':     'geometric_split',
+        })
+
+    for idx, a in enumerate(split_result, start=1):
         a['article_id'] = idx
-        
+
-    return final_articles
+    return split_result

requirements.txt

@@ -6,3 +6,5 @@ paddleocr==3.5.0
 paddlepaddle==3.0.0
 pytesseract>=0.3.10
 anthropic>=0.40
+surya-ocr==0.4.15
+fpdf2>=2.7.0

scratch_missing.txt

@@ -0,0 +1,6 @@
+output/20260614_164012/pages\page_002.triage_log.json - p002_a003: దిగుమతులు లేక నిలిచిపోయిన
+ట్రక్టర్జు
+eo
+ూమాలీల కొరతతో ఇబ్దందులు పడుతున్న రెతులు
+25
+త్యర్జూరు, మే 16 

scratch_test_grouper.py

@@ -0,0 +1,17 @@
+import json
+from pathlib import Path
+import geometric_grouper
+
+regions_file = Path("output/20260614_124012/pages/page_004.regions.json")
+img_file = Path("output/20260614_124012/pages/page_004.png")
+
+with open(regions_file, encoding='utf-8') as f:
+    regions = json.load(f)["regions"]
+
+valid_regions = [r for r in regions if r["type"] not in ("header", "footer", "page_number")]
+
+articles = geometric_grouper.group_surya_regions_geometrically(valid_regions, img_path=str(img_file))
+
+print(f"Total articles identified: {len(articles)}")
+for a in articles:
+    print(f"  Article {a['article_id']}: {len(a['member_regions'])} regions, bbox {a['bbox']}, grouped_by={a['grouped_by']}")

scratch_test_lines.py

@@ -0,0 +1,80 @@
+import cv2
+import numpy as np
+import sys
+from pathlib import Path
+
+def test_lines(img_path, out_path):
+    img = cv2.imread(img_path)
+    if img is None:
+        print("Could not read image")
+        return
+        
+    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    
+    # 1. Use standard binary thresholding instead of adaptive to reduce noise from text
+    # Newspapers usually have very black lines on white background
+    _, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)
+    
+    # We want lines that are at least 300px long (a typical column width)
+    line_length = 300
+    
+    # 2. Detect horizontal lines (MUST be thin: e.g., height <= 5px)
+    horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (line_length, 1))
+    horizontal_lines_raw = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, horizontal_kernel, iterations=2)
+    
+    # 3. Detect vertical lines (MUST be thin: e.g., width <= 5px)
+    vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, line_length))
+    vertical_lines_raw = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, vertical_kernel, iterations=2)
+    
+    # 4. Filter contours to ensure they are actually THIN lines, not thick blocks/images
+    h_contours_raw, _ = cv2.findContours(horizontal_lines_raw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    v_contours_raw, _ = cv2.findContours(vertical_lines_raw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    
+    h_contours = []
+    for cnt in h_contours_raw:
+        x, y, w, h = cv2.boundingRect(cnt)
+        if h <= 15:  # Line must not be thicker than 15px
+            h_contours.append(cnt)
+            
+    v_contours = []
+    for cnt in v_contours_raw:
+        x, y, w, h = cv2.boundingRect(cnt)
+        if w <= 15:  # Line must not be thicker than 15px
+            v_contours.append(cnt)
+            
+    print(f"Horizontal separators found: {len(h_contours)} (filtered from {len(h_contours_raw)})")
+    print(f"Vertical separators found: {len(v_contours)} (filtered from {len(v_contours_raw)})")
+    
+    # 5. Create clean masks from only the filtered thin lines
+    clean_h_mask = np.zeros_like(thresh)
+    cv2.drawContours(clean_h_mask, h_contours, -1, 255, thickness=cv2.FILLED)
+    
+    clean_v_mask = np.zeros_like(thresh)
+    cv2.drawContours(clean_v_mask, v_contours, -1, 255, thickness=cv2.FILLED)
+    
+    grid = cv2.add(clean_h_mask, clean_v_mask)
+    rect_contours, _ = cv2.findContours(grid, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
+    
+    box_count = 0
+    debug_img = img.copy()
+    
+    for cnt in rect_contours:
+        epsilon = 0.02 * cv2.arcLength(cnt, True)
+        approx = cv2.approxPolyDP(cnt, epsilon, True)
+        
+        if len(approx) == 4:
+            x, y, w, h = cv2.boundingRect(approx)
+            if w > 300 and h > 200:
+                box_count += 1
+                cv2.rectangle(debug_img, (x, y), (x+w, y+h), (255, 0, 255), 8)
+                
+    print(f"Rectangular boxes found: {box_count}")
+    
+    cv2.drawContours(debug_img, h_contours, -1, (0, 0, 255), 3) # Red for horizontal
+    cv2.drawContours(debug_img, v_contours, -1, (255, 0, 0), 3) # Blue for vertical
+    
+    cv2.imwrite(out_path, debug_img)
+    print(f"Saved debug image to {out_path}")
+
+if __name__ == "__main__":
+    test_lines(sys.argv[1], sys.argv[2])