deepkoluguri
/
sundeep-news-scan
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
sundeep-news-scan/smart_extractor.py

6658 lines
341 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""
Smart Extractor — One Claude call per page for political article detection + cropping.
PaddleOCR finds precise regions → Claude identifies ONLY political region groups → crop.
Usage:
python smart_extractor.py newspaper.pdf
python smart_extractor.py newspaper.pdf --output ./my_output
"""
import os
import sys
import json
import base64
import argparse
from pathlib import Path
from io import BytesIO
from datetime import datetime
import fitz # PyMuPDF
from PIL import Image
from difflib import SequenceMatcher
# ---------------------------------------------------------------------------
# Step 1: PDF to page images
# ---------------------------------------------------------------------------
def render_pdf_pages(pdf_path, pages_dir, target_w=4200, target_h=7400):
doc = fitz.open(pdf_path)
pages = []
for i, page in enumerate(doc, 1):
scale_x = target_w / page.rect.width
scale_y = target_h / page.rect.height
scale = min(scale_x, scale_y)
mat = fitz.Matrix(scale, scale)
pix = page.get_pixmap(matrix=mat)
out_path = str(pages_dir / f"page_{i:03d}.png")
pix.save(out_path)
pages.append({"page": i, "path": out_path, "width": pix.width, "height": pix.height})
print(f" Page {i}: {pix.width}x{pix.height} px")
doc.close()
return pages
# ---------------------------------------------------------------------------
# Step 2: PaddleOCR layout detection
# ---------------------------------------------------------------------------
def detect_regions(page_png_path, threshold=None):
import paddleocr
import numpy as np
# Load model (cached after first call). Default (threshold=None) uses
# PP-DocLayout's stock ~0.5 cutoff — what Claude grouping is tuned to see.
# A lower threshold can be passed for experimentation (e.g. via the
# _replay_regions.py harness), but is NOT used in the production pipeline:
# lowering it floods grouping with low-confidence + duplicate boxes.
if threshold is None:
layout = paddleocr.LayoutDetection()
else:
layout = paddleocr.LayoutDetection(threshold=threshold)
raw = layout.predict(page_png_path)
regions = []
if isinstance(raw, list) and len(raw) > 0:
for det_result in raw:
boxes = None
if hasattr(det_result, 'get'):
boxes = det_result.get('boxes', None)
elif hasattr(det_result, 'boxes'):
boxes = det_result.boxes
if boxes is not None:
for box in boxes:
rid = len(regions) + 1
coord = None
if isinstance(box, dict):
coord = box.get('coordinate', box.get('bbox', None))
label = box.get('label', box.get('type', 'unknown')).lower()
score = float(box.get('score', 0))
elif hasattr(box, 'coordinate'):
coord = box.coordinate
label = getattr(box, 'label', 'unknown').lower()
score = float(getattr(box, 'score', 0))
else:
label, score = 'unknown', 0.0
bbox = [0, 0, 0, 0]
if coord is not None:
if hasattr(coord, 'tolist'):
coord = coord.tolist()
coord = [float(v) for v in coord]
if len(coord) == 4:
bbox = [int(v) for v in coord]
regions.append({"id": rid, "type": label, "bbox": bbox, "confidence": score})
print(f" {len(regions)} layout regions detected")
return regions
def log_region_inventory(regions, page_num, pre_counts=None):
"""Diagnostic log: a clearly-sectioned, column-aligned inventory of the page —
counts per region type, then a table of every doc_title and paragraph_title
(id, the four bbox numbers, width, height, confidence). Region boxes carry no
text, so this is the structural inventory only."""
from collections import Counter
bar = "=" * 64
counts = Counter(r.get("type", "unknown") for r in regions)
print()
print(" " + bar)
print(f" PAGE {page_num} — REGION INVENTORY ({len(regions)} regions total)")
print(" " + bar)
# --- counts by type, dotted leaders, right-aligned numbers ---
if pre_counts is not None:
print(" Counts by type (before promotion -> after promotion)")
all_types = sorted(set(counts) | set(pre_counts),
key=lambda t: (-counts.get(t, 0), t))
for t in all_types:
before, after = pre_counts.get(t, 0), counts.get(t, 0)
dots = "." * max(2, 20 - len(t))
mark = " <== changed by promotion" if before != after else ""
print(f" {t} {dots} {before:>3d} -> {after:>3d}{mark}")
else:
print(" Counts by type")
for t, c in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])):
dots = "." * max(2, 20 - len(t))
print(f" {t} {dots} {c:>3d}")
# --- aligned table for headline-type regions ---
def table(label):
items = sorted([r for r in regions if r.get("type") == label],
key=lambda z: (z["bbox"][1], z["bbox"][0]))
print()
print(f" {label.upper()} ({len(items)})")
print(f" {'ID':>3} {'X1':>5} {'Y1':>5} {'X2':>5} {'Y2':>5} | {'W':>5} {'H':>4} {'CONF':>4} FLAG")
print(" " + "-" * 58)
for r in items:
b = r["bbox"]
w, h = b[2] - b[0], b[3] - b[1]
# a full-width paragraph_title is very likely a real headline mis-tagged
flag = "<- wide, likely a headline" if (label == "paragraph_title" and w >= 1000) else ""
print(f" {r['id']:>3} {b[0]:>5} {b[1]:>5} {b[2]:>5} {b[3]:>5} | "
f"{w:>5} {h:>4} {r.get('confidence', 0):>4.2f} {flag}")
table("doc_title")
table("paragraph_title")
print(" " + bar)
print()
# ---------------------------------------------------------------------------
# Headline promotion: fix mis-tagged headlines (paragraph_title -> doc_title)
# ---------------------------------------------------------------------------
def _dateline_in_text(text, paper):
"""Return the dateline string (incl. its colon) if the first line opens with one,
else None. Sakshi = '<town>:' (no date). Andhra Jyothi = '<town>, <date> (...):'.
A dateline often WRAPS across several lines in narrow columns — e.g.
'హుస్నా బాద్,' / 'జూన్ 1 (ఆంధ్ర' / 'జ్యోతి): …'. Accumulate the first few lines
until the closing colon appears, capped at 3 lines so a colon deep in body text
is never pulled in (the prefix length/word guards below reject false matches)."""
lines = [l.strip() for l in (text or "").splitlines() if l.strip()]
if not lines:
return None
# Andhra Jyothi's closing colon is sometimes OCR'd as '=' (e.g. '(ఆంధ్రజ్యోతి)='),
# so accept '=' as a delimiter for that paper only — other papers stay strict.
# Tesseract also routinely misreads the dateline colon as a SEMICOLON ('మే 19 ;'), so ';'
# is accepted everywhere (the digit + short-prefix + no-sentence-punctuation guards below
# keep a mid-body semicolon from matching). This recovers e.g. JG R18 'చిల్పూరు, మే 19 ;'.
delims = ":：=;" if paper == "andhra_jyothi" else ":：;"
line = lines[0]
for nxt in lines[1:3]: # join up to 3 leading lines (narrow cols)
if any(c in line for c in delims):
break
line = line + " " + nxt
pos = next((i for i, c in enumerate(line) if c in delims), -1)
if pos <= 0:
return None
prefix = line[:pos].strip()
# Multi-town datelines join several towns with '/' ('గజ్వేల్ రూరల్/మర్కూక్, …'),
# which pushes the prefix past the normal 45-char cap — allow more length when a
# slash is present (the digit/masthead guard below still gates false matches).
max_len = 70 if "/" in prefix else 45
if not (0 < len(prefix) <= max_len) or len(prefix.split()) > 8:
return None
# A colon matched mid-sentence (prefix carrying sentence punctuation) is not a
# dateline — '!?।' always disqualify. The separator comma after the town is often
# OCR'd as a '.', so for ANDHRA JYOTHI ONLY a single '.' OR ',' is tolerated — but
# not both (both together signals a real sentence). Other papers keep the strict
# rule: any '.' disqualifies.
if any(p in prefix for p in "!?।"):
return None
if paper == "andhra_jyothi":
if "." in prefix and "," in prefix:
return None
elif "." in prefix:
return None
if paper == "andhra_jyothi":
# Andhra Jyothi datelines carry a date (digits) and/or the masthead.
if not (any(ch.isdigit() for ch in prefix) or "ఆంధ్రజ్యోతి" in line[:pos + 2]):
return None
elif paper == "namaste_telangana":
# Namaste Telangana: '<town>, <date e.g. మే 19>:' — a date (digits), no masthead.
if not any(ch.isdigit() for ch in prefix):
return None
return line[:pos + 1]
def _bbox_overlap_ratio(a, b):
ix = max(0, min(a[2], b[2]) - max(a[0], b[0]))
iy = max(0, min(a[3], b[3]) - max(a[1], b[1]))
inter = ix * iy
if inter == 0:
return 0.0
area_a = (a[2] - a[0]) * (a[3] - a[1])
area_b = (b[2] - b[0]) * (b[3] - b[1])
return inter / max(1, min(area_a, area_b))
def _bbox_iou(a, b):
ix = max(0, min(a[2], b[2]) - max(a[0], b[0]))
iy = max(0, min(a[3], b[3]) - max(a[1], b[1]))
inter = ix * iy
if inter == 0:
return 0.0
area_a = (a[2] - a[0]) * (a[3] - a[1])
area_b = (b[2] - b[0]) * (b[3] - b[1])
return inter / max(1, area_a + area_b - inter)
def dedup_overlapping_regions(regions, thresh=0.85):
"""Drop near-duplicate detections (PaddleOCR sometimes emits the SAME element
twice at the identical box, e.g. one heading tagged both doc_title and
paragraph_title). Uses IoU so only NEARLY-IDENTICAL boxes are merged — a small
region that merely sits inside a bigger one (text over a photo, a caption) is
NOT dropped. Keep the better one: doc_title > paragraph_title, then higher conf."""
rank = {"doc_title": 2, "paragraph_title": 1}
drop = set()
for i in range(len(regions)):
if regions[i]["id"] in drop:
continue
for j in range(i + 1, len(regions)):
if regions[j]["id"] in drop:
continue
if _bbox_iou(regions[i]["bbox"], regions[j]["bbox"]) > thresh:
a, b = regions[i], regions[j]
ka = (rank.get(a.get("type"), 0), a.get("confidence", 0))
kb = (rank.get(b.get("type"), 0), b.get("confidence", 0))
loser, keep = (b, a) if ka >= kb else (a, b)
drop.add(loser["id"])
print(f" · dedup: dropped region {loser['id']} ({loser.get('type')}, "
f"conf={loser.get('confidence',0):.2f}) — near-duplicate of region "
f"{keep['id']} ({keep.get('type')}, conf={keep.get('confidence',0):.2f})")
return [r for r in regions if r["id"] not in drop]
def _box_ink_color(page_img, bbox, cache):
"""Return (ink_density, mean_dark_color) for a box — used to compare font
weight (how much ink) and text color between two stacked title lines."""
key = tuple(bbox)
if key in cache:
return cache[key]
c = page_img.crop(key).convert("RGB")
if c.width == 0 or c.height == 0:
cache[key] = (0.0, (0, 0, 0)); return cache[key]
s = c.resize((min(200, c.width), min(60, c.height)))
px = s.load()
dark = []
for y in range(s.height):
for x in range(s.width):
r, g, b = px[x, y][:3]
if 0.299 * r + 0.587 * g + 0.114 * b < 140: # dark = ink
dark.append((r, g, b))
density = len(dark) / max(1, s.width * s.height)
if dark:
n = len(dark)
col = (sum(p[0] for p in dark) / n, sum(p[1] for p in dark) / n, sum(p[2] for p in dark) / n)
else:
col = (0, 0, 0)
cache[key] = (density, col)
return cache[key]
def merge_stacked_title_lines(regions, page_img_path=None, max_gap=90, x_frac=0.5,
h_ratio=1.8, color_max=60, density_max=0.10):
"""Merge consecutive title-type boxes that are really the lines of ONE headline
(PaddleOCR boxes each wrapped line of a headline separately). Conservative:
both title-type, same column (x-overlap), small vertical gap, similar height,
lower line not wider, NOTHING between them, AND (if the page image is given)
the two lines share font weight (ink density) and text color."""
page_img = None
if page_img_path:
try:
page_img = Image.open(page_img_path).convert("RGB")
except Exception:
page_img = None
feat_cache = {}
def is_title(r):
return r.get("type") in ("doc_title", "paragraph_title")
def x_overlap(a, b):
ov = max(0, min(a[2], b[2]) - max(a[0], b[0]))
return ov / max(1, min(a[2] - a[0], b[2] - b[0]))
def something_between(a, b):
x1, x2 = max(a[0], b[0]), min(a[2], b[2])
for r in regions:
if is_title(r):
continue
rb = r["bbox"]
cy = (rb[1] + rb[3]) / 2
if a[3] < cy < b[1] and not (rb[2] <= x1 or rb[0] >= x2):
return True
return False
regions = list(regions)
changed = True
while changed:
changed = False
titles = sorted([r for r in regions if is_title(r)], key=lambda z: z["bbox"][1])
for A in titles:
best, best_gap = None, 1e9
for B in titles:
if B is A:
continue
a, b = A["bbox"], B["bbox"]
if b[1] < a[3] - 5: # B must sit below A
continue
gap = b[1] - a[3]
if gap > max_gap or x_overlap(a, b) < x_frac:
continue
ha, hb = a[3] - a[1], b[3] - b[1]
if max(ha, hb) / max(1, min(ha, hb)) > h_ratio:
continue
# Wrapped headline lines: the lower line is the same width or
# SHORTER than the upper (line 1 fills the column, line 2 is
# shorter). If the lower box is clearly WIDER, the upper one is a
# section label / kicker sitting above a wider headline — do NOT merge.
wa, wb = a[2] - a[0], b[2] - b[0]
if wb > wa * 1.15:
continue
if something_between(a, b):
continue
# Style match: same font weight (ink density) and same text color.
if page_img is not None:
da, ca = _box_ink_color(page_img, a, feat_cache)
db, cb = _box_ink_color(page_img, b, feat_cache)
color_dist = sum((p - q) ** 2 for p, q in zip(ca, cb)) ** 0.5
if color_dist > color_max or abs(da - db) > density_max:
continue
if gap < best_gap:
best, best_gap = B, gap
if best is not None:
a, b = A["bbox"], best["bbox"]
A["bbox"] = [min(a[0], b[0]), min(a[1], b[1]), max(a[2], b[2]), max(a[3], b[3])]
if best.get("type") == "doc_title":
A["type"] = "doc_title"
A["confidence"] = max(A.get("confidence", 0), best.get("confidence", 0))
A.setdefault("_merged_lines", [A["id"]]).append(best["id"])
print(f" ⊕ merged stacked headline line: region {best['id']} into region "
f"{A['id']} (gap={best_gap:.0f}px) → bbox {A['bbox']}")
regions = [r for r in regions if r is not best]
changed = True
break
return regions
def drop_masthead_regions(regions, paper, page_num, page_img_path):
"""Remove the front-page masthead (the fixed newspaper header) so it is never
mis-grouped as a news article.
Namaste Telangana opens page 1 with a banner holding the giant edition
nameplate (e.g. 'జనగామ'). PaddleOCR detects it three different ways, all
handled here:
• an explicit 'header'-typed region,
• an 'image' region with the nameplate as a doc_title sitting INSIDE it, or
• a bare nameplate doc_title at the very top (banner not detected at all).
Either way Claude sometimes seeds an 'article' from the nameplate (which then
over-grows into the real story beside it). A genuine page-1 headline never sits
in the top ~10% strip (real stories start well below the masthead), never lives
inside an image, and a 'header' is never a story — so we drop the masthead
banner, the nameplate, and the date-line beneath it.
Gated to page 1 (mastheads are front-page) so inner-page content is untouched.
"""
if paper == "andhra_jyothi":
# Andhra Jyothi section fronts open with a nameplate band — the 'ఆంధ్రజ్యోతి'
# logo + the coloured 'సిద్దిపేట / date' banner — as image/header regions
# pinned to the very TOP strip of the page. Real article photos start well
# below it, so any image/header whose TOP sits in the top ~5% is masthead and
# must never be folded into a story. Drop them on every page (inner pages
# simply have nothing up there).
try:
page_h = Image.open(page_img_path).height
except Exception:
page_h = 6833
strip = page_h * 0.05
drop_ids = {r["id"] for r in regions
if r["bbox"][1] < strip and r.get("type") in ("image", "header")}
if drop_ids:
print(f" 🗞️ Masthead: dropped {len(drop_ids)} Andhra Jyothi nameplate "
f"region(s) {sorted(drop_ids)} (top-strip banner)")
return [r for r in regions if r["id"] not in drop_ids]
if paper == "sakshi":
# Sakshi opens every page with a furniture band ABOVE the first headline:
# page 1 = nameplate + weather box + photo-brief teasers (with '-8లో' jump
# pointers) + the full-width date strip; inner pages = the section header.
# With no Sakshi masthead rule this band folds into the top articles (the
# weather box into the lead story) and the date strip even SEEDS a fake
# dateline article ('బుధవారం 20 మే 2026' OCRs as a dateline). Everything
# that ends above the topmost headline-type region is that band → drop.
tops = [r["bbox"][1] for r in regions
if r.get("type") in ("doc_title", "paragraph_title")]
try:
page_h = Image.open(page_img_path).height
except Exception:
page_h = 7055
if tops:
cut = min(tops)
if cut < page_h * 0.30: # sanity: band must be page furniture
drop_ids = {r["id"] for r in regions if r["bbox"][3] <= cut}
if drop_ids:
print(f" 🗞️ Masthead: dropped {len(drop_ids)} Sakshi top-band "
f"region(s) {sorted(drop_ids)} (above first headline y={cut})")
return [r for r in regions if r["id"] not in drop_ids]
return regions
if paper != "namaste_telangana" or page_num != 1:
return regions
try:
page_h = Image.open(page_img_path).height
except Exception:
page_h = 7400
top_band = page_h * 0.18
nameplate_band = page_h * 0.10 # the very top strip the masthead occupies
def _inside(inner, outer, frac=0.8):
ix = max(0, min(inner[2], outer[2]) - max(inner[0], outer[0]))
iy = max(0, min(inner[3], outer[3]) - max(inner[1], outer[1]))
ia = (inner[2] - inner[0]) * (inner[3] - inner[1])
return ia > 0 and (ix * iy) / ia >= frac
def _hoverlap(a, b):
return max(0, min(a[2], b[2]) - max(a[0], b[0]))
# Banner regions: explicit header, or an image hosting the nameplate doc_title.
banner_regs = [
r for r in regions if r["bbox"][1] < top_band and (
r.get("type") == "header"
or (r.get("type") == "image"
and any(d.get("type") == "doc_title" and _inside(d["bbox"], r["bbox"])
for d in regions))
)
]
# Bare nameplate: a doc_title whose top sits in the top ~10% strip.
nameplate_titles = [
r for r in regions
if r.get("type") == "doc_title" and r["bbox"][1] < nameplate_band
]
if not banner_regs and not nameplate_titles:
return regions
drop_ids = set()
for im in banner_regs:
drop_ids.add(im["id"])
for r in regions:
if r["id"] != im["id"] and _inside(r["bbox"], im["bbox"]):
drop_ids.add(r["id"])
for nt in nameplate_titles:
drop_ids.add(nt["id"])
nb = nt["bbox"]
# Sweep the date-line text directly beneath the nameplate (same x-span).
for r in regions:
rb = r["bbox"]
if (r["id"] not in drop_ids and r.get("type") == "text"
and nb[3] <= rb[1] <= nb[3] + 250
and _hoverlap(rb, nb) > 0):
drop_ids.add(r["id"])
if drop_ids:
print(f" 🗞️ Masthead: dropped {len(drop_ids)} header region(s) "
f"{sorted(drop_ids)} (Namaste Telangana page-1 nameplate)")
return [r for r in regions if r["id"] not in drop_ids]
def drop_classifieds_regions(regions, page_img_path):
"""Drop CLASSIFIEDS blocks — the 'Classified Ads / phone-number listing' columns — which
are page FURNITURE, not news. Left in, a neighbouring article's column-fold grows down
into them (e.g. JG R65 swallowing the whole left column). A region is classifieds when
its OCR text carries a classifieds keyword OR ≥5 phone-like numbers (a news article
rarely has >3). Only TALL regions are OCR-scanned (a classifieds listing is a tall
column), which keeps the cost down and avoids touching ordinary text. Court/legal
notices are intentionally NOT dropped here."""
try:
import pytesseract
import re as _re
from PIL import Image as _Image
if "eng" not in set(pytesseract.get_languages(config="")):
return regions
except Exception:
return regions
try:
img = _Image.open(page_img_path).convert("RGB")
except Exception:
return regions
KW = ("classified", "classifieds", "contact", "క్లాసిఫైడ")
drop = set()
for r in regions:
b = r["bbox"]
if (b[3] - b[1]) < 400: # classifieds listings are tall columns
continue
try:
# Cheap OCR: phone-runs (\d{6,}) and the English keyword are the real signals,
# so a half-res ENG-only pass suffices — ~2× faster than full-res tel+eng and
# verified to give the IDENTICAL drop set on the JG/AJ/NT corpus.
c = img.crop(tuple(b))
c = c.resize((max(1, c.width // 2), max(1, c.height // 2)))
t = pytesseract.image_to_string(c, lang="eng")
except Exception:
continue
tl = t.lower()
phones = len(_re.findall(r"\d{6,}", t))
if any(k in tl for k in KW) or phones >= 5:
drop.add(r["id"])
if drop:
print(f" 🗞️ Classifieds: dropped {len(drop)} region(s) {sorted(drop)} "
f"(phone-listing / 'Classified Ads' furniture)")
return [r for r in regions if r["id"] not in drop]
def tag_legal_notices(regions, page_img_path):
"""Tag COURT / LEGAL-NOTICE regions (set r['_legal']='core'|'support'). A legal notice
(court summons, name-change / matrimonial petition, public 'by order of the court' notice)
is page FURNITURE like classifieds — but, per the brief, it is KEPT and cropped as its OWN
section rather than dropped or absorbed into a neighbouring article (e.g. JG R65 swallowed a
court summons stacked in its left column). These notices are set in English legal
boilerplate, so a half-res ENG-only OCR detects them reliably and cheaply. The actual
carve-into-its-own-box happens later inside `cluster_all_article_blocks` (which reads these
tags, no OCR), so every consumer — the crops AND the debug boundary map — sees the section."""
try:
import pytesseract
import statistics as _st
from PIL import Image as _Image
if "eng" not in set(pytesseract.get_languages(config="")):
return regions
except Exception:
return regions
try:
img = _Image.open(page_img_path).convert("RGB")
except Exception:
return regions
# CORE = unambiguous court/petition boilerplate; SUPPORT = corroborating (won't seed a
# cluster on its own — a carve requires ≥1 CORE hit — but is folded in when adjacent).
CORE = ("hon'ble", "honble", "honourable", "in the court of", "by order of", "plaintiff",
"defendant", "petitioner", "respondent", "summons", "decree", "civil judge",
"district judge", "munsif", "o.s.no", "o.s. no", "i.a.no", "i.a. no", "sub court",
"appeal suit", "notice is hereby")
SUPPORT = ("advocate", "vakalat", "vakalath", "whereas")
_tw = [r["bbox"][2] - r["bbox"][0] for r in regions if r.get("type") == "text"]
pitch = _st.median(_tw) if _tw else 600
for r in regions:
if r.get("type") not in ("text", "paragraph_title"):
continue
b = r["bbox"]; w = b[2] - b[0]; h = b[3] - b[1]
if h < 30 or w < 60 or w > 1.4 * pitch: # single-column notice lines only
continue
try:
c = img.crop(tuple(b))
c = c.resize((max(1, c.width // 2), max(1, c.height // 2)))
t = pytesseract.image_to_string(c, lang="eng").lower()
except Exception:
continue
if any(k in t for k in CORE):
r["_legal"] = "core"
elif any(k in t for k in SUPPORT):
r["_legal"] = "support"
return regions
def recover_orphan_text_headlines(blocks, regions, page_img_path):
"""Recover a headline PaddleOCR mis-tagged as `text` — but ONLY when it ended up as a
stray ORPHAN block. That orphan-ness is the real signature of a lost headline: a
photo CAPTION is also short + bold + above a photo, but it always FOLDS into its
article (never orphans), so scoping to orphans leaves captions untouched. For a
qualifying orphan (short + bold vs. body ink + sitting above a photo), re-tag its text
region as a doc_title and return the re-tagged ids — the caller re-clusters so the
recovered headline anchors its article."""
try:
page = Image.open(page_img_path).convert("RGB")
except Exception:
return []
cache = {}
rmap = {r["id"]: r for r in regions}
texts = [r for r in regions if r.get("type") == "text"]
if not texts:
return []
dens = sorted(_box_ink_color(page, r["bbox"], cache)[0] for r in texts)
med_dens = dens[len(dens) // 2] if dens else 0 # body-text ink baseline
tws = sorted((r["bbox"][2] - r["bbox"][0]) for r in texts)
pitch = tws[len(tws) // 2] if tws else 0
def _xov(a, b):
return max(0, min(a[2], b[2]) - max(a[0], b[0]))
retag = []
for blk in blocks:
if blk.get("kind") != "O":
continue # ONLY stray orphans
for m in blk.get("members", []):
r = rmap.get(m)
if not r or r.get("type") != "text":
continue
b = r["bbox"]
w, h = b[2] - b[0], b[3] - b[1]
if h > 190 or w < 0.45 * max(1, pitch): # short (≤~2 lines), column-ish wide
continue
if _box_ink_color(page, b, cache)[0] < 1.4 * max(0.01, med_dens):
continue # bold (≥1.4× body ink)
if not any(o.get("type") == "image" and o["bbox"][1] >= b[3] - 10
and o["bbox"][1] - b[3] <= 220 and _xov(b, o["bbox"]) >= 0.4 * w
for o in regions):
continue # must sit above a photo
r["type"] = "doc_title"
r["_headline_from_text"] = True
retag.append(m)
print(f" ⬆ ORPHAN HEADLINE: R{m} re-tagged text→doc_title "
f"(bold orphan sitting above a photo)")
return retag
def promote_paragraph_titles(regions, page_img_path, paper, width_min=1000,
col_frac=0.8, abs_floor=250):
"""Reclassify a paragraph_title to a doc_title ONLY when BOTH hold:
(1) it is full-column wide (>= width_min px), AND
(2) a fresh dateline body starts directly beneath it.
Logs each promotion with the dateline text found, for debugging."""
try:
import pytesseract
if "tel" not in set(pytesseract.get_languages(config="")):
print(" (Headline promotion skipped — Telugu Tesseract 'tel' not installed)")
return regions
except Exception:
print(" (Headline promotion skipped — pytesseract not available)")
return regions
page = Image.open(page_img_path)
def first_line(bbox):
l, t, r, b = bbox
strip_h = min(220, b - t) # tall enough for a 3-line wrap in narrow columns
if strip_h <= 0:
return ""
try:
return pytesseract.image_to_string(page.crop((l, t, r, t + strip_h)), lang="tel")
except Exception:
return ""
promoted = 0
for reg in regions:
if reg.get("type") != "paragraph_title":
continue
b = reg["bbox"]
w = b[2] - b[0]
if w < abs_floor: # ignore tiny inline labels outright
continue
# skip if it duplicates an existing doc_title (PaddleOCR double-detects the
# same heading as both doc_title and paragraph_title — promoting would make
# two doc_titles for one heading)
if any(_bbox_overlap_ratio(b, dt["bbox"]) > 0.6
for dt in regions if dt is not reg and dt.get("type") == "doc_title"):
print(f" · not promoted: region {reg['id']} (W={w}) — already covered by an existing doc_title")
continue
# Body blocks beneath the heading. Use horizontal OVERLAP, not centre
# containment: a wide headline often sits above a MULTI-COLUMN body split
# around a photo, so its centre can land on the photo with no text under it.
below = [x for x in regions if x.get("type") == "text"
and x["bbox"][1] >= b[3] - 10
and max(0, min(x["bbox"][2], b[2]) - max(x["bbox"][0], b[0])) >= 0.2 * w]
if not below:
print(f" · not promoted: region {reg['id']} (W={w}) — no body block below it")
continue
body = min(below, key=lambda x: x["bbox"][1] - b[3])
body_w = body["bbox"][2] - body["bbox"][0]
# condition 1 (column-relative): the heading spans its own column (≈ the
# width of the body beneath it), OR is full-page wide. This catches a real
# headline sitting in a narrow column (which an absolute px rule misses).
# For a multi-column body the heading is far wider than one column, so this
# holds comfortably.
spans_column = (w >= col_frac * body_w) or (w >= width_min)
if not spans_column:
print(f" · not promoted: region {reg['id']} (W={w}) — not full-column width "
f"(body region {body['id']} W={body_w}; needs ≥ {int(col_frac * body_w)})")
continue
# condition 2: a fresh dateline opens the body. With a multi-column body the
# dateline is in the first column, which may not be the centremost block, so
# scan the blocks of the immediate row beneath the heading (top-then-left).
gap_top = body["bbox"][1] - b[3]
# SAME-COLUMN only: the dateline that promotes this heading must sit in a body
# that lies under the heading's OWN column — the heading must cover at least
# half of that body's width — not merely clip a neighbouring column.
def _under_heading(xb):
ov = max(0, min(b[2], xb[2]) - max(b[0], xb[0]))
return ov >= 0.5 * max(1, xb[2] - xb[0])
row = sorted([x for x in below
if x["bbox"][1] - b[3] <= gap_top + 120 and _under_heading(x["bbox"])],
key=lambda x: (x["bbox"][1], x["bbox"][0]))
def _dateline_in_body_top(bbox):
# The dateline that OPENS the article may sit a line or two BELOW a merged
# photo caption / sub-headline inside the same body region — so scan the
# first several lines, not just the top one.
l, t, rr, bo = bbox
sh = min(360, bo - t)
if sh <= 0:
return None
try:
txt = pytesseract.image_to_string(page.crop((l, t, rr, t + sh)), lang="tel")
except Exception:
return None
lines = [x for x in txt.splitlines() if x.strip()]
for i in range(min(len(lines), 6)):
d = _dateline_in_text("\n".join(lines[i:i + 3]), paper)
if d:
return d
return None
dl = None
for x in row[:4]:
dl = _dateline_in_body_top(x["bbox"])
if dl:
break
if dl: # condition 2: dateline directly below
reg["type"] = "doc_title"
reg["_promoted_from"] = "paragraph_title"
reg["_dateline"] = dl
promoted += 1
print(f" ⬆ PROMOTED region {reg['id']}: paragraph_title → doc_title "
f"[paper={paper}] (W={w} spans column [body {body['id']} W={body_w}]; "
f"dateline below in region {body['id']}: {dl!r})")
else:
print(f" · not promoted: region {reg['id']} (W={w} spans column) — "
f"[paper={paper}] no dateline under it (body region {body['id']})")
if promoted:
print(f" → headline promotion: {promoted} paragraph_title(s) reclassified as doc_title")
return regions
def detect_sakshi_headline_bands(regions, page_img_path):
"""SAKSHI ONLY: pixel-level recovery of display headlines PaddleOCR never boxed.
Sakshi sets very large display headlines that the layout model frequently
misses outright (JG pg1: the lift-1 and SIR-mapping banners have NO region at
all) or boxes only partially (the 'అనూహ్య స్పందన' headline is missing its red
kicker half). Sakshi also has no ruled grid, so a missing headline makes the
whitespace clusterer fold that article's photo/body into the story above or
below it — the single biggest source of bad Sakshi crops. Two pixel passes:
1. WIDEN — a doc_title grows sideways over adjacent large-ink columns that
no other region covers (the un-boxed kicker half of its own headline).
Widest titles first, and each extension joins the coverage map, so a
narrow neighbour can never steal a wide headline's kicker.
2. SYNTHESIZE — a band of big-type ink covered by NO region, at least a
column wide and 55-280px tall, with article content following below it,
becomes a synthetic doc_title anchor so the clusterer splits there.
Bands above the first real headline are never synthesized (that strip is the
masthead/weather furniture, already dropped). Returns the updated regions.
"""
import numpy as np
try:
img = Image.open(page_img_path).convert("L")
except Exception:
return regions
W, H = img.size
g = np.asarray(img)
ink = g < 160
cov = np.zeros((H, W), dtype=bool)
for r in regions:
b = r["bbox"]
cov[max(0, b[1] - 6):min(H, b[3] + 6), max(0, b[0] - 6):min(W, b[2] + 6)] = True
_tw = sorted(r["bbox"][2] - r["bbox"][0] for r in regions if r.get("type") == "text")
col_w = _tw[len(_tw) // 2] if _tw else 600
# ---- pass 1: WIDEN partially-boxed doc_titles over uncovered kicker ink ----
titles = sorted([r for r in regions if r.get("type") == "doc_title"],
key=lambda r: -(r["bbox"][2] - r["bbox"][0]))
for t in titles:
b = t["bbox"]
y1, y2 = max(0, b[1]), min(H, b[3])
bh = max(1, y2 - y1)
band_ink = ink[y1:y2, :]
# columns covered by OTHER regions in this y-band
ocov = np.zeros(W, dtype=bool)
for r in regions:
if r is t:
continue
rb = r["bbox"]
if rb[1] < y2 and rb[3] > y1:
ocov[max(0, rb[0] - 6):min(W, rb[2] + 6)] = True
inkcol = band_ink.sum(axis=0) > 0.12 * bh
new_x = {}
for side, rng in (("L", range(b[0] - 1, -1, -1)), ("R", range(b[2] + 1, W))):
gap, inky, xstop = 0, [], None
for x in rng:
if ocov[x]:
xstop = x
break
if inkcol[x]:
inky.append(x); gap = 0
else:
gap += 1
if gap > 110:
break
# Ink within 25px of a foreign region's edge is that neighbour's own glyph
# overflow (PaddleOCR boxes are tight) — not kicker ink. Drop it so the
# widened headline never laps into the next column's margin.
if xstop is not None:
inky = [x for x in inky
if ((x >= xstop + 25) if side == "L" else (x <= xstop - 25))]
reach = (min(inky) if side == "L" else max(inky)) if inky else None
if reach is not None and abs(reach - (b[0] if side == "L" else b[2])) >= 60:
new_x[side] = reach
if new_x:
ob = list(b)
if "L" in new_x:
t["bbox"][0] = max(0, new_x["L"] - 8)
if "R" in new_x:
t["bbox"][2] = min(W, new_x["R"] + 8)
t["_widened"] = True
cov[max(0, b[1] - 6):min(H, b[3] + 6),
max(0, t["bbox"][0] - 6):min(W, t["bbox"][2] + 6)] = True
print(f" 📐 Sakshi headline WIDEN: R{t['id']} {ob} → {t['bbox']} "
f"(un-boxed kicker ink alongside)")
# ---- pass 2: SYNTHESIZE doc_titles for completely un-boxed headline bands ----
free_ink = ink & ~cov
title_tops = [r["bbox"][1] for r in regions
if r.get("type") in ("doc_title", "paragraph_title")]
floor_y = (min(title_tops) - 40) if title_tops else 0
rowink = free_ink.sum(axis=1)
hot = rowink > 320
bands = []
yy = 0
while yy < H:
if hot[yy]:
y0 = yy
gap = 0
while yy < H and gap <= 18:
gap = 0 if hot[yy] else gap + 1
yy += 1
bands.append((y0, yy - gap))
else:
yy += 1
# Text-column LEFT edges (clustered): two different articles' headlines can share
# pixel rows with only a ~50px gutter between them — the same size as a word gap —
# so a gap alone cannot split them. A gap that CONTAINS a known column-left IS the
# gutter between two columns of text → split there; a word gap inside one headline
# never coincides with a column start → bridge it.
_clefts = []
for r in regions:
if r.get("type") != "text":
continue
x0 = r["bbox"][0]
for c in _clefts:
if abs(c[0] / c[1] - x0) <= 25:
c[0] += x0; c[1] += 1
break
else:
_clefts.append([x0, 1])
col_lefts = [c[0] / c[1] for c in _clefts if c[1] >= 2]
next_id = max((r["id"] for r in regions), default=0) + 1
new_regs = []
for (by1, by2) in bands:
if by1 < floor_y:
continue
colhot = free_ink[by1:by2, :].sum(axis=0) > 0.25 * (by2 - by1)
# split the row band into independent column runs (two headlines can share rows)
x = 0
runs = []
seg = None
gapstart = None
while x <= W:
on = colhot[x] if x < W else False
if on:
if seg is None:
seg = [x, x]
elif gapstart is not None:
gw = x - gapstart
crosses_col = any(gapstart - 5 <= c <= x + 25 for c in col_lefts)
if gw > 150 or (gw >= 35 and crosses_col):
runs.append((seg[0], seg[1] + 1))
seg = [x, x]
else:
seg[1] = x
gapstart = None
seg[1] = x
else:
if seg is not None and gapstart is None:
gapstart = x
if seg is not None and gapstart is not None and x - gapstart > 150:
runs.append((seg[0], seg[1] + 1))
seg, gapstart = None, None
x += 1
if seg is not None:
runs.append((seg[0], seg[1] + 1))
for (rx1, rx2) in runs:
if (rx2 - rx1) < 0.75 * col_w:
continue
sub = free_ink[by1:by2, rx1:rx2]
rows_any = np.where(sub.sum(axis=1) > 40)[0]
if rows_any.size == 0:
continue
sy1, sy2 = by1 + int(rows_any.min()), by1 + int(rows_any.max()) + 1
if not (55 <= (sy2 - sy1) <= 400):
continue # 400 admits tall display/script faces (NT 'నమ్మిండు..')
# while still rejecting photos (their frac fails too)
frac = free_ink[sy1:sy2, rx1:rx2].mean()
if not (0.05 <= frac <= 0.45):
continue # too sparse = stray marks; too dense = a filled
# box / photo (the గమనిక notice header is 0.51;
# real display headlines measure 0.30-0.35)
if not any(r.get("type") in ("text", "paragraph_title")
and sy2 <= r["bbox"][1] <= sy2 + 500
and min(rx2, r["bbox"][2]) - max(rx1, r["bbox"][0])
>= 0.3 * (r["bbox"][2] - r["bbox"][0])
for r in regions):
continue # no article content below → not a headline
nb = [max(0, rx1 - 6), max(0, sy1 - 6), min(W, rx2 + 6), min(H, sy2 + 6)]
new_regs.append({"id": next_id, "type": "doc_title", "bbox": nb,
"confidence": 0.50, "_synth_band": True})
print(f" 📐 Sakshi headline SYNTH: R{next_id} doc_title {nb} "
f"(un-boxed display headline band, ink={frac:.2f})")
next_id += 1
return regions + new_regs
def mark_bullet_titles(regions, page_img_path):
"""Flag title regions that VISUALLY begin with a bullet glyph (●/•). A bullet is a
deck / sub-point, NEVER a real headline — so the clusterer can demote it when a real
headline sits above it. Detected by pixels (not OCR, which mangles '●' into a stray
'అ'): in the title's first-line strip, look for a narrow, vertically-filled dark blob
at the very left (the disk) followed by a whitespace gap before the text begins."""
try:
page = Image.open(page_img_path).convert("L")
except Exception:
return regions
px = page.load()
W, H = page.size
for r in regions:
if r.get("type") not in ("doc_title", "paragraph_title"):
continue
x1, y1, x2, y2 = r["bbox"]
h = min(y2 - y1, 90)
sx2 = min(x2, x1 + 140)
if h <= 0 or sx2 - x1 < 20:
continue
cols = [sum(1 for y in range(y1, y1 + h) if 0 <= y < H and px[min(x, W - 1), y] < 110)
for x in range(x1, sx2)]
i = 0
while i < len(cols) and cols[i] < 2: # skip leading blank margin
i += 1
j = i
while j < len(cols) and cols[j] >= 2: # first ink run (the bullet disk)
j += 1
k = j
while k < len(cols) and cols[k] < 2: # whitespace gap after it
k += 1
run_w, gap = j - i, k - j
fill = (max(cols[i:j]) / h) if j > i else 0
# bullet disk: a narrow run (8–55px), well filled vertically (≥0.5), a clear gap
if 8 <= run_w <= 55 and gap >= 8 and fill >= 0.5:
r["_bullet"] = True
n = sum(1 for r in regions if r.get("_bullet"))
if n:
print(f" • bullet titles flagged: {n} (deck/sub-point, never a headline)")
return regions
def find_article_starts_by_dateline(regions, page_img_path, paper):
"""CLUE scan (Tesseract, no Claude): OCR the first line of every text region;
any that OPENS with a dateline ('<town>, మే 19:' etc.) is the START of an
article — even if PaddleOCR never detected that article's headline. This is a
diagnostic cross-check: the number of dateline starts should match the number
of articles, exposing headlines the layout model missed."""
try:
import pytesseract
if "tel" not in set(pytesseract.get_languages(config="")):
print(" (dateline clue scan skipped — Telugu Tesseract 'tel' not installed)")
return []
except Exception:
print(" (dateline clue scan skipped — pytesseract not available)")
return []
page = Image.open(page_img_path)
starts = []
for r in regions:
if r.get("type") != "text":
continue
b = r["bbox"]
sh = min(220, b[3] - b[1]) # tall enough for a 3-line wrap in narrow columns
if sh <= 0:
continue
try:
txt = pytesseract.image_to_string(page.crop((b[0], b[1], b[2], b[1] + sh)), lang="tel")
except Exception:
txt = ""
dl = _dateline_in_text(txt, paper)
if dl:
starts.append({"region_id": r["id"], "dateline": dl, "bbox": b})
n = len(starts)
print(f" 📰 DATELINE CLUE [paper={paper}]: {n} dateline(s) by the '<town>, date:' pattern "
f"→ MINIMUM {n} article(s) on this page (actual may be more; never fewer).")
for s in sorted(starts, key=lambda z: z["bbox"][1]):
print(f" body R{s['region_id']} opens an article — dateline {s['dateline']!r}")
# --- DETECTED vs MISSED: does each dateline article have a detected headline? ---
start_bboxes = [s["bbox"] for s in starts]
def headline_above(body, max_gap=1000):
cx = (body[0] + body[2]) / 2
cands = [r for r in regions
if (r.get("type") == "doc_title" or r.get("_promoted_from"))
and r["bbox"][0] <= cx <= r["bbox"][2]
and r["bbox"][3] <= body[1] + 40
and (body[1] - r["bbox"][3]) <= max_gap] # must be reasonably close above
if not cands:
return None
h = max(cands, key=lambda r: r["bbox"][3]) # nearest headline above
for ob in start_bboxes: # another article's dateline between?
if ob is body:
continue
ocx = (ob[0] + ob[2]) / 2
if h["bbox"][3] < ob[1] < body[1] and cx - 120 <= ocx <= cx + 120:
return None
return h["id"]
detected, missed = [], []
for s in starts:
h = headline_above(s["bbox"])
s["headline_region"] = h # doc_title id above it, or None if missed
(detected if h else missed).append((s, h))
n_dt = sum(1 for r in regions if r.get("type") == "doc_title")
print(f" 🔎 HEADLINE COVERAGE: {len(detected)}/{n} articles have a detected headline; "
f"{len(missed)} MISSED by PaddleOCR (doc_title count={n_dt}).")
if missed:
print(f" ⚠ {len(missed)} article(s) Claude must catch by EYE (no headline box) — these would "
f"be lost if we only used doc_titles:")
for s, _ in sorted(missed, key=lambda z: z[0]["bbox"][1]):
print(f" body R{s['region_id']} {s['dateline']!r} ← headline not detected")
return starts
def draw_titles_debug(page_img_path, regions, pages_dir, page_num):
"""Save a per-page image highlighting every headline-type region:
blue = doc_title, orange = paragraph_title, purple = figure_title,
green = promoted headline. Each box is labelled R<id> <type> W<width>."""
try:
from PIL import ImageDraw, ImageFont
except Exception as e:
print(f" (titles debug image skipped: {e})")
return
try:
font = ImageFont.truetype("DejaVuSans-Bold.ttf", 30)
except Exception:
try:
font = ImageFont.load_default(size=30)
except Exception:
font = ImageFont.load_default()
img = Image.open(page_img_path).convert("RGB")
draw = ImageDraw.Draw(img)
n_doc = n_pt = n_fig = n_promo = 0
for r in regions:
t = r.get("type")
if t not in ("doc_title", "paragraph_title", "figure_title"):
continue
b = r["bbox"]
w = b[2] - b[0]
if r.get("_promoted_from"):
color, lw, tag = (0, 160, 0), 10, f"R{r['id']} PROMOTED->doc_title W{w}"
n_promo += 1
elif t == "doc_title":
color, lw, tag = (0, 90, 220), 9, f"R{r['id']} doc_title W{w}"
n_doc += 1
elif t == "paragraph_title":
color, lw, tag = (230, 120, 0), 5, f"R{r['id']} paragraph_title W{w}"
n_pt += 1
else: # figure_title
color, lw, tag = (150, 50, 200), 5, f"R{r['id']} figure_title W{w}"
n_fig += 1
draw.rectangle([b[0], b[1], b[2], b[3]], outline=color, width=lw)
ty = max(0, b[1] - 34)
draw.rectangle([b[0], ty, b[0] + len(tag) * 16 + 12, ty + 32], fill=color)
draw.text((b[0] + 6, ty + 4), tag, fill="white", font=font)
out = str(pages_dir / f"page_{page_num:03d}_titles_debug.png")
img.save(out)
print(f" TITLES DEBUG IMAGE saved: {out} "
f"(doc_title={n_doc}, paragraph_title={n_pt}, figure_title={n_fig}, promoted={n_promo})")
# distinct color per PaddleOCR region type (RGB)
_TYPE_COLORS = {
"doc_title": (0, 90, 220), # blue
"paragraph_title": (230, 120, 0), # orange
"figure_title": (150, 50, 200), # purple
"text": (110, 120, 135), # grey
"image": (20, 160, 90), # teal-green
"figure": (20, 160, 90), # teal-green
"table": (200, 40, 40), # red
"header": (210, 160, 0), # gold
"footer": (140, 90, 40), # brown
"number": (90, 90, 90), # dark grey
}
def draw_all_regions_debug(page_img_path, regions, pages_dir, page_num):
"""Save a per-page image highlighting EVERY region from PaddleOCR, color-coded
by type and labelled R<id> <type>. A complete map of what was detected."""
try:
from PIL import ImageDraw, ImageFont
except Exception as e:
print(f" (all-regions debug image skipped: {e})")
return
try:
font = ImageFont.truetype("DejaVuSans-Bold.ttf", 26)
except Exception:
try:
font = ImageFont.load_default(size=26)
except Exception:
font = ImageFont.load_default()
from collections import Counter
img = Image.open(page_img_path).convert("RGB")
draw = ImageDraw.Draw(img)
for r in regions:
t = r.get("type", "unknown")
b = r["bbox"]
color = _TYPE_COLORS.get(t, (90, 90, 90))
# promoted headlines stand out in green regardless of base type
if r.get("_promoted_from"):
color = (0, 160, 0)
lw = 7 if t in ("doc_title", "paragraph_title", "figure_title") else 4
tag = f"R{r['id']} {t}"
draw.rectangle([b[0], b[1], b[2], b[3]], outline=color, width=lw)
ty = max(0, b[1] - 30)
draw.rectangle([b[0], ty, b[0] + len(tag) * 14 + 10, ty + 28], fill=color)
draw.text((b[0] + 5, ty + 3), tag, fill="white", font=font)
out = str(pages_dir / f"page_{page_num:03d}_allregions_debug.png")
img.save(out)
counts = Counter(r.get("type", "unknown") for r in regions)
summary = ", ".join(f"{k}={v}" for k, v in sorted(counts.items(), key=lambda kv: -kv[1]))
print(f" ALL-REGIONS DEBUG IMAGE saved: {out} ({len(regions)} regions: {summary})")
def _wall_between_datelines(upper_box, lower_box, doc_titles, h_lines):
"""Is there a real article WALL — a headline (doc_title) or a horizontal
separator line — in the vertical gap between two datelines (upper above lower),
within their shared column? A wall means the lower dateline starts a genuinely
separate article; no wall means continuous body text (one roundup story)."""
top, bot = upper_box[3], lower_box[1] # bottom of upper dl → top of lower dl
if bot <= top:
return False
cx0, cx1 = max(upper_box[0], lower_box[0]), min(upper_box[2], lower_box[2])
cw = cx1 - cx0
if cw <= 0: # no column overlap → use lower column
cx0, cx1 = lower_box[0], lower_box[2]
cw = cx1 - cx0
for rb in doc_titles: # a headline in the gap
if rb[1] >= top - 10 and rb[3] <= bot + 10 \
and max(0, min(cx1, rb[2]) - max(cx0, rb[0])) >= 0.3 * max(1, cw):
return True
for L in h_lines: # a separator line crossing the gap
if top <= L["y"] <= bot \
and max(0, min(cx1, L["x2"]) - max(cx0, L["x1"])) >= 0.3 * max(1, cw):
return True
return False
def split_blocks_on_datelines(blocks, regions, dateline_starts, sep_lines=None):
"""Apply the one-dateline-per-crop rule to the geometric article blocks (the
source of all_article_crops), using the SAME wall rule as the political-article
splitter: a block holding 2+ datelines is split at every same-column dateline
that has a real WALL (headline / separator line) above it. Datelines in a
different column always split; same-column datelines with only body text
between them (a roundup) stay in one block."""
if not dateline_starts:
return blocks
region_map = {r["id"]: r for r in regions}
dl_ids = {s["region_id"] for s in dateline_starts}
doc_titles = [r["bbox"] for r in regions if r.get("type") == "doc_title"]
h_lines = [L for L in (sep_lines or []) if not L.get("vert") and "y" in L]
# A BANNER (a doc_title ≥1.6 column-widths) heads ONE article even when its body
# carries several mandal datelines (a roundup). Compute the column pitch so we can
# recognise such banners and keep their sub-datelines together.
_tw = sorted((r["bbox"][2] - r["bbox"][0]) for r in regions if r.get("type") == "text")
_pitch = _tw[len(_tw) // 2] if _tw else 0
out = []
for blk in blocks:
members = blk.get("members", [])
dls = [rid for rid in members if rid in dl_ids and rid in region_map]
if len(dls) <= 1:
out.append(blk)
continue
# Banner-roundup exception: if this block's OWN banner headline (a doc_title
# ≥1.6 column-widths, member of the block) sits ABOVE all its datelines and
# spans their columns (each dateline's centre lies under the banner), the whole
# thing is ONE roundup story under one headline — keep it together, do NOT split.
dl_cx = [((region_map[d]["bbox"][0] + region_map[d]["bbox"][2]) / 2) for d in dls]
dl_top = min(region_map[d]["bbox"][1] for d in dls)
block_banners = [region_map[m]["bbox"] for m in members
if region_map.get(m, {}).get("type") == "doc_title"
and _pitch and (region_map[m]["bbox"][2] - region_map[m]["bbox"][0]) >= 1.6 * _pitch]
if any(bt[3] <= dl_top + 40 and all(bt[0] - 30 <= cx <= bt[2] + 30 for cx in dl_cx)
for bt in block_banners):
out.append(blk)
continue
dl_boxes = [(rid, region_map[rid]["bbox"]) for rid in dls]
def owner(rid):
if rid not in region_map:
return dls[0]
b = region_map[rid]["bbox"]
cx = (b[0] + b[2]) / 2
in_col = [(d, db) for d, db in dl_boxes if db[0] - 40 <= cx <= db[2] + 40]
if region_map[rid].get("type") == "doc_title": # headline owns dl below it
below = [(d, db) for d, db in in_col if db[1] >= b[3] - 30]
if below:
return min(below, key=lambda z: z[1][1])[0]
above = [(d, db) for d, db in in_col if db[1] <= b[1] + 30] # body → dl above
if above:
return max(above, key=lambda z: z[1][1])[0]
if in_col:
return min(in_col, key=lambda z: z[1][1])[0]
return min(dls, key=lambda d: region_map[d]["bbox"][1])
groups = {}
for rid in members:
groups.setdefault(owner(rid), []).append(rid)
if len(groups) <= 1:
out.append(blk)
continue
# keeper = group holding the block's anchor (else the topmost dateline group)
keeper = blk["anchor_id"] if blk["anchor_id"] in groups \
else (owner(blk["anchor_id"]) if blk["anchor_id"] in region_map
else min(dls, key=lambda d: region_map[d]["bbox"][1]))
if keeper not in groups:
keeper = max(groups, key=lambda k: len(groups[k]))
keeper_ids = list(groups[keeper])
split_groups = []
for dl, ids in groups.items():
if dl == keeper:
continue
db = region_map[dl]["bbox"]
cx = (db[0] + db[2]) / 2
above = [region_map[d]["bbox"] for d in dls
if d != dl and region_map[d]["bbox"][1] < db[1]
and region_map[d]["bbox"][0] - 40 <= cx <= region_map[d]["bbox"][2] + 40]
upper = max(above, key=lambda b: b[1]) if above else None
# Same-column dateline above → wall decides. None above → different column
# (side-by-side) → always its own block.
split = _wall_between_datelines(upper, db, doc_titles, h_lines) \
if upper is not None else True
(split_groups.append((dl, ids)) if split else keeper_ids.extend(ids))
if not split_groups:
out.append(blk)
continue
def _bbox_of(ids):
bxs = [region_map[i]["bbox"] for i in ids if i in region_map]
return [min(b[0] for b in bxs), min(b[1] for b in bxs),
max(b[2] for b in bxs), max(b[3] for b in bxs)]
print(f" ✂ Block dateline split: block R{blk['anchor_id']} held {len(dls)} "
f"datelines → 1 + {len(split_groups)} (headline/separator-line boundary)")
out.append({**blk, "members": sorted(keeper_ids), "bbox": _bbox_of(keeper_ids)})
for dl, ids in sorted(split_groups, key=lambda kv: region_map[kv[0]]["bbox"][1]):
out.append({"anchor_id": dl, "kind": "DL", "bbox": _bbox_of(ids),
"members": sorted(ids)})
return out
# Paper currently being processed (set by process_pdf / the replay scripts before clustering).
# Gates paper-specific clustering passes — e.g. SAKSHI (a no-ruled-grid, headline-light paper)
# needs aggressive horizontal-rule splitting that would over-split the AJ/NT boxed grids.
_ACTIVE_PAPER = None
def cluster_all_article_blocks(regions, dateline_starts=None, sep_lines=None):
"""Reconstruct EVERY candidate article block on a page (political and not),
purely geometrically, BEFORE any Stage-3 filtering. Shared by the all-articles
debug overlay and the per-article crop dump so both stay in sync.
Each headline (doc_title or promoted) and each headline-less dateline start is
an article 'anchor'; a paragraph_title that sits at the very top of its column
(no headline above it in that column) is promoted to an anchor too. Every other
region is folded into the nearest anchor above it in the same column (else the
nearest below, else the globally nearest). Page furniture (header/number) is
dropped.
Returns a list of blocks ordered top-to-bottom, each a dict:
{"anchor_id", "kind" ('H'|'DL'|'P'), "bbox" [x1,y1,x2,y2], "members" [ids]}
"""
if not regions:
return []
# Drop printer colour-registration bars: short image strips that sit BELOW all
# article text at the page bottom (CMYK dots, etc.) — page furniture, never an
# article. No real photo sits beneath all body text without a caption.
_txt_bottom = max((r["bbox"][3] for r in regions if r.get("type") == "text"), default=0)
if _txt_bottom > 0:
regions = [r for r in regions if not (
r.get("type") == "image"
and r["bbox"][1] >= _txt_bottom - 5
and (r["bbox"][3] - r["bbox"][1]) < 150)]
def _cx(b):
return (b[0] + b[2]) / 2
def _col_overlap(ab, b):
return (ab[0] <= _cx(b) <= ab[2]) or (b[0] <= _cx(ab) <= b[2])
sep = sep_lines or []
vsep = [L for L in sep if L.get("vert")] # vertical column-divider rules
sep = [L for L in sep if not L.get("vert")] # horizontal article separators
# A printed rule whose midpoint lands well inside a photo box is the photo's own
# internal/border rule, NOT an article separator. PaddleOCR inflates photo boxes,
# so such a rule can sit inside the photo of the SAME article. Body text still
# respects every rule, but a photo/caption is allowed to fold up to its article
# across one of these in-photo rules (see the fold loop's `ignore_inphoto`).
_img_boxes = [r["bbox"] for r in regions if r.get("type") == "image"]
def _in_photo(L, margin=20):
mx = (L["x1"] + L["x2"]) / 2
for ix1, iy1, ix2, iy2 in _img_boxes:
if ix1 + margin <= mx <= ix2 - margin and iy1 + margin <= L["y"] <= iy2 - margin:
return True
# SAKSHI: a rule hugging an image's BOTTOM edge (within 14px below it,
# horizontally inside the photo span ±25px) is the photo's own frame /
# caption rule, never a story separator. PaddleOCR's photo box ends a few
# px above the printed border, so the interior test alone misses it — and
# the stray border then fences the article's own caption and column body
# off their anchor (JG pg1 lift-1: R61/R46/R23 fell into the story below).
if (_ACTIVE_PAPER == "sakshi"
and iy2 - 6 <= L["y"] <= iy2 + 14
and L["x1"] >= ix1 - 25 and L["x2"] <= ix2 + 25):
return True
return False
sep_real = [L for L in sep if not _in_photo(L)] # rules that aren't inside a photo
def _is_box_edge(L):
"""SAKSHI ONLY: a horizontal rule with a PARALLEL partner rule 80-400px away
(≥70% x-overlap) is the top/bottom EDGE of a small boxed insert (quote box /
bullet box) INSIDE an article — never a story separator. Without this, the
orange bullet-box borders under a Sakshi headline fence the article's own
column bodies off their anchor (JG pg1 lift-1: R46/R23/R61 fell into the
story below). Not applied to the ruled papers, where two stacked stories
can legitimately sit between parallel rules this close."""
if _ACTIVE_PAPER != "sakshi":
return False
w = max(1, L["x2"] - L["x1"])
for M in sep:
if M is L or "y" not in M:
continue
if not (80 <= abs(M["y"] - L["y"]) <= 400):
continue
ov = min(L["x2"], M["x2"]) - max(L["x1"], M["x1"])
if ov >= 0.7 * min(w, max(1, M["x2"] - M["x1"])):
return True
return False
def _barrier_between(top_box, bot_box, lines=None):
"""True if a separator rule cuts between an upper box and a lower box — i.e.
they're different articles and must not be folded/grown together. The rule
must sit below the upper box's bottom and no deeper than the lower box's
vertical midpoint (OCR inflates photo boxes UP past the rule, so we can't
require the rule to clear the lower box's top), and overlap the lower box's
column (line fragments are short, so any real overlap counts).
Default line set is `sep_real` (rules NOT inside a photo): a rule whose
midpoint lands inside an image box is the photo's own internal/border mark,
never an article separator, so it must not fence anything — body text
included. Pass an explicit list to override."""
bx1, by1, bx2, by2 = bot_box
bmid = (by1 + by2) / 2
for L in (sep_real if lines is None else lines):
if top_box[3] <= L["y"] <= bmid:
ov = max(0, min(bx2, L["x2"]) - max(bx1, L["x1"]))
if ov >= 0.25 * min(bx2 - bx1, L["x2"] - L["x1"]) and not _is_box_edge(L):
return True
return False
dl_starts = dateline_starts or []
missed_ids = {s["region_id"] for s in dl_starts if not s.get("headline_region")}
# A narrow doc_title sitting directly beneath a WIDER doc_title (no body text
# between them) is that headline's kicker/deck — not a separate article. It must
# NOT become its own anchor: otherwise a multi-column-header article gets split,
# and a dateline under it binds to the kicker instead of the real (wide) header.
# Demoting the kicker lets it fold into the wide header so the article stays whole
# and grows across all of the header's columns.
_doc_titles = [r for r in regions if r.get("type") == "doc_title"]
def _is_kicker(k):
kb = k["bbox"]
kw = kb[2] - kb[0]
for w in _doc_titles:
if w["id"] == k["id"]:
continue
wb = w["bbox"]
if (wb[2] - wb[0]) < kw * 1.4:
continue # not meaningfully wider
if not (wb[0] - 30 <= kb[0] and kb[2] <= wb[2] + 30):
continue # wide header must span kicker
if not (0 <= kb[1] - wb[3] <= 130):
continue # kicker sits directly below
if any(t.get("type") == "text"
and wb[3] <= t["bbox"][1] and t["bbox"][3] <= kb[1] + 5
and _col_overlap(kb, t["bbox"]) for t in regions):
continue # body text between → separate
return True
return False
kicker_ids = {r["id"] for r in _doc_titles if _is_kicker(r)}
# A doc_title that VISUALLY starts with a bullet (flagged by mark_bullet_titles) is a
# deck/sub-point, never a headline. If a real doc_title sits ABOVE it in the same
# column, demote the bullet so it folds into that article instead of anchoring its
# own block (fixes a banner article being split by its bulleted sub-head). Unlike
# the kicker rule, this holds even when a photo/body sits between them.
bullet_demoted = {
r["id"] for r in _doc_titles
if r.get("_bullet") and any(
o["id"] != r["id"] and o.get("type") == "doc_title" and not o.get("_bullet")
and _col_overlap(o["bbox"], r["bbox"]) and o["bbox"][1] < r["bbox"][1]
for o in _doc_titles)}
def _is_base(r):
return ((r.get("type") == "doc_title" or r.get("_promoted_from")
or r["id"] in missed_ids)
and r["id"] not in kicker_ids and r["id"] not in bullet_demoted)
base_anchors = [r for r in regions if _is_base(r)]
base_anchor_ids = {a["id"] for a in base_anchors}
# --- Grow-up recovery for headless ('missed') datelines --------------------
# An article whose bold headline PaddleOCR did NOT tag as doc_title has that
# headline (plus any bullet sub-title, caption, and photo) sitting ABOVE its
# dateline body. Walk UP from each such dateline and claim the contiguous run
# of title-type + image boxes that overlap the dateline's column — allowing
# boxes WIDER than the column (multi-column banner headlines). Stop at the
# first body 'text' box, another headline (doc_title/promoted), or another
# dateline. Those boxes attach DOWN into this dateline's block instead of
# folding into the article above, so the crop becomes complete.
def _hoverlap_frac(a, b):
ix = max(0, min(a[2], b[2]) - max(a[0], b[0]))
return ix / max(1, min(a[2] - a[0], b[2] - b[0]))
dl_body_ids = {s["region_id"] for s in dl_starts}
grow_claims = {} # region_id -> dateline-anchor id that claimed it upward
for s in dl_starts:
if s.get("headline_region"):
continue # already has a headline
dl_id, dl_box = s["region_id"], s["bbox"]
cands = [r for r in regions
if r["id"] != dl_id
and r["bbox"][3] <= dl_box[1] + 20 # bottom at/above dateline top
and _hoverlap_frac(dl_box, r["bbox"]) >= 0.5]
cands.sort(key=lambda r: r["bbox"][1], reverse=True) # nearest above first
def _is_caption(box):
"""A short text line sitting directly under a photo is that photo's
caption (PaddleOCR often tags it 'text', not 'figure_title'). Treat it as
claimable so grow-up can pass THROUGH it to reach the photo above."""
if box[3] - box[1] > 200:
return False # too tall → body paragraph
for o in regions:
if o.get("type") != "image":
continue
ob = o["bbox"]
if (ob[3] <= box[1] and (box[1] - ob[3]) <= 150
and _hoverlap_frac(box, ob) >= 0.5):
return True
return False
claimed_img = False # an article has one photo group
# Photo-unit detection: when the nearest content region directly above the
# dateline is a photo (or its caption), the separator just under it is the
# caption/photo UNDERLINE — part of THIS article, not an article boundary —
# so grow-up may cross it. Narrowly gated: if the nearest region above is
# ordinary body text, the separator is a real boundary and still stops grow-up.
cross_underline = False
for r in cands:
t0 = r.get("type")
if t0 in ("header", "number", "footer"):
continue
cross_underline = (t0 == "image"
or (t0 == "text" and _is_caption(r["bbox"])))
break # inspect only the nearest region
for r in cands:
t = r.get("type")
if _barrier_between(r["bbox"], dl_box):
_photo_part = (t == "image" or t in ("paragraph_title", "figure_title")
or (t == "text" and _is_caption(r["bbox"])))
if not (cross_underline and _photo_part):
break # real boundary → top of article
# else: photo/caption underline → cross it, keep claiming the photo unit
if t in ("header", "number", "footer"):
continue # skip furniture, keep going up
if (r["id"] in dl_body_ids or r["id"] in base_anchor_ids):
break # another article → stop
if t == "text":
if _is_caption(r["bbox"]):
grow_claims[r["id"]] = dl_id # photo caption → claim & go up
continue
break # body text → stop
if t == "image":
if claimed_img:
break # 2nd photo → next article's → stop
claimed_img = True
grow_claims[r["id"]] = dl_id
elif t == "paragraph_title":
grow_claims[r["id"]] = dl_id # the story's own bold heading
break # the heading IS the article's top — anything above it
# (the previous story's column body, its photos) is not
# ours (Sakshi pg2: R14's walk climbed past subhead R121
# and stole R20, the భక్తులకు article's column body)
elif t == "figure_title":
grow_claims[r["id"]] = dl_id # photo caption → keep climbing
else:
break
# --- Stacked-headline recovery ABOVE a detected headline anchor (SAME COLUMN) --
# A wrapped headline (or a header + sub-heading stack) is split into several title
# boxes: the lowest becomes the doc_title anchor, the upper line(s) sit above it.
# They belong to the anchor's article, not the previous same-column story. Grow UP
# from each doc_title anchor claiming contiguous bold title-type boxes; stop at the
# first normal body text (the previous article's), a separator rule, or another
# anchor. NO gap-size rule: the gap INSIDE a headline can equal the gap to the
# previous article (both ~40px observed), so only row TYPE — bold title vs normal
# body text — reliably separates them.
_TITLE_UP = ("paragraph_title", "figure_title", "header")
headline_stack_claims = {} # region_id -> doc_title anchor id
for a in base_anchors:
if a.get("type") != "doc_title":
continue
ab = a["bbox"]
cur_top = ab[1]
while True:
ups = [r for r in regions
if r["id"] != a["id"]
and r["id"] not in headline_stack_claims
and r["id"] not in base_anchor_ids
and r["id"] not in grow_claims
and r.get("type") in _TITLE_UP
and r["bbox"][3] <= cur_top + 8
and _hoverlap_frac(ab, r["bbox"]) >= 0.5] # same column only
if not ups:
break
r = max(ups, key=lambda r: r["bbox"][1]) # nearest title above
rb = r["bbox"]
# Normal body text between this title and the stack → previous article → stop.
if any(o.get("type") == "text"
and rb[3] <= o["bbox"][1] and o["bbox"][3] <= cur_top + 5
and _hoverlap_frac(ab, o["bbox"]) >= 0.5 for o in regions):
break
if _barrier_between(rb, [ab[0], cur_top, ab[2], cur_top + 1]):
break
# A figure_title sitting directly UNDER an image is that photo's CAPTION,
# never a wrapped headline line — it ends the stack (Sakshi: the story
# above's photo caption must not be pulled into a synthesized headline).
if (r.get("type") == "figure_title"
and any(o.get("type") == "image"
and 0 <= rb[1] - o["bbox"][3] <= 90
and (min(rb[2], o["bbox"][2]) - max(rb[0], o["bbox"][0]))
>= 0.4 * (rb[2] - rb[0])
for o in regions)):
break
headline_stack_claims[r["id"]] = a["id"]
cur_top = rb[1]
extra_ids = set()
extra_anchors = []
for r in regions:
if _is_base(r) or r.get("type") != "paragraph_title":
continue
if r["id"] in grow_claims or r["id"] in headline_stack_claims: # claimed below/stack
continue
b = r["bbox"]
# A paragraph_title sitting DIRECTLY ABOVE a base doc_title (same column,
# tiny gap, no rule between) is the upper line of that two-line headline —
# not a new article start. Let it fold down into that doc_title's block.
head_below = any(
a.get("type") == "doc_title" and _col_overlap(a["bbox"], b)
and 0 <= a["bbox"][1] - b[3] <= 120
and not _barrier_between(b, a["bbox"])
for a in base_anchors)
if head_below:
continue
_above = [a for a in base_anchors
if _col_overlap(a["bbox"], b) and a["bbox"][1] <= b[1]
and not _barrier_between(a["bbox"], b)]
if not _above:
extra_ids.add(r["id"])
extra_anchors.append(r)
elif _ACTIVE_PAPER == "sakshi":
# SAKSHI: a column-wide paragraph_title sitting FAR below the nearest
# detected headline in its column, with its own content directly
# underneath, is the HEADING of a new stacked story whose display type
# PaddleOCR mistyped (pg2 R125 'విమానయాన రంగంలో..' sat 1200px under the
# statue story's doc_title and the whole column merged into one
# page-tall crop). A deck/quote-box pt near its parent stays demoted.
_tw3 = sorted(q["bbox"][2] - q["bbox"][0] for q in regions
if q.get("type") == "text")
_colw3 = _tw3[len(_tw3) // 2] if _tw3 else 600
_gap3 = b[1] - max(a["bbox"][3] for a in _above)
# 900px ≈ 1.4 columns of body: a feature package's own sub-head sits
# closer to its headline than that (pg1 R94 at 614px must stay folded);
# a genuinely separate stacked story sits further (pg2 R125 at 1201px).
if (_gap3 > 900 and (b[2] - b[0]) >= 0.55 * _colw3
and any(q.get("type") in ("text", "image")
and 0 <= q["bbox"][1] - b[3] <= 150
and (min(q["bbox"][2], b[2]) - max(q["bbox"][0], b[0]))
>= 0.4 * (b[2] - b[0])
for q in regions)):
extra_ids.add(r["id"])
extra_anchors.append(r)
print(f" ⚓ Sakshi stacked-story heading: paragraph_title R{r['id']} "
f"promoted to anchor ({_gap3}px below the headline above)")
anchors = base_anchors + extra_anchors
if not anchors:
return []
anchor_ids = {a["id"] for a in anchors}
# Deck-band protection: whatever sits BELOW a headline and ABOVE that article's
# dateline (same column) is the article's deck — bullet/summary lines, a lead
# photo, a caption. It is PART OF THE ARTICLE and can never be split off as an
# orphan, even when a decorative bullet underline reads as a separator rule.
# Claim every region inside that header→dateline band for the headline.
_aby = {a["id"]: a for a in anchors}
band_bot_for = {} # headline id -> top y of its dateline body
for s in dl_starts:
h = s.get("headline_region")
if h in anchor_ids:
band_bot_for[h] = min(band_bot_for.get(h, s["bbox"][1]), s["bbox"][1])
band_claims = {} # region_id -> headline anchor id
for h, band_bot in band_bot_for.items():
hb = _aby[h]["bbox"]
band_top = hb[3]
if band_bot <= band_top:
continue
for r in regions:
if r["id"] in anchor_ids or r.get("type") in ("header", "number", "footer"):
continue
rb = r["bbox"]
if (rb[1] >= band_top - 10 and rb[3] <= band_bot + 10
and _col_overlap(hb, rb)):
band_claims[r["id"]] = h
# Faint top-boundary rules above multi-column headers (tagged "mc" by
# separator_barriers): anchor_id -> boundary y. A region whose box is wholly
# above this y must not fold into that multi-column anchor's block.
mc_bound = {L["anchor"]: L["y"] for L in sep
if L.get("mc") and L.get("anchor") in anchor_ids}
def _mc_blocked(a, box):
y0 = mc_bound.get(a["id"])
return y0 is not None and box[3] <= y0
clusters = {a["id"]: [(a["id"], a["bbox"])] for a in anchors}
orphans = []
for r in regions:
if r["id"] in anchor_ids or r.get("type") in ("header", "number", "footer"):
continue
b = r["bbox"]
if r["id"] in band_claims and band_claims[r["id"]] in clusters:
clusters[band_claims[r["id"]]].append((r["id"], b)) # deck band → headline
continue
if r["id"] in grow_claims and grow_claims[r["id"]] in clusters:
clusters[grow_claims[r["id"]]].append((r["id"], b)) # grown upward
continue
if r["id"] in headline_stack_claims and headline_stack_claims[r["id"]] in clusters:
clusters[headline_stack_claims[r["id"]]].append((r["id"], b)) # upper headline line
continue
col = [a for a in anchors if _col_overlap(a["bbox"], b)]
# A separator rule blocks folding ACROSS it for body text. Two exceptions:
# (1) a dateline body must always stay with its own headline (a rule never
# sits between a headline and its dateline); (2) a photo or its caption may
# fold up to its article across an IN-PHOTO rule (the photo's own border),
# but still respects real gutter separators — so it uses `sep_real`.
is_dl_body = r["id"] in dl_body_ids
is_photo = r.get("type") in ("image", "figure_title")
bar_lines = sep_real if is_photo else None
# A multi-column header's faint top-boundary rule clamps ITS OWN block: a
# region lying entirely above that rule must not fold down into the
# multi-column article below it. This is scoped to the multi-col anchor only
# (via mc_bound), so ordinary body text elsewhere is unaffected.
above = [a for a in col if a["bbox"][1] <= b[1] + 40
and (is_dl_body or not _barrier_between(a["bbox"], b, bar_lines))
and not _mc_blocked(a, b)]
if above:
best = max(above, key=lambda a: a["bbox"][1])
else:
# No anchor above in-column: fall through to the nearest anchor, prefer
# one whose column overlaps it. But a region fenced off from a candidate
# by a printed separator rule belongs to a DIFFERENT article — exclude
# those. A region cut off from EVERY candidate is a true orphan (e.g. a
# self-contained sub-article whose headline OCR mis-tagged): it must not
# be force-folded into a dateline story, so it goes to its own block.
cands = [a for a in anchors if not _mc_blocked(a, b)] or anchors
col_c = [a for a in cands if _col_overlap(a["bbox"], b)]
pool0 = col_c or cands
def _fenced(a):
ab = a["bbox"]
return (_barrier_between(ab, b, bar_lines) if ab[1] <= b[1]
else _barrier_between(b, ab, bar_lines))
pool = [a for a in pool0 if not _fenced(a)]
if not pool:
orphans.append((r["id"], b))
continue
best = min(pool,
key=lambda a: (_cx(a["bbox"]) - _cx(b)) ** 2
+ (a["bbox"][1] - b[1]) ** 2)
clusters[best["id"]].append((r["id"], b))
# Group the fenced-off regions by column-overlap + vertical adjacency. A group is
# only treated as a genuine ORPHAN ARTICLE (kept out of the dateline stories and
# emitted on its own) when it looks self-contained: it carries a title-type region
# AND has more than one member (e.g. a photo + mis-tagged headline + body whose
# OCR headline wasn't detected). A lone stray text/image fragment is NOT an article
# — it falls back to its nearest in-column anchor so dateline body isn't stripped.
rtype = {r["id"]: r.get("type") for r in regions}
orphans.sort(key=lambda z: z[1][1])
groups = []
for rid, ob in orphans:
placed = False
for grp in groups:
gb = grp["bbox"]
if _col_overlap(gb, ob) and -40 <= ob[1] - gb[3] <= 320:
grp["members"].append((rid, ob))
grp["bbox"] = [min(gb[0], ob[0]), min(gb[1], ob[1]),
max(gb[2], ob[2]), max(gb[3], ob[3])]
placed = True
break
if not placed:
groups.append({"anchor_id": rid, "bbox": list(ob),
"members": [(rid, ob)]})
_title_t = ("figure_title", "paragraph_title", "doc_title")
def _cluster_ubbox(aid):
ms = clusters[aid]
return [min(bb[0] for _, bb in ms), min(bb[1] for _, bb in ms),
max(bb[2] for _, bb in ms), max(bb[3] for _, bb in ms)]
def _wide_rule_between(ab, gtop, frac=0.55):
"""True if a WIDE article-boundary rule (spanning ≥frac of the header's
width) sits between the header bottom `ab[3]` and `gtop`. Narrow bullet /
caption underlines do not count — only a full-width rule that separates two
stacked stories. Used to stop folds from crossing a real article boundary."""
y0 = ab[3]
if gtop <= y0:
return False
hw = ab[2] - ab[0]
for L in sep:
if L.get("vert") or "y" not in L:
continue
if y0 - 4 <= L["y"] <= gtop + 4:
ov = (min(ab[2], L.get("x2", 0)) - max(ab[0], L.get("x1", 0)))
if ov >= frac * hw:
return True
return False
orphan_groups = []
for grp in groups:
mem_ids = [m for m, _ in grp["members"]]
# A real (mis-tagged) article carries a headline-ish title AND body text. A
# bare photo+caption (title-type but no body text) is just a figure belonging
# to a neighbouring story, so it folds back rather than orphaning.
is_article = (any(rtype.get(m) in _title_t for m in mem_ids)
and any(rtype.get(m) == "text" for m in mem_ids))
if is_article:
orphan_groups.append(grp)
continue
gb = grp["bbox"]
# A photo (+caption) sitting geometrically INSIDE an article's bounding box
# belongs to that article: a banner header spans the whole article width, so
# any image beneath it within that span — e.g. a 2nd photo in another column —
# is part of the story, not a separate crop. Fold it into the tightest
# containing cluster, overriding any decorative / in-photo rule between them —
# but NOT across a real article-boundary rule (a wide full-width rule that
# separates two stacked stories), otherwise a lower article's lead photo would
# be swallowed by the article above it.
if any(rtype.get(m) == "image" for m in mem_ids):
host, host_area = None, None
for a in anchors:
cb = _cluster_ubbox(a["id"])
if (cb[0] - 15 <= gb[0] and gb[2] <= cb[2] + 15
and cb[1] - 15 <= gb[1] and gb[3] <= cb[3] + 15
and not _wide_rule_between(a["bbox"], gb[1])):
area = (cb[2] - cb[0]) * (cb[3] - cb[1])
if host is None or area < host_area:
host, host_area = a["id"], area
if host is not None:
for rid, ob in grp["members"]:
clusters[host].append((rid, ob))
continue
# --- In-column body belongs to its header (dateline-independent) ---------
# MOST IMPORTANT RULE: every paragraph / bullet / text line BELOW a header,
# in the header's columns, is that article's own body — it must NEVER become
# an orphan just because a decorative bullet / sub-heading underline reads as
# a separator rule. The ONLY thing that ends an article downward is a REAL
# article-boundary rule: a WIDE horizontal rule spanning most of the header's
# width. A narrow bullet underline does not. This does NOT depend on dateline
# detection (often absent), so it holds even with zero datelines.
# Scope to TEXT bodies (paragraphs / bullets). A group carrying an image is a
# self-contained block — a photo, an advertisement / greetings poster — not
# flowing article text, so it is NOT pulled up by this rule (inline article
# photos are handled separately by the containment fold above).
_text_body = (any(rtype.get(m) == "text" for m in mem_ids)
and not any(rtype.get(m) == "image" for m in mem_ids))
bhost = None
if _text_body:
for a in sorted((x for x in anchors if x.get("type") == "doc_title"),
key=lambda x: -x["bbox"][3]): # nearest header above first
ab = a["bbox"]
if not _col_overlap(ab, gb) or gb[1] < ab[3] - 10:
continue # group must sit below this header
if _wide_rule_between(ab, gb[1]):
continue # real boundary rule → different story
bhost = a["id"]
break
if bhost is not None:
for rid, ob in grp["members"]:
clusters[bhost].append((rid, ob))
continue
# Not a clearly self-contained article. Try to fold the group back into a
# neighbouring story — but a width-matched grey rule BOUNDS an article, so the
# fold-back must respect it: only anchors NOT fenced from the group are valid
# targets. If the group is fenced from EVERY in-column anchor (e.g. a promo /
# teaser box below a story's full-width bottom rule, whose bold heading OCR
# merged into plain text), it is a separate 'other' article → emit on its own.
gcands = [a for a in anchors
if not (mc_bound.get(a["id"]) is not None and gb[3] <= mc_bound[a["id"]])]
gcands = gcands or anchors
gcol = [a for a in gcands if _col_overlap(a["bbox"], gb)] or gcands
def _grp_fenced(a):
ab = a["bbox"]
return (_barrier_between(ab, gb) if ab[1] <= gb[1]
else _barrier_between(gb, ab))
unfenced = [a for a in gcol if not _grp_fenced(a)]
if not unfenced:
orphan_groups.append(grp) # fenced from all → separate article
continue
for rid, ob in grp["members"]: # fold each member to nearest unfenced
best = min(unfenced, key=lambda a: (_cx(a["bbox"]) - _cx(ob)) ** 2
+ (a["bbox"][1] - ob[1]) ** 2)
clusters[best["id"]].append((rid, ob))
# --- Grow-right column walk for multi-column (banner) headers ---------------
# A banner headline only sets the article's MINIMUM width — the body can run
# into further columns to the right (overflow text, a photo) that the header
# does not span. Starting from a multi-column header block, walk RIGHTWARD one
# column-pitch at a time and absorb the next column when it is a genuine
# continuation: it carries NO dateline/headline of its own anywhere in the
# column, is not fenced by a printed vertical rule, is ~one column wide
# (±10%, photos exempt), and its content covers ≥75% of the article's band.
# Stop at the first column holding a dateline/headline, a vertical rule, or
# the page edge.
rby = {r["id"]: r for r in regions}
tws = sorted((r["bbox"][2] - r["bbox"][0]) for r in regions
if r.get("type") == "text")
pitch = tws[len(tws) // 2] if tws else 0
hard_ids = set(dl_body_ids) | base_anchor_ids # datelines + headlines
dissolved = set()
def _ubbox(members):
xs = [b for _, b in members]
return [min(z[0] for z in xs), min(z[1] for z in xs),
max(z[2] for z in xs), max(z[3] for z in xs)]
# current owner of every region (its cluster anchor, or an orphan group index)
loc = {}
for aid, mem in clusters.items():
for i, _ in mem:
loc[i] = ("c", aid)
for gi, grp in enumerate(orphan_groups):
for i, _ in grp["members"]:
loc[i] = ("o", gi)
if pitch > 0:
# Grow-right starters: any article anchor — a doc_title of ANY width (a banner
# need not be ≥1.6×pitch; a narrow ~1.5-column headline like లారీలు grows too)
# OR a headless dateline article (గజ్వేల్ with no detected headline). The strict
# column-twin guards below (same width ±10% / photo-exempt, same gutter, band
# alignment ≥85%, stop at a vertical rule or a column owning its own headline/
# dateline) are what keep this from over-merging — so the START can be liberal.
banners = [a for a in anchors
if a["id"] not in dissolved
and (a.get("type") == "doc_title" or a["id"] in missed_ids)]
banners.sort(key=lambda a: a["bbox"][1])
for H in banners:
if H["id"] in dissolved:
continue
hid = H["id"]
def _relocate(rid):
where = loc.get(rid)
if where is None or where == ("c", hid):
return
kind, key = where
if kind == "c":
clusters[key] = [(i, b) for i, b in clusters[key] if i != rid]
else:
grp = orphan_groups[key]
grp["members"] = [(i, b) for i, b in grp["members"] if i != rid]
clusters[hid].append((rid, rby[rid]["bbox"]))
loc[rid] = ("c", hid)
def _absorb(rid):
# An extra (paragraph_title) anchor in the column is an in-article
# subhead: move its WHOLE cluster and dissolve it. Plain regions
# move individually.
if rid in anchor_ids and rid != hid:
for i, _ in list(clusters.get(rid, [])):
_relocate(i)
clusters[rid] = []
dissolved.add(rid)
else:
_relocate(rid)
hb = _ubbox(clusters[hid])
band_top, band_bot, right = hb[1], hb[3], hb[2]
# This article's OWN column geometry (rules 2 & 3). Derive the column
# WIDTH and inter-column GUTTER from the header's existing body columns
# so the grow-right tests compare against THIS article's metrics — not a
# page-wide median that mixes in narrow side stories. Single-column
# spanning members (the banner header itself, ≥1.5×pitch wide) are
# excluded so they don't merge two columns into one.
_bodies = [b for i, b in clusters[hid]
if i != hid and (b[2] - b[0]) < 1.5 * pitch]
_acols = [] # [left, right] per body column
for bx in sorted(_bodies, key=lambda z: z[0]):
for c in _acols:
ov = min(c[1], bx[2]) - max(c[0], bx[0])
if ov > 0.5 * min(c[1] - c[0], bx[2] - bx[0]):
c[0] = min(c[0], bx[0]); c[1] = max(c[1], bx[2])
break
else:
_acols.append([bx[0], bx[2]])
_acols.sort()
art_colw = max((c[1] - c[0] for c in _acols), default=pitch)
_gaps = [_acols[i + 1][0] - _acols[i][1] for i in range(len(_acols) - 1)]
art_gutter = min(_gaps) if _gaps else None
while True:
lo, hi = right, right + int(1.4 * pitch)
col = [r for r in regions
if r.get("type") not in ("header", "number", "footer")
and lo + 5 <= _cx(r["bbox"]) <= hi]
if not col:
break # nothing → page edge
# A dateline/headline stops the walk only when it sits WITHIN this
# article's vertical band. A different story stacked ABOVE/BELOW in
# the same column (its headline outside our band) is not a barrier —
# only the column-segment beside our own rows decides ownership.
if any(r["id"] in hard_ids
and min(r["bbox"][3], band_bot) > max(r["bbox"][1], band_top)
for r in col):
break # dateline/headline → stop
inband = [r for r in col
if (min(r["bbox"][3], band_bot) - max(r["bbox"][1], band_top))
>= 0.75 * max(1, r["bbox"][3] - r["bbox"][1])]
if not inband:
break
cb = _ubbox([(r["id"], r["bbox"]) for r in inband])
# A vertical rule in the GUTTER *before* this column (between our current
# right edge and the column's left edge) is the article boundary → stop.
# A rule at the column's own RIGHT edge is just its divider — absorb the
# column; the next iteration's gutter check stops the walk there.
if any(right - 5 <= L["x"] <= cb[0] + 5
and L["y1"] <= band_bot and L["y2"] >= band_top
for L in vsep):
break # vertical rule in gutter → stop
cw = cb[2] - cb[0]
gutter = cb[0] - right # gap to this column
has_img = any(r.get("type") == "image" for r in inband)
# Rule 3 — the next column's WIDTH must be within ±10% of THIS
# article's column width (photos exempt: a photo column need not
# match the text column width). A different width = different story.
if not (has_img or abs(cw - art_colw) <= 0.10 * art_colw):
break # width mismatch → stop
# Rule 2 — the GUTTER to the next column must be within ±10% of the
# article's own inter-column gutter (photos exempt). A wider gap is
# the whitespace before a DIFFERENT story, so stop the walk there.
if (art_gutter is not None and not has_img
and abs(gutter - art_gutter) > 0.10 * art_gutter):
break # gutter widened → stop
# The column must SPAN the article's band height (header + the
# first columns) within ±15% — paragraph gaps inside the column do
# not count against it; only the top-to-bottom extent matters.
col_top = min(r["bbox"][1] for r in inband)
col_bot = max(r["bbox"][3] for r in inband)
band_h = band_bot - band_top
if band_h > 0 and (col_bot - col_top) < 0.85 * band_h:
break # column too short → stop
for r in inband:
_absorb(r["id"])
nb = _ubbox(clusters[hid])
band_top, band_bot = nb[1], nb[3]
if cb[2] <= right:
break # no progress → stop
right = cb[2]
blocks = []
for a in anchors:
if a["id"] in dissolved:
continue
members = clusters[a["id"]]
if not members:
continue
bxs = [b for _, b in members]
bbox = [min(z[0] for z in bxs), min(z[1] for z in bxs),
max(z[2] for z in bxs), max(z[3] for z in bxs)]
if a["id"] in extra_ids:
kind = "P"
elif a["id"] in missed_ids and a.get("type") != "doc_title":
kind = "DL"
else:
kind = "H"
# Missed-headline article: its stylised banner headline wasn't tagged as a
# region, so it sits in the whitespace ABOVE the topmost detected member with
# nothing to anchor on. Extend the block's top UP to the article-boundary rule
# directly above — but ONLY across a band that is empty of any other region's
# content. If a foreign region sits in the gap, the rule belongs to a different
# story above and extending would drag it in, so we leave the top as-is.
if kind == "DL":
member_ids = {m for m, _ in members}
cand = [L["y"] for L in sep
if not L.get("vert") and "y" in L
and bbox[1] - 420 <= L["y"] <= bbox[1] - 20
and max(0, min(bbox[2], L["x2"]) - max(bbox[0], L["x1"]))
>= 0.5 * (bbox[2] - bbox[0])]
if cand:
ry = max(cand)
band_clear = not any(
r["id"] not in member_ids
and _col_overlap(bbox, r["bbox"])
and r["bbox"][3] > ry + 6 and r["bbox"][1] < bbox[1] - 6
for r in regions)
if band_clear:
bbox[1] = ry + 6
blocks.append({"anchor_id": a["id"], "kind": kind, "bbox": bbox,
"members": [m for m, _ in members]})
orphan_blocks = []
for grp in orphan_groups:
if not grp["members"]:
continue
gbxs = [b for _, b in grp["members"]]
gbbox = [min(z[0] for z in gbxs), min(z[1] for z in gbxs),
max(z[2] for z in gbxs), max(z[3] for z in gbxs)]
orphan_blocks.append({"anchor_id": grp["anchor_id"], "kind": "O",
"bbox": gbbox,
"members": [m for m, _ in grp["members"]]})
# Orphan blocks must not overlap each other. When two overlap, the smaller is
# almost always a subset/fragment of the bigger main block — keep the larger
# and drop the smaller. Process biggest-first; drop any orphan that overlaps a
# kept (larger) orphan by ≥50% of its own area.
def _ob_area(b):
return max(0, b[2] - b[0]) * max(0, b[3] - b[1])
orphan_blocks.sort(key=lambda b: _ob_area(b["bbox"]), reverse=True)
kept_orphans = []
for ob in orphan_blocks:
bigger = next((k for k in kept_orphans
if _bbox_overlap_ratio(ob["bbox"], k["bbox"]) >= 0.5), None)
if bigger is not None:
print(f" · orphan overlap: block R{ob['anchor_id']} overlaps larger "
f"orphan R{bigger['anchor_id']} — dropped (smaller/subset)")
continue
kept_orphans.append(ob)
blocks.extend(kept_orphans)
# Enforce one-dateline-per-crop on the geometric blocks too (same wall rule as
# the political-article splitter), so all_article_crops never merge two stories.
blocks = split_blocks_on_datelines(blocks, regions, dateline_starts, sep_lines=sep_lines)
# Multi-column body under a WIDE photo: a plain-body orphan block sitting directly
# BELOW a photo that belongs to an article (within the photo's column span, small
# gap, no separator between) is that article's continuation in the next column —
# merge it in, so the crop isn't left with a masked-white gap where that text sits.
_rby = {r["id"]: r for r in regions}
def _plain_body(blk):
return blk["kind"] == "O" and blk["members"] and all(
_rby.get(m, {}).get("type") == "text" and m not in dl_body_ids
for m in blk["members"])
photo_owner = {} # photo region id -> its (non-orphan) article block
for blk in blocks:
if blk["kind"] == "O":
continue
for m in blk["members"]:
if _rby.get(m, {}).get("type") == "image":
photo_owner[m] = blk
kept = []
for blk in blocks:
if _plain_body(blk):
ob = blk["bbox"]
owner = None
for pid, ow in photo_owner.items():
pb = _rby[pid]["bbox"]
xov = max(0, min(pb[2], ob[2]) - max(pb[0], ob[0]))
if (pb[3] <= ob[1] + 20 and ob[1] - pb[3] <= 200 # photo directly above
and xov >= 0.5 * max(1, ob[2] - ob[0]) # orphan under the photo
and not _barrier_between(pb, ob)): # no separator between
owner = ow
break
if owner is not None:
owner["members"] = sorted(set(owner["members"]) | set(blk["members"]))
mb = [_rby[m]["bbox"] for m in owner["members"] if m in _rby]
owner["bbox"] = [min(b[0] for b in mb), min(b[1] for b in mb),
max(b[2] for b in mb), max(b[3] for b in mb)]
print(f" ⇲ Multi-column body: orphan R{blk['anchor_id']} folded under photo "
f"into article R{owner['anchor_id']} (next-column continuation)")
continue
kept.append(blk)
blocks = kept
# Caption binding + suppression: a caption — a `figure_title` (any height) OR a short
# `text` (≤ ~2 lines / 200px) sitting directly under an image — that ended up as its
# OWN orphan block belongs to the article that owns the IMAGE above it. Bind it there.
# Any caption-only orphan that can't be bound to a photo's article is page furniture,
# never an article → drop it (e.g. the stray "మాట్లాడుతున్న ప్రొఫెసర్…" orphan).
_rbc = {r["id"]: r for r in regions}
def _xovc(a, b):
return max(0, min(a[2], b[2]) - max(a[0], b[0]))
def _img_above(cb):
cw = cb[2] - cb[0]
cands = [_rbc[i]["bbox"] for i in _rbc if _rbc[i].get("type") == "image"
and _rbc[i]["bbox"][3] <= cb[1] + 10 and cb[1] - _rbc[i]["bbox"][3] <= 150
and _xovc(cb, _rbc[i]["bbox"]) >= 0.4 * max(1, cw)]
if not cands:
return None
return max(cands, key=lambda ib: (_xovc(cb, ib), -(cb[1] - ib[3])))
def _is_caption_member(r):
t = r.get("type")
if t == "figure_title":
return True
return t == "text" and (r["bbox"][3] - r["bbox"][1]) <= 200 and _img_above(r["bbox"]) is not None
img_owner = {} # image region id -> the block that holds it
for blk in blocks:
for m in blk["members"]:
if _rbc.get(m, {}).get("type") == "image":
img_owner[m] = blk
kept_c = []
for blk in blocks:
members = [m for m in blk["members"] if m in _rbc]
if blk["kind"] == "O" and members and all(_is_caption_member(_rbc[m]) for m in members):
bound = 0
for m in members:
ib = _img_above(_rbc[m]["bbox"])
owner = next((img_owner[i] for i in img_owner
if _rbc[i]["bbox"] == ib and img_owner[i] is not blk), None) if ib else None
if owner is not None:
owner["members"] = sorted(set(owner["members"]) | {m})
bx = [_rbc[x]["bbox"] for x in owner["members"] if x in _rbc]
owner["bbox"] = [min(b[0] for b in bx), min(b[1] for b in bx),
max(b[2] for b in bx), max(b[3] for b in bx)]
bound += 1
if bound:
print(f" ⊞ Caption bound: orphan R{blk['anchor_id']} → folded into its photo's article")
else:
print(f" ⊠ Caption orphan dropped: R{blk['anchor_id']} (caption with no photo article)")
continue # caption-only orphan never survives as a crop
kept_c.append(blk)
blocks = kept_c
# Banner fill: a true banner (a wide doc_title, ≥1.6×pitch) owns EVERY column under
# its width, down to the article's bottom. A column that got intercepted — e.g. a
# caption that anchored its own headless fragment block (R78: caption + col-3 body) —
# is pulled back into the banner. Only HEADLESS FRAGMENT blocks (no dateline, no
# doc_title/promoted headline of their own) that lie ENTIRELY within the banner's
# x-span (+ a half-column overflow margin) and inside its body band are absorbed, so
# a real neighbouring article (which has its own headline/dateline) is never merged.
_rbf = {r["id"]: r for r in regions}
for blk in blocks:
if blk["kind"] != "H":
continue
a = _rbf.get(blk["anchor_id"])
if not a or a.get("type") != "doc_title" or (a["bbox"][2] - a["bbox"][0]) < 1.6 * pitch:
continue # banner only
bx1, bx2 = a["bbox"][0] - 270, a["bbox"][2] + 270 # banner span + ½-column overflow
btop, bbot = a["bbox"][3] - 10, blk["bbox"][3] + 10 # body band: below headline → bottom
for other in blocks:
if other is blk or not other.get("members"):
continue
if any(_rbf.get(m, {}).get("type") == "doc_title"
or _rbf.get(m, {}).get("_promoted_from") or m in dl_body_ids
for m in other["members"]):
continue # has its own headline/dateline → real article
if all((_rbf[m]["bbox"][0] + _rbf[m]["bbox"][2]) / 2 >= bx1
and (_rbf[m]["bbox"][0] + _rbf[m]["bbox"][2]) / 2 <= bx2
and _rbf[m]["bbox"][1] >= btop and _rbf[m]["bbox"][3] <= bbot
for m in other["members"] if m in _rbf):
blk["members"] = sorted(set(blk["members"]) | set(other["members"]))
bx = [_rbf[m]["bbox"] for m in blk["members"] if m in _rbf]
blk["bbox"] = [min(b[0] for b in bx), min(b[1] for b in bx),
max(b[2] for b in bx), max(b[3] for b in bx)]
print(f" ▦ Banner fill: fragment R{other['anchor_id']} pulled under "
f"banner R{blk['anchor_id']} (column below the banner)")
other["members"] = [] # emptied → dropped below
blocks = [b for b in blocks if b.get("members")]
# Headless-DL headline-strip recovery (NO OCR): a dateline article with NO detected
# headline has its real headline in the UNDETECTED strip ABOVE the block's top edge —
# whether that top is a PHOTO (anganwadi case) or the BODY (top-of-page articles whose
# banner headline PaddleOCR missed entirely). Extend the block's TOP up over that strip
# so the headline pixels land in the crop. Bound it by a separator rule above, the
# previous article's content, or a detected header/masthead (a hard floor — never
# extend INTO it). For a top-of-page article with nothing above, extend up by a capped
# amount to the page margin. Scoped strictly to HEADLESS DL articles.
_rb3 = {r["id"]: r for r in regions}
_hsep = [L for L in (sep_lines or []) if not L.get("vert") and "y" in L]
HSTRIP_CAP = 300
def _xov3(a, b):
return max(0, min(a[2], b[2]) - max(a[0], b[0]))
for blk in blocks:
if blk["kind"] != "DL":
continue
if any(_rb3.get(m, {}).get("type") in ("doc_title", "paragraph_title")
for m in blk["members"]):
continue # has a headline already → not headless
bb = blk["bbox"]
top, w = bb[1], bb[2] - bb[0]
# Boundary above the block top, in its column: separators + the bottoms of
# NON-member regions (previous article / a header). A header/masthead is a hard
# floor — never extend into it.
ups, header_floor = [], None
for r in regions:
if r["id"] in blk["members"]:
continue
rb = r["bbox"]
if rb[3] <= top + 5 and top - rb[3] <= HSTRIP_CAP and _xov3(bb, rb) >= 0.3 * w:
ups.append(rb[3])
if r.get("type") in ("header", "footer", "number"):
header_floor = max(header_floor or 0, rb[3])
seps = [L["y"] for L in _hsep
if top - HSTRIP_CAP <= L["y"] <= top + 5
and _xov3(bb, [L["x1"], 0, L["x2"], 0]) >= 0.3 * w]
cands = ups + seps
if cands:
boundary = max(cands)
elif top <= HSTRIP_CAP: # top-of-page article → page margin
boundary = max(0, top - HSTRIP_CAP)
else:
continue # mid-page, nothing above → no strip
if header_floor is not None:
boundary = max(boundary, header_floor) # never cross a header/masthead
# No detected image inside the strip (boundary→top) — keep it a pure headline band.
if any(_rb3.get(r["id"], {}).get("type") == "image"
and boundary < r["bbox"][3] <= top - 5 and _xov3(bb, r["bbox"]) >= 0.3 * w
for r in regions):
continue
strip_h = top - boundary
if not (45 <= strip_h <= HSTRIP_CAP): # must be a headline-sized band
continue
new_top = int(boundary) + 4
if new_top < top:
blk["bbox"] = [bb[0], new_top, bb[2], bb[3]]
print(f" ⬆ Headline-strip: DL block R{blk['anchor_id']} top {top}→{new_top} "
f"(captured undetected headline above the article)")
# Floor-trim: drop trailing page-bottom furniture (a footer line, an empty/tiny text
# fragment) that sits FAR below a block's real content, separated by a large empty
# gap. Such regions fold into the nearest anchor above and inflate the crop with
# white space. If a big vertical gap (>300px) separates the lowest members from the
# rest AND everything below the gap is small/furniture, drop them and pull the bbox
# up to the real content.
_rb4 = {r["id"]: r for r in regions}
for blk in blocks:
mem = sorted([(m, _rb4[m]["bbox"]) for m in blk["members"] if m in _rb4],
key=lambda x: x[1][1])
if len(mem) < 2:
continue
for i in range(len(mem) - 1, 0, -1):
gap = mem[i][1][1] - max(b[3] for _, b in mem[:i])
below = mem[i:]
small_furniture = all(
(b[3] - b[1]) < 150
and _rb4[m].get("type") in ("text", "footer", "number", "figure_title")
for m, b in below)
if gap > 300 and small_furniture:
keep = [m for m, _ in mem[:i]]
bx = [_rb4[m]["bbox"] for m in keep]
blk["members"] = keep
blk["bbox"] = [min(b[0] for b in bx), min(b[1] for b in bx),
max(b[2] for b in bx), max(b[3] for b in bx)]
print(f" ⬓ Floor-trim: block R{blk['anchor_id']} dropped {len(below)} "
f"trailing furniture region(s) below a {int(gap)}px gap")
break
# Ceiling-trim: the mirror of floor-trim, for LEADING furniture. A headless dateline
# (DL) block's real content is at/below the dateline — nothing above belongs to it. But
# because every region folds into the nearest anchor, a tall top-of-column banner/photo
# that has no anchor of its own falls DOWN into the first dated article beneath it,
# ballooning the crop up to the page top. If the members ABOVE the dateline are
# separated from it by a big gap (>300px), drop them and pull the bbox down. Gated to
# DL blocks + >300px gap, so a normal photo/headline just above a dateline (small gap)
# is never touched.
_dlc = {s["region_id"] for s in (dateline_starts or [])}
for blk in blocks:
if blk["kind"] != "DL":
continue
anc = _rb4.get(blk["anchor_id"], {}).get("bbox")
if not anc:
continue
atop = anc[1]
above = [(m, _rb4[m]["bbox"]) for m in blk["members"]
if m in _rb4 and m != blk["anchor_id"] and _rb4[m]["bbox"][3] <= atop + 5]
if not above:
continue
gap = atop - max(b[3] for _, b in above)
if gap > 300:
drop = {m for m, _ in above}
keep = [m for m in blk["members"] if m not in drop]
bx = [_rb4[m]["bbox"] for m in keep if m in _rb4]
blk["members"] = keep
blk["bbox"] = [min(b[0] for b in bx), min(b[1] for b in bx),
max(b[2] for b in bx), max(b[3] for b in bx)]
print(f" ⬒ Ceiling-trim: block R{blk['anchor_id']} dropped {len(above)} "
f"leading furniture region(s) above a {int(gap)}px gap")
# Lead-photo-with-caption rebind: a PHOTO → CAPTION → HEADLINE vertical stack is the
# standard lead-photo layout (the photo illustrates the headline's story below it). When
# the photo/caption folded UP into the article above instead, move BOTH the photo and
# its caption down into the headline's block. Gated to that exact 3-region stack (image,
# a figure_title directly below it, a doc_title directly below that) so a photo at the
# BOTTOM of an article — which has body text (not a headline) under its caption — is
# never moved (that was the H93-type false positive).
_rlp = {r["id"]: r for r in regions}
_blkof = {}
for bl in blocks:
for m in bl["members"]:
_blkof[m] = bl
_imgs = [r for r in regions if r.get("type") == "image"]
_caps = [r for r in regions if r.get("type") == "figure_title"]
_docs = [r for r in regions if r.get("type") == "doc_title"]
def _hovl(a, b):
return max(0, min(a[2], b[2]) - max(a[0], b[0]))
for im in _imgs:
ib = im["bbox"]
cap = next((c for c in _caps if 0 <= c["bbox"][1] - ib[3] <= 90
and _hovl(ib, c["bbox"]) >= 0.4 * (ib[2] - ib[0])), None)
if not cap:
continue
cb = cap["bbox"]
dt = next((d for d in _docs if 0 <= d["bbox"][1] - cb[3] <= 90
and _hovl(cb, d["bbox"]) >= 0.3 * (cb[2] - cb[0])), None)
if not dt:
continue
if dt.get("_synth_band"):
continue # a SYNTHESIZED headline's lead photo sits BELOW its band by
# construction; a photo+caption ABOVE it is the tail of the
# article above (Sakshi R36/R69 must stay with the story above)
src = _blkof.get(im["id"]); tgt = _blkof.get(dt["id"])
if src is None or tgt is None or src is tgt:
continue
moved = [r for r in (im, cap) if _blkof.get(r["id"]) is src]
for r in moved:
src["members"].remove(r["id"]); tgt["members"].append(r["id"]); _blkof[r["id"]] = tgt
for blk in (src, tgt):
bx = [_rlp[m]["bbox"] for m in blk["members"] if m in _rlp]
if bx:
blk["bbox"] = [min(b[0] for b in bx), min(b[1] for b in bx),
max(b[2] for b in bx), max(b[3] for b in bx)]
print(f" 📸 Lead-photo rebind: photo R{im['id']} (+cap R{cap['id']}) → headline "
f"R{dt['id']}'s block (from R{src['anchor_id']})")
# NOTE: a "below-divider cut" (cut content sitting below a vertical divider's END out of
# the right-column article above — e.g. JG pg1 DL2 wrongly holding the maize story's
# wide photo R30 that straddles the divider) was implemented and REVERTED. The cut/merge
# destabilised the dense JG front page: the freed content merged into the wrong neighbour
# (R49 instead of the maize story R29), and the emptied DL2 regrew (Step 4) and re-absorbed
# stray regions. It is SAFE for AJ/NT (the straddle gate matched only JG R2), but the
# JG merge-target + regrow interaction needs more work before it can land. Deferred.
# Body-less headline merge: a large headline sometimes lands in its OWN H block
# together with the photo/caption ABOVE it, leaving the bullets + dated body as a
# SEPARATE block right underneath. Such a headline owns no story text of its own, yet
# a dated story sits directly below it in the same columns — the headline clearly
# belongs to THAT story. Fold the headline (and its photo) down into the story below.
# Gated tight: A must be body-LESS, its bottom member must be the headline (doc_title),
# and B must sit ≤90px below with ≥60% x-overlap AND carry real body text. The overlap
# gate means a WIDE section banner over a single narrow column won't merge (roundups).
_rbm = {r["id"]: r for r in regions}
_dropped = set()
_TITLE = ("doc_title", "paragraph_title")
for A in blocks:
# A body-less HEADER block — a lone doc_title (kind H) OR a lone paragraph_title /
# quote-box header (kind P) with no story text of its own — that sits directly above
# a block carrying real body text belongs to that story. (P covers cases like a
# pull-quote/sidebar header whose body folded into the article, leaving the header
# cropped as a tiny strip.)
if A["kind"] not in ("H", "P") or id(A) in _dropped:
continue
amem = [(m, _rbm[m]) for m in A["members"] if m in _rbm]
if any(r["type"] == "text" for _, r in amem):
continue # A must be body-LESS
if not any(r["type"] in _TITLE for _, r in amem):
continue
bottom_m = max(amem, key=lambda t: t[1]["bbox"][3])
if bottom_m[1]["type"] not in _TITLE:
continue # header must touch the body below
ax1, ay1, ax2, ay2 = A["bbox"]
best = None
for B in blocks:
if B is A or id(B) in _dropped:
continue
bx1, by1, bx2, by2 = B["bbox"]
if not (-10 <= by1 - ay2 <= 90):
continue
if max(0, min(ax2, bx2) - max(ax1, bx1)) < 0.6 * (ax2 - ax1):
continue
if not any(_rbm.get(m, {}).get("type") == "text" for m in B["members"]):
continue
if best is None or by1 < best[0]:
best = (by1, B)
if best:
B = best[1]
B["members"] = list(dict.fromkeys(A["members"] + B["members"]))
bx = [_rbm[m]["bbox"] for m in B["members"] if m in _rbm]
B["bbox"] = [min(b[0] for b in bx), min(b[1] for b in bx),
max(b[2] for b in bx), max(b[3] for b in bx)]
_dropped.add(id(A))
print(f" ⬇ Headline-merge: body-less headline R{A['anchor_id']} folded into "
f"dated story R{B['anchor_id']}")
if _dropped:
blocks = [b for b in blocks if id(b) not in _dropped]
# Containment-merge: if a block X sits almost entirely (≥90% of ITS area) inside a
# strictly LARGER block Y, X is a duplicate fragment of Y — a split-off column, an
# orphan tail, or a sub-story sitting under a roundup banner. Fold X's members into Y
# and drop X, so that content is cropped ONCE (inside Y) instead of twice. Runs BEFORE
# the rectangle pass so a reunited article (e.g. a paragraph_title header whose 2nd
# column had split off) again reads as multi-column and can be rectangled. The 0.9
# containment + larger-Y gate is strict: two side-by-side articles never qualify
# (neither is ~entirely inside the other), so unrelated articles are untouched.
_rbc = {r["id"]: r for r in regions}
def _area(b):
return max(1, (b[2] - b[0]) * (b[3] - b[1]))
_gone = set()
for X in blocks:
if id(X) in _gone:
continue
a = X["bbox"]
best = None
for Y in blocks:
if Y is X or id(Y) in _gone:
continue
c = Y["bbox"]
ox = max(0, min(a[2], c[2]) - max(a[0], c[0]))
oy = max(0, min(a[3], c[3]) - max(a[1], c[1]))
if _area(c) > _area(a) and ox * oy / _area(a) >= 0.9:
if best is None or _area(c) < _area(best["bbox"]):
best = Y # tightest wrapping container
if best is not None:
best["members"] = list(dict.fromkeys(best["members"] + X["members"]))
bx = [_rbc[m]["bbox"] for m in best["members"] if m in _rbc]
best["bbox"] = [min(b[0] for b in bx), min(b[1] for b in bx),
max(b[2] for b in bx), max(b[3] for b in bx)]
_gone.add(id(X))
print(f" ⊎ Containment-merge: R{X['anchor_id']}({X['kind']}) folded into "
f"R{best['anchor_id']}({best['kind']})")
if _gone:
blocks = [b for b in blocks if id(b) not in _gone]
# Header-split: a headline (doc_title) heads the story BELOW it. If an H block also
# holds a dateline sitting clearly ABOVE its headline, that dateline belongs to a
# SEPARATE earlier article that got glued on — either the column above had no anchor of
# its own and folded down into this block, or a contained DL article was absorbed into
# this block's over-extended box. Cut at the headline: everything above the headline
# becomes its own dated article; the headline and everything below it stay. Gated to a
# dateline >20px above the headline — the exact over-merge signature (measured to never
# match a correctly-clustered single article on AJ/NT).
_rbh = {r["id"]: r for r in regions}
_dlh = {s["region_id"] for s in (dateline_starts or [])}
_hs_new = []
for blk in list(blocks):
if blk["kind"] != "H":
continue
hb = _rbh.get(blk["anchor_id"], {}).get("bbox")
if not hb:
continue
htop = hb[1]
dl_above = [m for m in blk["members"]
if m in _dlh and _rbh[m]["bbox"][1] < htop - 20]
if not dl_above:
continue
above = [m for m in blk["members"]
if m in _rbh and m != blk["anchor_id"] and _rbh[m]["bbox"][3] <= htop + 10]
below = [m for m in blk["members"] if m not in set(above)]
if not above or len(below) < 1:
continue
top_dl = min(dl_above, key=lambda m: _rbh[m]["bbox"][1])
abx = [_rbh[m]["bbox"] for m in above if m in _rbh]
_hs_new.append({"anchor_id": top_dl, "kind": "DL", "members": sorted(above),
"bbox": [min(b[0] for b in abx), min(b[1] for b in abx),
max(b[2] for b in abx), max(b[3] for b in abx)]})
bbx = [_rbh[m]["bbox"] for m in below if m in _rbh]
blk["members"] = sorted(below)
blk["bbox"] = [min(b[0] for b in bbx), min(b[1] for b in bbx),
max(b[2] for b in bbx), max(b[3] for b in bbx)]
blk.pop("_raw_rect", None) # geometry changed; re-evaluate below
print(f" ✂ Header-split: R{blk['anchor_id']}(H) split off the article ABOVE its "
f"headline (new DL anchor R{top_dl}, {len(above)} region(s))")
blocks.extend(_hs_new)
# Banner crop-the-rectangle: a BANNER-headed MULTI-COLUMN article is cropped as the
# FULL rectangle — the banner's top, across its columns, down to the detected bottom
# boundary — taking everything in the box (raw, no masking). This fills any residual
# gap and keeps a section header's stories together (one crop per banner, by design).
# Strictly gated: (a) banner ≥1.6 columns AND article spans ≥2 columns, and (b) the
# bottom boundary is SURE — a separator rule spanning the banner, the next banner
# below, or the page bottom. If the boundary is uncertain, leave the member-based crop.
_rbr = {r["id"]: r for r in regions}
_hbar = [L for L in (sep_lines or []) if not L.get("vert") and "y" in L]
_page_bot = max((r["bbox"][3] for r in regions), default=0)
_dlids = {s["region_id"] for s in (dateline_starts or [])}
_sec_dropped = set()
for blk in blocks:
# A doc_title-headed block (kind H) is banner-detected on its doc_title; a
# paragraph_title-headed block (kind P) on its paragraph_title. H behaviour is
# unchanged — P just lets a wide paragraph_title header act as the banner too, so
# multi-column paragraph_title articles crop as one rectangle like doc_title ones.
if blk["kind"] == "H":
_btype = "doc_title"
elif blk["kind"] == "P":
_btype = "paragraph_title"
else:
continue
bts = [_rbr[m]["bbox"] for m in blk["members"]
if _rbr.get(m, {}).get("type") == _btype
and (_rbr[m]["bbox"][2] - _rbr[m]["bbox"][0]) >= 1.6 * pitch]
if not bts:
continue
banner = max(bts, key=lambda b: b[2] - b[0])
# ── Wide SECTION banner (roundup) ──────────────────────────────────────
# A banner ≥2.5 columns that heads MULTIPLE separate dated sub-stories is a
# SECTION header, not a single article. Crop the whole section as ONE raw
# rectangle: banner-top → the next full-width separator that spans the banner,
# and absorb (drop) every sub-block sitting inside it. The "≥2 headed sub-blocks
# inside" gate is what separates a section header from a wide SINGLE-article
# banner (which has no dated sub-stories beneath it) — so single articles and
# already-merged roundups are never caught.
b0l, b0t, b0r, b0b = banner
ban_w = b0r - b0l
if blk["kind"] == "H" and ban_w >= 2.5 * pitch:
spanning = [L["y"] for L in _hbar if L["y"] > b0b + 20
and max(0, min(b0r, L["x2"]) - max(b0l, L["x1"])) >= 0.8 * ban_w]
sec_bot = min(spanning) if spanning else None
if sec_bot is not None:
inside = [o for o in blocks if o is not blk
and o["bbox"][0] >= b0l - 30 and o["bbox"][2] <= b0r + 30
and o["bbox"][1] >= b0t - 10 and o["bbox"][3] <= sec_bot + 10]
headed_inside = [o for o in inside
if any(m in _dlids for m in o["members"])
or any(_rbr.get(m, {}).get("type") in ("doc_title", "paragraph_title")
for m in o["members"])]
if len(headed_inside) >= 2:
blk["bbox"] = [int(b0l), int(b0t), int(b0r), int(sec_bot)]
blk["_raw_rect"] = True
blk["_section"] = True # TRUE Type-1 section: a wide banner + a spanning
# bottom RULE (sec_bot) — never Type-2 split this.
for o in inside:
_sec_dropped.add(id(o))
print(f" ▭▭ Section banner: R{blk['anchor_id']} cropped as ONE rectangle "
f"x[{int(b0l)},{int(b0r)}] y[{int(b0t)},{int(sec_bot)}], "
f"absorbed {len(inside)} sub-block(s)")
continue
# ───────────────────────────────────────────────────────────────────────
# x-span = UNION of banner AND every member: the banner is sometimes detected
# narrower than the article's body columns, so clamping to the banner edge would
# clip the outer column(s). Take the widest extent that the article actually fills.
mboxes = [_rbr[m]["bbox"] for m in blk["members"] if m in _rbr]
bl = min([banner[0]] + [b[0] for b in mboxes])
brt = max([banner[2]] + [b[2] for b in mboxes])
btop = min([banner[1]] + [b[1] for b in mboxes]) # incl. a photo ABOVE the banner
cols = {round(((_rbr[m]["bbox"][0] + _rbr[m]["bbox"][2]) / 2) / max(1, pitch))
for m in blk["members"] if _rbr.get(m, {}).get("type") == "text"}
if len(cols) < 2:
continue # must be multi-column
# The article's REAL bottom comes from THIS block's own clustered members —
# NOT every region geometrically under the banner's wide x-span (that would pull
# a top banner all the way to the page bottom and swallow the stories below it).
content_bot = max((_rbr[m]["bbox"][3] for m in blk["members"] if m in _rbr),
default=blk["bbox"][3])
# SURE bottom boundary: a clean cut must exist JUST below the content (within
# ~70px) — a separator rule spanning the banner, the next banner down, or the
# page bottom. This confirms the rectangle ends with the article, not mid-story.
WIN = 70
sep_below = [L["y"] for L in _hbar
if content_bot - 12 <= L["y"] <= content_bot + WIN
and max(0, min(brt, L["x2"]) - max(bl, L["x1"])) >= 0.5 * (brt - bl)]
next_banner = [r["bbox"][1] for r in regions
if r.get("type") == "doc_title"
and (r["bbox"][2] - r["bbox"][0]) >= 1.6 * pitch
and content_bot - 12 <= r["bbox"][1] <= content_bot + WIN + 40
and max(0, min(brt, r["bbox"][2]) - max(bl, r["bbox"][0])) >= 0.3 * (brt - bl)]
near_page_bot = content_bot >= _page_bot - 80
if not (sep_below or next_banner or near_page_bot):
continue # boundary NOT sure → leave as-is
bottom = min([content_bot + 12] + sep_below + [(nb - 8) for nb in next_banner])
bottom = max(bottom, content_bot + 4) # never cut into the content
blk["bbox"] = [int(bl), int(btop), int(brt), int(bottom)]
blk["_raw_rect"] = True
print(f" ▭ Banner rectangle: R{blk['anchor_id']} cropped top-to-bottom "
f"x[{int(bl)},{int(brt)}] y[{int(btop)},{int(bottom)}] (raw, {len(cols)} cols)")
if _sec_dropped:
blocks = [b for b in blocks if id(b) not in _sec_dropped]
# Rightmost-column page-edge extension: an article in the RIGHTMOST column has the
# page's right margin as its natural right boundary (a "dead-end"). If a block's right
# edge already sits within half a column of the page-content right edge, extend it out
# to that page edge so the rightmost story isn't cropped tight / clipped. Strictly the
# right edge only — never a column-add, and INTERIOR articles (right edge well left of
# the page edge) are never touched.
_cright = max((r["bbox"][2] for r in regions), default=0)
for blk in blocks:
x1, y1, x2, y2 = blk["bbox"]
if _cright - 0.5 * pitch <= x2 < _cright:
blk["bbox"] = [x1, y1, int(_cright), y2]
print(f" ▶ Right-edge→page: R{blk['anchor_id']} right {int(x2)}→{int(_cright)} "
f"(rightmost column, +{int(_cright - x2)}px)")
# Drop graphic-label orphans: an ORPHAN (O) block that is pure text and sits almost
# entirely (≥80%) inside a FOREIGN image region is a label / caption baked into a photo
# or banner graphic — not an article. Remove it. (Measured: only 1 such block across
# AJ+NT+JG — a stray label inside a front-page banner image; AJ/NT have none.)
_rbj = {r["id"]: r for r in regions}
_imgsj = [(r["id"], r["bbox"]) for r in regions if r.get("type") == "image"]
_junk = set()
for blk in blocks:
if blk["kind"] != "O":
continue
if any(_rbj.get(m, {}).get("type") != "text" for m in blk["members"]):
continue # pure-text orphans only
a = blk["bbox"]; aarea = max(1, (a[2] - a[0]) * (a[3] - a[1]))
for iid, c in _imgsj:
if iid in blk["members"]:
continue
ox = max(0, min(a[2], c[2]) - max(a[0], c[0]))
oy = max(0, min(a[3], c[3]) - max(a[1], c[1]))
if ox * oy / aarea >= 0.8:
_junk.add(id(blk))
print(f" ✗ Drop graphic-label orphan: R{blk['anchor_id']} (pure text "
f"inside foreign image R{iid})")
break
if _junk:
blocks = [b for b in blocks if id(b) not in _junk]
# NOTE: orphan-TEXT absorb (fold a pure-text orphan up into the article above) was tried
# and REVERTED — its only match (NT pg4 R75→R132) was a CLASSIFIEDS "change of name"
# notice where it could not be verified whether the orphan was part of that ad or a
# separate adjacent notice. Merging distinct classifieds is a regression risk, so this
# remains deferred (see CROP_STRATEGY.md backlog).
# No-overlap reconciliation: when two block boxes overlap and exactly ONE of them has
# NO members inside the overlap, that box is merely over-reaching into its neighbour's
# territory (empty space for a masked crop, a DUPLICATE for a raw crop). Clip the
# over-reaching EDGE of the 0-member box back to its own members so the boxes no longer
# overlap. Only the 0-member side is trimmed (never grown); pairs where BOTH sides have
# members in the overlap are genuine interleaves (L-shaped articles) and are left alone.
_rbo = {r["id"]: r for r in regions}
def _mbb(blk):
bx = [_rbo[m]["bbox"] for m in blk["members"] if m in _rbo]
return ([min(b[0] for b in bx), min(b[1] for b in bx),
max(b[2] for b in bx), max(b[3] for b in bx)] if bx else list(blk["bbox"]))
def _memcount(blk, O):
return sum(1 for m in blk["members"] if m in _rbo
and not (_rbo[m]["bbox"][2] < O[0] or _rbo[m]["bbox"][0] > O[2]
or _rbo[m]["bbox"][3] < O[1] or _rbo[m]["bbox"][1] > O[3]))
for i in range(len(blocks)):
for j in range(i + 1, len(blocks)):
A, B = blocks[i], blocks[j]
a, b = A["bbox"], B["bbox"]
O = [max(a[0], b[0]), max(a[1], b[1]), min(a[2], b[2]), min(a[3], b[3])]
if O[0] >= O[2] or O[1] >= O[3]:
continue
iarea = (O[2] - O[0]) * (O[3] - O[1])
if iarea < 0.10 * min((a[2]-a[0])*(a[3]-a[1]), (b[2]-b[0])*(b[3]-b[1])):
continue
ca, cb = _memcount(A, O), _memcount(B, O)
if (ca == 0) == (cb == 0):
continue # both empty or both occupied → leave
X = A if ca == 0 else B # the over-reaching, content-free box
other = B if X is A else A
xb = list(X["bbox"])
mx = _mbb(X) # X's own members (must NOT be cut)
# neighbour's OCCUPIED region within the overlap (the content X is covering)
oboxes = [_rbo[mm]["bbox"] for mm in other["members"]
if mm in _rbo and not (_rbo[mm]["bbox"][2] < O[0] or _rbo[mm]["bbox"][0] > O[2]
or _rbo[mm]["bbox"][3] < O[1] or _rbo[mm]["bbox"][1] > O[3])]
if not oboxes:
continue
oc = [min(b[0] for b in oboxes), min(b[1] for b in oboxes),
max(b[2] for b in oboxes), max(b[3] for b in oboxes)]
# Retreat X's facing edge to just clear `oc`, WITHOUT cutting any of X's members.
cands = []
if oc[1] - 4 >= mx[3]: cands.append((xb[3] - (oc[1] - 4), [xb[0], xb[1], xb[2], oc[1] - 4])) # clip bottom (X above)
if oc[3] + 4 <= mx[1]: cands.append(((oc[3] + 4) - xb[1], [xb[0], oc[3] + 4, xb[2], xb[3]])) # clip top (X below)
if oc[0] - 4 >= mx[2]: cands.append((xb[2] - (oc[0] - 4), [xb[0], xb[1], oc[0] - 4, xb[3]])) # clip right (X left)
if oc[2] + 4 <= mx[0]: cands.append(((oc[2] + 4) - xb[0], [oc[2] + 4, xb[1], xb[2], xb[3]])) # clip left (X right)
cands = [(loss, nb) for loss, nb in cands if loss > 0 and nb[2] > nb[0] and nb[3] > nb[1]]
if not cands:
continue # X wraps the neighbour → genuine, leave alone
X["bbox"] = min(cands, key=lambda t: t[0])[1]
print(f" ▢ No-overlap: trimmed R{X['anchor_id']}({X['kind']}) to clear overlap "
f"with R{other['anchor_id']}")
# Ragged-bottom sidebar split: an L-shaped article whose ONE column runs FAR below the
# others, where that trailing tail is a SELF-CONTAINED sub-box (starts with its own
# paragraph_title sub-head) and the vacated corner is occupied by a DIFFERENT article
# (so one rectangle would mask that neighbour as white space). Split the tail into its
# own crop so the main article becomes a clean rectangle. STRICT gate: the MAIN part
# (block minus tail) must REMAIN multi-column (≥2 text columns) — so a block whose main
# would collapse to a lone column is left intact (avoids mis-splitting e.g. AJ R53).
_rs = {r["id"]: r for r in regions}
_tws = sorted((r["bbox"][2] - r["bbox"][0]) for r in regions if r.get("type") == "text")
_pitch_s = _tws[len(_tws) // 2] if _tws else 530
_split_new = []
for blk in list(blocks):
mem = [(m, _rs[m]["bbox"]) for m in blk["members"] if m in _rs]
if len(mem) < 4:
continue
colbot = {}
for m, bb in mem:
if _rs[m]["type"] in ("text", "paragraph_title"):
c = round(((bb[0] + bb[2]) / 2) / _pitch_s)
colbot[c] = max(colbot.get(c, -1), bb[3])
if len(colbot) < 2:
continue
tailcol = max(colbot, key=lambda c: colbot[c])
others_bot = max(v for c, v in colbot.items() if c != tailcol)
if colbot[tailcol] - others_bot < 250:
continue
tail = [(m, bb) for m, bb in mem
if round(((bb[0] + bb[2]) / 2) / _pitch_s) == tailcol and bb[1] >= others_bot - 30]
if not tail:
continue
tail.sort(key=lambda t: t[1][1])
if _rs[tail[0][0]]["type"] != "paragraph_title":
continue
if tail[-1][1][3] - tail[0][1][1] < 200:
continue
tail_ids = {m for m, _ in tail}
main_ids = [m for m in blk["members"] if m not in tail_ids]
maincols = {round(((_rs[m]["bbox"][0] + _rs[m]["bbox"][2]) / 2) / _pitch_s)
for m in main_ids if _rs.get(m, {}).get("type") == "text"}
if len(maincols) < 2:
continue # main would collapse → leave intact
ty1 = tail[0][1][1]; ty2 = max(bb[3] for _, bb in tail); minx = min(bb[0] for _, bb in tail)
if not any(o is not blk and not (o["bbox"][3] < ty1 or o["bbox"][1] > ty2)
and o["bbox"][0] < minx - 20 for o in blocks):
continue # nothing in the vacated corner → no white space
mbx = [_rs[m]["bbox"] for m in main_ids if m in _rs]
blk["members"] = sorted(main_ids)
blk["bbox"] = [min(b[0] for b in mbx), min(b[1] for b in mbx),
max(b[2] for b in mbx), max(b[3] for b in mbx)]
tbx = [bb for _, bb in tail]
_split_new.append({"anchor_id": tail[0][0], "kind": "P", "members": sorted(tail_ids),
"bbox": [min(b[0] for b in tbx), min(b[1] for b in tbx),
max(b[2] for b in tbx), max(b[3] for b in tbx)]})
print(f" ⑃ Sidebar-split: R{blk['anchor_id']} → main + sidebar R{tail[0][0]} "
f"({len(tail_ids)} regs, clears L-shape white space)")
blocks.extend(_split_new)
# Seed / lock CONFIDENT cells: a block is high-confidence ("locked") when it is a
# COMPLETE article — carries BOTH a dateline AND a detected headline
# (doc_title/paragraph_title) — OR it is FULLY BOXED (≥3 of its 4 sides coincide with a
# printed separator rule). These are the partition's fixed seeds; later boundary work
# operates on the UN-locked remainder.
_hb_l = [L for L in (sep_lines or []) if not L.get("vert") and "y" in L]
_vb_l = [L for L in (sep_lines or []) if L.get("vert")]
_dl_l = {s["region_id"] for s in (dateline_starts or [])}
_rl = {r["id"]: r for r in regions}
for blk in blocks:
x1, y1, x2, y2 = blk["bbox"]; bw = max(1, x2 - x1); bhh = max(1, y2 - y1)
n_dl = sum(1 for m in blk["members"] if m in _dl_l) + (1 if blk["anchor_id"] in _dl_l
and blk["anchor_id"] not in blk["members"] else 0)
single_dl = (n_dl == 1) # exactly one dated story = not an over-merge
has_head = any(_rl.get(m, {}).get("type") in ("doc_title", "paragraph_title")
for m in blk["members"])
sides = sum([
any(abs(L["y"] - y1) <= 45 and max(0, min(x2, L["x2"]) - max(x1, L["x1"])) >= 0.6 * bw for L in _hb_l),
any(abs(L["y"] - y2) <= 45 and max(0, min(x2, L["x2"]) - max(x1, L["x1"])) >= 0.6 * bw for L in _hb_l),
any(abs(L["x"] - x1) <= 45 and max(0, min(y2, L["y2"]) - max(y1, L["y1"])) >= 0.6 * bhh for L in _vb_l),
any(abs(L["x"] - x2) <= 45 and max(0, min(y2, L["y2"]) - max(y1, L["y1"])) >= 0.6 * bhh for L in _vb_l),
])
blk["_locked"] = bool((single_dl and has_head) or sides >= 3)
# Carve COURT / LEGAL NOTICES into their OWN section box. Regions tagged r['_legal'] by
# tag_legal_notices are court summons / name-change petitions / 'by order of the court'
# notices — page furniture that must be cropped as its own block, not absorbed into a
# neighbour (JG R65 swallowed a court summons in its left column). Group adjacent tagged
# regions; carve a cluster out ONLY when it is a real multi-part notice (≥2 tagged regions,
# ≥1 of them 'core') that sits ENTIRELY inside ONE WIDE (masked) host block — so the host
# whites the carved area out and nothing is duplicated. (A notice straddling a NARROW
# straight-cropped block, e.g. NT pg3, is intentionally left alone: carving it would
# duplicate, since a narrow block is cropped verbatim with no white-out.) Runs AFTER
# seed/lock so the new box is force-locked and Step 4 treats it as a fixed obstacle.
_legal_regs = [r for r in regions if r.get("_legal")]
if _legal_regs:
_legal_regs.sort(key=lambda r: r["bbox"][1])
_used = set(); _clusters = []
for r in _legal_regs:
if r["id"] in _used:
continue
grp = [r]; _used.add(r["id"]); changed = True
while changed:
changed = False
gx1 = min(g["bbox"][0] for g in grp); gx2 = max(g["bbox"][2] for g in grp)
gy1 = min(g["bbox"][1] for g in grp); gy2 = max(g["bbox"][3] for g in grp)
for s in _legal_regs:
if s["id"] in _used:
continue
sb = s["bbox"]
ox = min(gx2, sb[2]) - max(gx1, sb[0])
if ox < 0.4 * min(gx2 - gx1, sb[2] - sb[0]):
continue # different column
if sb[1] > gy2 + 180 or sb[3] < gy1 - 180:
continue # too far vertically
grp.append(s); _used.add(s["id"]); changed = True
_clusters.append(grp)
_lbyid = {r["id"]: r for r in regions}
for grp in _clusters:
ids = {g["id"] for g in grp}
if len(grp) < 2 or not any(g.get("_legal") == "core" for g in grp):
continue # need a real multi-part court notice
hosts = [b for b in blocks if ids & (set(b["members"]) | {b["anchor_id"]})]
if len(hosts) != 1:
continue # spread across blocks → leave alone
host = hosts[0]
if not ids <= (set(host["members"]) | {host["anchor_id"]}):
continue
hx1, _hy1, hx2, _hy2 = host["bbox"]
if (hx2 - hx1) <= 1100:
continue # NARROW host crops verbatim → would duplicate
mem = sorted(ids)
lbx = [_lbyid[i]["bbox"] for i in mem]
lbbox = [min(b[0] for b in lbx), min(b[1] for b in lbx),
max(b[2] for b in lbx), max(b[3] for b in lbx)]
anchor = min(mem, key=lambda i: _lbyid[i]["bbox"][1]) # topmost region = header
host["members"] = [m for m in host["members"] if m not in ids]
blocks.append({"anchor_id": anchor, "kind": "L", "bbox": lbbox,
"members": mem, "_locked": True, "_legal": True})
print(f" ⚖️ Legal notice: carved R{anchor} ({len(mem)} regs) out of "
f"R{host['anchor_id']} as its own section {lbbox}")
# ── Type-2 SPLIT: separate stacked articles that share NO common bottom rule ──────────
# Two layouts both look like one big box but need OPPOSITE treatment (AJ / Namaste — ruled
# papers):
# • Type 1 — several sub-stories under ONE wide banner with a COMMON BOTTOM horizontal rule.
# That banner+rule made the block a `_section` (the section-banner pass) → crop as ONE
# rectangle. NEVER split it.
# • Type 2 — independent articles stacked one below another, EACH with its OWN headline AND
# its own dateline, with NO common bottom rule between them (e.g. JG R65: కలెక్టర్/సుందరయ్య/
# పోస్టుమార్టం; R64: జాతీయ రహదారి / ఉపసర్పంచుల; R97: గుండె చికిత్స / మృతుడి పరామర్శ). Over-merge
# → split into one article per headline.
# DISCRIMINATOR = the `_section` flag (a real spanning bottom RULE), NOT the coarse `_raw_rect`
# flag — a single wide headline raw'd by the banner-rectangle pass (which merely extends to the
# next banner, NO rule) can still hide a 2nd article below it (R64), so those ARE eligible.
# ANCHORS = doc_title OR paragraph_title (PaddleOCR often tags the 2nd headline a
# paragraph_title — R84, R58) that OWNS a dateline. Ownership is COLUMN-AWARE: each dateline
# goes to the nearest headline above it WHOSE x-span covers the dateline (so a caption in a
# different column never steals a dateline). A single article with a headline + a sub-headline
# sharing ONE dateline yields ONE owner ⇒ not split (AJ R50/R57, R35/R79).
_new_split = []
_split_done = []
for blk in list(blocks):
if blk.get("_section") or blk.get("_legal"):
continue # true Type-1 section / legal → keep whole
dts = sorted([m for m in blk["members"]
if _rl.get(m, {}).get("type") in ("doc_title", "paragraph_title")],
key=lambda m: _rl[m]["bbox"][1])
_owns = {}
for d in [m for m in blk["members"] if m in _dl_l]:
db = _rl[d]["bbox"]; dy = db[1]; dcx = (db[0] + db[2]) / 2
above = [h for h in dts if _rl[h]["bbox"][1] <= dy + 8
and _rl[h]["bbox"][0] - 40 <= dcx <= _rl[h]["bbox"][2] + 40] # same column
if above:
_owns.setdefault(max(above, key=lambda h: _rl[h]["bbox"][1]), []).append(d)
anchors = sorted(_owns.keys(), key=lambda h: _rl[h]["bbox"][1])
if len(anchors) < 2:
continue
# require the headlines to be VERTICALLY STACKED (≥120px apart) — not two columns headed
# side-by-side at the same height (that is one section, not a vertical over-merge).
ys = [_rl[a]["bbox"][1] for a in anchors]
if any(abs(ys[i + 1] - ys[i]) < 120 for i in range(len(ys) - 1)):
continue
# Assign each member to its article using BOTH y and x: the nearest headline at/above it
# (the y-band), but only if the member sits within that headline's COLUMN x-span (± margin).
# A member that lands in a headline's y-band but a DIFFERENT column is unrelated content
# sharing the page real-estate — e.g. JG R65's left column below the carved court notice
# holds a classifieds logo + a small job ad + a గమనిక notice, none of which belong to the
# centre-right సుందరయ్య / పోస్టుమార్టం stories. Such members fall out as LEFTOVER and become a
# `_furniture` section: excluded from the article crops (a separate block masks white), drawn
# on the debug boundary map, and NOT saved as an article crop.
_marg = 0.3 * pitch
groups = {a: [] for a in anchors}
leftover = []
for m in set(blk["members"]) | {blk["anchor_id"]}:
if m not in _rl:
continue
mb = _rl[m]["bbox"]; mcx = (mb[0] + mb[2]) / 2; my = mb[1]
above = [a for a in anchors if _rl[a]["bbox"][1] <= my + 8]
tgt = max(above, key=lambda a: _rl[a]["bbox"][1]) if above else anchors[0]
hb = _rl[tgt]["bbox"]
if hb[0] - _marg <= mcx <= hb[2] + _marg:
groups[tgt].append(m)
else:
leftover.append(m)
for a in anchors:
mem = sorted(set(groups[a]))
if not mem:
continue
bxs = [_rl[m]["bbox"] for m in mem]
_new_split.append({"anchor_id": a, "kind": "H",
"bbox": [min(b[0] for b in bxs), min(b[1] for b in bxs),
max(b[2] for b in bxs), max(b[3] for b in bxs)],
"members": mem, "_locked": True})
# group the leftover (out-of-column) regions into furniture section(s) by column +
# proximity — but NEVER across a separator rule: two stacked items in the leftover
# column with a rule between them are different stories (pg2 col4: the విద్యార్థిని
# girl's story and the శ్రీనివాస్ tribute tail were one group until the y=4679 rule
# was honoured).
_fclusters = []
for m in sorted(leftover, key=lambda m: _rl[m]["bbox"][1]):
mb = _rl[m]["bbox"]
placed = False
for fc in _fclusters:
fx1 = min(_rl[i]["bbox"][0] for i in fc); fx2 = max(_rl[i]["bbox"][2] for i in fc)
fy1 = min(_rl[i]["bbox"][1] for i in fc)
fy2 = max(_rl[i]["bbox"][3] for i in fc)
if (min(fx2, mb[2]) - max(fx1, mb[0])) >= 0.3 * min(fx2 - fx1, mb[2] - mb[0]) \
and mb[1] <= fy2 + 260 \
and not (_ACTIVE_PAPER == "sakshi"
and _barrier_between([fx1, fy1, fx2, fy2], mb)):
fc.append(m); placed = True; break
if not placed:
_fclusters.append([m])
for fc in _fclusters:
bxs = [_rl[i]["bbox"] for i in fc]
fbb = [min(b[0] for b in bxs), min(b[1] for b in bxs),
max(b[2] for b in bxs), max(b[3] for b in bxs)]
top_id = min(fc, key=lambda i: _rl[i]["bbox"][1])
# A leftover group that LEADS with its own column-wide heading and carries body
# text is a real stacked ARTICLE whose column none of the split anchors span —
# emit it as an article, not furniture (Sakshi pg2: the విద్యార్థిని ఆత్మహత్య story,
# heading R88 on top, fell out of the R41 split and was never cropped). The
# width gate matters twice over: a narrow bold label is not a story heading (a
# classifieds WANTED box stays furniture), and a narrow SUB-head atop a story's
# continuation tail ('పలువురి నివాళి') must fall through to the continuation
# check below, not start a phantom article. This check runs BEFORE band-attach:
# a group with its own real heading is a new story even when it sits gap-close
# beside another article's band.
if (_rl[top_id].get("type") in ("doc_title", "paragraph_title")
and any(_rl[i].get("type") == "text" for i in fc)
and (_rl[top_id]["bbox"][2] - _rl[top_id]["bbox"][0])
>= 0.6 * max(1, fbb[2] - fbb[0])):
_new_split.append({"anchor_id": top_id, "kind": "P", "bbox": fbb,
"members": sorted(fc), "_locked": True})
print(f" ✂️ Type-2 leftover → ARTICLE R{top_id} (leads with its own "
f"heading; {len(fc)} regions)")
continue
# A heading-less leftover group whose vertical band MATCHES an adjacent-column
# split article (top within 90px, ≥80% overlap, facing gap ≤80px) is that
# article's column CONTINUATION — its text even resumes mid-sentence (pg2:
# R119/R4/R97/R7 carry the విషాదయాత్ర story's ending + tributes, beside H93).
fh = max(1, fbb[3] - fbb[1])
host = None
for ns in (_new_split if _ACTIVE_PAPER == "sakshi" else []):
if ns.get("_furniture"):
continue
nb = ns["bbox"]
gap = (fbb[0] - nb[2]) if fbb[0] >= nb[2] else (nb[0] - fbb[2])
vov = min(fbb[3], nb[3]) - max(fbb[1], nb[1])
if (-20 <= gap <= 80 and abs(nb[1] - fbb[1]) <= 90
and vov >= 0.8 * min(fh, max(1, nb[3] - nb[1]))):
host = ns
break
if host is not None:
host["members"] = sorted(set(host["members"]) | set(fc))
host["bbox"] = [min(host["bbox"][0], fbb[0]), min(host["bbox"][1], fbb[1]),
max(host["bbox"][2], fbb[2]), max(host["bbox"][3], fbb[3])]
print(f" ✂️ Type-2 leftover → CONTINUATION of R{host['anchor_id']} "
f"(same band, adjacent column; {len(fc)} regions)")
continue
_new_split.append({"anchor_id": min(fc), "kind": "F", "bbox": fbb,
"members": sorted(fc), "_locked": True, "_furniture": True})
_split_done.append(id(blk))
print(f" ✂️ Type-2 split: R{blk['anchor_id']} → {len(anchors)} articles "
f"R{anchors}" + (f" + {len(_fclusters)} furniture section(s)" if _fclusters else "")
+ " (stacked headlines, no common bottom rule)")
if _split_done:
blocks = [b for b in blocks if id(b) not in _split_done] + _new_split
# ── SAKSHI / NT horizontal-rule split ────────────────────────────────────────────────
# Sakshi is a no-grid, headline-light paper, so the whitespace clusterer over-merges stacked
# stories into one block. But Sakshi DOES print horizontal RULES between stacked articles
# (full-width and ~1-column). Per the brief: wherever a rule LONGER than a caption (≥0.85 col)
# crosses a block's interior with real content both above and below, split the block there.
# NT (Namaste Telangana) gets the SAME split but only at rules ≥1.8 columns wide — per the
# line-pattern study a ≥2-col rule is an article boundary on NT (the boxed grid's ~1-col
# caption/box rules must NOT cut; NT Adilabad's feature-heavy flood front page over-merged
# the whole bottom band across a 4.4-col rule). AJ is untouched (whitespace layout).
# Each band's anchor is its own headline if it has one, else its topmost member.
if _ACTIVE_PAPER in ("sakshi", "namaste_telangana"):
_sk_minw = 0.85 if _ACTIVE_PAPER == "sakshi" else 1.8
_sk_tw = sorted(r["bbox"][2] - r["bbox"][0] for r in regions if r.get("type") == "text")
_sk_col = _sk_tw[len(_sk_tw) // 2] if _sk_tw else 600
_sk_hbars = sorted([L for L in (sep_lines or []) if not L.get("vert") and "y" in L],
key=lambda L: L["y"])
_sk_new = []
_sk_done = []
for blk in list(blocks):
if blk.get("_raw_rect") or blk.get("_legal") or blk.get("_furniture"):
continue
x1, y1, x2, y2 = blk["bbox"]; bw = max(1, x2 - x1)
mems = [m for m in (set(blk["members"]) | {blk["anchor_id"]}) if m in _rl]
if len(mems) < 2:
continue
# y-tops of every "article start" in the block (a detected headline or a dateline)
starts_y = [_rl[m]["bbox"][1] for m in mems
if _rl[m].get("type") in ("doc_title", "paragraph_title") or m in _dl_l]
# Cut ONLY at a rule that (a) is longer than a caption & spans the block, (b) is NOT a
# caption underline, AND (c) has a NEW article starting just below it (a headline/dateline
# within ~170px). Condition (c) is what stops a HEADLINE-underline rule from severing a
# headline from its own body (the old bug: R67 → 47px headline + orphan body).
cuts = [L["y"] for L in _sk_hbars
if y1 + 80 < L["y"] < y2 - 80
and (L["x2"] - L["x1"]) >= _sk_minw * _sk_col
and (min(x2, L["x2"]) - max(x1, L["x1"])) >= 0.6 * bw
and not _line_is_underline(L["y"], L["x1"], L["x2"], regions)
and (any(L["y"] - 10 <= sy <= L["y"] + 170 for sy in starts_y)
# NT: a ≥2.2-col rule is a STRUCTURAL band divider; it cuts even
# when the next band's headline isn't a member yet (NT Adilabad's
# bottom-band headline only merges in later via containment)
or (_ACTIVE_PAPER == "namaste_telangana"
and (L["x2"] - L["x1"]) >= 2.2 * _sk_col))]
if not cuts:
continue
bounds = [y1 - 1] + cuts + [y2 + 1]
bands = []
for i in range(len(bounds) - 1):
lo, hi = bounds[i], bounds[i + 1]
bm = [m for m in mems if lo <= (_rl[m]["bbox"][1] + _rl[m]["bbox"][3]) / 2 < hi]
if bm:
bands.append(bm)
if len(bands) < 2:
continue
for bm in bands:
bxs = [_rl[m]["bbox"] for m in bm]
heads = [m for m in bm if _rl[m].get("type") in ("doc_title", "paragraph_title")]
anch = (min(heads, key=lambda m: _rl[m]["bbox"][1]) if heads
else min(bm, key=lambda m: _rl[m]["bbox"][1]))
_sk_new.append({"anchor_id": anch, "kind": ("H" if heads else blk.get("kind", "O")),
"bbox": [min(b[0] for b in bxs), min(b[1] for b in bxs),
max(b[2] for b in bxs), max(b[3] for b in bxs)],
"members": sorted(bm), "_locked": True})
_sk_done.append(id(blk))
print(f" ─ Sakshi rule-split: R{blk['anchor_id']} → {len(bands)} bands at rules {cuts}")
if _sk_done:
blocks = [b for b in blocks if id(b) not in _sk_done] + _sk_new
# Step 4 — grow OPEN (non-confident) cells into surrounding EMPTY space, bounded by the
# nearest other cell (locked OR open), big vertical walls, and the page frame. LOCKED
# cells are NEVER moved. Each open cell extends each edge until just before the nearest
# neighbour/wall/frame in that direction, filling its gap WITHOUT overlapping anything.
# Grow-only (never shrinks). Greedy (uses already-grown neighbours) so two adjacent open
# cells meet at the gap instead of overlapping.
_GAP = 8
_frame_l = min((r["bbox"][0] for r in regions), default=0)
_frame_r = max((r["bbox"][2] for r in regions), default=0)
_frame_b = max((r["bbox"][3] for r in regions), default=0)
_frame_t = min((b["bbox"][1] for b in blocks), default=0) # content top (below masthead)
_vwx = [((L["x1"] + L["x2"]) // 2 if "x" not in L else L["x"],
min(L.get("y1", 0), L.get("y2", 0)), max(L.get("y1", 0), L.get("y2", 0)))
for L in (sep_lines or []) if L.get("vert")]
for X in [b for b in blocks if not b.get("_locked")]:
x1, y1, x2, y2 = X["bbox"]
others = [b["bbox"] for b in blocks if b is not X]
def _yband(c):
return not (c[3] <= y1 or c[1] >= y2)
def _xband(c):
return not (c[2] <= x1 or c[0] >= x2)
# right: nearest cell-left / wall to the right, else frame right
rb = [c[0] for c in others if _yband(c) and c[0] >= x2] \
+ [wx for wx, wy1, wy2 in _vwx if wx >= x2 and not (wy2 <= y1 or wy1 >= y2)]
nx2 = min([_frame_r] + rb) - _GAP
# left
lb = [c[2] for c in others if _yband(c) and c[2] <= x1] \
+ [wx for wx, wy1, wy2 in _vwx if wx <= x1 and not (wy2 <= y1 or wy1 >= y2)]
nx1 = max([_frame_l] + lb) + _GAP
# down
db = [c[1] for c in others if _xband(c) and c[1] >= y2]
ny2 = min([_frame_b] + db) - _GAP
# up
ub = [c[3] for c in others if _xband(c) and c[3] <= y1]
ny1 = max([_frame_t] + ub) + _GAP
# grow-only; keep current edge if the computed bound would shrink
gb = [min(x1, nx1), min(y1, ny1), max(x2, nx2), max(y2, ny2)]
# Clip-after-grow: independent edge growth can poke a CORNER into a diagonal
# neighbour. Retreat any grown edge back out of every other cell — but never below
# X's ORIGINAL box — so the grown box overlaps NOTHING (locked or open).
for c in others:
if gb[2] <= c[0] or gb[0] >= c[2] or gb[3] <= c[1] or gb[1] >= c[3]:
continue
if c[0] >= x2: gb[2] = min(gb[2], max(x2, c[0] - _GAP)) # neighbour to the right
elif c[2] <= x1: gb[0] = max(gb[0], min(x1, c[2] + _GAP)) # to the left
elif c[1] >= y2: gb[3] = min(gb[3], max(y2, c[1] - _GAP)) # below
elif c[3] <= y1: gb[1] = max(gb[1], min(y1, c[3] + _GAP)) # above
# else: c overlaps X's ORIGINAL box (pre-existing) — leave it to other passes
# STRICT GUARD: Step 4 must never increase overlap with a LOCKED (confident) cell.
# If the grown box would touch any locked cell more than the original did, abandon
# the growth entirely for this cell (revert to original).
def _ovl(p, q):
return max(0, min(p[2], q[2]) - max(p[0], q[0])) * max(0, min(p[3], q[3]) - max(p[1], q[1]))
orig = [x1, y1, x2, y2]
if any(b.get("_locked") and b["bbox"] is not X["bbox"]
and _ovl(gb, b["bbox"]) > _ovl(orig, b["bbox"]) + 1 for b in blocks if b is not X):
gb = orig
X["bbox"] = gb
# Step 5 — assign every REMAINING (unassigned) content region to a cell: the cell whose
# box CONTAINS the region's centre (smallest such). This catches regions a raw section
# rectangle covers visually but never recorded as members, and any stray content sitting
# inside an existing crop. Containment-only ⇒ the receiving box already covers it, so no
# box grows and no new overlap is created. Masthead-band / far furniture is left out.
_CONTENT5 = {"text", "image", "figure_title", "doc_title", "paragraph_title"}
_assigned5 = set()
for b in blocks:
_assigned5.update(b["members"])
for r in regions:
if r["id"] in _assigned5 or r.get("type") not in _CONTENT5:
continue
rb = r["bbox"]; rcx = (rb[0] + rb[2]) / 2; rcy = (rb[1] + rb[3]) / 2
if rcy < _frame_t - 5:
continue # above the content frame → furniture
cont = [b for b in blocks
if b["bbox"][0] <= rcx <= b["bbox"][2] and b["bbox"][1] <= rcy <= b["bbox"][3]]
if not cont:
continue # not inside any crop → leave (nearest-assign skipped: bbox-grow risk)
tgt = min(cont, key=lambda b: (b["bbox"][2] - b["bbox"][0]) * (b["bbox"][3] - b["bbox"][1]))
tgt["members"] = sorted(set(tgt["members"]) | {r["id"]})
_assigned5.add(r["id"])
# ── Bind ORPHAN PHOTOS — an image may NEVER stand alone; it must belong to an article. ──────
# An orphan block (no doc_title/paragraph_title headline AND no dateline) that is PHOTO-DOMINANT
# (its image area ≥ 50% of the block, or it is image-only) is a lead/inline photo that lost its
# article (common on Sakshi, whose headlines PaddleOCR often misses — JG R28, Sakshi pg2 photo
# row). Attach it to the best article: the one whose top sits just BELOW the photo in the same
# column (the photo is that article's lead photo), else just above, else the nearest
# column-overlapping article. A TEXT/body orphan is LEFT ALONE — an article may be orphan, only
# a photo may not.
def _is_article(b):
return any(_rl.get(m, {}).get("type") in ("doc_title", "paragraph_title")
for m in b["members"]) or any(m in _dl_l for m in b["members"])
_articles = [b for b in blocks if _is_article(b)
and not b.get("_furniture") and not b.get("_legal")] # photos bind to NEWS, not furniture
# A photo must not bind ACROSS a horizontal rule that is longer than a caption (>0.6 col): such
# a rule is an article separator (the user's cross-paper line rule). Used to keep e.g. a Sakshi
# top "student photo" from binding down through full-width rules into an unrelated story.
_op_hbars = [L for L in (sep_lines or []) if not L.get("vert") and "y" in L]
_op_tw = sorted(r["bbox"][2] - r["bbox"][0] for r in regions if r.get("type") == "text")
_op_col = _op_tw[len(_op_tw) // 2] if _op_tw else 600
def _sep_between(p, q):
lo, hi = (p[3], q[1]) if q[1] >= p[3] else (q[3], p[1])
if hi <= lo:
return False # boxes overlap vertically → no gap rule
return any(lo - 5 < L["y"] < hi + 5 and (L["x2"] - L["x1"]) > 0.6 * _op_col
and (min(p[2], L["x2"]) - max(p[0], L["x1"])) >= 0.3 * (p[2] - p[0])
for L in _op_hbars)
if _articles:
_ph_absorbed = []
for ob in blocks:
if _is_article(ob) or ob.get("_furniture") or ob.get("_legal") or not ob["members"]:
continue
imgs = [m for m in ob["members"] if _rl.get(m, {}).get("type") == "image"]
if not imgs:
continue # text-only orphan → an article may orphan
ox1, oy1, ox2, oy2 = ob["bbox"]
oa = max(1, (ox2 - ox1) * (oy2 - oy1))
ia = sum(max(0, _rl[m]["bbox"][2] - _rl[m]["bbox"][0])
* max(0, _rl[m]["bbox"][3] - _rl[m]["bbox"][1]) for m in imgs)
body = [m for m in ob["members"] if _rl.get(m, {}).get("type") == "text"]
if body and ia < 0.5 * oa:
continue # has BODY text and not image-dominant = article → leave
# else: image-only, or image + caption (figure_title) only, or image-dominant → a PHOTO
def _col(a):
return (min(ox2, a["bbox"][2]) - max(ox1, a["bbox"][0])) >= 0.4 * (ox2 - ox1)
# ADJACENT article (small gap ≤200px): the photo is its lead/inline photo — bind even if
# a rule sits between (that rule is the article's own headline border, not a separator).
below = [a for a in _articles if _col(a) and oy2 - 40 <= a["bbox"][1] <= oy2 + 200]
above = [a for a in _articles if _col(a) and oy1 - 200 <= a["bbox"][3] <= oy1 + 40]
if below:
tgt = min(below, key=lambda a: a["bbox"][1] - oy2) # lead photo: article just below
elif above:
tgt = min(above, key=lambda a: oy1 - a["bbox"][3]) # inline/tail photo just above
else:
# FAR fallback: nearest column-overlapping article, but NEVER across a separator rule
# (>0.6 col) — that is what kept the Sakshi student photo R52 out of R118 (a story two
# full-width rules + a page of white space away).
colov = [a for a in _articles if _col(a) and not _sep_between(ob["bbox"], a["bbox"])]
tgt = min(colov, key=lambda a: abs((a["bbox"][1] + a["bbox"][3]) / 2
- (oy1 + oy2) / 2)) if colov else None
if tgt is None:
continue
tgt["members"] = sorted(set(tgt["members"]) | set(ob["members"]))
tgt["bbox"] = [min(tgt["bbox"][0], ox1), min(tgt["bbox"][1], oy1),
max(tgt["bbox"][2], ox2), max(tgt["bbox"][3], oy2)]
_ph_absorbed.append(id(ob))
print(f" 🖼 Orphan photo: R{ob['anchor_id']} → article R{tgt['anchor_id']} "
f"(an image must belong to an article)")
if _ph_absorbed:
blocks = [b for b in blocks if id(b) not in _ph_absorbed]
# ── SAKSHI: fold a text-only CONTINUATION column into the boxed article beside it. ──
# Sakshi prints multi-column boxed briefs with no internal rules; when the right
# column carries no headline/dateline of its own, the clusterer leaves it as a text
# ORPHAN (pg1 R18 = the right half of the R79 box — its first word even continues
# mid-word from the left column). Merge a text-only orphan into an article block
# sitting DIRECTLY beside it with the SAME vertical band: that is the other column
# of the same box, never a different story (a story in the next column starts at
# its own y and has its own anchor, so the band test rejects it).
if _ACTIVE_PAPER == "sakshi":
_cont_done = []
for ob in blocks:
if (id(ob) in _cont_done or ob.get("_furniture") or ob.get("_legal")
or _is_article(ob) or not ob["members"]):
continue
if any(_rl.get(m, {}).get("type") == "image" for m in ob["members"]):
continue # photos are handled by the photo bind
ox1, oy1, ox2, oy2 = ob["bbox"]
oh = max(1, oy2 - oy1)
best = None
for tb in blocks:
if (tb is ob or id(tb) in _cont_done or not _is_article(tb)
or tb.get("_furniture") or tb.get("_legal")):
continue
tx1, ty1, tx2, ty2 = tb["bbox"]
gap = (ox1 - tx2) if ox1 >= tx2 else (tx1 - ox2)
if not (-20 <= gap <= 80):
continue # must be the directly adjacent column
vov = min(oy2, ty2) - max(oy1, ty1)
if abs(ty1 - oy1) > 90 or vov < 0.8 * min(oh, max(1, ty2 - ty1)):
continue # same vertical band = same box
fx1, fx2 = (tx2, ox1) if ox1 >= tx2 else (ox2, tx1)
if any(fx1 - 6 <= (L["x"] if "x" in L else (L["x1"] + L["x2"]) // 2) <= fx2 + 6
and (min(max(L.get("y1", 0), L.get("y2", 0)), oy2)
- max(min(L.get("y1", 0), L.get("y2", 0)), oy1)) >= 0.5 * vov
for L in vsep):
continue # a drawn wall between → different stories
if best is None or abs(ty1 - oy1) < abs(best["bbox"][1] - oy1):
best = tb
if best is not None:
best["members"] = sorted(set(best["members"]) | set(ob["members"]))
best["bbox"] = [min(best["bbox"][0], ox1), min(best["bbox"][1], oy1),
max(best["bbox"][2], ox2), max(best["bbox"][3], oy2)]
_cont_done.append(id(ob))
print(f" ⇤ Sakshi continuation column: orphan R{ob['anchor_id']} folded into "
f"R{best['anchor_id']} (same band, adjacent column)")
if _cont_done:
blocks = [b for b in blocks if id(b) not in _cont_done]
# ── SAKSHI: re-home a STRAY IMAGE a wide banner anchor claimed by the column rule. ──
# A synthesized 3-column headline column-overlaps everything beneath it, so a small
# inline image far below — whose own narrow article headline does NOT column-overlap
# it — falls to the banner (JG pg1 R44: the R50 story's 'సాక్షి ఎఫెక్ట్' clipping fell
# to the SIR banner 1100px above). If an image member sits vertically DETACHED from
# the rest of its block and fully INSIDE another article's box, it belongs there.
# Also enabled for NT (JG pg1: the heat feature's photos/continuation fell into the
# H28 story 2200px below; the same re-home family cures it). AJ stays untouched.
if _ACTIVE_PAPER in ("sakshi", "namaste_telangana"):
for B in blocks:
if B.get("_raw_rect") or B.get("_section") or B.get("_furniture"):
continue
for m in [x for x in list(B["members"]) if _rl.get(x, {}).get("type") == "image"]:
mb = _rl[m]["bbox"]
# The photo's own caption travels WITH it: a figure_title directly
# below (≤90px, ≥40% overlap) is part of the unit — and must be
# excluded from the envelope test, or a stolen photo+caption pair at
# a block's bottom never reads as detached (pg2 R47+R115: the dead
# farmer's portrait stuck to the briefs column above his story).
cap = next((x for x in B["members"]
if _rl.get(x, {}).get("type") == "figure_title"
and 0 <= _rl[x]["bbox"][1] - mb[3] <= 90
and (min(_rl[x]["bbox"][2], mb[2]) - max(_rl[x]["bbox"][0], mb[0]))
>= 0.4 * (mb[2] - mb[0])), None)
unit = [m] + ([cap] if cap is not None else [])
ubox = [min(_rl[u]["bbox"][0] for u in unit), min(_rl[u]["bbox"][1] for u in unit),
max(_rl[u]["bbox"][2] for u in unit), max(_rl[u]["bbox"][3] for u in unit)]
rest = [_rl[x]["bbox"] for x in B["members"] if x not in unit and x in _rl]
if not rest:
continue
ub = [min(b[0] for b in rest), min(b[1] for b in rest),
max(b[2] for b in rest), max(b[3] for b in rest)]
if not (ubox[1] >= ub[3] - 10 or ubox[3] <= ub[1] + 10):
continue # unit inside its block's envelope → fine
for C in blocks:
if C is B or C.get("_furniture") or C.get("_legal") or not _is_article(C):
continue
cb = C["bbox"]
if (cb[0] - 15 <= ubox[0] and ubox[2] <= cb[2] + 15
and cb[1] - 15 <= ubox[1] and ubox[3] <= cb[3] + 15):
for u in unit:
B["members"].remove(u)
C["members"] = sorted(set(C["members"]) | set(unit))
B["bbox"] = ub
C["bbox"] = [min(cb[0], ubox[0]), min(cb[1], ubox[1]),
max(cb[2], ubox[2]), max(cb[3], ubox[3])]
print(f" 🖼 Sakshi stray image re-home: R{unit} "
f"R{B['anchor_id']} → R{C['anchor_id']} (inside its box)")
break
# A CAPTION (figure_title) belongs with the photo it abuts. When the photo
# folded into one block but its caption into another (the dance-photo caption
# R104 fell 3800px down its column into the statue story H106, stretching that
# crop page-tall), move the caption to the photo's block and shrink the donor
# back to its remaining members.
_cap_owner = {}
for Bc in blocks:
for m in Bc["members"]:
_cap_owner[m] = Bc
for B in blocks:
if B.get("_raw_rect") or B.get("_section") or B.get("_furniture"):
continue
for m in [x for x in list(B["members"])
if _rl.get(x, {}).get("type") == "figure_title"]:
mb = _rl[m]["bbox"]
img = next((q for q in regions if q.get("type") == "image"
# -30 tolerance: PaddleOCR's photo box often overlaps the
# caption strip by a few px (new-run R104 vs R44: -11px)
and (-30 <= mb[1] - q["bbox"][3] <= 90 # caption below photo
or -30 <= q["bbox"][1] - mb[3] <= 90) # caption above photo
and (min(mb[2], q["bbox"][2]) - max(mb[0], q["bbox"][0]))
>= 0.4 * (mb[2] - mb[0])), None)
if img is None:
continue
C = _cap_owner.get(img["id"])
if C is None or C is B or C.get("_furniture") or C.get("_legal"):
continue
B["members"].remove(m)
C["members"] = sorted(set(C["members"]) | {m})
_cap_owner[m] = C
rest = [_rl[x]["bbox"] for x in B["members"] if x in _rl]
if rest:
B["bbox"] = [min(b[0] for b in rest), min(b[1] for b in rest),
max(b[2] for b in rest), max(b[3] for b in rest)]
cb = C["bbox"]
C["bbox"] = [min(cb[0], mb[0]), min(cb[1], mb[1]),
max(cb[2], mb[2]), max(cb[3], mb[3])]
print(f" 🏷 Sakshi stray caption re-home: R{m} "
f"R{B['anchor_id']} → R{C['anchor_id']} (its photo R{img['id']} lives there)")
# A member sitting entirely ABOVE its own block's HEADLINE is never that
# article's content unless it is title-typed (kicker/deck) — body text, photos
# and captions up there are the tail of the story above that fell down-column
# to this anchor (Sakshi pg2: the NPDCL story's last lines into H100; NT JG
# pg1: the heat feature's vendor/ice-cream photos + continuation column into
# H28, 2200px above its 'ఆలస్యంపై రైతుల ఆగ్రహం' headline). If another
# article's box contains it, move it there and shrink the donor.
for B in blocks:
if (B.get("kind") != "H" or B.get("_raw_rect") or B.get("_section")
or B.get("_furniture")):
continue
_ab = _rl.get(B["anchor_id"], {}).get("bbox")
if not _ab:
continue
for m in [x for x in list(B["members"])
if _rl.get(x, {}).get("type") in ("text", "image", "figure_title")
and _rl[x]["bbox"][3] <= _ab[1] + 10]:
mb = _rl[m]["bbox"]
host = None
for C in blocks:
if C is B or C.get("_furniture") or C.get("_legal") or not _is_article(C):
continue
cb = C["bbox"]
if (cb[0] - 15 <= mb[0] and mb[2] <= cb[2] + 15
and cb[1] - 15 <= mb[1] and mb[3] <= cb[3] + 15):
if host is None or ((cb[2] - cb[0]) * (cb[3] - cb[1])
< (host["bbox"][2] - host["bbox"][0])
* (host["bbox"][3] - host["bbox"][1])):
host = C
if host is None:
continue
B["members"].remove(m)
host["members"] = sorted(set(host["members"]) | {m})
rest = [_rl[x]["bbox"] for x in B["members"] if x in _rl]
if rest:
B["bbox"] = [min(b[0] for b in rest), min(b[1] for b in rest),
max(b[2] for b in rest), max(b[3] for b in rest)]
hb = host["bbox"]
host["bbox"] = [min(hb[0], mb[0]), min(hb[1], mb[1]),
max(hb[2], mb[2]), max(hb[3], mb[3])]
print(f" 📄 Sakshi stray text re-home: R{m} above R{B['anchor_id']}'s "
f"headline → R{host['anchor_id']} (inside its box)")
# ── NT LATE structural-rule cut ──────────────────────────────────────────────
# The mid-pipeline rule-split runs BEFORE the grow / containment-merge / orphan-
# photo passes, so a block that only became page-tall THROUGH those later passes
# is never re-checked (NT Adilabad pg1: DL5 swallowed the flood feature and the
# bottom band across a 4.4-column rule). Re-cut FINAL blocks at structural rules:
# ≥1.8 col wide, spanning the block, not a caption underline, and either a new
# article start just below or simply ≥2.2 col (a rule that wide IS a band divider
# on NT per the line study). Locked / raw-rect / section / furniture stay whole.
if _ACTIVE_PAPER == "namaste_telangana":
_lt_tw = sorted(r["bbox"][2] - r["bbox"][0] for r in regions if r.get("type") == "text")
_lt_col = _lt_tw[len(_lt_tw) // 2] if _lt_tw else 600
_lt_h = sorted([L for L in (sep_lines or []) if not L.get("vert") and "y" in L],
key=lambda L: L["y"])
_lt_new, _lt_done = [], []
for blk in list(blocks):
if (blk.get("_raw_rect") or blk.get("_section") or blk.get("_legal")
or blk.get("_furniture") or blk.get("_locked")):
continue
x1, y1, x2, y2 = blk["bbox"]; bw = max(1, x2 - x1)
mems = [m for m in (set(blk["members"]) | {blk["anchor_id"]}) if m in _rl]
if len(mems) < 2:
continue
starts_y = [_rl[m]["bbox"][1] for m in mems
if _rl[m].get("type") in ("doc_title", "paragraph_title") or m in _dl_l]
cuts = [L["y"] for L in _lt_h
if y1 + 80 < L["y"] < y2 - 80
and (L["x2"] - L["x1"]) >= 1.8 * _lt_col
and (min(x2, L["x2"]) - max(x1, L["x1"])) >= 0.6 * bw
and not _line_is_underline(L["y"], L["x1"], L["x2"], regions)
and (any(L["y"] - 10 <= sy <= L["y"] + 170 for sy in starts_y)
or (L["x2"] - L["x1"]) >= 2.2 * _lt_col)]
if not cuts:
continue
bounds = [y1 - 1] + sorted(set(cuts)) + [y2 + 1]
bands = []
for i in range(len(bounds) - 1):
lo, hi = bounds[i], bounds[i + 1]
bm = [m for m in mems if lo <= (_rl[m]["bbox"][1] + _rl[m]["bbox"][3]) / 2 < hi]
if bm:
bands.append(bm)
if len(bands) < 2:
continue
for bm in bands:
bxs = [_rl[m]["bbox"] for m in bm]
heads = [m for m in bm if _rl[m].get("type") in ("doc_title", "paragraph_title")]
anch = (min(heads, key=lambda m: _rl[m]["bbox"][1]) if heads
else min(bm, key=lambda m: _rl[m]["bbox"][1]))
_lt_new.append({"anchor_id": anch, "kind": ("H" if heads else blk.get("kind", "O")),
"bbox": [min(b[0] for b in bxs), min(b[1] for b in bxs),
max(b[2] for b in bxs), max(b[3] for b in bxs)],
"members": sorted(bm), "_locked": True})
_lt_done.append(id(blk))
print(f" ─ NT late rule-split: R{blk['anchor_id']} → {len(bands)} bands "
f"at structural rules {sorted(set(cuts))}")
if _lt_done:
blocks = [b for b in blocks if id(b) not in _lt_done] + _lt_new
blocks.sort(key=lambda blk: (blk["bbox"][1], blk["bbox"][0]))
return blocks
def draw_all_article_boundaries_debug(page_img_path, regions, pages_dir, page_num,
dateline_starts=None, sep_lines=None):
"""Save a per-page image outlining EVERY candidate article block on the page
BEFORE the political filter runs (see cluster_all_article_blocks). Each block
gets a distinct colour and a '#<anchor_id> [H|DL|P] (<n regions>)' label — the
full set of articles the page contains, political and not, so you can see the
raw layout that the Stage-3 political filter then narrows down."""
try:
from PIL import ImageDraw, ImageFont
except Exception as e:
print(f" (all-article-boundaries debug image skipped: {e})")
return
blocks = cluster_all_article_blocks(regions, dateline_starts, sep_lines=sep_lines)
if not blocks:
return
try:
font = ImageFont.truetype("DejaVuSans-Bold.ttf", 34)
except Exception:
try:
font = ImageFont.load_default(size=34)
except Exception:
font = ImageFont.load_default()
img = Image.open(page_img_path).convert("RGB")
draw = ImageDraw.Draw(img)
for idx, blk in enumerate(blocks):
x1, y1, x2, y2 = blk["bbox"]
color = _ARTICLE_COLORS[idx % len(_ARTICLE_COLORS)]
draw.rectangle([x1, y1, x2, y2], outline=color, width=12)
label = f"#{blk['anchor_id']} [{blk['kind']}] ({len(blk['members'])})"
ty = max(0, y1 - 42)
draw.rectangle([x1, ty, x1 + len(label) * 17 + 14, ty + 40], fill=color)
draw.text((x1 + 7, ty + 4), label, fill="white", font=font)
out = str(pages_dir / f"page_{page_num:03d}_all_articles_debug.png")
img.save(out)
print(f" ALL-ARTICLE-BOUNDARIES DEBUG IMAGE saved: {out} "
f"({len(blocks)} article blocks before political filter)")
def draw_boundary_map_debug(page_img_path, regions, pages_dir, page_num,
dateline_starts=None, sep_lines=None):
"""PRE-CROP DEBUG MAP: draw every article block's BOUNDARY + its HEADER on a BLANK
white canvas (an 'empty page' map), so each job leaves a per-page picture of exactly
where the article boxes landed BEFORE cropping — to eyeball overlaps, gaps, the
masthead band, and which header anchors each box. White = empty body; solid bar =
the header/dateline anchor; coloured outline = the article boundary. No paid API."""
try:
from PIL import ImageDraw, ImageFont
except Exception as e:
print(f" (boundary-map debug image skipped: {e})")
return
try:
W, H = Image.open(page_img_path).size
except Exception as e:
print(f" (boundary-map debug image skipped: {e})")
return
blocks = cluster_all_article_blocks(regions, dateline_starts, sep_lines=sep_lines)
rmap = {r["id"]: r for r in regions}
dl_ids = {s["region_id"] for s in (dateline_starts or [])}
try:
font = ImageFont.truetype("DejaVuSans-Bold.ttf", 30)
except Exception:
try:
font = ImageFont.load_default(size=30)
except Exception:
font = ImageFont.load_default()
# Local Telugu OCR (best-effort) to print each article's actual HEADER text on its box.
_ocr = None
try:
import pytesseract
if "tel" in set(pytesseract.get_languages(config="")):
_ocr = pytesseract
except Exception:
_ocr = None
_hfont = font
for _tf in ("NotoSansTelugu-Regular.ttf",
"/System/Library/Fonts/Supplemental/Telugu Sangam MN.ttc",
"/System/Library/Fonts/KohinoorTelugu.ttc",
"/usr/share/fonts/truetype/noto/NotoSansTelugu-Regular.ttf"):
try:
_hfont = ImageFont.truetype(_tf, 30)
break
except Exception:
continue
src = Image.open(page_img_path).convert("RGB") if _ocr else None
def _header_text(blk):
"""OCR the article's header: the headline region for H/P, else the top strip of
the dateline body for DL — so every box shows what story it is."""
if not _ocr:
return ""
ar = rmap.get(blk["anchor_id"], {})
b = ar.get("bbox")
if not b:
return ""
strip_h = min(170, b[3] - b[1])
if strip_h <= 0:
return ""
try:
t = _ocr.image_to_string(src.crop((b[0], b[1], b[2], b[1] + strip_h)), lang="tel")
return " ".join(t.split())[:42]
except Exception:
return ""
img = Image.new("RGB", (W, H), "white")
draw = ImageDraw.Draw(img)
# Masthead / page-header band = everything above the topmost article box.
top_wall = min((b["bbox"][1] for b in blocks), default=0)
if top_wall > 4:
draw.rectangle([0, 0, W, top_wall], fill=(232, 232, 232))
draw.line([0, top_wall, W, top_wall], fill=(140, 140, 140), width=5)
draw.text((14, max(2, top_wall // 2 - 16)),
"MASTHEAD / page-header band", fill=(110, 110, 110), font=font)
for idx, blk in enumerate(blocks):
x1, y1, x2, y2 = blk["bbox"]
color = _ARTICLE_COLORS[idx % len(_ARTICLE_COLORS)]
# Furniture (classifieds / notice / unrelated content) — drawn as a hatched GREY section
# so it is visible on the map but clearly NOT an article (it is never saved as a crop).
if blk.get("_furniture"):
fc = (150, 150, 150)
for sx in range(x1, x2, 30):
draw.line([sx, y1, min(sx + 30, x2), y2], fill=fc, width=2) # diagonal hatch
draw.rectangle([x1, y1, x2, y2], outline=fc, width=6)
flab = f"FURNITURE #{blk['anchor_id']} ({len(blk['members'])})"
draw.rectangle([x1 + 5, y1 + 6, x1 + 16 + len(flab) * 14, y1 + 40], fill=fc)
draw.text((x1 + 11, y1 + 10), flab, fill="white", font=font)
continue
# header anchor: solid bar for H/P headline; outlined box for a DL dateline.
ar = rmap.get(blk["anchor_id"])
if ar and ar.get("type") in ("doc_title", "paragraph_title"):
draw.rectangle(ar["bbox"], fill=color)
elif ar and blk["anchor_id"] in dl_ids:
draw.rectangle(ar["bbox"], outline=color, width=6)
# article boundary — LOCKED (confident) cells get a solid box; OPEN cells dashed.
locked = blk.get("_locked")
if locked:
draw.rectangle([x1, y1, x2, y2], outline=color, width=8)
else:
for sx in range(x1, x2, 40): # dashed outline for OPEN cells
draw.line([sx, y1, min(sx + 22, x2), y1], fill=color, width=8)
draw.line([sx, y2, min(sx + 22, x2), y2], fill=color, width=8)
for sy in range(y1, y2, 40):
draw.line([x1, sy, x1, min(sy + 22, y2)], fill=color, width=8)
draw.line([x2, sy, x2, min(sy + 22, y2)], fill=color, width=8)
lock_tag = "LOCK " if locked else "open "
label = f"{lock_tag}#{blk['anchor_id']} [{blk['kind']}] ({len(blk['members'])})"
ty = max(top_wall, y1) + 6
draw.rectangle([x1 + 5, ty, x1 + 16 + len(label) * 15, ty + 36], fill=color)
draw.text((x1 + 11, ty + 3), label, fill="white", font=font)
# actual header text (OCR) on a white strip just below the id label
head = _header_text(blk)
if head:
hy = ty + 40
draw.rectangle([x1 + 5, hy, min(x2 - 4, x1 + 24 + len(head) * 16), hy + 38],
fill="white", outline=color, width=2)
draw.text((x1 + 11, hy + 4), head, fill=(20, 20, 20), font=_hfont)
out = str(pages_dir / f"page_{page_num:03d}_boundary_map_debug.png")
img.save(out)
print(f" BOUNDARY-MAP DEBUG IMAGE saved: {out} "
f"({len(blocks)} article boxes on blank canvas)")
def log_identification_breakdown(page_img_path, regions, dateline_starts,
political, rejected, page_num, sep_lines=None):
"""Per-page log: the headlines CLAUDE identified (political + rejected) vs the
article blocks the CODE identified geometrically (cluster_all_article_blocks),
with each code block's anchor heading read by local Telugu OCR. No paid API.
Lets you eyeball, per page, what the model saw vs what the layout logic carved."""
blocks = cluster_all_article_blocks(regions, dateline_starts, sep_lines=sep_lines)
rmap = {r["id"]: r for r in regions}
# Local OCR for the code-block anchor heading (best-effort; regions store no text).
_ocr = None
try:
import pytesseract
if "tel" in set(pytesseract.get_languages(config="")):
_ocr = pytesseract
except Exception:
_ocr = None
page_img = Image.open(page_img_path) if _ocr else None
def anchor_heading(b):
if not _ocr:
return ""
sh = min(180, b[3] - b[1])
if sh <= 0:
return ""
try:
t = _ocr.image_to_string(
page_img.crop((b[0], b[1], b[2], b[1] + sh)), lang="tel")
return " ".join(t.split())[:60]
except Exception:
return ""
claude_total = len(political) + len(rejected)
print(f"\n{'='*70}")
print(f"PAGE {page_num} — IDENTIFICATION BREAKDOWN")
print(f"{'='*70}")
print(f" CLAUDE IDENTIFIED COUNT : {claude_total} "
f"({len(political)} political + {len(rejected)} rejected)")
for i, a in enumerate(political, 1):
print(f" P{i:<2} {a.get('headline_telugu','?')}")
for i, r in enumerate(rejected, 1):
print(f" R{i:<2} {r.get('headline','?')}")
print(f" CODE IDENTIFIED COUNT : {len(blocks)}")
for i, blk in enumerate(blocks, 1):
a = rmap.get(blk["anchor_id"], {})
head = anchor_heading(a.get("bbox", [0, 0, 0, 0])) or "(no OCR text)"
print(f" {i:<2} [{blk['kind']:<2}] R{blk['anchor_id']:<3} {head}")
print(f"{'='*70}\n")
def check_dateline_crop_integrity(regions, dateline_starts, page_num, sep_lines=None):
"""Dateline-vs-crop integrity check (deterministic, no paid API).
Rule: every OCR-detected dateline is a guaranteed article (the minimum-count
rule), and a properly cropped article carries BOTH its headline and its
dateline body. This flags all-article blocks that violate that — i.e. crops
that are split/incomplete:
• INCOMPLETE — a block anchored on a dateline body with NO detected headline
(kind 'DL'); its headline + photo most likely got cropped into a neighbour.
• MERGED — one block holds 2+ datelines (multiple articles in one crop).
Returns a list of dicts {page, anchor_id, kind, reason}; also prints a summary."""
from collections import defaultdict
blocks = cluster_all_article_blocks(regions, dateline_starts, sep_lines=sep_lines)
dl_ids = {s["region_id"] for s in (dateline_starts or [])}
rtype = {r["id"]: r.get("type") for r in regions}
title_types = ("doc_title", "paragraph_title", "figure_title", "image")
per_block_dls = defaultdict(list)
for blk in blocks:
for mid in blk["members"]:
if mid in dl_ids:
per_block_dls[blk["anchor_id"]].append(mid)
issues = []
for blk in blocks:
a = blk["anchor_id"]
if blk["kind"] == "DL":
# After grow-up, a DL block that recovered a headline/photo (a title or
# image member) is complete; flag only those still carrying none.
has_title = any(rtype.get(m) in title_types for m in blk["members"])
if not has_title:
issues.append({"page": page_num, "anchor_id": a, "kind": "INCOMPLETE",
"reason": "dateline body with NO headline/photo found "
"above it — crop is just the body"})
dls = per_block_dls.get(a, [])
if len(dls) >= 2:
issues.append({"page": page_num, "anchor_id": a, "kind": "MERGED",
"reason": f"{len(dls)} datelines in one crop "
f"(regions {sorted(dls)}) — articles merged"})
n_dl = len(dl_ids)
if issues:
print(f" ⚠ DATELINE/CROP INTEGRITY (page {page_num}): {len(issues)} issue(s) "
f"across {len(blocks)} blocks / {n_dl} datelines")
for it in issues:
print(f" [{it['kind']}] block R{it['anchor_id']}: {it['reason']}")
else:
print(f" ✅ DATELINE/CROP INTEGRITY (page {page_num}): every dateline maps "
f"to a headline-bearing crop ({len(blocks)} blocks / {n_dl} datelines)")
return issues
def crop_all_article_blocks(page_img_path, regions, out_base_dir, page_num,
dateline_starts=None, pad=12, sep_lines=None,
only_anchor_ids=None, overlay=False):
"""Crop EVERY candidate article block (no political filtering) to its own image
so each page's articles can be eyeballed in isolation. Writes one PNG per block
into <out_base_dir>/page_<NNN>/ named a<order>_<kind><anchor_id>.png (order
is top-to-bottom on the page). Uses the same clustering as the debug overlay.
When `only_anchor_ids` is given, only blocks with those anchors are SAVED, but
every other block is still whited out of the saved crops (so a dateline crop
stays clean of neighbouring orphan/other articles)."""
from PIL import ImageDraw
import numpy as np
from _lines import _faint_grey_band_below
blocks = cluster_all_article_blocks(regions, dateline_starts, sep_lines=sep_lines)
if not blocks:
return 0
page_dir = Path(out_base_dir) / f"page_{page_num:03d}"
page_dir.mkdir(parents=True, exist_ok=True)
img = Image.open(page_img_path).convert("RGB")
W, H = img.size
grayarr = np.asarray(img.convert("L"))
rmap = {r["id"]: r for r in regions}
owner = {}
for blk in blocks:
for mid in blk["members"] + [blk["anchor_id"]]:
owner[mid] = blk["anchor_id"]
def _intersect(a, b):
ix = min(a[2], b[2]) - max(a[0], b[0])
iy = min(a[3], b[3]) - max(a[1], b[1])
return ix > 0 and iy > 0
# A block whose rectangle is no wider than ~1.5 columns sits in a single
# column, so its bbox IS the article's true boundary — crop it straight from
# the page (no white-out) to keep display headlines PaddleOCR never boxed.
# Wider multi-column blocks are L-shaped: their rectangle overlaps neighbour
# articles, so those neighbours still need whiting out.
NARROW_W = 1100
saved = 0
for order, blk in enumerate(blocks, 1):
if only_anchor_ids is not None and blk["anchor_id"] not in only_anchor_ids:
continue
if blk.get("_furniture"):
continue # classifieds / notice / unrelated furniture — NOT an article crop
# (still present in `blocks`, so it whites out of neighbour crops and
# shows as a section on the debug boundary map)
bx1, by1, bx2, by2 = blk["bbox"]
# Drop a DETACHED top banner photo. When the topmost member of a wide block
# is an image that does NOT span the article width AND a tall pure-white
# pixel gutter separates it from the content below, that photo belongs to a
# neighbouring story sharing the banner row — keeping it would leave a big
# blank corner (the neighbour's blanked column). Start the crop where the
# content resumes below the gutter instead.
mems = sorted((rmap[m] for m in blk["members"] + [blk["anchor_id"]]
if m in rmap), key=lambda r: r["bbox"][1])
if mems and mems[0].get("type") == "image" and (bx2 - bx1) > NARROW_W:
top = mems[0]["bbox"]
below = [r for r in mems if r["bbox"][1] >= top[3] - 2]
if (top[2] - top[0]) < 0.7 * (bx2 - bx1) and below:
ny = min(r["bbox"][1] for r in below)
white = (grayarr[top[3]:ny, bx1:bx2] > 235).mean(axis=1) >= 0.99
run = 0; resume = None
for i, w in enumerate(white):
if w:
run += 1
else:
if run >= 30:
resume = top[3] + i
run = 0
if resume is not None:
by1 = resume
x1 = max(0, bx1 - pad); y1 = max(0, by1 - pad)
x2 = min(W, bx2 + pad)
# If a faint grey separator rule sits just below the article (spanning its
# column width), cut the bottom edge precisely AT that line; otherwise keep
# the body-extent bottom.
band_y = _faint_grey_band_below(grayarr, blk["bbox"])
y2 = min(H, band_y) if band_y is not None else min(H, by2 + pad)
rect = (x1, y1, x2, y2)
crop = img.crop(rect).copy()
if not blk.get("_raw_rect"):
pristine = crop.copy()
draw = ImageDraw.Draw(crop)
def _wbox(b, gpad=0):
cx1 = max(0, b[0] - x1 - gpad); cy1 = max(0, b[1] - y1 - gpad)
cx2 = min(x2 - x1, b[2] - x1 + gpad); cy2 = min(y2 - y1, b[3] - y1 + gpad)
if cx2 > cx1 and cy2 > cy1:
draw.rectangle([cx1, cy1, cx2, cy2], fill=(255, 255, 255))
if (bx2 - bx1) > NARROW_W:
for ob in blocks:
if ob["anchor_id"] == blk["anchor_id"]:
continue
if _intersect(rect, ob["bbox"]):
_wbox(ob["bbox"], gpad=8)
for r in regions:
if r["id"] in owner:
continue
if _intersect(rect, r["bbox"]):
_wbox(r["bbox"], gpad=4)
else:
# NARROW single-column crop: the rect is the article's true
# boundary, but the ±pad margin (or a clustering overlap) pulls
# in slivers of NEIGHBOUR articles — sliced glyph columns at the
# crop edges, a foreign headline poking in at the bottom. White
# out regions OWNED by other blocks; leave un-owned ink alone
# (it may be this article's own display headline that PaddleOCR
# never boxed). Members are repasted below, so an overlapping
# foreign box can never erase this article's own content.
for r in regions:
oa = owner.get(r["id"])
if oa is None or oa == blk["anchor_id"]:
continue
if _intersect(rect, r["bbox"]):
_wbox(r["bbox"], gpad=2)
for mid in blk["members"] + [blk["anchor_id"]]:
r = rmap.get(mid)
if not r:
continue
b = r["bbox"]
cx1 = max(0, b[0] - x1); cy1 = max(0, b[1] - y1)
cx2 = min(x2 - x1, b[2] - x1); cy2 = min(y2 - y1, b[3] - y1)
if cx2 > cx1 and cy2 > cy1:
crop.paste(pristine.crop((cx1, cy1, cx2, cy2)), (cx1, cy1))
# Trim the outer white margin (the blank column gutter to the next
# article on the right, plus any white border on the other sides).
# A border row/column is trimmed unless ≥1% of its pixels are ink,
# so a faint sparse strip bleeding in from a neighbour is dropped
# while genuine text columns and full-width grey rules survive.
ca = np.asarray(crop.convert("L"))
ink = ca < 245
colf = ink.mean(axis=0)
rowf = ink.mean(axis=1)
cols = np.where(colf > 0.01)[0]
rows = np.where(rowf > 0.01)[0]
trim_x = trim_y = 0
if cols.size and rows.size:
trim_x, trim_y = int(cols.min()), int(rows.min())
crop = crop.crop((trim_x, trim_y,
int(cols.max()) + 1, int(rows.max()) + 1))
if overlay:
# REVIEW overlay: draw every member region on the crop so coverage can
# be eyeballed — blue = anchor, magenta = synthesized headline band,
# green = other members. (The paid pipeline saves plain crops; only the
# replay/review path passes overlay=True.)
od = ImageDraw.Draw(crop)
offx, offy = x1 + trim_x, y1 + trim_y
for mid in blk["members"] + [blk["anchor_id"]]:
r = rmap.get(mid)
if not r:
continue
b = r["bbox"]
box = [b[0] - offx, b[1] - offy, b[2] - offx, b[3] - offy]
if box[2] <= 0 or box[3] <= 0 or box[0] >= crop.width or box[1] >= crop.height:
continue
colr = ((200, 0, 160) if r.get("_synth_band")
else (0, 90, 220) if mid == blk["anchor_id"]
else (0, 160, 0))
od.rectangle(box, outline=colr, width=4)
od.text((box[0] + 6, box[1] + 4),
f"R{mid} {r.get('type', '')[:14]}", fill=colr)
name = f"a{order:02d}_{blk['kind']}{blk['anchor_id']}.png"
crop.save(str(page_dir / name))
saved += 1
print(f" PER-ARTICLE CROPS saved: {page_dir} ({saved} article images)")
return saved
def _line_is_underline(ly, lx1, lx2, regions, band=45):
"""RULE: a horizontal rule with a photo/figure or its caption (figure_title)
sitting DIRECTLY ABOVE it is an UNDERLINE for that image/caption — NOT an
article separator. Such a line must never be used to split or bound articles
(Rule-B floor corroboration, banner bottom rule, etc.).
Returns True when an image / figure / figure_title region's BOTTOM edge lies
just above the line (within `band` px) and the line is roughly the WIDTH of
that image/caption. A real underline is drawn to the width of the photo it
sits under, so the line and the region span the same horizontal extent. A
full-width structural rule (e.g. a multi-column banner's bottom border) is
far wider than any single photo above it — it sticks out well past the
image on both sides — so it is NOT treated as an underline.
"""
lw = max(lx2 - lx1, 1)
for r in regions:
if str(r.get("type")) not in ("image", "figure", "figure_title"):
continue
b = r["bbox"]
if not (ly - band <= b[3] <= ly + 5): # region bottom hugs the line
continue
iw = max(b[2] - b[0], 1)
ov = min(b[2], lx2) - max(b[0], lx1)
# underline ⟺ image sits above the line AND the line is not much wider
# than the image (line width within 1.5× the image width). A banner
# bottom rule is several times wider than the photo above it -> skip.
if ov > 0.5 * min(iw, lw) and lw <= 1.5 * iw:
return True
return False
# distinct, high-contrast colours cycled per article boundary (RGB)
_ARTICLE_COLORS = [
(220, 30, 30), (30, 110, 220), (0, 160, 70), (210, 120, 0),
(150, 50, 200), (0, 170, 180), (200, 30, 130), (120, 90, 30),
(60, 60, 200), (90, 160, 0),
]
def draw_article_boundaries_debug(page_img_path, records, pages_dir, page_num):
"""Save a per-page image outlining the FINAL crop rectangle of every article
(after all geometric rules: foreign-headline clip, Rule-B floor, grow-right,
multi-column banner, and the calibration-strip bottom clamp). Each article
gets a distinct colour and a label 'p<pg>_a<id> [BANNER] <headline>'. This is
the ground-truth view of what each article crop actually contains.
`records` are the dicts returned by crop_political_articles (each carries the
final 'bbox'); when bbox is absent it is read back from the article info.json.
"""
try:
from PIL import ImageDraw, ImageFont
except Exception as e:
print(f" (article-boundaries debug image skipped: {e})")
return
page_recs = [r for r in records if r.get("page") == page_num]
if not page_recs:
return
try:
font = ImageFont.truetype("DejaVuSans-Bold.ttf", 34)
except Exception:
try:
font = ImageFont.load_default(size=34)
except Exception:
font = ImageFont.load_default()
img = Image.open(page_img_path).convert("RGB")
draw = ImageDraw.Draw(img)
for idx, rec in enumerate(sorted(page_recs, key=lambda r: r.get("article_id", 0))):
b = rec.get("bbox")
if b is None: # fall back to the stored info.json
try:
info = json.loads((Path(rec["dir"]) / "info.json").read_text())
b = info["bbox"]
except Exception:
continue
color = _ARTICLE_COLORS[idx % len(_ARTICLE_COLORS)]
draw.rectangle([b[0], b[1], b[2], b[3]], outline=color, width=14)
tag = rec.get("name", f"p{page_num:03d}_a{rec.get('article_id', idx + 1):03d}")
if rec.get("is_banner"):
tag += " [BANNER]"
hl = (rec.get("headline_telugu") or "")[:22]
label = f"{tag} {hl}" if hl else tag
ty = max(0, b[1] - 44)
draw.rectangle([b[0], ty, b[0] + len(label) * 17 + 16, ty + 42], fill=color)
try:
draw.text((b[0] + 8, ty + 4), label, fill="white", font=font)
except Exception:
draw.text((b[0] + 8, ty + 4), tag, fill="white", font=font)
out = str(pages_dir / f"page_{page_num:03d}_articles_debug.png")
img.save(out)
print(f" ARTICLE-BOUNDARIES DEBUG IMAGE saved: {out} ({len(page_recs)} articles)")
# ---------------------------------------------------------------------------
# Step 3: Claude — identify political articles + group regions (ONE call)
# ---------------------------------------------------------------------------
POLITICAL_PROMPT = """You are a senior opposition party strategist in Telangana state, India.
I am showing you a Telugu newspaper page image. I have also detected {num_regions} text/image regions on this page using OCR layout detection. Each region has an ID and precise pixel coordinates.
REGIONS:
{region_list}
{dateline_starts}
YOUR TASK: Look at the newspaper page and identify ONLY the politically significant articles. For each political article, tell me which region IDs belong to it so I can crop them together precisely.
For EACH article on the page, ask:
QUESTION 1: "Does this article describe actual DAMAGE, LOSS, DEATH, CORRUPTION, or SUFFERING happening to people in Telangana? And can the government (state OR central) be BLAMED for it?"
- Must be a NEGATIVE EVENT that already occurred — not a plan, announcement, or statistic.
- Must have a VICTIM — someone who died, lost money, was cheated, is suffering.
- Government (Telangana state OR central/BJP) can be blamed — for causing, failing to prevent, or failing to respond.
- Central government failures qualify IF actual harm to Telangana citizens.
- RESPONSIBILITY CHECK: If damage caused by nature, individual negligence, criminals, or private parties with NO government role → NO.
- If POSITIVE (new scheme, development, achievement) → NO.
- If PLAN or ANNOUNCEMENT → NO.
- If NO victim and NO damage → NO.
DO NOT RATIONALIZE A GOVERNMENT ANGLE THAT ISN'T REALLY THERE:
The proximate cause of the damage must clearly be a government act, a specific government policy, government inaction on a specific known issue, or systemic government failure. You may NOT stretch INDIVIDUAL CRIME articles to qualify by arguing things like:
- "The government should have provided better welfare for this community" (when the actual event was an individual crime)
- "The government should have prevented this" (when the actual cause was a private quarrel, criminal act, accident, or natural event with no government role)
- "This community suffers in general and the government doesn't help" (when the actual article is about ONE individual incident)
If you find yourself reaching for a generic "community welfare" or "should have done better" angle on an individual crime, REJECT it.
DO QUALIFY — ongoing policy-effect articles. These count as government damage even though they aren't a one-time "event":
- Petrol/diesel/LPG price hikes that hurt consumers, drivers, transporters (central government tax policy)
- Electricity tariff hikes that burden households (state policy)
- Procurement-center failures hurting farmers (state agriculture/civil-supplies policy)
- Scheme implementation failures: pension delays, ration shortages, scholarship non-payment (specific state/central scheme)
- Inflation/price rise of essential commodities tied to specific government policy
For these, "specific damage" can be a quantified ongoing burden (Rs.X per day, Rs.Y per family per month, N% increase) rather than a one-time incident. ACCEPT them as long as a named government policy or department is the cause and there are real victims with real numbers.
PROTESTS ARE EVIDENCE OF DAMAGE — QUALIFY THEM:
Farmer/citizen PROTESTS, rasta-rokos (road blockades), and dharnas held OVER A SPECIFIC GOVERNMENT FAILURE (procurement not happening, paddy/crops spoiling, unpaid dues, ration/pension/scholarship denial, water not released) DO QUALIFY. The protest is not "just a protest" — it is direct evidence that the underlying government failure has already harmed real victims (e.g. farmers blocking a road because grain has been sitting unprocured for a month and is sprouting). Do NOT reject these as "small", "brief", "minor", or "overlapping with another story". If it has its own headline, list it as its OWN article. (This is different from purely political/party rallies, organizational meetings, or student union protests with no specific government-caused damage — those still do NOT qualify.)
QUESTION 2: "Can the opposition party DIRECTLY USE this article to attack the government?"
Only include articles where BOTH answers are YES.
GROUPING RULES — READ CAREFULLY (this is critical for correct cropping):
- An "article" is ONE story occupying ONE contiguous rectangular block on the page (headline + its body text + its photo/caption directly under or beside that headline).
- A REAL ARTICLE has a main headline (OCR will label it as `doc_title`). Every entry you return in political_articles MUST include exactly one `doc_title` region in its member_region_ids — EXCEPT for the missed-headline articles flagged in the DATELINE SCAN below, which you anchor on their dateline body region instead.
- The member regions of a single article MUST be spatially adjacent — they touch or sit immediately next to each other. Use the bbox coordinates I gave you to verify.
- DO NOT merge regions from different columns of the page into one article unless they are literally the continuation of the same article (same headline, text flows column-to-column).
- DO NOT merge two visually separate stories just because they cover similar topics. If there are two headlines, that is TWO articles — list them separately (or reject one).
- If you are unsure whether two regions belong to the same article, KEEP THEM SEPARATE. Over-grouping is worse than under-grouping.
- Before finalizing each article, mentally draw the bounding box of all its member regions and ask: "Does this rectangle contain ONLY this one story, with no other articles inside it?" If not, remove the outlier regions.
ARTICLE BODY DATELINE — a strong signal for "this is where an article begins":
{dateline_rule}
SUB-BOXES AND QUOTE BOXES — DO NOT EMIT THESE AS SEPARATE ARTICLES:
Newspapers often embed sub-sections inside a larger article: victim testimonials, official statements, quote-boxes, infoboxes, "what they said" panels, side-stories. These typically have:
- A small `paragraph_title` heading instead of a big `doc_title` (no doc_title at all)
- A small portrait photo of one person plus a few sentences of text
- They sit INSIDE or DIRECTLY ADJACENT to the main article's bounding area (same column, same vertical zone, often in the rightmost narrow column of the parent article)
These are PART OF THE PARENT ARTICLE — include their regions in the parent's member_region_ids. DO NOT create a separate political_articles entry for them. If you cannot find a `doc_title` to anchor an "article", it is almost certainly a sub-box of another article — fold it in or drop it.
TELANGANA STATE politics only. Farmer issues are MOST IMPORTANT.
DO NOT SELECT:
- International news, other states news
- Government achievements, new schemes, welfare announcements, development projects
- Newspaper masthead, headers, ads, classifieds
- Sports, entertainment, cinema, horoscopes
- General statistics without actual damage
- Resignations (unless corruption/scam/fired/forced out)
- PM Modi speeches (unless specific policy failure hurting Telangana)
- Individual accidents (road, electric shock, drowning, fire) — only if MASS disaster (10+ deaths) or CLEAR systemic government negligence
- INDIVIDUAL CRIMES AND VIOLENCE: murders, assaults, robberies, kidnappings, domestic violence, family disputes, gang violence, personal enmity attacks, suicides — these are matters between private parties or criminals and the victim. REJECT them. They qualify ONLY if there is a clear, specific, named government failure that directly caused or enabled the event (e.g., police themselves committed the crime, a specific government policy directly produced the violence, or 10+ victims in one systemic incident). A generic "the community needs more government protection" or "welfare hasn't reached them" angle is NOT enough — REJECT.
- COMMUNITY-LEVEL GENERIC GRIEVANCES: articles that just describe a community's hardships in general without a specific named government action or inaction that caused a specific damage event — REJECT.
For each selected political article, return:
- "headline_telugu": exact Telugu headline from the page
- "headline_english": English translation
- "member_region_ids": list of region IDs that belong to this article (for precise cropping)
- "q1_reason": why this is government failure (1 sentence)
- "q2_reason": how opposition can use this (1 sentence)
- "attack_angle": what to say in press conference (1 sentence)
- "priority": "high" or "medium"
- "category": one of corruption, governance_failure, public_grievance, law_order, health, education, infrastructure, ruling_party_crisis, farmer_issues
Also return "rejected" list with brief reasoning for EVERY rejected article.
Return ONLY valid JSON:
{{
"political_articles": [...],
"rejected": [
{{"headline": "...", "reason": "..."}}
]
}}
If no political articles found, return {{"political_articles": [], "rejected": [...]}}"""
# --- Paper-specific dateline rules -----------------------------------------
# The dateline (the place-prefix that opens an article's body) is the most
# reliable "a new article starts here" signal, but its exact form differs by
# paper. We pick the rule from the PDF filename. Common shared rules (a new
# dateline forces a split; it overrides a mis-typed headline) are appended to
# every variant.
_DATELINE_COMMON = """A text region whose first words match this dateline pattern is the START of an article's body — it belongs to the headline directly above it.
Use the dateline as follows:
- A NEW dateline marks the START of a NEW article. If two datelines appear under what looks like one story, that is TWO articles — split them into separate entries, EVEN IF they cover the same topic and sit side by side.
- This OVERRIDES the region type label. PaddleOCR sometimes mislabels a real headline as `paragraph_title` instead of `doc_title`. If a heading sits directly above a fresh dateline, treat it as a real article headline and emit a SEPARATE article — do NOT fold it into a neighbour as a sub-box.
- A text region with NO dateline of its own (it continues the flow from the region above) is a continuation of the same article — keep it together.
- The sub-box / fold-in rule below applies ONLY to blocks that have NO dateline of their own."""
DATELINE_RULES = {
"sakshi": (
"This is a SAKSHI page. Sakshi opens each article body with just a place "
"name followed by a COLON (no date):\n"
" Examples: `జనగామ:` `జనగామ రూరల్:` `<Town>:`\n"
"The reliable marker is a short place-prefix ending in a colon \":\" at the "
"start of a body block.\n\n" + _DATELINE_COMMON
),
"andhra_jyothi": (
"This is an ANDHRA JYOTHI page. Andhra Jyothi opens each article body with "
"place + date + (ఆంధ్రజ్యోతి) + COLON:\n"
" Examples: `వరంగల్, మే 14 (ఆంధ్రజ్యోతి):` `<City/Town>, <Date> (ఆంధ్రజ్యోతి):`\n"
"The reliable marker is a prefix ending in `(ఆంధ్రజ్యోతి):` — a place, a "
"date, the masthead in brackets, then a colon.\n\n" + _DATELINE_COMMON
),
"namaste_telangana": (
"This is a NAMASTE TELANGANA page. Namaste Telangana opens each article body "
"with place + date (Telugu month + day, e.g. మే 19) + COLON, with NO masthead:\n"
" Examples: `వరంగల్, మే 19:` `జనగామ, మే 19:` `<Town>, <Telugu date>:`\n"
"The reliable marker is a short place-and-date prefix ending in a colon \":\" "
"(it contains a date but, unlike Andhra Jyothi, no `(ఆంధ్రజ్యోతి)` masthead).\n\n"
+ _DATELINE_COMMON
),
"unknown": (
"Telugu newspapers open each article body with a location dateline ending in "
"a colon. Two common formats:\n"
" - Andhra Jyothi: `<City/Town>, <Date> (ఆంధ్రజ్యోతి):`\n"
" - Sakshi: `<Town>:` (place name + colon, no date)\n"
"The reliable marker is a short place-prefix ending in a colon \":\" at the "
"start of a body block.\n\n" + _DATELINE_COMMON
),
}
def detect_paper(pdf_path):
"""Identify the newspaper from the PDF filename so we can pick its dateline rule."""
name = str(pdf_path).lower()
if "sakshi" in name:
return "sakshi"
if "namaste" in name or "namasthe" in name or "_nt_" in name or "nt_" in name:
return "namaste_telangana"
if "andhra" in name or "jyoth" in name or "_aj_" in name or "_aj" in name or "aj_" in name:
return "andhra_jyothi"
return "unknown"
def _build_dateline_block(dateline_starts):
"""Turn the deterministic dateline scan into prompt text that tells Claude how
many articles there are and which ones have a headline PaddleOCR missed."""
if not dateline_starts:
return ("DATELINE SCAN: (none available for this page — rely on the image and "
"region types.)")
lines = [
"DATELINE SCAN (deterministic, reliable): each region below OPENS its body with a "
"dateline, so each is a SEPARATE article. This is a MINIMUM article count — do NOT "
"return fewer political articles than the political ones among these:",
]
n_missed = 0
for s in sorted(dateline_starts, key=lambda z: z["bbox"][1]):
if s.get("headline_region"):
lines.append(f" - Region {s['region_id']} (dateline \"{s['dateline']}\"): headline "
f"already detected as region {s['headline_region']}.")
else:
n_missed += 1
lines.append(f" - Region {s['region_id']} (dateline \"{s['dateline']}\"): *** NO headline "
f"box detected *** — PaddleOCR MISSED this article's headline (it is large "
f"stylised text directly ABOVE region {s['region_id']}, which you CAN see in "
f"the image). Emit this as its OWN article anchored on region "
f"{s['region_id']}; read its headline from the image. Do NOT fold or drop it.")
if n_missed:
lines.append(f"IMPORTANT: {n_missed} article(s) above have NO detected headline box. They are "
f"real articles you must still consider — anchor each on its dateline body region.")
return "\n".join(lines)
def _extract_json_obj(text):
"""Pull the outermost JSON object out of a model response that may be empty,
wrapped in ``` / ```json fences, or padded with prose before/after the JSON.
Raises ValueError on an empty response; lets json.JSONDecodeError propagate
for genuinely malformed JSON."""
import re as _re
if not text or not text.strip():
raise ValueError("empty response")
t = text.strip()
# Prefer content inside a fenced block if one exists.
m = _re.search(r"```(?:json)?\s*(.*?)```", t, _re.DOTALL)
if m:
t = m.group(1).strip()
else:
t = t.replace("```json", "").replace("```", "").strip()
# Trim any leading/trailing prose around the object braces.
if not t.startswith("{"):
i = t.find("{")
if i != -1:
t = t[i:]
if not t.endswith("}"):
j = t.rfind("}")
if j != -1:
t = t[:j + 1]
if not t:
raise ValueError("no JSON object found in response")
return json.loads(t)
def _salvage_political_array(text):
"""Last-resort recovery when the whole JSON object will not parse — typically a
stray character somewhere in the (log-only) ``rejected`` list breaks the document
even though the ``political_articles`` array itself is well-formed. Locate just
that array by string-aware bracket matching and parse it on its own. Returns the
list of articles, or None if it cannot be recovered."""
import re as _re
if not text:
return None
m = _re.search(r'"political_articles"\s*:\s*\[', text)
if not m:
return None
start = m.end() - 1 # index of the opening '['
depth = 0
in_str = False
esc = False
for j in range(start, len(text)):
c = text[j]
if in_str:
if esc:
esc = False
elif c == "\\":
esc = True
elif c == '"':
in_str = False
continue
if c == '"':
in_str = True
elif c == "[":
depth += 1
elif c == "]":
depth -= 1
if depth == 0:
try:
arr = json.loads(text[start:j + 1])
return arr if isinstance(arr, list) else None
except json.JSONDecodeError:
return None
return None
def identify_and_group_political_articles(page_img_path, regions, page_num, paper="unknown",
dateline_starts=None):
api_key = os.environ.get("ANTHROPIC_API_KEY")
if not api_key:
print("ERROR: Set ANTHROPIC_API_KEY environment variable")
sys.exit(1)
import anthropic
client = anthropic.Anthropic(api_key=api_key)
# Resize image for API
img = Image.open(page_img_path)
max_w = 1500
if img.width > max_w:
ratio = max_w / img.width
img = img.resize((max_w, int(img.height * ratio)), Image.LANCZOS)
buf = BytesIO()
img.convert("RGB").save(buf, format="JPEG", quality=80)
img_data = base64.standard_b64encode(buf.getvalue()).decode("utf-8")
# Build region list
region_lines = []
for r in regions:
region_lines.append(f" ID {r['id']}: type={r['type']}, bbox=[{r['bbox'][0]}, {r['bbox'][1]}, {r['bbox'][2]}, {r['bbox'][3]}]")
region_list = "\n".join(region_lines)
dateline_rule = DATELINE_RULES.get(paper, DATELINE_RULES["unknown"])
prompt = POLITICAL_PROMPT.format(num_regions=len(regions), region_list=region_list,
dateline_rule=dateline_rule,
dateline_starts=_build_dateline_block(dateline_starts))
print(f" Sending page {page_num} to Claude Opus 4.8 ({len(regions)} regions)...")
def _call(strict=False):
ptext = prompt
if strict:
ptext += ("\n\nIMPORTANT: Return ONLY the raw JSON object described above — "
"no explanation, no markdown code fences, nothing before or after it.")
return client.messages.create(
model="claude-opus-4-8",
max_tokens=8192,
messages=[{"role": "user", "content": [
{"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": img_data}},
{"type": "text", "text": ptext},
]}],
)
def _text_of(resp):
return "".join(getattr(b, "text", "") for b in resp.content
if getattr(b, "type", None) == "text").strip()
dbg = Path(page_img_path).with_name(f"page_{page_num:03d}.badresponse.txt")
response = _call()
text = _text_of(response)
first_text = text
try:
result = _extract_json_obj(text)
except (ValueError, json.JSONDecodeError) as e:
try:
dbg.write_text(f"stop_reason={response.stop_reason}\n\n{text}")
except Exception:
pass
print(f" ⚠️ Page {page_num}: response was not valid JSON ({e}); "
f"stop_reason={response.stop_reason}. Retrying once (strict)...")
response = _call(strict=True)
text = _text_of(response)
try:
result = _extract_json_obj(text)
except (ValueError, json.JSONDecodeError) as e2:
try:
dbg.write_text(f"stop_reason={response.stop_reason}\n\n{text}")
except Exception:
pass
# Both parses failed outright. Before giving up, try to salvage just the
# political_articles array from either response — a stray char in the
# log-only `rejected` list should not cost us a whole page of articles.
salvaged = _salvage_political_array(first_text) or _salvage_political_array(text)
if salvaged is not None:
print(f" ♻️ Page {page_num}: full JSON was malformed but recovered "
f"{len(salvaged)} political article(s) from the partial response. "
f"Raw response saved to {dbg.name}")
return {"political_articles": salvaged, "rejected": []}
print(f" ❌ Page {page_num}: still no valid JSON after retry "
f"(stop_reason={response.stop_reason}). Treating page as having no "
f"political articles so the rest of the run continues. "
f"Raw response saved to {dbg.name}")
return {"political_articles": [], "rejected": []}
# Print reasoning
political = result.get("political_articles", [])
rejected = result.get("rejected", [])
print(f"\n=== PAGE {page_num} RESULTS ===")
print(f" Selected: {len(political)} political articles")
for i, art in enumerate(political, 1):
print(f" ✅ {i}. {art.get('headline_english', '?')}")
print(f" Regions: {art.get('member_region_ids', [])}")
print(f" Q1: {art.get('q1_reason', '')}")
print(f" Q2: {art.get('q2_reason', '')}")
print(f" Rejected: {len(rejected)} articles")
for r in rejected:
print(f" ❌ {r.get('headline', '?')[:50]} — {r.get('reason', '')[:60]}")
print(f"=== END PAGE {page_num} ===\n")
return result
# ---------------------------------------------------------------------------
# Safeguard: split articles whose member regions form spatially disjoint clusters.
#
# Claude sometimes bundles regions from different stories into one article
# (e.g. p001_a001 covering both "Petrol Shock" and the Indiramma housing article
# in the same group). Geometrically, those member regions form two or more
# disconnected clusters. This function detects that and splits them up.
# ---------------------------------------------------------------------------
def _bboxes_connected(a, b, gap=60):
"""Two bboxes are connected if they overlap or sit within `gap` pixels of each other."""
ax1, ay1, ax2, ay2 = a
bx1, by1, bx2, by2 = b
dx = max(0, max(bx1 - ax2, ax1 - bx2))
dy = max(0, max(by1 - ay2, ay1 - by2))
return dx <= gap and dy <= gap
def _cluster_regions(member_ids, region_map, gap=60):
"""Group region IDs into connected components via union-find on bbox proximity."""
ids = [rid for rid in member_ids if rid in region_map]
if not ids:
return []
parent = {rid: rid for rid in ids}
def find(x):
while parent[x] != x:
parent[x] = parent[parent[x]]
x = parent[x]
return x
def union(x, y):
rx, ry = find(x), find(y)
if rx != ry:
parent[rx] = ry
for i, a in enumerate(ids):
for b in ids[i + 1:]:
if _bboxes_connected(region_map[a]["bbox"], region_map[b]["bbox"], gap):
union(a, b)
clusters = {}
for rid in ids:
clusters.setdefault(find(rid), []).append(rid)
# Sort clusters by area (largest first) — main story should come first.
def area(group):
bs = [region_map[r]["bbox"] for r in group]
return (max(b[2] for b in bs) - min(b[0] for b in bs)) * \
(max(b[3] for b in bs) - min(b[1] for b in bs))
return sorted(clusters.values(), key=area, reverse=True)
# ---------------------------------------------------------------------------
# Headline verification via Claude vision (catches headline hallucinations).
#
# Claude sometimes claims a doc_title region contains one headline while the
# region actually contains a totally different article (e.g. labelling a
# US-Iran peace deal as "Deputy Collector's illegal assets"). The region IDs
# are honest but the text content is fabricated, so geometric safeguards
# cannot catch it. We crop the doc_title region and send JUST that crop to
# Claude with a tightly scoped prompt — "transcribe the Telugu text" — and
# compare to the headline Claude originally reported. PaddleOCR's Telugu
# model is unreliable, so we use Claude vision for this check.
# ---------------------------------------------------------------------------
def _normalize_telugu(s):
"""Strip whitespace and punctuation for comparison."""
if not s:
return ""
return "".join(ch for ch in s if ch.isalnum() or ord(ch) > 127)
def _claude_transcribe_region(client, page_img_path, bbox, pad=6):
"""Crop the bbox from the page image and ask Claude to transcribe its text."""
import anthropic
img = Image.open(page_img_path)
x1, y1, x2, y2 = bbox
crop = img.crop((max(0, x1 - pad), max(0, y1 - pad),
min(img.width, x2 + pad), min(img.height, y2 + pad)))
buf = BytesIO()
crop.convert("RGB").save(buf, format="JPEG", quality=90)
img_data = base64.standard_b64encode(buf.getvalue()).decode("utf-8")
prompt = (
"This image is a cropped newspaper headline region. "
"Transcribe the visible Telugu text EXACTLY as it appears. "
"Return ONLY the transcribed Telugu text, no English translation, "
"no explanation, no quotes. If the image is blank or unreadable, "
"return exactly the word: UNREADABLE"
)
resp = client.messages.create(
model="claude-haiku-4-5-20251001",
max_tokens=200,
messages=[{"role": "user", "content": [
{"type": "image", "source": {"type": "base64",
"media_type": "image/jpeg",
"data": img_data}},
{"type": "text", "text": prompt},
]}],
)
return resp.content[0].text.strip()
def verify_headlines_with_claude(political_articles, regions, page_img_path,
min_similarity=0.40):
"""
For each political article, crop the doc_title region Claude pointed to and
ask Claude (Haiku) to transcribe the actual Telugu text. Compare against
the headline Claude originally claimed.
If the claimed region's text doesn't match the claimed headline, SEARCH
every other doc_title region on the page for a match. If a different
doc_title's transcription matches the claimed headline, REMAP the article
onto that region (the downward-growth safeguard will then fill in body
regions). If no region matches, drop the article as hallucinated.
"""
if not political_articles:
return political_articles
api_key = os.environ.get("ANTHROPIC_API_KEY")
if not api_key:
print(" (Headline verification skipped — ANTHROPIC_API_KEY not set)")
return political_articles
try:
import anthropic
except ImportError:
print(" (Headline verification skipped — anthropic package not installed)")
return political_articles
client = anthropic.Anthropic(api_key=api_key)
region_map = {r["id"]: r for r in regions}
all_doc_titles = [r for r in regions if r.get("type") == "doc_title"]
# Spot oversized display banners (lead/section headlines set in large
# stylized type). OCR reliably mangles such headlines, so a low similarity
# there is NOT evidence of a hallucination and must not trigger a drop.
# The earlier median*1.8 rule was brittle: on a page with several big
# headers the median rises and a genuine banner (e.g. a 493px headline on
# a page whose median is 278) falls just under the bar. Use the SMALLEST
# doc_title (the typical body headline) as the baseline instead, plus an
# absolute floor, so a banner is caught regardless of what else is on the
# page.
_dt_heights = sorted(r["bbox"][3] - r["bbox"][1] for r in all_doc_titles)
_min_dt_h = _dt_heights[0] if _dt_heights else 0
_median_dt_h = _dt_heights[len(_dt_heights) // 2] if _dt_heights else 0
def _is_display_banner(rid):
h = region_map[rid]["bbox"][3] - region_map[rid]["bbox"][1]
if h < 220: # normal column headlines never qualify
return False
return (h >= 300 # absolute display floor
or (_min_dt_h and h >= 2.0 * _min_dt_h) # 2x the body headline
or (_median_dt_h and h >= 1.6 * _median_dt_h))
def _ocr_unreadable(txt):
# OCR failed to produce a usable transcription — treat as "can't verify",
# never as proof of a hallucination.
if not txt:
return True
t = txt.strip()
return t.upper() == "UNREADABLE" or len(_normalize_telugu(t)) < 6
# Cache transcriptions across articles so we don't re-transcribe the same
# doc_title region multiple times.
cache = {}
def transcribe(rid):
if rid in cache:
return cache[rid]
try:
txt = _claude_transcribe_region(client, page_img_path,
region_map[rid]["bbox"])
except Exception as e:
print(f" (transcription error for region {rid}: {e})")
txt = None
cache[rid] = txt
return txt
def sim_to_claim(rid, claim_norm):
actual = transcribe(rid)
if not actual or actual.strip().upper() == "UNREADABLE":
return 0.0, actual
return SequenceMatcher(None, _normalize_telugu(actual), claim_norm).ratio(), actual
kept = []
for art in political_articles:
member_ids = art.get("member_region_ids", []) or []
titles_in = [rid for rid in member_ids
if region_map.get(rid, {}).get("type") == "doc_title"]
claimed = (art.get("headline_telugu") or "").strip()
label = (art.get("headline_english") or "?")[:50]
if not titles_in or not claimed:
kept.append(art)
continue
primary_id = min(titles_in, key=lambda r: region_map[r]["bbox"][1])
claim_norm = _normalize_telugu(claimed)
if not claim_norm:
kept.append(art)
continue
sim, actual = sim_to_claim(primary_id, claim_norm)
if sim >= min_similarity:
print(f" ✓ headline verified: '{label}' at region {primary_id} (sim={sim:.2f})")
kept.append(art)
continue
# Wrapped-headline guard: OCR of the anchor often captures only the
# first line of a headline that wraps across several regions (e.g. the
# anchor reads "వంట గ్యాస్" while the full headline is "వంట గ్యాస్
# కొరతను నివారించాలి"), which scores low against the full claim. If the
# anchor's OCR is contained in the claim (or vice-versa), it IS the
# right headline — keep it and do NOT go hunting for another region to
# remap onto. Skipping this is what let the cooking-gas article get
# remapped onto the health headline.
actual_norm = _normalize_telugu(actual or "")
if (actual_norm and len(actual_norm) >= 5
and (actual_norm in claim_norm or claim_norm in actual_norm)):
print(f" ✓ headline partial-match kept: '{label}' at region {primary_id} "
f"(anchor OCR {actual_norm[:18]!r} is a fragment of the claim; sim={sim:.2f})")
kept.append(art)
continue
# Mismatch — search other doc_titles for the actual headline.
print(f" ⚠ headline mismatch for '{label}' at region {primary_id} "
f"(sim={sim:.2f}): claimed={claimed[:40]!r}, "
f"actual={(actual or '')[:40]!r}")
print(f" searching {len(all_doc_titles)-1} other doc_titles for the real region...")
best_id, best_sim, best_text = None, sim, actual
for r in all_doc_titles:
if r["id"] == primary_id:
continue
s, txt = sim_to_claim(r["id"], claim_norm)
if s > best_sim:
best_sim, best_id, best_text = s, r["id"], txt
# Remap needs a clear bar, but not so high that Telugu OCR noise on a
# GENUINE match (which routinely lands ~0.65-0.80, since OCR garbles a few
# characters) gets rejected. Unrelated headlines that merely share the
# "...చాలి / ...ించాలి" ending top out around ~0.52, so a 0.62 bar plus the
# requirement that the match clearly beats the wrong anchor (by >=0.15)
# accepts a real, OCR-degraded match while still rejecting suffix collisions.
remap_min = 0.62
if best_id is not None and best_sim >= remap_min and (best_sim - sim) >= 0.15:
print(f" ↻ REMAPPED '{label}' from region {primary_id} → region {best_id} "
f"(sim={best_sim:.2f}, text={(best_text or '')[:40]!r})")
# Start fresh with just the new doc_title; downward growth will
# fill in the article's body regions.
art["member_region_ids"] = [best_id]
art["_remapped_from"] = primary_id
art["_remapped_to"] = best_id
kept.append(art)
elif _is_display_banner(primary_id) or _ocr_unreadable(best_text):
# Either an oversized display banner, or OCR simply couldn't read
# the headline. In both cases the low similarity is an OCR failure,
# NOT evidence that Claude hallucinated the article — every article
# is anchored to a real doc_title region Claude saw in the layout.
# Dropping here is what made legit articles (e.g. the stylized
# "సూక్ష్మ సేద్యం.. తగ్గిన రాయితీ" banner) vanish between runs.
# Trust Claude's selection and keep the article.
reason = ("display banner too large for reliable OCR"
if _is_display_banner(primary_id)
else "OCR returned unreadable/garbage text")
h = region_map[primary_id]['bbox'][3] - region_map[primary_id]['bbox'][1]
print(f" 🛡 KEPT (unverified): '{label}' at region {primary_id} — {reason} "
f"(sim={best_sim:.2f}, h={h}px, min/median doc_title h={_min_dt_h}/{_median_dt_h}px); "
f"trusting Claude's selection")
art["_headline_unverified"] = True
kept.append(art)
else:
# OCR produced a confident, readable headline that clearly differs
# from the claim AND matches no other region — a real hallucination.
print(f" ✗ HALLUCINATION DROPPED: '{label}' — region {primary_id} OCRs to "
f"a different readable headline and no doc_title matches the claim "
f"(best sim={best_sim:.2f}, ocr={(best_text or '')[:40]!r})")
return kept
def _drop_blocked_by_other_doc_title(member_ids, region_map, all_regions,
col_overlap_threshold=0.3):
"""
Drop member regions that "belong" to a different article based on column-aware
headline blocking. For each non-headline member region R of this article:
Look at every OTHER doc_title D on the page (not this article's headline).
If D sits vertically between this article's headline and R, AND D's column
overlaps significantly with R's column, AND D's column does NOT overlap
with this article's headline → R is on the wrong side of D's headline and
most likely belongs to D's article. Drop R.
Also drop any doc_title region in `member_ids` that isn't this article's
primary (topmost) doc_title — a second headline is always wrong here.
"""
titles_in = [rid for rid in member_ids
if region_map.get(rid, {}).get("type") == "doc_title"]
if not titles_in:
return member_ids, []
primary_id = min(titles_in, key=lambda r: region_map[r]["bbox"][1])
primary_bbox = region_map[primary_id]["bbox"]
pt_x1, pt_y1, pt_x2, pt_y2 = primary_bbox
other_doc_titles = [r for r in all_regions
if r.get("type") == "doc_title"
and r["id"] != primary_id]
def col_overlap_ratio(a, b):
ov = max(0, min(a[2], b[2]) - max(a[0], b[0]))
return ov / max(1, min(a[2] - a[0], b[2] - b[0]))
kept, dropped = [], []
for rid in member_ids:
r = region_map.get(rid)
if r is None:
continue
# Always keep the primary headline.
if rid == primary_id:
kept.append(rid)
continue
# Drop any OTHER doc_title in the member list — it's another article's headline.
if r.get("type") == "doc_title":
dropped.append(rid)
continue
r_bbox = r["bbox"]
blocked = False
for ot in other_doc_titles:
ot_bbox = ot["bbox"]
# ot must sit vertically between the article's headline and R.
if not (pt_y2 < ot_bbox[1] and ot_bbox[3] <= r_bbox[3]):
continue
# ot's column must overlap R's column.
if col_overlap_ratio(ot_bbox, r_bbox) < col_overlap_threshold:
continue
# ot's column must NOT overlap this article's headline column —
# otherwise it could be a continuation, not a barrier.
if col_overlap_ratio(ot_bbox, primary_bbox) >= col_overlap_threshold:
continue
blocked = True
break
if blocked:
dropped.append(rid)
else:
kept.append(rid)
return kept, dropped
def _trim_above_headline(member_ids, region_map, tolerance=30):
"""
Drop member regions that sit entirely ABOVE the article's headline.
In a newspaper, an article's `doc_title` marks the TOP of the story —
anything whose bottom edge is above the headline belongs to the article
above and must not be included. `tolerance` allows a small overlap
(e.g. a decorative line or bullet that sits a few pixels above).
"""
titles = [
rid for rid in member_ids
if rid in region_map and region_map[rid].get("type") == "doc_title"
]
if not titles:
return member_ids, [] # no headline to anchor on — leave as-is
# Topmost headline = smallest y1
top_y1 = min(region_map[rid]["bbox"][1] for rid in titles)
boundary = top_y1 - tolerance
kept, dropped = [], []
for rid in member_ids:
r = region_map.get(rid)
if not r:
continue
# Always keep doc_title regions themselves.
if r.get("type") == "doc_title":
kept.append(rid)
continue
y2 = r["bbox"][3]
if y2 < boundary:
dropped.append(rid)
else:
kept.append(rid)
return kept, dropped
def _grow_article_downward(member_ids, region_map, all_regions, claimed_globally,
gap=50, x_tolerance=80, headline_continuation_gap=80,
dateline_starts=None):
"""
Pull in UNCLAIMED regions below the article's headline that are spatially
adjacent to the existing members. Bounded by:
- top: article's headline (with small tolerance)
- bottom: the next doc_title on the page (or page bottom)
- sides: the article's current horizontal extent (so we don't grow
into a neighboring column that belongs to another article)
Also handles multi-line headlines where OCR splits one headline into
multiple stacked doc_title regions: any doc_title sitting immediately
below the article's headline in the same column (gap <= 80 px) is treated
as a headline-continuation line, folded into the article, and NOT counted
as the next-article floor.
Claude sometimes under-includes body text regions; this catches them.
"""
titles_in = [rid for rid in member_ids
if region_map.get(rid, {}).get("type") == "doc_title"]
if not titles_in:
return member_ids, []
primary_id = min(titles_in, key=lambda r: region_map[r]["bbox"][1])
headline_y1 = region_map[primary_id]["bbox"][1]
# Horizontal extent of the article's current members — this defines the
# columns the article occupies. We allow `x_tolerance` slack on either side.
member_boxes = [region_map[rid]["bbox"] for rid in member_ids if rid in region_map]
art_x1 = min(b[0] for b in member_boxes) - x_tolerance
art_x2 = max(b[2] for b in member_boxes) + x_tolerance
def _shares_column(other_bbox):
return max(0, min(art_x2, other_bbox[2]) - max(art_x1, other_bbox[0])) > 0
def _shares_column_floor(other_bbox):
# A FLOOR (next article's headline/dateline) must genuinely sit in our
# columns — not merely clip the x_tolerance padding margin. A neighbouring
# column's article whose left edge falls within our slop overlaps by only a
# few px; treating it as a floor wrongly truncates our OWN body that extends
# a little past it. Require the overlap to exceed the padding we added.
return max(0, min(art_x2, other_bbox[2]) - max(art_x1, other_bbox[0])) > x_tolerance
# ── Detect multi-line headline ──────────────────────────────────────────
# Any doc_title sitting immediately below the current headline bottom in
# the same column is treated as a continuation line of the same headline.
# Iterate so we catch 3+ line headlines.
continuation_ids = []
current_bottom = region_map[primary_id]["bbox"][3]
candidates_for_continuation = [
r for r in all_regions
if r.get("type") == "doc_title"
and r["id"] != primary_id
and _shares_column(r["bbox"])
]
progressed = True
while progressed:
progressed = False
for r in candidates_for_continuation:
if r["id"] in continuation_ids:
continue
top, bot = r["bbox"][1], r["bbox"][3]
gap_above = top - current_bottom
if 0 <= gap_above <= headline_continuation_gap:
continuation_ids.append(r["id"])
current_bottom = max(current_bottom, bot)
progressed = True
headline_bottom = current_bottom
# Find the next REAL doc_title floor — one that's NOT a headline continuation.
other_titles_y = sorted(
r["bbox"][1] for r in all_regions
if r.get("type") == "doc_title"
and r["id"] != primary_id
and r["id"] not in continuation_ids
and r["bbox"][1] > headline_bottom + 30
and _shares_column_floor(r["bbox"])
)
floor = other_titles_y[0] if other_titles_y else float("inf")
# ── Identify an ORPHANED headline's own body dateline ────────────────────
# When Claude returns an article as HEADLINE ONLY (no body member of its own),
# the body that flows directly beneath the headline usually opens with this
# article's OWN dateline. That dateline must NOT act as a downward floor (it is
# our body, not the next story). Treat the topmost same-column dateline below the
# headline — provided no doc_title sits between it and the headline — as our own
# opening dateline.
_title_types = {"doc_title", "paragraph_title", "figure_title"}
has_body = any(region_map.get(rid, {}).get("type") not in _title_types
for rid in member_ids)
own_dateline_id = None
if not has_body and dateline_starts:
# Prefer a dateline body the scan already matched to THIS headline.
_own = [s for s in dateline_starts
if s.get("headline_region") == primary_id
and _shares_column_floor(s["bbox"])
and s["bbox"][1] > headline_bottom - 10
and s["region_id"] not in member_ids]
if _own:
own_dateline_id = min(_own, key=lambda s: s["bbox"][1])["region_id"]
else:
# Otherwise the topmost same-column dateline below the headline, as long
# as no doc_title sits between it and the headline, is our own body.
_col_dls = sorted(
(s for s in dateline_starts
if _shares_column_floor(s["bbox"])
and s["bbox"][1] > headline_bottom - 10
and s["region_id"] not in member_ids
and s.get("headline_region") != primary_id),
key=lambda s: s["bbox"][1])
if _col_dls and _col_dls[0]["bbox"][1] <= floor:
own_dateline_id = _col_dls[0]["region_id"]
# DATELINE FLOOR: the next article's dateline is a hard downward boundary — it
# works even when that next article's HEADLINE was missed by PaddleOCR. We only
# treat a dateline as a floor if it belongs to a DIFFERENT article: skip this
# article's own opening dateline (it sits right under the headline, and/or its
# scan-matched headline is this one).
if dateline_starts:
for s in dateline_starts:
b = s["bbox"]
if not _shares_column_floor(b):
continue
if b[1] <= headline_bottom + 150: # too close = this article's own dateline
continue
if s.get("headline_region") == primary_id: # explicitly this article's own
continue
if s["region_id"] == own_dateline_id: # orphaned headline's own body dateline
continue
floor = min(floor, b[1])
def _center_in_article_columns(r):
b = r["bbox"]
center_x = (b[0] + b[2]) / 2
return art_x1 <= center_x <= art_x2
# Candidate pool: unclaimed regions in the article's y-band AND within
# the article's horizontal column span.
candidates = [
r for r in all_regions
if r["id"] not in claimed_globally
and r["id"] not in member_ids
and r["id"] not in continuation_ids
and r["bbox"][1] >= headline_y1 - 30
and r["bbox"][3] <= floor + 30
and _center_in_article_columns(r)
]
# Seed the kept set with the original members + any continuation doc_titles
# (multi-line headline pieces). They count as "added by growth" so they
# surface in logs and show up in the crop.
kept = list(member_ids)
kept_set = set(kept)
added = []
for cid in continuation_ids:
if cid not in kept_set:
kept.append(cid)
kept_set.add(cid)
added.append(cid)
# ORPHANED-HEADLINE SEED: when Claude returns an article as headline-only, the
# first body line sits a little too far below the headline to connect directly
# (sub-head / standfirst spacing exceeds `gap`). The dateline scan, however,
# already matched a body region to THIS headline (own_dateline_id). Seed it so
# adjacency growth can chain the rest of the body from it. This anchors on a
# KNOWN body region rather than loosening the gap, so growth still stops at the
# gap before the next (possibly OCR-missed) article's photo/headline.
if (own_dateline_id is not None and own_dateline_id not in kept_set
and own_dateline_id not in claimed_globally
and own_dateline_id in region_map):
kept.append(own_dateline_id)
kept_set.add(own_dateline_id)
added.append(own_dateline_id)
# Iteratively grow: any candidate that touches the current set is added.
changed = True
while changed:
changed = False
for r in list(candidates):
if r["id"] in kept_set:
continue
for k in kept_set:
if k not in region_map:
continue
if _bboxes_connected(region_map[k]["bbox"], r["bbox"], gap):
kept.append(r["id"])
kept_set.add(r["id"])
added.append(r["id"])
changed = True
break
# ── Rightward column extension ──────────────────────────────────────────
# The growth above is bounded by the article's CURRENT column span, so a body
# that wraps into a column the HEADLINE does not sit over is never reached
# (headline over cols 1–2, body actually uses cols 1–3). Step into the column
# immediately to the right and pull in unclaimed continuation blocks.
#
# Purely ADDITIVE: it only runs after the span-bounded growth, and only adds
# regions that pass every gate below — so an article with no adjacent
# continuation column is returned unchanged. Each candidate must:
# • be unclaimed and not already kept,
# • NOT be a headline itself (doc_title) and NOT open its own dateline
# (either would mean a different article begins there),
# • sit in the next column to the right (its left edge just past our right
# edge) and touch the kept set (adjacency), and
# • sit ABOVE the first headline/dateline in ITS OWN column (per-column
# floor) — so a different story stacked lower in that column is left out.
def _hoverlap(a, b):
return max(0, min(a[2], b[2]) - max(a[0], b[0]))
def _voverlap(a, b):
return max(0, min(a[3], b[3]) - max(a[1], b[1]))
dateline_body_ids = {ds["region_id"] for ds in (dateline_starts or [])}
def _column_floor(r):
"""Top of the first NEW-article headline/dateline in r's column, below our
headline. r is only eligible if it sits entirely above this line."""
# A neighbour headline can start at nearly the SAME y as our own (parallel
# columns), so we exclude our own titles by IDENTITY (not by y) and accept
# any other title at/below our headline top. The `headline_y1 - 30` gate
# keeps mastheads ABOVE the headline from acting as a floor.
ys = [t["bbox"][1] for t in all_regions
if t.get("type") == "doc_title"
and t["id"] != primary_id and t["id"] not in continuation_ids
and t["id"] not in member_ids
and t["bbox"][1] > headline_y1 - 30
and _hoverlap(t["bbox"], r["bbox"]) > 0]
# A parallel/neighbouring article's headline is often a paragraph_title
# (not a doc_title) — e.g. two stories side by side, or a boxed item heading
# the next column. Treat any FOREIGN paragraph_title the same as a doc_title
# floor: if it hoverlaps the candidate's column and sits at/below our headline
# top, the candidate must stay ABOVE it. This means we only step into a
# continuation column from its TOP (unheaded text flowing from our column);
# if the column is headed by someone else's title, we stop. Under-reaching a
# continuation is safe — bleeding a neighbour's article is not.
for t in all_regions:
if (t.get("type") == "paragraph_title"
and t["id"] != primary_id and t["id"] not in continuation_ids
and t["id"] not in member_ids
and t["bbox"][1] > headline_y1 - 30
and _hoverlap(t["bbox"], r["bbox"]) > 0):
ys.append(t["bbox"][1])
for ds in (dateline_starts or []):
db = ds["bbox"]
if (ds.get("headline_region") != primary_id
and ds["region_id"] != own_dateline_id
and db[1] > headline_y1 - 30
and _hoverlap(db, r["bbox"]) > 0):
ys.append(db[1])
return min(ys) if ys else float("inf")
col_gap = 160 # gutter + slack between adjacent columns
changed = True
while changed:
changed = False
for r in all_regions:
rid = r["id"]
if (rid in kept_set or rid in claimed_globally or rid in member_ids
or r.get("type") == "doc_title" or rid in dateline_body_ids):
continue
rb = r["bbox"]
# Within the article's vertical band, above its column's next headline.
if rb[1] < headline_y1 - 30 or rb[3] > _column_floor(r) + 30:
continue
# The reading flow must WRAP into this column: a kept *body text* member
# has to sit immediately to its LEFT and overlap it vertically. This is
# what separates a genuine continuation column from an unrelated story
# that merely sits adjacent (whose left neighbour is a masthead, photo,
# or another article — not our body text).
wraps = any(
k in region_map
and region_map[k].get("type") == "text"
and 0 <= rb[0] - region_map[k]["bbox"][2] <= col_gap
and _voverlap(region_map[k]["bbox"], rb) >= 60
for k in kept_set
)
if not wraps:
continue
# GUARD: don't wrap into a region that is HEADED by its own column header.
# A neighbour article's headline is frequently mis-tagged as plain `text`
# by OCR (so it never becomes a doc_title/paragraph_title floor). If any
# FOREIGN region sits directly above `r` in its column — tightly stacked
# and covering most of r's width — then r begins that neighbour's story,
# not our continuation. Genuine continuation text has nothing foreign
# directly above it (only our own headline, or the column top).
r_w = max(1, rb[2] - rb[0])
headed_by_foreign = any(
t["id"] != primary_id
and t["id"] not in continuation_ids
and t["id"] not in member_ids
and t["id"] not in kept_set
and t.get("type") in ("text", "doc_title", "paragraph_title", "figure_title")
and 0 <= rb[1] - t["bbox"][3] <= 60
and _hoverlap(t["bbox"], rb) >= 0.5 * r_w
for t in all_regions
)
if headed_by_foreign:
continue
kept.append(rid)
kept_set.add(rid)
added.append(rid)
changed = True
return kept, added
def split_disjoint_articles(political_articles, regions, gap=60, min_regions=2,
dateline_starts=None):
"""For each Claude-returned article, split into spatially-connected sub-articles."""
region_map = {r["id"]: r for r in regions}
# ------ Pass 0: drop regions blocked by another doc_title in same column ------
# For each article, remove members that sit behind another article's
# headline (column-aware), and any duplicate doc_titles in the member list.
for art in political_articles:
member_ids = art.get("member_region_ids", [])
member_ids, dropped_blocked = _drop_blocked_by_other_doc_title(
member_ids, region_map, regions
)
if dropped_blocked:
print(f" ⛔ Column blocker: '{art.get('headline_english','?')[:40]}' — "
f"dropped {len(dropped_blocked)} region(s) blocked by another "
f"doc_title in their column: {dropped_blocked}")
art["member_region_ids"] = member_ids
# ------ Pass 1: trim above-headline strays from every article ------
trimmed_articles = []
for art in political_articles:
member_ids = art.get("member_region_ids", [])
member_ids, dropped_above = _trim_above_headline(member_ids, region_map)
if dropped_above:
print(f" ⚠️ Headline trim: '{art.get('headline_english','?')[:40]}' — "
f"dropped {len(dropped_above)} region(s) above the headline: {dropped_above}")
new_art = dict(art)
new_art["member_region_ids"] = member_ids
trimmed_articles.append(new_art)
# ------ Pass 2: grow each article downward into unclaimed regions ------
# Build the global "claimed" set so growth never steals from another article.
claimed = set()
for a in trimmed_articles:
claimed.update(a["member_region_ids"])
for art in trimmed_articles:
grown, added = _grow_article_downward(
art["member_region_ids"], region_map, regions, claimed,
dateline_starts=dateline_starts
)
if added:
print(f" ⬇ Growth: '{art.get('headline_english','?')[:40]}' — "
f"added {len(added)} adjacent region(s) below headline: {added}")
claimed.update(added)
art["member_region_ids"] = grown
# ------ Pass 3: geometric cluster split (catches remaining disjoint cases) ------
out = []
for art in trimmed_articles:
member_ids = art["member_region_ids"]
clusters = _cluster_regions(member_ids, region_map, gap)
if len(clusters) <= 1:
out.append(art)
continue
# Multi-cluster article. Headline (doc_title) is the article's identity —
# a real article must contain one. Use it to classify each cluster:
# - Cluster with a doc_title → real article content, keep
# - Cluster with NO doc_title → misattributed strays, drop
# - Multiple clusters each with their own doc_title → Claude bundled
# two real articles together; split them into separate output items.
with_title, without_title = [], []
for cluster in clusters:
has_title = any(
region_map.get(rid, {}).get("type") == "doc_title"
for rid in cluster
)
(with_title if has_title else without_title).append(cluster)
stray_count = sum(len(c) for c in without_title)
headline = art.get("headline_english", "?")[:40]
if len(with_title) == 0:
# No headline in any cluster — likely an OCR mislabel. Fall back to
# the largest cluster (matches previous behavior).
keeper = clusters[0]
print(f" ⚠️ Geometry split: '{headline}' — no doc_title in any cluster, "
f"kept largest ({len(keeper)} regions), dropped {len(member_ids)-len(keeper)}")
new_art = dict(art)
new_art["member_region_ids"] = keeper
new_art["_split_note"] = "no doc_title in any cluster; kept largest"
out.append(new_art)
elif len(with_title) == 1:
# One real cluster + N stray clusters → keep real, drop strays.
keeper = with_title[0]
if stray_count:
print(f" ⚠️ Geometry split: '{headline}' — kept the {len(keeper)} "
f"regions around the headline, dropped {stray_count} stray "
f"region(s) from {len(without_title)} disconnected cluster(s) "
f"with no doc_title")
new_art = dict(art)
new_art["member_region_ids"] = keeper
new_art["_split_note"] = (
f"dropped {stray_count} stray region(s) without doc_title"
)
out.append(new_art)
else:
# Multiple real articles bundled together → split into separate items.
print(f" ⚠️ Geometry split: '{headline}' — found {len(with_title)} "
f"doc_titles in disjoint clusters, splitting into {len(with_title)} "
f"articles (also dropped {stray_count} strays from "
f"{len(without_title)} cluster(s) without doc_title)")
for idx, cluster in enumerate(with_title, 1):
new_art = dict(art)
new_art["member_region_ids"] = cluster
if idx > 1:
new_art["headline_english"] = (
f"{art.get('headline_english','?')} (split {idx})"
)
new_art["_split_note"] = (
f"original group had {len(with_title)} doc_titles; this is part {idx}"
)
out.append(new_art)
# ------ Pass 4: merge sub-boxes (no doc_title) into their parent article ------
# Quote-boxes, testimonial boxes, and infoboxes inside a larger article have
# only a paragraph_title — no doc_title. If such an "article" sits inside or
# directly adjacent to another article's bounding box, fold it into that
# parent rather than emitting it as a separate (duplicate) crop.
out = _merge_subboxes_into_parents(out, region_map, dateline_starts=dateline_starts)
# ------ Pass 5: final re-growth over orphaned regions -------------------
# The Pass 2 growth used a `claimed` set built BEFORE the geometry split.
# If Claude mis-assigned a body region to a neighboring article, it counted
# as claimed and blocked the rightful article from growing into it — then
# Pass 3 dropped it from the wrong article as a disconnected stray, leaving
# it orphaned (unclaimed) with no owner. Rebuild `claimed` from the final
# article set and run one more growth pass so each article absorbs the
# still-unclaimed regions directly below its headline.
claimed = set()
for a in out:
claimed.update(a["member_region_ids"])
for art in out:
grown, added = _grow_article_downward(
art["member_region_ids"], region_map, regions, claimed,
dateline_starts=dateline_starts
)
if added:
print(f" ⬇ Re-growth: '{art.get('headline_english','?')[:40]}' — "
f"recovered {len(added)} orphaned region(s) below headline: {added}")
claimed.update(added)
art["member_region_ids"] = grown
return out
def split_articles_on_datelines(political_articles, regions, dateline_starts, sep_lines=None):
"""Correct Claude's OVER-grouping: if one article's member regions contain TWO
(or more) dateline bodies, it has swallowed >1 story — split it. Each region is
assigned to the dateline that OWNS it (nearest dateline above it in its column);
members above all datelines attach to the topmost. The group holding the
article's headline keeps the original metadata; the others are emitted as
new articles flagged _needs_review (their Q1/Q2 belong to the parent).
A non-keeper dateline group is split off as a SEPARATE article only when a real
article WALL — a headline (doc_title) or a horizontal separator line — sits in
the gap between it and the dateline directly above it. If the gap is just
continuous body text (no wall), the datelines belong to one roundup story
(same headline, several mandals) and are folded back together."""
if not dateline_starts:
return political_articles
region_map = {r["id"]: r for r in regions}
dl_ids = {s["region_id"] for s in dateline_starts}
# Article walls: detected headlines and horizontal separator lines.
doc_titles = [r["bbox"] for r in regions if r.get("type") == "doc_title"]
h_lines = [L for L in (sep_lines or []) if not L.get("vert") and "y" in L]
def wall_between(upper_box, lower_box):
return _wall_between_datelines(upper_box, lower_box, doc_titles, h_lines)
out = []
for art in political_articles:
members = art.get("member_region_ids", []) or []
dls = [rid for rid in members if rid in dl_ids and rid in region_map]
if len(dls) <= 1:
out.append(art)
continue
dl_boxes = [(rid, region_map[rid]["bbox"]) for rid in dls]
def owner(rid):
if rid not in region_map:
return dls[0]
b = region_map[rid]["bbox"]
cx = (b[0] + b[2]) / 2
in_col = [(d, db) for d, db in dl_boxes if db[0] - 40 <= cx <= db[2] + 40]
# A HEADLINE owns the dateline directly BELOW it (its body starts there),
# not the dateline above it.
if region_map[rid].get("type") == "doc_title":
below = [(d, db) for d, db in in_col if db[1] >= b[3] - 30]
if below:
return min(below, key=lambda z: z[1][1])[0] # nearest below
# A BODY region belongs to the nearest dateline ABOVE it.
above = [(d, db) for d, db in in_col if db[1] <= b[1] + 30]
if above:
return max(above, key=lambda z: z[1][1])[0]
if in_col: # above all → topmost in its column
return min(in_col, key=lambda z: z[1][1])[0]
return min(dls, key=lambda d: region_map[d]["bbox"][1])
groups = {}
for rid in members:
groups.setdefault(owner(rid), []).append(rid)
if len(groups) <= 1:
out.append(art)
continue
# which group keeps the parent metadata? the one holding the article's headline
doc_ids = [rid for rid in members
if region_map.get(rid, {}).get("type") == "doc_title"]
keeper = owner(min(doc_ids, key=lambda r: region_map[r]["bbox"][1])) if doc_ids \
else min(dls, key=lambda d: region_map[d]["bbox"][1])
if keeper not in groups:
keeper = max(groups, key=lambda k: len(groups[k]))
# Split a non-keeper dateline group off as its own article only when a real
# WALL (headline or separator line) sits between it and the dateline directly
# above it. No wall → continuous body text → same roundup story → fold back
# into the keeper rather than emit a spurious headless crop.
keeper_ids = list(groups[keeper])
review_groups = []
for dl, ids in groups.items():
if dl == keeper:
continue
db = region_map[dl]["bbox"]
cx = (db[0] + db[2]) / 2
above = [region_map[d]["bbox"] for d in dls
if d != dl and region_map[d]["bbox"][1] < db[1]
and region_map[d]["bbox"][0] - 40 <= cx <= region_map[d]["bbox"][2] + 40]
upper = max(above, key=lambda b: b[1]) if above else None
# Fold-back only applies WITHIN a column. If there is a same-column
# dateline above, decide by the wall in the gap. If there is none, this
# group sits in a DIFFERENT column (a side-by-side story) → always its own
# article; never fold two datelines together across columns.
has_wall = wall_between(upper, db) if upper is not None else True
if has_wall:
review_groups.append((dl, ids))
else:
keeper_ids.extend(ids)
if not review_groups:
out.append(art) # nothing was really a separate story
continue
print(f" ✂ Dateline split: '{art.get('headline_english','?')[:40]}' contained "
f"{len(dls)} datelines → kept 1 + split off {len(review_groups)} article(s) "
f"(headline/separator-line boundary)")
na = dict(art)
na["member_region_ids"] = sorted(keeper_ids)
out.append(na)
for dl, ids in sorted(review_groups, key=lambda kv: region_map[kv[0]]["bbox"][1]):
out.append({
"headline_telugu": "", "headline_english": "(dateline-split — needs analysis)",
"member_region_ids": sorted(ids), "priority": "medium", "category": "review",
"q1_reason": "", "q2_reason": "", "attack_angle": "",
"_split_on_dateline": True, "_needs_review": True,
})
return out
def trim_above_anchor(political_articles, regions, dateline_starts=None):
"""DATELINE CEILING (the mirror of the downward floor): an article must not
include regions that sit ABOVE its own start — those belong to the article
above it. The article's start (anchor) is its headline (`doc_title`), or, if
PaddleOCR missed the headline, its own dateline body. Any member region whose
bottom is above the anchor's top is dropped."""
region_map = {r["id"]: r for r in regions}
dl_list = [(s["region_id"], s["bbox"]) for s in (dateline_starts or [])
if s["region_id"] in region_map]
dl_ids = {d for d, _ in dl_list}
for art in political_articles:
members = art.get("member_region_ids", []) or []
doc_ids = [rid for rid in members
if region_map.get(rid, {}).get("type") == "doc_title"]
if doc_ids:
anchor = min(doc_ids, key=lambda r: region_map[r]["bbox"][1])
else:
dls = [rid for rid in members if rid in dl_ids and rid in region_map]
if not dls:
continue # no headline and no dateline → can't place a ceiling
anchor = min(dls, key=lambda r: region_map[r]["bbox"][1])
ab = region_map[anchor]["bbox"]
ceiling, acx = ab[1], (ab[0] + ab[2]) / 2
# the article's OWN dateline = the nearest dateline at/below the anchor in its column
own = set()
below_anchor = [(d, db) for d, db in dl_list
if db[0] - 40 <= acx <= db[2] + 40 and db[1] >= ab[3] - 30]
if below_anchor:
own.add(min(below_anchor, key=lambda z: z[1][1])[0])
kept, dropped = [], []
for rid in members:
r = region_map.get(rid)
if rid == anchor or not r:
kept.append(rid)
continue
b = r["bbox"]
cx = (b[0] + b[2]) / 2
above_anchor = b[3] <= ceiling + 5
# Is there a FOREIGN dateline at/above this region's level? (Not column-
# restricted, so it also catches the OTHER columns of a multi-column
# article sitting above this one.)
foreign_above = any(db[1] <= b[1] + 30 for d, db in dl_list if d not in own)
if dl_list and above_anchor and foreign_above:
dropped.append(rid) # below a foreign dateline → previous article (any type)
elif not dl_list and above_anchor and r.get("type") == "text":
dropped.append(rid) # no datelines → conservative: only body text
else:
kept.append(rid)
if dropped:
print(f" ⬆ Dateline ceiling: '{art.get('headline_english','?')[:40]}' — dropped "
f"{len(dropped)} region(s) above its start (belong to the article above): "
f"{sorted(dropped)}")
art["member_region_ids"] = sorted(kept)
return political_articles
def trim_below_floor(political_articles, regions, dateline_starts=None):
"""Drop a member region that sits ENTIRELY BELOW the article's own text AND at/below
the point where a FOREIGN article (a doc_title or dateline not in this article) has
already started. Catches a stray photo/box from the next article that got grouped in
even though there's no foreign headline directly above it in its own column (e.g. a
portrait placed beside the next article's headline). Conservative: only regions that
are below all of THIS article's text are eligible, so body text is never touched."""
region_map = {r["id"]: r for r in regions}
all_docs = [r for r in regions if r.get("type") == "doc_title"]
dl_list = [(s["region_id"], s["bbox"]) for s in (dateline_starts or [])]
for art in political_articles:
members = art.get("member_region_ids", []) or []
mset = set(members)
doc_ids = [i for i in members if region_map.get(i, {}).get("type") == "doc_title"]
text_boxes = [region_map[i]["bbox"] for i in members
if region_map.get(i, {}).get("type") == "text"]
if not doc_ids or not text_boxes:
continue
anchor_top = min(region_map[i]["bbox"][1] for i in doc_ids)
text_bottom = max(b[3] for b in text_boxes)
foreign_tops = [r["bbox"][1] for r in all_docs
if r["id"] not in mset and r["bbox"][1] > anchor_top]
foreign_tops += [db[1] for d, db in dl_list
if d not in mset and db[1] > anchor_top]
if not foreign_tops:
continue
kept, dropped = [], []
for rid in members:
r = region_map.get(rid)
if not r or rid in doc_ids:
kept.append(rid)
continue
b = r["bbox"]
below_text = b[1] > text_bottom # below ALL of this article's text
foreign_started = any(ft <= b[1] + 30 for ft in foreign_tops) # next article began at/above it
if below_text and foreign_started:
dropped.append(rid)
else:
kept.append(rid)
if dropped:
print(f" ⬇ Floor trim: '{art.get('headline_english','?')[:40]}' — dropped "
f"{len(dropped)} region(s) below its text that belong to the article below: "
f"{sorted(dropped)}")
art["member_region_ids"] = sorted(kept)
return political_articles
def grow_articles_final(political_articles, regions, dateline_starts=None):
"""Final downward+rightward growth pass on the SETTLED article shapes.
The first growth runs inside split_disjoint_articles, but the dateline splits
and trims that follow reshape articles (and mint new ones from dateline
splits), leaving those final shapes un-grown — so a body region adjacent to a
headline can be left out of the crop. Re-grow here, after all reshaping, so
every final article pulls in its adjacent body before cropping.
Strictly additive: `claimed` is the union of all articles' members (each
article's own members are excluded as candidates inside the grow), and newly
grown regions are claimed immediately, so no article ever steals another's.
"""
region_map = {r["id"]: r for r in regions}
claimed = set()
for a in political_articles:
claimed.update(a.get("member_region_ids", []) or [])
for art in political_articles:
members = art.get("member_region_ids", []) or []
if not members:
continue
grown, added = _grow_article_downward(
members, region_map, regions, claimed, dateline_starts=dateline_starts
)
if added:
print(f" ⬇ Final growth: '{art.get('headline_english','?')[:40]}' — "
f"added {len(added)} region(s): {sorted(added)}")
art["member_region_ids"] = grown
claimed.update(added)
return political_articles
def _bbox_of(member_ids, region_map):
bs = [region_map[r]["bbox"] for r in member_ids if r in region_map]
if not bs:
return None
return [min(b[0] for b in bs), min(b[1] for b in bs),
max(b[2] for b in bs), max(b[3] for b in bs)]
def _bbox_relation(child, parent, adjacency=80):
"""
Return ratio of `child` bbox area that is inside `parent` bbox.
If they don't overlap directly, also consider proximity:
if `child` is within `adjacency` pixels of `parent` (horizontally and
vertically) and shares column space, treat as adjacent and return 0.5.
"""
cx1, cy1, cx2, cy2 = child
px1, py1, px2, py2 = parent
x_ov = max(0, min(cx2, px2) - max(cx1, px1))
y_ov = max(0, min(cy2, py2) - max(cy1, py1))
child_area = max(1, (cx2 - cx1) * (cy2 - cy1))
contained_ratio = (x_ov * y_ov) / child_area
if contained_ratio > 0:
return contained_ratio
# No direct overlap — check adjacency in same column.
horiz_close = (x_ov > 0) or (min(abs(cx1 - px2), abs(px1 - cx2)) <= adjacency)
vert_close = (y_ov > 0) or (min(abs(cy1 - py2), abs(py1 - cy2)) <= adjacency)
horiz_shares_column = x_ov >= 0.5 * min(cx2 - cx1, px2 - px1)
if horiz_close and vert_close and horiz_shares_column:
return 0.5
return 0.0
def _merge_subboxes_into_parents(articles, region_map, min_relation=0.4, dateline_starts=None):
dl_ids = {s["region_id"] for s in (dateline_starts or [])}
parents, subs = [], []
for art in articles:
has_title = any(
region_map.get(rid, {}).get("type") == "doc_title"
for rid in art.get("member_region_ids", [])
)
(parents if has_title else subs).append(art)
if not subs:
return articles # nothing to merge
if not parents:
# No doc_title parent anywhere on the page. Keep a headless sub ONLY if it is
# a plausible standalone article — anchored on a dateline, or carrying a
# title-ish region. A sub that is just plain body text with neither is an
# orphan fragment (often a non-political block onto which Claude pasted a
# headline borrowed from another page) — drop it rather than emit junk.
kept = []
for s in subs:
mids = s.get("member_region_ids", []) or []
has_anchor = any(rid in dl_ids for rid in mids)
has_titleish = any(
region_map.get(rid, {}).get("type") in ("doc_title", "paragraph_title")
for rid in mids
)
if has_anchor or has_titleish:
s["_subbox_note"] = "no doc_title and no parent article candidate"
print(f" ⚠️ Sub-box without parent: '{s.get('headline_english','?')[:40]}' "
f"— kept as standalone (dateline-anchored or has title region)")
kept.append(s)
else:
print(f" ✗ Orphan sub-box dropped: '{s.get('headline_english','?')[:40]}' "
f"— no doc_title, no dateline anchor, no title region "
f"(likely a body fragment mislabeled as an article)")
return kept
# For each sub, find the parent with highest containment/adjacency ratio.
for sub in subs:
sub_bbox = _bbox_of(sub.get("member_region_ids", []), region_map)
if sub_bbox is None:
continue
best_parent, best_score = None, 0.0
for p in parents:
p_bbox = _bbox_of(p["member_region_ids"], region_map)
if p_bbox is None:
continue
score = _bbox_relation(sub_bbox, p_bbox)
if score > best_score:
best_score = score
best_parent = p
sub_label = sub.get("headline_english", "?")[:40]
if best_parent is not None and best_score >= min_relation:
parent_label = best_parent.get("headline_english", "?")[:40]
parent_ids = set(best_parent["member_region_ids"])
parent_ids.update(sub.get("member_region_ids", []))
best_parent["member_region_ids"] = list(parent_ids)
print(f" ⤴ Sub-box merge: '{sub_label}' folded into parent "
f"'{parent_label}' (overlap/adjacency ratio={best_score:.2f})")
else:
# No doc_title and no clear parent — this is almost certainly an
# internal sub-box of some article that Claude split out. Drop it
# rather than emit a duplicate / mislabeled crop.
print(f" ✗ Sub-box dropped: '{sub_label}' (no doc_title, "
f"best parent overlap={best_score:.2f}) — likely a quote-box "
f"or testimonial inside another article")
return parents
# ---------------------------------------------------------------------------
# Step 4: Crop using PaddleOCR's precise boxes
# ---------------------------------------------------------------------------
def crop_political_articles(page_img_path, political_articles, regions, output_dir, page_num,
dateline_starts=None, sep_lines=None):
img = Image.open(page_img_path)
pw, ph = img.size
# ── BLOCK-SNAP: prefer the validated geometric block crop ─────────────────────
# The geometric clusterer + crop_all_article_blocks carry ALL the per-paper
# cropping fixes (furniture drop, un-boxed headline recovery, neighbour masking,
# box-edge rules, continuation folds — see CROP_STRATEGY.md). Instead of
# re-deriving an article rectangle from Claude's member list with parallel
# ad-hoc geometry, map each political article onto the geometric block that owns
# the MAJORITY of its members and ship that block's crop. The legacy path below
# remains the fallback when no block matches (and for any article whose block
# was already claimed by a higher-priority article on the same page).
_snap_files, _snap_blocks, _snap_owner = {}, {}, {}
try:
import re as _re, tempfile as _tf, shutil as _sh
_blocks = cluster_all_article_blocks(regions, dateline_starts, sep_lines=sep_lines)
for _b in _blocks:
if _b.get("_furniture"):
continue
_snap_blocks[_b["anchor_id"]] = _b
for _m in _b["members"] + [_b["anchor_id"]]:
_snap_owner[_m] = _b["anchor_id"]
_bdir = Path(_tf.mkdtemp(prefix="polblocks_"))
crop_all_article_blocks(page_img_path, regions, str(_bdir), page_num,
dateline_starts, sep_lines=sep_lines)
for _p in (_bdir / f"page_{page_num:03d}").glob("a*.png"):
_mm = _re.match(r"a\d+_[A-Z]+(\d+)\.png$", _p.name)
if _mm:
_snap_files[int(_mm.group(1))] = _p
except Exception as _e:
print(f" (block-snap unavailable, falling back to legacy crops: {_e})")
_snap_files, _snap_blocks, _snap_owner = {}, {}, {}
_snap_used = set()
# Bottom of real content: y above which the page holds only the printer's
# CMYK colour-calibration strip + paper margin. Article crops are clamped to
# this so the registration dots never bleed into an article image. Returns the
# full height when there is no such strip (namaste, photo-filled bottoms).
try:
from _lines import detect_content_bottom
content_bottom = detect_content_bottom(page_img_path)
if not content_bottom:
content_bottom = ph
except Exception:
content_bottom = ph
# Build region lookup
region_map = {r["id"]: r for r in regions}
# Every detected headline on the page. A `doc_title` that does NOT belong to
# the article being cropped marks the START of a DIFFERENT story, so the crop
# must stop above it — otherwise the next article's headline + body bleed in.
all_doc_titles = [r for r in regions if str(r.get("type")) == "doc_title"]
records = []
for i, art in enumerate(political_articles, 1):
member_ids = list(art.get("member_region_ids", []))
if not member_ids:
continue
# Member regions that actually exist on the page.
member_regs = [region_map[rid] for rid in member_ids if rid in region_map]
if not member_regs:
continue
# --- block-snap: majority-vote the owning geometric block ------------
_votes = {}
for _r in member_regs:
_a = _snap_owner.get(_r["id"])
if _a is not None:
_votes[_a] = _votes.get(_a, 0) + 1
_snap = None
if _votes:
_best, _n = max(_votes.items(), key=lambda kv: kv[1])
if (_n >= 0.5 * len(member_regs) and _best in _snap_files
and _best not in _snap_used):
_snap = _best
if _snap is not None:
_blk = _snap_blocks[_snap]
art_name = f"p{page_num:03d}_a{i:03d}"
art_dir = output_dir / art_name
art_dir.mkdir(parents=True, exist_ok=True)
_sh.copyfile(_snap_files[_snap], art_dir / "article.png")
_bb = list(_blk["bbox"])
info = {
"page": page_num,
"article_id": i,
"bbox": _bb,
"member_region_ids": sorted(set(_blk["members"]) | {_snap}),
"headline_telugu": art.get("headline_telugu", ""),
"headline_english": art.get("headline_english", ""),
"q1_reason": art.get("q1_reason", ""),
"q2_reason": art.get("q2_reason", ""),
"attack_angle": art.get("attack_angle", ""),
"priority": art.get("priority", ""),
"category": art.get("category", ""),
"method": "smart_extractor_blocksnap",
}
(art_dir / "info.json").write_text(json.dumps(info, indent=2, ensure_ascii=False))
records.append({"dir": str(art_dir), "page": page_num, "article_id": i,
"name": art_name, "bbox": _bb,
"headline_telugu": art.get("headline_telugu", ""),
"is_banner": False})
art["_page"] = page_num
art["_art_name"] = art_name
art["_crop_area"] = (_bb[2] - _bb[0]) * (_bb[3] - _bb[1])
_snap_used.add(_snap)
print(f" 🧲 Block-snap: {art_name} → geometric block R{_snap} "
f"({_n}/{len(member_regs)} members, bbox={_bb})")
continue
# Raw union of member bounding boxes (no padding yet).
rx1 = min(r["bbox"][0] for r in member_regs)
ry1 = min(r["bbox"][1] for r in member_regs)
rx2 = max(r["bbox"][2] for r in member_regs)
ry2 = max(r["bbox"][3] for r in member_regs)
member_ids0 = list(member_ids) # original members (for banner restore)
# --- Multi-column BANNER headline: crop the whole multi-column block --
# A single WIDE headline spanning several columns heads ONE article; every
# column beneath it down to its bottom rule belongs to that article even
# though it contains internal sub-headlines and photos. The per-column
# clip / floor rules below would wrongly split such a banner, so when we
# recognise a banner bounded by side walls AND a bottom rule we take the
# whole block and skip those rules. If there is NO bottom rule we fall
# back to the normal per-column logic.
banner_box = None
_bt0 = sorted([r for r in member_regs if str(r.get("type")) == "doc_title"],
key=lambda r: r["bbox"][1])
if _bt0 and (_bt0[0]["bbox"][2] - _bt0[0]["bbox"][0]) >= 0.40 * pw:
hb0 = _bt0[0]["bbox"]
htop, hbot = hb0[1], hb0[3]
try:
from _lines import detect_separator_lines
_ls0 = detect_separator_lines(page_img_path)
vlines0, hlines0 = _ls0.get("v", []), _ls0.get("h", [])
except Exception:
vlines0, hlines0 = [], []
def _vcov(ln): # v-rule reaches up alongside headline
return ln.get("y1", 0) <= hbot and ln.get("y2", 0) >= htop - 20
lw = [ln.get("x1", ln.get("x", 0)) for ln in vlines0
if _vcov(ln) and ln.get("x1", ln.get("x", 0)) <= hb0[0] + 30]
rw = [ln.get("x1", ln.get("x", 0)) for ln in vlines0
if _vcov(ln) and ln.get("x1", ln.get("x", 0)) >= hb0[2] - 30]
left_wall = max(lw) if lw else rx1
right_wall = min(rw) if rw else rx2
bw = right_wall - left_wall
# Bottom rule: a horizontal line spanning most of the banner width, the
# first one BELOW the grouped content. The width test (>60% of banner)
# rejects short caption/photo underlines; _line_is_underline() also
# rejects a WIDE rule that merely underlines an image/caption (a line
# with a photo or figure_title directly above it is not a separator).
cands = [ln.get("y1", ln.get("y", 0)) for ln in hlines0
if ln.get("y1", ln.get("y", 0)) > max(hbot, ry2) + 5
and (min(ln.get("x2", 0), right_wall)
- max(ln.get("x1", 0), left_wall)) > 0.6 * bw
and not _line_is_underline(ln.get("y1", ln.get("y", 0)),
ln.get("x1", 0), ln.get("x2", 0), regions)]
if cands and bw > 0:
bottom_rule = min(cands)
if bottom_rule > htop + 200:
banner_box = [left_wall, htop, right_wall, bottom_rule]
# --- Clip out intruding foreign headlines (column-aware) ------------
# An article has ONE headline at the top. Any OTHER headline below it marks
# the START of the next story IN THE COLUMN(S) IT SPANS. Drop only the
# members that overlap that headline's column AND sit at/below it — body
# flowing in the article's OTHER columns is kept. A page-wide y-clip would
# truncate a multi-column article whenever a short neighbouring headline
# appears in just ONE of its columns (e.g. a col-1 headline below the
# article while its body continues lower in cols 2–4).
member_ids_set = {r["id"] for r in member_regs}
own_titles = sorted([r for r in member_regs if str(r.get("type")) == "doc_title"],
key=lambda r: r["bbox"][1])
primary_bottom = own_titles[0]["bbox"][3] if own_titles else ry1
# Span of this article's TEXTUAL content (excludes images/figures). Used to
# recognise a disjoint trailing member (e.g. a misgrouped far-side photo)
# that sits past the article's text AND below a foreign headline.
text_regs = [r for r in member_regs
if str(r.get("type")) in ("text", "doc_title", "paragraph_title")]
text_x1 = min((r["bbox"][0] for r in text_regs), default=rx1)
text_x2 = max((r["bbox"][2] for r in text_regs), default=rx2)
text_bottom = max((r["bbox"][3] for r in text_regs), default=ry2)
def _hov(a, b):
return max(0, min(a[2], b[2]) - max(a[0], b[0]))
drop_ids = set()
for dt in all_doc_titles:
if dt["id"] in member_ids_set: # our own headline / continuation
continue
db = dt["bbox"]
if db[1] <= primary_bottom: # must start below our headline
continue
if _hov(db, [rx1, ry1, rx2, ry2]) <= 0: # must fall in our columns
continue
# A foreign headline that sits BELOW all of our text marks pure trailing
# foreign content: anything starting at/below it is the next story. Drop
# such a member only if it's horizontally disjoint from our text column,
# so a legitimate bottom photo sitting under the text is never cut.
trailing = db[1] >= text_bottom
for r in member_regs:
rb = r["bbox"]
if rb[1] >= db[1] - 30:
if _hov(rb, db) > 0:
drop_ids.add(r["id"])
elif trailing and _hov(rb, [text_x1, 0, text_x2, 0]) <= 0:
drop_ids.add(r["id"])
if drop_ids:
kept = [r for r in member_regs if r["id"] not in drop_ids]
if kept:
member_regs = kept
member_ids = [r["id"] for r in kept]
rx1 = min(r["bbox"][0] for r in member_regs)
ry1 = min(r["bbox"][1] for r in member_regs)
rx2 = max(r["bbox"][2] for r in member_regs)
ry2 = max(r["bbox"][3] for r in member_regs)
print(f" ✂️ Clipped {len(drop_ids)} member(s) under a foreign headline "
f"out of p{page_num:03d}_a{i:03d}: {sorted(drop_ids)}")
# --- Clip members across a vertical separator rule -------------------
# A printed vertical rule running the height of the article marks a
# column/article boundary (the gutter between two stories). Drop members
# that fall entirely on the FAR side of such a rule from the article's
# primary headline — e.g. a neighbouring story's photo/caption bleeding
# in past the gutter line. The headline must sit cleanly on one side of
# the rule (a rule passing THROUGH the headline spans the article and is
# not a boundary).
prim = sorted([r for r in member_regs if str(r.get("type")) == "doc_title"],
key=lambda r: r["bbox"][1])
prim = prim[0] if prim else (sorted(member_regs, key=lambda r: r["bbox"][1])[0]
if member_regs else None)
if prim is not None and len(member_regs) > 1:
try:
from _lines import detect_separator_lines
vlines = detect_separator_lines(page_img_path).get("v", [])
except Exception:
vlines = []
hb = prim["bbox"]
vdrop = set()
for ln in vlines:
vx = ln.get("x1", ln.get("x", 0))
ly1, ly2 = ln.get("y1", 0), ln.get("y2", 0)
if (ly2 - ly1) < 600: # only long structural rules
continue
if hb[0] < vx < hb[2]: # rule passes through headline
continue
head_left = hb[2] <= vx
for r in member_regs:
if r["id"] == prim["id"]:
continue
b = r["bbox"]
if b[3] < ly1 or b[1] > ly2: # member must overlap rule span
continue
on_far_side = (b[0] >= vx) if head_left else (b[2] <= vx)
if on_far_side:
vdrop.add(r["id"])
if vdrop:
kept = [r for r in member_regs if r["id"] not in vdrop]
if kept:
member_regs = kept
member_ids = [r["id"] for r in kept]
rx1 = min(r["bbox"][0] for r in member_regs)
ry1 = min(r["bbox"][1] for r in member_regs)
rx2 = max(r["bbox"][2] for r in member_regs)
ry2 = max(r["bbox"][3] for r in member_regs)
print(f" ✂️ Clipped {len(vdrop)} member(s) across a vertical rule "
f"out of p{page_num:03d}_a{i:03d}: {sorted(vdrop)}")
# --- Floor at a second in-column headline ----------------------------
# An article has ONE headline. A SECOND headline-class member (doc_title
# or paragraph_title) below it, in the same column AND with its own body
# text beneath, is the start of a different story that was over-merged
# (e.g. a 'continued from page N' header). Treat the topmost such member
# as a floor: drop it and every member at/below it that overlaps the
# headline's column. Members in the article's OTHER columns are kept.
prim2 = sorted([r for r in member_regs if str(r.get("type")) == "doc_title"],
key=lambda r: r["bbox"][1])
if prim2:
pb = prim2[0]["bbox"]
pid = prim2[0]["id"]
# Horizontal rules on the page — a TRUE second-article boundary in the
# column is printed under a horizontal rule. An in-article sub-head /
# byline (a narrow paragraph_title with body beneath it, like a
# dateline) has NO rule above it and must NOT floor the crop.
try:
from _lines import detect_separator_lines
hlines = detect_separator_lines(page_img_path).get("h", [])
except Exception:
hlines = []
def _rule_above(cb):
ct, cw = cb[1], (cb[2] - cb[0])
for ln in hlines:
ly = ln.get("y1", ln.get("y", 0))
if pb[3] < ly <= ct + 20:
ox = (min(ln.get("x2", 0), cb[2]) - max(ln.get("x1", 0), cb[0]))
# A caption/photo underline (image or figure_title directly
# above the line) is NOT a section separator — skip it.
if ox > 0.4 * cw and not _line_is_underline(
ly, ln.get("x1", 0), ln.get("x2", 0), regions):
return True
return False
def _qualifies(r):
t = str(r.get("type"))
if t == "doc_title":
return True
# paragraph_title only floors with a corroborating rule above it
return t == "paragraph_title" and _rule_above(r["bbox"])
sec = [r for r in member_regs
if r["id"] != pid
and _qualifies(r)
and r["bbox"][1] > pb[3] + 5
and _hov(r["bbox"], pb) > 0.3 * (pb[2] - pb[0])]
if sec:
floor = min(r["bbox"][1] for r in sec)
has_body_below = any(
str(r.get("type")) == "text"
and r["bbox"][1] > floor
and _hov(r["bbox"], [pb[0], 0, pb[2], 0]) > 0
for r in member_regs)
if has_body_below:
fdrop = {r["id"] for r in member_regs
if r["bbox"][1] >= floor - 10
and _hov(r["bbox"], [pb[0], 0, pb[2], 0]) > 0}
if fdrop:
kept = [r for r in member_regs if r["id"] not in fdrop]
if kept:
member_regs = kept
member_ids = [r["id"] for r in kept]
rx1 = min(r["bbox"][0] for r in member_regs)
ry1 = min(r["bbox"][1] for r in member_regs)
rx2 = max(r["bbox"][2] for r in member_regs)
ry2 = max(r["bbox"][3] for r in member_regs)
print(f" ✂️ Floored p{page_num:03d}_a{i:03d} at a 2nd in-column "
f"headline (y={floor}); dropped {sorted(fdrop)}")
# --- Grow RIGHT to a vertical separator wall -------------------------
# Claude sometimes under-groups an article, omitting its right-most
# column even though a printed vertical rule clearly bounds the story on
# the right. If a long structural rule runs just past the crop's right
# edge within the article's vertical band, and the strip between the
# current right edge and that wall contains NO foreign headline
# (doc_title), the strip belongs to this article — absorb its text /
# image / caption members and extend the crop to the wall.
right_cap = None
if member_regs:
try:
from _lines import detect_separator_lines
vlines2 = detect_separator_lines(page_img_path).get("v", [])
except Exception:
vlines2 = []
art_h = ry2 - ry1
current_ids2 = {r["id"] for r in member_regs}
walls = [ln for ln in vlines2
if (ln.get("y2", 0) - ln.get("y1", 0)) >= 600
and ln.get("x1", ln.get("x", 0)) > rx2 + 10
and (min(ln.get("y2", 0), ry2) - max(ln.get("y1", 0), ry1)) > 0.5 * art_h]
if walls and art_h > 0:
wall_x = min(ln.get("x1", ln.get("x", 0)) for ln in walls)
# Bound the grow distance: a missing right-hand COLUMN cannot be
# wider than the article it attaches to. A far wall at the page's
# right margin would otherwise swallow whole neighbouring stories
# that happen to lack a detected gutter rule.
art_w = rx2 - rx1
too_far = (wall_x - rx2) > max(art_w, 1)
# No FOREIGN headline may live in the strip — that would mean the
# strip is a different story, not this article's missing column.
foreign = too_far or any(str(r.get("type")) == "doc_title"
and r["id"] not in current_ids2
and r["bbox"][2] > rx2 and r["bbox"][0] < wall_x
and r["bbox"][3] > ry1 and r["bbox"][1] < ry2
for r in regions)
if not foreign:
absorb = [r for r in regions
if str(r.get("type")) in ("text", "image",
"figure_title", "paragraph_title")
and r["id"] not in current_ids2
and rx2 < (r["bbox"][0] + r["bbox"][2]) / 2 < wall_x
and r["bbox"][3] > ry1 and r["bbox"][1] < ry2]
if absorb:
member_regs = member_regs + absorb
member_ids = [r["id"] for r in member_regs]
rx1 = min(r["bbox"][0] for r in member_regs)
ry1 = min(r["bbox"][1] for r in member_regs)
rx2 = max(max(r["bbox"][2] for r in member_regs), int(wall_x))
ry2 = max(r["bbox"][3] for r in member_regs)
right_cap = int(wall_x)
print(f" ➡️ Grew p{page_num:03d}_a{i:03d} right to a vertical "
f"wall (x={int(wall_x)}); absorbed {sorted(r['id'] for r in absorb)}")
# A recognised banner overrides the per-column clip/floor/grow results:
# take the whole multi-column block (side walls + bottom rule) and restore
# the full member set the clips may have pruned.
if banner_box is not None:
rx1, ry1, rx2, ry2 = banner_box
right_cap = None
member_ids = list(member_ids0)
print(f" 🪧 Banner p{page_num:03d}_a{i:03d}: cropped full multi-column "
f"block x[{rx1}..{rx2}] y[{ry1}..{ry2}]")
# Pad and clamp to page bounds.
x1 = max(0, rx1 - 15)
y1 = max(0, ry1 - 15)
x2 = min(pw, rx2 + 15)
y2 = min(ph, ry2 + 15, content_bottom)
if right_cap is not None:
x2 = min(x2, right_cap)
if (x2 - x1) < 50 or (y2 - y1) < 50:
continue
# Crop
art_name = f"p{page_num:03d}_a{i:03d}"
art_dir = output_dir / art_name
art_dir.mkdir(parents=True, exist_ok=True)
cropped = img.crop((x1, y1, x2, y2))
cropped.save(str(art_dir / "article.png"))
info = {
"page": page_num,
"article_id": i,
"bbox": [x1, y1, x2, y2],
"member_region_ids": member_ids,
"headline_telugu": art.get("headline_telugu", ""),
"headline_english": art.get("headline_english", ""),
"q1_reason": art.get("q1_reason", ""),
"q2_reason": art.get("q2_reason", ""),
"attack_angle": art.get("attack_angle", ""),
"priority": art.get("priority", ""),
"category": art.get("category", ""),
"method": "smart_extractor",
}
(art_dir / "info.json").write_text(json.dumps(info, indent=2, ensure_ascii=False))
records.append({"dir": str(art_dir), "page": page_num, "article_id": i,
"name": art_name, "bbox": [x1, y1, x2, y2],
"headline_telugu": art.get("headline_telugu", ""),
"is_banner": banner_box is not None})
# Annotate the article dict so the cross-page de-dup step (Step 5) can
# compare instances of the same story and keep the most complete crop.
art["_page"] = page_num
art["_art_name"] = art_name
art["_crop_area"] = (x2 - x1) * (y2 - y1)
print(f" Cropped: {art_name} ({len(member_ids)} regions, bbox={[x1,y1,x2,y2]})")
if _snap_files:
try:
_sh.rmtree(next(iter(_snap_files.values())).parent.parent)
except Exception:
pass
return records
# ---------------------------------------------------------------------------
# Generate output files
# ---------------------------------------------------------------------------
def generate_brief(all_articles, output_dir):
high = [a for a in all_articles if a.get("priority") == "high"]
medium = [a for a in all_articles if a.get("priority") == "medium"]
brief_path = output_dir / "opposition_daily_brief.txt"
with open(brief_path, "w", encoding="utf-8") as f:
f.write("=" * 70 + "\n")
f.write(" OPPOSITION DAILY BRIEF\n")
f.write("=" * 70 + "\n\n")
f.write(f"Total: {len(all_articles)} | HIGH: {len(high)} | MEDIUM: {len(medium)}\n")
f.write("-" * 70 + "\n\n")
for label, group in [("HIGH PRIORITY", high), ("MEDIUM PRIORITY", medium)]:
if group:
f.write(f">>> {label} <<<\n\n")
for i, a in enumerate(group, 1):
f.write(f"{i}. {a.get('headline_telugu', '')}\n")
f.write(f" ({a.get('headline_english', '')})\n")
f.write(f" WHY: {a.get('q1_reason', '')}\n")
f.write(f" ATTACK: {a.get('attack_angle', '')}\n\n")
f.write("-" * 70 + "\n")
f.write("END OF BRIEF\n")
print(f" Brief saved: {brief_path}")
def generate_pdf(all_records, output_dir):
images = []
for rec in sorted(all_records, key=lambda r: (r["page"], r["article_id"])):
img_path = Path(rec["dir"]) / "article.png"
if img_path.exists():
images.append(Image.open(img_path).convert("RGB"))
if images:
pdf_path = output_dir / "political_articles.pdf"
images[0].save(str(pdf_path), save_all=True, append_images=images[1:])
print(f" PDF saved: {pdf_path}")
def generate_combined_pdf(crop_paths, out_path):
"""Stitch every cropped article image across ALL processed PDFs into one PDF,
one crop per page, in the given order."""
images = []
for p in crop_paths:
try:
images.append(Image.open(p).convert("RGB"))
except Exception as e:
print(f" ⚠️ Skipping unreadable crop {p}: {e}")
if images:
images[0].save(str(out_path), save_all=True, append_images=images[1:])
print(f" Combined PDF saved: {out_path} ({len(images)} crops)")
else:
print(" No crops found to combine into a PDF.")
def copy_all_images(all_records, output_dir):
import shutil
img_dir = output_dir / "all_political_images"
img_dir.mkdir(exist_ok=True)
for rec in all_records:
src = Path(rec["dir"]) / "article.png"
if src.exists():
shutil.copy2(str(src), str(img_dir / f"{rec['name']}.png"))
print(f" All images copied to: {img_dir}")
def write_review_report(all_political, pages_dir, output_dir, tiny_area=500000,
crop_integrity_issues=None):
"""Lightweight post-run validator. Flags crops that probably need a human look —
empty Telugu headline, a suspiciously small crop area, or a _needs_review marker —
plus any page whose model response failed to parse (a *.badresponse.txt exists).
Writes <output_dir>/_REVIEW.txt. Returns the number of flagged items."""
lines = []
flagged = 0
bad_pages = sorted(Path(pages_dir).glob("*.badresponse.txt"))
if bad_pages:
lines.append("PAGES WITH UNPARSEABLE MODEL RESPONSE (articles may be missing):")
for bp in bad_pages:
lines.append(f" - {bp.name}")
lines.append("")
flagged += len(bad_pages)
art_issues = []
for art in all_political:
reasons = []
if not (art.get("headline_telugu") or "").strip():
reasons.append("empty headline_telugu")
area = art.get("_crop_area")
if isinstance(area, (int, float)) and area < tiny_area:
reasons.append(f"tiny crop area ({int(area):,} < {tiny_area:,})")
if art.get("_needs_review"):
reasons.append("flagged _needs_review")
if reasons:
art_issues.append((art, reasons))
if art_issues:
lines.append("CROPS TO REVIEW:")
for art, reasons in art_issues:
name = art.get("_art_name", "?")
page = art.get("_page", "?")
hl = (art.get("headline_telugu") or art.get("headline_english") or "").strip() or "(no headline)"
lines.append(f" - p{page} {name}: {hl}")
lines.append(f" → {'; '.join(reasons)}")
lines.append("")
flagged += len(art_issues)
if crop_integrity_issues:
lines.append("INCOMPLETE / IMPROPER CROPS (dateline-vs-crop integrity):")
for it in crop_integrity_issues:
lines.append(f" - p{it['page']} block R{it['anchor_id']} [{it['kind']}]: {it['reason']}")
lines.append("")
flagged += len(crop_integrity_issues)
header = (f"REVIEW REPORT — {flagged} item(s) flagged\n"
f"{'=' * 48}\n\n") if flagged else "REVIEW REPORT — no issues flagged.\n"
(Path(output_dir) / "_REVIEW.txt").write_text(header + "\n".join(lines), encoding="utf-8")
if flagged:
print(f" ⚠️ Review report: {flagged} item(s) flagged → {output_dir / '_REVIEW.txt'}")
else:
print(f" ✅ Review report: no issues flagged → {output_dir / '_REVIEW.txt'}")
return flagged
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def process_pdf(pdf_path, output_dir, target_w=4200, target_h=7400):
"""Run the full extraction pipeline on ONE PDF, writing everything into output_dir."""
pdf_path = Path(pdf_path)
output_dir = Path(output_dir)
pages_dir = output_dir / "pages"
articles_dir = output_dir / "articles"
pages_dir.mkdir(parents=True, exist_ok=True)
articles_dir.mkdir(parents=True, exist_ok=True)
print(f"PDF: {pdf_path}")
print(f"Output: {output_dir}")
print()
# Step 1: PDF → page images
print("STEP 1: Rendering PDF pages...")
pages = render_pdf_pages(pdf_path, pages_dir, target_w, target_h)
paper = detect_paper(pdf_path)
global _ACTIVE_PAPER
_ACTIVE_PAPER = paper # gates paper-specific clustering passes (e.g. Sakshi)
print(f" Detected paper: {paper} (dateline rule selected accordingly)")
print()
all_records = []
all_political = []
page_summary = [] # per-page counts: (page, all_blocks/crops, political)
crop_integrity_issues = [] # dateline-vs-crop integrity flags (split/merged crops)
for p in pages:
n = p["page"]
page_img_path = p["path"]
# Step 2: PaddleOCR layout detection
print(f"STEP 2: Page {n} — detecting layout regions...")
regions = detect_regions(page_img_path)
# Drop near-duplicate detections (same heading detected twice).
regions = dedup_overlapping_regions(regions)
# Merge the lines of a headline that wrapped onto 2+ lines (boxed separately).
regions = merge_stacked_title_lines(regions, page_img_path)
# Drop the fixed front-page masthead (Namaste Telangana edition nameplate)
# so it is never mis-grouped as a news article.
regions = drop_masthead_regions(regions, paper, n, page_img_path)
# Drop CLASSIFIEDS columns (phone-listing furniture) so a neighbouring article's
# column-fold can't grow down into them (JG R65 mega-over-merge). News untouched.
regions = drop_classifieds_regions(regions, page_img_path)
# Tag COURT / LEGAL NOTICES (kept, but carved into their own section by the clusterer
# so they aren't absorbed into a neighbouring article).
regions = tag_legal_notices(regions, page_img_path)
# Snapshot type counts BEFORE promotion so the inventory can show before/after.
from collections import Counter as _Counter
pre_promo_counts = _Counter(r.get("type", "unknown") for r in regions)
# Reclassify mis-tagged headlines: a full-width paragraph_title that has a
# fresh dateline body directly beneath it is really a headline → doc_title.
regions = promote_paragraph_titles(regions, page_img_path, paper)
# SAKSHI/NT: recover display headlines PaddleOCR never boxed (pixel pass) —
# widen partially-boxed headlines over their kicker, synthesize anchors
# for completely un-boxed headline bands (NT JG pg1: the script-font
# 'నమ్మిండు..' and DL2's heading have no region at all). NT is FRONT PAGE
# ONLY: inner NT pages are dense and well-detected; the synth there split
# protected roundup sections (Siddipet pg4 25→28).
if paper == "sakshi" or (paper == "namaste_telangana" and n == 1):
regions = detect_sakshi_headline_bands(regions, page_img_path)
# Save regions (post-promotion)
(pages_dir / f"page_{n:03d}.regions.json").write_text(
json.dumps({"page": n, "regions": regions}, indent=2, ensure_ascii=False)
)
# Diagnostic: per-page region-type counts (before/after promotion) + headline list
log_region_inventory(regions, n, pre_promo_counts)
# Flag bullet-prefixed titles (decks/sub-points) so the clusterer demotes them
# when a real headline sits above — a bullet is never an article headline.
regions = mark_bullet_titles(regions, page_img_path)
# Article-boundary rules: horizontal separator lines (without a bold headline
# directly above) mark where one stacked story ends and the next begins. Used
# as hard barriers so the clusterer never folds/grows a block across them.
try:
from _lines import separator_barriers
sep_lines = separator_barriers(page_img_path, regions)
except Exception as e:
print(f" (separator-line detection skipped: {e})")
sep_lines = []
# CLUE: find article-starts by the dateline pattern (catches articles whose
# headline PaddleOCR missed). Fed into the Stage-3 prompt so Claude emits them.
dateline_starts = find_article_starts_by_dateline(regions, page_img_path, paper)
# Orphan-scoped headline recovery: cluster once, then re-tag any BOLD orphan text
# sitting above a photo as a doc_title (a lost headline PaddleOCR mis-typed). Only
# orphans qualify, so captions — which always fold into an article — are untouched.
# If anything was recovered, re-detect datelines (they can now link to the headline)
# and the downstream cluster/crop runs use the corrected regions.
_probe_blocks = cluster_all_article_blocks(regions, dateline_starts, sep_lines=sep_lines)
if recover_orphan_text_headlines(_probe_blocks, regions, page_img_path):
dateline_starts = find_article_starts_by_dateline(regions, page_img_path, paper)
# Diagnostic image: highlight headline-type boxes (doc/paragraph/figure_title)
draw_titles_debug(page_img_path, regions, pages_dir, n)
# Diagnostic image: EVERY region from PaddleOCR, color-coded by type
draw_all_regions_debug(page_img_path, regions, pages_dir, n)
# Diagnostic image: EVERY candidate article block BEFORE political filtering
draw_all_article_boundaries_debug(page_img_path, regions, pages_dir, n,
dateline_starts, sep_lines=sep_lines)
# Diagnostic image: BLANK-canvas boundary map (headers + boundaries) before cropping
draw_boundary_map_debug(page_img_path, regions, pages_dir, n,
dateline_starts, sep_lines=sep_lines)
# Per-article crops (all articles, unfiltered) into output_dir/all_article_crops/page_NNN/
n_all_crops = crop_all_article_blocks(page_img_path, regions,
Path(output_dir) / "all_article_crops", n, dateline_starts,
sep_lines=sep_lines)
# Flag crops that violate the dateline minimum-count rule (split/merged).
crop_integrity_issues += check_dateline_crop_integrity(regions, dateline_starts, n,
sep_lines=sep_lines)
# Step 3: Claude — identify political articles + group regions (ONE call)
print(f"STEP 3: Page {n} — identifying political articles (1 Claude call)...")
result = identify_and_group_political_articles(page_img_path, regions, n, paper,
dateline_starts)
political = result.get("political_articles", [])
rejected = result.get("rejected", [])
# Headline verification: crop each doc_title region and ask Claude
# (via a small, scoped vision call) to transcribe the actual Telugu
# text. Drop articles where that transcription doesn't match the
# headline Claude originally reported. Catches the case where Claude
# hallucinates content (e.g. labelling a US-Iran peace deal article
# as "Deputy Collector's illegal assets").
print(f"STEP 3b: Page {n} — verifying {len(political)} headline(s) against image...")
political = verify_headlines_with_claude(political, regions, page_img_path)
# Geometry safeguard: split or trim Claude's groupings if member regions
# form spatially disjoint clusters (catches the multi-article bundling bug).
political = split_disjoint_articles(political, regions, dateline_starts=dateline_starts)
# Correct over-grouping: split any article whose regions span 2+ datelines.
political = split_articles_on_datelines(political, regions, dateline_starts, sep_lines=sep_lines)
# Dateline ceiling: drop regions above an article's start (they belong to
# the article above). Mirror of the downward floor; needs no datelines when
# the headline was detected.
political = trim_above_anchor(political, regions, dateline_starts)
# Floor trim: drop a stray region below the article's text that belongs to
# the next article (e.g. a portrait beside the next headline).
political = trim_below_floor(political, regions, dateline_starts)
# Final growth on the SETTLED shapes: the in-split grow pass can't see the
# reshaping done by the dateline split + trims, so under-grown articles get
# one more additive downward/rightward pass here.
political = grow_articles_final(political, regions, dateline_starts)
# Save full result (post-safeguard)
(pages_dir / f"page_{n:03d}.political_result.json").write_text(
json.dumps({"political_articles": political, "rejected": rejected},
indent=2, ensure_ascii=False),
encoding="utf-8",
)
# Per-page count log. Two independent counters, never blended:
# • Code side — datelines from the OCR scan + article blocks cropped,
# broken down by anchor kind (H headline / P paragraph / DL dateline-
# only / O orphan-or-other).
# • Claude side — political kept, rejected, and their total.
from collections import Counter as _KindCounter
_blocks = cluster_all_article_blocks(regions, dateline_starts, sep_lines=sep_lines)
_kind = _KindCounter(b["kind"] for b in _blocks)
n_datelines = len(dateline_starts)
n_headline = _kind.get("H", 0)
n_para = _kind.get("P", 0)
n_dl_block = _kind.get("DL", 0)
n_orphan = _kind.get("O", 0)
n_claude = len(political)
n_rejected = len(rejected)
print(f"\n 📊 PAGE {n} COUNTS:")
print(f" Code — datelines found : {n_datelines}")
print(f" Code — article blocks cropped : {n_all_crops}")
print(f" headline-anchored (H) : {n_headline}")
print(f" paragraph-anchored (P) : {n_para}")
print(f" dateline-only (DL) : {n_dl_block}")
print(f" orphans / other (O) : {n_orphan}")
print(f" Claude — political (kept) : {n_claude}")
print(f" Claude — rejected : {n_rejected}")
print(f" Claude — total identified : {n_claude + n_rejected} (kept + rejected)")
# Per-page: Claude-identified headings vs Code-identified block headings.
log_identification_breakdown(page_img_path, regions, dateline_starts,
political, rejected, n, sep_lines=sep_lines)
# --- DETAILED LOGGING ---
region_map = {r["id"]: r for r in regions}
print(f"\n{'='*70}")
print(f"SMART EXTRACTOR — PAGE {n} RESULTS")
print(f"{'='*70}")
print(f" Input: {len(regions)} regions")
print(f" Output: {len(political)} political articles selected, {len(rejected)} rejected")
print()
# Log each selected political article with region details
for i, art in enumerate(political, 1):
member_ids = art.get("member_region_ids", [])
print(f" ✅ POLITICAL ARTICLE {i}: \"{art.get('headline_english', '?')}\"")
print(f" Telugu: {art.get('headline_telugu', '?')}")
print(f" Priority: {art.get('priority', '?')} | Category: {art.get('category', '?')}")
print(f" Q1: {art.get('q1_reason', '?')}")
print(f" Q2: {art.get('q2_reason', '?')}")
print(f" Attack: {art.get('attack_angle', '?')}")
print(f" Member regions ({len(member_ids)}): {member_ids}")
if member_ids:
min_x, min_y, max_x, max_y = 99999, 99999, 0, 0
for rid in member_ids:
r = region_map.get(rid, {})
rb = r.get("bbox", [0,0,0,0])
print(f" Region {rid:3d}: type={r.get('type','?'):15s} bbox=[{rb[0]:5d},{rb[1]:5d},{rb[2]:5d},{rb[3]:5d}] size={rb[2]-rb[0]:5d}x{rb[3]-rb[1]:5d}px")
min_x = min(min_x, rb[0])
min_y = min(min_y, rb[1])
max_x = max(max_x, rb[2])
max_y = max(max_y, rb[3])
print(f" Merged bbox: [{min_x},{min_y},{max_x},{max_y}] size={max_x-min_x}x{max_y-min_y}px")
print()
# Log rejected articles
if rejected:
print(f" REJECTED ARTICLES:")
for r in rejected:
print(f" ❌ {r.get('headline', '?')[:60]} — {r.get('reason', '?')[:60]}")
print()
# Stats
if political:
region_counts = [len(art.get("member_region_ids", [])) for art in political]
print(f" STATS:")
print(f" Political articles: {len(political)}")
print(f" Regions per article: min={min(region_counts)}, max={max(region_counts)}, avg={sum(region_counts)/len(region_counts):.1f}")
large = [i+1 for i,c in enumerate(region_counts) if c > 6]
if large:
print(f" ⚠️ Large articles (>6 regions): {large} — may have merged multiple stories")
print(f"{'='*70}\n")
# Draw debug image — political articles only with colored boundaries
try:
from PIL import ImageDraw
debug_img = Image.open(page_img_path).copy()
draw = ImageDraw.Draw(debug_img)
article_colors = [
(255, 0, 0), (0, 0, 255), (0, 150, 0), (180, 0, 180),
(200, 100, 0), (0, 150, 150), (150, 0, 0), (0, 0, 150),
(100, 100, 0), (200, 0, 100), (0, 100, 200), (150, 75, 0),
]
for idx, art in enumerate(political):
color = article_colors[idx % len(article_colors)]
member_ids = art.get("member_region_ids", [])
# Draw each member region
for rid in member_ids:
r = region_map.get(rid, {})
rb = r.get("bbox", [0,0,0,0])
draw.rectangle([rb[0], rb[1], rb[2], rb[3]], outline=color, width=5)
draw.text((rb[0]+8, rb[1]+8), f"R{rid}", fill=color)
# Draw merged outer boundary
if member_ids:
all_bboxes = [region_map.get(rid, {}).get("bbox", [0,0,0,0]) for rid in member_ids]
mx1 = min(b[0] for b in all_bboxes)
my1 = min(b[1] for b in all_bboxes)
mx2 = max(b[2] for b in all_bboxes)
my2 = max(b[3] for b in all_bboxes)
draw.rectangle([mx1-15, my1-15, mx2+15, my2+15], outline=color, width=15)
# Label
label = f"POLITICAL {idx+1}: {art.get('headline_english','')[:40]}"
draw.rectangle([mx1-15, my1-50, mx1 + len(label)*11, my1-15], fill=color)
draw.text((mx1-10, my1-48), label, fill="white")
debug_path = page_img_path.replace(".png", "_political_debug.png")
debug_img.save(debug_path)
print(f"POLITICAL DEBUG IMAGE saved: {debug_path}")
print(f" Only political articles shown with colored boundaries.")
print()
except Exception as e:
print(f" Could not create debug image: {e}")
page_summary.append((n, n_datelines, n_all_crops, n_orphan, n_claude, n_rejected))
if not political:
print(f" Page {n}: no political articles found")
continue
# Step 4: Crop using precise PaddleOCR boxes
print(f"STEP 4: Page {n} — cropping {len(political)} political articles...")
records = crop_political_articles(page_img_path, political, regions, articles_dir, n,
dateline_starts=dateline_starts, sep_lines=sep_lines)
all_records.extend(records)
all_political.extend(political)
# Diagnostic image: final crop rectangle of every article on the page.
draw_article_boundaries_debug(page_img_path, records, pages_dir, n)
print()
# Per-page article-count summary table (logged for every job).
# "Actual articles" mirrors the code-identified article-block count.
if page_summary:
print(f"{'='*84}")
print("PER-PAGE ARTICLE COUNT SUMMARY")
print(f"{'='*84}")
print(f" {'Page':<6}{'Datelines':<12}{'Blocks (code)':<16}{'Orphans/Other':<16}"
f"{'Claude kept':<14}{'Rejected':<10}")
t_dl = t_bl = t_or = t_cl = t_rj = 0
for n, n_dl, n_bl, n_or, n_cl, n_rj in page_summary:
print(f" {n:<6}{n_dl:<12}{n_bl:<16}{n_or:<16}{n_cl:<14}{n_rj:<10}")
t_dl += n_dl; t_bl += n_bl; t_or += n_or; t_cl += n_cl; t_rj += n_rj
print(f" {'-'*78}")
print(f" {'Total':<6}{t_dl:<12}{t_bl:<16}{t_or:<16}{t_cl:<14}{t_rj:<10}")
print(f"{'='*84}\n")
# Cross-page handling: KEEP BOTH (per-page).
# A story that shares a headline across pages (e.g. a front-page lead plus
# the inside-page body) is NOT de-duplicated here. Deleting the smaller crop
# was wrong: an over-merged crop on one page could "win" on raw area and
# silently delete a correctly-cropped article on another page (front-page
# leads were disappearing this way). True continuations are stitched
# separately by _continuation.py; every page's selected article is retained.
if all_political:
# Informational only — report same-headline matches without dropping any.
seen_by_key = {}
for art in all_political:
key = _normalize_telugu(art.get("headline_telugu", "")) \
or (art.get("headline_english", "") or "").strip().lower()
if not key:
continue
seen_by_key.setdefault(key, []).append(art)
for key, arts in seen_by_key.items():
if len(arts) > 1:
pages_str = ", ".join(str(a.get("_page", "?")) for a in arts)
print(f" ↔ Same headline on pages {pages_str}: "
f"'{arts[0].get('headline_english','?')[:40]}' — keeping all (per-page).")
# Step 5: Generate outputs
if all_political:
print("STEP 5: Generating outputs...")
(output_dir / "political_articles.json").write_text(
json.dumps(all_political, indent=2, ensure_ascii=False), encoding="utf-8"
)
generate_brief(all_political, output_dir)
copy_all_images(all_records, output_dir)
generate_pdf(all_records, output_dir)
# Post-run validator — surface crops/pages that likely need a human look.
write_review_report(all_political, pages_dir, output_dir,
crop_integrity_issues=crop_integrity_issues)
print()
print(f"DONE! {len(all_records)} political articles found across {len(pages)} pages.")
print(f"Output: {output_dir}")
return output_dir
def _collect_pdfs(paths):
"""Expand a mix of PDF files and folders into a de-duplicated, ordered PDF list."""
pdfs = []
for raw in paths:
p = Path(raw)
if p.is_dir():
pdfs.extend(sorted(p.glob("*.pdf")))
elif p.is_file() and p.suffix.lower() == ".pdf":
pdfs.append(p)
else:
print(f" ⚠️ Skipping (not a PDF or folder): {p}")
seen, unique = set(), []
for p in pdfs:
rp = p.resolve()
if rp not in seen:
seen.add(rp)
unique.append(p)
return unique
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
parser = argparse.ArgumentParser(description="Smart Extractor — political article detection")
parser.add_argument("paths", nargs="+",
help="One or more PDF files and/or folders containing PDFs")
parser.add_argument("--output", default=None,
help="Output base directory (default: ./output). "
"Each PDF gets its own subfolder <stem>_<timestamp>.")
parser.add_argument("--target-w", type=int, default=4200, help="Target page width in pixels")
parser.add_argument("--target-h", type=int, default=7400, help="Target page height in pixels")
args = parser.parse_args()
pdfs = _collect_pdfs(args.paths)
if not pdfs:
print("ERROR: no PDF files found in the given path(s).")
sys.exit(1)
base = Path(args.output) if args.output else Path("output")
print(f"Found {len(pdfs)} PDF(s) to process:")
for p in pdfs:
print(f" - {p.name}")
print()
batch_ts = datetime.now().strftime("%Y%m%d_%H%M%S")
results = []
all_crop_paths = []
for i, pdf in enumerate(pdfs, 1):
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
out_dir = base / f"{pdf.stem}_{timestamp}"
print("=" * 70)
print(f"[{i}/{len(pdfs)}] {pdf.name}")
print("=" * 70)
try:
process_pdf(pdf, out_dir, args.target_w, args.target_h)
all_crop_paths.extend(sorted((out_dir / "all_political_images").glob("*.png")))
results.append(("ok", pdf.name, out_dir))
except Exception as e:
import traceback
traceback.print_exc()
print(f" ❌ Failed on {pdf.name}: {e} — continuing with the next PDF.")
results.append(("FAILED", pdf.name, out_dir))
# One combined PDF of every cropped article across all processed PDFs.
if all_crop_paths:
print()
print(f"Building combined PDF of {len(all_crop_paths)} crop(s) from all PDFs...")
generate_combined_pdf(all_crop_paths, base / f"ALL_crops_{batch_ts}.pdf")
print()
print("=" * 70)
print(f"BATCH COMPLETE — {len(pdfs)} PDF(s)")
for status, name, out_dir in results:
print(f" {status:>7} {name} → {out_dir}")
if all_crop_paths:
print(f" Combined crops PDF → {base / f'ALL_crops_{batch_ts}.pdf'}")
print("=" * 70)
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink