"""Construct one boundary box per article by fusing three signals: 1. DATELINE pattern -> the MINIMUM set of articles (one per ', date:'). 2. HEADLINE above -> the article's top anchor / column span (doc_title). 3. STRAIGHT RULES -> hard walls: snap top/bottom to horizontal rules and left/right to vertical rules between stories. Pure geometry on saved fixtures + the page PNG. No Claude, no API. construct_boundaries(regions, datelines, lines, size) -> list of: {region_id, dateline, headline_region, bbox:[l,t,r,b], walls:{...}} """ def _cx(b): return (b[0] + b[2]) / 2 def construct_boundaries(regions, datelines, lines, size): W, H = size rmap = {r["id"]: r for r in regions} h_rules = sorted(lines["h"], key=lambda L: L["y1"]) v_rules = sorted(lines["v"], key=lambda L: L["x1"]) dls = sorted(datelines, key=lambda s: s["bbox"][1]) # ---- duplicate-dateline dedup: PaddleOCR sometimes boxes the SAME physical text # twice (and a false dateline match — an ad/phone line — then appears as two # near-identical bodies). Two datelines are one article when EITHER their bodies # overlap heavily (IoU>0.6) OR one body is mostly nested inside the other AND the # dateline text is identical (the same town double-boxed as a thin line + a tall # paragraph — IoU stays low but containment is near 1). Keep the larger body and # inherit a headline from the dropped twin if the keeper had none. def _area(b): return (b[2]-b[0]) * (b[3]-b[1]) def _same_dateline(sa, sb): a, b = sa["bbox"], sb["bbox"] ix = max(0, min(a[2], b[2]) - max(a[0], b[0])) iy = max(0, min(a[3], b[3]) - max(a[1], b[1])) inter = ix * iy if not inter: return False aa, ba = _area(a), _area(b) iou = inter / (aa + ba - inter) if (aa + ba - inter) else 0.0 if iou > 0.6: return True contain = inter / min(aa, ba) if min(aa, ba) else 0.0 ta = (sa.get("dateline") or "").strip() tb = (sb.get("dateline") or "").strip() return contain > 0.8 and ta != "" and ta == tb deduped = [] for s in dls: dup = next((k for k in deduped if _same_dateline(s, k)), None) if dup is None: deduped.append(s) else: # keep larger body; inherit a missing headline keep, drop = (s, dup) if _area(s["bbox"]) > _area(dup["bbox"]) else (dup, s) if keep.get("headline_region") not in rmap and drop.get("headline_region") in rmap: keep = {**keep, "headline_region": drop["headline_region"]} deduped[deduped.index(dup)] = keep dls = deduped # ---- shared-headline dedup: a doc_title can only head ONE article. When two # datelines both matched the same headline (stacked in a column, or a 2-col # headline with a missed neighbour), keep it for the dateline with the SMALLEST # positive gap below the headline; the others lose it (anchor on their own body). by_hl = {} for s in dls: h = s.get("headline_region") if h in rmap: by_hl.setdefault(h, []).append(s) hid_of = {id(s): s.get("headline_region") for s in dls} for h, group in by_hl.items(): if len(group) <= 1: continue hb = rmap[h]["bbox"] below = [s for s in group if s["bbox"][1] >= hb[3] - 30] keeper = min(below, key=lambda s: s["bbox"][1] - hb[3]) if below \ else min(group, key=lambda s: abs(s["bbox"][1] - hb[3])) for s in group: if s is not keeper: hid_of[id(s)] = None def _anchor_rect(ns): """A neighbour's article ANCHOR = its headline (if it owns one) UNION its dateline body. Floors are computed against this rect — not the bare body — so a story ends at the TOP of the next story's HEADLINE, and a wide 2-column headline that reaches sideways into this column still acts as a wall.""" b = ns["bbox"] h = hid_of.get(id(ns)) if h in rmap: hb = rmap[h]["bbox"] return (min(b[0], hb[0]), min(b[1], hb[1]), max(b[2], hb[2]), max(b[3], hb[3])) return (b[0], b[1], b[2], b[3]) def _covers(L, y0, y1): # the rule must span the MAJORITY of [y0,y1], not merely touch an edge ov = min(L["y2"], y1) - max(L["y1"], y0) return ov >= 0.5 * max(1, y1 - y0) def vwall_left(cx, y0, y1): cands = [L for L in v_rules if L["x1"] < cx - 20 and _covers(L, y0, y1)] return max((L["x1"] for L in cands), default=0) def vwall_right(cx, y0, y1): cands = [L for L in v_rules if L["x1"] > cx + 20 and _covers(L, y0, y1)] return min((L["x1"] for L in cands), default=W) def hrule_between(x0, x1, y_lo, y_hi, want_above): """Return the horizontal rule whose span covers [x0,x1] within (y_lo,y_hi). want_above=True -> the LOWEST such rule (closest above the body's top is the bottom-most of those above); we pick nearest to the reference edge.""" hits = [L for L in h_rules if L["y1"] > y_lo and L["y1"] < y_hi and L["x1"] <= x1 - 30 and L["x2"] >= x0 + 30] if not hits: return None # nearest to the reference edge: above -> max y (closest below ceiling ref), # below -> min y (closest above floor ref) return max(hits, key=lambda L: L["y1"]) if want_above \ else min(hits, key=lambda L: L["y1"]) out = [] for i, s in enumerate(dls): db = s["bbox"] cx = _cx(db) hid = hid_of[id(s)] hb = rmap[hid]["bbox"] if (hid in rmap) else None # ---- horizontal span: UNION of headline + dateline body so the body is # always contained; top anchor is the headline top if present. if hb: left0, right0 = min(hb[0], db[0]), max(hb[2], db[2]) top_anchor = hb[1] else: left0, right0 = db[0], db[2] top_anchor = db[1] # ---- CEILING: snap up to a horizontal rule just above the anchor (but no # higher than the previous article's dateline in this column). prev_y = 0 for ps in dls: if ps is s: continue pb = ps["bbox"] if pb[0] - 40 <= cx <= pb[2] + 40 and pb[3] <= top_anchor: prev_y = max(prev_y, pb[3]) ceil_rule = hrule_between(left0, right0, prev_y, top_anchor + 20, want_above=True) # a ceiling rule only counts if it sits a sane distance ABOVE the anchor # (a far-away rule is a different story's divider, not this article's top). top = top_anchor if ceil_rule and 0 <= (top_anchor - ceil_rule["y1"]) <= 500: top = ceil_rule["y1"] else: ceil_rule = None top = min(top, top_anchor) # never cut INTO the article from above # ---- FLOOR: nearest boundary directly below in THIS dateline's OWN column # (own column, not the wide headline span — a neighbour under a 2-col # headline must clip the SIDE, not act as the floor). own_l, own_r = db[0], db[2] next_y = H for ns in dls: if ns is s: continue na = _anchor_rect(ns) # headline ∪ body of neighbour if na[1] > db[1] + 30 and not (na[2] < own_l + 20 or na[0] > own_r - 20): next_y = min(next_y, na[1]) floor_rule = hrule_between(own_l, own_r, db[3], next_y + 5, want_above=False) bottom = floor_rule["y1"] if floor_rule else next_y bottom = max(bottom, db[3]) # never cut INTO the article from below # ---- SIDE-CLIP: a neighbour dateline that sits beside this body within the # (wide) headline span clips the corresponding edge, so a left-column # story never swallows the right column's separate story. for ns in dls: if ns is s: continue nb = ns["bbox"] if nb[3] > top_anchor + 20 and nb[1] < bottom - 20: # vertical overlap if own_r - 10 <= nb[0] < right0: # neighbour to the right right0 = min(right0, nb[0]) if left0 < nb[2] <= own_l + 10: # neighbour to the left left0 = max(left0, nb[2]) # ---- side walls: snap to vertical rules spanning the article's y-band lwall = vwall_left(cx, top, bottom) rwall = vwall_right(cx, top, bottom) left = max(left0 - 20, lwall) right = min(right0 + 20, rwall) left = min(left, db[0]) # guarantee the dateline body fits right = max(right, db[2]) out.append({ "region_id": s["region_id"], "dateline": s.get("dateline", ""), "headline_region": hid, "bbox": [int(left), int(top), int(right), int(bottom)], "walls": {"ceil_rule": bool(ceil_rule), "floor_rule": bool(floor_rule), "lwall": int(lwall), "rwall": int(rwall)}, }) return out