148 lines
5.4 KiB
Python
148 lines
5.4 KiB
Python
"""Production-path regression harness for grow_articles_final.
|
|
|
|
Unlike _regress_grow.py (which calls _grow_article_downward directly with
|
|
dateline_starts=None and so over-estimates wrapping), this mirrors the REAL
|
|
pipeline tail: it computes dateline_starts via find_article_starts_by_dateline
|
|
and runs grow_articles_final exactly as production does. A diff of two snapshots
|
|
therefore isolates what the code change does ON THE SHIPPING PATH.
|
|
|
|
Usage:
|
|
venv/bin/python3 _regress_final.py snapshot.json # snapshot current code
|
|
venv/bin/python3 _regress_final.py --diff a.json b.json # compare two snapshots
|
|
"""
|
|
import json
|
|
import sys
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
OUT = Path("output")
|
|
|
|
|
|
def paper_of_height(h):
|
|
if 7040 <= h <= 7060:
|
|
return "sakshi"
|
|
if h in (6833, 6549):
|
|
return "andhra_jyothi"
|
|
if h == 6422:
|
|
return "namaste_telangana"
|
|
return "unknown"
|
|
|
|
|
|
def union_bbox(member_ids, region_map):
|
|
bs = [region_map[r]["bbox"] for r in member_ids if r in region_map]
|
|
if not bs:
|
|
return None
|
|
return [min(b[0] for b in bs), min(b[1] for b in bs),
|
|
max(b[2] for b in bs), max(b[3] for b in bs)]
|
|
|
|
|
|
def run():
|
|
import copy
|
|
from PIL import Image
|
|
from smart_extractor import find_article_starts_by_dateline, grow_articles_final
|
|
|
|
runs = sorted([d for d in OUT.iterdir() if d.is_dir()],
|
|
key=lambda d: d.stat().st_mtime, reverse=True)
|
|
snapshot = {}
|
|
papers = Counter()
|
|
for run_dir in runs:
|
|
pages_dir = run_dir / "pages"
|
|
if not pages_dir.exists():
|
|
continue
|
|
for reg_path in sorted(pages_dir.glob("page_*.regions.json")):
|
|
pol_path = reg_path.with_name(reg_path.name.replace(
|
|
".regions.json", ".political_result.json"))
|
|
png = reg_path.with_name(reg_path.name.replace(".regions.json", ".png"))
|
|
if not pol_path.exists() or not png.exists():
|
|
continue
|
|
regions = json.loads(reg_path.read_text())["regions"]
|
|
region_map = {r["id"]: r for r in regions}
|
|
political = json.loads(pol_path.read_text()).get("political_articles", [])
|
|
if not political:
|
|
continue
|
|
try:
|
|
paper = paper_of_height(Image.open(png).height)
|
|
dateline_starts = find_article_starts_by_dateline(
|
|
regions, str(png), paper)
|
|
grown = grow_articles_final(
|
|
copy.deepcopy(political), regions, dateline_starts)
|
|
except Exception as e:
|
|
print(f" SKIP {run_dir.name}/{reg_path.stem}: {e}")
|
|
continue
|
|
|
|
for i, art in enumerate(grown):
|
|
key = f"{run_dir.name}/{reg_path.stem}/art{i}"
|
|
members = sorted(art.get("member_region_ids", []) or [])
|
|
snapshot[key] = {
|
|
"paper": paper,
|
|
"headline": art.get("headline_english", "")[:50],
|
|
"members": members,
|
|
"bbox": union_bbox(members, region_map),
|
|
}
|
|
papers[paper] += 1
|
|
print(f" papers: {dict(papers)}")
|
|
return snapshot
|
|
|
|
|
|
def diff(path_a, path_b):
|
|
a = json.loads(Path(path_a).read_text())
|
|
b = json.loads(Path(path_b).read_text())
|
|
keys = sorted(set(a) | set(b))
|
|
|
|
per_total = Counter()
|
|
per_changed = Counter()
|
|
grew = Counter()
|
|
shrank = Counter()
|
|
samples = []
|
|
|
|
for k in keys:
|
|
ra, rb = a.get(k), b.get(k)
|
|
paper = (rb or ra).get("paper", "unknown")
|
|
per_total[paper] += 1
|
|
if ra is None or rb is None:
|
|
per_changed[paper] += 1
|
|
samples.append((k, paper, "ONLY IN ONE SNAPSHOT", None, None))
|
|
continue
|
|
sa, sb = set(ra["members"]), set(rb["members"])
|
|
if sa != sb:
|
|
per_changed[paper] += 1
|
|
added = sorted(sb - sa)
|
|
removed = sorted(sa - sb)
|
|
if added:
|
|
grew[paper] += 1
|
|
if removed:
|
|
shrank[paper] += 1
|
|
samples.append((k, paper, rb["headline"], added, removed))
|
|
|
|
print("\n=== PRODUCTION-PATH DIFF (A=baseline/OLD -> B=new) ===")
|
|
print(f"{'paper':<18}{'articles':>10}{'changed':>9}{'grew':>7}{'shrank':>8}")
|
|
for paper in sorted(per_total):
|
|
print(f"{paper:<18}{per_total[paper]:>10}{per_changed[paper]:>9}"
|
|
f"{grew[paper]:>7}{shrank[paper]:>8}")
|
|
tot = sum(per_total.values())
|
|
chg = sum(per_changed.values())
|
|
print(f"{'TOTAL':<18}{tot:>10}{chg:>9}{sum(grew.values()):>7}{sum(shrank.values()):>8}")
|
|
|
|
if samples:
|
|
print(f"\n--- {len(samples)} changed article(s) ---")
|
|
for k, paper, hl, added, removed in samples:
|
|
print(f"\n[{paper}] {k}")
|
|
print(f" headline: {hl}")
|
|
if added:
|
|
print(f" + added : {added}")
|
|
if removed:
|
|
print(f" - removed: {removed}")
|
|
else:
|
|
print("\nNo differences — member sets identical across all fixtures.")
|
|
return chg
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) >= 4 and sys.argv[1] == "--diff":
|
|
diff(sys.argv[2], sys.argv[3])
|
|
else:
|
|
out_path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/final_snapshot.json"
|
|
snap = run()
|
|
Path(out_path).write_text(json.dumps(snap, indent=2, ensure_ascii=False))
|
|
print(f"wrote {len(snap)} article snapshots to {out_path}")
|