35 lines
2.1 KiB
Python
35 lines
2.1 KiB
Python
import os,json,tempfile,contextlib,io,glob,re
|
|
tempfile.tempdir=os.path.abspath("_tess_tmp"); os.environ["TMPDIR"]=tempfile.tempdir
|
|
import sys; sys.path.insert(0,"."); from pathlib import Path
|
|
import smart_extractor as se
|
|
from _lines import separator_barriers
|
|
o=open("_testcls.txt","w")
|
|
def paper_of(n): return "andhra_jyothi" if n.lower().replace(" ","").startswith("andhrajyothi") else "namaste_telangana"
|
|
for run in sorted(glob.glob("output/*/")):
|
|
name=Path(run).name; paper=paper_of(name); D=Path(run)/"pages"
|
|
pages=sorted(set(int(re.search(r"page_(\d+)\.png$",p).group(1)) for p in glob.glob(str(D/"page_*.png")) if re.search(r"page_\d+\.png$",p)))
|
|
if not pages: continue
|
|
o.write(f"\n=== {name} [{paper}] ===\n")
|
|
for pg in pages:
|
|
png=str(D/f"page_{pg:03d}.png"); rf=D/f"page_{pg:03d}.regions.json"
|
|
if not rf.exists(): continue
|
|
regs=json.loads(rf.read_text())["regions"]
|
|
buf=io.StringIO()
|
|
try:
|
|
with contextlib.redirect_stdout(buf):
|
|
regs=se.dedup_overlapping_regions(regs); regs=se.merge_stacked_title_lines(regs,png)
|
|
regs=se.drop_masthead_regions(regs,paper,pg,png)
|
|
regs=se.drop_classifieds_regions(regs,png)
|
|
regs=se.promote_paragraph_titles(regs,png,paper); regs=se.mark_bullet_titles(regs,png)
|
|
ds=se.find_article_starts_by_dateline(regs,png,paper); bars=separator_barriers(png,regs)
|
|
_p=se.cluster_all_article_blocks(regs,ds,sep_lines=bars)
|
|
if se.recover_orphan_text_headlines(_p,regs,png): ds=se.find_article_starts_by_dateline(regs,png,paper)
|
|
blocks=se.cluster_all_article_blocks(regs,ds,sep_lines=bars)
|
|
dl={s['region_id'] for s in ds}
|
|
over=[(b['anchor_id'],sum(1 for m in b['members'] if m in dl)) for b in blocks if sum(1 for m in b['members'] if m in dl)>=2]
|
|
cls=[l.strip() for l in buf.getvalue().splitlines() if 'Classifieds:' in l]
|
|
o.write(f" pg{pg}: {len(blocks)} blocks | 2+dl={over} | {cls[0] if cls else 'no classifieds'}\n")
|
|
except Exception as e:
|
|
o.write(f" pg{pg}: ERROR {e}\n")
|
|
o.close();print("WROTE")
|