deepkoluguri
/
sundeep-news-scan
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: refactor status tracking to use meta.json directly for reliable UI progress updates

This commit is contained in:
Deep Koluguri 2026-06-13 00:28:30 -04:00
parent bb504efd02
commit ec798a06ba
2 changed files with 35 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
app.py
View File

@ -41,8 +41,7 @@ if hasattr(sys.stderr, 'reconfigure'):
from extractor import process_pdf from extractor import process_pdf
JOBS = {}
JOB_LOCK = threading.Lock()
ROOT = Path(__file__).parent ROOT = Path(__file__).parent
UPLOAD_DIR = ROOT / "uploads" UPLOAD_DIR = ROOT / "uploads"
@ -124,8 +123,12 @@ def upload():
def _run_job(job_id, pdf_path, job_dir, target_w, target_h): def _run_job(job_id, pdf_path, job_dir, target_w, target_h):
try: try:
with JOB_LOCK: meta_path = Path(job_dir) / "meta.json"
JOBS[job_id] = {"status": "running", "message": "Starting..."} meta = json.loads(meta_path.read_text())
meta["status"] = "running"
meta["message"] = "Starting..."
meta_path.write_text(json.dumps(meta, indent=2))
result = process_pdf(pdf_path, job_dir, target_w, target_h, job_id) result = process_pdf(pdf_path, job_dir, target_w, target_h, job_id)
meta_path = Path(job_dir) / "meta.json" meta_path = Path(job_dir) / "meta.json"
meta = json.loads(meta_path.read_text()) meta = json.loads(meta_path.read_text())
@ -134,9 +137,9 @@ def _run_job(job_id, pdf_path, job_dir, target_w, target_h):
"pages": result["pages"], "pages": result["pages"],
"articles": result["articles"], "articles": result["articles"],
}) })
meta["status"] = "done"
meta["message"] = "Finished."
meta_path.write_text(json.dumps(meta, indent=2)) meta_path.write_text(json.dumps(meta, indent=2))
with JOB_LOCK:
JOBS[job_id] = {"status": "done", "message": "Finished."}
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
meta_path = Path(job_dir) / "meta.json" meta_path = Path(job_dir) / "meta.json"
@ -144,11 +147,10 @@ def _run_job(job_id, pdf_path, job_dir, target_w, target_h):
meta = json.loads(meta_path.read_text()) meta = json.loads(meta_path.read_text())
meta["status"] = "error" meta["status"] = "error"
meta["error"] = str(e) meta["error"] = str(e)
meta["message"] = str(e)
meta_path.write_text(json.dumps(meta, indent=2)) meta_path.write_text(json.dumps(meta, indent=2))
except Exception: except Exception:
pass pass
with JOB_LOCK:
JOBS[job_id] = {"status": "error", "message": str(e)}
@app.route("/job/<job_id>") @app.route("/job/<job_id>")
@ -182,16 +184,17 @@ def job_view(job_id):
@app.route("/job/<job_id>/status") @app.route("/job/<job_id>/status")
def job_status(job_id): def job_status(job_id):
with JOB_LOCK:
status = JOBS.get(job_id, {"status": "unknown"})
meta_path = OUTPUT_DIR / job_id / "meta.json" meta_path = OUTPUT_DIR / job_id / "meta.json"
status_data = {"status": "unknown"}
if meta_path.exists(): if meta_path.exists():
try: try:
meta = json.loads(meta_path.read_text()) meta = json.loads(meta_path.read_text())
status["meta"] = meta status_data["status"] = meta.get("status", "unknown")
status_data["message"] = meta.get("message", "")
status_data["meta"] = meta
except Exception: except Exception:
pass pass
return jsonify(status) return jsonify(status_data)
@app.route("/job/<job_id>/page/<int:page_num>") @app.route("/job/<job_id>/page/<int:page_num>")

36
extractor.py
View File

@ -40,17 +40,21 @@ if not os.environ.get("ANTHROPIC_API_KEY"):
import fitz # PyMuPDF import fitz # PyMuPDF
from PIL import Image from PIL import Image
JOBS = {}
JOB_LOCK = threading.Lock()
_LAYOUT = None _LAYOUT = None
_LAYOUT_LOCK = threading.Lock() _LAYOUT_LOCK = threading.Lock()
def _set_status(job_id, msg): def _set_status(job_dir, msg):
with JOB_LOCK: try:
if job_id in JOBS: meta_path = Path(job_dir) / "meta.json"
JOBS[job_id]["message"] = msg if meta_path.exists():
meta = json.loads(meta_path.read_text(encoding="utf-8"))
meta["message"] = msg
meta_path.write_text(json.dumps(meta, indent=2, ensure_ascii=False), encoding="utf-8")
except Exception:
pass
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@ -693,7 +697,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
articles_root.mkdir(parents=True, exist_ok=True) articles_root.mkdir(parents=True, exist_ok=True)
# Step 1: Render PDF pages # Step 1: Render PDF pages
_set_status(job_id, "Rendering PDF pages...") _set_status(job_dir, "Rendering PDF pages...")
pages = render_pdf_pages(pdf_path, pages_dir, target_w, target_h, max_pages=max_pages, pages_to_process=pages_to_process) pages = render_pdf_pages(pdf_path, pages_dir, target_w, target_h, max_pages=max_pages, pages_to_process=pages_to_process)
all_records = [] all_records = []
@ -704,7 +708,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
page_img_path = p["path"] page_img_path = p["path"]
# Step 2: PaddleOCR layout detection — precise bounding boxes # Step 2: PaddleOCR layout detection — precise bounding boxes
_set_status(job_id, f"Page {n}: detecting layout regions...") _set_status(job_dir, f"Page {n}: detecting layout regions...")
regions = detect_regions(page_img_path) regions = detect_regions(page_img_path)
(pages_dir / f"page_{n:03d}.regions.json").write_text( (pages_dir / f"page_{n:03d}.regions.json").write_text(
json.dumps({"page": n, "regions": regions}, indent=2, ensure_ascii=False), encoding="utf-8" json.dumps({"page": n, "regions": regions}, indent=2, ensure_ascii=False), encoding="utf-8"
@ -712,7 +716,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
print(f" Page {n}: {len(regions)} layout regions detected") print(f" Page {n}: {len(regions)} layout regions detected")
# Step 3: Group regions into articles # Step 3: Group regions into articles
_set_status(job_id, f"Page {n}: grouping articles...") _set_status(job_dir, f"Page {n}: grouping articles...")
page_img = Image.open(page_img_path) page_img = Image.open(page_img_path)
articles, enriched = group_into_articles(page_img_path, regions, p["width"], page_img=page_img) articles, enriched = group_into_articles(page_img_path, regions, p["width"], page_img=page_img)
@ -845,14 +849,14 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
except Exception as e: except Exception as e:
print(f" Could not create articles debug image: {e}") print(f" Could not create articles debug image: {e}")
# Step 4: Crop and OCR HEADLINES only # Step 4: Crop and OCR HEADLINES only
_set_status(job_id, f"Page {n}: cropping {len(articles)} headlines...") _set_status(job_dir, f"Page {n}: cropping {len(articles)} headlines...")
headline_records = crop_headlines(page_img_path, articles, regions, articles_root, n) headline_records = crop_headlines(page_img_path, articles, regions, articles_root, n)
_set_status(job_id, f"Page {n}: running fast OCR on {len(headline_records)} headlines...") _set_status(job_dir, f"Page {n}: running fast OCR on {len(headline_records)} headlines...")
ocr_headlines(headline_records) ocr_headlines(headline_records)
# Step 5: Triage with Claude Haiku based on headlines # Step 5: Triage with Claude Haiku based on headlines
_set_status(job_id, f"Page {n}: triaging headlines with Claude...") _set_status(job_dir, f"Page {n}: triaging headlines with Claude...")
selected_ids = _triage_headlines_with_claude(headline_records, n, job_dir=str(job_dir)) selected_ids = _triage_headlines_with_claude(headline_records, n, job_dir=str(job_dir))
# Cleanup non-selected directories # Cleanup non-selected directories
@ -865,16 +869,16 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
continue continue
# Step 6: Crop FULL articles for the selected ones # Step 6: Crop FULL articles for the selected ones
_set_status(job_id, f"Page {n}: cropping {len(selected_ids)} FULL articles...") _set_status(job_dir, f"Page {n}: cropping {len(selected_ids)} FULL articles...")
records = crop_full_articles(page_img_path, selected_ids, articles, articles_root, n) records = crop_full_articles(page_img_path, selected_ids, articles, articles_root, n)
print(f" Page {n}: cropped {len(records)} full articles") print(f" Page {n}: cropped {len(records)} full articles")
# Step 7: OCR full articles # Step 7: OCR full articles
_set_status(job_id, f"Page {n}: running heavy OCR on {len(records)} full articles...") _set_status(job_dir, f"Page {n}: running heavy OCR on {len(records)} full articles...")
ocr_articles(records) ocr_articles(records)
# Step 8: Deep Analysis with Claude Opus # Step 8: Deep Analysis with Claude Opus
_set_status(job_id, f"Page {n}: deep political analysis with Claude...") _set_status(job_dir, f"Page {n}: deep political analysis with Claude...")
selected = _analyze_articles_with_claude(records, n, job_dir=str(job_dir)) selected = _analyze_articles_with_claude(records, n, job_dir=str(job_dir))
if selected: if selected:
@ -996,5 +1000,5 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
if all_selected and generate_political_pdf: if all_selected and generate_political_pdf:
generate_political_pdf(all_selected, job_dir) generate_political_pdf(all_selected, job_dir)
_set_status(job_id, "Done!") _set_status(job_dir, "Done!")
return {"pages": len(pages), "articles": len(all_records)} return {"pages": len(pages), "articles": len(all_records)}