diff --git a/app.py b/app.py index 98e2906..6a70874 100644 --- a/app.py +++ b/app.py @@ -41,8 +41,7 @@ if hasattr(sys.stderr, 'reconfigure'): from extractor import process_pdf -JOBS = {} -JOB_LOCK = threading.Lock() + ROOT = Path(__file__).parent UPLOAD_DIR = ROOT / "uploads" @@ -124,8 +123,12 @@ def upload(): def _run_job(job_id, pdf_path, job_dir, target_w, target_h): try: - with JOB_LOCK: - JOBS[job_id] = {"status": "running", "message": "Starting..."} + meta_path = Path(job_dir) / "meta.json" + meta = json.loads(meta_path.read_text()) + meta["status"] = "running" + meta["message"] = "Starting..." + meta_path.write_text(json.dumps(meta, indent=2)) + result = process_pdf(pdf_path, job_dir, target_w, target_h, job_id) meta_path = Path(job_dir) / "meta.json" meta = json.loads(meta_path.read_text()) @@ -134,9 +137,9 @@ def _run_job(job_id, pdf_path, job_dir, target_w, target_h): "pages": result["pages"], "articles": result["articles"], }) + meta["status"] = "done" + meta["message"] = "Finished." meta_path.write_text(json.dumps(meta, indent=2)) - with JOB_LOCK: - JOBS[job_id] = {"status": "done", "message": "Finished."} except Exception as e: traceback.print_exc() meta_path = Path(job_dir) / "meta.json" @@ -144,11 +147,10 @@ def _run_job(job_id, pdf_path, job_dir, target_w, target_h): meta = json.loads(meta_path.read_text()) meta["status"] = "error" meta["error"] = str(e) + meta["message"] = str(e) meta_path.write_text(json.dumps(meta, indent=2)) except Exception: pass - with JOB_LOCK: - JOBS[job_id] = {"status": "error", "message": str(e)} @app.route("/job/") @@ -182,16 +184,17 @@ def job_view(job_id): @app.route("/job//status") def job_status(job_id): - with JOB_LOCK: - status = JOBS.get(job_id, {"status": "unknown"}) meta_path = OUTPUT_DIR / job_id / "meta.json" + status_data = {"status": "unknown"} if meta_path.exists(): try: meta = json.loads(meta_path.read_text()) - status["meta"] = meta + status_data["status"] = meta.get("status", "unknown") + status_data["message"] = meta.get("message", "") + status_data["meta"] = meta except Exception: pass - return jsonify(status) + return jsonify(status_data) @app.route("/job//page/") diff --git a/extractor.py b/extractor.py index de8d09e..991e83e 100644 --- a/extractor.py +++ b/extractor.py @@ -40,17 +40,21 @@ if not os.environ.get("ANTHROPIC_API_KEY"): import fitz # PyMuPDF from PIL import Image -JOBS = {} -JOB_LOCK = threading.Lock() + _LAYOUT = None _LAYOUT_LOCK = threading.Lock() -def _set_status(job_id, msg): - with JOB_LOCK: - if job_id in JOBS: - JOBS[job_id]["message"] = msg +def _set_status(job_dir, msg): + try: + meta_path = Path(job_dir) / "meta.json" + if meta_path.exists(): + meta = json.loads(meta_path.read_text(encoding="utf-8")) + meta["message"] = msg + meta_path.write_text(json.dumps(meta, indent=2, ensure_ascii=False), encoding="utf-8") + except Exception: + pass # --------------------------------------------------------------------------- @@ -693,7 +697,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p articles_root.mkdir(parents=True, exist_ok=True) # Step 1: Render PDF pages - _set_status(job_id, "Rendering PDF pages...") + _set_status(job_dir, "Rendering PDF pages...") pages = render_pdf_pages(pdf_path, pages_dir, target_w, target_h, max_pages=max_pages, pages_to_process=pages_to_process) all_records = [] @@ -704,7 +708,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p page_img_path = p["path"] # Step 2: PaddleOCR layout detection — precise bounding boxes - _set_status(job_id, f"Page {n}: detecting layout regions...") + _set_status(job_dir, f"Page {n}: detecting layout regions...") regions = detect_regions(page_img_path) (pages_dir / f"page_{n:03d}.regions.json").write_text( json.dumps({"page": n, "regions": regions}, indent=2, ensure_ascii=False), encoding="utf-8" @@ -712,7 +716,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p print(f" Page {n}: {len(regions)} layout regions detected") # Step 3: Group regions into articles - _set_status(job_id, f"Page {n}: grouping articles...") + _set_status(job_dir, f"Page {n}: grouping articles...") page_img = Image.open(page_img_path) articles, enriched = group_into_articles(page_img_path, regions, p["width"], page_img=page_img) @@ -845,14 +849,14 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p except Exception as e: print(f" Could not create articles debug image: {e}") # Step 4: Crop and OCR HEADLINES only - _set_status(job_id, f"Page {n}: cropping {len(articles)} headlines...") + _set_status(job_dir, f"Page {n}: cropping {len(articles)} headlines...") headline_records = crop_headlines(page_img_path, articles, regions, articles_root, n) - _set_status(job_id, f"Page {n}: running fast OCR on {len(headline_records)} headlines...") + _set_status(job_dir, f"Page {n}: running fast OCR on {len(headline_records)} headlines...") ocr_headlines(headline_records) # Step 5: Triage with Claude Haiku based on headlines - _set_status(job_id, f"Page {n}: triaging headlines with Claude...") + _set_status(job_dir, f"Page {n}: triaging headlines with Claude...") selected_ids = _triage_headlines_with_claude(headline_records, n, job_dir=str(job_dir)) # Cleanup non-selected directories @@ -865,16 +869,16 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p continue # Step 6: Crop FULL articles for the selected ones - _set_status(job_id, f"Page {n}: cropping {len(selected_ids)} FULL articles...") + _set_status(job_dir, f"Page {n}: cropping {len(selected_ids)} FULL articles...") records = crop_full_articles(page_img_path, selected_ids, articles, articles_root, n) print(f" Page {n}: cropped {len(records)} full articles") # Step 7: OCR full articles - _set_status(job_id, f"Page {n}: running heavy OCR on {len(records)} full articles...") + _set_status(job_dir, f"Page {n}: running heavy OCR on {len(records)} full articles...") ocr_articles(records) # Step 8: Deep Analysis with Claude Opus - _set_status(job_id, f"Page {n}: deep political analysis with Claude...") + _set_status(job_dir, f"Page {n}: deep political analysis with Claude...") selected = _analyze_articles_with_claude(records, n, job_dir=str(job_dir)) if selected: @@ -996,5 +1000,5 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p if all_selected and generate_political_pdf: generate_political_pdf(all_selected, job_dir) - _set_status(job_id, "Done!") + _set_status(job_dir, "Done!") return {"pages": len(pages), "articles": len(all_records)}