feat: refactor status tracking to use meta.json directly for reliable UI progress updates
This commit is contained in:
parent
bb504efd02
commit
ec798a06ba
27
app.py
27
app.py
|
|
@ -41,8 +41,7 @@ if hasattr(sys.stderr, 'reconfigure'):
|
|||
|
||||
from extractor import process_pdf
|
||||
|
||||
JOBS = {}
|
||||
JOB_LOCK = threading.Lock()
|
||||
|
||||
|
||||
ROOT = Path(__file__).parent
|
||||
UPLOAD_DIR = ROOT / "uploads"
|
||||
|
|
@ -124,8 +123,12 @@ def upload():
|
|||
|
||||
def _run_job(job_id, pdf_path, job_dir, target_w, target_h):
|
||||
try:
|
||||
with JOB_LOCK:
|
||||
JOBS[job_id] = {"status": "running", "message": "Starting..."}
|
||||
meta_path = Path(job_dir) / "meta.json"
|
||||
meta = json.loads(meta_path.read_text())
|
||||
meta["status"] = "running"
|
||||
meta["message"] = "Starting..."
|
||||
meta_path.write_text(json.dumps(meta, indent=2))
|
||||
|
||||
result = process_pdf(pdf_path, job_dir, target_w, target_h, job_id)
|
||||
meta_path = Path(job_dir) / "meta.json"
|
||||
meta = json.loads(meta_path.read_text())
|
||||
|
|
@ -134,9 +137,9 @@ def _run_job(job_id, pdf_path, job_dir, target_w, target_h):
|
|||
"pages": result["pages"],
|
||||
"articles": result["articles"],
|
||||
})
|
||||
meta["status"] = "done"
|
||||
meta["message"] = "Finished."
|
||||
meta_path.write_text(json.dumps(meta, indent=2))
|
||||
with JOB_LOCK:
|
||||
JOBS[job_id] = {"status": "done", "message": "Finished."}
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
meta_path = Path(job_dir) / "meta.json"
|
||||
|
|
@ -144,11 +147,10 @@ def _run_job(job_id, pdf_path, job_dir, target_w, target_h):
|
|||
meta = json.loads(meta_path.read_text())
|
||||
meta["status"] = "error"
|
||||
meta["error"] = str(e)
|
||||
meta["message"] = str(e)
|
||||
meta_path.write_text(json.dumps(meta, indent=2))
|
||||
except Exception:
|
||||
pass
|
||||
with JOB_LOCK:
|
||||
JOBS[job_id] = {"status": "error", "message": str(e)}
|
||||
|
||||
|
||||
@app.route("/job/<job_id>")
|
||||
|
|
@ -182,16 +184,17 @@ def job_view(job_id):
|
|||
|
||||
@app.route("/job/<job_id>/status")
|
||||
def job_status(job_id):
|
||||
with JOB_LOCK:
|
||||
status = JOBS.get(job_id, {"status": "unknown"})
|
||||
meta_path = OUTPUT_DIR / job_id / "meta.json"
|
||||
status_data = {"status": "unknown"}
|
||||
if meta_path.exists():
|
||||
try:
|
||||
meta = json.loads(meta_path.read_text())
|
||||
status["meta"] = meta
|
||||
status_data["status"] = meta.get("status", "unknown")
|
||||
status_data["message"] = meta.get("message", "")
|
||||
status_data["meta"] = meta
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify(status)
|
||||
return jsonify(status_data)
|
||||
|
||||
|
||||
@app.route("/job/<job_id>/page/<int:page_num>")
|
||||
|
|
|
|||
36
extractor.py
36
extractor.py
|
|
@ -40,17 +40,21 @@ if not os.environ.get("ANTHROPIC_API_KEY"):
|
|||
import fitz # PyMuPDF
|
||||
from PIL import Image
|
||||
|
||||
JOBS = {}
|
||||
JOB_LOCK = threading.Lock()
|
||||
|
||||
|
||||
_LAYOUT = None
|
||||
_LAYOUT_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _set_status(job_id, msg):
|
||||
with JOB_LOCK:
|
||||
if job_id in JOBS:
|
||||
JOBS[job_id]["message"] = msg
|
||||
def _set_status(job_dir, msg):
|
||||
try:
|
||||
meta_path = Path(job_dir) / "meta.json"
|
||||
if meta_path.exists():
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
meta["message"] = msg
|
||||
meta_path.write_text(json.dumps(meta, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -693,7 +697,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
|
|||
articles_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Step 1: Render PDF pages
|
||||
_set_status(job_id, "Rendering PDF pages...")
|
||||
_set_status(job_dir, "Rendering PDF pages...")
|
||||
pages = render_pdf_pages(pdf_path, pages_dir, target_w, target_h, max_pages=max_pages, pages_to_process=pages_to_process)
|
||||
|
||||
all_records = []
|
||||
|
|
@ -704,7 +708,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
|
|||
page_img_path = p["path"]
|
||||
|
||||
# Step 2: PaddleOCR layout detection — precise bounding boxes
|
||||
_set_status(job_id, f"Page {n}: detecting layout regions...")
|
||||
_set_status(job_dir, f"Page {n}: detecting layout regions...")
|
||||
regions = detect_regions(page_img_path)
|
||||
(pages_dir / f"page_{n:03d}.regions.json").write_text(
|
||||
json.dumps({"page": n, "regions": regions}, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
|
|
@ -712,7 +716,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
|
|||
print(f" Page {n}: {len(regions)} layout regions detected")
|
||||
|
||||
# Step 3: Group regions into articles
|
||||
_set_status(job_id, f"Page {n}: grouping articles...")
|
||||
_set_status(job_dir, f"Page {n}: grouping articles...")
|
||||
page_img = Image.open(page_img_path)
|
||||
articles, enriched = group_into_articles(page_img_path, regions, p["width"], page_img=page_img)
|
||||
|
||||
|
|
@ -845,14 +849,14 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
|
|||
except Exception as e:
|
||||
print(f" Could not create articles debug image: {e}")
|
||||
# Step 4: Crop and OCR HEADLINES only
|
||||
_set_status(job_id, f"Page {n}: cropping {len(articles)} headlines...")
|
||||
_set_status(job_dir, f"Page {n}: cropping {len(articles)} headlines...")
|
||||
headline_records = crop_headlines(page_img_path, articles, regions, articles_root, n)
|
||||
|
||||
_set_status(job_id, f"Page {n}: running fast OCR on {len(headline_records)} headlines...")
|
||||
_set_status(job_dir, f"Page {n}: running fast OCR on {len(headline_records)} headlines...")
|
||||
ocr_headlines(headline_records)
|
||||
|
||||
# Step 5: Triage with Claude Haiku based on headlines
|
||||
_set_status(job_id, f"Page {n}: triaging headlines with Claude...")
|
||||
_set_status(job_dir, f"Page {n}: triaging headlines with Claude...")
|
||||
selected_ids = _triage_headlines_with_claude(headline_records, n, job_dir=str(job_dir))
|
||||
|
||||
# Cleanup non-selected directories
|
||||
|
|
@ -865,16 +869,16 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
|
|||
continue
|
||||
|
||||
# Step 6: Crop FULL articles for the selected ones
|
||||
_set_status(job_id, f"Page {n}: cropping {len(selected_ids)} FULL articles...")
|
||||
_set_status(job_dir, f"Page {n}: cropping {len(selected_ids)} FULL articles...")
|
||||
records = crop_full_articles(page_img_path, selected_ids, articles, articles_root, n)
|
||||
print(f" Page {n}: cropped {len(records)} full articles")
|
||||
|
||||
# Step 7: OCR full articles
|
||||
_set_status(job_id, f"Page {n}: running heavy OCR on {len(records)} full articles...")
|
||||
_set_status(job_dir, f"Page {n}: running heavy OCR on {len(records)} full articles...")
|
||||
ocr_articles(records)
|
||||
|
||||
# Step 8: Deep Analysis with Claude Opus
|
||||
_set_status(job_id, f"Page {n}: deep political analysis with Claude...")
|
||||
_set_status(job_dir, f"Page {n}: deep political analysis with Claude...")
|
||||
selected = _analyze_articles_with_claude(records, n, job_dir=str(job_dir))
|
||||
|
||||
if selected:
|
||||
|
|
@ -996,5 +1000,5 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p
|
|||
if all_selected and generate_political_pdf:
|
||||
generate_political_pdf(all_selected, job_dir)
|
||||
|
||||
_set_status(job_id, "Done!")
|
||||
_set_status(job_dir, "Done!")
|
||||
return {"pages": len(pages), "articles": len(all_records)}
|
||||
|
|
|
|||
Loading…
Reference in New Issue