from app.tasks.celery_app import celery from app.pipeline.orchestrator import Pipeline from app.db.session import SessionLocal from app.db.models import Article, ArticleImage, UploadJob import hashlib, os pipeline = None @celery.task(bind=True) def process_pdf(self, job_id, pdf_path, newspaper, edition, date): db = SessionLocal() job = db.query(UploadJob).filter_by(id=job_id).first() if job: job.status = "PROCESSING" db.commit() global pipeline if pipeline is None: from app.pipeline.orchestrator import Pipeline pipeline = Pipeline() job_dir = os.path.join("/data/storage", os.path.basename(pdf_path) + "_crops") self.update_state(state="PROCESSING", meta={"stage": "layout"}) try: articles = pipeline.process(pdf_path, newspaper, edition, date, job_dir) for a in articles: chash = hashlib.sha256(a["text"][:500].encode()).hexdigest() if db.query(Article).filter_by(content_hash=chash).first(): continue # dedup / syndicated art = Article(content_hash=chash, **{k: a[k] for k in [ "date","newspaper","edition","page","headline","category", "subcategory","text","summary","keywords","persons","locations", "organizations","sentiment","language","image_path", "bounding_box","confidence","embedding"]}) art.upload_job_id = job_id db.add(art); db.flush() for fp in a["figure_paths"]: db.add(ArticleImage(article_id=art.id, image_path=fp)) if job: job.status = "COMPLETED" db.commit() except Exception as e: if job: job.status = "FAILED" job.error_message = str(e) db.commit() raise e finally: db.close() return {"articles": len(articles)}