51 lines
1.9 KiB
Python
51 lines
1.9 KiB
Python
from app.tasks.celery_app import celery
|
|
from app.pipeline.orchestrator import Pipeline
|
|
from app.db.session import SessionLocal
|
|
from app.db.models import Article, ArticleImage, UploadJob
|
|
import hashlib, os
|
|
|
|
pipeline = None
|
|
|
|
@celery.task(bind=True)
|
|
def process_pdf(self, job_id, pdf_path, newspaper, edition, date):
|
|
db = SessionLocal()
|
|
job = db.query(UploadJob).filter_by(id=job_id).first()
|
|
if job:
|
|
job.status = "PROCESSING"
|
|
db.commit()
|
|
|
|
global pipeline
|
|
if pipeline is None:
|
|
from app.pipeline.orchestrator import Pipeline
|
|
pipeline = Pipeline()
|
|
|
|
job_dir = os.path.join("/data/storage", os.path.basename(pdf_path) + "_crops")
|
|
self.update_state(state="PROCESSING", meta={"stage": "layout"})
|
|
try:
|
|
articles = pipeline.process(pdf_path, newspaper, edition, date, job_dir)
|
|
for a in articles:
|
|
chash = hashlib.sha256(a["text"][:500].encode()).hexdigest()
|
|
if db.query(Article).filter_by(content_hash=chash).first():
|
|
continue # dedup / syndicated
|
|
art = Article(content_hash=chash, **{k: a[k] for k in [
|
|
"date","newspaper","edition","page","headline","category",
|
|
"subcategory","text","summary","keywords","persons","locations",
|
|
"organizations","sentiment","language","image_path",
|
|
"bounding_box","confidence","embedding"]})
|
|
art.upload_job_id = job_id
|
|
db.add(art); db.flush()
|
|
for fp in a["figure_paths"]:
|
|
db.add(ArticleImage(article_id=art.id, image_path=fp))
|
|
if job:
|
|
job.status = "COMPLETED"
|
|
db.commit()
|
|
except Exception as e:
|
|
if job:
|
|
job.status = "FAILED"
|
|
job.error_message = str(e)
|
|
db.commit()
|
|
raise e
|
|
finally:
|
|
db.close()
|
|
return {"articles": len(articles)}
|