260 lines
7.9 KiB
Python
260 lines
7.9 KiB
Python
"""
|
|
Telugu Newspaper Article Extractor — local web app.
|
|
|
|
Pipeline:
|
|
1. User uploads a PDF.
|
|
2. App renders each page to PNG at a target pixel size (~4200x7400).
|
|
3. PaddleOCR (PP-Structure) finds layout regions per page.
|
|
4. Grouping function joins regions into articles, anchored on the
|
|
"<place>, మే <day> (ఆంధ్రజ్యోతి):" dateline pattern.
|
|
5. Each article is cropped to its own PNG.
|
|
6. Each article's text is OCR'd with Tesseract (telugu).
|
|
7. The user can browse results in the browser and download a zip.
|
|
|
|
Run:
|
|
pip install -r requirements.txt
|
|
python app.py
|
|
-> open http://localhost:5000
|
|
"""
|
|
import io
|
|
import os
|
|
import re
|
|
import json
|
|
import shutil
|
|
import zipfile
|
|
import tempfile
|
|
import threading
|
|
import traceback
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
|
|
from flask import (
|
|
Flask, request, render_template, redirect, url_for,
|
|
send_from_directory, send_file, abort, jsonify,
|
|
)
|
|
|
|
import sys
|
|
if hasattr(sys.stdout, 'reconfigure'):
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
if hasattr(sys.stderr, 'reconfigure'):
|
|
sys.stderr.reconfigure(encoding='utf-8')
|
|
|
|
from extractor import process_pdf
|
|
|
|
JOBS = {}
|
|
JOB_LOCK = threading.Lock()
|
|
|
|
ROOT = Path(__file__).parent
|
|
UPLOAD_DIR = ROOT / "uploads"
|
|
OUTPUT_DIR = ROOT / "output"
|
|
UPLOAD_DIR.mkdir(exist_ok=True)
|
|
OUTPUT_DIR.mkdir(exist_ok=True)
|
|
|
|
app = Flask(__name__)
|
|
app.config["MAX_CONTENT_LENGTH"] = 200 * 1024 * 1024 # 200 MB upload cap
|
|
|
|
|
|
@app.route("/")
|
|
def index():
|
|
# list existing jobs newest-first
|
|
jobs = []
|
|
for d in sorted(OUTPUT_DIR.iterdir(), reverse=True):
|
|
if d.is_dir():
|
|
meta_path = d / "meta.json"
|
|
meta = {}
|
|
if meta_path.exists():
|
|
try:
|
|
meta = json.loads(meta_path.read_text())
|
|
except Exception:
|
|
pass
|
|
jobs.append({
|
|
"id": d.name,
|
|
"filename": meta.get("filename", "?"),
|
|
"status": meta.get("status", "unknown"),
|
|
"pages": meta.get("pages", 0),
|
|
"articles": meta.get("articles", 0),
|
|
"created": meta.get("created", ""),
|
|
})
|
|
return render_template("index.html", jobs=jobs)
|
|
|
|
|
|
@app.route("/upload", methods=["POST"])
|
|
def upload():
|
|
f = request.files.get("pdf")
|
|
if not f or not f.filename.lower().endswith(".pdf"):
|
|
return "Upload a PDF file", 400
|
|
|
|
try:
|
|
target_w = int(request.form.get("width", "4200"))
|
|
target_h = int(request.form.get("height", "7400"))
|
|
except ValueError:
|
|
target_w, target_h = 4200, 7400
|
|
|
|
# Job folder
|
|
job_id = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
job_dir = OUTPUT_DIR / job_id
|
|
job_dir.mkdir()
|
|
|
|
pdf_path = job_dir / "input.pdf"
|
|
f.save(str(pdf_path))
|
|
|
|
# Write initial meta
|
|
meta = {
|
|
"id": job_id,
|
|
"filename": f.filename,
|
|
"status": "queued",
|
|
"target_width": target_w,
|
|
"target_height": target_h,
|
|
"pages": 0,
|
|
"articles": 0,
|
|
"created": datetime.now().isoformat(timespec="seconds"),
|
|
}
|
|
(job_dir / "meta.json").write_text(json.dumps(meta, indent=2))
|
|
|
|
# Run in background thread so UI stays responsive
|
|
t = threading.Thread(
|
|
target=_run_job,
|
|
args=(job_id, str(pdf_path), str(job_dir), target_w, target_h),
|
|
daemon=True,
|
|
)
|
|
t.start()
|
|
|
|
return redirect(url_for("job_view", job_id=job_id))
|
|
|
|
|
|
def _run_job(job_id, pdf_path, job_dir, target_w, target_h):
|
|
try:
|
|
with JOB_LOCK:
|
|
JOBS[job_id] = {"status": "running", "message": "Starting..."}
|
|
result = process_pdf(pdf_path, job_dir, target_w, target_h, job_id)
|
|
meta_path = Path(job_dir) / "meta.json"
|
|
meta = json.loads(meta_path.read_text())
|
|
meta.update({
|
|
"status": "done",
|
|
"pages": result["pages"],
|
|
"articles": result["articles"],
|
|
})
|
|
meta_path.write_text(json.dumps(meta, indent=2))
|
|
with JOB_LOCK:
|
|
JOBS[job_id] = {"status": "done", "message": "Finished."}
|
|
except Exception as e:
|
|
traceback.print_exc()
|
|
meta_path = Path(job_dir) / "meta.json"
|
|
try:
|
|
meta = json.loads(meta_path.read_text())
|
|
meta["status"] = "error"
|
|
meta["error"] = str(e)
|
|
meta_path.write_text(json.dumps(meta, indent=2))
|
|
except Exception:
|
|
pass
|
|
with JOB_LOCK:
|
|
JOBS[job_id] = {"status": "error", "message": str(e)}
|
|
|
|
|
|
@app.route("/job/<job_id>")
|
|
def job_view(job_id):
|
|
job_dir = OUTPUT_DIR / job_id
|
|
if not job_dir.exists():
|
|
abort(404)
|
|
|
|
meta = json.loads((job_dir / "meta.json").read_text())
|
|
|
|
# collect article info if available
|
|
articles = []
|
|
for art_dir in sorted((job_dir / "articles").glob("*")) if (job_dir / "articles").exists() else []:
|
|
if art_dir.is_dir():
|
|
info_path = art_dir / "info.json"
|
|
info = {}
|
|
if info_path.exists():
|
|
info = json.loads(info_path.read_text())
|
|
articles.append({
|
|
"name": art_dir.name,
|
|
"page": info.get("page", "?"),
|
|
"headline_preview": info.get("headline_preview", ""),
|
|
"dateline": info.get("dateline", ""),
|
|
"grouped_by": info.get("grouped_by", ""),
|
|
"has_image": (art_dir / "article.png").exists(),
|
|
"has_text": (art_dir / "article.txt").exists(),
|
|
})
|
|
|
|
return render_template("job.html", meta=meta, job_id=job_id, articles=articles)
|
|
|
|
|
|
@app.route("/job/<job_id>/status")
|
|
def job_status(job_id):
|
|
status = {"status": "unknown"}
|
|
with JOB_LOCK:
|
|
if job_id in JOBS:
|
|
status.update(JOBS[job_id])
|
|
|
|
meta_path = OUTPUT_DIR / job_id / "meta.json"
|
|
if meta_path.exists():
|
|
try:
|
|
meta = json.loads(meta_path.read_text())
|
|
if "status" in meta:
|
|
status["status"] = meta["status"]
|
|
if "error" in meta and not status.get("message"):
|
|
status["message"] = meta["error"]
|
|
elif "message" in meta:
|
|
status["message"] = meta["message"]
|
|
status["meta"] = meta
|
|
except Exception:
|
|
pass
|
|
|
|
return jsonify(status)
|
|
|
|
|
|
@app.route("/job/<job_id>/page/<int:page_num>")
|
|
def page_image(job_id, page_num):
|
|
page_path = OUTPUT_DIR / job_id / "pages" / f"page_{page_num:03d}.png"
|
|
if not page_path.exists():
|
|
abort(404)
|
|
return send_file(str(page_path))
|
|
|
|
|
|
@app.route("/job/<job_id>/article/<art_name>/image")
|
|
def article_image(job_id, art_name):
|
|
p = OUTPUT_DIR / job_id / "articles" / art_name / "article.png"
|
|
if not p.exists():
|
|
abort(404)
|
|
return send_file(str(p))
|
|
|
|
|
|
@app.route("/job/<job_id>/article/<art_name>/text")
|
|
def article_text(job_id, art_name):
|
|
p = OUTPUT_DIR / job_id / "articles" / art_name / "article.txt"
|
|
if not p.exists():
|
|
return "(no text)", 404
|
|
return p.read_text(encoding="utf-8"), 200, {"Content-Type": "text/plain; charset=utf-8"}
|
|
|
|
|
|
@app.route("/job/<job_id>/download")
|
|
def download_zip(job_id):
|
|
job_dir = OUTPUT_DIR / job_id
|
|
if not job_dir.exists():
|
|
abort(404)
|
|
|
|
buf = io.BytesIO()
|
|
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
for root, dirs, files in os.walk(job_dir):
|
|
# skip the source pdf and big page images to keep the zip lean
|
|
for fn in files:
|
|
if fn == "input.pdf":
|
|
continue
|
|
full = Path(root) / fn
|
|
rel = full.relative_to(job_dir)
|
|
zf.write(full, arcname=str(rel))
|
|
buf.seek(0)
|
|
return send_file(
|
|
buf,
|
|
mimetype="application/zip",
|
|
as_attachment=True,
|
|
download_name=f"{job_id}_articles.zip",
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print("Telugu Article Extractor")
|
|
print("Open http://localhost:5000")
|
|
app.run(host="127.0.0.1", port=5000, debug=False, threaded=True)
|