feat: add newspaper-extractor deployment
This commit is contained in:
parent
0fe322cc53
commit
226d683a46
|
|
@ -0,0 +1,40 @@
|
|||
name: Build Newspaper Extractor Backend
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'newspaper-extractor/backend/**'
|
||||
|
||||
jobs:
|
||||
build-newspaper-backend:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DOCKER_HOST: tcp://172.17.0.1:2375
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Build and push Newspaper Extractor Backend (Kaniko)
|
||||
run: |
|
||||
# Use short SHA for reproducible, auditable image tags.
|
||||
SHORT_SHA="${GITHUB_SHA::7}"
|
||||
IMAGE_TAG="192.168.8.250:5000/agentic-os/newspaper-extractor-backend:${SHORT_SHA}"
|
||||
IMAGE_LATEST="192.168.8.250:5000/agentic-os/newspaper-extractor-backend:latest"
|
||||
|
||||
JOB_CONTAINER=$(docker ps --format '{{.Names}}' | grep 'GITEA-ACTIONS-TASK' | head -1)
|
||||
echo "Using volumes from: $JOB_CONTAINER"
|
||||
echo "Building image: $IMAGE_TAG"
|
||||
|
||||
docker run --rm \
|
||||
--volumes-from "$JOB_CONTAINER" \
|
||||
gcr.io/kaniko-project/executor:latest \
|
||||
--context=dir:///workspace/deepkoluguri/agentic-os/newspaper-extractor/backend \
|
||||
--dockerfile=Dockerfile \
|
||||
--destination="${IMAGE_TAG}" \
|
||||
--destination="${IMAGE_LATEST}" \
|
||||
--insecure \
|
||||
--skip-tls-verify
|
||||
|
||||
echo "Built and pushed: ${IMAGE_TAG}"
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
name: Build Newspaper Extractor Frontend
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'newspaper-extractor/frontend/**'
|
||||
|
||||
jobs:
|
||||
build-newspaper-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DOCKER_HOST: tcp://172.17.0.1:2375
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Build and push Newspaper Extractor Frontend (Kaniko)
|
||||
run: |
|
||||
SHORT_SHA="${GITHUB_SHA::7}"
|
||||
IMAGE_TAG="192.168.8.250:5000/agentic-os/newspaper-extractor-frontend:${SHORT_SHA}"
|
||||
IMAGE_LATEST="192.168.8.250:5000/agentic-os/newspaper-extractor-frontend:latest"
|
||||
|
||||
JOB_CONTAINER=$(docker ps --format '{{.Names}}' | grep 'GITEA-ACTIONS-TASK' | head -1)
|
||||
echo "Using volumes from: $JOB_CONTAINER"
|
||||
echo "Building image: $IMAGE_TAG"
|
||||
|
||||
docker run --rm \
|
||||
--volumes-from "$JOB_CONTAINER" \
|
||||
gcr.io/kaniko-project/executor:latest \
|
||||
--context=dir:///workspace/deepkoluguri/agentic-os/newspaper-extractor/frontend \
|
||||
--dockerfile=Dockerfile \
|
||||
--destination="${IMAGE_TAG}" \
|
||||
--destination="${IMAGE_LATEST}" \
|
||||
--insecure \
|
||||
--skip-tls-verify
|
||||
|
||||
echo "Built and pushed: ${IMAGE_TAG}"
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: newspaper-backend
|
||||
namespace: ai-agents
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: newspaper-backend
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: newspaper-backend
|
||||
spec:
|
||||
containers:
|
||||
- name: backend
|
||||
image: 192.168.8.250:5000/agentic-os/newspaper-extractor-backend:latest
|
||||
imagePullPolicy: Always
|
||||
env:
|
||||
- name: DATABASE_URL
|
||||
value: "postgresql://myuser:mypassword@newspaper-db.ai-agents.svc.cluster.local:5432/mydatabase"
|
||||
- name: CELERY_BROKER_URL
|
||||
value: "redis://newspaper-redis.ai-agents.svc.cluster.local:6379/0"
|
||||
- name: CELERY_RESULT_BACKEND
|
||||
value: "redis://newspaper-redis.ai-agents.svc.cluster.local:6379/0"
|
||||
- name: REDIS_URL
|
||||
value: "redis://newspaper-redis.ai-agents.svc.cluster.local:6379/0"
|
||||
ports:
|
||||
- containerPort: 8000
|
||||
volumeMounts:
|
||||
- name: shared-data
|
||||
mountPath: /app/data
|
||||
volumes:
|
||||
- name: shared-data
|
||||
persistentVolumeClaim:
|
||||
claimName: newspaper-data-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: newspaper-backend
|
||||
namespace: ai-agents
|
||||
spec:
|
||||
selector:
|
||||
app: newspaper-backend
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8000
|
||||
targetPort: 8000
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: newspaper-data-pvc
|
||||
namespace: ai-agents
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: newspaper-db
|
||||
namespace: ai-agents
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: newspaper-db
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: newspaper-db
|
||||
spec:
|
||||
containers:
|
||||
- name: postgres
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: myuser
|
||||
- name: POSTGRES_PASSWORD
|
||||
value: mypassword
|
||||
- name: POSTGRES_DB
|
||||
value: mydatabase
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
volumeMounts:
|
||||
- name: pgdata
|
||||
mountPath: /var/lib/postgresql/data
|
||||
volumes:
|
||||
- name: pgdata
|
||||
persistentVolumeClaim:
|
||||
claimName: newspaper-db-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: newspaper-db-pvc
|
||||
namespace: ai-agents
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: newspaper-db
|
||||
namespace: ai-agents
|
||||
spec:
|
||||
selector:
|
||||
app: newspaper-db
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: newspaper-frontend
|
||||
namespace: ai-agents
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: newspaper-frontend
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: newspaper-frontend
|
||||
spec:
|
||||
containers:
|
||||
- name: frontend
|
||||
image: 192.168.8.250:5000/agentic-os/newspaper-extractor-frontend:latest
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- containerPort: 3000
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: newspaper-frontend
|
||||
namespace: ai-agents
|
||||
spec:
|
||||
selector:
|
||||
app: newspaper-frontend
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 3000
|
||||
targetPort: 3000
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: newspaper-ingress
|
||||
namespace: ai-agents
|
||||
annotations:
|
||||
k8s.apisix.apache.org/enable-websocket: "true"
|
||||
spec:
|
||||
ingressClassName: apisix
|
||||
rules:
|
||||
- host: newspaper.applaude.net
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: newspaper-frontend
|
||||
port:
|
||||
number: 3000
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: newspaper-redis
|
||||
namespace: ai-agents
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: newspaper-redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: newspaper-redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:alpine
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: newspaper-redis
|
||||
namespace: ai-agents
|
||||
spec:
|
||||
selector:
|
||||
app: newspaper-redis
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 6379
|
||||
targetPort: 6379
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: newspaper-worker
|
||||
namespace: ai-agents
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: newspaper-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: newspaper-worker
|
||||
spec:
|
||||
containers:
|
||||
- name: worker
|
||||
image: 192.168.8.250:5000/agentic-os/newspaper-extractor-backend:latest
|
||||
imagePullPolicy: Always
|
||||
command: ["celery", "-A", "app.tasks.celery_app", "worker", "--loglevel=info", "-P", "solo"]
|
||||
env:
|
||||
- name: DATABASE_URL
|
||||
value: "postgresql://myuser:mypassword@newspaper-db.ai-agents.svc.cluster.local:5432/mydatabase"
|
||||
- name: CELERY_BROKER_URL
|
||||
value: "redis://newspaper-redis.ai-agents.svc.cluster.local:6379/0"
|
||||
- name: CELERY_RESULT_BACKEND
|
||||
value: "redis://newspaper-redis.ai-agents.svc.cluster.local:6379/0"
|
||||
- name: REDIS_URL
|
||||
value: "redis://newspaper-redis.ai-agents.svc.cluster.local:6379/0"
|
||||
volumeMounts:
|
||||
- name: shared-data
|
||||
mountPath: /app/data
|
||||
volumes:
|
||||
- name: shared-data
|
||||
persistentVolumeClaim:
|
||||
claimName: newspaper-data-pvc
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
FROM python:3.10-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install system dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
build-essential \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
poppler-utils \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
RUN pip install 'git+https://github.com/facebookresearch/detectron2.git'
|
||||
|
||||
COPY . .
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import or_, text
|
||||
from app.db.session import get_db
|
||||
from app.pipeline.enricher import Enricher
|
||||
|
||||
router = APIRouter(prefix="/api")
|
||||
enricher = Enricher()
|
||||
|
||||
@router.get("/search")
|
||||
def search(q: str = "", category: str = None, person: str = None,
|
||||
date: str = None, page: int = None, job_id: int = None, semantic: bool = False, db=Depends(get_db)):
|
||||
from app.db.models import Article
|
||||
query = db.query(Article)
|
||||
if category: query = query.filter(Article.category == category)
|
||||
if date: query = query.filter(Article.date == date)
|
||||
if page: query = query.filter(Article.page == page)
|
||||
if job_id: query = query.filter(Article.upload_job_id == job_id)
|
||||
if person: query = query.filter(Article.persons.any(person))
|
||||
if q and not semantic:
|
||||
query = query.filter(or_(Article.headline.ilike(f"%{q}%"),
|
||||
Article.text.ilike(f"%{q}%")))
|
||||
return query.limit(50).all()
|
||||
if semantic and q:
|
||||
vec = enricher.embed(q)
|
||||
# pgvector cosine distance
|
||||
return db.execute(text("""
|
||||
SELECT * FROM articles ORDER BY embedding <=> :v LIMIT 30
|
||||
"""), {"v": str(vec)}).fetchall()
|
||||
return query.limit(50).all()
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
from fastapi import APIRouter, UploadFile, File, Form, Depends
|
||||
from app.tasks.ingest import process_pdf
|
||||
from app.db.session import get_db
|
||||
from app.db.models import UploadJob
|
||||
import shutil, os, uuid
|
||||
|
||||
router = APIRouter(prefix="/api")
|
||||
|
||||
@router.post("/upload")
|
||||
async def upload(file: UploadFile = File(...),
|
||||
newspaper: str = Form(...), edition: str = Form("Main"),
|
||||
date: str = Form(...), db=Depends(get_db)):
|
||||
path = f"/data/storage/{uuid.uuid4().hex}_{file.filename}"
|
||||
os.makedirs("/data/storage", exist_ok=True)
|
||||
with open(path, "wb") as f:
|
||||
shutil.copyfileobj(file.file, f)
|
||||
|
||||
job = UploadJob(filename=file.filename, newspaper=newspaper, edition=edition, date=date)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
|
||||
task = process_pdf.delay(job.id, path, newspaper, edition, date)
|
||||
|
||||
job.celery_task_id = task.id
|
||||
db.commit()
|
||||
|
||||
return {"job_id": job.id, "task_id": task.id, "status": "queued"}
|
||||
|
||||
@router.get("/jobs")
|
||||
def get_jobs(db=Depends(get_db)):
|
||||
jobs = db.query(UploadJob).order_by(UploadJob.created_time.desc()).all()
|
||||
return jobs
|
||||
|
||||
@router.get("/status/{task_id}")
|
||||
def status(task_id: str):
|
||||
from app.tasks.celery_app import celery
|
||||
res = celery.AsyncResult(task_id)
|
||||
return {"state": res.state, "info": res.info}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
from pydantic_settings import BaseSettings
|
||||
|
||||
class Settings(BaseSettings):
|
||||
DATABASE_URL: str = "postgresql+psycopg2://user:pass@db:5432/news"
|
||||
REDIS_URL: str = "redis://redis:6379/0"
|
||||
STORAGE_DIR: str = "/data/storage"
|
||||
DPI: int = 300
|
||||
LAYOUT_MODEL: str = "PubLayNet" # or DocLayNet / custom YOLOv11
|
||||
OCR_LANGS: list = ["en", "hi", "te", "ta"]
|
||||
LLM_PROVIDER: str = "gemini" # gemini | openai | local
|
||||
LLM_API_KEY: str = ""
|
||||
EMBED_MODEL: str = "sentence-transformers/all-MiniLM-L6-v2"
|
||||
USE_GPU: bool = True
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
|
||||
settings = Settings()
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
from sqlalchemy import Column, Integer, String, Text, Float, ForeignKey, DateTime, JSON
|
||||
from sqlalchemy.dialects.postgresql import ARRAY
|
||||
from sqlalchemy.orm import relationship, declarative_base
|
||||
from pgvector.sqlalchemy import Vector
|
||||
import datetime
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
class UploadJob(Base):
|
||||
__tablename__ = "upload_jobs"
|
||||
id = Column(Integer, primary_key=True)
|
||||
filename = Column(String)
|
||||
newspaper = Column(String)
|
||||
edition = Column(String)
|
||||
date = Column(String)
|
||||
celery_task_id = Column(String)
|
||||
status = Column(String, default="PENDING")
|
||||
error_message = Column(Text)
|
||||
created_time = Column(DateTime, default=datetime.datetime.utcnow)
|
||||
articles = relationship("Article", back_populates="upload_job")
|
||||
|
||||
class Article(Base):
|
||||
__tablename__ = "articles"
|
||||
id = Column(Integer, primary_key=True)
|
||||
upload_job_id = Column(Integer, ForeignKey("upload_jobs.id"), nullable=True)
|
||||
date = Column(String, index=True)
|
||||
newspaper = Column(String, index=True)
|
||||
edition = Column(String)
|
||||
page = Column(Integer, index=True)
|
||||
headline = Column(Text, index=True)
|
||||
subheadline = Column(Text)
|
||||
category = Column(String, index=True)
|
||||
subcategory = Column(String, index=True)
|
||||
language = Column(String)
|
||||
author = Column(String)
|
||||
text = Column(Text)
|
||||
summary = Column(Text)
|
||||
keywords = Column(ARRAY(String))
|
||||
persons = Column(ARRAY(String))
|
||||
locations = Column(ARRAY(String))
|
||||
organizations = Column(ARRAY(String))
|
||||
sentiment = Column(String)
|
||||
image_path = Column(String)
|
||||
bounding_box = Column(JSON)
|
||||
confidence = Column(Float)
|
||||
embedding = Column(Vector(384)) # pgvector for semantic search
|
||||
content_hash = Column(String, index=True) # for dedup
|
||||
created_time = Column(DateTime, default=datetime.datetime.utcnow)
|
||||
images = relationship("ArticleImage", back_populates="article")
|
||||
upload_job = relationship("UploadJob", back_populates="articles")
|
||||
|
||||
class ArticleImage(Base):
|
||||
__tablename__ = "article_images"
|
||||
id = Column(Integer, primary_key=True)
|
||||
article_id = Column(Integer, ForeignKey("articles.id"))
|
||||
image_path = Column(String)
|
||||
caption = Column(Text)
|
||||
width = Column(Integer)
|
||||
height = Column(Integer)
|
||||
article = relationship("Article", back_populates="images")
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from app.config import settings
|
||||
|
||||
engine = create_engine(settings.DATABASE_URL)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
from fastapi import FastAPI
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from app.api import routes_upload, routes_search
|
||||
from app.db.models import Base
|
||||
from app.db.session import engine
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector;"))
|
||||
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
app = FastAPI(title="Newspaper Extractor API")
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"], # Allow all origins for development
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Serve uploaded images
|
||||
import os
|
||||
os.makedirs("/data/storage", exist_ok=True)
|
||||
app.mount("/api/images", StaticFiles(directory="/data/storage"), name="images")
|
||||
|
||||
app.include_router(routes_upload.router)
|
||||
app.include_router(routes_search.router)
|
||||
|
||||
@app.get("/")
|
||||
def root():
|
||||
return {"message": "Newspaper Extractor API is running."}
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
import numpy as np
|
||||
from scipy.spatial import distance
|
||||
|
||||
class ArticleSegmenter:
|
||||
def __init__(self, col_gap_ratio=0.02):
|
||||
self.col_gap_ratio = col_gap_ratio
|
||||
|
||||
def _vertical_overlap(self, a, b):
|
||||
top = max(a[1], b[1]); bot = min(a[3], b[3])
|
||||
return max(0, bot - top)
|
||||
|
||||
def _horizontal_gap(self, a, b):
|
||||
return b[0] - a[2]
|
||||
|
||||
def segment(self, blocks, page_width, page_height):
|
||||
# 1. Remove ads
|
||||
content = [b for b in blocks if not b["is_ad"] and b["type"] != "Advertisement"]
|
||||
|
||||
# 2. Sort reading order: column-aware (top-to-bottom within columns)
|
||||
content = self._reading_order(content, page_width)
|
||||
|
||||
# 3. Seed articles at each Title block
|
||||
articles = []
|
||||
current = None
|
||||
for b in content:
|
||||
if b["type"] == "Title":
|
||||
if current:
|
||||
articles.append(current)
|
||||
current = {"blocks": [b], "title_block": b}
|
||||
else:
|
||||
if current is None:
|
||||
current = {"blocks": [b], "title_block": None}
|
||||
else:
|
||||
if self._belongs(current, b, page_width):
|
||||
current["blocks"].append(b)
|
||||
else:
|
||||
articles.append(current)
|
||||
current = {"blocks": [b], "title_block": None}
|
||||
if current:
|
||||
articles.append(current)
|
||||
|
||||
# 4. Merge cross-column continuation of same article
|
||||
articles = self._merge_columns(articles, page_width)
|
||||
|
||||
# 5. Compute union bounding box per article
|
||||
for a in articles:
|
||||
a["bbox"] = self._union_bbox(a["blocks"])
|
||||
a["confidence"] = float(np.mean([blk["score"] for blk in a["blocks"]]))
|
||||
return articles
|
||||
|
||||
def _reading_order(self, blocks, page_width, n_cols_guess=None):
|
||||
if not blocks:
|
||||
return blocks
|
||||
xs = np.array([b["bbox"][0] for b in blocks]).reshape(-1, 1)
|
||||
# Estimate columns via 1D clustering on x-start
|
||||
from sklearn.cluster import KMeans
|
||||
k = self._estimate_columns(xs, page_width)
|
||||
km = KMeans(n_clusters=k, n_init=5).fit(xs)
|
||||
for b, lbl in zip(blocks, km.labels_):
|
||||
b["col"] = int(lbl)
|
||||
# Order columns left→right, blocks top→bottom
|
||||
col_order = np.argsort([xs[km.labels_ == c].mean() for c in range(k)])
|
||||
ordered = []
|
||||
for c in col_order:
|
||||
col_blocks = [b for b in blocks if b["col"] == c]
|
||||
col_blocks.sort(key=lambda b: b["bbox"][1])
|
||||
ordered.extend(col_blocks)
|
||||
return ordered
|
||||
|
||||
def _estimate_columns(self, xs, page_width):
|
||||
from sklearn.metrics import silhouette_score
|
||||
best_k, best_score = 1, -1
|
||||
for k in range(1, min(7, len(xs))):
|
||||
if k == 1:
|
||||
continue
|
||||
from sklearn.cluster import KMeans
|
||||
labels = KMeans(n_clusters=k, n_init=3).fit_predict(xs)
|
||||
try:
|
||||
s = silhouette_score(xs, labels)
|
||||
if s > best_score:
|
||||
best_score, best_k = s, k
|
||||
except Exception:
|
||||
pass
|
||||
return max(best_k, 1)
|
||||
|
||||
def _belongs(self, article, block, page_width):
|
||||
"""Block belongs to current article if vertically continuous & aligned."""
|
||||
last = article["blocks"][-1]["bbox"]
|
||||
cur = block["bbox"]
|
||||
same_col = abs(last[0] - cur[0]) < 0.05 * page_width
|
||||
gap = cur[1] - last[3]
|
||||
return same_col and gap < 0.04 * page_width
|
||||
|
||||
def _merge_columns(self, articles, page_width):
|
||||
# Merge article fragments whose headline spans wide / text flows to next col
|
||||
# Simplified: if an article has no title and sits at top of next column
|
||||
# right after a titled article, merge. Production: use LayoutLMv3 relations.
|
||||
return articles
|
||||
|
||||
def _union_bbox(self, blocks):
|
||||
xs1 = min(b["bbox"][0] for b in blocks)
|
||||
ys1 = min(b["bbox"][1] for b in blocks)
|
||||
xs2 = max(b["bbox"][2] for b in blocks)
|
||||
ys2 = max(b["bbox"][3] for b in blocks)
|
||||
return [xs1, ys1, xs2 - xs1, ys2 - ys1]
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
import json
|
||||
from app.config import settings
|
||||
|
||||
CATEGORIES = ["Politics","National","International","Sports","Business","Technology",
|
||||
"Education","Health","Science","Entertainment","Crime","Court","Weather",
|
||||
"Editorial","Opinion","Local News","State News","Environment","Lifestyle",
|
||||
"Automobile","Travel","Agriculture","Jobs","Classifieds","Others"]
|
||||
|
||||
SUBCATS = {"Sports": ["Cricket","Football","Tennis","Others"],
|
||||
"Business": ["Finance","Markets","Economy"],
|
||||
"Technology": ["AI","Gadgets","Software"],
|
||||
"Entertainment": ["Movies","TV","Music"],
|
||||
"Politics": ["State Politics","National Politics","International"]}
|
||||
|
||||
PROMPT = """You are a news classifier. Given the headline and text, return strict JSON:
|
||||
{{"category": one of {cats}, "subcategory": string, "confidence": 0-1,
|
||||
"summary": 2-sentence summary, "keywords": [up to 8],
|
||||
"persons": [], "locations": [], "organizations": [], "dates": [], "sentiment": "pos|neu|neg"}}
|
||||
|
||||
HEADLINE: {headline}
|
||||
TEXT: {text}
|
||||
"""
|
||||
|
||||
class Categorizer:
|
||||
def __init__(self):
|
||||
self.provider = settings.LLM_PROVIDER
|
||||
if self.provider == "gemini":
|
||||
import google.generativeai as genai
|
||||
genai.configure(api_key=settings.LLM_API_KEY)
|
||||
self.model = genai.GenerativeModel("gemini-1.5-flash")
|
||||
|
||||
def classify(self, headline, text):
|
||||
prompt = PROMPT.format(cats=CATEGORIES, headline=headline, text=text[:4000])
|
||||
try:
|
||||
if self.provider == "gemini":
|
||||
resp = self.model.generate_content(prompt)
|
||||
raw = resp.text.strip().strip("```json").strip("```")
|
||||
return json.loads(raw)
|
||||
except Exception as e:
|
||||
return self._fallback(headline, text)
|
||||
|
||||
def _fallback(self, headline, text):
|
||||
# Zero-shot via local model if LLM unavailable
|
||||
from transformers import pipeline
|
||||
clf = pipeline("zero-shot-classification",
|
||||
model="facebook/bart-large-mnli")
|
||||
res = clf(headline + ". " + text[:500], CATEGORIES)
|
||||
return {"category": res["labels"][0], "subcategory": "",
|
||||
"confidence": res["scores"][0], "summary": "", "keywords": [],
|
||||
"persons": [], "locations": [], "organizations": [],
|
||||
"dates": [], "sentiment": "neu"}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
import cv2, os, uuid
|
||||
from app.config import settings
|
||||
|
||||
def crop_article(page_image, bbox, out_dir):
|
||||
x, y, w, h = [int(v) for v in bbox]
|
||||
crop = page_image[y:y+h, x:x+w]
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
path = os.path.join(out_dir, f"article_{uuid.uuid4().hex[:8]}.png")
|
||||
cv2.imwrite(path, cv2.cvtColor(crop, cv2.COLOR_RGB2BGR))
|
||||
return path
|
||||
|
||||
def crop_figures(page_image, article, out_dir):
|
||||
paths = []
|
||||
for b in article["blocks"]:
|
||||
if b["type"] == "Figure":
|
||||
paths.append(crop_article(page_image, _xywh(b["bbox"]), out_dir))
|
||||
return paths
|
||||
|
||||
def _xywh(box):
|
||||
return [box[0], box[1], box[2]-box[0], box[3]-box[1]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
from sentence_transformers import SentenceTransformer
|
||||
from app.config import settings
|
||||
|
||||
class Enricher:
|
||||
def __init__(self):
|
||||
self.model = SentenceTransformer(settings.EMBED_MODEL)
|
||||
def embed(self, text: str):
|
||||
return self.model.encode(text, normalize_embeddings=True).tolist()
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
def detect_headline(article, page, ocr_engine):
|
||||
tb = article.get("title_block")
|
||||
if tb:
|
||||
text, _ = ocr_engine.extract_text(page, _xywh(tb["bbox"]), page["native_words"])
|
||||
return text.strip().split("\n")[0]
|
||||
# Fallback: largest-font text block
|
||||
# font size ~ block height / line count
|
||||
candidates = sorted(article["blocks"], key=lambda b: b["bbox"][3]-b["bbox"][1], reverse=True)
|
||||
if candidates:
|
||||
text, _ = ocr_engine.extract_text(page, _xywh(candidates[0]["bbox"]), page["native_words"])
|
||||
return text.strip().split("\n")[0]
|
||||
return ""
|
||||
|
||||
def _xywh(box):
|
||||
return [box[0], box[1], box[2]-box[0], box[3]-box[1]]
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
import layoutparser as lp
|
||||
import numpy as np
|
||||
|
||||
CLASS_MAP = {
|
||||
0: "Text", 1: "Title", 2: "List",
|
||||
3: "Table", 4: "Figure", 5: "Advertisement",
|
||||
}
|
||||
|
||||
class LayoutDetector:
|
||||
def __init__(self):
|
||||
try:
|
||||
# Detectron2 PubLayNet — swap with custom DocLayNet/YOLOv11 weights
|
||||
self.model = lp.Detectron2LayoutModel(
|
||||
config_path="lp://PubLayNet/mask_rcnn_X_101_32x8d_FPN_3x/config",
|
||||
extra_config=["MODEL.ROI_HEADS.SCORE_THRESH_TEST", 0.55],
|
||||
label_map={0:"Text",1:"Title",2:"List",3:"Table",4:"Figure"},
|
||||
)
|
||||
except Exception:
|
||||
self.model = None
|
||||
# Optional secondary ad classifier (custom-trained)
|
||||
self.ad_classifier = AdClassifier()
|
||||
|
||||
def detect(self, image: np.ndarray):
|
||||
if self.model is None:
|
||||
# Fallback to single block if detectron2 isn't installed
|
||||
h, w = image.shape[:2]
|
||||
return [{
|
||||
"type": "Text",
|
||||
"bbox": [0, 0, w, h],
|
||||
"score": 1.0,
|
||||
"is_ad": False
|
||||
}]
|
||||
|
||||
layout = self.model.detect(image)
|
||||
blocks = []
|
||||
for b in layout:
|
||||
box = b.coordinates # (x1,y1,x2,y2)
|
||||
block = {
|
||||
"type": b.type,
|
||||
"bbox": [int(c) for c in box],
|
||||
"score": float(b.score),
|
||||
}
|
||||
block["is_ad"] = self.ad_classifier.predict(image, box)
|
||||
blocks.append(block)
|
||||
return blocks
|
||||
|
||||
|
||||
class AdClassifier:
|
||||
"""
|
||||
Ads rarely match column grids and contain logos/price patterns.
|
||||
Train a lightweight CNN (MobileNet) on labeled ad/non-ad crops,
|
||||
OR use heuristics + LLM verification. Stub below.
|
||||
"""
|
||||
def predict(self, image, box) -> bool:
|
||||
# Heuristic placeholder: replace with trained model
|
||||
return False
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
from paddleocr import PaddleOCR
|
||||
import numpy as np
|
||||
|
||||
class OCREngine:
|
||||
def __init__(self, langs=["en", "te"]):
|
||||
# PaddleOCR v2 — one instance per language, angle cls enabled
|
||||
self.engines = {
|
||||
l: PaddleOCR(use_angle_cls=True, lang=l, show_log=False)
|
||||
for l in langs
|
||||
}
|
||||
|
||||
def ocr_crop(self, image: np.ndarray, lang="en"):
|
||||
engine = self.engines.get(lang, self.engines["en"])
|
||||
result = engine.ocr(image, cls=True)
|
||||
lines = []
|
||||
for line in (result[0] or []):
|
||||
box, (text, conf) = line
|
||||
lines.append({"text": text, "conf": conf, "box": box})
|
||||
return lines
|
||||
|
||||
def extract_text(self, page, article_bbox, native_words):
|
||||
x, y, w, h = article_bbox
|
||||
if native_words: # searchable PDF — use native text inside bbox
|
||||
words = [t for (wx0,wy0,wx1,wy1,t) in native_words
|
||||
if wx0 >= x and wy0 >= y and wx1 <= x+w and wy1 <= y+h]
|
||||
text = " ".join(words)
|
||||
if len(text) > 50:
|
||||
return text, 0.99
|
||||
# Fallback to OCR
|
||||
crop = page["image"][int(y):int(y+h), int(x):int(x+w)]
|
||||
lang = self.detect_language(crop)
|
||||
lines = self.ocr_crop(crop, lang)
|
||||
text = "\n".join(l["text"] for l in lines)
|
||||
conf = float(np.mean([l["conf"] for l in lines])) if lines else 0.0
|
||||
return text, conf
|
||||
|
||||
def detect_language(self, crop):
|
||||
# Quick langdetect on a fast English OCR pass, or script detection
|
||||
return "en" # plug in script-detection model
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
from app.pipeline.pdf_reader import PDFReader
|
||||
from app.pipeline.layout_detector import LayoutDetector
|
||||
from app.pipeline.article_segmenter import ArticleSegmenter
|
||||
from app.pipeline.ocr_engine import OCREngine
|
||||
from app.pipeline.headline import detect_headline
|
||||
from app.pipeline.cropper import crop_article, crop_figures
|
||||
from app.pipeline.categorizer import Categorizer
|
||||
from app.pipeline.enricher import Enricher
|
||||
from app.config import settings
|
||||
import os
|
||||
|
||||
class Pipeline:
|
||||
def __init__(self):
|
||||
self.layout = LayoutDetector()
|
||||
self.segmenter = ArticleSegmenter()
|
||||
self.ocr = OCREngine(settings.OCR_LANGS)
|
||||
self.cat = Categorizer()
|
||||
self.enricher = Enricher()
|
||||
|
||||
def process(self, pdf_path, newspaper, edition, date, job_dir):
|
||||
reader = PDFReader(pdf_path)
|
||||
results = []
|
||||
for page in reader.iter_pages():
|
||||
h, w = page["image"].shape[:2]
|
||||
blocks = self.layout.detect(page["image"])
|
||||
articles = self.segmenter.segment(blocks, w, h)
|
||||
for art in articles:
|
||||
text, conf = self.ocr.extract_text(page, art["bbox"], page["native_words"])
|
||||
if len(text.strip()) < 80: # skip noise/ads
|
||||
continue
|
||||
headline = detect_headline(art, page, self.ocr)
|
||||
img_path = crop_article(page["image"], art["bbox"], job_dir)
|
||||
fig_paths = crop_figures(page["image"], art, job_dir)
|
||||
meta = self.cat.classify(headline, text)
|
||||
embedding = self.enricher.embed(headline + " " + text[:1000])
|
||||
results.append({
|
||||
"date": date, "newspaper": newspaper, "edition": edition,
|
||||
"page": page["page_number"], "headline": headline,
|
||||
"category": meta["category"], "subcategory": meta.get("subcategory"),
|
||||
"text": text, "summary": meta.get("summary"),
|
||||
"keywords": meta.get("keywords"), "persons": meta.get("persons"),
|
||||
"locations": meta.get("locations"), "organizations": meta.get("organizations"),
|
||||
"sentiment": meta.get("sentiment"), "language": "en",
|
||||
"image_path": img_path, "figure_paths": fig_paths,
|
||||
"bounding_box": art["bbox"],
|
||||
"confidence": round(art["confidence"] * meta.get("confidence", 1), 3),
|
||||
"embedding": embedding,
|
||||
})
|
||||
return results
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
import fitz # PyMuPDF
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from app.config import settings
|
||||
|
||||
class PDFReader:
|
||||
def __init__(self, path: str):
|
||||
self.doc = fitz.open(path)
|
||||
|
||||
def is_searchable(self, page) -> bool:
|
||||
"""Detect if a page has a usable native text layer."""
|
||||
text = page.get_text("text").strip()
|
||||
return len(text) > 100 # heuristic
|
||||
|
||||
def page_to_image(self, page) -> np.ndarray:
|
||||
zoom = settings.DPI / 72
|
||||
mat = fitz.Matrix(zoom, zoom)
|
||||
pix = page.get_pixmap(matrix=mat, alpha=False)
|
||||
img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
|
||||
return np.array(img)
|
||||
|
||||
def native_text_blocks(self, page):
|
||||
"""Returns word boxes from searchable PDF: [(x0,y0,x1,y1,text)]"""
|
||||
words = page.get_text("words")
|
||||
scale = settings.DPI / 72
|
||||
return [(w[0]*scale, w[1]*scale, w[2]*scale, w[3]*scale, w[4]) for w in words]
|
||||
|
||||
def iter_pages(self):
|
||||
for i, page in enumerate(self.doc):
|
||||
yield {
|
||||
"page_number": i + 1,
|
||||
"image": self.page_to_image(page),
|
||||
"searchable": self.is_searchable(page),
|
||||
"native_words": self.native_text_blocks(page) if self.is_searchable(page) else [],
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
from celery import Celery
|
||||
from app.config import settings
|
||||
|
||||
celery = Celery(
|
||||
"newspaper_tasks",
|
||||
broker=settings.REDIS_URL,
|
||||
backend=settings.REDIS_URL,
|
||||
include=["app.tasks.ingest"]
|
||||
)
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
from app.tasks.celery_app import celery
|
||||
from app.pipeline.orchestrator import Pipeline
|
||||
from app.db.session import SessionLocal
|
||||
from app.db.models import Article, ArticleImage, UploadJob
|
||||
import hashlib, os
|
||||
|
||||
pipeline = None
|
||||
|
||||
@celery.task(bind=True)
|
||||
def process_pdf(self, job_id, pdf_path, newspaper, edition, date):
|
||||
db = SessionLocal()
|
||||
job = db.query(UploadJob).filter_by(id=job_id).first()
|
||||
if job:
|
||||
job.status = "PROCESSING"
|
||||
db.commit()
|
||||
|
||||
global pipeline
|
||||
if pipeline is None:
|
||||
from app.pipeline.orchestrator import Pipeline
|
||||
pipeline = Pipeline()
|
||||
|
||||
job_dir = os.path.join("/data/storage", os.path.basename(pdf_path) + "_crops")
|
||||
self.update_state(state="PROCESSING", meta={"stage": "layout"})
|
||||
try:
|
||||
articles = pipeline.process(pdf_path, newspaper, edition, date, job_dir)
|
||||
for a in articles:
|
||||
chash = hashlib.sha256(a["text"][:500].encode()).hexdigest()
|
||||
if db.query(Article).filter_by(content_hash=chash).first():
|
||||
continue # dedup / syndicated
|
||||
art = Article(content_hash=chash, **{k: a[k] for k in [
|
||||
"date","newspaper","edition","page","headline","category",
|
||||
"subcategory","text","summary","keywords","persons","locations",
|
||||
"organizations","sentiment","language","image_path",
|
||||
"bounding_box","confidence","embedding"]})
|
||||
art.upload_job_id = job_id
|
||||
db.add(art); db.flush()
|
||||
for fp in a["figure_paths"]:
|
||||
db.add(ArticleImage(article_id=art.id, image_path=fp))
|
||||
if job:
|
||||
job.status = "COMPLETED"
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
if job:
|
||||
job.status = "FAILED"
|
||||
job.error_message = str(e)
|
||||
db.commit()
|
||||
raise e
|
||||
finally:
|
||||
db.close()
|
||||
return {"articles": len(articles)}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
fastapi
|
||||
uvicorn
|
||||
pydantic-settings
|
||||
sqlalchemy
|
||||
psycopg2-binary
|
||||
pgvector
|
||||
celery
|
||||
redis
|
||||
PyMuPDF
|
||||
Pillow
|
||||
numpy
|
||||
scipy
|
||||
scikit-learn
|
||||
layoutparser[detectron2]
|
||||
paddleocr==2.7.3
|
||||
paddlepaddle==2.6.2
|
||||
sentence-transformers
|
||||
google-generativeai
|
||||
transformers
|
||||
opencv-python-headless
|
||||
python-multipart
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
version: "3.9"
|
||||
services:
|
||||
db:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: user
|
||||
POSTGRES_PASSWORD: pass
|
||||
POSTGRES_DB: news
|
||||
volumes: ["pgdata:/var/lib/postgresql/data"]
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
|
||||
backend:
|
||||
build: ./backend
|
||||
env_file: .env
|
||||
depends_on: [db, redis]
|
||||
volumes: ["./data:/data", "./backend:/app"]
|
||||
ports: ["8080:8000"]
|
||||
command: uvicorn app.main:app --host 0.0.0.0 --port 8000
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices: [{capabilities: ["gpu"]}] # GPU acceleration
|
||||
|
||||
worker:
|
||||
build: ./backend
|
||||
env_file: .env
|
||||
depends_on: [db, redis]
|
||||
volumes: ["./data:/data", "./backend:/app"]
|
||||
command: celery -A app.tasks.celery_app worker --loglevel=info --concurrency=2
|
||||
|
||||
frontend:
|
||||
build: ./frontend
|
||||
ports: ["3000:3000"]
|
||||
volumes:
|
||||
- "./frontend:/app"
|
||||
- "/app/node_modules"
|
||||
command: npm run dev
|
||||
depends_on: [backend]
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
node_modules
|
||||
.next
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
/.pnp
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/versions
|
||||
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# next.js
|
||||
/.next/
|
||||
/out/
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
*.pem
|
||||
|
||||
# debug
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
next-env.d.ts
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
<!-- BEGIN:nextjs-agent-rules -->
|
||||
# This is NOT the Next.js you know
|
||||
|
||||
This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` before writing any code. Heed deprecation notices.
|
||||
<!-- END:nextjs-agent-rules -->
|
||||
|
|
@ -0,0 +1 @@
|
|||
@AGENTS.md
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
FROM node:20-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package*.json ./
|
||||
RUN npm install
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN npm run build
|
||||
|
||||
CMD ["npm", "start"]
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app).
|
||||
|
||||
## Getting Started
|
||||
|
||||
First, run the development server:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
# or
|
||||
yarn dev
|
||||
# or
|
||||
pnpm dev
|
||||
# or
|
||||
bun dev
|
||||
```
|
||||
|
||||
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
|
||||
|
||||
You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
|
||||
|
||||
This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel.
|
||||
|
||||
## Learn More
|
||||
|
||||
To learn more about Next.js, take a look at the following resources:
|
||||
|
||||
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
|
||||
- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
|
||||
|
||||
You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
|
||||
|
||||
## Deploy on Vercel
|
||||
|
||||
The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
|
||||
|
||||
Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details.
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
import { defineConfig, globalIgnores } from "eslint/config";
|
||||
import nextVitals from "eslint-config-next/core-web-vitals";
|
||||
import nextTs from "eslint-config-next/typescript";
|
||||
|
||||
const eslintConfig = defineConfig([
|
||||
...nextVitals,
|
||||
...nextTs,
|
||||
// Override default ignores of eslint-config-next.
|
||||
globalIgnores([
|
||||
// Default ignores of eslint-config-next:
|
||||
".next/**",
|
||||
"out/**",
|
||||
"build/**",
|
||||
"next-env.d.ts",
|
||||
]),
|
||||
]);
|
||||
|
||||
export default eslintConfig;
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
import type { NextConfig } from "next";
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
async rewrites() {
|
||||
return [
|
||||
{
|
||||
source: '/api/:path*',
|
||||
destination: 'http://backend:8000/api/:path*',
|
||||
},
|
||||
];
|
||||
},
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,29 @@
|
|||
{
|
||||
"name": "frontend",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "eslint"
|
||||
},
|
||||
"dependencies": {
|
||||
"clsx": "^2.1.1",
|
||||
"lucide-react": "^1.22.0",
|
||||
"next": "16.2.9",
|
||||
"react": "19.2.4",
|
||||
"react-dom": "19.2.4",
|
||||
"tailwind-merge": "^3.6.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/postcss": "^4",
|
||||
"@types/node": "^20",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"eslint": "^9",
|
||||
"eslint-config-next": "16.2.9",
|
||||
"tailwindcss": "^4",
|
||||
"typescript": "^5"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
const config = {
|
||||
plugins: {
|
||||
"@tailwindcss/postcss": {},
|
||||
},
|
||||
};
|
||||
|
||||
export default config;
|
||||
|
|
@ -0,0 +1 @@
|
|||
<svg fill="none" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path d="M14.5 13.5V5.41a1 1 0 0 0-.3-.7L9.8.29A1 1 0 0 0 9.08 0H1.5v13.5A2.5 2.5 0 0 0 4 16h8a2.5 2.5 0 0 0 2.5-2.5m-1.5 0v-7H8v-5H3v12a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1M9.5 5V2.12L12.38 5zM5.13 5h-.62v1.25h2.12V5zm-.62 3h7.12v1.25H4.5zm.62 3h-.62v1.25h7.12V11z" clip-rule="evenodd" fill="#666" fill-rule="evenodd"/></svg>
|
||||
|
After Width: | Height: | Size: 391 B |
|
|
@ -0,0 +1 @@
|
|||
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><g clip-path="url(#a)"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.27 14.1a6.5 6.5 0 0 0 3.67-3.45q-1.24.21-2.7.34-.31 1.83-.97 3.1M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16m.48-1.52a7 7 0 0 1-.96 0H7.5a4 4 0 0 1-.84-1.32q-.38-.89-.63-2.08a40 40 0 0 0 3.92 0q-.25 1.2-.63 2.08a4 4 0 0 1-.84 1.31zm2.94-4.76q1.66-.15 2.95-.43a7 7 0 0 0 0-2.58q-1.3-.27-2.95-.43a18 18 0 0 1 0 3.44m-1.27-3.54a17 17 0 0 1 0 3.64 39 39 0 0 1-4.3 0 17 17 0 0 1 0-3.64 39 39 0 0 1 4.3 0m1.1-1.17q1.45.13 2.69.34a6.5 6.5 0 0 0-3.67-3.44q.65 1.26.98 3.1M8.48 1.5l.01.02q.41.37.84 1.31.38.89.63 2.08a40 40 0 0 0-3.92 0q.25-1.2.63-2.08a4 4 0 0 1 .85-1.32 7 7 0 0 1 .96 0m-2.75.4a6.5 6.5 0 0 0-3.67 3.44 29 29 0 0 1 2.7-.34q.31-1.83.97-3.1M4.58 6.28q-1.66.16-2.95.43a7 7 0 0 0 0 2.58q1.3.27 2.95.43a18 18 0 0 1 0-3.44m.17 4.71q-1.45-.12-2.69-.34a6.5 6.5 0 0 0 3.67 3.44q-.65-1.27-.98-3.1" fill="#666"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h16v16H0z"/></clipPath></defs></svg>
|
||||
|
After Width: | Height: | Size: 1.0 KiB |
|
|
@ -0,0 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
|
|
@ -0,0 +1 @@
|
|||
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1155 1000"><path d="m577.3 0 577.4 1000H0z" fill="#fff"/></svg>
|
||||
|
After Width: | Height: | Size: 128 B |
|
|
@ -0,0 +1 @@
|
|||
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M1.5 2.5h13v10a1 1 0 0 1-1 1h-11a1 1 0 0 1-1-1zM0 1h16v11.5a2.5 2.5 0 0 1-2.5 2.5h-11A2.5 2.5 0 0 1 0 12.5zm3.75 4.5a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5M7 4.75a.75.75 0 1 1-1.5 0 .75.75 0 0 1 1.5 0m1.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5" fill="#666"/></svg>
|
||||
|
After Width: | Height: | Size: 385 B |
|
|
@ -0,0 +1,36 @@
|
|||
"use client";
|
||||
import { useState } from "react";
|
||||
|
||||
export default function Dashboard() {
|
||||
const [task, setTask] = useState<string|null>(null);
|
||||
const [status, setStatus] = useState("");
|
||||
|
||||
async function upload(file: File) {
|
||||
const fd = new FormData();
|
||||
fd.append("file", file);
|
||||
fd.append("newspaper", "The Hindu");
|
||||
fd.append("date", new Date().toISOString().slice(0,10));
|
||||
const r = await fetch("http://localhost:8000/api/upload", {method:"POST", body:fd});
|
||||
const j = await r.json();
|
||||
setTask(j.task_id);
|
||||
poll(j.task_id);
|
||||
}
|
||||
|
||||
async function poll(id: string) {
|
||||
const r = await fetch(`http://localhost:8000/api/status/${id}`);
|
||||
const j = await r.json();
|
||||
setStatus(j.state);
|
||||
if (j.state !== "SUCCESS" && j.state !== "FAILURE")
|
||||
setTimeout(() => poll(id), 2000);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="p-8">
|
||||
<h1 className="text-2xl font-bold mb-4">Newspaper Extractor</h1>
|
||||
<input type="file" accept="application/pdf"
|
||||
onChange={e => e.target.files && upload(e.target.files[0])}
|
||||
className="border p-4 rounded" />
|
||||
{task && <p className="mt-4">Status: {status}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
|
|
@ -0,0 +1,44 @@
|
|||
@import "tailwindcss";
|
||||
|
||||
@theme inline {
|
||||
--color-background: #030712; /* gray-950 */
|
||||
--color-foreground: #f9fafb; /* gray-50 */
|
||||
--font-sans: var(--font-geist-sans), ui-sans-serif, system-ui, sans-serif;
|
||||
--font-mono: var(--font-geist-mono), ui-monospace, SFMono-Regular, monospace;
|
||||
}
|
||||
|
||||
:root {
|
||||
--background: #030712;
|
||||
--foreground: #f9fafb;
|
||||
}
|
||||
|
||||
body {
|
||||
background-color: var(--background);
|
||||
color: var(--foreground);
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
}
|
||||
|
||||
/* Custom Scrollbar for a premium feel */
|
||||
::-webkit-scrollbar {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
}
|
||||
::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
::-webkit-scrollbar-thumb {
|
||||
background: #374151;
|
||||
border-radius: 4px;
|
||||
}
|
||||
::-webkit-scrollbar-thumb:hover {
|
||||
background: #4b5563;
|
||||
}
|
||||
|
||||
/* Glassmorphism utilities */
|
||||
.glass {
|
||||
background: rgba(17, 24, 39, 0.7);
|
||||
backdrop-filter: blur(12px);
|
||||
-webkit-backdrop-filter: blur(12px);
|
||||
border: 1px solid rgba(255, 255, 255, 0.05);
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
import type { Metadata } from "next";
|
||||
import { Geist, Geist_Mono } from "next/font/google";
|
||||
import "./globals.css";
|
||||
|
||||
const geistSans = Geist({
|
||||
variable: "--font-geist-sans",
|
||||
subsets: ["latin"],
|
||||
});
|
||||
|
||||
const geistMono = Geist_Mono({
|
||||
variable: "--font-geist-mono",
|
||||
subsets: ["latin"],
|
||||
});
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: "Create Next App",
|
||||
description: "Generated by create next app",
|
||||
};
|
||||
|
||||
export default function RootLayout({
|
||||
children,
|
||||
}: Readonly<{
|
||||
children: React.ReactNode;
|
||||
}>) {
|
||||
return (
|
||||
<html
|
||||
lang="en"
|
||||
className={`${geistSans.variable} ${geistMono.variable} h-full antialiased`}
|
||||
>
|
||||
<body className="min-h-full flex flex-col">{children}</body>
|
||||
</html>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,136 @@
|
|||
'use client';
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import { SearchBar } from '@/components/SearchBar';
|
||||
import { ArticleCard } from '@/components/ArticleCard';
|
||||
import { UploadModal } from '@/components/UploadModal';
|
||||
import UploadQueue from '@/components/UploadQueue';
|
||||
import { searchArticles, Article } from '@/lib/api';
|
||||
import { Newspaper, Upload, Sparkles, AlertCircle } from 'lucide-react';
|
||||
|
||||
export default function Home() {
|
||||
const [articles, setArticles] = useState<Article[]>([]);
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [isUploadModalOpen, setIsUploadModalOpen] = useState(false);
|
||||
const [hasSearched, setHasSearched] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [selectedJobId, setSelectedJobId] = useState<number | null>(null);
|
||||
|
||||
// Fetch initial generic results on load
|
||||
useEffect(() => {
|
||||
handleSearch('', false);
|
||||
}, []);
|
||||
|
||||
const handleSearch = async (query: string, semantic: boolean, jobId: number | null = selectedJobId) => {
|
||||
setIsLoading(true);
|
||||
setHasSearched(true);
|
||||
setError(null);
|
||||
try {
|
||||
const results = await searchArticles(query, semantic, jobId || undefined);
|
||||
setArticles(results || []);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
setError('Failed to fetch articles. Please ensure the backend is running.');
|
||||
setArticles([]);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleJobSelect = (jobId: number | null) => {
|
||||
setSelectedJobId(jobId);
|
||||
handleSearch('', false, jobId);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-[#030712] text-zinc-50 font-sans selection:bg-indigo-500/30">
|
||||
|
||||
{/* Navbar */}
|
||||
<nav className="fixed top-0 left-0 right-0 z-40 glass border-b border-white/5">
|
||||
<div className="max-w-7xl mx-auto px-6 h-16 flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="bg-indigo-500/10 p-2 rounded-lg border border-indigo-500/20">
|
||||
<Newspaper className="text-indigo-400" size={24} />
|
||||
</div>
|
||||
<span className="font-bold tracking-tight text-xl bg-clip-text text-transparent bg-gradient-to-r from-zinc-100 to-zinc-400">
|
||||
NeuralNews
|
||||
</span>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => setIsUploadModalOpen(true)}
|
||||
className="flex items-center gap-2 bg-white/5 hover:bg-white/10 border border-white/10 px-4 py-2 rounded-lg text-sm font-medium transition-all"
|
||||
>
|
||||
<Upload size={16} />
|
||||
Ingest PDF
|
||||
</button>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
{/* Hero Section */}
|
||||
<main className="pt-32 pb-20 px-6 max-w-7xl mx-auto">
|
||||
<div className="flex flex-col items-center text-center mb-16 space-y-6">
|
||||
<div className="inline-flex items-center gap-2 px-3 py-1 rounded-full bg-indigo-500/10 text-indigo-400 border border-indigo-500/20 text-sm font-medium">
|
||||
<Sparkles size={16} />
|
||||
Powered by OCR & Layout Parsing
|
||||
</div>
|
||||
<h1 className="text-5xl md:text-7xl font-extrabold tracking-tight">
|
||||
Intelligent <span className="text-transparent bg-clip-text bg-gradient-to-r from-indigo-400 to-cyan-400">Newspaper</span> <br className="hidden md:block"/> Extraction
|
||||
</h1>
|
||||
<p className="max-w-2xl text-lg text-zinc-400">
|
||||
Search through semantically embedded newspaper articles, categorized by layout and translated via AI.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Main Content Layout with Sidebar */}
|
||||
<div className="flex flex-col lg:flex-row gap-8">
|
||||
{/* Main Feed */}
|
||||
<div className="flex-1">
|
||||
{/* Search Bar */}
|
||||
<div className="mb-8">
|
||||
<SearchBar onSearch={(q, s) => handleSearch(q, s, selectedJobId)} isLoading={isLoading} />
|
||||
</div>
|
||||
|
||||
{/* Results */}
|
||||
{error ? (
|
||||
<div className="flex flex-col items-center justify-center py-20 text-center">
|
||||
<AlertCircle className="text-red-400 mb-4" size={48} />
|
||||
<h2 className="text-xl font-medium text-red-100 mb-2">Connection Error</h2>
|
||||
<p className="text-red-400/80">{error}</p>
|
||||
</div>
|
||||
) : isLoading ? (
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-6 animate-pulse">
|
||||
{[1, 2, 3, 4].map(i => (
|
||||
<div key={i} className="h-64 rounded-2xl bg-zinc-900 border border-zinc-800"></div>
|
||||
))}
|
||||
</div>
|
||||
) : articles.length > 0 ? (
|
||||
<div className="grid grid-cols-1 xl:grid-cols-2 gap-6">
|
||||
{articles.map(article => (
|
||||
<ArticleCard key={article.id} article={article} />
|
||||
))}
|
||||
</div>
|
||||
) : hasSearched ? (
|
||||
<div className="text-center py-20">
|
||||
<h3 className="text-2xl font-semibold text-zinc-300">No articles found</h3>
|
||||
<p className="text-zinc-500 mt-2">Try adjusting your search query or uploading a new newspaper.</p>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
{/* Sidebar */}
|
||||
<div className="w-full lg:w-80 flex-shrink-0 h-[calc(100vh-200px)] sticky top-24">
|
||||
<UploadQueue
|
||||
onSelectJob={handleJobSelect}
|
||||
selectedJobId={selectedJobId}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<UploadModal
|
||||
isOpen={isUploadModalOpen}
|
||||
onClose={() => setIsUploadModalOpen(false)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
'use client';
|
||||
|
||||
import { Article } from '@/lib/api';
|
||||
import { Calendar, Newspaper, Hash, Smile, Frown, Meh, Image as ImageIcon } from 'lucide-react';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
interface ArticleCardProps {
|
||||
article: Article;
|
||||
}
|
||||
|
||||
export function ArticleCard({ article }: ArticleCardProps) {
|
||||
// Determine sentiment badge
|
||||
const sentimentScore = parseFloat(article.sentiment || "0");
|
||||
const isPositive = sentimentScore > 0.3;
|
||||
const isNegative = sentimentScore < -0.3;
|
||||
|
||||
// Create an image url if image path exists
|
||||
// The path is internal to docker like /data/storage/cropped_XYZ.jpg
|
||||
// Assuming the backend mounts /data/storage at /api/images
|
||||
const imageUrl = article.image_path
|
||||
? `http://localhost:8000/api/images/${article.image_path.split('/').pop()}`
|
||||
: null;
|
||||
|
||||
return (
|
||||
<div className="group relative flex flex-col gap-4 rounded-2xl border border-zinc-800 bg-zinc-900/50 p-6 transition-all hover:bg-zinc-900 hover:shadow-xl hover:shadow-indigo-500/5 hover:-translate-y-1">
|
||||
{/* Header info */}
|
||||
<div className="flex items-start justify-between">
|
||||
<div className="flex flex-wrap items-center gap-3 text-xs font-medium text-zinc-400">
|
||||
<span className="flex items-center gap-1.5 rounded-full bg-zinc-800/50 px-2.5 py-1">
|
||||
<Newspaper size={14} className="text-zinc-500" />
|
||||
{article.newspaper} ({article.edition}) - P.{article.page}
|
||||
</span>
|
||||
<span className="flex items-center gap-1.5 rounded-full bg-zinc-800/50 px-2.5 py-1">
|
||||
<Calendar size={14} className="text-zinc-500" />
|
||||
{article.date}
|
||||
</span>
|
||||
<span className="flex items-center gap-1.5 rounded-full bg-zinc-800/50 px-2.5 py-1">
|
||||
<Hash size={14} className="text-zinc-500" />
|
||||
{article.category} {article.subcategory ? `/ ${article.subcategory}` : ''}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Sentiment Badge */}
|
||||
<div className={cn(
|
||||
"flex items-center gap-1.5 rounded-full px-2.5 py-1 text-xs font-semibold",
|
||||
isPositive ? "bg-emerald-500/10 text-emerald-400" :
|
||||
isNegative ? "bg-rose-500/10 text-rose-400" :
|
||||
"bg-blue-500/10 text-blue-400"
|
||||
)}>
|
||||
{isPositive ? <Smile size={14} /> : isNegative ? <Frown size={14} /> : <Meh size={14} />}
|
||||
{isPositive ? "Positive" : isNegative ? "Negative" : "Neutral"}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Title & Content */}
|
||||
<div className="flex flex-col md:flex-row gap-6">
|
||||
<div className="flex-1 space-y-3">
|
||||
<h3 className="text-xl font-bold tracking-tight text-zinc-100 group-hover:text-indigo-400 transition-colors">
|
||||
{article.headline || "Untitled Article"}
|
||||
</h3>
|
||||
{article.subheadline && (
|
||||
<h4 className="text-sm font-medium text-zinc-400">
|
||||
{article.subheadline}
|
||||
</h4>
|
||||
)}
|
||||
|
||||
<p className="text-sm leading-relaxed text-zinc-300">
|
||||
{article.summary || article.text?.slice(0, 250) + "..."}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Optional Image */}
|
||||
{imageUrl ? (
|
||||
<div className="shrink-0">
|
||||
<img
|
||||
src={imageUrl}
|
||||
alt="Article snippet"
|
||||
className="h-32 w-48 object-cover rounded-xl border border-zinc-800 bg-zinc-950"
|
||||
onError={(e) => {
|
||||
// fallback if image fails to load
|
||||
e.currentTarget.style.display = 'none';
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<div className="shrink-0 flex h-32 w-48 items-center justify-center rounded-xl border border-dashed border-zinc-800 bg-zinc-900/50 text-zinc-600">
|
||||
<ImageIcon size={24} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Entities / Tags */}
|
||||
<div className="mt-2 flex flex-wrap gap-2">
|
||||
{article.persons?.map(p => (
|
||||
<span key={`p-${p}`} className="rounded-md bg-indigo-500/10 px-2 py-1 text-xs font-medium text-indigo-400 border border-indigo-500/20">
|
||||
👤 {p}
|
||||
</span>
|
||||
))}
|
||||
{article.locations?.map(l => (
|
||||
<span key={`l-${l}`} className="rounded-md bg-amber-500/10 px-2 py-1 text-xs font-medium text-amber-400 border border-amber-500/20">
|
||||
📍 {l}
|
||||
</span>
|
||||
))}
|
||||
{article.organizations?.map(o => (
|
||||
<span key={`o-${o}`} className="rounded-md bg-cyan-500/10 px-2 py-1 text-xs font-medium text-cyan-400 border border-cyan-500/20">
|
||||
🏢 {o}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
'use client';
|
||||
|
||||
import { Search, Sparkles } from 'lucide-react';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { useState } from 'react';
|
||||
|
||||
interface SearchBarProps {
|
||||
onSearch: (query: string, semantic: boolean) => void;
|
||||
isLoading: boolean;
|
||||
}
|
||||
|
||||
export function SearchBar({ onSearch, isLoading }: SearchBarProps) {
|
||||
const [query, setQuery] = useState('');
|
||||
const [isSemantic, setIsSemantic] = useState(false);
|
||||
|
||||
const handleSubmit = (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
onSearch(query, isSemantic);
|
||||
};
|
||||
|
||||
return (
|
||||
<form onSubmit={handleSubmit} className="w-full max-w-3xl mx-auto">
|
||||
<div className="relative group flex items-center bg-zinc-900/80 rounded-2xl border border-zinc-800 focus-within:border-indigo-500/50 focus-within:ring-2 focus-within:ring-indigo-500/20 shadow-2xl transition-all duration-300">
|
||||
<div className="pl-6 flex-1 flex items-center">
|
||||
<Search className="text-zinc-500 mr-3" size={20} />
|
||||
<input
|
||||
type="text"
|
||||
value={query}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
placeholder={isSemantic ? "Describe the concept or event you're looking for..." : "Search headlines, text, or keywords..."}
|
||||
className="w-full bg-transparent py-5 text-zinc-100 placeholder-zinc-500 focus:outline-none text-lg"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="pr-4 pl-4 flex items-center gap-3 border-l border-zinc-800">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setIsSemantic(!isSemantic)}
|
||||
className={cn(
|
||||
"flex items-center gap-2 px-3 py-1.5 rounded-lg text-sm font-medium transition-all duration-300",
|
||||
isSemantic
|
||||
? "bg-indigo-500/10 text-indigo-400 border border-indigo-500/20"
|
||||
: "text-zinc-500 hover:text-zinc-300 border border-transparent"
|
||||
)}
|
||||
>
|
||||
<Sparkles size={16} className={isSemantic ? "animate-pulse" : ""} />
|
||||
Semantic AI
|
||||
</button>
|
||||
|
||||
<button
|
||||
type="submit"
|
||||
disabled={isLoading || !query.trim()}
|
||||
className="bg-zinc-100 text-zinc-900 px-6 py-2.5 rounded-xl font-semibold hover:bg-white active:scale-95 transition-all disabled:opacity-50 disabled:cursor-not-allowed"
|
||||
>
|
||||
{isLoading ? 'Searching...' : 'Search'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
'use client';
|
||||
|
||||
import { useState } from 'react';
|
||||
import { uploadPDF } from '@/lib/api';
|
||||
import { X, UploadCloud, Loader2 } from 'lucide-react';
|
||||
|
||||
interface UploadModalProps {
|
||||
isOpen: boolean;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export function UploadModal({ isOpen, onClose }: UploadModalProps) {
|
||||
const [file, setFile] = useState<File | null>(null);
|
||||
const [newspaper, setNewspaper] = useState('Andhra Jyothi');
|
||||
const [edition, setEdition] = useState('Main');
|
||||
const [date, setDate] = useState(new Date().toISOString().split('T')[0]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [taskId, setTaskId] = useState<string | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
if (!isOpen) return null;
|
||||
|
||||
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
if (e.target.files && e.target.files.length > 0) {
|
||||
setFile(e.target.files[0]);
|
||||
}
|
||||
};
|
||||
|
||||
const handleUpload = async () => {
|
||||
if (!file) {
|
||||
setError('Please select a PDF file.');
|
||||
return;
|
||||
}
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const res = await uploadPDF(file, newspaper, edition, date);
|
||||
setTaskId(res.task_id);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/60 backdrop-blur-sm">
|
||||
<div className="w-full max-w-md rounded-2xl border border-zinc-800 bg-zinc-950 p-6 shadow-2xl">
|
||||
<div className="flex items-center justify-between mb-6">
|
||||
<h2 className="text-xl font-semibold text-zinc-100">Upload Newspaper</h2>
|
||||
<button onClick={onClose} className="rounded-full p-2 text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100">
|
||||
<X size={20} />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{taskId ? (
|
||||
<div className="text-center py-8">
|
||||
<div className="mb-4 inline-flex h-12 w-12 items-center justify-center rounded-full bg-emerald-500/20 text-emerald-400">
|
||||
<UploadCloud size={24} />
|
||||
</div>
|
||||
<h3 className="mb-2 text-lg font-medium text-zinc-100">Upload Successful</h3>
|
||||
<p className="text-sm text-zinc-400">
|
||||
Task queued for processing. It may take a few minutes for the pipeline to extract all data.
|
||||
</p>
|
||||
<p className="mt-4 text-xs font-mono text-zinc-500">Task ID: {taskId}</p>
|
||||
<button
|
||||
onClick={() => {
|
||||
setTaskId(null);
|
||||
setFile(null);
|
||||
onClose();
|
||||
}}
|
||||
className="mt-6 w-full rounded-lg bg-zinc-800 py-2 text-sm font-medium text-zinc-200 hover:bg-zinc-700"
|
||||
>
|
||||
Close
|
||||
</button>
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<label className="mb-1 block text-sm font-medium text-zinc-400">Newspaper PDF</label>
|
||||
<div className="relative flex w-full cursor-pointer flex-col items-center justify-center rounded-xl border-2 border-dashed border-zinc-800 bg-zinc-900/50 py-8 hover:border-zinc-700 hover:bg-zinc-900">
|
||||
<input
|
||||
type="file"
|
||||
accept="application/pdf"
|
||||
className="absolute inset-0 z-10 h-full w-full cursor-pointer opacity-0"
|
||||
onChange={handleFileChange}
|
||||
/>
|
||||
<UploadCloud className="mb-2 text-zinc-500" size={32} />
|
||||
<span className="text-sm font-medium text-zinc-300">
|
||||
{file ? file.name : 'Click or drag PDF to upload'}
|
||||
</span>
|
||||
<span className="mt-1 text-xs text-zinc-500">Max size 50MB</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label className="mb-1 block text-sm font-medium text-zinc-400">Newspaper</label>
|
||||
<input
|
||||
type="text"
|
||||
value={newspaper}
|
||||
onChange={e => setNewspaper(e.target.value)}
|
||||
className="w-full rounded-lg border border-zinc-800 bg-zinc-900 px-3 py-2 text-sm text-zinc-100 placeholder-zinc-600 focus:border-indigo-500 focus:outline-none focus:ring-1 focus:ring-indigo-500"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label className="mb-1 block text-sm font-medium text-zinc-400">Edition</label>
|
||||
<input
|
||||
type="text"
|
||||
value={edition}
|
||||
onChange={e => setEdition(e.target.value)}
|
||||
className="w-full rounded-lg border border-zinc-800 bg-zinc-900 px-3 py-2 text-sm text-zinc-100 placeholder-zinc-600 focus:border-indigo-500 focus:outline-none focus:ring-1 focus:ring-indigo-500"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="mb-1 block text-sm font-medium text-zinc-400">Date</label>
|
||||
<input
|
||||
type="date"
|
||||
value={date}
|
||||
onChange={e => setDate(e.target.value)}
|
||||
className="w-full rounded-lg border border-zinc-800 bg-zinc-900 px-3 py-2 text-sm text-zinc-100 focus:border-indigo-500 focus:outline-none focus:ring-1 focus:ring-indigo-500"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="rounded-lg bg-red-500/10 p-3 text-sm text-red-400 border border-red-500/20">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<button
|
||||
onClick={handleUpload}
|
||||
disabled={loading || !file}
|
||||
className="mt-2 flex w-full items-center justify-center rounded-lg bg-indigo-600 py-2.5 text-sm font-semibold text-white transition-colors hover:bg-indigo-500 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
{loading ? (
|
||||
<>
|
||||
<Loader2 className="mr-2 animate-spin" size={16} />
|
||||
Uploading...
|
||||
</>
|
||||
) : (
|
||||
'Start Ingestion Pipeline'
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
'use client';
|
||||
import { useEffect, useState } from 'react';
|
||||
import { getJobs, UploadJob } from '@/lib/api';
|
||||
import { FileText, Clock, CheckCircle2, XCircle, RefreshCw } from 'lucide-react';
|
||||
|
||||
export default function UploadQueue({
|
||||
onSelectJob,
|
||||
selectedJobId
|
||||
}: {
|
||||
onSelectJob: (jobId: number | null) => void;
|
||||
selectedJobId: number | null;
|
||||
}) {
|
||||
const [jobs, setJobs] = useState<UploadJob[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
const fetchJobs = async () => {
|
||||
try {
|
||||
const data = await getJobs();
|
||||
setJobs(data);
|
||||
} catch (err) {
|
||||
console.error("Failed to fetch jobs:", err);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
fetchJobs();
|
||||
const interval = setInterval(fetchJobs, 5000);
|
||||
return () => clearInterval(interval);
|
||||
}, []);
|
||||
|
||||
const getStatusIcon = (status: string) => {
|
||||
switch(status) {
|
||||
case 'COMPLETED': return <CheckCircle2 className="w-4 h-4 text-emerald-400" />;
|
||||
case 'FAILED': return <XCircle className="w-4 h-4 text-red-400" />;
|
||||
case 'PROCESSING': return <RefreshCw className="w-4 h-4 text-blue-400 animate-spin" />;
|
||||
default: return <Clock className="w-4 h-4 text-slate-400" />;
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="bg-slate-800/50 rounded-xl border border-slate-700 p-4 h-full flex flex-col">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<h2 className="text-lg font-semibold text-slate-200">Upload History</h2>
|
||||
{selectedJobId && (
|
||||
<button
|
||||
onClick={() => onSelectJob(null)}
|
||||
className="text-xs text-indigo-400 hover:text-indigo-300"
|
||||
>
|
||||
Show All
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex-1 overflow-y-auto pr-2 space-y-2 custom-scrollbar">
|
||||
{loading && jobs.length === 0 ? (
|
||||
<div className="text-center text-slate-500 py-4 text-sm">Loading...</div>
|
||||
) : jobs.length === 0 ? (
|
||||
<div className="text-center text-slate-500 py-4 text-sm">No uploads yet.</div>
|
||||
) : (
|
||||
jobs.map(job => (
|
||||
<button
|
||||
key={job.id}
|
||||
onClick={() => job.status === 'COMPLETED' ? onSelectJob(job.id) : null}
|
||||
className={`w-full text-left p-3 rounded-lg border transition-colors flex items-start gap-3
|
||||
${selectedJobId === job.id
|
||||
? 'bg-indigo-500/10 border-indigo-500/50 ring-1 ring-indigo-500/20'
|
||||
: 'bg-slate-800/80 border-slate-700/50 hover:bg-slate-700/50'
|
||||
}
|
||||
${job.status !== 'COMPLETED' ? 'cursor-default opacity-80' : 'cursor-pointer hover:border-slate-600'}
|
||||
`}
|
||||
>
|
||||
<div className="mt-0.5">{getStatusIcon(job.status)}</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="text-sm font-medium text-slate-200 truncate" title={job.filename}>
|
||||
{job.filename}
|
||||
</div>
|
||||
<div className="text-xs text-slate-400 mt-1 flex items-center justify-between">
|
||||
<span>{job.newspaper} • {job.date}</span>
|
||||
<span className="text-[10px] uppercase tracking-wider font-semibold opacity-70">
|
||||
{job.status}
|
||||
</span>
|
||||
</div>
|
||||
{job.error_message && (
|
||||
<div className="text-xs text-red-400 mt-1 truncate">
|
||||
{job.error_message}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</button>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
export interface Article {
|
||||
id: number;
|
||||
newspaper: string;
|
||||
edition: string;
|
||||
page: number;
|
||||
date: string;
|
||||
headline: string;
|
||||
subheadline: string;
|
||||
category: string;
|
||||
subcategory: string;
|
||||
language: string;
|
||||
author: string;
|
||||
text: string;
|
||||
summary: string;
|
||||
keywords: string[];
|
||||
persons: string[];
|
||||
locations: string[];
|
||||
organizations: string[];
|
||||
sentiment: string;
|
||||
image_path: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
export interface UploadJob {
|
||||
id: number;
|
||||
filename: string;
|
||||
newspaper: string;
|
||||
edition: string;
|
||||
date: string;
|
||||
status: string;
|
||||
error_message?: string;
|
||||
created_time: string;
|
||||
}
|
||||
|
||||
export async function searchArticles(query: string, isSemantic: boolean, jobId?: number): Promise<Article[]> {
|
||||
const url = new URL('/api/search', window.location.origin);
|
||||
if (query) url.searchParams.append('q', query);
|
||||
if (jobId) url.searchParams.append('job_id', jobId.toString());
|
||||
if (isSemantic) url.searchParams.append('semantic', 'true');
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
// Prevent Next.js from caching dynamic search results
|
||||
cache: 'no-store'
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch articles');
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export async function uploadPDF(file: File, newspaper: string, edition: string, date: string) {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
formData.append('newspaper', newspaper);
|
||||
formData.append('edition', edition);
|
||||
formData.append('date', date);
|
||||
|
||||
const response = await fetch('/api/upload', {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to upload PDF');
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export async function getJobs(): Promise<UploadJob[]> {
|
||||
const response = await fetch('/api/jobs', {
|
||||
method: 'GET',
|
||||
cache: 'no-store'
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch jobs');
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
import { type ClassValue, clsx } from "clsx"
|
||||
import { twMerge } from "tailwind-merge"
|
||||
|
||||
export function cn(...inputs: ClassValue[]) {
|
||||
return twMerge(clsx(inputs))
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2017",
|
||||
"lib": ["dom", "dom.iterable", "esnext"],
|
||||
"allowJs": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"module": "esnext",
|
||||
"moduleResolution": "bundler",
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"jsx": "react-jsx",
|
||||
"incremental": true,
|
||||
"plugins": [
|
||||
{
|
||||
"name": "next"
|
||||
}
|
||||
],
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": [
|
||||
"next-env.d.ts",
|
||||
"**/*.ts",
|
||||
"**/*.tsx",
|
||||
".next/types/**/*.ts",
|
||||
".next/dev/types/**/*.ts",
|
||||
"**/*.mts"
|
||||
],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
Loading…
Reference in New Issue