diff --git a/newspaper-extractor/backend/app/api/routes_search.py b/newspaper-extractor/backend/app/api/routes_search.py index 804b26f..08bc344 100644 --- a/newspaper-extractor/backend/app/api/routes_search.py +++ b/newspaper-extractor/backend/app/api/routes_search.py @@ -1,4 +1,4 @@ -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException, Body from sqlalchemy import or_, text from app.db.session import get_db from app.pipeline.enricher import Enricher @@ -27,3 +27,22 @@ def search(q: str = "", category: str = None, person: str = None, SELECT * FROM articles ORDER BY embedding <=> :v LIMIT 30 """), {"v": str(vec)}).fetchall() return query.limit(50).all() + +@router.put("/articles/{article_id}") +def update_article(article_id: int, updates: dict = Body(...), db=Depends(get_db)): + from app.db.models import Article + article = db.query(Article).filter(Article.id == article_id).first() + if not article: + raise HTTPException(status_code=404, detail="Article not found") + + # Update allowed fields + allowed = ["headline", "category", "subcategory", "summary", "sentiment", "keywords", "persons", "locations", "organizations"] + for k, v in updates.items(): + if k in allowed: + setattr(article, k, v) + + # Mark as verified for RAG training + article.human_verified = True + db.commit() + db.refresh(article) + return article diff --git a/newspaper-extractor/backend/app/db/models.py b/newspaper-extractor/backend/app/db/models.py index 45d267d..fbb5905 100644 --- a/newspaper-extractor/backend/app/db/models.py +++ b/newspaper-extractor/backend/app/db/models.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, Integer, String, Text, Float, ForeignKey, DateTime, JSON +from sqlalchemy import Column, Integer, String, Text, Float, ForeignKey, DateTime, JSON, Boolean from sqlalchemy.dialects.postgresql import ARRAY from sqlalchemy.orm import relationship, declarative_base from pgvector.sqlalchemy import Vector @@ -45,6 +45,7 @@ class Article(Base): confidence = Column(Float) embedding = Column(Vector(384)) # pgvector for semantic search content_hash = Column(String, index=True) # for dedup + human_verified = Column(Boolean, default=False, index=True) created_time = Column(DateTime, default=datetime.datetime.utcnow) images = relationship("ArticleImage", back_populates="article") upload_job = relationship("UploadJob", back_populates="articles") diff --git a/newspaper-extractor/backend/app/pipeline/categorizer.py b/newspaper-extractor/backend/app/pipeline/categorizer.py index 3f96365..b0565a8 100644 --- a/newspaper-extractor/backend/app/pipeline/categorizer.py +++ b/newspaper-extractor/backend/app/pipeline/categorizer.py @@ -17,6 +17,7 @@ PROMPT = """You are a news classifier. Given the headline and text, return stric "summary": 2-sentence summary, "keywords": [up to 8], "persons": [], "locations": [], "organizations": [], "dates": [], "sentiment": "pos|neu|neg"}} +{few_shot_context} HEADLINE: {headline} TEXT: {text} """ @@ -29,8 +30,15 @@ class Categorizer: genai.configure(api_key=settings.LLM_API_KEY) self.model = genai.GenerativeModel("gemini-1.5-flash") - def classify(self, headline, text): - prompt = PROMPT.format(cats=CATEGORIES, headline=headline, text=text[:4000]) + def classify(self, headline, text, few_shot_examples=None): + few_shot_context = "" + if few_shot_examples: + few_shot_context = "Here are some past examples of how similar articles were classified by a human:\n\n" + for i, ex in enumerate(few_shot_examples): + few_shot_context += f"EXAMPLE {i+1}:\nHEADLINE: {ex['headline']}\nTEXT: {ex['text'][:500]}...\nCLASSIFICATION (JSON): {json.dumps(ex['classification'])}\n\n" + few_shot_context += "Now, classify the following new article based on these patterns:\n\n" + + prompt = PROMPT.format(cats=CATEGORIES, few_shot_context=few_shot_context, headline=headline, text=text[:4000]) try: if self.provider == "gemini": resp = self.model.generate_content(prompt) diff --git a/newspaper-extractor/backend/app/pipeline/orchestrator.py b/newspaper-extractor/backend/app/pipeline/orchestrator.py index bcd7ca3..800a9f9 100644 --- a/newspaper-extractor/backend/app/pipeline/orchestrator.py +++ b/newspaper-extractor/backend/app/pipeline/orchestrator.py @@ -31,15 +31,54 @@ class Pipeline: headline = detect_headline(art, page, self.ocr) img_path = crop_article(page["image"], art["bbox"], job_dir) fig_paths = crop_figures(page["image"], art, job_dir) - meta = self.cat.classify(headline, text) - embedding = self.enricher.embed(headline + " " + text[:1000]) + + # Semantic Embedding for RAG + raw_text_for_embed = headline + " " + text[:1000] + embedding = self.enricher.embed(raw_text_for_embed) + + # Fetch few-shot examples from DB + few_shot_examples = [] + try: + from app.db.session import SessionLocal + from sqlalchemy import text as sqla_text + db = SessionLocal() + # Query for up to 3 verified examples using cosine distance + rows = db.execute(sqla_text(""" + SELECT headline, text, category, subcategory, summary, sentiment, keywords, persons, locations, organizations + FROM articles + WHERE human_verified = TRUE + ORDER BY embedding <=> :v LIMIT 3 + """), {"v": str(embedding)}).fetchall() + for r in rows: + ex = { + "headline": r.headline, + "text": r.text, + "classification": { + "category": r.category, + "subcategory": r.subcategory, + "summary": r.summary, + "sentiment": r.sentiment, + "keywords": r.keywords, + "persons": r.persons, + "locations": r.locations, + "organizations": r.organizations + } + } + few_shot_examples.append(ex) + except Exception as e: + print(f"RAG fetch failed: {e}") + finally: + if 'db' in locals(): db.close() + + meta = self.cat.classify(headline, text, few_shot_examples=few_shot_examples) + results.append({ "date": date, "newspaper": newspaper, "edition": edition, "page": page["page_number"], "headline": headline, - "category": meta["category"], "subcategory": meta.get("subcategory"), + "category": meta.get("category", "Others"), "subcategory": meta.get("subcategory"), "text": text, "summary": meta.get("summary"), - "keywords": meta.get("keywords"), "persons": meta.get("persons"), - "locations": meta.get("locations"), "organizations": meta.get("organizations"), + "keywords": meta.get("keywords", []), "persons": meta.get("persons", []), + "locations": meta.get("locations", []), "organizations": meta.get("organizations", []), "sentiment": meta.get("sentiment"), "language": "en", "image_path": img_path, "figure_paths": fig_paths, "bounding_box": art["bbox"], diff --git a/newspaper-extractor/frontend/src/components/ArticleCard.tsx b/newspaper-extractor/frontend/src/components/ArticleCard.tsx index beb883b..e23d08f 100644 --- a/newspaper-extractor/frontend/src/components/ArticleCard.tsx +++ b/newspaper-extractor/frontend/src/components/ArticleCard.tsx @@ -1,28 +1,51 @@ 'use client'; -import { Article } from '@/lib/api'; -import { Calendar, Newspaper, Hash, Smile, Frown, Meh, Image as ImageIcon } from 'lucide-react'; +import { useState } from 'react'; +import { Article, updateArticle } from '@/lib/api'; +import { Calendar, Newspaper, Hash, Smile, Frown, Meh, Image as ImageIcon, Edit2, Save, X } from 'lucide-react'; import { cn } from '@/lib/utils'; interface ArticleCardProps { article: Article; } -export function ArticleCard({ article }: ArticleCardProps) { +export function ArticleCard({ article: initialArticle }: ArticleCardProps) { + const [article, setArticle] = useState(initialArticle); + const [isEditing, setIsEditing] = useState(false); + const [editData, setEditData] = useState({ category: article.category, summary: article.summary }); + const [isSaving, setIsSaving] = useState(false); + // Determine sentiment badge const sentimentScore = parseFloat(article.sentiment || "0"); const isPositive = sentimentScore > 0.3; const isNegative = sentimentScore < -0.3; - // Create an image url if image path exists - // The path is internal to docker like /data/storage/cropped_XYZ.jpg - // Assuming the backend mounts /data/storage at /api/images const imageUrl = article.image_path ? `/api/images/${article.image_path.split('/').pop()}` : null; + const handleSave = async () => { + setIsSaving(true); + try { + const updated = await updateArticle(article.id, { + category: editData.category, + summary: editData.summary + }); + setArticle(updated); + setIsEditing(false); + } catch (e) { + console.error(e); + alert("Failed to save changes"); + } finally { + setIsSaving(false); + } + }; + return ( -
+
{/* Header info */}
@@ -36,19 +59,48 @@ export function ArticleCard({ article }: ArticleCardProps) { - {article.category} {article.subcategory ? `/ ${article.subcategory}` : ''} + {isEditing ? ( + setEditData({...editData, category: e.target.value})} + className="bg-zinc-950 border border-zinc-700 px-1 py-0.5 rounded text-zinc-200" + /> + ) : ( + <>{article.category} {article.subcategory ? `/ ${article.subcategory}` : ''} + )} + {(article as any).human_verified && ( + ✓ Verified + )}
- {/* Sentiment Badge */} -
- {isPositive ? : isNegative ? : } - {isPositive ? "Positive" : isNegative ? "Negative" : "Neutral"} + {/* Actions & Sentiment Badge */} +
+ {isEditing ? ( +
+ + +
+ ) : ( + + )} + +
+ {isPositive ? : isNegative ? : } + {isPositive ? "Positive" : isNegative ? "Negative" : "Neutral"} +
@@ -64,9 +116,17 @@ export function ArticleCard({ article }: ArticleCardProps) { )} -

- {article.summary || article.text?.slice(0, 250) + "..."} -

+ {isEditing ? ( +