feat: Add continuous learning self-correction pipeline

newspaper-extractor/backend/app/api/routes_search.py

@@ -1,4 +1,4 @@
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, HTTPException, Body
 from sqlalchemy import or_, text
 from app.db.session import get_db
 from app.pipeline.enricher import Enricher
@@ -27,3 +27,22 @@ def search(q: str = "", category: str = None, person: str = None,
             SELECT * FROM articles ORDER BY embedding <=> :v LIMIT 30
         """), {"v": str(vec)}).fetchall()
     return query.limit(50).all()
+
+@router.put("/articles/{article_id}")
+def update_article(article_id: int, updates: dict = Body(...), db=Depends(get_db)):
+    from app.db.models import Article
+    article = db.query(Article).filter(Article.id == article_id).first()
+    if not article:
+        raise HTTPException(status_code=404, detail="Article not found")
+    
+    # Update allowed fields
+    allowed = ["headline", "category", "subcategory", "summary", "sentiment", "keywords", "persons", "locations", "organizations"]
+    for k, v in updates.items():
+        if k in allowed:
+            setattr(article, k, v)
+    
+    # Mark as verified for RAG training
+    article.human_verified = True
+    db.commit()
+    db.refresh(article)
+    return article

newspaper-extractor/backend/app/db/models.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, String, Text, Float, ForeignKey, DateTime, JSON
+from sqlalchemy import Column, Integer, String, Text, Float, ForeignKey, DateTime, JSON, Boolean
 from sqlalchemy.dialects.postgresql import ARRAY
 from sqlalchemy.orm import relationship, declarative_base
 from pgvector.sqlalchemy import Vector
@@ -45,6 +45,7 @@ class Article(Base):
     confidence = Column(Float)
     embedding = Column(Vector(384))          # pgvector for semantic search
     content_hash = Column(String, index=True) # for dedup
+    human_verified = Column(Boolean, default=False, index=True)
     created_time = Column(DateTime, default=datetime.datetime.utcnow)
     images = relationship("ArticleImage", back_populates="article")
     upload_job = relationship("UploadJob", back_populates="articles")

newspaper-extractor/backend/app/pipeline/categorizer.py

@@ -17,6 +17,7 @@ PROMPT = """You are a news classifier. Given the headline and text, return stric
 "summary": 2-sentence summary, "keywords": [up to 8],
 "persons": [], "locations": [], "organizations": [], "dates": [], "sentiment": "pos|neu|neg"}}
 
+{few_shot_context}
 HEADLINE: {headline}
 TEXT: {text}
 """
@@ -29,8 +30,15 @@ class Categorizer:
             genai.configure(api_key=settings.LLM_API_KEY)
             self.model = genai.GenerativeModel("gemini-1.5-flash")
 
-    def classify(self, headline, text):
-        prompt = PROMPT.format(cats=CATEGORIES, headline=headline, text=text[:4000])
+    def classify(self, headline, text, few_shot_examples=None):
+        few_shot_context = ""
+        if few_shot_examples:
+            few_shot_context = "Here are some past examples of how similar articles were classified by a human:\n\n"
+            for i, ex in enumerate(few_shot_examples):
+                few_shot_context += f"EXAMPLE {i+1}:\nHEADLINE: {ex['headline']}\nTEXT: {ex['text'][:500]}...\nCLASSIFICATION (JSON): {json.dumps(ex['classification'])}\n\n"
+            few_shot_context += "Now, classify the following new article based on these patterns:\n\n"
+
+        prompt = PROMPT.format(cats=CATEGORIES, few_shot_context=few_shot_context, headline=headline, text=text[:4000])
         try:
             if self.provider == "gemini":
                 resp = self.model.generate_content(prompt)

newspaper-extractor/backend/app/pipeline/orchestrator.py

@@ -31,15 +31,54 @@ class Pipeline:
                 headline = detect_headline(art, page, self.ocr)
                 img_path = crop_article(page["image"], art["bbox"], job_dir)
                 fig_paths = crop_figures(page["image"], art, job_dir)
-                meta = self.cat.classify(headline, text)
-                embedding = self.enricher.embed(headline + " " + text[:1000])
+                
+                # Semantic Embedding for RAG
+                raw_text_for_embed = headline + " " + text[:1000]
+                embedding = self.enricher.embed(raw_text_for_embed)
+                
+                # Fetch few-shot examples from DB
+                few_shot_examples = []
+                try:
+                    from app.db.session import SessionLocal
+                    from sqlalchemy import text as sqla_text
+                    db = SessionLocal()
+                    # Query for up to 3 verified examples using cosine distance
+                    rows = db.execute(sqla_text("""
+                        SELECT headline, text, category, subcategory, summary, sentiment, keywords, persons, locations, organizations 
+                        FROM articles 
+                        WHERE human_verified = TRUE 
+                        ORDER BY embedding <=> :v LIMIT 3
+                    """), {"v": str(embedding)}).fetchall()
+                    for r in rows:
+                        ex = {
+                            "headline": r.headline,
+                            "text": r.text,
+                            "classification": {
+                                "category": r.category,
+                                "subcategory": r.subcategory,
+                                "summary": r.summary,
+                                "sentiment": r.sentiment,
+                                "keywords": r.keywords,
+                                "persons": r.persons,
+                                "locations": r.locations,
+                                "organizations": r.organizations
+                            }
+                        }
+                        few_shot_examples.append(ex)
+                except Exception as e:
+                    print(f"RAG fetch failed: {e}")
+                finally:
+                    if 'db' in locals(): db.close()
+
+                meta = self.cat.classify(headline, text, few_shot_examples=few_shot_examples)
+                
                 results.append({
                     "date": date, "newspaper": newspaper, "edition": edition,
                     "page": page["page_number"], "headline": headline,
-                    "category": meta["category"], "subcategory": meta.get("subcategory"),
+                    "category": meta.get("category", "Others"), "subcategory": meta.get("subcategory"),
                     "text": text, "summary": meta.get("summary"),
-                    "keywords": meta.get("keywords"), "persons": meta.get("persons"),
-                    "locations": meta.get("locations"), "organizations": meta.get("organizations"),
+                    "keywords": meta.get("keywords", []), "persons": meta.get("persons", []),
+                    "locations": meta.get("locations", []), "organizations": meta.get("organizations", []),
                     "sentiment": meta.get("sentiment"), "language": "en",
                     "image_path": img_path, "figure_paths": fig_paths,
                     "bounding_box": art["bbox"],

newspaper-extractor/frontend/src/components/ArticleCard.tsx

@@ -1,28 +1,51 @@
 'use client';
 
-import { Article } from '@/lib/api';
-import { Calendar, Newspaper, Hash, Smile, Frown, Meh, Image as ImageIcon } from 'lucide-react';
+import { useState } from 'react';
+import { Article, updateArticle } from '@/lib/api';
+import { Calendar, Newspaper, Hash, Smile, Frown, Meh, Image as ImageIcon, Edit2, Save, X } from 'lucide-react';
 import { cn } from '@/lib/utils';
 
 interface ArticleCardProps {
   article: Article;
 }
 
-export function ArticleCard({ article }: ArticleCardProps) {
+export function ArticleCard({ article: initialArticle }: ArticleCardProps) {
+  const [article, setArticle] = useState(initialArticle);
+  const [isEditing, setIsEditing] = useState(false);
+  const [editData, setEditData] = useState({ category: article.category, summary: article.summary });
+  const [isSaving, setIsSaving] = useState(false);
+
   // Determine sentiment badge
   const sentimentScore = parseFloat(article.sentiment || "0");
   const isPositive = sentimentScore > 0.3;
   const isNegative = sentimentScore < -0.3;
   
-  // Create an image url if image path exists
-  // The path is internal to docker like /data/storage/cropped_XYZ.jpg
-  // Assuming the backend mounts /data/storage at /api/images
   const imageUrl = article.image_path 
     ? `/api/images/${article.image_path.split('/').pop()}` 
     : null;
 
+  const handleSave = async () => {
+    setIsSaving(true);
+    try {
+      const updated = await updateArticle(article.id, {
+        category: editData.category,
+        summary: editData.summary
+      });
+      setArticle(updated);
+      setIsEditing(false);
+    } catch (e) {
+      console.error(e);
+      alert("Failed to save changes");
+    } finally {
+      setIsSaving(false);
+    }
+  };
+
   return (
-    <div className="group relative flex flex-col gap-4 rounded-2xl border border-zinc-800 bg-zinc-900/50 p-6 transition-all hover:bg-zinc-900 hover:shadow-xl hover:shadow-indigo-500/5 hover:-translate-y-1">
+    <div className={cn(
+      "group relative flex flex-col gap-4 rounded-2xl border bg-zinc-900/50 p-6 transition-all hover:bg-zinc-900 hover:shadow-xl hover:shadow-indigo-500/5 hover:-translate-y-1",
+      (article as any).human_verified ? "border-indigo-500/50" : "border-zinc-800"
+    )}>
       {/* Header info */}
       <div className="flex items-start justify-between">
         <div className="flex flex-wrap items-center gap-3 text-xs font-medium text-zinc-400">
@@ -36,19 +59,48 @@ export function ArticleCard({ article }: ArticleCardProps) {
           </span>
           <span className="flex items-center gap-1.5 rounded-full bg-zinc-800/50 px-2.5 py-1">
             <Hash size={14} className="text-zinc-500" />
-            {article.category} {article.subcategory ? `/ ${article.subcategory}` : ''}
+            {isEditing ? (
+              <input 
+                type="text" 
+                value={editData.category}
+                onChange={e => setEditData({...editData, category: e.target.value})}
+                className="bg-zinc-950 border border-zinc-700 px-1 py-0.5 rounded text-zinc-200"
+              />
+            ) : (
+              <>{article.category} {article.subcategory ? `/ ${article.subcategory}` : ''}</>
+            )}
           </span>
+          {(article as any).human_verified && (
+            <span className="text-indigo-400 font-bold ml-2">✓ Verified</span>
+          )}
         </div>
         
-        {/* Sentiment Badge */}
-        <div className={cn(
-          "flex items-center gap-1.5 rounded-full px-2.5 py-1 text-xs font-semibold",
-          isPositive ? "bg-emerald-500/10 text-emerald-400" : 
-          isNegative ? "bg-rose-500/10 text-rose-400" : 
-          "bg-blue-500/10 text-blue-400"
-        )}>
-          {isPositive ? <Smile size={14} /> : isNegative ? <Frown size={14} /> : <Meh size={14} />}
-          {isPositive ? "Positive" : isNegative ? "Negative" : "Neutral"}
+        {/* Actions & Sentiment Badge */}
+        <div className="flex items-center gap-3">
+          {isEditing ? (
+            <div className="flex items-center gap-2">
+              <button onClick={handleSave} disabled={isSaving} className="flex items-center gap-1 text-xs text-emerald-400 bg-emerald-400/10 hover:bg-emerald-400/20 px-2 py-1 rounded">
+                <Save size={14} /> {isSaving ? "Saving..." : "Save"}
+              </button>
+              <button onClick={() => { setIsEditing(false); setEditData({ category: article.category, summary: article.summary }); }} className="flex items-center gap-1 text-xs text-rose-400 bg-rose-400/10 hover:bg-rose-400/20 px-2 py-1 rounded">
+                <X size={14} /> Cancel
+              </button>
+            </div>
+          ) : (
+            <button onClick={() => setIsEditing(true)} className="flex items-center gap-1 text-xs text-zinc-400 hover:text-indigo-400 transition-colors bg-zinc-800/50 hover:bg-zinc-800 px-2 py-1 rounded opacity-0 group-hover:opacity-100">
+              <Edit2 size={14} /> Edit
+            </button>
+          )}
+
+          <div className={cn(
+            "flex items-center gap-1.5 rounded-full px-2.5 py-1 text-xs font-semibold",
+            isPositive ? "bg-emerald-500/10 text-emerald-400" : 
+            isNegative ? "bg-rose-500/10 text-rose-400" : 
+            "bg-blue-500/10 text-blue-400"
+          )}>
+            {isPositive ? <Smile size={14} /> : isNegative ? <Frown size={14} /> : <Meh size={14} />}
+            {isPositive ? "Positive" : isNegative ? "Negative" : "Neutral"}
+          </div>
         </div>
       </div>
 
@@ -64,9 +116,17 @@ export function ArticleCard({ article }: ArticleCardProps) {
             </h4>
           )}
           
-          <p className="text-sm leading-relaxed text-zinc-300">
-            {article.summary || article.text?.slice(0, 250) + "..."}
-          </p>
+          {isEditing ? (
+            <textarea 
+              value={editData.summary}
+              onChange={e => setEditData({...editData, summary: e.target.value})}
+              className="w-full bg-zinc-950 border border-zinc-700 p-2 rounded-md text-sm text-zinc-200 min-h-[100px]"
+            />
+          ) : (
+            <p className="text-sm leading-relaxed text-zinc-300">
+              {article.summary || article.text?.slice(0, 250) + "..."}
+            </p>
+          )}
         </div>
 
         {/* Optional Image */}
@@ -77,7 +137,6 @@ export function ArticleCard({ article }: ArticleCardProps) {
               alt="Article snippet" 
               className="h-32 w-48 object-cover rounded-xl border border-zinc-800 bg-zinc-950"
               onError={(e) => {
-                // fallback if image fails to load
                 e.currentTarget.style.display = 'none';
               }}
             />

newspaper-extractor/frontend/src/lib/api.ts

@@ -83,3 +83,17 @@ export async function getJobs(): Promise<UploadJob[]> {
   }
   return response.json();
 }
+
+export async function updateArticle(id: number, updates: Partial<Article>): Promise<Article> {
+  const response = await fetch(`/api/articles/${id}`, {
+    method: 'PUT',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(updates),
+  });
+  if (!response.ok) {
+    throw new Error('Failed to update article');
+  }
+  return response.json();
+}