fix: UI polling bug and paddleocr dependency version

app.py

@@ -182,15 +182,25 @@ def job_view(job_id):
 
 @app.route("/job/<job_id>/status")
 def job_status(job_id):
+    status = {"status": "unknown"}
     with JOB_LOCK:
-        status = JOBS.get(job_id, {"status": "unknown"})
+        if job_id in JOBS:
+            status.update(JOBS[job_id])
+            
     meta_path = OUTPUT_DIR / job_id / "meta.json"
     if meta_path.exists():
         try:
             meta = json.loads(meta_path.read_text())
+            if "status" in meta:
+                status["status"] = meta["status"]
+            if "error" in meta and not status.get("message"):
+                status["message"] = meta["error"]
+            elif "message" in meta:
+                status["message"] = meta["message"]
             status["meta"] = meta
         except Exception:
             pass
+            
     return jsonify(status)
 
 

extractor.py

@@ -581,40 +581,32 @@ I have run OCR on the cropped articles of page {page_num} of a Telugu newspaper.
 
 For EACH article, ask yourself these TWO questions:
 
-QUESTION 1: "Does this article describe actual DAMAGE, LOSS, DEATH, CORRUPTION, or SUFFERING happening to people in Telangana? And can the government (state OR central) be BLAMED for it — either for causing it, failing to prevent it, or failing to respond to it?"
+QUESTION 1: "Is this article POLITICALLY SIGNIFICANT for an opposition party in Telangana?"
-- The article must describe a NEGATIVE EVENT that has already occurred — not a future plan, not an announcement, not a statistic.
+- Political significance includes: MLA/MLC/MP statements, press conferences, padayatras, political protests, and party activities.
-- There must be a VICTIM — someone who died, lost money, was cheated, is suffering, or was harmed.
+- It ALSO includes government scheme announcements tied to politicians, or Collector administrative directives.
-- The government (Telangana state OR central/BJP) can be blamed — for causing it, failing to prevent it, or failing to respond to it.
+- It ALSO includes government failures, corruption, price hikes, farmer distress, infrastructure failures, and public grievances.
-- Central government failures also qualify IF the article describes actual harm to Telangana citizens (e.g., NEET paper leak harming Telangana students, central policy causing job losses in Telangana).
+- If it is just sports, cinema, horoscopes, advertisements, or purely neutral non-political news -> answer is NO.
-- If the article describes something POSITIVE (new scheme, development, achievement) -> answer is NO.
-- If the article describes a PLAN, PETITION, or ANNOUNCEMENT (e.g. MLA/civilians submitting a representation requesting funds, work commencement, inspections, or temple matters) -> answer is NO. It must describe an active issue/damage event, not requests for future work.
-- If there is NO victim and NO damage described -> answer is NO.
-- RESPONSIBILITY CHECK: Ask "WHO caused this damage?" If the damage was caused by nature (fire, flood, lightning), individual negligence (reckless driving, personal accident), criminals (theft, murder), or private parties — and the government had NO role in causing or preventing it — answer is NO. Only answer YES if the government is CLEARLY responsible through its policy, negligence, corruption, or failure to act.
 - If YES -> proceed to Question 2
 - If NO -> REJECT this article immediately
 
-QUESTION 2: "Can the opposition party DIRECTLY USE this article to attack the Telangana government in a press conference, assembly session, or public rally?"
+QUESTION 2: "Can the opposition party USE this article in any way (e.g. to attack the government, to track ruling party activities, or to monitor scheme implementations)?"
-- If YES -> SELECT this article
+- If the article is politically relevant and useful for an opposition war room to track -> answer is YES.
-- If NO -> REJECT this article
+- If it is completely irrelevant to state politics -> answer is NO.
 
 BOTH answers must be YES to select. If either answer is NO, reject.
 
-DEDUPLICATION: If two or more articles cover the SAME issue or event (e.g., same farmer protest, same accident, same scam), only SELECT the ONE with the most complete coverage or the larger article. REJECT the duplicates. Do not keep two articles about the same news story.
+DEDUPLICATION: If two or more articles cover the SAME issue or event, only SELECT the ONE with the most complete coverage.
 
 NOW go through EACH article ID I gave you. Ask Q1 and Q2 for each. Only select if BOTH are YES.
 
-IMPORTANT: This application is for TELANGANA STATE politics only. Farmer issues are the MOST IMPORTANT — always select if farmers are suffering.
+IMPORTANT: This application is for TELANGANA STATE politics.
 
 DO NOT SELECT — automatically REJECT:
 - International news (wars, foreign affairs, immigration)
-- Government achievements, new schemes, welfare announcements, development projects
 - Newspaper masthead, headers, advertisements, classifieds
 - Sports, entertainment, cinema, horoscopes
-- General statistics without actual damage (birth rate, population, GDP)
+- General statistics without political context
-- News about other states
+- News about other states completely unrelated to Telangana
-- Resignations — ALWAYS REJECT unless the article explicitly contains words like corruption, scam, fired, forced out, or political pressure. A person resigning for personal, health, family, or workload reasons is NOT a government failure.
-- PM Modi speeches or party meetings (unless specific policy failure hurting Telangana people)
-- Individual accidents (road accidents, electric shocks, drownings, fire accidents) — these are NOT government failures, they can happen due to personal negligence or bad luck. Only select if there is a MASS disaster (10+ deaths) or CLEAR systemic government negligence (e.g., bridge collapse, building collapse, factory fire due to no safety inspections).
 
 Return a JSON object with TWO lists:
 

geometric_grouper.py

@@ -7,11 +7,15 @@ def get_horizontal_overlap(bbox1, bbox2):
     x_right = min(bbox1[2], bbox2[2])
     return max(0, x_right - x_left)
 
-def extract_barriers(img_path):
+def extract_barriers(img_path, regions):
     """Extracts horizontal and vertical barrier lines using OpenCV."""
     if not img_path:
         return [], []
         
+    import statistics
+    text_widths = [(r['bbox'][2] - r['bbox'][0]) for r in regions if r['type'] == 'text']
+    median_col_width = statistics.median(text_widths) if text_widths else 300
+    
     img = cv2.imread(str(img_path))
     if img is None:
         return [], []
@@ -19,7 +23,7 @@ def extract_barriers(img_path):
     gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
     _, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)
     
-    line_length = 300
+    line_length = int(median_col_width * 0.5)
     h_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (line_length, 1))
     h_lines_raw = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, h_kernel, iterations=2)
     
@@ -32,7 +36,8 @@ def extract_barriers(img_path):
     h_lines = []
     for cnt in h_contours_raw:
         x, y, w, h = cv2.boundingRect(cnt)
-        if h <= 15:  # Thin lines only
+        # Ignore short decorative lines (caption underlines) which are <= 0.6 col
+        if h <= 15 and w >= (median_col_width * 0.6):
             h_lines.append((x, y, x + w, y + h))
             
     v_lines = []
@@ -89,7 +94,7 @@ def group_surya_regions_geometrically(regions, img_path=None):
         return []
         
     # Extract barriers using OpenCV
-    h_lines, v_lines = extract_barriers(img_path)
+    h_lines, v_lines = extract_barriers(img_path, regions)
     if img_path:
         print(f"  [grouper] Extracted {len(h_lines)} horizontal and {len(v_lines)} vertical OpenCV barriers")
 
@@ -255,7 +260,58 @@ def group_surya_regions_geometrically(regions, img_path=None):
     if current_orphan:
         merged_orphans.append(current_orphan)
         
-    final_articles = valid_articles + merged_orphans
+    # Orphan Adoption: If an orphan sits directly below a valid article with high overlap and small gap, adopt it (ignoring photo borders)
+    adopted_orphans = set()
+    for o in merged_orphans:
+        best_parent = None
+        min_gap = float('inf')
+        for a in valid_articles:
+            overlap = get_horizontal_overlap(a['bbox'], o['bbox'])
+            gap = o['bbox'][1] - a['bbox'][3]
+            if overlap > 0 and 0 <= gap < 200:
+                width_overlap_ratio = overlap / min((a['bbox'][2] - a['bbox'][0]), (o['bbox'][2] - o['bbox'][0]))
+                if width_overlap_ratio > 0.8:
+                    if gap < min_gap:
+                        min_gap = gap
+                        best_parent = a
+        if best_parent:
+            best_parent['member_regions'].extend(o['member_regions'])
+            best_parent['bbox'][0] = min(best_parent['bbox'][0], o['bbox'][0])
+            best_parent['bbox'][1] = min(best_parent['bbox'][1], o['bbox'][1])
+            best_parent['bbox'][2] = max(best_parent['bbox'][2], o['bbox'][2])
+            best_parent['bbox'][3] = max(best_parent['bbox'][3], o['bbox'][3])
+            adopted_orphans.add(o['article_id'])
+            
+    final_orphans_pass_1 = [o for o in merged_orphans if o['article_id'] not in adopted_orphans]
+    
+    # Strict Orphan Photo Binding: An image may never stand alone.
+    remaining_orphans = []
+    for o in final_orphans_pass_1:
+        has_image = any(region_by_id[rid]['type'] == 'image' for rid in o['member_regions'] if rid in region_by_id)
+        has_title = any(region_by_id[rid]['type'] in ('doc_title', 'paragraph_title') for rid in o['member_regions'] if rid in region_by_id)
+        
+        if has_image and not has_title:
+            best_article = None
+            min_dist = float('inf')
+            
+            for a in valid_articles:
+                dist = min(abs(a['bbox'][3] - o['bbox'][1]), abs(o['bbox'][3] - a['bbox'][1]))
+                overlap = get_horizontal_overlap(a['bbox'], o['bbox'])
+                if overlap > 0 and dist < min_dist:
+                    min_dist = dist
+                    best_article = a
+            
+            if best_article:
+                best_article['member_regions'].extend(o['member_regions'])
+                best_article['bbox'][0] = min(best_article['bbox'][0], o['bbox'][0])
+                best_article['bbox'][1] = min(best_article['bbox'][1], o['bbox'][1])
+                best_article['bbox'][2] = max(best_article['bbox'][2], o['bbox'][2])
+                best_article['bbox'][3] = max(best_article['bbox'][3], o['bbox'][3])
+                continue # successfully bound
+                
+        remaining_orphans.append(o)
+        
+    final_articles = valid_articles + remaining_orphans
 
     MAX_SPLIT_HEIGHT  = 2500
     MIN_MEMBERS_SPLIT = 5

political_filter.py

@@ -48,6 +48,10 @@ def scan_pages_for_political_articles(client, page_images, model="claude-sonnet-
 Scan ALL pages of this Telugu newspaper carefully. Identify EVERY article that is politically significant — specifically issues that an opposition party can use for political mileage.
 
 INCLUDE articles about:
+- MLA/MLC/MP statements, press conferences, and speeches
+- Political party activities, padayatras, and protests
+- Government scheme announcements tied to politicians
+- Collector administrative directives or warnings
 - Government failures, delays, broken promises
 - Corruption, scams, misuse of funds by ruling party leaders or officials
 - Price hikes, inflation affecting common people
@@ -65,7 +69,6 @@ IGNORE articles about:
 - Horoscopes, puzzles, classifieds
 - International news with no local political angle
 - Advertisements
-- Purely factual/neutral government announcements with no controversy
 
 For each politically significant article, provide:
 1. page_number: which page it's on

requirements.txt

@@ -2,7 +2,7 @@ Flask>=3.0
 PyMuPDF>=1.23
 Pillow>=10.0
 numpy>=1.24
-paddleocr==3.5.0
+paddleocr==2.8.1
 paddlepaddle==3.0.0
 pytesseract>=0.3.10
 anthropic>=0.40

scratch_test_grouper.py

@@ -2,8 +2,8 @@ import json
 from pathlib import Path
 import geometric_grouper
 
-regions_file = Path("output/20260614_124012/pages/page_004.regions.json")
+regions_file = Path(r"output\20260614_225443\pages\page_004.regions.json")
-img_file = Path("output/20260614_124012/pages/page_004.png")
+img_file = Path(r"output\20260614_225443\pages\page_004.png")
 
 with open(regions_file, encoding='utf-8') as f:
     regions = json.load(f)["regions"]
@@ -14,4 +14,9 @@ articles = geometric_grouper.group_surya_regions_geometrically(valid_regions, im
 
 print(f"Total articles identified: {len(articles)}")
 for a in articles:
-    print(f"  Article {a['article_id']}: {len(a['member_regions'])} regions, bbox {a['bbox']}, grouped_by={a['grouped_by']}")
+    print(f"  Article {a['article_id']}: {len(a['member_regions'])} regions, bbox {a['bbox']}")
+    for rid in a['member_regions']:
+        r = next((x for x in regions if x['id'] == rid), None)
+        if r:
+            print(f"    Region {rid}: type={r['type']:15s} bbox={r['bbox']}")
+    print()