diff --git a/crash_log.txt b/crash_log.txt new file mode 100644 index 0000000..12f61ab Binary files /dev/null and b/crash_log.txt differ diff --git a/extractor.py b/extractor.py index 09de8b4..d3147a7 100644 --- a/extractor.py +++ b/extractor.py @@ -43,6 +43,11 @@ from PIL import Image JOBS = {} JOB_LOCK = threading.Lock() +# Minimum article bounding-box thresholds — filters out layout artifacts +# (tiny title fragments, page decorations) before they waste OCR compute. +MIN_ARTICLE_AREA = 10000 # pixels² (e.g. 100×100 px minimum) +MIN_ARTICLE_DIM = 60 # pixels either dimension below this → skip + _LAYOUT = None _LAYOUT_LOCK = threading.Lock() @@ -203,7 +208,7 @@ def group_into_articles(page_png_path, regions, page_width, page_img=None): print("[running geometric column grouping]") import geometric_grouper - articles = geometric_grouper.group_surya_regions_geometrically(valid_regions) + articles = geometric_grouper.group_surya_regions_geometrically(valid_regions, page_png_path) return articles, valid_regions @@ -211,6 +216,11 @@ def group_into_articles(page_png_path, regions, page_width, page_img=None): # Stage 4 — crop each article # --------------------------------------------------------------------------- def crop_headlines(page_png_path, articles, regions, out_dir, page_num): + # How many pixels below the title region to include in the headline crop. + # Including a few lines of body text stabilises Telugu OCR output and gives + # Claude's triage enough context to make a reliable keep/reject decision. + HEADLINE_BODY_EXTENSION = 300 + out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) img = Image.open(page_png_path) @@ -223,10 +233,16 @@ def crop_headlines(page_png_path, articles, regions, out_dir, page_num): title_id = art.get("title_region") if title_id and title_id in region_by_id: l, t, r, b = region_by_id[title_id]["bbox"] + # Extend downward into the body to give OCR more context. + # A bare 68px title strip in Telugu is too short for stable OCR; + # including the first few lines of body text below it anchors the + # reading and reduces non-determinism in Claude's triage input. + art_bottom = art["bbox"][3] + b = min(art_bottom, b + HEADLINE_BODY_EXTENSION) else: - # Fallback to the top 15% of the article if no title region + # Fallback: top 20% of the article bbox when no title region exists l, t, r, b = art["bbox"] - b = min(b, int(t + (b - t) * 0.15)) + b = min(b, int(t + (b - t) * 0.20)) # Pad slightly l = max(0, l - 10) @@ -396,13 +412,12 @@ def _ocr_with_paddleocr(img_path): for row in rows: lines.append(" ".join(p[2] for p in row)) else: - # Last resort: try str representation - try: - s = str(det_result) - if len(s) > 10: - return s - except: - pass + # No text found via any path — return None cleanly. + # (Caller will substitute "[No text extracted by OCR]".) + # NOTE: do NOT fall back to str(det_result) here — that + # returns the raw PaddleOCR result dict as a string, + # which poisons Claude's triage with file-path garbage. + pass return "\n".join(lines) if lines else None except Exception as e: @@ -478,9 +493,31 @@ I have extracted the HEADLINES from page {page_num} of a Telugu newspaper. Here {articles_json_str} -Return the IDs of articles that MIGHT describe DAMAGE, LOSS, CORRUPTION, or SUFFERING happening to people in Telangana, where the government could be blamed. -Also return the IDs of any articles about farmer protests, NEET paper leaks, or protests against the ruling party. -If you are unsure, INCLUDE it. We just want to filter out obvious ads, sports, and unrelated filler. +Your job is a BROAD FIRST PASS — keep anything that COULD be politically useful. You are NOT the final judge; a deeper analysis step will follow. + +KEEP articles about ANY of these: +- Government failures, broken promises, delays, inaction +- Price hikes, inflation, fuel costs, essential goods +- Farmer distress, crop losses, water/power issues for farming +- Crime, law & order failures, police negligence +- Infrastructure problems (roads, hospitals, schools in bad shape) +- Corruption, scams, misuse of funds by officials or politicians +- Protests, strikes, public grievances against authorities +- Health emergencies, hospital negligence, shortage of medicines +- Job losses, unemployment, labour disputes +- Education failures (school closures, NEET, exam problems) +- Accidents or disasters with possible government negligence angle +- Any ruling party internal conflict or embarrassment + +DO NOT KEEP (filter these out only if you are certain): +- Pure sports scores or match results +- Pure entertainment / cinema news with zero political angle +- Horoscopes, puzzles, crosswords +- Pure classified advertisements or product promotions +- International news with zero Telangana/India relevance +- Completely blank or unreadable OCR regions + +When in doubt, KEEP IT. It is far better to pass 10 borderline articles to the next stage than to miss one important story. Return ONLY a valid JSON list of IDs. Example: ["p001_a002", "p001_a005"]. No prose.""" @@ -831,7 +868,21 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p print() except Exception as e: print(f" Could not create articles debug image: {e}") - # Step 4: Crop and OCR HEADLINES only + # Step 4: Filter out garbage/tiny article regions before cropping. + # Surya sometimes produces title fragments (e.g. 29×33 px) that aren't + # real articles — they waste OCR compute and pollute triage results. + articles_before = len(articles) + articles = [ + a for a in articles + if (lambda b: (b[2]-b[0]) * (b[3]-b[1]) >= MIN_ARTICLE_AREA and (b[2]-b[0]) >= MIN_ARTICLE_DIM and (b[3]-b[1]) >= MIN_ARTICLE_DIM)(a["bbox"]) + ] + filtered_count = articles_before - len(articles) + if filtered_count: + print(f" Page {n}: filtered out {filtered_count} tiny/garbage articles " + f"(< {MIN_ARTICLE_DIM}px or < {MIN_ARTICLE_AREA}px\u00b2), " + f"{len(articles)} remain for cropping.") + + # Step 4b: Crop and OCR HEADLINES only _set_status(job_id, f"Page {n}: cropping {len(articles)} headlines...") headline_records = crop_headlines(page_img_path, articles, regions, articles_root, n) @@ -842,10 +893,33 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p _set_status(job_id, f"Page {n}: triaging headlines with Claude...") selected_ids = _triage_headlines_with_claude(headline_records, n, job_dir=str(job_dir)) - # Cleanup non-selected directories + # --- TRIAGE LOG: save all decisions for auditing --- + rejected_dir = job_dir / "rejected" + rejected_dir.mkdir(exist_ok=True) + triage_log = { + "page": n, + "total": len(headline_records), + "selected_count": len(selected_ids), + "rejected_count": len(headline_records) - len(selected_ids), + "articles": [] + } + for rec in headline_records: + triage_log["articles"].append({ + "id": rec["name"], + "headline_text": rec.get("headline_text", "")[:200], + "decision": "KEEP" if rec["name"] in selected_ids else "REJECT", + }) + (pages_dir / f"page_{n:03d}.triage_log.json").write_text( + json.dumps(triage_log, indent=2, ensure_ascii=False), encoding="utf-8" + ) + + # Move (not delete) rejected dirs to rejected/ so they can be audited for rec in headline_records: if rec["name"] not in selected_ids: - shutil.rmtree(rec["dir"], ignore_errors=True) + src = Path(rec["dir"]) + dst = rejected_dir / src.name + if src.exists() and not dst.exists(): + shutil.move(str(src), str(dst)) if not selected_ids: print(f" Page {n}: 0 political articles found during triage.") @@ -868,7 +942,7 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p final_ids = {s["id"] for s in selected} print(f" Page {n}: {len(selected)} politically significant articles: {sorted(final_ids)}") - # Keep only selected articles, remove the rest + # Keep selected, move rejected to rejected/ for audit (never permanently delete) for rec in records: art_dir = Path(rec["dir"]) art_name = art_dir.name @@ -889,7 +963,10 @@ def process_pdf(pdf_path, job_dir, target_w, target_h, job_id, max_pages=None, p sel_info["image_file"] = f"{art_name}.png" all_selected.append(sel_info) else: - shutil.rmtree(str(art_dir), ignore_errors=True) + # Move to rejected/ for audit instead of deleting + dst = rejected_dir / art_name + if art_dir.exists() and not dst.exists(): + shutil.move(str(art_dir), str(dst)) else: # No API key or no political articles — keep all cropped articles print(f" Page {n}: no final political filter applied, keeping all {len(records)} articles") diff --git a/generate_pdf.py b/generate_pdf.py index 43d4dd9..99c7d14 100644 --- a/generate_pdf.py +++ b/generate_pdf.py @@ -49,11 +49,13 @@ def generate_political_pdf(selected_articles, job_dir): pdf.multi_cell(0, 8, f"Category: {clean_text(art.get('category')).upper()}", new_x="LMARGIN", new_y="NEXT") pdf.set_font("helvetica", "", 12) - pdf.multi_cell(0, 8, f"Why: {clean_text(art.get('reasoning'))}", new_x="LMARGIN", new_y="NEXT") + pdf.multi_cell(0, 8, f"Why: {clean_text(art.get('political_significance'))}", new_x="LMARGIN", new_y="NEXT") + pdf.multi_cell(0, 8, f"Attack Angle: {clean_text(art.get('attack_angle'))}", new_x="LMARGIN", new_y="NEXT") pdf.ln(5) # Image - img_name = art.get("image_file") + art_id = art.get("id") + img_name = f"{art_id}.png" if art_id else None if img_name: img_path = images_dir / img_name if img_path.exists(): diff --git a/geometric_grouper.py b/geometric_grouper.py index d57a8b9..7ee6474 100644 --- a/geometric_grouper.py +++ b/geometric_grouper.py @@ -1,23 +1,103 @@ import math +import cv2 +import numpy as np def get_horizontal_overlap(bbox1, bbox2): x_left = max(bbox1[0], bbox2[0]) x_right = min(bbox1[2], bbox2[2]) return max(0, x_right - x_left) -def group_surya_regions_geometrically(regions): +def extract_barriers(img_path): + """Extracts horizontal and vertical barrier lines using OpenCV.""" + if not img_path: + return [], [] + + img = cv2.imread(str(img_path)) + if img is None: + return [], [] + + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + _, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV) + + line_length = 300 + h_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (line_length, 1)) + h_lines_raw = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, h_kernel, iterations=2) + + v_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, line_length)) + v_lines_raw = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, v_kernel, iterations=2) + + h_contours_raw, _ = cv2.findContours(h_lines_raw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + v_contours_raw, _ = cv2.findContours(v_lines_raw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + h_lines = [] + for cnt in h_contours_raw: + x, y, w, h = cv2.boundingRect(cnt) + if h <= 15: # Thin lines only + h_lines.append((x, y, x + w, y + h)) + + v_lines = [] + for cnt in v_contours_raw: + x, y, w, h = cv2.boundingRect(cnt) + if w <= 15: # Thin lines only + v_lines.append((x, y, x + w, y + h)) + + return h_lines, v_lines + +def is_blocked_by_horizontal_barrier(top_y, bottom_y, item_left, item_right, h_lines): """ - Groups regions spatially. - Instead of a linear reading order, this associates text and images - with the headline that is directly above them in the same column. + Checks if a horizontal line sits between top_y and bottom_y and spans across the item. """ + item_width = item_right - item_left + tolerance = item_width * 0.2 + + for (lx1, ly1, lx2, ly2) in h_lines: + line_y = ly1 + if top_y < line_y < bottom_y: + # Does it span across the item? + if lx1 <= (item_left + tolerance) and lx2 >= (item_right - tolerance): + return True + return False + +def is_blocked_by_vertical_barrier(t_bbox, item_bbox, v_lines): + """ + Checks if a vertical line sits between the centers of two regions. + """ + t_cx = (t_bbox[0] + t_bbox[2]) / 2 + item_cx = (item_bbox[0] + item_bbox[2]) / 2 + + left_x = min(t_cx, item_cx) + right_x = max(t_cx, item_cx) + + # Only check if they are in different columns + if right_x - left_x < 100: + return False + + y_overlap_top = max(t_bbox[1], item_bbox[1]) + y_overlap_bottom = min(t_bbox[3], item_bbox[3]) + + for (vx1, vy1, vx2, vy2) in v_lines: + line_x = vx1 + if left_x < line_x < right_x: + # Does the line overlap them vertically? + # A true vertical column separator will be very tall + if vy1 < y_overlap_bottom and vy2 > y_overlap_top: + return True + return False + +def group_surya_regions_geometrically(regions, img_path=None): if not regions: return [] + # Extract barriers using OpenCV + h_lines, v_lines = extract_barriers(img_path) + if img_path: + print(f" [grouper] Extracted {len(h_lines)} horizontal and {len(v_lines)} vertical OpenCV barriers") + titles = [r for r in regions if r['type'] in ('doc_title', 'paragraph_title')] non_titles = [r for r in regions if r['type'] not in ('doc_title', 'paragraph_title')] + + region_by_id = {r['id']: r for r in regions} - # If no titles exist, return everything as one article if not titles: member_ids = [r['id'] for r in regions] min_x = min(r['bbox'][0] for r in regions) @@ -42,69 +122,86 @@ def group_surya_regions_geometrically(regions): best_title = None - # ------------------------------------------------------------- - # Rule 1: Find titles ABOVE the item that share the same column - # ------------------------------------------------------------- + max_dist_above = 2500 if item['type'] == 'image' else 1500 candidates_above = [] for t in titles: t_bbox = t['bbox'] if t_bbox[1] <= item_cy: + # Barrier Check + if is_blocked_by_horizontal_barrier(t_bbox[3], item_bbox[1], item_bbox[0], item_bbox[2], h_lines): + continue + if is_blocked_by_vertical_barrier(t_bbox, item_bbox, v_lines): + continue + overlap = get_horizontal_overlap(item_bbox, t_bbox) t_width = t_bbox[2] - t_bbox[0] t_cx = (t_bbox[0] + t_bbox[2]) / 2 if overlap > (item_width * 0.1) or overlap > (t_width * 0.1) or abs(item_cx - t_cx) < 150: distance = item_bbox[1] - t_bbox[3] - if distance < 1500: # MAX DISTANCE ABOVE + if distance < max_dist_above: candidates_above.append(t) if candidates_above: - best_title = min(candidates_above, key=lambda t: abs(item_bbox[1] - t['bbox'][3])) + def _above_score(t): + vert = item_bbox[1] - t['bbox'][3] + horiz = abs(item_cx - (t['bbox'][0] + t['bbox'][2]) / 2) + return vert + horiz * 0.8 + best_title = min(candidates_above, key=_above_score) else: - # ------------------------------------------------------------- - # Rule 2: If no title is above, find titles BELOW the item - # ------------------------------------------------------------- candidates_below = [] for t in titles: t_bbox = t['bbox'] if t_bbox[1] > item_cy: + # Barrier Check + if is_blocked_by_horizontal_barrier(item_bbox[3], t_bbox[1], item_bbox[0], item_bbox[2], h_lines): + continue + if is_blocked_by_vertical_barrier(t_bbox, item_bbox, v_lines): + continue + overlap = get_horizontal_overlap(item_bbox, t_bbox) t_width = t_bbox[2] - t_bbox[0] t_cx = (t_bbox[0] + t_bbox[2]) / 2 if overlap > (item_width * 0.1) or overlap > (t_width * 0.1) or abs(item_cx - t_cx) < 150: distance = t_bbox[1] - item_bbox[3] - if distance < 800: # MAX DISTANCE BELOW (stricter because text usually flows down, not up) + if distance < 800: candidates_below.append(t) if candidates_below: - best_title = min(candidates_below, key=lambda t: abs(t['bbox'][1] - item_bbox[3])) + def _below_score(t): + vert = t['bbox'][1] - item_bbox[3] + horiz = abs(item_cx - (t['bbox'][0] + t['bbox'][2]) / 2) + return vert + horiz * 0.8 + best_title = min(candidates_below, key=_below_score) else: - # ------------------------------------------------------------- - # Rule 3: Absolute fallback. No column overlap at all. - # Find the absolute closest title via Euclidean distance. - # ------------------------------------------------------------- - closest_t = min(titles, key=lambda t: math.hypot( - item_cx - ((t['bbox'][0] + t['bbox'][2]) / 2), - item_cy - ((t['bbox'][1] + t['bbox'][3]) / 2) - )) - # Only attach if it's reasonably close, else leave as orphan - dist = math.hypot(item_cx - ((closest_t['bbox'][0] + closest_t['bbox'][2]) / 2), - item_cy - ((closest_t['bbox'][1] + closest_t['bbox'][3]) / 2)) - if dist < 1500: - best_title = closest_t + # Absolute fallback + valid_closest = [] + for t in titles: + if is_blocked_by_horizontal_barrier(min(item_bbox[3], t['bbox'][3]), max(item_bbox[1], t['bbox'][1]), item_bbox[0], item_bbox[2], h_lines): + continue + if is_blocked_by_vertical_barrier(t['bbox'], item_bbox, v_lines): + continue + valid_closest.append(t) + + if valid_closest: + closest_t = min(valid_closest, key=lambda t: math.hypot( + item_cx - ((t['bbox'][0] + t['bbox'][2]) / 2), + item_cy - ((t['bbox'][1] + t['bbox'][3]) / 2) + )) + dist = math.hypot(item_cx - ((closest_t['bbox'][0] + closest_t['bbox'][2]) / 2), + item_cy - ((closest_t['bbox'][1] + closest_t['bbox'][3]) / 2)) + if dist < 1500: + best_title = closest_t if best_title: article_groups[best_title['id']].append(item) else: - # Treat as an orphan (create a new article group for it, or group orphans together) - # For simplicity, we'll assign it a virtual title ID based on its own ID so it becomes a standalone article virtual_id = f"orphan_{item['id']}" article_groups[virtual_id] = [item] # Format output articles = [] - # Sort groups top-to-bottom, left-to-right based on title position sorted_titles = sorted(titles, key=lambda t: (t['bbox'][1], t['bbox'][0])) for idx, (title_id, group_regions) in enumerate(article_groups.items(), start=1): @@ -116,7 +213,6 @@ def group_surya_regions_geometrically(regions): max_x = max(r['bbox'][2] for r in group_regions) max_y = max(r['bbox'][3] for r in group_regions) - # Determine if it's an orphan group is_orphan = str(title_id).startswith("orphan_") articles.append({ @@ -127,12 +223,9 @@ def group_surya_regions_geometrically(regions): "grouped_by": "geometric_orphan" if is_orphan else "geometric" }) - # We should merge orphans that are vertically adjacent into the same orphan group to prevent shattering - # Simple vertical merge for orphans orphan_articles = [a for a in articles if a['grouped_by'] == 'geometric_orphan'] valid_articles = [a for a in articles if a['grouped_by'] == 'geometric'] - # Sort orphans top-to-bottom orphan_articles.sort(key=lambda a: a['bbox'][1]) merged_orphans = [] @@ -143,28 +236,101 @@ def group_surya_regions_geometrically(regions): current_orphan = o continue - # Check if they overlap horizontally and are close vertically overlap = get_horizontal_overlap(current_orphan['bbox'], o['bbox']) vertical_dist = o['bbox'][1] - current_orphan['bbox'][3] if overlap > 0 and vertical_dist < 400: - # Merge - current_orphan['member_regions'].extend(o['member_regions']) - current_orphan['bbox'][0] = min(current_orphan['bbox'][0], o['bbox'][0]) - current_orphan['bbox'][1] = min(current_orphan['bbox'][1], o['bbox'][1]) - current_orphan['bbox'][2] = max(current_orphan['bbox'][2], o['bbox'][2]) - current_orphan['bbox'][3] = max(current_orphan['bbox'][3], o['bbox'][3]) - else: - merged_orphans.append(current_orphan) - current_orphan = o + # Check barrier before merging orphans + if not is_blocked_by_horizontal_barrier(current_orphan['bbox'][3], o['bbox'][1], current_orphan['bbox'][0], current_orphan['bbox'][2], h_lines): + current_orphan['member_regions'].extend(o['member_regions']) + current_orphan['bbox'][0] = min(current_orphan['bbox'][0], o['bbox'][0]) + current_orphan['bbox'][1] = min(current_orphan['bbox'][1], o['bbox'][1]) + current_orphan['bbox'][2] = max(current_orphan['bbox'][2], o['bbox'][2]) + current_orphan['bbox'][3] = max(current_orphan['bbox'][3], o['bbox'][3]) + continue + + merged_orphans.append(current_orphan) + current_orphan = o if current_orphan: merged_orphans.append(current_orphan) final_articles = valid_articles + merged_orphans - - # Reassign IDs - for idx, a in enumerate(final_articles, start=1): + + MAX_SPLIT_HEIGHT = 2500 + MIN_MEMBERS_SPLIT = 5 + MIN_SPLIT_GAP = 30 + MIN_LOWER_HEIGHT = 400 + + split_result = [] + for art in final_articles: + _, art_t, _, art_b = art['bbox'] + if (art_b - art_t) <= MAX_SPLIT_HEIGHT or len(art['member_regions']) < MIN_MEMBERS_SPLIT: + split_result.append(art) + continue + + members = [region_by_id[rid] for rid in art['member_regions'] if rid in region_by_id] + members.sort(key=lambda reg: reg['bbox'][1]) + + best_gap = MIN_SPLIT_GAP + split_idx = -1 + max_bottom_so_far = 0 + for i in range(len(members) - 1): + cur_bottom = members[i]['bbox'][3] + max_bottom_so_far = max(max_bottom_so_far, cur_bottom) + + next_top = members[i + 1]['bbox'][1] + gap = next_top - max_bottom_so_far + + # If there is a barrier line crossing this gap, artificially inflate the gap score to force a split here + is_barrier = is_blocked_by_horizontal_barrier(max_bottom_so_far, next_top, members[i]['bbox'][0], members[i]['bbox'][2], h_lines) + effective_gap = gap + 10000 if is_barrier else gap + + if effective_gap > best_gap: + lower_group = members[i + 1:] + if not any(reg['type'] == 'image' for reg in lower_group): + best_gap = effective_gap + split_idx = i + + if split_idx < 0: + split_result.append(art) + continue + + upper = members[:split_idx + 1] + lower = members[split_idx + 1:] + + lower_bottom = max(reg['bbox'][3] for reg in lower) + if lower_bottom - lower[0]['bbox'][1] < MIN_LOWER_HEIGHT: + split_result.append(art) + continue + + def _bbox(regs): + return [min(reg['bbox'][0] for reg in regs), min(reg['bbox'][1] for reg in regs), + max(reg['bbox'][2] for reg in regs), max(reg['bbox'][3] for reg in regs)] + + real_gap = members[split_idx+1]['bbox'][1] - max(r['bbox'][3] for r in upper) + print(f" [split] Article {art['article_id']}: split at gap={real_gap}px after region " + f"{upper[-1]['id']} (upper {len(upper)} regions, lower {len(lower)} regions)") + + upper_ids = [reg['id'] for reg in upper] + lower_ids = [reg['id'] for reg in lower] + + split_result.append({ + 'article_id': art['article_id'], + 'title_region': art['title_region'] if art['title_region'] in upper_ids else None, + 'member_regions': upper_ids, + 'bbox': _bbox(upper), + 'grouped_by': art['grouped_by'], + }) + split_result.append({ + 'article_id': None, + 'title_region': art['title_region'] if art['title_region'] in lower_ids else None, + 'member_regions': lower_ids, + 'bbox': _bbox(lower), + 'grouped_by': 'geometric_split', + }) + + for idx, a in enumerate(split_result, start=1): a['article_id'] = idx - - return final_articles + + return split_result diff --git a/requirements.txt b/requirements.txt index 0d1e398..7f26790 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,5 @@ paddleocr==3.5.0 paddlepaddle==3.0.0 pytesseract>=0.3.10 anthropic>=0.40 +surya-ocr==0.4.15 +fpdf2>=2.7.0 diff --git a/scratch_missing.txt b/scratch_missing.txt new file mode 100644 index 0000000..62b9830 --- /dev/null +++ b/scratch_missing.txt @@ -0,0 +1,6 @@ +output/20260614_164012/pages\page_002.triage_log.json - p002_a003: దిగుమతులు లేక నిలిచిపోయిన +ట్రక్టర్జు +eo +ూమాలీల కొరతతో ఇబ్దందులు పడుతున్న రెతులు +25 +త్యర్జూరు, మే 16 diff --git a/scratch_test_grouper.py b/scratch_test_grouper.py new file mode 100644 index 0000000..14870b1 --- /dev/null +++ b/scratch_test_grouper.py @@ -0,0 +1,17 @@ +import json +from pathlib import Path +import geometric_grouper + +regions_file = Path("output/20260614_124012/pages/page_004.regions.json") +img_file = Path("output/20260614_124012/pages/page_004.png") + +with open(regions_file, encoding='utf-8') as f: + regions = json.load(f)["regions"] + +valid_regions = [r for r in regions if r["type"] not in ("header", "footer", "page_number")] + +articles = geometric_grouper.group_surya_regions_geometrically(valid_regions, img_path=str(img_file)) + +print(f"Total articles identified: {len(articles)}") +for a in articles: + print(f" Article {a['article_id']}: {len(a['member_regions'])} regions, bbox {a['bbox']}, grouped_by={a['grouped_by']}") diff --git a/scratch_test_lines.py b/scratch_test_lines.py new file mode 100644 index 0000000..0065db6 --- /dev/null +++ b/scratch_test_lines.py @@ -0,0 +1,80 @@ +import cv2 +import numpy as np +import sys +from pathlib import Path + +def test_lines(img_path, out_path): + img = cv2.imread(img_path) + if img is None: + print("Could not read image") + return + + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + + # 1. Use standard binary thresholding instead of adaptive to reduce noise from text + # Newspapers usually have very black lines on white background + _, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV) + + # We want lines that are at least 300px long (a typical column width) + line_length = 300 + + # 2. Detect horizontal lines (MUST be thin: e.g., height <= 5px) + horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (line_length, 1)) + horizontal_lines_raw = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, horizontal_kernel, iterations=2) + + # 3. Detect vertical lines (MUST be thin: e.g., width <= 5px) + vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, line_length)) + vertical_lines_raw = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, vertical_kernel, iterations=2) + + # 4. Filter contours to ensure they are actually THIN lines, not thick blocks/images + h_contours_raw, _ = cv2.findContours(horizontal_lines_raw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + v_contours_raw, _ = cv2.findContours(vertical_lines_raw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + h_contours = [] + for cnt in h_contours_raw: + x, y, w, h = cv2.boundingRect(cnt) + if h <= 15: # Line must not be thicker than 15px + h_contours.append(cnt) + + v_contours = [] + for cnt in v_contours_raw: + x, y, w, h = cv2.boundingRect(cnt) + if w <= 15: # Line must not be thicker than 15px + v_contours.append(cnt) + + print(f"Horizontal separators found: {len(h_contours)} (filtered from {len(h_contours_raw)})") + print(f"Vertical separators found: {len(v_contours)} (filtered from {len(v_contours_raw)})") + + # 5. Create clean masks from only the filtered thin lines + clean_h_mask = np.zeros_like(thresh) + cv2.drawContours(clean_h_mask, h_contours, -1, 255, thickness=cv2.FILLED) + + clean_v_mask = np.zeros_like(thresh) + cv2.drawContours(clean_v_mask, v_contours, -1, 255, thickness=cv2.FILLED) + + grid = cv2.add(clean_h_mask, clean_v_mask) + rect_contours, _ = cv2.findContours(grid, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE) + + box_count = 0 + debug_img = img.copy() + + for cnt in rect_contours: + epsilon = 0.02 * cv2.arcLength(cnt, True) + approx = cv2.approxPolyDP(cnt, epsilon, True) + + if len(approx) == 4: + x, y, w, h = cv2.boundingRect(approx) + if w > 300 and h > 200: + box_count += 1 + cv2.rectangle(debug_img, (x, y), (x+w, y+h), (255, 0, 255), 8) + + print(f"Rectangular boxes found: {box_count}") + + cv2.drawContours(debug_img, h_contours, -1, (0, 0, 255), 3) # Red for horizontal + cv2.drawContours(debug_img, v_contours, -1, (255, 0, 0), 3) # Blue for vertical + + cv2.imwrite(out_path, debug_img) + print(f"Saved debug image to {out_path}") + +if __name__ == "__main__": + test_lines(sys.argv[1], sys.argv[2])