deepkoluguri
/
sundeep-news-scan
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: add gunicorn and opencv-python to requirements

This commit is contained in:
Deep Koluguri 2026-06-13 10:46:33 -04:00
parent 8f69c0fddc
commit 65c1501d89
3 changed files with 38 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
claude_ocr.py
View File

@ -18,8 +18,23 @@ OUTPUT_DIR = Path(__file__).parent / "output"
def read_article_image(client, img_path, model="claude-sonnet-4-6"):
"""Send article image to Claude and get Telugu text back."""
with open(img_path, "rb") as f:
img_data = base64.standard_b64encode(f.read()).decode("utf-8")
from PIL import Image
import io
with Image.open(img_path) as img:
# Resize if too large to fit in 10MB limit (usually > 4000x4000)
max_dim = 2800
if max(img.width, img.height) > max_dim:
ratio = max_dim / max(img.width, img.height)
new_w = int(img.width * ratio)
new_h = int(img.height * ratio)
print(f" [Claude OCR] Resizing {img.width}x{img.height} to {new_w}x{new_h} to fit API limits")
img = img.resize((new_w, new_h), Image.LANCZOS)
# Save to bytes
buffer = io.BytesIO()
img.save(buffer, format="PNG", optimize=True)
img_data = base64.standard_b64encode(buffer.getvalue()).decode("utf-8")
response = client.messages.create(
model=model,

115
extractor.py
View File

@ -334,103 +334,22 @@ def crop_full_articles(page_png_path, selected_ids, articles, out_dir, page_num)
# ---------------------------------------------------------------------------
# Stage 5 — OCR each cropped article (Telugu)
# ---------------------------------------------------------------------------
_paddle_ocr_instance = None
def _ocr_with_paddleocr(img_path):
"""Use PaddleOCR for better reading-order-aware Telugu OCR."""
global _paddle_ocr_instance
temp_path = None
try:
from PIL import Image
with Image.open(img_path) as im:
px = im.width * im.height
if px > 4000000:
# Instead of skipping, resize the article image to prevent PaddleOCR deadlock
ratio = (3800000 / px) ** 0.5
new_w = int(im.width * ratio)
new_h = int(im.height * ratio)
print(f"Article image {img_path.name} is too large ({im.width}x{im.height} = {px}px). Resizing to {new_w}x{new_h} for safe PaddleOCR.")
resized_im = im.resize((new_w, new_h), Image.LANCZOS)
temp_path = img_path.parent / f"temp_resized_{img_path.name}"
resized_im.save(temp_path)
import paddleocr
if _paddle_ocr_instance is None:
_paddle_ocr_instance = paddleocr.PaddleOCR(lang='te')
path_to_ocr = str(temp_path) if temp_path else str(img_path)
result = _paddle_ocr_instance.predict(path_to_ocr)
lines = []
if isinstance(result, list) and len(result) > 0:
det_result = result[0]
# PaddleOCR v3.5 returns DetResult with 'rec_texts' or nested structure
rec_texts = None
if hasattr(det_result, 'get'):
rec_texts = det_result.get('rec_texts', None)
if rec_texts:
lines = list(rec_texts)
else:
# Try to extract from boxes/text pairs
boxes = det_result.get('dt_polys', []) if hasattr(det_result, 'get') else []
texts = det_result.get('rec_texts', []) if hasattr(det_result, 'get') else []
if texts:
# Sort by y-coordinate (top to bottom), then x (left to right)
# to get proper reading order
pairs = []
for i, txt in enumerate(texts):
if i < len(boxes):
box = boxes[i]
if hasattr(box, 'tolist'):
box = box.tolist()
# Get top-left y coordinate for sorting
if isinstance(box, (list, tuple)) and len(box) >= 4:
y = min(box[j] for j in range(1, len(box), 2)) if len(box) >= 8 else box[1]
x = min(box[j] for j in range(0, len(box), 2)) if len(box) >= 8 else box[0]
else:
y, x = 0, 0
pairs.append((y, x, txt))
else:
pairs.append((0, 0, txt))
# Sort: primarily by y (row), then x (column within row)
# Group into rows based on y proximity
if pairs:
pairs.sort(key=lambda p: (p[0], p[1]))
row_threshold = 20 # pixels
rows = []
current_row = [pairs[0]]
for p in pairs[1:]:
if abs(p[0] - current_row[0][0]) < row_threshold:
current_row.append(p)
else:
current_row.sort(key=lambda p: p[1]) # sort by x within row
rows.append(current_row)
current_row = [p]
current_row.sort(key=lambda p: p[1])
rows.append(current_row)
for row in rows:
lines.append(" ".join(p[2] for p in row))
else:
# Last resort: try str representation
try:
s = str(det_result)
if len(s) > 10:
return s
except:
pass
return "\n".join(lines) if lines else None
except Exception as e:
print(f"PaddleOCR text extraction failed: {e}")
def _ocr_with_claude(img_path):
"""Use Claude Vision API for Telugu OCR as a fallback since PaddleOCR is missing."""
import os
api_key = os.environ.get("ANTHROPIC_API_KEY")
if not api_key:
print("ANTHROPIC_API_KEY not set. Skipping OCR.")
return None
finally:
if temp_path and temp_path.exists():
try:
temp_path.unlink()
except:
pass
import anthropic
import claude_ocr
client = anthropic.Anthropic(api_key=api_key)
return claude_ocr.read_article_image(client, str(img_path))
except Exception as e:
print(f"Claude text extraction failed: {e}")
return None
def ocr_headlines(headline_records):
@ -439,7 +358,7 @@ def ocr_headlines(headline_records):
if not img_path.exists():
rec["headline_text"] = ""
continue
text = _ocr_with_paddleocr(img_path)
text = _ocr_with_claude(img_path)
rec["headline_text"] = (text or "").strip()
(img_path.parent / "headline.txt").write_text(rec["headline_text"], encoding="utf-8")
@ -450,12 +369,14 @@ def ocr_articles(article_records):
if not img_path.exists():
continue
text = _ocr_with_paddleocr(img_path)
text = _ocr_with_claude(img_path)
(art_dir / "article.txt").write_text(text or "", encoding="utf-8")
info_path = art_dir / "info.json"
if info_path.exists():
import json
info = json.loads(info_path.read_text(encoding="utf-8"))
info["ocr_method"] = "claude_vision"
info_path.write_text(json.dumps(info, indent=2, ensure_ascii=False), encoding="utf-8")

2
requirements.txt
View File

@ -8,3 +8,5 @@ pytesseract>=0.3.10
anthropic>=0.40
surya-ocr==0.4.15
fpdf2>=2.7.0
gunicorn>=21.0.0
opencv-python>=4.8.0