"""Separator-rule detection for newspaper pages (classical CV, no ML / no API).

Detects the thin printed grey/black straight lines the paper uses to divide
stories, with a TEXT-ISOLATION filter to reject false positives from dense
Telugu text (a real rule sits in a whitespace gutter; a text stroke has ink
immediately above/below it).

Public:
    detect_separator_lines(png_path) -> {"h": [...], "v": [...], "size": (W,H)}
        each line is a dict: {x1,y1,x2,y2,len,gray} (gray = mean intensity).
"""
import cv2
import numpy as np


def _candidates(bw, horizontal, min_len, max_thick=55):
    if horizontal:
        k = cv2.getStructuringElement(cv2.MORPH_RECT, (min_len // 2, 1))
    else:
        k = cv2.getStructuringElement(cv2.MORPH_RECT, (1, min_len // 2))
    opened = cv2.morphologyEx(bw, cv2.MORPH_OPEN, k, iterations=1)
    # bridge small collinear gaps along the line direction
    if horizontal:
        opened = cv2.dilate(opened, cv2.getStructuringElement(cv2.MORPH_RECT, (25, 3)))
    else:
        opened = cv2.dilate(opened, cv2.getStructuringElement(cv2.MORPH_RECT, (3, 25)))
    cnts, _ = cv2.findContours(opened, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    out = []
    for c in cnts:
        x, y, w, h = cv2.boundingRect(c)
        if horizontal and w >= min_len and h <= max_thick:
            out.append((x, y + h // 2, x + w, y + h // 2, w))
        if (not horizontal) and h >= min_len and w <= max_thick:
            out.append((x + w // 2, y, x + w // 2, y + h, h))
    return out


def _isolated(gray, line, horizontal, ink_thresh=110, gap=(5, 16),
              max_adjacent_ink=0.12):
    """A true separator sits in a whitespace gutter: the tight band on BOTH sides
    (parallel to the line) is mostly paper. Returns (is_isolated, mean_gray)."""
    H, W = gray.shape
    x1, y1, x2, y2, ln = line
    if horizontal:
        seg = gray[y1, x1:x2]
        lo, hi = gap
        above = gray[max(0, y1 - hi):max(0, y1 - lo), x1:x2]
        below = gray[min(H, y1 + lo):min(H, y1 + hi), x1:x2]
    else:
        seg = gray[y1:y2, x1]
        lo, hi = gap
        above = gray[y1:y2, max(0, x1 - hi):max(0, x1 - lo)]
        below = gray[y1:y2, min(W, x1 + lo):min(W, x1 + hi)]
    if seg.size == 0 or above.size == 0 or below.size == 0:
        return False, 255
    ink_a = float((above < ink_thresh).mean())
    ink_b = float((below < ink_thresh).mean())
    isolated = (ink_a <= max_adjacent_ink) and (ink_b <= max_adjacent_ink)
    return isolated, float(seg.mean())


def detect_separator_lines(png_path, h_frac=0.045, v_frac=0.06):
    img = cv2.imread(str(png_path))
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    H, W = gray.shape
    blur = cv2.GaussianBlur(gray, (3, 3), 0)
    bw = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]

    h_min = int(h_frac * W)
    v_min = int(v_frac * H)
    h_out, v_out = [], []
    for ln in _candidates(bw, True, h_min):
        iso, g = _isolated(gray, ln, True)
        if iso:
            h_out.append({"x1": ln[0], "y1": ln[1], "x2": ln[2], "y2": ln[3],
                          "len": ln[4], "gray": round(g, 1)})
    for ln in _candidates(bw, False, v_min):
        iso, g = _isolated(gray, ln, False)
        if iso:
            v_out.append({"x1": ln[0], "y1": ln[1], "x2": ln[2], "y2": ln[3],
                          "len": ln[4], "gray": round(g, 1)})
    return {"h": h_out, "v": v_out, "size": (W, H)}


def detect_content_bottom(png_path, search_frac=0.12, color_thr=0.06,
                          blank_color=0.02, blank_ink=0.035, peak_thr=0.15,
                          max_band=160, min_gap=25, min_margin=18):
    """Find the y below which a page holds only the printer's COLOUR-CALIBRATION
    strip (the row of CMYK registration dots + grey bars) and the bottom paper
    margin — i.e. the bottom of real article content.

    The calibration strip is a thin band of strongly-saturated colour dots near
    the page bottom, isolated by a white paper margin BELOW it (down to the page
    edge) and a white gap ABOVE it (separating it from article content). Crops
    should be clamped to the returned y so the dots never bleed into an article.

    Returns the y at the TOP of the calibration band, or the full page height H
    when no such strip is found (nothing to clamp)."""
    img = cv2.imread(str(png_path))
    if img is None:
        return None
    H, W = img.shape[:2]
    b, g, r = img[..., 0].astype(np.int16), img[..., 1].astype(np.int16), img[..., 2].astype(np.int16)
    mx = np.maximum(np.maximum(r, g), b)
    mn = np.minimum(np.minimum(r, g), b)
    sat = mx - mn
    colored = (sat > 55) & (mx > 80)          # saturated colour (registration dots)
    dark = mx < 110                           # ink / dark
    col_frac = colored.mean(axis=1)           # per-row colour fraction
    ink_frac = dark.mean(axis=1)              # per-row ink fraction
    blank = (col_frac < blank_color) & (ink_frac < blank_ink)

    y0 = int(H * (1 - search_frac))
    # The calibration strip is a band of strongly-coloured rows; identify it as the
    # bottom-most run of rows whose colour fraction clears `color_thr`.
    strip_rows = [y for y in range(y0, H) if col_frac[y] >= color_thr]
    if not strip_rows:
        return H                               # no coloured strip near the bottom
    band_bottom = strip_rows[-1]
    # A real calibration strip is followed by white paper margin to the page edge.
    below = blank[band_bottom + 1:H]
    if below.size < min_margin or float(below.mean()) < 0.9:
        return H
    # Walk up from band_bottom while still in the coloured band (tolerate the small
    # white gaps between dot rows).
    band_top = band_bottom
    gap = 0
    y = band_bottom - 1
    while y > y0:
        if col_frac[y] >= color_thr:
            band_top = y
            gap = 0
        else:
            gap += 1
            if gap > 12:
                break
        y -= 1
    if (band_bottom - band_top + 1) > max_band:
        return H                               # too tall -> fused with a photo, bail
    # A genuine printer calibration strip is a dense row of saturated CMYK
    # registration dots: its peak per-row colour fraction is high (~0.22-0.45 across
    # Andhra Jyothi / Sakshi). A single-hue coloured ad/notice box embedded in
    # article text peaks far lower (~0.11) -> reject it as a false positive.
    peak = float(col_frac[band_top:band_bottom + 1].max())
    if peak < peak_thr:
        return H
    return band_top


def _hspan_overlap(ax1, ax2, bx1, bx2):
    ix = max(0, min(ax2, bx2) - max(ax1, bx1))
    return ix / max(1, min(ax2 - ax1, bx2 - bx1))


def _faint_rule_above(gray, hbbox, max_gap=170, min_gap=6, gray_lo=140, gray_hi=215,
                      cover=0.6, max_dark=0.06):
    """Find a faint GREY horizontal rule sitting in the whitespace directly above a
    (multi-column) header. Such a light rule reads as background to the ink-based
    Otsu detector in `detect_separator_lines`, so we look for a thin row that is
    mostly uniform mid-grey across the header's width and is isolated by white paper
    above and below. Returns the rule's y, or None. (Used only above multi-column
    headers, where it marks the article's TOP boundary.)"""
    H, W = gray.shape
    hx1, hy, hx2 = hbbox[0], hbbox[1], hbbox[2]
    y_hi = max(0, hy - min_gap)
    y_lo = max(0, hy - max_gap)
    for y in range(y_hi, y_lo, -1):
        row = gray[y, hx1:hx2].astype(np.int16)
        if row.size == 0:
            continue
        mid = float(((row >= gray_lo) & (row <= gray_hi)).mean())
        dark = float((row < gray_lo).mean())
        if mid >= cover and dark <= max_dark:
            above = gray[max(0, y - 6):max(0, y - 2), hx1:hx2]
            below = gray[min(H, y + 2):min(H, y + 6), hx1:hx2]
            if (above.size and below.size
                    and float((above > gray_hi).mean()) > 0.85
                    and float((below > gray_hi).mean()) > 0.85):
                return y
    return None


def _faint_grey_band_above(gray, hbbox, max_gap=185, min_gap=6, max_thick=12,
                           gray_lo=135, gray_hi=232, cover=0.8, max_dark=0.08,
                           white_thr=235, white_cover=0.9):
    """Find a faint GREY horizontal rule (1-`max_thick` px thick) sitting in the
    whitespace directly above a SINGLE-column header. Per the layout convention a
    grey rule whose length matches the article/header width marks the boundary
    between two stacked stories, so the next story's headline must not fold into
    the article above the rule. Such light rules read as background to the ink-based
    Otsu detector in `detect_separator_lines`, and `_faint_rule_above` only accepts
    a 1-px isolated rule, so a thicker grey band is missed; here we group the band
    and require it to span the header's width and be isolated by white paper above
    and below. Returns the band's centre y, or None."""
    H, W = gray.shape
    hx1, hy, hx2 = int(hbbox[0]), int(hbbox[1]), int(hbbox[2])
    y_hi = max(0, hy - min_gap)
    y_lo = max(0, hy - max_gap)
    cols = slice(hx1, hx2)
    grey_rows = []
    for y in range(y_lo, y_hi):
        row = gray[y, cols].astype(np.int16)
        if row.size == 0:
            continue
        mid = float(((row >= gray_lo) & (row <= gray_hi)).mean())
        dark = float((row < gray_lo).mean())
        if mid >= cover and dark <= max_dark:
            grey_rows.append(y)
    if not grey_rows:
        return None
    bands = []
    s = p = grey_rows[0]
    for y in grey_rows[1:]:
        if y == p + 1:
            p = y
        else:
            bands.append((s, p)); s = p = y
    bands.append((s, p))
    for top, bot in sorted(bands, key=lambda b: -b[1]):   # nearest the header first
        if bot - top + 1 > max_thick:
            continue
        above = gray[max(0, top - 4):top, cols]
        below = gray[bot + 1:min(H, bot + 5), cols]
        if (above.size and below.size
                and float((above > white_thr).mean()) >= white_cover
                and float((below > white_thr).mean()) >= white_cover):
            return (top + bot) // 2
    return None


def _faint_grey_band_below(gray, bbox, max_gap=150, min_gap=4, max_thick=12,
                           gray_lo=135, gray_hi=232, cover=0.8, max_dark=0.08,
                           white_thr=235, white_cover=0.9):
    """Mirror of `_faint_grey_band_above`, searching DOWNWARD: find the faint grey
    rule that sits in the whitespace just below an article (within `max_gap` px of
    its bottom edge) and spans its column width. That rule is the article's BOTTOM
    boundary, so the crop can be cut precisely at it. Returns the band's top y (the
    cut line), or None when no clean full-width grey rule is found below."""
    H, W = gray.shape
    bx1, by, bx2 = int(bbox[0]), int(bbox[3]), int(bbox[2])
    y_lo = min(H, by + min_gap)
    y_hi = min(H, by + max_gap)
    cols = slice(bx1, bx2)
    grey_rows = []
    for y in range(y_lo, y_hi):
        row = gray[y, cols].astype(np.int16)
        if row.size == 0:
            continue
        mid = float(((row >= gray_lo) & (row <= gray_hi)).mean())
        dark = float((row < gray_lo).mean())
        if mid >= cover and dark <= max_dark:
            grey_rows.append(y)
    if not grey_rows:
        return None
    bands = []
    s = p = grey_rows[0]
    for y in grey_rows[1:]:
        if y == p + 1:
            p = y
        else:
            bands.append((s, p)); s = p = y
    bands.append((s, p))
    for top, bot in sorted(bands, key=lambda b: b[0]):    # nearest below first
        if bot - top + 1 > max_thick:
            continue
        # Skip the 1-px anti-aliased fade row hugging the rule before checking for
        # the white paper gutter above and below it.
        above = gray[max(0, top - 7):max(0, top - 2), cols]
        below = gray[bot + 2:min(H, bot + 7), cols]
        if (above.size and below.size
                and float((above > white_thr).mean()) >= white_cover
                and float((below > white_thr).mean()) >= white_cover):
            return top
    return None


def separator_barriers(png_path, regions, near=80, min_overlap=0.3,
                       title_types=("doc_title", "paragraph_title", "figure_title"),
                       multicol_frac=0.22):
    """Article-boundary rules: the horizontal printed lines that mark the END of a
    story. Per the layout convention, a rule with NORMAL text (or whitespace) above
    it separates two stacked articles, while a rule sitting right under a BOLD
    headline/title is a decorative flourish and must NOT split anything.

    We approximate 'bold above' by a title-type region (doc_title / paragraph_title
    / figure_title) whose bottom edge is within `near` px above the rule and that
    horizontally overlaps it. Such rules are dropped; the rest are returned as
    barriers {y, x1, x2} for the clusterer to forbid cross-line folding.

    Additionally, for every MULTI-COLUMN header (a doc_title at least `multicol_frac`
    of the page width), we look for a faint grey rule directly above it and add it as
    a barrier spanning the header's width: that rule is the article's TOP boundary, so
    nothing above it folds into the multi-column article below."""
    res = detect_separator_lines(png_path)

    # A real article separator sits in a WHITESPACE GUTTER between two stacked
    # boxes; it never lands deep inside a body-text region. PaddleOCR's morphology
    # occasionally mistakes a dense Telugu text stroke for a thin rule, producing a
    # phantom line straight through a paragraph. Reject any detected rule whose
    # midpoint falls well inside (margin-shrunk) a 'text' box it column-overlaps.
    _text_boxes = [r["bbox"] for r in regions if r.get("type") == "text"]

    def _in_text(y, lx1, lx2, margin=28):
        mx = (lx1 + lx2) / 2
        for tx1, ty1, tx2, ty2 in _text_boxes:
            if (tx1 <= mx <= tx2
                    and ty1 + margin <= y <= ty2 - margin
                    and _hspan_overlap(lx1, lx2, tx1, tx2) >= min_overlap):
                return True
        return False

    # Column width (median text-region width). A horizontal line counts as an article
    # separator only when it spans most of a column. NOTE: a strict ≥1.0× column floor is
    # too aggressive — a SINGLE-column article's own boundary rule is itself <1 column wide,
    # so ≥1.0× drops real separators and merges articles (measured: NT pg4 R121 -> 2 datelines).
    # 0.6× column is the safe floor: it removes only the short caption / decorative underlines
    # and merges NO articles on AJ / NT / JG.
    _twid = sorted((r["bbox"][2] - r["bbox"][0]) for r in regions if r.get("type") == "text")
    _pitch = _twid[len(_twid) // 2] if _twid else 0
    _len_floor = 0.6 * _pitch

    bars = []
    for L in res["h"]:
        y, lx1, lx2 = L["y1"], L["x1"], L["x2"]
        if _in_text(y, lx1, lx2):
            continue
        if _pitch and (lx2 - lx1) < _len_floor:
            continue                       # too short to be an article separator (caption/underline)
        bold_above = False
        title_w_above = 0
        for r in regions:
            b = r["bbox"]
            if (b[3] <= y and (y - b[3]) <= near
                    and r.get("type") in title_types
                    and _hspan_overlap(lx1, lx2, b[0], b[2]) >= min_overlap):
                bold_above = True
                title_w_above = max(title_w_above, b[2] - b[0])
        # A rule under a bold title is a decorative flourish ONLY when it merely
        # underlines that title (rule width ≈ title width). A rule spanning much
        # WIDER than the title above it runs across several columns — e.g. a photo
        # caption (figure_title) sits above ONE column while the rule runs the FULL
        # article width — so it is a genuine article boundary and must be kept.
        if (not bold_above) or ((lx2 - lx1) > 1.5 * title_w_above):
            bars.append({"y": y, "x1": lx1, "x2": lx2})

    gray = cv2.cvtColor(cv2.imread(str(png_path)), cv2.COLOR_BGR2GRAY)
    W = gray.shape[1]
    _img_boxes = [r["bbox"] for r in regions if r.get("type") == "image"]

    def _lead_photo_above(yb, hx1, hx2, gap=80):
        """A faint rule directly under the article's OWN lead photo is that photo's
        caption underline, not the article's top boundary. If an image sits flush
        (within `gap`px) above the rule and overlaps the header's width, the rule must
        NOT bound the article — otherwise the lead photo gets cut off."""
        for ix1, iy1, ix2, iy2 in _img_boxes:
            if (0 <= yb - iy2 <= gap
                    and _hspan_overlap(hx1, hx2, ix1, ix2) >= min_overlap):
                return True
        return False

    for r in regions:
        if r.get("type") != "doc_title":
            continue
        b = r["bbox"]
        if (b[2] - b[0]) < multicol_frac * W:
            continue                                  # single-column header -> skip
        yb = _faint_rule_above(gray, b)
        if yb is not None and not _lead_photo_above(yb, b[0], b[2]):
            bars.append({"y": yb, "x1": b[0], "x2": b[2],
                         "mc": True, "anchor": r["id"]})

    # Same rule for SINGLE-column headers: a faint grey rule spanning the header's
    # width directly above it bounds the article ABOVE it, so the header (and its
    # story) cannot fold upward across the rule. Plain barrier (no mc clamp).
    for r in regions:
        if r.get("type") not in ("doc_title", "paragraph_title"):
            continue
        b = r["bbox"]
        if (b[2] - b[0]) >= multicol_frac * W:
            continue                                  # multi-column -> handled above
        yb = _faint_grey_band_above(gray, b)
        if yb is not None and not _lead_photo_above(yb, b[0], b[2]):
            bars.append({"y": yb, "x1": b[0], "x2": b[2]})

    # Vertical column-divider rules. detect_separator_lines already requires each
    # line to sit in a whitespace gutter (isolated on both sides), so photo/text
    # internal strokes are filtered out. These fence rightward column-growth: an
    # article must not grow across a printed vertical rule into the next column.
    for L in res["v"]:
        bars.append({"vert": True, "x": (L["x1"] + L["x2"]) // 2,
                     "y1": min(L["y1"], L["y2"]), "y2": max(L["y1"], L["y2"])})
    return bars


def overlay(png_path, lines, out_path):
    img = cv2.imread(str(png_path))
    for L in lines["h"]:
        cv2.line(img, (L["x1"], L["y1"]), (L["x2"], L["y2"]), (0, 0, 255), 6)
    for L in lines["v"]:
        cv2.line(img, (L["x1"], L["y1"]), (L["x2"], L["y2"]), (255, 0, 0), 6)
    cv2.imwrite(str(out_path), img)


if __name__ == "__main__":
    import sys
    from pathlib import Path
    p = Path(sys.argv[1])
    res = detect_separator_lines(p)
    print(f"{p.name}  {res['size']}  H={len(res['h'])} V={len(res['v'])}")
    for L in sorted(res["h"], key=lambda d: d["y1"]):
        print(f"  H y={L['y1']:>5} x[{L['x1']:>4}..{L['x2']:>4}] len={L['len']:>4} gray={L['gray']}")
    for L in sorted(res["v"], key=lambda d: d["x1"]):
        print(f"  V x={L['x1']:>5} y[{L['y1']:>4}..{L['y2']:>4}] len={L['len']:>4} gray={L['gray']}")
    out = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(f"/tmp/lines_{p.stem}.png")
    overlay(p, res, out)
    print("overlay:", out)
