"""Extract translatable BLOCKS (whole paragraph / list item / heading level units)
from the Sphinx single-page HTML dump, keeping inline markup intact so that
context-aware translation is possible.

Output: _i18n/blocks.json  -> list of {id, path, html, text}
Then translator fills _i18n/trans/<n>.json as {"id": "<translated html>"}.
"""
import json, re
from bs4 import BeautifulSoup, NavigableString, Comment

SRC = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/index.html"
OUT = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/_i18n/blocks.json"

BLOCK_TAGS = {"p", "h1", "h2", "h3", "h4", "h5", "h6", "td", "th", "dt", "dd",
              "figcaption", "caption", "li", "blockquote"}
# inline tags that must never be translated / never contain translatable prose
OPAQUE = {"code", "pre", "kbd", "samp", "var", "tt", "img", "script", "style"}
SKIP_CLASS = {"headerlink", "version-switch", "language-switch", "wy-breadcrumbs-aside"}

html = open(SRC, encoding="utf-8").read()
soup = BeautifulSoup(html, "lxml")

blocks = []


def clean_html(node):
    """inner HTML of node with headerlink anchors and comments removed."""
    clone = BeautifulSoup(str(node), "lxml")
    for tag in clone.find_all(True):
        cls = tag.get("class") or []
        if tag.name == "a" and "headerlink" in cls:
            tag.decompose()
    out = clone.decode_contents()
    out = re.sub(r"<!--.*?-->", "", out, flags=re.S)
    return out.strip()


def is_candidate(node):
    if node.name not in BLOCK_TAGS:
        return False
    # li that directly contains p -> the p is the block, li is container
    if node.name == "li" and node.find("p", recursive=False):
        return False
    for p in node.parents:
        if getattr(p, "name", None) in OPAQUE:
            return False
    return True


seen = set()
for node in soup.find_all(True):
    if not is_candidate(node):
        continue
    if node.name == "blockquote" and node.find(["p", "li"]):
        continue  # inner p handled separately
    h = clean_html(node)
    txt = node.get_text(" ", strip=True)
    if not re.search(r"[A-Za-z]", txt):
        continue
    key = (node.name, h)
    path = "/".join([p.get("id") or p.name for p in list(node.parents)[:3][::-1] if getattr(p, "name", None)])
    blocks.append({"tag": node.name, "path": path, "html": h, "text": txt})
    seen.add(key)

# title + sidebar captions / nav labels are handled as generic elements too
extra = []
for t in soup.find_all(["title"]):
    extra.append({"tag": "title", "path": "head", "html": t.get_text(), "text": t.get_text()})
for cap in soup.select("p.caption span.caption-text, nav a, .wy-menu a"):
    txt = cap.get_text(" ", strip=True)
    if not re.search(r"[A-Za-z]", txt):
        continue
    extra.append({"tag": "nav", "path": "nav", "html": clean_html(cap) or txt, "text": txt})

allb = extra + blocks
json.dump(allb, open(OUT, "w", encoding="utf-8"), ensure_ascii=False, indent=1)

uniq = {}
for b in allb:
    uniq.setdefault(b["html"], b)
print("blocks:", len(allb), "unique html:", len(uniq), "unique chars:", sum(len(k) for k in uniq))
from collections import Counter
print(Counter(b["tag"] for b in allb).most_common())