84 lines
3.2 KiB
Python
84 lines
3.2 KiB
Python
"""Extract translatable BLOCKS (whole paragraph / list item / heading level units)
|
|
from the Sphinx single-page HTML dump, keeping inline markup intact so that
|
|
context-aware translation is possible.
|
|
|
|
Output: _i18n/blocks.json -> list of {id, path, html, text}
|
|
Then translator fills _i18n/trans/<n>.json as {"id": "<translated html>"}.
|
|
"""
|
|
import json, re
|
|
from bs4 import BeautifulSoup, NavigableString, Comment
|
|
|
|
SRC = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/index.html"
|
|
OUT = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/_i18n/blocks.json"
|
|
|
|
BLOCK_TAGS = {"p", "h1", "h2", "h3", "h4", "h5", "h6", "td", "th", "dt", "dd",
|
|
"figcaption", "caption", "li", "blockquote"}
|
|
# inline tags that must never be translated / never contain translatable prose
|
|
OPAQUE = {"code", "pre", "kbd", "samp", "var", "tt", "img", "script", "style"}
|
|
SKIP_CLASS = {"headerlink", "version-switch", "language-switch", "wy-breadcrumbs-aside"}
|
|
|
|
html = open(SRC, encoding="utf-8").read()
|
|
soup = BeautifulSoup(html, "lxml")
|
|
|
|
blocks = []
|
|
|
|
|
|
def clean_html(node):
|
|
"""inner HTML of node with headerlink anchors and comments removed."""
|
|
clone = BeautifulSoup(str(node), "lxml")
|
|
for tag in clone.find_all(True):
|
|
cls = tag.get("class") or []
|
|
if tag.name == "a" and "headerlink" in cls:
|
|
tag.decompose()
|
|
out = clone.decode_contents()
|
|
out = re.sub(r"<!--.*?-->", "", out, flags=re.S)
|
|
return out.strip()
|
|
|
|
|
|
def is_candidate(node):
|
|
if node.name not in BLOCK_TAGS:
|
|
return False
|
|
# li that directly contains p -> the p is the block, li is container
|
|
if node.name == "li" and node.find("p", recursive=False):
|
|
return False
|
|
for p in node.parents:
|
|
if getattr(p, "name", None) in OPAQUE:
|
|
return False
|
|
return True
|
|
|
|
|
|
seen = set()
|
|
for node in soup.find_all(True):
|
|
if not is_candidate(node):
|
|
continue
|
|
if node.name == "blockquote" and node.find(["p", "li"]):
|
|
continue # inner p handled separately
|
|
h = clean_html(node)
|
|
txt = node.get_text(" ", strip=True)
|
|
if not re.search(r"[A-Za-z]", txt):
|
|
continue
|
|
key = (node.name, h)
|
|
path = "/".join([p.get("id") or p.name for p in list(node.parents)[:3][::-1] if getattr(p, "name", None)])
|
|
blocks.append({"tag": node.name, "path": path, "html": h, "text": txt})
|
|
seen.add(key)
|
|
|
|
# title + sidebar captions / nav labels are handled as generic elements too
|
|
extra = []
|
|
for t in soup.find_all(["title"]):
|
|
extra.append({"tag": "title", "path": "head", "html": t.get_text(), "text": t.get_text()})
|
|
for cap in soup.select("p.caption span.caption-text, nav a, .wy-menu a"):
|
|
txt = cap.get_text(" ", strip=True)
|
|
if not re.search(r"[A-Za-z]", txt):
|
|
continue
|
|
extra.append({"tag": "nav", "path": "nav", "html": clean_html(cap) or txt, "text": txt})
|
|
|
|
allb = extra + blocks
|
|
json.dump(allb, open(OUT, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
|
|
|
uniq = {}
|
|
for b in allb:
|
|
uniq.setdefault(b["html"], b)
|
|
print("blocks:", len(allb), "unique html:", len(uniq), "unique chars:", sum(len(k) for k in uniq))
|
|
from collections import Counter
|
|
print(Counter(b["tag"] for b in allb).most_common())
|