58 lines
1.8 KiB
Python
58 lines
1.8 KiB
Python
"""Extract translatable text nodes from the Sphinx single-page HTML dump."""
|
|
import json, re, sys
|
|
from bs4 import BeautifulSoup, NavigableString, Comment
|
|
|
|
SRC = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/index.html"
|
|
OUT = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/_i18n/strings.json"
|
|
|
|
SKIP_TAGS = {"script", "style", "code", "pre", "kbd", "samp", "var", "tt"}
|
|
SKIP_CLASS = {"headerlink", "wy-breadcrumbs-aside", "version-switch", "language-switch"}
|
|
|
|
html = open(SRC, encoding="utf-8").read()
|
|
soup = BeautifulSoup(html, "lxml")
|
|
|
|
records = []
|
|
counter = 0
|
|
|
|
|
|
def parent_skipped(node):
|
|
for p in node.parents:
|
|
if getattr(p, "name", None) is None:
|
|
continue
|
|
if p.name in SKIP_TAGS:
|
|
return True
|
|
cls = p.get("class") or []
|
|
if any(c in SKIP_CLASS for c in cls):
|
|
return True
|
|
return False
|
|
|
|
|
|
def has_letter(t):
|
|
return bool(re.search(r"[A-Za-z]", t)) or bool(re.search(r"[\u4e00-\u9fff]", t))
|
|
|
|
|
|
for node in soup.find_all(string=True):
|
|
if isinstance(node, Comment):
|
|
continue
|
|
t = str(node)
|
|
if not has_letter(t):
|
|
continue
|
|
if parent_skipped(node):
|
|
continue
|
|
# keep trailing/leading spaces for reconstruction
|
|
stripped = t.strip()
|
|
if not stripped:
|
|
continue
|
|
counter += 1
|
|
records.append({
|
|
"id": counter,
|
|
"text": t,
|
|
"parents": [p.name for p in node.parents if getattr(p, "name", None)][:4],
|
|
})
|
|
|
|
json.dump(records, open(OUT, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
|
total_chars = sum(len(r["text"]) for r in records)
|
|
print("nodes:", len(records), "chars:", total_chars)
|
|
from collections import Counter
|
|
print(Counter(tuple(r["parents"][:2]) for r in records).most_common(20))
|