121 lines
4.2 KiB
Python
121 lines
4.2 KiB
Python
"""Serialize translatable blocks into compact marker templates.
|
|
|
|
Marker scheme (short => cheap to reproduce while translating):
|
|
<n/> opaque inline element (code / img / kbd / abbr-with-code-text) - kept verbatim
|
|
<l n>..</l> link (<a>): inner text is translatable, anchor markup is restored later
|
|
<b>..</b> <strong>
|
|
<i>..</i> <em>
|
|
everything else inline is transparent (unwrapped) but its text is translatable
|
|
"""
|
|
import json, re, sys
|
|
from bs4 import BeautifulSoup, NavigableString, Comment
|
|
|
|
SRC = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/index.html"
|
|
OUT = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/_i18n/blocks.json"
|
|
|
|
BLOCK_TAGS = {"p", "h1", "h2", "h3", "h4", "h5", "h6", "td", "th", "dt", "dd",
|
|
"figcaption", "caption", "li"}
|
|
OPAQUE = {"code", "kbd", "samp", "var", "tt", "img", "pre", "script", "style", "math"}
|
|
TRANSPARENT = {"span", "cite", "markdown", "mark", "small", "sub", "sup", "u", "s", "q"}
|
|
|
|
|
|
class Ser:
|
|
def __init__(self):
|
|
self.markers = {}
|
|
self.n = 0
|
|
|
|
def add(self, html):
|
|
self.n += 1
|
|
self.markers[str(self.n)] = html
|
|
return self.n
|
|
|
|
def walk(self, node, out):
|
|
for ch in node.children:
|
|
if isinstance(ch, Comment):
|
|
continue
|
|
if isinstance(ch, NavigableString):
|
|
s = re.sub(r"\s+", " ", str(ch))
|
|
if s:
|
|
out.append(s)
|
|
continue
|
|
name = ch.name
|
|
cls = ch.get("class") or []
|
|
if name == "a" and "headerlink" in cls:
|
|
continue
|
|
if name in OPAQUE or name == "abbr":
|
|
out.append("<%d/>" % self.add(str(ch)))
|
|
continue
|
|
if name in TRANSPARENT:
|
|
self.walk(ch, out)
|
|
continue
|
|
if name == "a":
|
|
idx = self.add(str(ch))
|
|
out.append("<l%d>" % idx)
|
|
self.walk(ch, out)
|
|
out.append("</l>")
|
|
continue
|
|
if name == "strong":
|
|
out.append("<b>")
|
|
self.walk(ch, out)
|
|
out.append("</b>")
|
|
continue
|
|
if name in ("em", "dfn"):
|
|
out.append("<i>")
|
|
self.walk(ch, out)
|
|
out.append("</i>")
|
|
continue
|
|
self.walk(ch, out)
|
|
|
|
def run(self, tag):
|
|
out = []
|
|
self.walk(tag, out)
|
|
tpl = "".join(out)
|
|
tpl = re.sub(r"[ \t]*\n[ \t]*", "\n", tpl)
|
|
tpl = re.sub(r"\n{2,}", "\n", tpl).strip()
|
|
return tpl
|
|
|
|
|
|
def serialize(node, kind, feed):
|
|
s = Ser()
|
|
tpl = s.run(node)
|
|
if not re.search(r"[A-Za-z]", re.sub(r"<[^>]*>", "", tpl)):
|
|
return
|
|
feed.append({"kind": kind, "tag": node.name, "tpl": tpl, "markers": s.markers,
|
|
"node": node})
|
|
|
|
|
|
def collect(soup):
|
|
"""Walk the document exactly the way the extractor did."""
|
|
feed = []
|
|
body = soup.find("div", {"role": "main"})
|
|
for node in body.find_all(True):
|
|
if node.name not in BLOCK_TAGS:
|
|
continue
|
|
if node.name == "li" and node.find("p", recursive=False):
|
|
continue
|
|
for p in node.parents:
|
|
if getattr(p, "name", None) in ("pre", "code"):
|
|
break
|
|
else:
|
|
serialize(node, "body", feed)
|
|
for cap in soup.select("p.caption span.caption-text"):
|
|
serialize(cap, "nav", feed)
|
|
for a in soup.select("div.wy-menu a, nav a"):
|
|
if a.find_parent("div", class_="wy-menu") is None:
|
|
continue
|
|
serialize(a, "nav", feed)
|
|
return feed
|
|
|
|
|
|
if __name__ == "__main__":
|
|
blocks = collect(BeautifulSoup(open(SRC, encoding="utf-8").read(), "lxml"))
|
|
plain = [{"kind": b["kind"], "tag": b["tag"], "path": "", "html": b["tpl"],
|
|
"text": re.sub(r"<[^>]*>", "", b["tpl"]), "markers": b["markers"]}
|
|
for b in blocks]
|
|
json.dump(plain, open(OUT, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
|
uniq = {}
|
|
for b in blocks:
|
|
uniq.setdefault(b["tpl"] + "\x00" + json.dumps(b["markers"], sort_keys=True), b)
|
|
print("blocks:", len(blocks), "unique:", len(uniq),
|
|
"unique tpl chars:", sum(len(b["tpl"]) for b in uniq.values()))
|