docs\limboai-stable

2026-10-15 16:30:44 +08:00
parent ecebf76f47
commit 18e7b2daa8
149 changed files with 97312 additions and 0 deletions
@@ -0,0 +1,120 @@
+"""Serialize translatable blocks into compact marker templates.
+
+Marker scheme (short => cheap to reproduce while translating):
+   <n/>       opaque inline element (code / img / kbd / abbr-with-code-text) - kept verbatim
+   <l n>..</l> link (<a>): inner text is translatable, anchor markup is restored later
+   <b>..</b>   <strong>
+   <i>..</i>   <em>
+   everything else inline is transparent (unwrapped) but its text is translatable
+"""
+import json, re, sys
+from bs4 import BeautifulSoup, NavigableString, Comment
+
+SRC = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/index.html"
+OUT = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/_i18n/blocks.json"
+
+BLOCK_TAGS = {"p", "h1", "h2", "h3", "h4", "h5", "h6", "td", "th", "dt", "dd",
+              "figcaption", "caption", "li"}
+OPAQUE = {"code", "kbd", "samp", "var", "tt", "img", "pre", "script", "style", "math"}
+TRANSPARENT = {"span", "cite", "markdown", "mark", "small", "sub", "sup", "u", "s", "q"}
+
+
+class Ser:
+    def __init__(self):
+        self.markers = {}
+        self.n = 0
+
+    def add(self, html):
+        self.n += 1
+        self.markers[str(self.n)] = html
+        return self.n
+
+    def walk(self, node, out):
+        for ch in node.children:
+            if isinstance(ch, Comment):
+                continue
+            if isinstance(ch, NavigableString):
+                s = re.sub(r"\s+", " ", str(ch))
+                if s:
+                    out.append(s)
+                continue
+            name = ch.name
+            cls = ch.get("class") or []
+            if name == "a" and "headerlink" in cls:
+                continue
+            if name in OPAQUE or name == "abbr":
+                out.append("<%d/>" % self.add(str(ch)))
+                continue
+            if name in TRANSPARENT:
+                self.walk(ch, out)
+                continue
+            if name == "a":
+                idx = self.add(str(ch))
+                out.append("<l%d>" % idx)
+                self.walk(ch, out)
+                out.append("</l>")
+                continue
+            if name == "strong":
+                out.append("<b>")
+                self.walk(ch, out)
+                out.append("</b>")
+                continue
+            if name in ("em", "dfn"):
+                out.append("<i>")
+                self.walk(ch, out)
+                out.append("</i>")
+                continue
+            self.walk(ch, out)
+
+    def run(self, tag):
+        out = []
+        self.walk(tag, out)
+        tpl = "".join(out)
+        tpl = re.sub(r"[ \t]*\n[ \t]*", "\n", tpl)
+        tpl = re.sub(r"\n{2,}", "\n", tpl).strip()
+        return tpl
+
+
+def serialize(node, kind, feed):
+    s = Ser()
+    tpl = s.run(node)
+    if not re.search(r"[A-Za-z]", re.sub(r"<[^>]*>", "", tpl)):
+        return
+    feed.append({"kind": kind, "tag": node.name, "tpl": tpl, "markers": s.markers,
+                 "node": node})
+
+
+def collect(soup):
+    """Walk the document exactly the way the extractor did."""
+    feed = []
+    body = soup.find("div", {"role": "main"})
+    for node in body.find_all(True):
+        if node.name not in BLOCK_TAGS:
+            continue
+        if node.name == "li" and node.find("p", recursive=False):
+            continue
+        for p in node.parents:
+            if getattr(p, "name", None) in ("pre", "code"):
+                break
+        else:
+            serialize(node, "body", feed)
+    for cap in soup.select("p.caption span.caption-text"):
+        serialize(cap, "nav", feed)
+    for a in soup.select("div.wy-menu a, nav a"):
+        if a.find_parent("div", class_="wy-menu") is None:
+            continue
+        serialize(a, "nav", feed)
+    return feed
+
+
+if __name__ == "__main__":
+    blocks = collect(BeautifulSoup(open(SRC, encoding="utf-8").read(), "lxml"))
+    plain = [{"kind": b["kind"], "tag": b["tag"], "path": "", "html": b["tpl"],
+              "text": re.sub(r"<[^>]*>", "", b["tpl"]), "markers": b["markers"]}
+             for b in blocks]
+    json.dump(plain, open(OUT, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
+    uniq = {}
+    for b in blocks:
+        uniq.setdefault(b["tpl"] + "\x00" + json.dumps(b["markers"], sort_keys=True), b)
+    print("blocks:", len(blocks), "unique:", len(uniq),
+          "unique tpl chars:", sum(len(b["tpl"]) for b in uniq.values()))