docs\limboai-stable

2026-10-15 16:30:44 +08:00
parent ecebf76f47
commit 18e7b2daa8
149 changed files with 97312 additions and 0 deletions
@@ -0,0 +1,42 @@
+"""Assign stable uids to unique translatable templates and slice them into batch files."""
+import json, re, os
+from bs4 import BeautifulSoup
+from serialize import collect, SRC
+
+OUTDIR = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/_i18n"
+
+soup = BeautifulSoup(open(SRC, encoding="utf-8").read(), "lxml")
+blocks = collect(soup)
+blocks.append({"kind": "title", "tag": "title",
+               "tpl": soup.find("title").get_text(), "markers": {}, "node": None})
+
+os.makedirs(OUTDIR + "/batches", exist_ok=True)
+os.makedirs(OUTDIR + "/trans", exist_ok=True)
+
+
+def key(b):
+    return b["tpl"] + "\x00" + json.dumps(b["markers"], sort_keys=True)
+
+
+uniq = {}
+for b in blocks:                      # document order preserved
+    k = key(b)
+    if k not in uniq:
+        uniq[k] = {"uid": len(uniq) + 1, "tpl": b["tpl"], "markers": b["markers"]}
+
+json.dump({"uniq": uniq},
+          open(f"{OUTDIR}/uniq.json", "w", encoding="utf-8"), ensure_ascii=False, indent=1)
+
+CHARS = 12000
+batch, size, nb = [], 0, 0
+for v in uniq.values():
+    batch.append("<%d> %s" % (v["uid"], v["tpl"].replace("\n", " ⏎ ")))
+    size += len(v["tpl"]) + 12
+    if size >= CHARS:
+        nb += 1
+        open(f"{OUTDIR}/batches/batch{nb:02d}.txt", "w", encoding="utf-8").write("\n".join(batch))
+        batch, size = [], 0
+if batch:
+    nb += 1
+    open(f"{OUTDIR}/batches/batch{nb:02d}.txt", "w", encoding="utf-8").write("\n".join(batch))
+print("unique:", len(uniq), "batches:", nb)