43 lines
1.4 KiB
Python
43 lines
1.4 KiB
Python
"""Assign stable uids to unique translatable templates and slice them into batch files."""
|
|
import json, re, os
|
|
from bs4 import BeautifulSoup
|
|
from serialize import collect, SRC
|
|
|
|
OUTDIR = r"C:/Users/yvmou/Downloads/limboai-readthedocs-io-en-stable/limboai-stable/_i18n"
|
|
|
|
soup = BeautifulSoup(open(SRC, encoding="utf-8").read(), "lxml")
|
|
blocks = collect(soup)
|
|
blocks.append({"kind": "title", "tag": "title",
|
|
"tpl": soup.find("title").get_text(), "markers": {}, "node": None})
|
|
|
|
os.makedirs(OUTDIR + "/batches", exist_ok=True)
|
|
os.makedirs(OUTDIR + "/trans", exist_ok=True)
|
|
|
|
|
|
def key(b):
|
|
return b["tpl"] + "\x00" + json.dumps(b["markers"], sort_keys=True)
|
|
|
|
|
|
uniq = {}
|
|
for b in blocks: # document order preserved
|
|
k = key(b)
|
|
if k not in uniq:
|
|
uniq[k] = {"uid": len(uniq) + 1, "tpl": b["tpl"], "markers": b["markers"]}
|
|
|
|
json.dump({"uniq": uniq},
|
|
open(f"{OUTDIR}/uniq.json", "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
|
|
|
CHARS = 12000
|
|
batch, size, nb = [], 0, 0
|
|
for v in uniq.values():
|
|
batch.append("<%d> %s" % (v["uid"], v["tpl"].replace("\n", " ⏎ ")))
|
|
size += len(v["tpl"]) + 12
|
|
if size >= CHARS:
|
|
nb += 1
|
|
open(f"{OUTDIR}/batches/batch{nb:02d}.txt", "w", encoding="utf-8").write("\n".join(batch))
|
|
batch, size = [], 0
|
|
if batch:
|
|
nb += 1
|
|
open(f"{OUTDIR}/batches/batch{nb:02d}.txt", "w", encoding="utf-8").write("\n".join(batch))
|
|
print("unique:", len(uniq), "batches:", nb)
|