Files
web-Iot/tmp_tokens.cjs
2026-08-14 09:02:52 +08:00

13 lines
462 B
JavaScript

const a = require('./tmp_missing.json');
const z = a.zhAdd;
const tokens = new Set();
for (const ns of Object.keys(z)) {
for (const leaf of Object.keys(z[ns])) {
const words = leaf.replace(/([a-z])([A-Z])/g, '$1 $2').replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2').split(/[\s._]+/).filter(Boolean);
for (const w of words) tokens.add(w.toLowerCase());
}
}
const arr = [...tokens].sort();
console.log('Token count:', arr.length);
console.log(arr.join(' '));