13 lines
462 B
JavaScript
13 lines
462 B
JavaScript
const a = require('./tmp_missing.json');
|
|
const z = a.zhAdd;
|
|
const tokens = new Set();
|
|
for (const ns of Object.keys(z)) {
|
|
for (const leaf of Object.keys(z[ns])) {
|
|
const words = leaf.replace(/([a-z])([A-Z])/g, '$1 $2').replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2').split(/[\s._]+/).filter(Boolean);
|
|
for (const w of words) tokens.add(w.toLowerCase());
|
|
}
|
|
}
|
|
const arr = [...tokens].sort();
|
|
console.log('Token count:', arr.length);
|
|
console.log(arr.join(' '));
|