"""Character-local packets, preserving every original field and source link."""
import json,pathlib,unicodedata
root=pathlib.Path(__file__).resolve().parents[1];p=root/'dist';read=lambda n:json.loads((p/n).read_text())
lex=read('lexicon.json');regional=read('regional-data.json');words=read('word-examples.json');components=read('components.json');history=read('history-sources.json');origins=read('origins.json');origin={x['char']:x for x in origins['entries']};memory=read('memory-tips.json')['characters']
chars=set(lex['characters'])|set(regional['characters'])|set(words['characters'])|set(components['characters'])|set(history)|set(origin)
shards=[{}for _ in range(128)]
for c in sorted(chars):
 info=lex['characters'].get(c);related={c,unicodedata.normalize('NFKC',c)}|{v['char'] for v in (info or {}).get('variants',[]) if not v.get('ambiguous')}
 x={}
 for key,source in [('info',lex['characters']),('regional',regional['characters']),('words',words['characters']),('components',components['characters'])]:
  if c in source:x[key]=source[c]
 if c in memory:x['memory']=memory[c]
 x['history']={k:history[k] for k in related if k in history};x['origins']=[origin[k]for k in related if k in origin]
 shards[ord(c)%128][c]=x
out=p/'learning-details';out.mkdir(exist_ok=True)
for i,s in enumerate(shards):(out/f'{i:03d}.json').write_text(json.dumps(s,ensure_ascii=False,separators=(',',':')))
print('learning details bytes',sum(f.stat().st_size for f in out.glob('*.json')),'max packet',max(f.stat().st_size for f in out.glob('*.json')))
