import sys, re, json
sys.stdout.reconfigure(encoding='utf-8')

def kata(s):
    return ''.join(chr(ord(c)+0x60) if 'ぁ'<=c<='ゖ' else c for c in s)

with open(r'N:\stream-comments\data\_analysis\kanji_readings.json', encoding='utf-8') as fh:
    KR = {k: set(v) for k, v in json.load(fh).items()}

VOICE = {'カ':'ガ','キ':'ギ','ク':'グ','ケ':'ゲ','コ':'ゴ','サ':'ザ','シ':'ジ','ス':'ズ',
 'セ':'ゼ','ソ':'ゾ','タ':'ダ','チ':'ヂ','ツ':'ヅ','テ':'デ','ト':'ド',
 'ハ':'バ','ヒ':'ビ','フ':'ブ','ヘ':'ベ','ホ':'ボ'}
HANDAKU = {'ハ':'パ','ヒ':'ピ','フ':'プ','ヘ':'ペ','ホ':'ポ'}

def variants(r, first):
    outs = {r}
    if not first:
        c0 = r[0]
        if c0 in VOICE: outs.add(VOICE[c0] + r[1:])
        if c0 in HANDAKU: outs.add(HANDAKU[c0] + r[1:])
    exp = set()
    for o in outs:
        if len(o) >= 2: exp.add(o[:-1] + 'ッ')
    return outs | exp

def is_kanji(ch):
    return '一' <= ch <= '鿿' or '㐀' <= ch <= '䶿'

def can_segment(token, reading):
    reading = kata(reading)
    m = len(reading)
    reach = {0}
    kanji_seen = 0
    for ch in token:
        nxt = set()
        if is_kanji(ch):
            rs = KR.get(ch)
            if not rs:
                return None
            first = (kanji_seen == 0)
            for p in reach:
                for base in rs:
                    for v in variants(base, first):
                        if reading.startswith(v, p):
                            nxt.add(p + len(v))
            kanji_seen += 1
        elif ch == '々':  # 々
            return None
        else:
            cc = kata(ch)
            for p in reach:
                if reading.startswith(cc, p):
                    nxt.add(p + len(cc))
        if not nxt:
            return False
        reach = nxt
    return m in reach

allowed = re.compile(r'^[ぁ-ゟ゠-ヿ一-鿿㐀-䶿]+$')
flagged = []
skipped = 0
with open(r'N:\stream-comments\data\_analysis\token_readings.tsv', encoding='utf-8') as f:
    for line in f:
        p = line.rstrip('\n').split('\t')
        if len(p) < 3 or not p[0].isdigit():
            continue
        cnt = int(p[0]); tok = p[1]; rd = p[2]
        if cnt < 8:
            break
        if not allowed.match(tok):
            continue
        if not any(is_kanji(c) for c in tok):
            continue
        res = can_segment(tok, rd)
        if res is None:
            skipped += 1
            continue
        if res is False:
            flagged.append((cnt, tok, rd))

flagged.sort(key=lambda x: -x[0])
print("unexplained readings (cnt>=8): %d  (skipped undecidable: %d)" % (len(flagged), skipped))
for cnt, tok, rd in flagged[:120]:
    print("  %4d  %s  ->  %s" % (cnt, tok, rd))