"""全漢字スイープ: 字ごとに 「その字を含む頻出熟語」 を読ませ、 その字の読み方ごとにまとめる。

誤読は 「同じ字の少数派の読み」 に固まりやすい (上=うえ に落ちた 事実上 など) ので、
字 × 位置 (語頭 / 語末) ごとに読みの分布を出し、 少数派を上に並べたレポートを作る。

    python scripts/kanji_sweep.py loop                  # 常駐: 全字を頻度順に回し続ける (収集機)
    python scripts/kanji_sweep.py run 上 下 後          # 字を指定して 1 回
    python scripts/kanji_sweep.py done 上 下            # 処理済みにする (ledger に語句を登録)
    python scripts/kanji_sweep.py status

- レポート: $SWEEP_OUT/<字>.md と index.md (少数派の語数が多い字から並ぶ)
- loop は 1 周ごとに辞書を git pull し、 新しいバックアップがあればコーパスを作り直す
- 「試した語句」 の累計 (probe_ledger) には、 人が処理した字 (done) の語句だけを入れる
"""
from __future__ import annotations

import argparse
import collections
import datetime as dt
import json
import os
import pathlib
import re
import sqlite3
import subprocess
import sys
import time

sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
import probe_ledger as pl  # noqa: E402

ROOT = pathlib.Path(__file__).resolve().parent.parent
CORPUS = pathlib.Path(os.environ.get("SWEEP_CORPUS", ROOT / "data" / "snapshot" / "corpus.tsv"))
OUT = pathlib.Path(os.environ.get("SWEEP_OUT", ROOT / "data" / "sweep"))
BACKUPS = pathlib.Path(
    os.environ.get("SWEEP_BACKUPS", pathlib.Path.home() / "backups" / "stream-comments" / "daily")
)
KANJI = r"[一-鿿㐀-䶿々]"
MIN_COUNT = 3
MAX_WORDS = 400
CHUNK = 20
NL = "\n"
TAB = "\t"


def log(msg: str) -> None:
    print(f"[sweep] {msg}", flush=True)


def kata2hira(s: str) -> str:
    return "".join(chr(ord(c) - 0x60) if "ァ" <= c <= "ヶ" else c for c in s)


def weight(count: int) -> int:
    """出現回数の重み。 同じコメントの連投 (草 / メンバー名コール) やジャンルの流量差で
    順位が決まらないよう、 回数をそのまま使わず 1 + floor(log2(回数)) に丸める
    (1 回 = 1、 2〜3 回 = 2、 1,000 回 ≈ 10)。 ★2026-09-15 コーパス偏り是正"""
    return max(1, count).bit_length()


def load_corpus():
    with CORPUS.open(encoding="utf-8") as f:
        for line in f:
            n, _, body = line.rstrip(NL).partition(TAB)
            yield weight(int(n or 1)), body


def kanji_freq() -> collections.Counter:
    c: collections.Counter = collections.Counter()
    for n, body in load_corpus():
        for ch in re.findall(KANJI, body):
            c[ch] += n
    return c


HIRA = r"[ぁ-ゖ]"
POSITIONS = ("語末", "語頭", "送り仮名")


def collect_words(targets: set[str]) -> dict:
    """字 → 位置 → 語 → 出現数。

    - 語末 / 語頭: 前後が漢字でない 2〜4 字の漢字列
    - 送り仮名: 前が漢字でない 1〜3 字の漢字列 + 直後のひらがな 1〜3 字 (効率上がって / 見上げる)。
      語は 「漢字列|ひらがな」 の形で持つ (読みの切り出しで境目が要るため)
    """
    words: dict = collections.defaultdict(collections.Counter)
    pat = re.compile(rf"(?<!{KANJI})({KANJI}{{2,4}})(?!{KANJI})")
    okuri = re.compile(rf"(?<!{KANJI})({KANJI}{{1,3}})({HIRA}{{1,3}})")
    for n, body in load_corpus():
        for m in pat.finditer(body):
            w = m.group(1)
            if w[-1] in targets:
                words[(w[-1], "語末")][w] += n
            if w[0] in targets and w[0] != w[-1]:
                words[(w[0], "語頭")][w] += n
        for m in okuri.finditer(body):
            k, h = m.group(1), m.group(2)
            if k[-1] in targets:
                words[(k[-1], "送り仮名")][f"{k}|{h}"] += n
    return words


def read(items: list[str]) -> list[str]:
    return subprocess.run(
        [str(pl.BIN), "lookup", "--mode", "hiragana", *pl.dict_args(pl.DICT), "--batch"],
        input=NL.join(items) + NL,
        capture_output=True,
        text=True,
        encoding="utf-8",
        check=True,
    ).stdout.splitlines()


def char_reading(reading: str, rest_reading: str, pos: str, okuri: str = "") -> str:
    """語の読みから、 対象字の読み部分を取り出す (残りの字だけの読みとの差分)。"""
    if pos == "語末" and reading.startswith(rest_reading):
        return reading[len(rest_reading):] or "?"
    if pos == "語頭" and reading.endswith(rest_reading):
        return reading[: len(reading) - len(rest_reading)] or "?"
    if pos == "送り仮名" and reading.startswith(rest_reading) and reading.endswith(okuri):
        core = reading[len(rest_reading): len(reading) - len(okuri)]
        return core or "(送り仮名と一体)"
    return "(一体読み)"


def split_items(ws: list[str], pos: str) -> tuple[list[str], list[str], list[str]]:
    """(読ませる表記, 対象字を除いた残りの表記, 送り仮名) を返す。"""
    if pos == "送り仮名":
        full, rest, ok = [], [], []
        for w in ws:
            k, h = w.split("|")
            full.append(k + h)
            rest.append(k[:-1])
            ok.append(h)
        return full, rest, ok
    rests = [w[:-1] if pos == "語末" else w[1:] for w in ws]
    return ws, rests, [""] * len(ws)


def read_rest(rests: list[str]) -> list[str]:
    """残りの表記を読む。 空文字 (1 字 + 送り仮名の語) は読ませずに空の読みを返す。"""
    idx = [i for i, r in enumerate(rests) if r]
    out = [""] * len(rests)
    if idx:
        for i, r in zip(idx, read([rests[i] for i in idx])):
            out[i] = r
    return out


def sweep(chars: list[str]) -> dict[str, dict]:
    """レポートを書き、 字ごとの {words: [...], minority: 少数派の語数} を返す。"""
    OUT.mkdir(parents=True, exist_ok=True)
    words = collect_words(set(chars))
    result: dict[str, dict] = {}
    for ch in chars:
        lines = [f"# 「{ch}」 スイープ ({dt.datetime.now():%Y-%m-%d %H:%M})", ""]
        all_words: list[str] = []
        minority = 0
        for pos in POSITIONS:
            cnt = words.get((ch, pos), collections.Counter())
            top = [(w, n) for w, n in cnt.most_common(MAX_WORDS) if n >= MIN_COUNT]
            if not top:
                continue
            full, rests, okuris = split_items([w for w, _ in top], pos)
            rd, rr = read(full), read_rest(rests)
            groups: dict = collections.defaultdict(list)
            for (_, n), f, r, rest_r, ok in zip(top, full, rd, rr, okuris):
                key = char_reading(kata2hira(r), kata2hira(rest_r), pos, ok)
                groups[key].append((f, r, n))
            ws = full
            all_words += ws
            tot = sum(n for _, n in top)
            for key, members in groups.items():
                if not key.startswith("(") and sum(m[2] for m in members) < tot * 0.1:
                    minority += len(members)
            lines += [f"## {pos}の「{ch}」 ({len(ws)} 語)", ""]
            for key, members in sorted(groups.items(), key=lambda kv: sum(m[2] for m in kv[1])):
                s = sum(m[2] for m in members)
                ex = " / ".join(
                    f"{w}={r}({n})" for w, r, n in sorted(members, key=lambda m: -m[2])[:40]
                )
                lines.append(f"- **{key}** — {len(members)} 語 / 重み {s:,}: {ex}")
            lines.append("")
        (OUT / f"{ch}.md").write_text(NL.join(lines), encoding="utf-8")
        result[ch] = {"words": all_words, "minority": minority}
    return result


COUNTER_NUMS = ("1", "3", "6", "8", "10", "100", "何")


def counter_report() -> None:
    """数字 + 漢字 (助数詞) を集め、 読みが変わりやすい数で読ませた一覧 counters.md を書く。

    数字は漢字スイープに乗らないので別に回す。 1 / 6 / 8 / 10 / 100 は促音、 3 / 何 は連濁が出やすい。
    """
    pat = re.compile(rf"[0-9０-９]+({KANJI}{{1,3}})")
    cnt: collections.Counter = collections.Counter()
    for n, body in load_corpus():
        for m in pat.finditer(body):
            cnt[m.group(1)] += n
    top = [(c, n) for c, n in cnt.most_common(MAX_WORDS * 2) if n >= MIN_COUNT]
    items = [num + c for c, _ in top for num in COUNTER_NUMS]
    rd = read(items)
    lines = [
        f"# 数字 + 助数詞 ({dt.datetime.now():%Y-%m-%d %H:%M}、 {len(top)} 種)",
        "",
        "数字に付く漢字を出現順に、 " + " / ".join(COUNTER_NUMS) + " で読ませたもの。",
        "",
        "| 助数詞 | 出現 | " + " | ".join(COUNTER_NUMS) + " |",
        "|---|---:|" + "---|" * len(COUNTER_NUMS),
    ]
    k = len(COUNTER_NUMS)
    for i, (c, n) in enumerate(top):
        rs = [kata2hira(r) for r in rd[i * k: (i + 1) * k]]
        lines.append(f"| {c} | {n:,} | " + " | ".join(rs) + " |")
    (OUT / "counters.md").write_text(NL.join(lines) + NL, encoding="utf-8")
    log(f"counters.md: {len(top)} 種")


def state_path() -> pathlib.Path:
    return OUT / "state.json"


def load_done() -> set[str]:
    dp = OUT / "done.txt"
    return set(dp.read_text(encoding="utf-8").split()) if dp.exists() else set()


def write_index(rank: dict[str, int], state: dict) -> None:
    done = load_done()
    rows = sorted(
        state["chars"].items(), key=lambda kv: (-kv[1]["minority"], rank.get(kv[0], 10**9))
    )
    idx = [
        f"# 全漢字スイープ候補 (更新 {dt.datetime.now():%Y-%m-%d %H:%M})",
        "",
        f"周回 {state['cycle']} / この周の進捗 {state['pos']} / {len(rank)} 字。 "
        "少数派 (その字の読みのうち重み付き 10% 未満、 重み = 1+log2(出現回数)) に入った語数の多い順。 済 = 処理済み。",
        "",
        "| 字 | 頻度順位 | 語数 | 少数派の語数 | 更新 | 状態 |",
        "|---|---:|---:|---:|---|---|",
    ]
    for ch, v in rows:
        mark = "済" if ch in done else ""
        idx.append(
            f"| [{ch}]({ch}.md) | {rank.get(ch, 0) + 1} | {v['n']} | {v['minority']} | {v['at']} | {mark} |"
        )
    (OUT / "index.md").write_text(NL.join(idx) + NL, encoding="utf-8")


def refresh_corpus() -> bool:
    """最新の日次バックアップが今のコーパスより新しければ TSV を作り直す。"""
    snaps = sorted(BACKUPS.glob("comments-*.sqlite.zst"))
    if not snaps:
        return False
    latest = snaps[-1]
    if CORPUS.exists() and CORPUS.stat().st_mtime >= latest.stat().st_mtime:
        return False
    tmpdb = CORPUS.with_suffix(".sqlite.tmp")
    subprocess.run(["zstd", "-q", "-d", "-f", str(latest), "-o", str(tmpdb)], check=True)
    tmp = CORPUS.with_suffix(".tsv.tmp")
    db = sqlite3.connect(str(tmpdb))
    with tmp.open("w", encoding="utf-8") as f:
        for b, n in db.execute("select b, count from comments"):
            f.write(str(n) + TAB + b.replace(NL, " ").replace(TAB, " ") + NL)
    db.close()
    tmpdb.unlink()
    tmp.replace(CORPUS)
    log(f"corpus refreshed from {latest.name}")
    return True


def git_pull() -> None:
    r = subprocess.run(
        ["git", "-C", str(pl.DICT), "pull", "-q", "--ff-only"], capture_output=True, text=True
    )
    if r.returncode != 0:
        log(f"git pull failed: {r.stderr.strip()[:200]}")


def loop() -> None:
    OUT.mkdir(parents=True, exist_ok=True)
    CORPUS.parent.mkdir(parents=True, exist_ok=True)
    sp = state_path()
    state = (
        json.loads(sp.read_text(encoding="utf-8")) if sp.exists() else {"cycle": 1, "pos": 0, "chars": {}}
    )
    pause = float(os.environ.get("SWEEP_PAUSE", "30"))
    while True:
        if state["pos"] == 0:
            git_pull()
            refresh_corpus()
            counter_report()
        freq = kanji_freq()
        order = [ch for ch, n in freq.most_common() if n >= MIN_COUNT and ch != "々"]
        rank = {ch: i for i, ch in enumerate(order)}
        chunk = order[state["pos"]: state["pos"] + CHUNK]
        if not chunk:
            log(f"cycle {state['cycle']} finished")
            state["cycle"] += 1
            state["pos"] = 0
            sp.write_text(json.dumps(state, ensure_ascii=False), encoding="utf-8")
            continue
        t = time.monotonic()
        res = sweep(chunk)
        now = f"{dt.datetime.now():%m-%d %H:%M}"
        for ch, v in res.items():
            state["chars"][ch] = {"n": len(v["words"]), "minority": v["minority"], "at": now}
        state["pos"] += len(chunk)
        sp.write_text(json.dumps(state, ensure_ascii=False), encoding="utf-8")
        write_index(rank, state)
        log(
            f"cycle {state['cycle']} {state['pos']}/{len(order)} ({''.join(chunk)}) "
            f"in {time.monotonic() - t:.0f}s"
        )
        time.sleep(pause)


def mark_done(chars: list[str]) -> None:
    """処理した字を済にし、 その字のレポートに載っていた語句を ledger に登録する。"""
    con = pl.connect()
    res = sweep(chars)
    for ch, v in res.items():
        n_in, n_new = pl.register(con, "terms", v["words"], f"sweep:{ch}")
        pl.upsert_round(con, f"sweep:{ch}", terms_in=n_in, terms_new=n_new)
        log(f"{ch}: done ({n_in} 語、 新規 {n_new})")
    con.commit()
    dp = OUT / "done.txt"
    dp.write_text(NL.join(sorted(load_done() | set(chars))) + NL, encoding="utf-8")


def main() -> None:
    for s in (sys.stdout, sys.stderr):
        s.reconfigure(encoding="utf-8")
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    sub = ap.add_subparsers(dest="cmd", required=True)
    sub.add_parser("loop")
    a = sub.add_parser("run")
    a.add_argument("chars", nargs="+")
    a = sub.add_parser("done")
    a.add_argument("chars", nargs="+")
    sub.add_parser("status")
    sub.add_parser("counters")
    ns = ap.parse_args()
    if ns.cmd == "counters":
        counter_report()
        return
    if ns.cmd == "loop":
        loop()
    elif ns.cmd == "run":
        for ch in sweep(ns.chars):
            log(f"{ch} → {OUT / (ch + '.md')}")
    elif ns.cmd == "done":
        mark_done(ns.chars)
    else:
        sp = state_path()
        st = json.loads(sp.read_text(encoding="utf-8")) if sp.exists() else {}
        print(
            json.dumps({k: st.get(k) for k in ("cycle", "pos")}, ensure_ascii=False),
            "reports:",
            len(list(OUT.glob("*.md"))),
            "done:",
            len(load_done()),
        )


if __name__ == "__main__":
    main()