"""Fediverse 収集 (Misskey / Mastodon の公開ローカルタイムライン、 認証不要、 2026-09-21)。

    python scripts/fediverse.py https://misskey.io --dry --duration 20
    python scripts/fediverse.py https://mstdn.jp --dry --duration 20
    daemon: platform = "fediverse", id = "https://misskey.io" (種別は API を叩いて自動判定)

polling (既定 60 秒) で新着だけ取り、 本文の HTML を剥がして mask → NFKC → SqliteAppender。 投稿者・URL は保存しない。
"""

from __future__ import annotations

import html
import json
import pathlib
import re
import sys
import threading
import time
import urllib.request

from import_youtube_live import SqliteAppender, mask, normalize

ROOT = pathlib.Path(__file__).resolve().parent.parent
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Gecko/20100101 Firefox/120.0"
MENTION_RE = re.compile(r"@[\w.\-]+(?:@[\w.\-]+)?")
EMOJI_RE = re.compile(r":[\w+\-]+:")  # Misskey カスタム絵文字 (:blobcat: 等)


def http(url: str, data: dict | None = None) -> object:
    body = json.dumps(data).encode() if data is not None else None
    req = urllib.request.Request(url, data=body, headers={"User-Agent": UA, "Content-Type": "application/json"})
    with urllib.request.urlopen(req, timeout=30) as r:
        return json.load(r)


def kind(base: str) -> str:
    try:
        http(base + "/api/meta", {"detail": False})
        return "misskey"
    except Exception:
        return "mastodon"


def fetch(base: str, k: str, since: str | None):
    """新着 (id 降順) のテキスト列と最新 id を返す。"""
    if k == "misskey":
        d = {"limit": 100, "withReplies": False, "withFiles": False}
        if since:
            d["sinceId"] = since
        notes = http(base + "/api/notes/local-timeline", d)
        texts = [(n.get("id"), n.get("text") or "") for n in notes if n.get("text")]
    else:
        url = base + "/api/v1/timelines/public?local=true&limit=40" + (f"&since_id={since}" if since else "")
        st = http(url)
        texts = []
        for s in st:
            t = re.sub(r"<br\s*/?>|</p>", "\n", s.get("content") or "", flags=re.I)
            t = html.unescape(re.sub(r"<[^>]+>", "", t))
            texts.append((s.get("id"), t))
    newest = max((i for i, _ in texts if i), default=since)
    return texts, newest


def collect_fediverse(base: str, out_dir: pathlib.Path, genre: str | None = None, stop_event: threading.Event | None = None,
                      dry: bool = False, duration_sec: int = 0, log_tag: str = "", interval: int = 60) -> None:
    prefix = f"[{log_tag}] " if log_tag else f"[fedi {base}] "
    k = kind(base)
    writer = None if dry else SqliteAppender.shared(out_dir / "comments.sqlite")
    deadline = time.time() + duration_sec if duration_sec > 0 else None
    since = None
    count = 0
    print(f"{prefix}[start] kind={k} dry={dry}", flush=True)
    try:
        while True:
            if stop_event is not None and stop_event.is_set():
                break
            if deadline is not None and time.time() >= deadline:
                break
            try:
                texts, since = fetch(base, k, since)
            except Exception as e:
                print(f"{prefix}[err] {e!r}", flush=True)
                texts = []
            for _, text in texts:
                for line in text.split("\n"):
                    body = normalize(mask(EMOJI_RE.sub(" ", MENTION_RE.sub("", line))))
                    if len(body) < 2:
                        continue
                    if dry:
                        if count < 25:
                            print("  ", body[:80], flush=True)
                    else:
                        writer.write(int(time.time() * 1000), body, genre)
                    count += 1
            for _ in range(interval):
                if (stop_event is not None and stop_event.is_set()) or (deadline is not None and time.time() >= deadline):
                    break
                time.sleep(1)
    finally:
        if writer is not None:
            writer.close()
        print(f"{prefix}[done] seen={count}", flush=True)


def main() -> int:
    import argparse

    ap = argparse.ArgumentParser()
    ap.add_argument("base")
    ap.add_argument("--dry", action="store_true")
    ap.add_argument("--duration", type=int, default=0)
    ap.add_argument("--genre", default=None)
    ap.add_argument("--interval", type=int, default=60)
    ap.add_argument("--out-dir", default=str(ROOT / "data"))
    a = ap.parse_args()
    collect_fediverse(a.base.rstrip("/"), pathlib.Path(a.out_dir), a.genre, None, a.dry, a.duration, interval=a.interval)
    return 0


if __name__ == "__main__":
    sys.exit(main())