"""Generic analysis for a comparison post. Needs plan.json and a mapping.py next to it.

  python3 analyze.py --plan plan.json --peek     # top-level keys + one sample per actor (write mapping.py from this)
  python3 analyze.py --plan plan.json            # data.json + report

mapping.py must define canon(family: str, item: dict) -> dict with the plan's canonical keys
(missing/empty -> None). Optional keys it may set: "_filler" (bool, billed non-item rows),
"_repost_original_len" (int, length of the original text attached to a repost).
"""
import argparse, datetime as dt, importlib.util, json, statistics, sys
from collections import Counter, defaultdict
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from common import load_plan


def parse_dt(s):
    if not s:
        return None
    for fmt in ("%a %b %d %H:%M:%S %z %Y",):
        try:
            return dt.datetime.strptime(s, fmt)
        except ValueError:
            pass
    try:
        return dt.datetime.fromisoformat(str(s).replace("Z", "+00:00"))
    except (ValueError, TypeError):
        return None


def fill(vals):
    nonnull = [v for v in vals if v is not None and v != ""]
    return round(100 * len(nonnull) / len(vals), 1) if vals and nonnull else None


def load(plan, slug, task):
    d = plan["_raw"] / slug.replace("/", "__")
    p = d / f"{task}.items.json"
    if not p.exists():
        return None
    items = json.loads(p.read_text())
    run = json.loads((d / f"{task}.run.json").read_text()) if (d / f"{task}.run.json").exists() else {}
    inp = json.loads((d / f"{task}.input.json").read_text()) if (d / f"{task}.input.json").exists() else {}
    return items, run, inp


def run_facts(plan, slug, task, items, run, rec):
    stats = run.get("stats") or {}
    ev = (((run.get("pricingInfo") or {}).get("pricingPerEvent") or {}).get("actorChargeEvents") or {})
    listed = None
    for e in ev.values():
        if e.get("isPrimaryEvent") or listed is None:
            listed = e.get("eventPriceUsd")
    n = len(items); usd = run.get("usageTotalUsd")
    s, f = parse_dt(run.get("startedAt")), parse_dt(run.get("finishedAt"))
    return {"slug": slug, "task": task, "run_id": run.get("id"), "status": run.get("status"), "status_message": run.get("statusMessage"),
            "build": run.get("buildNumber"), "requested": plan["requested"].get(task), "n_items": n, "started_at": run.get("startedAt"),
            "finished_at": run.get("finishedAt"), "duration_s": round((f - s).total_seconds(), 1) if s and f else None, "wall_s": rec.get("wall_s"),
            "usd_total": usd, "usd_per_1k": round(usd / n * 1000, 4) if usd is not None and n else None,
            "listed_price_per_item": listed, "charged_events": run.get("chargedEventCounts"), "billing_model": run.get("platformUsageBillingModel"),
            "platform_usage_usd": sum((run.get("usageUsd") or {}).values()) if run.get("usageUsd") else None,
            "memory_mb": (run.get("options") or {}).get("memoryMbytes"), "compute_units": stats.get("computeUnits"),
            "bytes_per_item": round(len(json.dumps(items, ensure_ascii=False).encode()) / n) if n else None,
            "keys_per_item": round(statistics.mean(len(i) for i in items), 1) if n else None,
            "key_union": len({k for i in items for k in i}) if n else None, "input": rec.get("input")}


def item_facts(plan, kind, cs):
    canon = plan["canon"]; ids = [c["id"] for c in cs if c["id"]]; dts = [c["_dt"] for c in cs if c["_dt"]]
    ordered = None
    if len(dts) > 1:
        pairs = list(zip(dts, dts[1:])); ordered = round(100 * sum(1 for a, b in pairs if a >= b) / len(pairs), 1)
    f = {"n": len(cs), "unique_ids": len(set(ids)), "dups": len(ids) - len(set(ids)),
         "newest": max(dts).isoformat() if dts else None, "oldest": min(dts).isoformat() if dts else None,
         "span_hours": round((max(dts) - min(dts)).total_seconds() / 3600, 1) if dts else None, "newest_first_pct": ordered,
         "text_max": max((c["_len"] for c in cs), default=0), "text_mean": round(statistics.mean(c["_len"] for c in cs), 1) if cs else None,
         "text_over_long": sum(1 for c in cs if c["_len"] > plan.get("canon_text_long", 300)),
         "text_ellipsis": sum(1 for c in cs if (c["text"] or "").rstrip().endswith("…")),
         "repost_by_text": sum(1 for c in cs if c["_repost_text"]),
         "repost_truncated": sum(1 for c in cs if c["_repost_text"] and (c["text"] or "").rstrip().endswith("…")),
         "repost_with_original": sum(1 for c in cs if c["_repost_text"] and c.get("_repost_original_len", 0) > 0),
         "reply_flagged": sum(1 for c in cs if c["is_reply"]) if any(c["is_reply"] is not None for c in cs) else None,
         "reply_by_text": sum(1 for c in cs if (c["text"] or "").startswith("@")),
         "media_items": sum(1 for c in cs if c["media"]) if any(c["media"] is not None for c in cs) else None,
         "filler": sum(1 for c in cs if c.get("_filler")), "real_items": sum(1 for c in cs if not c.get("_filler")),
         "langs": Counter(c["lang"] for c in cs if c["lang"]).most_common(4), "fill": {k: fill([c[k] for c in cs]) for k in canon}}
    if kind == "search" and cs:
        q = plan["query"].lower(); words = q.split()
        f["query_phrase_pct"] = round(100 * sum(1 for c in cs if q in (c["text"] or "").lower()) / len(cs), 1)
        f["query_words_pct"] = round(100 * sum(1 for c in cs if all(w in (c["text"] or "").lower() for w in words)) / len(cs), 1)
    if kind == "timeline" and cs:
        f["author_match_pct"] = round(100 * sum(1 for c in cs if (c["author_username"] or "").lower() == plan["handle"].lower()) / len(cs), 1)
    return f


def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--plan", required=True); ap.add_argument("--peek", action="store_true"); a = ap.parse_args()
    plan = load_plan(a.plan); order = list(plan["actors"]); kinds = plan["kinds"]
    if a.peek:
        task = next(t for t in plan["tasks"] if kinds.get(t) == "search")
        for slug in order:
            got = load(plan, slug, task)
            if not got: print(f"== {slug}: no {task} items"); continue
            items = got[0]; it = items[0] if items else {}
            print(f"\n== {slug} ({len(items)} items; keys/item≈{round(statistics.mean(len(i) for i in items),1) if items else 0})")
            print("keys:", list(it.keys())[:70]); print("sample:", json.dumps(it, ensure_ascii=False)[:600])
        return
    spec = importlib.util.spec_from_file_location("mapping", plan["_plan_dir"] / "mapping.py"); mapping = importlib.util.module_from_spec(spec); spec.loader.exec_module(mapping)
    manifest = {}
    for line in (plan["_raw"] / "manifest.jsonl").read_text().splitlines():
        r = json.loads(line); manifest[(r["slug"], r["task"])] = r
    runs, canon_by = {}, {}
    for slug in order:
        fam = plan["actors"][slug]["family"]
        for task in plan["tasks"]:
            got = load(plan, slug, task)
            if not got: continue
            items, run, inp = got; rec = dict(manifest.get((slug, task), {})); rec["input"] = inp
            cs = []
            for it in items:
                c = mapping.canon(fam, it)
                for k in plan["canon"]: c.setdefault(k, None)
                if c["id"] is not None: c["id"] = str(c["id"])
                c["_dt"] = parse_dt(c["created_at"]); c["_len"] = len(c["text"] or ""); c["_repost_text"] = bool((c["text"] or "").startswith("RT @"))
                cs.append(c)
            canon_by[(slug, task)] = cs
            runs[(slug, task)] = run_facts(plan, slug, task, items, run, rec); runs[(slug, task)]["items"] = item_facts(plan, kinds.get(task), cs)
    cross = {}
    for task in [t for t in plan["tasks"] if kinds.get(t) in ("search", "timeline")]:
        sets = {s: {c["id"] for c in canon_by.get((s, task), []) if c["id"]} for s in order if (s, task) in canon_by}
        if not sets: continue
        union = set().union(*sets.values()); seen_by = Counter(i for v in sets.values() for i in v)
        matrix = {x: {y: len(sets[x] & sets[y]) for y in sets} for x in sets}
        vals = defaultdict(lambda: defaultdict(dict))
        for s in sets:
            for c in canon_by[(s, task)]:
                if c["id"] and seen_by[c["id"]] >= 2:
                    for m in ("views", "likes", "reposts", "replies"):
                        if c[m] is not None: vals[c["id"]][m][s] = c[m]
        agree = {}
        for m in ("views", "likes", "reposts", "replies"):
            tot = sum(1 for i in vals if len(vals[i][m]) >= 2); same = sum(1 for i in vals if len(vals[i][m]) >= 2 and len(set(vals[i][m].values())) == 1)
            agree[m] = {"ids": tot, "identical": same}
        lens = defaultdict(dict)
        for s in sets:
            for c in canon_by[(s, task)]:
                if c["id"]: lens[c["id"]][s] = c["_len"]
        long_ids = {i for i in lens if max(lens[i].values()) > plan.get("canon_text_long", 300)}
        longform = {}
        for s in sets:
            shared = [i for i in long_ids if s in lens[i]]; full = [i for i in shared if lens[i][s] >= 0.95 * max(lens[i].values())]
            longform[s] = {"long_ids_seen": len(shared), "full_text": len(full), "pct": round(100 * len(full) / len(shared), 1) if shared else None}
        cross[task] = {"union": len(union), "per_actor": {s: len(v) for s, v in sets.items()}, "in_all": sum(1 for k in seen_by.values() if k == len(sets)),
                       "only_one": sum(1 for k in seen_by.values() if k == 1), "matrix": matrix, "metric_agreement": agree, "longform_capture": longform}
    lookups = {}
    for task in [t for t in plan["tasks"] if kinds.get(t) == "lookup"]:
        rows = {}
        for s in order:
            cs = canon_by.get((s, task))
            if cs is None: continue
            real = [c for c in cs if not c.get("_filler")]; c = real[0] if real else None
            rows[s] = None if c is None else {k: c[k] for k in ("id", "text", "likes", "reposts", "replies", "quotes", "views", "bookmarks", "created_at", "lang", "author_username", "is_long")} | \
                {"text_len": c["_len"], "n_items": len(cs), "filler": sum(1 for x in cs if x.get("_filler")), "media_n": len(c["media"]) if c["media"] else (0 if c["media"] is not None else None)}
        lookups[task] = rows
    data = {"generated_at": dt.datetime.now(dt.timezone.utc).isoformat(), "plan": {k: v for k, v in plan.items() if not k.startswith("_")},
            "order": order, "runs": {f"{s}|{t}": v for (s, t), v in runs.items()}, "cross": cross, "lookups": lookups}
    (plan["_out"] / "data.json").write_text(json.dumps(data, indent=1, ensure_ascii=False, default=str))
    # ---- report
    for task in plan["tasks"]:
        print(f"\n=== {task}")
        print(f"{'actor':46} {'n':>4} {'dur':>6} {'$total':>8} {'$/1k':>7} {'applied':>8} {'bill':>5} {'B/it':>6} {'keys':>5} {'dup':>3} {'new1st':>6} {'txtmax':>6} {'long':>4} {'…':>3} {'RP':>3} {'@re':>4} {'media':>5} extra")
        for s in order:
            r = runs.get((s, task))
            if not r: continue
            it = r["items"]; extra = ""
            if kinds.get(task) == "search": extra = f"phrase={it.get('query_phrase_pct')}% words={it.get('query_words_pct')}% newest={it['newest']} span={it['span_hours']}h"
            if kinds.get(task) == "timeline": extra = f"author={it.get('author_match_pct')}% newest={it['newest']} oldest={it['oldest']}"
            if it["filler"]: extra += f" FILLER={it['filler']}"
            print(f"{s[:46]:46} {it['n']:>4} {str(r['duration_s']):>6} {str(r['usd_total'])[:8]:>8} {str(r['usd_per_1k'])[:7]:>7} {str(r['listed_price_per_item']):>8} {str(r['billing_model'])[:5]:>5} {str(r['bytes_per_item']):>6} {str(r['keys_per_item']):>5} {it['dups']:>3} {str(it['newest_first_pct']):>6} {it['text_max']:>6} {it['text_over_long']:>4} {it['text_ellipsis']:>3} {it['repost_by_text']:>3} {it['reply_by_text']:>4} {str(it['media_items']):>5} {extra}")
    st = next((t for t in plan["tasks"] if kinds.get(t) == "search"), None)
    if st:
        print(f"\n=== fill rates ({st})"); hdr = [s.split('/')[0][:10] for s in order if (s, st) in runs]
        print(f"{'field':18} " + " ".join(f"{h:>10}" for h in hdr))
        for k in plan["canon"]:
            print(f"{k:18} " + " ".join(f"{str(runs[(s, st)]['items']['fill'][k]):>10}" for s in order if (s, st) in runs))
    for task, x in cross.items():
        print(f"\n=== cross {task}: union={x['union']} in_all={x['in_all']} only_one={x['only_one']}")
        for a_, row in x["matrix"].items(): print(f"  {a_[:40]:40} " + " ".join(f"{v:>4}" for v in row.values()))
        print("  metric agreement:", json.dumps(x["metric_agreement"])); print("  long-text capture:", json.dumps(x["longform_capture"]))
    for task, rows in lookups.items():
        print(f"\n=== {task}")
        for s, r in rows.items(): print(f"  {s[:45]:45} {json.dumps(r, ensure_ascii=False)[:220] if r else 'NO ITEMS'}")
    print(f"\nwrote {plan['_out'] / 'data.json'}")


if __name__ == "__main__":
    main()