dickshow-index/aggregate.py

"""Merge per-episode analyses into ONE master JSON:
data/dickshow_analysis.json

Contents:
  meta          counts, generated_at, rubric reference
  episodes      full per-episode analyses, sorted by ep_num
  indexes       derived views for fast lookup:
                by_person, by_tag_topic, edginess_ranking, saga_map
"""
import json
import os
import re
from datetime import datetime, timezone

from config import DATA_DIR
from analyze import ANALYSIS_DIR, MASTER_JSON
from feed import load_episodes

_EP_FILE_RE = re.compile(r"^ep\d{4}\.json$")


def load_analyses() -> list[dict]:
    out = []
    if not os.path.isdir(ANALYSIS_DIR):
        return out
    for f in sorted(os.listdir(ANALYSIS_DIR)):
        if not _EP_FILE_RE.match(f):
            continue  # ignore stray files like epNNNN_corrected.json
        try:
            with open(os.path.join(ANALYSIS_DIR, f), encoding="utf-8") as fh:
                out.append(json.load(fh))
        except (json.JSONDecodeError, OSError) as e:
            print(f"warning: skipping bad analysis file {f}: {e}")
    return out


def aggregate() -> None:
    analyses = load_analyses()
    eps_meta = {e["ep_num"]: e for e in load_episodes() if e["ep_num"]}

    # attach runtime from episode metadata when available + verification status
    for a in analyses:
        m = eps_meta.get(a.get("ep_num"))
        if m and m.get("duration_sec"):
            a["runtime_minutes"] = round(m["duration_sec"] / 60)
        v = a.get("_verification") or {}
        a["verified"] = v.get("verdict") if isinstance(v, dict) else None

    analyses.sort(key=lambda a: (a.get("ep_num") is None,
                                 a.get("ep_num") or 0))

    by_person: dict[str, list] = {}
    by_topic: dict[str, list] = {}
    quotes_ranked = []
    ti_scores = []
    for a in analyses:
        epn = a.get("ep_num")
        label = f"Ep {epn}" if epn else (a.get("title") or "?")[:20]
        seen_p = set()
        seen_t = set()
        for tp in a.get("talking_points", []):
            for p in tp.get("people", []):
                key = p.lower().strip()
                seen_p.add(key)
                by_person.setdefault(key, []).append(
                    {"episode": epn, "topic": tp.get("topic"),
                     "ts": tp.get("start_ts")})
            topic = (tp.get("topic") or "").strip().lower()
            if topic:
                seen_t.add(topic)
        for t in sorted(seen_t):
            by_topic.setdefault(t, []).append(epn)
        for q in a.get("notable_quotes", []):
            quotes_ranked.append({
                "episode": epn, "ts": q.get("ts"), "ti": q.get("ti", 0),
                "quote": q.get("quote"), "why": q.get("why"),
            })
        tg = a.get("transgression", {})
        if isinstance(tg.get("overall_ti"), int):
            ti_scores.append({"episode": epn, "overall_ti": tg["overall_ti"],
                              "sincerity": tg.get("sincerity"),
                              "targets": tg.get("targets", [])})

    quotes_ranked.sort(key=lambda q: -q.get("ti", 0))
    ti_scores.sort(key=lambda s: -s.get("overall_ti", 0))

    n_verified = sum(1 for a in analyses if a.get("verified") in ("pass", "fix"))

    master = {
        "meta": {
            "generated_at": datetime.now(timezone.utc).isoformat(),
            "podcast": "The Dick Show",
            "feed": "https://thedickshow.com/feed/podcast/",
            "episodes_analyzed": len(analyses),
            "episodes_factchecked": n_verified,
            "total_runtime_minutes": sum(a.get("runtime_minutes") or 0
                                         for a in analyses),
            "scale": "Transgression Index 0-10, see rubric.py / README",
            "note": ("Companion file for routine opencode access. "
                     "Per-episode analyst files live in data/analysis/, "
                     "raw transcripts in transcripts/. Each episode carries "
                     "'verified': pass|fix|null from the dossier fact-check "
                     "pass (verify.py). Treat verified=null as unconfirmed."),
        },
        "episodes": analyses,
        "indexes": {
            "by_person": {k: v for k, v in sorted(by_person.items())},
            "topics_by_episode": {k: sorted(set(x for x in v if x))
                                  for k, v in sorted(by_topic.items())},
            "edginess_ranking": {
                "top_overall_ti": ti_scores[:25],
                "top_quotes": quotes_ranked[:50],
            },
        },
    }

    tmp = MASTER_JSON + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        json.dump(master, f, ensure_ascii=False, indent=1)
    os.replace(tmp, MASTER_JSON)
    print(f"master -> {MASTER_JSON} "
          f"({len(analyses)} episodes, {len(quotes_ranked)} ranked quotes)")


if __name__ == "__main__":
    aggregate()