Initial commit: Dick Show Index pipeline with install script and docs

aggregate.py

@@ -0,0 +1,123 @@
+"""Merge per-episode analyses into ONE master JSON:
+data/dickshow_analysis.json
+
+Contents:
+  meta          counts, generated_at, rubric reference
+  episodes      full per-episode analyses, sorted by ep_num
+  indexes       derived views for fast lookup:
+                by_person, by_tag_topic, edginess_ranking, saga_map
+"""
+import json
+import os
+from datetime import datetime, timezone
+
+from config import DATA_DIR
+from analyze import ANALYSIS_DIR, MASTER_JSON
+from feed import load_episodes
+
+
+def load_analyses() -> list[dict]:
+    out = []
+    if not os.path.isdir(ANALYSIS_DIR):
+        return out
+    for f in sorted(os.listdir(ANALYSIS_DIR)):
+        if not f.endswith(".json"):
+            continue
+        try:
+            with open(os.path.join(ANALYSIS_DIR, f), encoding="utf-8") as fh:
+                out.append(json.load(fh))
+        except (json.JSONDecodeError, OSError) as e:
+            print(f"warning: skipping bad analysis file {f}: {e}")
+    return out
+
+
+def aggregate() -> None:
+    analyses = load_analyses()
+    eps_meta = {e["ep_num"]: e for e in load_episodes() if e["ep_num"]}
+
+    # attach runtime from episode metadata when available + verification status
+    for a in analyses:
+        m = eps_meta.get(a.get("ep_num"))
+        if m and m.get("duration_sec"):
+            a["runtime_minutes"] = round(m["duration_sec"] / 60)
+        v = a.get("_verification") or {}
+        a["verified"] = v.get("verdict") if isinstance(v, dict) else None
+
+    analyses.sort(key=lambda a: (a.get("ep_num") is None,
+                                 a.get("ep_num") or 0))
+
+    by_person: dict[str, list] = {}
+    by_topic: dict[str, list] = {}
+    quotes_ranked = []
+    ti_scores = []
+    for a in analyses:
+        epn = a.get("ep_num")
+        label = f"Ep {epn}" if epn else (a.get("title") or "?")[:20]
+        seen_p = set()
+        seen_t = set()
+        for tp in a.get("talking_points", []):
+            for p in tp.get("people", []):
+                key = p.lower().strip()
+                seen_p.add(key)
+                by_person.setdefault(key, []).append(
+                    {"episode": epn, "topic": tp.get("topic"),
+                     "ts": tp.get("start_ts")})
+            topic = (tp.get("topic") or "").strip().lower()
+            if topic:
+                seen_t.add(topic)
+        for t in sorted(seen_t):
+            by_topic.setdefault(t, []).append(epn)
+        for q in a.get("notable_quotes", []):
+            quotes_ranked.append({
+                "episode": epn, "ts": q.get("ts"), "ti": q.get("ti", 0),
+                "quote": q.get("quote"), "why": q.get("why"),
+            })
+        tg = a.get("transgression", {})
+        if isinstance(tg.get("overall_ti"), int):
+            ti_scores.append({"episode": epn, "overall_ti": tg["overall_ti"],
+                              "sincerity": tg.get("sincerity"),
+                              "targets": tg.get("targets", [])})
+
+    quotes_ranked.sort(key=lambda q: -q.get("ti", 0))
+    ti_scores.sort(key=lambda s: -s.get("overall_ti", 0))
+
+    n_verified = sum(1 for a in analyses if a.get("verified") in ("pass", "fix"))
+
+    master = {
+        "meta": {
+            "generated_at": datetime.now(timezone.utc).isoformat(),
+            "podcast": "The Dick Show",
+            "feed": "https://thedickshow.com/feed/podcast/",
+            "episodes_analyzed": len(analyses),
+            "episodes_factchecked": n_verified,
+            "total_runtime_minutes": sum(a.get("runtime_minutes") or 0
+                                         for a in analyses),
+            "scale": "Transgression Index 0-10, see rubric.py / README",
+            "note": ("Companion file for routine opencode access. "
+                     "Per-episode analyst files live in data/analysis/, "
+                     "raw transcripts in transcripts/. Each episode carries "
+                     "'verified': pass|fix|null from the dossier fact-check "
+                     "pass (verify.py). Treat verified=null as unconfirmed."),
+        },
+        "episodes": analyses,
+        "indexes": {
+            "by_person": {k: v for k, v in sorted(by_person.items())},
+            "topics_by_episode": {k: sorted(set(x for x in v if x))
+                                  for k, v in sorted(by_topic.items())},
+            "edginess_ranking": {
+                "top_overall_ti": ti_scores[:25],
+                "top_quotes": quotes_ranked[:50],
+            },
+        },
+    }
+
+    tmp = MASTER_JSON + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump(master, f, ensure_ascii=False, indent=1)
+    os.replace(tmp, MASTER_JSON)
+    print(f"master -> {MASTER_JSON} "
+          f"({len(analyses)} episodes, {len(quotes_ranked)} ranked quotes)")
+
+
+if __name__ == "__main__":
+    aggregate()