dickshow-index/verify.py

"""Second-pass fact-check: verify an episode analysis against the research
dossier (Dax_Herrera_Dick_Masterson_Profile.txt) via a second opencode run.

Spots "strange assumptions" like attributing a 2023 cohost seat to Maddox.
Usage:
    python verify.py            # verify ALL existing analyses
    python verify.py 361 373   # verify specific episodes

Retries, model fallback and provider cooldown live in llm.py (shared).
"""
import json
import os
import sys
import time

# console-safe output (model errors may contain any unicode)
try:
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
except Exception:
    pass

from llm import run as llm_run
from llm import RateLimitError, ProviderError
from llm import VERIFY_TIMEOUT
from config import BASE_DIR
from feed import load_episodes
from download import slug as ep_slug
from facts import DOSSIER_PATH, CAST_SHEET
from analyze import (ANALYSIS_DIR, OPENCODE, strip_to_json, validate)
from export_txt import txt_path

VERIFY_PROMPT = """You are the fact-checking archivist for The Dick Show project. Three files are attached:

  FILE 1: the episode transcript (timestamped lines [HH:MM:SS]).
  FILE 2: a researched dossier about the host, Dax Herrera aka Dick Masterson (verified court records, timeline, cast roster, feuds).
  FILE 3: a candidate ANALYSIS JSON of that episode produced by another analyst pass.

Your job: cross-check FILE 3 against FILE 1 and FILE 2 for FACTUAL consistency only. Do not judge opinions, humor, or TI scores. Check specifically:
- Speaker/role attribution. {cast}
- Names, spellings, and who is who (e.g. Sean vs Vito vs Asterios vs Maddox).
- Timeline claims vs the dossier (what was happening when this episode aired; which feud is active).
- Events/people attributed to the wrong era or wrong person.
- Claims about guests that contradict the transcript itself.

Output ONLY one valid JSON object, no fences, no commentary:
{{
 "verdict": "pass" or "fix",
 "issues": [
   {{"field": "guests|talking_points|arguments|ideas|notable_quotes|transgression",
     "claim": "...", "issue": "...", "correction": "..."}}
 ],
 "corrected_analysis": {{ ... full corrected analysis JSON, same schema as FILE 3, or null if verdict=pass ... }}
}}

Rules:
- If everything checks out: verdict="pass", issues=[], corrected_analysis=null.
- If ANY factual issue affects content: verdict="fix" and corrected_analysis MUST contain the complete corrected JSON (not a diff).
- Keep all scores/timestamps unless they are part of an identified error.
Begin now."""


def build_verify_prompt() -> str:
    return VERIFY_PROMPT.replace("{cast}", CAST_SHEET.strip())


def verify_episode(ep: dict, force: bool = False) -> dict | None:
    apath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json")
    if not os.path.exists(apath):
        return None
    with open(apath, encoding="utf-8") as f:
        current = json.load(f)

    attach_txt = txt_path(ep)
    if not os.path.exists(attach_txt):
        from export_txt import export_txt
        attach_txt = export_txt(ep)
    if not os.path.exists(DOSSIER_PATH):
        print(f"  dossier missing at {DOSSIER_PATH}; cannot verify",
              file=sys.stderr)
        return None

    prompt = build_verify_prompt()

    def parse_verdict(raw: str) -> dict:
        v = strip_to_json(raw)
        verdict = v.get("verdict")
        if verdict not in ("pass", "fix"):
            raise ValueError(f"bad verdict: {verdict!r}")
        if verdict == "fix":
            fixed = v.get("corrected_analysis")
            if not isinstance(fixed, dict):
                raise ValueError("verdict=fix but no corrected_analysis")
            validate(fixed)
        return v

    from datetime import datetime, timezone as _tz
    try:
        verdict = llm_run(prompt, [attach_txt, DOSSIER_PATH, apath],
                          timeout=VERIFY_TIMEOUT, label="verify",
                          parse=parse_verdict)
    except (RateLimitError, ProviderError) as e:
        print(f"  verification NOT RUN for {ep['title']}: {e}", flush=True)
        return None

    v = verdict.get("verdict")
    if v == "pass":
        current.setdefault("_verification", {
            "verdict": "pass",
            "checked_at": datetime.now(_tz.utc).isoformat(), "issues": []})
        _save(apath, current)
        print(f"  verified OK (no issues)", flush=True)
        return current
    fixed = verdict["corrected_analysis"]
    issues = verdict.get("issues", [])
    fixed["_verification"] = {"verdict": "fix",
                              "checked_at": datetime.now(_tz.utc).isoformat(),
                              "issues": issues}
    fixed["ep_num"] = ep["ep_num"] if ep["ep_num"] else fixed.get("ep_num")
    _save(apath, fixed)
    print(f"  FIXED {len(issues)} issue(s):", flush=True)
    for i in issues[:6]:
        print(f"    - [{i.get('field')}] {str(i.get('issue'))[:90]}",
              flush=True)
    return fixed


def run_opencode_multi(prompt: str, attachments: list[str]) -> str:
    # compat shim: raw output only, JSON parsed by the caller
    return llm_run(prompt, attachments, timeout=VERIFY_TIMEOUT,
                   label="verify")


def _now() -> str:
    from datetime import datetime, timezone
    return datetime.now(timezone.utc).isoformat()


def _save(apath: str, obj: dict) -> None:
    tmp = apath + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        json.dump(obj, f, ensure_ascii=False, indent=1)
    os.replace(tmp, apath)


def main() -> None:
    args = [a for a in sys.argv[1:] if not a.startswith("-")]
    eps = load_episodes()
    todo = [e for e in eps
            if os.path.exists(os.path.join(ANALYSIS_DIR,
                                           f"{ep_slug(e)}.json"))
            and (not args or e["ep_num"] in {int(a) for a in args})]
    print(f"verifying {len(todo)} analysis file(s)", flush=True)
    for i, ep in enumerate(todo, 1):
        label = f"Ep {ep['ep_num']}" if ep['ep_num'] else ep['title'][:40]
        print(f"[{i}/{len(todo)}] verifying {label} ...", flush=True)
        verify_episode(ep)
    from aggregate import aggregate
    aggregate()


if __name__ == "__main__":
    main()