Initial commit: Dick Show Index pipeline with install script and docs
This commit is contained in:
n3wk
@@ -0,0 +1,177 @@
|
||||
"""Second-pass fact-check: verify an episode analysis against the research
|
||||
dossier (Dax_Herrera_Dick_Masterson_Profile.txt) via a second opencode run.
|
||||
|
||||
Spots "strange assumptions" like attributing a 2023 cohost seat to Maddox.
|
||||
Usage:
|
||||
python verify.py # verify ALL existing analyses
|
||||
python verify.py 361 373 # verify specific episodes
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
# console-safe output (model errors may contain any unicode)
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from config import BASE_DIR
|
||||
from feed import load_episodes
|
||||
from download import slug as ep_slug
|
||||
from facts import DOSSIER_PATH, CAST_SHEET
|
||||
from analyze import (ANALYSIS_DIR, OPENCODE, run_opencode, strip_to_json,
|
||||
validate)
|
||||
from export_txt import txt_path
|
||||
|
||||
VERIFY_PROMPT = """You are the fact-checking archivist for The Dick Show project. Three files are attached:
|
||||
|
||||
FILE 1: the episode transcript (timestamped lines [HH:MM:SS]).
|
||||
FILE 2: a researched dossier about the host, Dax Herrera aka Dick Masterson (verified court records, timeline, cast roster, feuds).
|
||||
FILE 3: a candidate ANALYSIS JSON of that episode produced by another analyst pass.
|
||||
|
||||
Your job: cross-check FILE 3 against FILE 1 and FILE 2 for FACTUAL consistency only. Do not judge opinions, humor, or TI scores. Check specifically:
|
||||
- Speaker/role attribution. {cast}
|
||||
- Names, spellings, and who is who (e.g. Sean vs Vito vs Asterios vs Maddox).
|
||||
- Timeline claims vs the dossier (what was happening when this episode aired; which feud is active).
|
||||
- Events/people attributed to the wrong era or wrong person.
|
||||
- Claims about guests that contradict the transcript itself.
|
||||
|
||||
Output ONLY one valid JSON object, no fences, no commentary:
|
||||
{{
|
||||
"verdict": "pass" or "fix",
|
||||
"issues": [
|
||||
{{"field": "guests|talking_points|arguments|ideas|notable_quotes|transgression",
|
||||
"claim": "...", "issue": "...", "correction": "..."}}
|
||||
],
|
||||
"corrected_analysis": {{ ... full corrected analysis JSON, same schema as FILE 3, or null if verdict=pass ... }}
|
||||
}}
|
||||
|
||||
Rules:
|
||||
- If everything checks out: verdict="pass", issues=[], corrected_analysis=null.
|
||||
- If ANY factual issue affects content: verdict="fix" and corrected_analysis MUST contain the complete corrected JSON (not a diff).
|
||||
- Keep all scores/timestamps unless they are part of an identified error.
|
||||
Begin now."""
|
||||
|
||||
|
||||
def build_verify_prompt() -> str:
|
||||
return VERIFY_PROMPT.replace("{cast}", CAST_SHEET.strip())
|
||||
|
||||
|
||||
def verify_episode(ep: dict, force: bool = False) -> dict | None:
|
||||
apath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json")
|
||||
if not os.path.exists(apath):
|
||||
return None
|
||||
with open(apath, encoding="utf-8") as f:
|
||||
current = json.load(f)
|
||||
|
||||
attach_txt = txt_path(ep)
|
||||
if not os.path.exists(attach_txt):
|
||||
from export_txt import export_txt
|
||||
attach_txt = export_txt(ep)
|
||||
if not os.path.exists(DOSSIER_PATH):
|
||||
print(f" dossier missing at {DOSSIER_PATH}; cannot verify",
|
||||
file=sys.stderr)
|
||||
return None
|
||||
|
||||
prompt = build_verify_prompt()
|
||||
from analyze import RateLimitError
|
||||
waits = [30, 120, 300, 600]
|
||||
attempt = 0
|
||||
last_err = None
|
||||
while attempt < len(waits):
|
||||
attempt += 1
|
||||
try:
|
||||
raw = run_opencode_multi(prompt, [attach_txt, DOSSIER_PATH, apath])
|
||||
verdict = strip_to_json(raw)
|
||||
v = verdict.get("verdict")
|
||||
if v not in ("pass", "fix"):
|
||||
raise ValueError(f"bad verdict: {v!r}")
|
||||
if v == "pass":
|
||||
current.setdefault("_verification", {
|
||||
"verdict": "pass", "checked_at": _now(), "issues": []})
|
||||
_save(apath, current)
|
||||
print(f" verified OK (no issues)", flush=True)
|
||||
return current
|
||||
fixed = verdict.get("corrected_analysis")
|
||||
if not isinstance(fixed, dict):
|
||||
raise ValueError("verdict=fix but no corrected_analysis")
|
||||
validate(fixed)
|
||||
issues = verdict.get("issues", [])
|
||||
fixed["_verification"] = {"verdict": "fix", "checked_at": _now(),
|
||||
"issues": issues}
|
||||
fixed["ep_num"] = ep["ep_num"] if ep["ep_num"] else fixed.get("ep_num")
|
||||
_save(apath, fixed)
|
||||
print(f" FIXED {len(issues)} issue(s):", flush=True)
|
||||
for i in issues[:6]:
|
||||
print(f" - [{i.get('field')}] {str(i.get('issue'))[:90]}",
|
||||
flush=True)
|
||||
return fixed
|
||||
except RateLimitError as e:
|
||||
last_err = e
|
||||
wait = waits[min(attempt, len(waits) - 1)]
|
||||
print(f" verify rate-limited (attempt {attempt}); "
|
||||
f"sleeping {wait}s ...", flush=True)
|
||||
time.sleep(wait)
|
||||
except Exception as e: # noqa: BLE001
|
||||
last_err = e
|
||||
wait = waits[min(attempt - 1, len(waits) - 1)]
|
||||
print(f" verify attempt {attempt} failed: {e}; "
|
||||
f"retry in {wait}s", flush=True)
|
||||
time.sleep(wait)
|
||||
print(f" verification FAILED for {ep['title']}: {last_err}", flush=True)
|
||||
return None
|
||||
|
||||
|
||||
def run_opencode_multi(prompt: str, attachments: list[str]) -> str:
|
||||
from analyze import OPENCODE_MODEL
|
||||
cmd = [OPENCODE, "run", "--auto"]
|
||||
for a in attachments:
|
||||
cmd += ["-f", a]
|
||||
cmd += ["--dir", BASE_DIR]
|
||||
if OPENCODE_MODEL:
|
||||
cmd += ["-m", OPENCODE_MODEL]
|
||||
cmd.append(prompt)
|
||||
import subprocess
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace",
|
||||
cwd=BASE_DIR, timeout=900)
|
||||
out = proc.stdout or ""
|
||||
if not out.strip() and proc.stderr:
|
||||
raise RuntimeError(f"opencode produced no stdout: "
|
||||
f"{proc.stderr[-400:]}")
|
||||
return out
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
from datetime import datetime, timezone
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _save(apath: str, obj: dict) -> None:
|
||||
tmp = apath + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(obj, f, ensure_ascii=False, indent=1)
|
||||
os.replace(tmp, apath)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = [a for a in sys.argv[1:] if not a.startswith("-")]
|
||||
eps = load_episodes()
|
||||
todo = [e for e in eps
|
||||
if os.path.exists(os.path.join(ANALYSIS_DIR,
|
||||
f"{ep_slug(e)}.json"))
|
||||
and (not args or e["ep_num"] in {int(a) for a in args})]
|
||||
print(f"verifying {len(todo)} analysis file(s)", flush=True)
|
||||
for i, ep in enumerate(todo, 1):
|
||||
label = f"Ep {ep['ep_num']}" if ep['ep_num'] else ep['title'][:40]
|
||||
print(f"[{i}/{len(todo)}] verifying {label} ...", flush=True)
|
||||
verify_episode(ep)
|
||||
from aggregate import aggregate
|
||||
aggregate()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user