Resilient LLM runner: model fallback chain, provider-death blacklist, cooldown, ANSI-repair

- llm.py: single retry/fallback/cooldown authority used by analyze/verify/pipeline
- provider [404] blacklists model + switches; rate limit escalates then cooldown
- unparsable model output (bad JSON/ANSI noise) skips to next model
- verify.py: DS_VERIFY_TIMEOUT, true rate-limit handling, honest FAILED reporting
- pipeline_all.py: honors model, reports verify status accurately
- aggregate.py: skip epNNNN_corrected.json strays (was duplicating ep 304)
- gitignore data outputs; drop stray analysis/analyst/analysts dirs

.gitignore

@@ -18,6 +18,9 @@ data/analysis/
 data/index.db
 data/index.db-wal
 data/index.db-shm
+data/episodes.jsonl
+data/dickshow_analysis.json
+data/lm_blacklist.json
 
 # IDE
 .idea/

aggregate.py

@@ -9,20 +9,23 @@ Contents:
 """
 import json
 import os
+import re
 from datetime import datetime, timezone
 
 from config import DATA_DIR
 from analyze import ANALYSIS_DIR, MASTER_JSON
 from feed import load_episodes
 
+_EP_FILE_RE = re.compile(r"^ep\d{4}\.json$")
+
 
 def load_analyses() -> list[dict]:
     out = []
     if not os.path.isdir(ANALYSIS_DIR):
         return out
     for f in sorted(os.listdir(ANALYSIS_DIR)):
-        if not f.endswith(".json"):
+        if not _EP_FILE_RE.match(f):
-            continue
+            continue  # ignore stray files like epNNNN_corrected.json
         try:
             with open(os.path.join(ANALYSIS_DIR, f), encoding="utf-8") as fh:
                 out.append(json.load(fh))

analyze.py

@@ -1,14 +1,15 @@
-"""Analyze episodes via the local `opencode run` CLI.
+"""Analyze episodes via the local `opencode run` CLI, routed through the
+resilient runner in llm.py (model fallback chain + provider-death blacklist).
 
 For each transcript:
   1. export readable timestamped txt (export_txt.py)
   2. call: opencode run -f <txt> "<prompt with rubric + schema>"
-  3. parse/validate the returned JSON (retry up to 3x on bad output)
+  3. parse/validate the returned JSON (model fallback chain handles retries)
   4. save data/analysis/epXXXX.json
 
 Then aggregate all analyses into one master file for routine opencode access:
 data/dickshow_analysis.json   (run aggregate.py, or it runs automatically
-                               after each new analysis in this script)
+                                after each new analysis in this script)
 
 Resumable: existing per-episode analyses are skipped unless --force.
 """
@@ -16,11 +17,12 @@ import argparse
 import json
 import os
 import re
-import subprocess
 import sys
 import time
 from datetime import datetime, timezone
 
+from llm import run as llm_run
+from llm import RateLimitError, ProviderError  # noqa: F401  (compat)
 from config import DATA_DIR, BASE_DIR
 from feed import load_episodes
 from download import slug as ep_slug, audio_path
@@ -36,7 +38,7 @@ OPENCODE = os.environ.get(
     "OPENCODE_BIN", os.path.expanduser(r"~\.opencode\bin\opencode.exe"))
 # analyzer model override, e.g. opencode/x-preview-f-free
 OPENCODE_MODEL = os.environ.get("OPENCODE_ANALYZE_MODEL", "").strip()
-OPENCODE_TIMEOUT = int(os.environ.get("DS_ANALYZE_TIMEOUT", "900"))  # 15 min
+OPENCODE_TIMEOUT = int(os.environ.get("DS_ANALYZE_TIMEOUT", "1800"))  # 30 min
 
 PROMPT_TEMPLATE = """You are the show archivist for The Dick Show podcast. Attached is the FULL transcript of one episode (timestamped lines like [HH:MM:SS]).
 
@@ -69,14 +71,26 @@ def build_prompt() -> str:
                                   schema=SCHEMA_EXAMPLE.strip())
 
 
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
+# "tool call" display lines opencode may echo before the model's reply:
+# `→ Read file ...` / spinner glyphs / "Press Enter..." prompts.
+_TOOLCALL_RE = re.compile(
+    r"(?m)^.*\x1b?\[?→.*$|^\s*(Press Enter|•|→)\s*$")
+
+
 def _repair(text: str) -> str:
-    """Common LLM-JSON repairs: fences, trailing commas, stray text, missing commas."""
+    """Common LLM-JSON repairs: ANSI codes, fences, trailing commas, missing commas."""
-    text = FENCES_RE.sub("", text.strip()).strip()
+    text = _ANSI_RE.sub("", text)
+    text = FENCES_RE.sub("", text).strip()
+    # extract just the outermost {...} JSON object if the reply is noisy
     start = text.find("{")
     end = text.rfind("}")
-    if start > 0 or end < len(text) - 1:
+    if start >= 0 and end > start:
-        if start != -1 and end > start:
+        text = text[start:end + 1]
-            text = text[start:end + 1]
+    elif text.strip():
+        return text.strip()
+    else:
+        return ""
     # trailing commas before } or ]
     text = re.sub(r",(\s*[}\]])", r"\1", text)
     # missing commas between object members: "key": "value" "key2": "value2" -> add comma
@@ -141,101 +155,69 @@ def validate(obj: dict) -> None:
 
 
 def run_opencode(prompt: str, attach: str) -> str:
-    # NOTE: deliberately NO `-c`/`-s` session reuse: `run -c` continues the
+    # NOTE: deliberately NO `-c`/`-s` session reuse. The resilient runner
-    # globally-last session, which under concurrent pipelines cross-pollinates
+    # (llm.py) owns fallback/retry/cooldown; this is the compat shim used by
-    # analyses between episodes. Every call is a fresh session.
+    # verify.py for single-file analysis (JSON parsed by the caller).
-    # --auto: headless runs must not stall on permission prompts (the model
+    return llm_run(prompt, [attach], timeout=OPENCODE_TIMEOUT,
-    # sometimes Reads attached files for more context); scope is our own data.
+                   label="analyze")
-    cmd = [OPENCODE, "run", "--auto", "-f", attach,
-           "--dir", BASE_DIR]
-    if OPENCODE_MODEL:
-        cmd += ["-m", OPENCODE_MODEL]
-    cmd.append(prompt)
-    proc = subprocess.run(cmd, capture_output=True, text=True,
-                          encoding="utf-8", errors="replace",
-                          cwd=BASE_DIR, timeout=OPENCODE_TIMEOUT)
-    out = proc.stdout or ""
-    err = proc.stderr or ""
-    combined = (out + err).lower()
-    if "rate limit" in combined or "429" in combined:
-        raise RateLimitError("provider rate limit exceeded")
-    if not out.strip():
-        raise RuntimeError(f"opencode produced no stdout: {err[-400:]}")
-    return out
 
 
+# legacy alias kept for imports
 class RateLimitError(RuntimeError):
     pass
 
 
 # When the provider quota dries up, stop *attempting* analyses pipeline-wide
-# for a while; transcription (local) keeps running. Probe again after cooldown.
+# for a while; transcription (local) keeps running. Cooldown state lives in
+# llm.py and is shared with verify + pipeline_all.
 _RATE_COOLDOWN = {"until": 0.0}
 COOLDOWN_SECONDS = int(os.environ.get("DS_RATE_COOLDOWN", "900"))
 
 
-def analyze_episode(ep: dict, force: bool = False, retries: int = 3) -> dict | None:
+def analyze_episode(ep: dict, force: bool = False, retries: int = 1) -> dict | None:
+    """Analyze one episode. Retries/fallback/cooldown handled by llm.run.
+
+    Returns the analysis dict, or None if the episode was already done or the
+    provider is unavailable (callers catch up on a later pass).
+    """
     os.makedirs(ANALYSIS_DIR, exist_ok=True)
     apath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json")
     if os.path.exists(apath) and not force:
         return None
-    if time.time() < _RATE_COOLDOWN["until"]:
-        print(f"  (analysis cooldown {int(_RATE_COOLDOWN['until'] - time.time())}s "
-              f"- will catch up later)", flush=True)
-        return None
     if not os.path.exists(tpath(ep)):
         print(f"  no transcript for {ep['title']}; skipping", flush=True)
         return None
+
     attach = export_txt(ep)
     prompt = build_prompt()
-    last_err = None
+
-    # escalating waits; rate limits need minutes, not seconds
+    def parse_obj(raw: str) -> dict:
-    waits = [10, 60, 180, 420, 600, 900]
+        obj = strip_to_json(raw)
-    attempt = 0
+        validate(obj)
-    while attempt < len(waits):
+        ensure_content_match(obj, attach)
-        attempt += 1
+        return obj
-        t0 = time.time()
+
-        try:
+    try:
-            raw = run_opencode(
+        obj = llm_run(prompt, [attach], timeout=OPENCODE_TIMEOUT,
-                (REPAIR_PREAMBLE + prompt) if attempt > 1 else prompt, attach)
+                      label="analyze", parse=parse_obj)
-            obj = strip_to_json(raw)
+    except (RateLimitError, ProviderError) as e:
-            validate(obj)
+        print(f"  ANALYSIS NOT RUN {ep['title']}: {e}", flush=True)
-            ensure_content_match(obj, attach)
+        return None
-            obj["ep_num"] = ep["ep_num"] if ep["ep_num"] else obj.get("ep_num")
+
-            obj["title"] = obj.get("title") or ep["title"]
+    obj["ep_num"] = ep["ep_num"] if ep["ep_num"] else obj.get("ep_num")
-            obj["pub_date"] = ep["pub_date"]
+    obj["title"] = obj.get("title") or ep["title"]
-            obj["_analyzed_at"] = datetime.now(timezone.utc).isoformat()
+    obj["pub_date"] = ep["pub_date"]
-            obj["_source_transcript"] = os.path.basename(tpath(ep))
+    obj["_analyzed_at"] = datetime.now(timezone.utc).isoformat()
-            with open(apath + ".tmp", "w", encoding="utf-8") as f:
+    obj["_source_transcript"] = os.path.basename(tpath(ep))
-                json.dump(obj, f, ensure_ascii=False, indent=1)
+    with open(apath + ".tmp", "w", encoding="utf-8") as f:
-            os.replace(apath + ".tmp", apath)
+        json.dump(obj, f, ensure_ascii=False, indent=1)
-            dt = time.time() - t0
+    os.replace(apath + ".tmp", apath)
-            ti = obj["transgression"]["overall_ti"]
+    ti = obj["transgression"]["overall_ti"]
-            n_tp = len(obj.get("talking_points", []))
+    n_tp = len(obj.get("talking_points", []))
-            n_q = len(obj.get("notable_quotes", []))
+    n_q = len(obj.get("notable_quotes", []))
-            print(f"  OK {os.path.basename(apath)} in {dt:.0f}s "
+    print(f"  OK {os.path.basename(apath)} "
-                  f"(TI={ti}, {n_tp} points, {n_q} quotes)", flush=True)
+          f"(TI={ti}, {n_tp} points, {n_q} quotes)", flush=True)
-            return obj
+    return obj
-        except RateLimitError as e:
-            last_err = e
-            wait = waits[min(attempt, len(waits) - 1)]
-            print(f"  rate-limited (attempt {attempt}); "
-                  f"sleeping {wait}s ...", flush=True)
-            time.sleep(wait)
-        except Exception as e:  # noqa: BLE001
-            last_err = e
-            wait = waits[min(attempt - 1, len(waits) - 1)]
-            print(f"  attempt {attempt} failed: {e}; retry in {wait}s",
-                  flush=True)
-            time.sleep(wait)
-    print(f"  GIVING UP on {ep['title']} after {attempt} attempts: "
-          f"{last_err}", flush=True)
-    if isinstance(last_err, RateLimitError):
-        _RATE_COOLDOWN["until"] = time.time() + COOLDOWN_SECONDS
-        print(f"  entering analysis cooldown for {COOLDOWN_SECONDS}s "
-              f"(transcription continues; analyses catch up later)",
-              flush=True)
-    return None
 
 
 def main() -> None:

llm.py

@@ -0,0 +1,284 @@
+"""Resilient opencode runner for analysis/verify passes.
+
+Why this exists (after a string of outages):
+  * The opencode "Console" provider models are unstable: some get retired or
+    break upstream and then return "[404] Provider returned error" while still
+    appearing in `opencode models` (nemotron-3.5-lightning-free, 2026-08-25).
+    Retrying the SAME broken model loops for hours and always fails.
+  * Rate limits are account-wide (shared by all opencode/* models), so a 429
+    should wait out the cooldown, NOT flip models.
+  * Provider-dead errors should flip to the next WORKING model immediately.
+  * A model that returns text but garbage/non-JSON output should be skipped
+    in favor of the next candidate, not sent back through an outer retry loop.
+
+This module is the SINGLE retry/fallback/cooldown authority. Callers
+(analyze.py, verify.py, pipeline_all.py) just call run() and handle the
+outcome or the raised RateLimitError/ProviderError.
+"""
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+
+from config import BASE_DIR, DATA_DIR
+
+OPENCODE = os.environ.get(
+    "OPENCODE_BIN", os.path.expanduser(r"~\.opencode\bin\opencode.exe"))
+# analyzer model override, e.g. opencode/x-preview-f-free
+OPENCODE_MODEL = os.environ.get("OPENCODE_ANALYZE_MODEL", "").strip()
+ANALYZE_TIMEOUT = int(os.environ.get("DS_ANALYZE_TIMEOUT", "1800"))  # 30 min
+VERIFY_TIMEOUT = int(os.environ.get("DS_VERIFY_TIMEOUT", "900"))     # 15 min
+
+# Preferred model first; everything else is a fallback tried in order until
+# one returns usable output. Updated from `opencode models` 2026-08-25.
+MODEL_CHAIN = [
+    "opencode/big-pickle",
+    "opencode/nemotron-3-ultra-free",
+    "opencode/hy3-free",
+    "opencode/ling-3.0-flash-fin-free",
+    "opencode/mimo-v2.5-free",
+    "opencode/muse-spark-1.2-contributor-free",
+    "opencode/nemotron-3.5-lightning-free",
+]
+
+# Models that hard-failed with provider 404 / model-not-found at runtime.
+# Persisted so later runs skip them immediately instead of burning retries.
+BLACKLIST_JSON = os.path.join(DATA_DIR, "lm_blacklist.json")
+
+# Cooldown after consecutive rate limits, shared by every caller.
+COOLDOWN_SECONDS = int(os.environ.get("DS_RATE_COOLDOWN", "900"))
+_COOLDOWN = {"until": 0.0}
+
+# Escalating wait between attempts *for the same model* on any failure.
+WAITS = [10, 60, 180, 420, 600, 900]
+# Per-model retry budget for blips before moving to the next model.
+BLIP_RETRIES = 2
+# Rate-limit turns on the same model before bailing to the cooldown.
+RATE_ESCALATION = 3
+
+
+class RateLimitError(RuntimeError):
+    pass
+
+
+class ProviderError(RuntimeError):
+    pass
+
+
+# --- error classification -------------------------------------------------
+# ORDER MATTERS: rate first, then 404/model-death, then generic transient.
+_RATE_RE = re.compile(
+    r"rate limit|too many requests|\b429\b|quota exceeded", re.IGNORECASE)
+_DEAD_RE = re.compile(
+    r"\[404\]|provider returned error[^\n]*\b4\d\d\b|model not found|"
+    r"model_not_found|model does not exist|not available",
+    re.IGNORECASE)
+_TRANSIENT_RE = re.compile(
+    r"\b5\d\d\b|502|503|504|bad gateway|temporar|timed? ?out|connection|"
+    r"\bEOF\b|upstream request failed|http 500|500",
+    re.IGNORECASE)
+
+
+def _load_blacklist() -> list[str]:
+    try:
+        with open(BLACKLIST_JSON, encoding="utf-8") as f:
+            data = json.load(f)
+            return [m for m in data.get("models", []) if isinstance(m, str)]
+    except (OSError, json.JSONDecodeError):
+        return []
+
+
+def _save_blacklist(models: list[str]) -> None:
+    os.makedirs(os.path.dirname(BLACKLIST_JSON), exist_ok=True)
+    tmp = BLACKLIST_JSON + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump({"models": sorted(set(models))}, f, indent=1)
+    os.replace(tmp, BLACKLIST_JSON)
+
+
+def _blacklist_model(model: str) -> None:
+    models = _load_blacklist()
+    if model not in models:
+        models.append(model)
+        _save_blacklist(models)
+    print(f"  [lm] BLACKLISTED {model} (provider-dead)",
+          file=sys.stderr, flush=True)
+
+
+def active_models(env_override: str = "") -> list[str]:
+    """Ordered fallback chain, minus blacklisted. Env override goes first."""
+    blacklist = set(_load_blacklist())
+    chain: list[str] = []
+    if env_override and env_override not in blacklist:
+        chain.append(env_override)
+    for m in MODEL_CHAIN:
+        if m not in chain and m not in blacklist:
+            chain.append(m)
+    if not chain:
+        # everything blacklisted - reset and warn
+        print("  [lm] ALL models blacklisted; resetting blacklist",
+              file=sys.stderr, flush=True)
+        _save_blacklist([])
+        chain = [m for m in MODEL_CHAIN if m != env_override][:1]
+    return chain
+
+
+def classify_combined(out: str, err: str) -> str | None:
+    """Return 'rate' | 'dead' | 'transient' | None based on provider output."""
+    combined = (out + "\n" + err)
+    if _RATE_RE.search(combined):
+        return "rate"
+    if _DEAD_RE.search(combined):
+        return "dead"
+    if _TRANSIENT_RE.search(combined):
+        return "transient"
+    return None
+
+
+def run(prompt: str, attachments: list[str], *,
+        timeout: int | None = None, label: str = "opencode",
+        parse=None, env_override: str | None = None,
+        print_progress: bool = True):
+    """Resilient `opencode run` against the model chain.
+
+    Returns parse(raw_stdout) if `parse` is given (and it must return/raise),
+    else the raw stdout text. Raises RateLimitError or ProviderError once the
+    chain and escalation budgets are exhausted.
+    """
+    if env_override is None:
+        env_override = os.environ.get("OPENCODE_ANALYZE_MODEL", "").strip()
+    timeout = timeout or ANALYZE_TIMEOUT
+
+    if time.time() < _COOLDOWN["until"]:
+        raise RateLimitError(
+            f"in cooldown for {int(_COOLDOWN['until'] - time.time())}s")
+
+    models = active_models(env_override)
+    if not models:
+        raise ProviderError("no usable models after blacklist")
+
+    last_transient = None
+    last_rate = None
+    saw_rate = False
+    bad_output = []  # models that answered but failed `parse`
+
+    for model in models:
+        blips_left = BLIP_RETRIES
+        for round_no in range(RATE_ESCALATION + 1):
+            cmd = [OPENCODE, "run", "--auto"]
+            for a in attachments:
+                cmd += ["-f", a]
+            cmd += ["--dir", BASE_DIR, "-m", model, prompt]
+            if print_progress:
+                print(f"  [{label}] running analysis "
+                      f"(attempt {round_no + 1} ({model}, {timeout}s))...",
+                      flush=True)
+            t0 = time.time()
+            try:
+                proc = subprocess.run(
+                    cmd, capture_output=True, text=True,
+                    encoding="utf-8", errors="replace",
+                    cwd=BASE_DIR, timeout=timeout)
+            except subprocess.TimeoutExpired:
+                blips_left -= 1
+                last_transient = f"{model} hit timeout {timeout}s"
+                if blips_left <= 0:
+                    break  # next model
+                wait = min(60, 2 ** blips_left * 15)
+                if print_progress:
+                    print(f"  [{label}] timed out after {timeout}s; "
+                          f"retry in {wait}s", flush=True)
+                time.sleep(wait)
+                continue
+
+            out = proc.stdout or ""
+            err = proc.stderr or ""
+            cls = classify_combined(out, err)
+
+            if cls == "rate":
+                saw_rate = True
+                last_rate = (model, err[-200:])
+                if round_no >= RATE_ESCALATION:
+                    _COOLDOWN["until"] = time.time() + COOLDOWN_SECONDS
+                    raise RateLimitError(
+                        f"{model} rate-limited (entering "
+                        f"{COOLDOWN_SECONDS}s cooldown)")
+                wait = WAITS[min(round_no, len(WAITS) - 1)]
+                if print_progress:
+                    print(f"  [{label}] rate-limited ({model}); "
+                          f"sleeping {wait}s ...", flush=True)
+                time.sleep(wait)
+                continue  # same model, longer wait
+
+            if cls == "dead":
+                _blacklist_model(model)
+                break  # next model
+
+            if cls == "transient":
+                blips_left -= 1
+                last_transient = f"{model}: {err[-200:]}"
+                if blips_left <= 0:
+                    break  # next model
+                wait = min(60, 2 ** blips_left * 15)
+                if print_progress:
+                    print(f"  [{label}] transient error ({model}); "
+                          f"retry in {wait}s", flush=True)
+                time.sleep(wait)
+                continue
+
+            if not out.strip():
+                blips_left -= 1
+                last_transient = f"{model}: no stdout ({err[-200:].strip()})"
+                if blips_left <= 0:
+                    break  # next model
+                wait = min(60, 2 ** blips_left * 15)
+                if print_progress:
+                    print(f"  [{label}] no output from {model}; "
+                          f"retry in {wait}s", flush=True)
+                time.sleep(wait)
+                continue
+
+            elapsed = time.time() - t0
+            if parse is not None:
+                try:
+                    result = parse(out)
+                except Exception as e:  # bad output: skip this model
+                    bad_output.append((model, str(e)[:120]))
+                    if print_progress:
+                        print(f"  [{label}] {model} gave unparsable output "
+                              f"({str(e)[:90]}); next model", flush=True)
+                    break
+                if print_progress:
+                    print(f"  [{label}] {model} OK in {elapsed:.0f}s",
+                          flush=True)
+                return result
+            if print_progress:
+                print(f"  [{label}] {model} returned "
+                      f"{len(out)} chars in {elapsed:.0f}s", flush=True)
+            return out
+
+    if saw_rate:
+        _COOLDOWN["until"] = time.time() + COOLDOWN_SECONDS
+        raise RateLimitError("all models rate-limited: "
+                             f"{last_rate[1] if last_rate else 'unknown'}")
+    raise ProviderError((
+        f"no model produced valid output; models_with_bad_output="
+        f"{bad_output}; last_transient={last_transient}"
+    ))
+
+
+if __name__ == "__main__":
+    # smoke test: pick the first live model and ask it to say hello
+    t0 = time.time()
+    try:
+        res = run("Reply with exactly: OK", [], timeout=90, label="smoke")
+        print(f"SMOKE OK: {res.strip()[:80]} in {time.time()-t0:.0f}s")
+        print(f"active models: {active_models()}")
+    except RateLimitError as e:
+        print(f"SMOKE RATE-LIMITED: {e}")
+        sys.exit(3)
+    except ProviderError as e:
+        print(f"SMOKE EXHAUSTED: {e}")
+        sys.exit(4)

pipeline_all.py

@@ -17,6 +17,7 @@ import sys
 import time
 import traceback
 import subprocess
+from pathlib import Path
 
 try:
     sys.stdout.reconfigure(encoding="utf-8", errors="replace")
@@ -143,16 +144,20 @@ def do_transcribe(model, ep: dict) -> None:
                 pass
 
 
+from llm import run as llm_run, RateLimitError, ProviderError
+from llm import ANALYZE_TIMEOUT
+
+
 def run_analysis_with_timeout(ep: dict) -> tuple:
     """
-    Run analysis + verification using opencode subprocess with hard timeout.
+    Run analysis + verification using the resilient opencode runner (model
-    Returns (success: bool, result: dict | None, error: str | None)
+    fallback chain + provider-death blacklist). Returns (success, result, error).
     """
-    from analyze import ANALYSIS_DIR, OPENCODE, build_prompt, strip_to_json, validate
+    from analyze import ANALYSIS_DIR, build_prompt, validate, strip_to_json
+    from analyze import ensure_content_match
     from verify import verify_episode
     from download import slug as ep_slug
-    from export_txt import txt_path
+    from export_txt import export_txt
-    from config import BASE_DIR
 
     label = f"Ep {ep['ep_num']}"
     apath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json")
@@ -167,70 +172,39 @@ def run_analysis_with_timeout(ep: dict) -> tuple:
     prompt = build_prompt()
     attach = export_txt(ep)
 
-    # Run analysis with timeout + rate-limit retries
+    def parse_obj(raw: str) -> dict:
-    cmd = [OPENCODE, "run", "--auto", "-f", attach, "--dir", BASE_DIR, prompt]
+        obj = strip_to_json(raw)
+        validate(obj)
+        ensure_content_match(obj, attach)
+        return obj
 
-    # Rate limit retry waits (seconds)
+    # Resilient runner handles model fallback + retries internally.
-    rate_limit_waits = [10, 60, 180, 420, 600]
+    try:
+        print(f"  [{time.strftime('%H:%M:%S')}] running analysis "
+              f"(timeout {ANALYZE_TIMEOUT}s)...", flush=True)
+        obj = llm_run(prompt, [attach], timeout=ANALYZE_TIMEOUT,
+                      label="analyze", parse=parse_obj)
+    except (RateLimitError, ProviderError) as e:
+        return (False, None, str(e))
 
-    for attempt, wait_time in enumerate(rate_limit_waits, 1):
+    obj["title"] = obj.get("title") or ep["title"]
-        try:
+    obj["pub_date"] = ep["pub_date"]
-            print(f"  [{time.strftime('%H:%M:%S')}] running analysis (attempt {attempt}, timeout {ANALYSIS_TIMEOUT}s)...", flush=True)
+    obj["_analyzed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
-            proc = subprocess.run(cmd, capture_output=True, text=True,
+    obj["_source_transcript"] = os.path.basename(transcript_path(ep))
-                                  encoding="utf-8", errors="replace",
+    obj["ep_num"] = ep["ep_num"] if ep["ep_num"] else obj.get("ep_num")
-                                  cwd=BASE_DIR, timeout=ANALYSIS_TIMEOUT)
-            out = proc.stdout or ""
-            err = proc.stderr or ""
-            combined = (out + err).lower()
 
-            if "rate limit" in combined or "429" in combined:
+    # Save analysis
-                raise RuntimeError("provider rate limit exceeded")
+    apath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json")
+    with open(apath + ".tmp", "w", encoding="utf-8") as f:
+        json.dump(obj, f, ensure_ascii=False, indent=1)
+    os.replace(apath + ".tmp", apath)
 
-            if not out.strip() and err:
+    # Now run verification
-                raise RuntimeError(f"opencode produced no stdout: {err[-400:]}")
+    print(f"  [{time.strftime('%H:%M:%S')}] fact-checking...", flush=True)
+    from verify import verify_episode
+    verified = verify_episode(ep)
 
-            # Parse result
+    return (True, obj, None if verified is not None else "verification failed")
-            from analyze import _repair
-            obj = json.loads(_repair(out), strict=False)
-            validate(obj)
-            
-            # Add metadata
-            obj["ep_num"] = ep["ep_num"] if ep["ep_num"] else obj.get("ep_num")
-            obj["title"] = obj.get("title") or ep["title"]
-            obj["pub_date"] = ep["pub_date"]
-            obj["_analyzed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
-            obj["_source_transcript"] = os.path.basename(transcript_path(ep))
-            
-            # Save analysis
-            from analyze import ANALYSIS_DIR as ANALYSIS_DIR2
-            from download import slug as ep_slug
-            apath = os.path.join(ANALYSIS_DIR2, f"{ep_slug(ep)}.json")
-            with open(apath + ".tmp", "w", encoding="utf-8") as f:
-                json.dump(obj, f, ensure_ascii=False, indent=1)
-            os.replace(apath + ".tmp", apath)
-            
-            # Now run verification
-            print(f"  [{time.strftime('%H:%M:%S')}] fact-checking...", flush=True)
-            from verify import verify_episode
-            verify_episode(ep)
-            
-            return (True, obj, None)
-            
-        except RuntimeError as e:
-            if "rate limit" in str(e).lower():
-                if attempt < len(rate_limit_waits):
-                    wait = rate_limit_waits[attempt]
-                    print(f"  [{time.strftime('%H:%M:%S')}] rate limited (attempt {attempt}), sleeping {wait}s...", flush=True)
-                    time.sleep(wait)
-                    continue
-                else:
-                    return (False, None, f"Rate limit exceeded after {attempt} attempts")
-            else:
-                return (False, None, str(e))
-        except Exception as e:
-            return (False, None, str(e))
-    
-    return (False, None, "Max retries exceeded")
 
 
 def main() -> None:
@@ -304,9 +278,14 @@ def main() -> None:
             success, result, error = run_analysis_with_timeout(ep)
             if success:
                 done_a += 1
-                print(f"  [{time.strftime('%H:%M:%S')}] analyzed {label} (TI={result['transgression']['overall_ti']}, "
+                ti = result.get("transgression", {}).get("overall_ti", "?")
-                      f"{len(result['talking_points'])} points, {len(result['notable_quotes'])} quotes)", flush=True)
+                print(f"  [{time.strftime('%H:%M:%S')}] analyzed {label} (TI={ti}, "
-                print(f"  [{time.strftime('%H:%M:%S')}] verified {label}", flush=True)
+                      f"{len(result.get('talking_points', []))} points, "
+                      f"{len(result.get('notable_quotes', []))} quotes)", flush=True)
+                if error:
+                    print(f"  [{time.strftime('%H:%M:%S')}] NOT verified {label}: {error}", flush=True)
+                else:
+                    print(f"  [{time.strftime('%H:%M:%S')}] verified {label}", flush=True)
             else:
                 print(f"  [{time.strftime('%H:%M:%S')}] ANALYSIS FAILED {label}: {error}", flush=True)
         except Exception as ex:

verify.py

@@ -5,6 +5,8 @@ Spots "strange assumptions" like attributing a 2023 cohost seat to Maddox.
 Usage:
     python verify.py            # verify ALL existing analyses
     python verify.py 361 373   # verify specific episodes
+
+Retries, model fallback and provider cooldown live in llm.py (shared).
 """
 import json
 import os
@@ -18,12 +20,14 @@ try:
 except Exception:
     pass
 
+from llm import run as llm_run
+from llm import RateLimitError, ProviderError
+from llm import VERIFY_TIMEOUT
 from config import BASE_DIR
 from feed import load_episodes
 from download import slug as ep_slug
 from facts import DOSSIER_PATH, CAST_SHEET
-from analyze import (ANALYSIS_DIR, OPENCODE, run_opencode, strip_to_json,
+from analyze import (ANALYSIS_DIR, OPENCODE, strip_to_json, validate)
-                     validate)
 from export_txt import txt_path
 
 VERIFY_PROMPT = """You are the fact-checking archivist for The Dick Show project. Three files are attached:
@@ -77,72 +81,54 @@ def verify_episode(ep: dict, force: bool = False) -> dict | None:
         return None
 
     prompt = build_verify_prompt()
-    from analyze import RateLimitError
+
-    waits = [30, 120, 300, 600]
+    def parse_verdict(raw: str) -> dict:
-    attempt = 0
+        v = strip_to_json(raw)
-    last_err = None
+        verdict = v.get("verdict")
-    while attempt < len(waits):
+        if verdict not in ("pass", "fix"):
-        attempt += 1
+            raise ValueError(f"bad verdict: {verdict!r}")
-        try:
+        if verdict == "fix":
-            raw = run_opencode_multi(prompt, [attach_txt, DOSSIER_PATH, apath])
+            fixed = v.get("corrected_analysis")
-            verdict = strip_to_json(raw)
-            v = verdict.get("verdict")
-            if v not in ("pass", "fix"):
-                raise ValueError(f"bad verdict: {v!r}")
-            if v == "pass":
-                current.setdefault("_verification", {
-                    "verdict": "pass", "checked_at": _now(), "issues": []})
-                _save(apath, current)
-                print(f"  verified OK (no issues)", flush=True)
-                return current
-            fixed = verdict.get("corrected_analysis")
             if not isinstance(fixed, dict):
                 raise ValueError("verdict=fix but no corrected_analysis")
             validate(fixed)
-            issues = verdict.get("issues", [])
+        return v
-            fixed["_verification"] = {"verdict": "fix", "checked_at": _now(),
+
-                                      "issues": issues}
+    from datetime import datetime, timezone as _tz
-            fixed["ep_num"] = ep["ep_num"] if ep["ep_num"] else fixed.get("ep_num")
+    try:
-            _save(apath, fixed)
+        verdict = llm_run(prompt, [attach_txt, DOSSIER_PATH, apath],
-            print(f"  FIXED {len(issues)} issue(s):", flush=True)
+                          timeout=VERIFY_TIMEOUT, label="verify",
-            for i in issues[:6]:
+                          parse=parse_verdict)
-                print(f"    - [{i.get('field')}] {str(i.get('issue'))[:90]}",
+    except (RateLimitError, ProviderError) as e:
-                      flush=True)
+        print(f"  verification NOT RUN for {ep['title']}: {e}", flush=True)
-            return fixed
+        return None
-        except RateLimitError as e:
+
-            last_err = e
+    v = verdict.get("verdict")
-            wait = waits[min(attempt, len(waits) - 1)]
+    if v == "pass":
-            print(f"  verify rate-limited (attempt {attempt}); "
+        current.setdefault("_verification", {
-                  f"sleeping {wait}s ...", flush=True)
+            "verdict": "pass",
-            time.sleep(wait)
+            "checked_at": datetime.now(_tz.utc).isoformat(), "issues": []})
-        except Exception as e:  # noqa: BLE001
+        _save(apath, current)
-            last_err = e
+        print(f"  verified OK (no issues)", flush=True)
-            wait = waits[min(attempt - 1, len(waits) - 1)]
+        return current
-            print(f"  verify attempt {attempt} failed: {e}; "
+    fixed = verdict["corrected_analysis"]
-                  f"retry in {wait}s", flush=True)
+    issues = verdict.get("issues", [])
-            time.sleep(wait)
+    fixed["_verification"] = {"verdict": "fix",
-    print(f"  verification FAILED for {ep['title']}: {last_err}", flush=True)
+                              "checked_at": datetime.now(_tz.utc).isoformat(),
-    return None
+                              "issues": issues}
+    fixed["ep_num"] = ep["ep_num"] if ep["ep_num"] else fixed.get("ep_num")
+    _save(apath, fixed)
+    print(f"  FIXED {len(issues)} issue(s):", flush=True)
+    for i in issues[:6]:
+        print(f"    - [{i.get('field')}] {str(i.get('issue'))[:90]}",
+              flush=True)
+    return fixed
 
 
 def run_opencode_multi(prompt: str, attachments: list[str]) -> str:
-    from analyze import OPENCODE_MODEL
+    # compat shim: raw output only, JSON parsed by the caller
-    cmd = [OPENCODE, "run", "--auto"]
+    return llm_run(prompt, attachments, timeout=VERIFY_TIMEOUT,
-    for a in attachments:
+                   label="verify")
-        cmd += ["-f", a]
-    cmd += ["--dir", BASE_DIR]
-    if OPENCODE_MODEL:
-        cmd += ["-m", OPENCODE_MODEL]
-    cmd.append(prompt)
-    import subprocess
-    proc = subprocess.run(cmd, capture_output=True, text=True,
-                          encoding="utf-8", errors="replace",
-                          cwd=BASE_DIR, timeout=900)
-    out = proc.stdout or ""
-    if not out.strip() and proc.stderr:
-        raise RuntimeError(f"opencode produced no stdout: "
-                           f"{proc.stderr[-400:]}")
-    return out
 
 
 def _now() -> str: