Resilient LLM runner: model fallback chain, provider-death blacklist, cooldown, ANSI-repair

- llm.py: single retry/fallback/cooldown authority used by analyze/verify/pipeline - provider [404] blacklists model + switches; rate limit escalates then cooldown - unparsable model output (bad JSON/ANSI noise) skips to next model - verify.py: DS_VERIFY_TIMEOUT, true rate-limit handling, honest FAILED reporting - pipeline_all.py: honors model, reports verify status accurately - aggregate.py: skip epNNNN_corrected.json strays (was duplicating ep 304) - gitignore data outputs; drop stray analysis/analyst/analysts dirs
2026-08-29 03:26:56 +02:00
parent 06724d7dc1
commit 2719e4319d
6 changed files with 463 additions and 226 deletions
@@ -0,0 +1,284 @@
+"""Resilient opencode runner for analysis/verify passes.
+
+Why this exists (after a string of outages):
+  * The opencode "Console" provider models are unstable: some get retired or
+    break upstream and then return "[404] Provider returned error" while still
+    appearing in `opencode models` (nemotron-3.5-lightning-free, 2026-08-25).
+    Retrying the SAME broken model loops for hours and always fails.
+  * Rate limits are account-wide (shared by all opencode/* models), so a 429
+    should wait out the cooldown, NOT flip models.
+  * Provider-dead errors should flip to the next WORKING model immediately.
+  * A model that returns text but garbage/non-JSON output should be skipped
+    in favor of the next candidate, not sent back through an outer retry loop.
+
+This module is the SINGLE retry/fallback/cooldown authority. Callers
+(analyze.py, verify.py, pipeline_all.py) just call run() and handle the
+outcome or the raised RateLimitError/ProviderError.
+"""
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+
+from config import BASE_DIR, DATA_DIR
+
+OPENCODE = os.environ.get(
+    "OPENCODE_BIN", os.path.expanduser(r"~\.opencode\bin\opencode.exe"))
+# analyzer model override, e.g. opencode/x-preview-f-free
+OPENCODE_MODEL = os.environ.get("OPENCODE_ANALYZE_MODEL", "").strip()
+ANALYZE_TIMEOUT = int(os.environ.get("DS_ANALYZE_TIMEOUT", "1800"))  # 30 min
+VERIFY_TIMEOUT = int(os.environ.get("DS_VERIFY_TIMEOUT", "900"))     # 15 min
+
+# Preferred model first; everything else is a fallback tried in order until
+# one returns usable output. Updated from `opencode models` 2026-08-25.
+MODEL_CHAIN = [
+    "opencode/big-pickle",
+    "opencode/nemotron-3-ultra-free",
+    "opencode/hy3-free",
+    "opencode/ling-3.0-flash-fin-free",
+    "opencode/mimo-v2.5-free",
+    "opencode/muse-spark-1.2-contributor-free",
+    "opencode/nemotron-3.5-lightning-free",
+]
+
+# Models that hard-failed with provider 404 / model-not-found at runtime.
+# Persisted so later runs skip them immediately instead of burning retries.
+BLACKLIST_JSON = os.path.join(DATA_DIR, "lm_blacklist.json")
+
+# Cooldown after consecutive rate limits, shared by every caller.
+COOLDOWN_SECONDS = int(os.environ.get("DS_RATE_COOLDOWN", "900"))
+_COOLDOWN = {"until": 0.0}
+
+# Escalating wait between attempts *for the same model* on any failure.
+WAITS = [10, 60, 180, 420, 600, 900]
+# Per-model retry budget for blips before moving to the next model.
+BLIP_RETRIES = 2
+# Rate-limit turns on the same model before bailing to the cooldown.
+RATE_ESCALATION = 3
+
+
+class RateLimitError(RuntimeError):
+    pass
+
+
+class ProviderError(RuntimeError):
+    pass
+
+
+# --- error classification -------------------------------------------------
+# ORDER MATTERS: rate first, then 404/model-death, then generic transient.
+_RATE_RE = re.compile(
+    r"rate limit|too many requests|\b429\b|quota exceeded", re.IGNORECASE)
+_DEAD_RE = re.compile(
+    r"\[404\]|provider returned error[^\n]*\b4\d\d\b|model not found|"
+    r"model_not_found|model does not exist|not available",
+    re.IGNORECASE)
+_TRANSIENT_RE = re.compile(
+    r"\b5\d\d\b|502|503|504|bad gateway|temporar|timed? ?out|connection|"
+    r"\bEOF\b|upstream request failed|http 500|500",
+    re.IGNORECASE)
+
+
+def _load_blacklist() -> list[str]:
+    try:
+        with open(BLACKLIST_JSON, encoding="utf-8") as f:
+            data = json.load(f)
+            return [m for m in data.get("models", []) if isinstance(m, str)]
+    except (OSError, json.JSONDecodeError):
+        return []
+
+
+def _save_blacklist(models: list[str]) -> None:
+    os.makedirs(os.path.dirname(BLACKLIST_JSON), exist_ok=True)
+    tmp = BLACKLIST_JSON + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump({"models": sorted(set(models))}, f, indent=1)
+    os.replace(tmp, BLACKLIST_JSON)
+
+
+def _blacklist_model(model: str) -> None:
+    models = _load_blacklist()
+    if model not in models:
+        models.append(model)
+        _save_blacklist(models)
+    print(f"  [lm] BLACKLISTED {model} (provider-dead)",
+          file=sys.stderr, flush=True)
+
+
+def active_models(env_override: str = "") -> list[str]:
+    """Ordered fallback chain, minus blacklisted. Env override goes first."""
+    blacklist = set(_load_blacklist())
+    chain: list[str] = []
+    if env_override and env_override not in blacklist:
+        chain.append(env_override)
+    for m in MODEL_CHAIN:
+        if m not in chain and m not in blacklist:
+            chain.append(m)
+    if not chain:
+        # everything blacklisted - reset and warn
+        print("  [lm] ALL models blacklisted; resetting blacklist",
+              file=sys.stderr, flush=True)
+        _save_blacklist([])
+        chain = [m for m in MODEL_CHAIN if m != env_override][:1]
+    return chain
+
+
+def classify_combined(out: str, err: str) -> str | None:
+    """Return 'rate' | 'dead' | 'transient' | None based on provider output."""
+    combined = (out + "\n" + err)
+    if _RATE_RE.search(combined):
+        return "rate"
+    if _DEAD_RE.search(combined):
+        return "dead"
+    if _TRANSIENT_RE.search(combined):
+        return "transient"
+    return None
+
+
+def run(prompt: str, attachments: list[str], *,
+        timeout: int | None = None, label: str = "opencode",
+        parse=None, env_override: str | None = None,
+        print_progress: bool = True):
+    """Resilient `opencode run` against the model chain.
+
+    Returns parse(raw_stdout) if `parse` is given (and it must return/raise),
+    else the raw stdout text. Raises RateLimitError or ProviderError once the
+    chain and escalation budgets are exhausted.
+    """
+    if env_override is None:
+        env_override = os.environ.get("OPENCODE_ANALYZE_MODEL", "").strip()
+    timeout = timeout or ANALYZE_TIMEOUT
+
+    if time.time() < _COOLDOWN["until"]:
+        raise RateLimitError(
+            f"in cooldown for {int(_COOLDOWN['until'] - time.time())}s")
+
+    models = active_models(env_override)
+    if not models:
+        raise ProviderError("no usable models after blacklist")
+
+    last_transient = None
+    last_rate = None
+    saw_rate = False
+    bad_output = []  # models that answered but failed `parse`
+
+    for model in models:
+        blips_left = BLIP_RETRIES
+        for round_no in range(RATE_ESCALATION + 1):
+            cmd = [OPENCODE, "run", "--auto"]
+            for a in attachments:
+                cmd += ["-f", a]
+            cmd += ["--dir", BASE_DIR, "-m", model, prompt]
+            if print_progress:
+                print(f"  [{label}] running analysis "
+                      f"(attempt {round_no + 1} ({model}, {timeout}s))...",
+                      flush=True)
+            t0 = time.time()
+            try:
+                proc = subprocess.run(
+                    cmd, capture_output=True, text=True,
+                    encoding="utf-8", errors="replace",
+                    cwd=BASE_DIR, timeout=timeout)
+            except subprocess.TimeoutExpired:
+                blips_left -= 1
+                last_transient = f"{model} hit timeout {timeout}s"
+                if blips_left <= 0:
+                    break  # next model
+                wait = min(60, 2 ** blips_left * 15)
+                if print_progress:
+                    print(f"  [{label}] timed out after {timeout}s; "
+                          f"retry in {wait}s", flush=True)
+                time.sleep(wait)
+                continue
+
+            out = proc.stdout or ""
+            err = proc.stderr or ""
+            cls = classify_combined(out, err)
+
+            if cls == "rate":
+                saw_rate = True
+                last_rate = (model, err[-200:])
+                if round_no >= RATE_ESCALATION:
+                    _COOLDOWN["until"] = time.time() + COOLDOWN_SECONDS
+                    raise RateLimitError(
+                        f"{model} rate-limited (entering "
+                        f"{COOLDOWN_SECONDS}s cooldown)")
+                wait = WAITS[min(round_no, len(WAITS) - 1)]
+                if print_progress:
+                    print(f"  [{label}] rate-limited ({model}); "
+                          f"sleeping {wait}s ...", flush=True)
+                time.sleep(wait)
+                continue  # same model, longer wait
+
+            if cls == "dead":
+                _blacklist_model(model)
+                break  # next model
+
+            if cls == "transient":
+                blips_left -= 1
+                last_transient = f"{model}: {err[-200:]}"
+                if blips_left <= 0:
+                    break  # next model
+                wait = min(60, 2 ** blips_left * 15)
+                if print_progress:
+                    print(f"  [{label}] transient error ({model}); "
+                          f"retry in {wait}s", flush=True)
+                time.sleep(wait)
+                continue
+
+            if not out.strip():
+                blips_left -= 1
+                last_transient = f"{model}: no stdout ({err[-200:].strip()})"
+                if blips_left <= 0:
+                    break  # next model
+                wait = min(60, 2 ** blips_left * 15)
+                if print_progress:
+                    print(f"  [{label}] no output from {model}; "
+                          f"retry in {wait}s", flush=True)
+                time.sleep(wait)
+                continue
+
+            elapsed = time.time() - t0
+            if parse is not None:
+                try:
+                    result = parse(out)
+                except Exception as e:  # bad output: skip this model
+                    bad_output.append((model, str(e)[:120]))
+                    if print_progress:
+                        print(f"  [{label}] {model} gave unparsable output "
+                              f"({str(e)[:90]}); next model", flush=True)
+                    break
+                if print_progress:
+                    print(f"  [{label}] {model} OK in {elapsed:.0f}s",
+                          flush=True)
+                return result
+            if print_progress:
+                print(f"  [{label}] {model} returned "
+                      f"{len(out)} chars in {elapsed:.0f}s", flush=True)
+            return out
+
+    if saw_rate:
+        _COOLDOWN["until"] = time.time() + COOLDOWN_SECONDS
+        raise RateLimitError("all models rate-limited: "
+                             f"{last_rate[1] if last_rate else 'unknown'}")
+    raise ProviderError((
+        f"no model produced valid output; models_with_bad_output="
+        f"{bad_output}; last_transient={last_transient}"
+    ))
+
+
+if __name__ == "__main__":
+    # smoke test: pick the first live model and ask it to say hello
+    t0 = time.time()
+    try:
+        res = run("Reply with exactly: OK", [], timeout=90, label="smoke")
+        print(f"SMOKE OK: {res.strip()[:80]} in {time.time()-t0:.0f}s")
+        print(f"active models: {active_models()}")
+    except RateLimitError as e:
+        print(f"SMOKE RATE-LIMITED: {e}")
+        sys.exit(3)
+    except ProviderError as e:
+        print(f"SMOKE EXHAUSTED: {e}")
+        sys.exit(4)