"""Resilient opencode runner for analysis/verify passes. Why this exists (after a string of outages): * The opencode "Console" provider models are unstable: some get retired or break upstream and then return "[404] Provider returned error" while still appearing in `opencode models` (nemotron-3.5-lightning-free, 2026-08-25). Retrying the SAME broken model loops for hours and always fails. * Rate limits are account-wide (shared by all opencode/* models), so a 429 should wait out the cooldown, NOT flip models. * Provider-dead errors should flip to the next WORKING model immediately. * A model that returns text but garbage/non-JSON output should be skipped in favor of the next candidate, not sent back through an outer retry loop. This module is the SINGLE retry/fallback/cooldown authority. Callers (analyze.py, verify.py, pipeline_all.py) just call run() and handle the outcome or the raised RateLimitError/ProviderError. """ import json import os import re import subprocess import sys import time from config import BASE_DIR, DATA_DIR OPENCODE = os.environ.get( "OPENCODE_BIN", os.path.expanduser(r"~\.opencode\bin\opencode.exe")) # analyzer model override, e.g. opencode/x-preview-f-free OPENCODE_MODEL = os.environ.get("OPENCODE_ANALYZE_MODEL", "").strip() ANALYZE_TIMEOUT = int(os.environ.get("DS_ANALYZE_TIMEOUT", "1800")) # 30 min VERIFY_TIMEOUT = int(os.environ.get("DS_VERIFY_TIMEOUT", "900")) # 15 min # Preferred model first; everything else is a fallback tried in order until # one returns usable output. Updated from `opencode models` 2026-08-25. MODEL_CHAIN = [ "opencode/big-pickle", "opencode/nemotron-3-ultra-free", "opencode/hy3-free", "opencode/ling-3.0-flash-fin-free", "opencode/mimo-v2.5-free", "opencode/muse-spark-1.2-contributor-free", "opencode/nemotron-3.5-lightning-free", ] # Models that hard-failed with provider 404 / model-not-found at runtime. # Persisted so later runs skip them immediately instead of burning retries. BLACKLIST_JSON = os.path.join(DATA_DIR, "lm_blacklist.json") # Cooldown after consecutive rate limits, shared by every caller. COOLDOWN_SECONDS = int(os.environ.get("DS_RATE_COOLDOWN", "900")) _COOLDOWN = {"until": 0.0} # Escalating wait between attempts *for the same model* on any failure. WAITS = [10, 60, 180, 420, 600, 900] # Per-model retry budget for blips before moving to the next model. BLIP_RETRIES = 2 # Rate-limit turns on the same model before bailing to the cooldown. RATE_ESCALATION = 3 class RateLimitError(RuntimeError): pass class ProviderError(RuntimeError): pass # --- error classification ------------------------------------------------- # ORDER MATTERS: rate first, then 404/model-death, then generic transient. _RATE_RE = re.compile( r"rate limit|too many requests|\b429\b|quota exceeded", re.IGNORECASE) _DEAD_RE = re.compile( r"\[404\]|provider returned error[^\n]*\b4\d\d\b|model not found|" r"model_not_found|model does not exist|not available", re.IGNORECASE) _TRANSIENT_RE = re.compile( r"\b5\d\d\b|502|503|504|bad gateway|temporar|timed? ?out|connection|" r"\bEOF\b|upstream request failed|http 500|500", re.IGNORECASE) def _load_blacklist() -> list[str]: try: with open(BLACKLIST_JSON, encoding="utf-8") as f: data = json.load(f) return [m for m in data.get("models", []) if isinstance(m, str)] except (OSError, json.JSONDecodeError): return [] def _save_blacklist(models: list[str]) -> None: os.makedirs(os.path.dirname(BLACKLIST_JSON), exist_ok=True) tmp = BLACKLIST_JSON + ".tmp" with open(tmp, "w", encoding="utf-8") as f: json.dump({"models": sorted(set(models))}, f, indent=1) os.replace(tmp, BLACKLIST_JSON) def _blacklist_model(model: str) -> None: models = _load_blacklist() if model not in models: models.append(model) _save_blacklist(models) print(f" [lm] BLACKLISTED {model} (provider-dead)", file=sys.stderr, flush=True) def active_models(env_override: str = "") -> list[str]: """Ordered fallback chain, minus blacklisted. Env override goes first.""" blacklist = set(_load_blacklist()) chain: list[str] = [] if env_override and env_override not in blacklist: chain.append(env_override) for m in MODEL_CHAIN: if m not in chain and m not in blacklist: chain.append(m) if not chain: # everything blacklisted - reset and warn print(" [lm] ALL models blacklisted; resetting blacklist", file=sys.stderr, flush=True) _save_blacklist([]) chain = [m for m in MODEL_CHAIN if m != env_override][:1] return chain def classify_combined(out: str, err: str) -> str | None: """Return 'rate' | 'dead' | 'transient' | None based on provider output.""" combined = (out + "\n" + err) if _RATE_RE.search(combined): return "rate" if _DEAD_RE.search(combined): return "dead" if _TRANSIENT_RE.search(combined): return "transient" return None def run(prompt: str, attachments: list[str], *, timeout: int | None = None, label: str = "opencode", parse=None, env_override: str | None = None, print_progress: bool = True): """Resilient `opencode run` against the model chain. Returns parse(raw_stdout) if `parse` is given (and it must return/raise), else the raw stdout text. Raises RateLimitError or ProviderError once the chain and escalation budgets are exhausted. """ if env_override is None: env_override = os.environ.get("OPENCODE_ANALYZE_MODEL", "").strip() timeout = timeout or ANALYZE_TIMEOUT if time.time() < _COOLDOWN["until"]: raise RateLimitError( f"in cooldown for {int(_COOLDOWN['until'] - time.time())}s") models = active_models(env_override) if not models: raise ProviderError("no usable models after blacklist") last_transient = None last_rate = None saw_rate = False bad_output = [] # models that answered but failed `parse` for model in models: blips_left = BLIP_RETRIES for round_no in range(RATE_ESCALATION + 1): cmd = [OPENCODE, "run", "--auto"] for a in attachments: cmd += ["-f", a] cmd += ["--dir", BASE_DIR, "-m", model, prompt] if print_progress: print(f" [{label}] running analysis " f"(attempt {round_no + 1} ({model}, {timeout}s))...", flush=True) t0 = time.time() try: proc = subprocess.run( cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", cwd=BASE_DIR, timeout=timeout) except subprocess.TimeoutExpired: blips_left -= 1 last_transient = f"{model} hit timeout {timeout}s" if blips_left <= 0: break # next model wait = min(60, 2 ** blips_left * 15) if print_progress: print(f" [{label}] timed out after {timeout}s; " f"retry in {wait}s", flush=True) time.sleep(wait) continue out = proc.stdout or "" err = proc.stderr or "" cls = classify_combined(out, err) if cls == "rate": saw_rate = True last_rate = (model, err[-200:]) if round_no >= RATE_ESCALATION: _COOLDOWN["until"] = time.time() + COOLDOWN_SECONDS raise RateLimitError( f"{model} rate-limited (entering " f"{COOLDOWN_SECONDS}s cooldown)") wait = WAITS[min(round_no, len(WAITS) - 1)] if print_progress: print(f" [{label}] rate-limited ({model}); " f"sleeping {wait}s ...", flush=True) time.sleep(wait) continue # same model, longer wait if cls == "dead": _blacklist_model(model) break # next model if cls == "transient": blips_left -= 1 last_transient = f"{model}: {err[-200:]}" if blips_left <= 0: break # next model wait = min(60, 2 ** blips_left * 15) if print_progress: print(f" [{label}] transient error ({model}); " f"retry in {wait}s", flush=True) time.sleep(wait) continue if not out.strip(): blips_left -= 1 last_transient = f"{model}: no stdout ({err[-200:].strip()})" if blips_left <= 0: break # next model wait = min(60, 2 ** blips_left * 15) if print_progress: print(f" [{label}] no output from {model}; " f"retry in {wait}s", flush=True) time.sleep(wait) continue elapsed = time.time() - t0 if parse is not None: try: result = parse(out) except Exception as e: # bad output: skip this model bad_output.append((model, str(e)[:120])) if print_progress: print(f" [{label}] {model} gave unparsable output " f"({str(e)[:90]}); next model", flush=True) break if print_progress: print(f" [{label}] {model} OK in {elapsed:.0f}s", flush=True) return result if print_progress: print(f" [{label}] {model} returned " f"{len(out)} chars in {elapsed:.0f}s", flush=True) return out if saw_rate: _COOLDOWN["until"] = time.time() + COOLDOWN_SECONDS raise RateLimitError("all models rate-limited: " f"{last_rate[1] if last_rate else 'unknown'}") raise ProviderError(( f"no model produced valid output; models_with_bad_output=" f"{bad_output}; last_transient={last_transient}" )) if __name__ == "__main__": # smoke test: pick the first live model and ask it to say hello t0 = time.time() try: res = run("Reply with exactly: OK", [], timeout=90, label="smoke") print(f"SMOKE OK: {res.strip()[:80]} in {time.time()-t0:.0f}s") print(f"active models: {active_models()}") except RateLimitError as e: print(f"SMOKE RATE-LIMITED: {e}") sys.exit(3) except ProviderError as e: print(f"SMOKE EXHAUSTED: {e}") sys.exit(4)