n3wk
2719e4319d
- llm.py: single retry/fallback/cooldown authority used by analyze/verify/pipeline - provider [404] blacklists model + switches; rate limit escalates then cooldown - unparsable model output (bad JSON/ANSI noise) skips to next model - verify.py: DS_VERIFY_TIMEOUT, true rate-limit handling, honest FAILED reporting - pipeline_all.py: honors model, reports verify status accurately - aggregate.py: skip epNNNN_corrected.json strays (was duplicating ep 304) - gitignore data outputs; drop stray analysis/analyst/analysts dirs
284 lines
11 KiB
Python
284 lines
11 KiB
Python
"""Resilient opencode runner for analysis/verify passes.
|
|
|
|
Why this exists (after a string of outages):
|
|
* The opencode "Console" provider models are unstable: some get retired or
|
|
break upstream and then return "[404] Provider returned error" while still
|
|
appearing in `opencode models` (nemotron-3.5-lightning-free, 2026-08-25).
|
|
Retrying the SAME broken model loops for hours and always fails.
|
|
* Rate limits are account-wide (shared by all opencode/* models), so a 429
|
|
should wait out the cooldown, NOT flip models.
|
|
* Provider-dead errors should flip to the next WORKING model immediately.
|
|
* A model that returns text but garbage/non-JSON output should be skipped
|
|
in favor of the next candidate, not sent back through an outer retry loop.
|
|
|
|
This module is the SINGLE retry/fallback/cooldown authority. Callers
|
|
(analyze.py, verify.py, pipeline_all.py) just call run() and handle the
|
|
outcome or the raised RateLimitError/ProviderError.
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from config import BASE_DIR, DATA_DIR
|
|
|
|
OPENCODE = os.environ.get(
|
|
"OPENCODE_BIN", os.path.expanduser(r"~\.opencode\bin\opencode.exe"))
|
|
# analyzer model override, e.g. opencode/x-preview-f-free
|
|
OPENCODE_MODEL = os.environ.get("OPENCODE_ANALYZE_MODEL", "").strip()
|
|
ANALYZE_TIMEOUT = int(os.environ.get("DS_ANALYZE_TIMEOUT", "1800")) # 30 min
|
|
VERIFY_TIMEOUT = int(os.environ.get("DS_VERIFY_TIMEOUT", "900")) # 15 min
|
|
|
|
# Preferred model first; everything else is a fallback tried in order until
|
|
# one returns usable output. Updated from `opencode models` 2026-08-25.
|
|
MODEL_CHAIN = [
|
|
"opencode/big-pickle",
|
|
"opencode/nemotron-3-ultra-free",
|
|
"opencode/hy3-free",
|
|
"opencode/ling-3.0-flash-fin-free",
|
|
"opencode/mimo-v2.5-free",
|
|
"opencode/muse-spark-1.2-contributor-free",
|
|
"opencode/nemotron-3.5-lightning-free",
|
|
]
|
|
|
|
# Models that hard-failed with provider 404 / model-not-found at runtime.
|
|
# Persisted so later runs skip them immediately instead of burning retries.
|
|
BLACKLIST_JSON = os.path.join(DATA_DIR, "lm_blacklist.json")
|
|
|
|
# Cooldown after consecutive rate limits, shared by every caller.
|
|
COOLDOWN_SECONDS = int(os.environ.get("DS_RATE_COOLDOWN", "900"))
|
|
_COOLDOWN = {"until": 0.0}
|
|
|
|
# Escalating wait between attempts *for the same model* on any failure.
|
|
WAITS = [10, 60, 180, 420, 600, 900]
|
|
# Per-model retry budget for blips before moving to the next model.
|
|
BLIP_RETRIES = 2
|
|
# Rate-limit turns on the same model before bailing to the cooldown.
|
|
RATE_ESCALATION = 3
|
|
|
|
|
|
class RateLimitError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class ProviderError(RuntimeError):
|
|
pass
|
|
|
|
|
|
# --- error classification -------------------------------------------------
|
|
# ORDER MATTERS: rate first, then 404/model-death, then generic transient.
|
|
_RATE_RE = re.compile(
|
|
r"rate limit|too many requests|\b429\b|quota exceeded", re.IGNORECASE)
|
|
_DEAD_RE = re.compile(
|
|
r"\[404\]|provider returned error[^\n]*\b4\d\d\b|model not found|"
|
|
r"model_not_found|model does not exist|not available",
|
|
re.IGNORECASE)
|
|
_TRANSIENT_RE = re.compile(
|
|
r"\b5\d\d\b|502|503|504|bad gateway|temporar|timed? ?out|connection|"
|
|
r"\bEOF\b|upstream request failed|http 500|500",
|
|
re.IGNORECASE)
|
|
|
|
|
|
def _load_blacklist() -> list[str]:
|
|
try:
|
|
with open(BLACKLIST_JSON, encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
return [m for m in data.get("models", []) if isinstance(m, str)]
|
|
except (OSError, json.JSONDecodeError):
|
|
return []
|
|
|
|
|
|
def _save_blacklist(models: list[str]) -> None:
|
|
os.makedirs(os.path.dirname(BLACKLIST_JSON), exist_ok=True)
|
|
tmp = BLACKLIST_JSON + ".tmp"
|
|
with open(tmp, "w", encoding="utf-8") as f:
|
|
json.dump({"models": sorted(set(models))}, f, indent=1)
|
|
os.replace(tmp, BLACKLIST_JSON)
|
|
|
|
|
|
def _blacklist_model(model: str) -> None:
|
|
models = _load_blacklist()
|
|
if model not in models:
|
|
models.append(model)
|
|
_save_blacklist(models)
|
|
print(f" [lm] BLACKLISTED {model} (provider-dead)",
|
|
file=sys.stderr, flush=True)
|
|
|
|
|
|
def active_models(env_override: str = "") -> list[str]:
|
|
"""Ordered fallback chain, minus blacklisted. Env override goes first."""
|
|
blacklist = set(_load_blacklist())
|
|
chain: list[str] = []
|
|
if env_override and env_override not in blacklist:
|
|
chain.append(env_override)
|
|
for m in MODEL_CHAIN:
|
|
if m not in chain and m not in blacklist:
|
|
chain.append(m)
|
|
if not chain:
|
|
# everything blacklisted - reset and warn
|
|
print(" [lm] ALL models blacklisted; resetting blacklist",
|
|
file=sys.stderr, flush=True)
|
|
_save_blacklist([])
|
|
chain = [m for m in MODEL_CHAIN if m != env_override][:1]
|
|
return chain
|
|
|
|
|
|
def classify_combined(out: str, err: str) -> str | None:
|
|
"""Return 'rate' | 'dead' | 'transient' | None based on provider output."""
|
|
combined = (out + "\n" + err)
|
|
if _RATE_RE.search(combined):
|
|
return "rate"
|
|
if _DEAD_RE.search(combined):
|
|
return "dead"
|
|
if _TRANSIENT_RE.search(combined):
|
|
return "transient"
|
|
return None
|
|
|
|
|
|
def run(prompt: str, attachments: list[str], *,
|
|
timeout: int | None = None, label: str = "opencode",
|
|
parse=None, env_override: str | None = None,
|
|
print_progress: bool = True):
|
|
"""Resilient `opencode run` against the model chain.
|
|
|
|
Returns parse(raw_stdout) if `parse` is given (and it must return/raise),
|
|
else the raw stdout text. Raises RateLimitError or ProviderError once the
|
|
chain and escalation budgets are exhausted.
|
|
"""
|
|
if env_override is None:
|
|
env_override = os.environ.get("OPENCODE_ANALYZE_MODEL", "").strip()
|
|
timeout = timeout or ANALYZE_TIMEOUT
|
|
|
|
if time.time() < _COOLDOWN["until"]:
|
|
raise RateLimitError(
|
|
f"in cooldown for {int(_COOLDOWN['until'] - time.time())}s")
|
|
|
|
models = active_models(env_override)
|
|
if not models:
|
|
raise ProviderError("no usable models after blacklist")
|
|
|
|
last_transient = None
|
|
last_rate = None
|
|
saw_rate = False
|
|
bad_output = [] # models that answered but failed `parse`
|
|
|
|
for model in models:
|
|
blips_left = BLIP_RETRIES
|
|
for round_no in range(RATE_ESCALATION + 1):
|
|
cmd = [OPENCODE, "run", "--auto"]
|
|
for a in attachments:
|
|
cmd += ["-f", a]
|
|
cmd += ["--dir", BASE_DIR, "-m", model, prompt]
|
|
if print_progress:
|
|
print(f" [{label}] running analysis "
|
|
f"(attempt {round_no + 1} ({model}, {timeout}s))...",
|
|
flush=True)
|
|
t0 = time.time()
|
|
try:
|
|
proc = subprocess.run(
|
|
cmd, capture_output=True, text=True,
|
|
encoding="utf-8", errors="replace",
|
|
cwd=BASE_DIR, timeout=timeout)
|
|
except subprocess.TimeoutExpired:
|
|
blips_left -= 1
|
|
last_transient = f"{model} hit timeout {timeout}s"
|
|
if blips_left <= 0:
|
|
break # next model
|
|
wait = min(60, 2 ** blips_left * 15)
|
|
if print_progress:
|
|
print(f" [{label}] timed out after {timeout}s; "
|
|
f"retry in {wait}s", flush=True)
|
|
time.sleep(wait)
|
|
continue
|
|
|
|
out = proc.stdout or ""
|
|
err = proc.stderr or ""
|
|
cls = classify_combined(out, err)
|
|
|
|
if cls == "rate":
|
|
saw_rate = True
|
|
last_rate = (model, err[-200:])
|
|
if round_no >= RATE_ESCALATION:
|
|
_COOLDOWN["until"] = time.time() + COOLDOWN_SECONDS
|
|
raise RateLimitError(
|
|
f"{model} rate-limited (entering "
|
|
f"{COOLDOWN_SECONDS}s cooldown)")
|
|
wait = WAITS[min(round_no, len(WAITS) - 1)]
|
|
if print_progress:
|
|
print(f" [{label}] rate-limited ({model}); "
|
|
f"sleeping {wait}s ...", flush=True)
|
|
time.sleep(wait)
|
|
continue # same model, longer wait
|
|
|
|
if cls == "dead":
|
|
_blacklist_model(model)
|
|
break # next model
|
|
|
|
if cls == "transient":
|
|
blips_left -= 1
|
|
last_transient = f"{model}: {err[-200:]}"
|
|
if blips_left <= 0:
|
|
break # next model
|
|
wait = min(60, 2 ** blips_left * 15)
|
|
if print_progress:
|
|
print(f" [{label}] transient error ({model}); "
|
|
f"retry in {wait}s", flush=True)
|
|
time.sleep(wait)
|
|
continue
|
|
|
|
if not out.strip():
|
|
blips_left -= 1
|
|
last_transient = f"{model}: no stdout ({err[-200:].strip()})"
|
|
if blips_left <= 0:
|
|
break # next model
|
|
wait = min(60, 2 ** blips_left * 15)
|
|
if print_progress:
|
|
print(f" [{label}] no output from {model}; "
|
|
f"retry in {wait}s", flush=True)
|
|
time.sleep(wait)
|
|
continue
|
|
|
|
elapsed = time.time() - t0
|
|
if parse is not None:
|
|
try:
|
|
result = parse(out)
|
|
except Exception as e: # bad output: skip this model
|
|
bad_output.append((model, str(e)[:120]))
|
|
if print_progress:
|
|
print(f" [{label}] {model} gave unparsable output "
|
|
f"({str(e)[:90]}); next model", flush=True)
|
|
break
|
|
if print_progress:
|
|
print(f" [{label}] {model} OK in {elapsed:.0f}s",
|
|
flush=True)
|
|
return result
|
|
if print_progress:
|
|
print(f" [{label}] {model} returned "
|
|
f"{len(out)} chars in {elapsed:.0f}s", flush=True)
|
|
return out
|
|
|
|
if saw_rate:
|
|
_COOLDOWN["until"] = time.time() + COOLDOWN_SECONDS
|
|
raise RateLimitError("all models rate-limited: "
|
|
f"{last_rate[1] if last_rate else 'unknown'}")
|
|
raise ProviderError((
|
|
f"no model produced valid output; models_with_bad_output="
|
|
f"{bad_output}; last_transient={last_transient}"
|
|
))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# smoke test: pick the first live model and ask it to say hello
|
|
t0 = time.time()
|
|
try:
|
|
res = run("Reply with exactly: OK", [], timeout=90, label="smoke")
|
|
print(f"SMOKE OK: {res.strip()[:80]} in {time.time()-t0:.0f}s")
|
|
print(f"active models: {active_models()}")
|
|
except RateLimitError as e:
|
|
print(f"SMOKE RATE-LIMITED: {e}")
|
|
sys.exit(3)
|
|
except ProviderError as e:
|
|
print(f"SMOKE EXHAUSTED: {e}")
|
|
sys.exit(4) |