Resilient LLM runner: model fallback chain, provider-death blacklist, cooldown, ANSI-repair
- llm.py: single retry/fallback/cooldown authority used by analyze/verify/pipeline - provider [404] blacklists model + switches; rate limit escalates then cooldown - unparsable model output (bad JSON/ANSI noise) skips to next model - verify.py: DS_VERIFY_TIMEOUT, true rate-limit handling, honest FAILED reporting - pipeline_all.py: honors model, reports verify status accurately - aggregate.py: skip epNNNN_corrected.json strays (was duplicating ep 304) - gitignore data outputs; drop stray analysis/analyst/analysts dirs
This commit is contained in:
n3wk
+55
-76
@@ -17,6 +17,7 @@ import sys
|
||||
import time
|
||||
import traceback
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
@@ -143,94 +144,67 @@ def do_transcribe(model, ep: dict) -> None:
|
||||
pass
|
||||
|
||||
|
||||
from llm import run as llm_run, RateLimitError, ProviderError
|
||||
from llm import ANALYZE_TIMEOUT
|
||||
|
||||
|
||||
def run_analysis_with_timeout(ep: dict) -> tuple:
|
||||
"""
|
||||
Run analysis + verification using opencode subprocess with hard timeout.
|
||||
Returns (success: bool, result: dict | None, error: str | None)
|
||||
Run analysis + verification using the resilient opencode runner (model
|
||||
fallback chain + provider-death blacklist). Returns (success, result, error).
|
||||
"""
|
||||
from analyze import ANALYSIS_DIR, OPENCODE, build_prompt, strip_to_json, validate
|
||||
from analyze import ANALYSIS_DIR, build_prompt, validate, strip_to_json
|
||||
from analyze import ensure_content_match
|
||||
from verify import verify_episode
|
||||
from download import slug as ep_slug
|
||||
from export_txt import txt_path
|
||||
from config import BASE_DIR
|
||||
|
||||
from export_txt import export_txt
|
||||
|
||||
label = f"Ep {ep['ep_num']}"
|
||||
apath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json")
|
||||
|
||||
|
||||
# Check if already analyzed
|
||||
if os.path.exists(apath):
|
||||
with open(apath, encoding="utf-8") as f:
|
||||
existing = json.load(f)
|
||||
return (True, existing, None)
|
||||
|
||||
|
||||
# Build prompt
|
||||
prompt = build_prompt()
|
||||
attach = export_txt(ep)
|
||||
|
||||
# Run analysis with timeout + rate-limit retries
|
||||
cmd = [OPENCODE, "run", "--auto", "-f", attach, "--dir", BASE_DIR, prompt]
|
||||
|
||||
# Rate limit retry waits (seconds)
|
||||
rate_limit_waits = [10, 60, 180, 420, 600]
|
||||
|
||||
for attempt, wait_time in enumerate(rate_limit_waits, 1):
|
||||
try:
|
||||
print(f" [{time.strftime('%H:%M:%S')}] running analysis (attempt {attempt}, timeout {ANALYSIS_TIMEOUT}s)...", flush=True)
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace",
|
||||
cwd=BASE_DIR, timeout=ANALYSIS_TIMEOUT)
|
||||
out = proc.stdout or ""
|
||||
err = proc.stderr or ""
|
||||
combined = (out + err).lower()
|
||||
|
||||
if "rate limit" in combined or "429" in combined:
|
||||
raise RuntimeError("provider rate limit exceeded")
|
||||
|
||||
if not out.strip() and err:
|
||||
raise RuntimeError(f"opencode produced no stdout: {err[-400:]}")
|
||||
|
||||
# Parse result
|
||||
from analyze import _repair
|
||||
obj = json.loads(_repair(out), strict=False)
|
||||
validate(obj)
|
||||
|
||||
# Add metadata
|
||||
obj["ep_num"] = ep["ep_num"] if ep["ep_num"] else obj.get("ep_num")
|
||||
obj["title"] = obj.get("title") or ep["title"]
|
||||
obj["pub_date"] = ep["pub_date"]
|
||||
obj["_analyzed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
obj["_source_transcript"] = os.path.basename(transcript_path(ep))
|
||||
|
||||
# Save analysis
|
||||
from analyze import ANALYSIS_DIR as ANALYSIS_DIR2
|
||||
from download import slug as ep_slug
|
||||
apath = os.path.join(ANALYSIS_DIR2, f"{ep_slug(ep)}.json")
|
||||
with open(apath + ".tmp", "w", encoding="utf-8") as f:
|
||||
json.dump(obj, f, ensure_ascii=False, indent=1)
|
||||
os.replace(apath + ".tmp", apath)
|
||||
|
||||
# Now run verification
|
||||
print(f" [{time.strftime('%H:%M:%S')}] fact-checking...", flush=True)
|
||||
from verify import verify_episode
|
||||
verify_episode(ep)
|
||||
|
||||
return (True, obj, None)
|
||||
|
||||
except RuntimeError as e:
|
||||
if "rate limit" in str(e).lower():
|
||||
if attempt < len(rate_limit_waits):
|
||||
wait = rate_limit_waits[attempt]
|
||||
print(f" [{time.strftime('%H:%M:%S')}] rate limited (attempt {attempt}), sleeping {wait}s...", flush=True)
|
||||
time.sleep(wait)
|
||||
continue
|
||||
else:
|
||||
return (False, None, f"Rate limit exceeded after {attempt} attempts")
|
||||
else:
|
||||
return (False, None, str(e))
|
||||
except Exception as e:
|
||||
return (False, None, str(e))
|
||||
|
||||
return (False, None, "Max retries exceeded")
|
||||
|
||||
def parse_obj(raw: str) -> dict:
|
||||
obj = strip_to_json(raw)
|
||||
validate(obj)
|
||||
ensure_content_match(obj, attach)
|
||||
return obj
|
||||
|
||||
# Resilient runner handles model fallback + retries internally.
|
||||
try:
|
||||
print(f" [{time.strftime('%H:%M:%S')}] running analysis "
|
||||
f"(timeout {ANALYZE_TIMEOUT}s)...", flush=True)
|
||||
obj = llm_run(prompt, [attach], timeout=ANALYZE_TIMEOUT,
|
||||
label="analyze", parse=parse_obj)
|
||||
except (RateLimitError, ProviderError) as e:
|
||||
return (False, None, str(e))
|
||||
|
||||
obj["title"] = obj.get("title") or ep["title"]
|
||||
obj["pub_date"] = ep["pub_date"]
|
||||
obj["_analyzed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
obj["_source_transcript"] = os.path.basename(transcript_path(ep))
|
||||
obj["ep_num"] = ep["ep_num"] if ep["ep_num"] else obj.get("ep_num")
|
||||
|
||||
# Save analysis
|
||||
apath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json")
|
||||
with open(apath + ".tmp", "w", encoding="utf-8") as f:
|
||||
json.dump(obj, f, ensure_ascii=False, indent=1)
|
||||
os.replace(apath + ".tmp", apath)
|
||||
|
||||
# Now run verification
|
||||
print(f" [{time.strftime('%H:%M:%S')}] fact-checking...", flush=True)
|
||||
from verify import verify_episode
|
||||
verified = verify_episode(ep)
|
||||
|
||||
return (True, obj, None if verified is not None else "verification failed")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -304,9 +278,14 @@ def main() -> None:
|
||||
success, result, error = run_analysis_with_timeout(ep)
|
||||
if success:
|
||||
done_a += 1
|
||||
print(f" [{time.strftime('%H:%M:%S')}] analyzed {label} (TI={result['transgression']['overall_ti']}, "
|
||||
f"{len(result['talking_points'])} points, {len(result['notable_quotes'])} quotes)", flush=True)
|
||||
print(f" [{time.strftime('%H:%M:%S')}] verified {label}", flush=True)
|
||||
ti = result.get("transgression", {}).get("overall_ti", "?")
|
||||
print(f" [{time.strftime('%H:%M:%S')}] analyzed {label} (TI={ti}, "
|
||||
f"{len(result.get('talking_points', []))} points, "
|
||||
f"{len(result.get('notable_quotes', []))} quotes)", flush=True)
|
||||
if error:
|
||||
print(f" [{time.strftime('%H:%M:%S')}] NOT verified {label}: {error}", flush=True)
|
||||
else:
|
||||
print(f" [{time.strftime('%H:%M:%S')}] verified {label}", flush=True)
|
||||
else:
|
||||
print(f" [{time.strftime('%H:%M:%S')}] ANALYSIS FAILED {label}: {error}", flush=True)
|
||||
except Exception as ex:
|
||||
|
||||
Reference in New Issue
Block a user