n3wk
351 lines
14 KiB
Python
351 lines
14 KiB
Python
"""Full-backlog driver: Ep 1 -> latest. One pass, fully resumable, verbose.
|
|
|
|
Per episode:
|
|
1. skip if transcript + verified analysis already exist
|
|
2. download mp3 (skipped if present)
|
|
3. transcribe (re-does partial transcripts, e.g. old smoke tests)
|
|
4. analyze via opencode (fresh session, WITH HARD TIMEOUT) + dossier fact-check
|
|
5. every 10 episodes: rebuild FTS index + master JSON
|
|
|
|
Run: python pipeline_all.py [--eps 84 87 89 ...] [--start-ep N] [--dry-run]
|
|
Log: stdout (use with redirection)
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import subprocess
|
|
|
|
try:
|
|
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
|
except Exception:
|
|
pass
|
|
|
|
from config import TRANSCRIPT_DIR, KEEP_AUDIO
|
|
from feed import load_episodes
|
|
from download import audio_path, slug as ep_slug, download_episode
|
|
from preprocess import prepare_audio, prepared_path
|
|
from transcribe import transcript_path
|
|
from lexicon import build_lexicon, tags_for_text
|
|
from export_txt import export_txt
|
|
|
|
# seconds between opencode calls - provider rate limits are real
|
|
ANALYSIS_PAUSE = int(os.environ.get("DS_ANALYZE_PAUSE", "45"))
|
|
# hard timeout for entire analysis (seconds) - opencode can hang
|
|
ANALYSIS_TIMEOUT = int(os.environ.get("DS_ANALYZE_TIMEOUT", "1800")) # 30 minutes
|
|
|
|
# memory guard threshold (bytes) - pause if available RAM drops below this
|
|
MIN_FREE_RAM = int(os.environ.get("DS_MIN_FREE_RAM", str(1024 * 1024 * 1024))) # 1 GB
|
|
|
|
# encoding validation interval (episodes)
|
|
ENCODING_CHECK_INTERVAL = int(os.environ.get("DS_ENCODING_CHECK", "50"))
|
|
|
|
|
|
def has_partial(transcript_file: str) -> bool:
|
|
try:
|
|
with open(transcript_file, encoding="utf-8") as f:
|
|
return json.load(f).get("partial_minutes") is not None
|
|
except (OSError, json.JSONDecodeError):
|
|
return False
|
|
|
|
|
|
def verify_utf8_encoding(root: Path = Path('.')) -> tuple[int, int]:
|
|
"""Verify all JSON files are valid UTF-8. Returns (errors, files_checked)."""
|
|
errors = 0
|
|
checked = 0
|
|
for path in Path('.').rglob("*.json"):
|
|
if any(part.startswith('.') for part in path.parts):
|
|
continue
|
|
if path.name.startswith('.'):
|
|
continue
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as f:
|
|
json.load(path)
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as e:
|
|
print(f" [ENCODING ERROR] {path}: {e}", flush=True)
|
|
errors += 1
|
|
checked += 1
|
|
return errors, checked
|
|
|
|
|
|
def has_partial(transcript_file: str) -> bool:
|
|
try:
|
|
with open(transcript_file, encoding="utf-8") as f:
|
|
return json.load(f).get("partial_minutes") is not None
|
|
except (OSError, json.JSONDecodeError):
|
|
return False
|
|
|
|
|
|
def check_memory() -> bool:
|
|
"""Check if we have enough free RAM. Return True if OK, False if low."""
|
|
try:
|
|
import psutil
|
|
avail = psutil.virtual_memory().available
|
|
if avail < MIN_FREE_RAM:
|
|
return False
|
|
except ImportError:
|
|
pass # psutil not installed, skip check
|
|
except Exception:
|
|
pass
|
|
return True
|
|
|
|
|
|
def wait_for_memory() -> None:
|
|
"""Block until enough RAM is available."""
|
|
print(f" [MEM] Low memory (< {MIN_FREE_RAM/1e9:.1f} GB free), waiting...", flush=True)
|
|
while not check_memory():
|
|
time.sleep(30)
|
|
print(f" [MEM] Memory recovered, continuing...", flush=True)
|
|
|
|
|
|
def do_transcribe(model, ep: dict) -> None:
|
|
from transcribe import transcribe_episode
|
|
from preprocess import load_pcm
|
|
src = audio_path(ep)
|
|
if not os.path.exists(src):
|
|
print(f" [{time.strftime('%H:%M:%S')}] downloading {ep['audio_url'][-40:]} ...", flush=True)
|
|
download_episode(ep)
|
|
prep = prepared_path(ep)
|
|
ok = False
|
|
t0 = time.time()
|
|
try:
|
|
# Check memory before loading episode
|
|
wait_for_memory()
|
|
print(f" [{time.strftime('%H:%M:%S')}] transcribing Ep {ep['ep_num']} ({ep['title'][:50]})", flush=True)
|
|
result = transcribe_episode(model, ep, None)
|
|
outp = transcript_path(ep)
|
|
with open(outp + ".tmp", "w", encoding="utf-8") as f:
|
|
json.dump(result, f, ensure_ascii=False)
|
|
os.replace(outp + ".tmp", outp)
|
|
dur = result["duration_sec"] or 1
|
|
rate = dur / max(time.time() - t0, 0.001)
|
|
lex = build_lexicon()
|
|
tags = set()
|
|
for seg in result["segments"]:
|
|
tags.update(tags_for_text(seg["text"], lex))
|
|
print(f" [{time.strftime('%H:%M:%S')}] TRANSCRIBED: {len(result['segments'])} segments, "
|
|
f"{dur/60:.0f} min, {rate:.1f}x realtime", flush=True)
|
|
print(f" TAGS: {' '.join(sorted('#' + t for t in tags)) or '(none)'}", flush=True)
|
|
ok = True
|
|
finally:
|
|
if os.path.exists(prep):
|
|
try:
|
|
os.remove(prep)
|
|
except OSError:
|
|
pass
|
|
if not KEEP_AUDIO and ok and os.path.exists(src):
|
|
try:
|
|
os.remove(src)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def run_analysis_with_timeout(ep: dict) -> tuple:
|
|
"""
|
|
Run analysis + verification using opencode subprocess with hard timeout.
|
|
Returns (success: bool, result: dict | None, error: str | None)
|
|
"""
|
|
from analyze import ANALYSIS_DIR, OPENCODE, build_prompt, strip_to_json, validate
|
|
from verify import verify_episode
|
|
from download import slug as ep_slug
|
|
from export_txt import txt_path
|
|
from config import BASE_DIR
|
|
|
|
label = f"Ep {ep['ep_num']}"
|
|
apath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json")
|
|
|
|
# Check if already analyzed
|
|
if os.path.exists(apath):
|
|
with open(apath, encoding="utf-8") as f:
|
|
existing = json.load(f)
|
|
return (True, existing, None)
|
|
|
|
# Build prompt
|
|
prompt = build_prompt()
|
|
attach = export_txt(ep)
|
|
|
|
# Run analysis with timeout + rate-limit retries
|
|
cmd = [OPENCODE, "run", "--auto", "-f", attach, "--dir", BASE_DIR, prompt]
|
|
|
|
# Rate limit retry waits (seconds)
|
|
rate_limit_waits = [10, 60, 180, 420, 600]
|
|
|
|
for attempt, wait_time in enumerate(rate_limit_waits, 1):
|
|
try:
|
|
print(f" [{time.strftime('%H:%M:%S')}] running analysis (attempt {attempt}, timeout {ANALYSIS_TIMEOUT}s)...", flush=True)
|
|
proc = subprocess.run(cmd, capture_output=True, text=True,
|
|
encoding="utf-8", errors="replace",
|
|
cwd=BASE_DIR, timeout=ANALYSIS_TIMEOUT)
|
|
out = proc.stdout or ""
|
|
err = proc.stderr or ""
|
|
combined = (out + err).lower()
|
|
|
|
if "rate limit" in combined or "429" in combined:
|
|
raise RuntimeError("provider rate limit exceeded")
|
|
|
|
if not out.strip() and err:
|
|
raise RuntimeError(f"opencode produced no stdout: {err[-400:]}")
|
|
|
|
# Parse result
|
|
from analyze import _repair
|
|
obj = json.loads(_repair(out), strict=False)
|
|
validate(obj)
|
|
|
|
# Add metadata
|
|
obj["ep_num"] = ep["ep_num"] if ep["ep_num"] else obj.get("ep_num")
|
|
obj["title"] = obj.get("title") or ep["title"]
|
|
obj["pub_date"] = ep["pub_date"]
|
|
obj["_analyzed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
obj["_source_transcript"] = os.path.basename(transcript_path(ep))
|
|
|
|
# Save analysis
|
|
from analyze import ANALYSIS_DIR as ANALYSIS_DIR2
|
|
from download import slug as ep_slug
|
|
apath = os.path.join(ANALYSIS_DIR2, f"{ep_slug(ep)}.json")
|
|
with open(apath + ".tmp", "w", encoding="utf-8") as f:
|
|
json.dump(obj, f, ensure_ascii=False, indent=1)
|
|
os.replace(apath + ".tmp", apath)
|
|
|
|
# Now run verification
|
|
print(f" [{time.strftime('%H:%M:%S')}] fact-checking...", flush=True)
|
|
from verify import verify_episode
|
|
verify_episode(ep)
|
|
|
|
return (True, obj, None)
|
|
|
|
except RuntimeError as e:
|
|
if "rate limit" in str(e).lower():
|
|
if attempt < len(rate_limit_waits):
|
|
wait = rate_limit_waits[attempt]
|
|
print(f" [{time.strftime('%H:%M:%S')}] rate limited (attempt {attempt}), sleeping {wait}s...", flush=True)
|
|
time.sleep(wait)
|
|
continue
|
|
else:
|
|
return (False, None, f"Rate limit exceeded after {attempt} attempts")
|
|
else:
|
|
return (False, None, str(e))
|
|
except Exception as e:
|
|
return (False, None, str(e))
|
|
|
|
return (False, None, "Max retries exceeded")
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--eps", nargs="*", type=int, default=None,
|
|
help="Specific episode numbers to process (default: all from start-ep)")
|
|
ap.add_argument("--start-ep", type=int, default=0)
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
args = ap.parse_args()
|
|
|
|
from faster_whisper import WhisperModel
|
|
from config import WHISPER_MODEL, DEVICE, COMPUTE_TYPE
|
|
from analyze import analyze_episode, OPENCODE
|
|
from verify import verify_episode
|
|
|
|
eps = load_episodes()
|
|
if args.eps:
|
|
eps = [e for e in eps if e["ep_num"] in args.eps]
|
|
else:
|
|
eps = [e for e in eps if (e["ep_num"] or 0) >= args.start_ep]
|
|
|
|
print(f"[{time.strftime('%H:%M:%S')}] pipeline over {len(eps)} episodes "
|
|
f"(model={WHISPER_MODEL}/{DEVICE}, analyzer={os.path.basename(OPENCODE)})", flush=True)
|
|
|
|
if args.dry_run:
|
|
for e in eps[:20]:
|
|
tp = transcript_path(e)
|
|
t_exists = os.path.exists(tp) and not has_partial(tp)
|
|
a_exists = os.path.exists(os.path.join(
|
|
os.path.dirname(TRANSCRIPT_DIR), "analysis", f"{e['ep_num']:04d}.json")) if e["ep_num"] else False
|
|
print(f" ep {e['ep_num']}: transcript={'Y' if t_exists else 'N'} analysis={'Y' if a_exists else 'N'}", flush=True)
|
|
return
|
|
|
|
model = None
|
|
done_t = done_a = 0
|
|
t_start = time.time()
|
|
|
|
for i, ep in enumerate(eps, 1):
|
|
num = ep["ep_num"] if ep["ep_num"] else ep["title"][:30]
|
|
tp = transcript_path(ep)
|
|
|
|
need_transcript = (not os.path.exists(tp)) or has_partial(tp)
|
|
if need_transcript and model is None:
|
|
print(f"[{time.strftime('%H:%M:%S')}] loading whisper model '{WHISPER_MODEL}'...", flush=True)
|
|
model = WhisperModel(WHISPER_MODEL, device=DEVICE, compute_type=COMPUTE_TYPE)
|
|
|
|
if has_partial(tp):
|
|
print(f"[{time.strftime('%H:%M:%S')}] [{i}/{len(eps)}] Ep {num}: redo partial transcript", flush=True)
|
|
os.remove(tp)
|
|
elif need_transcript:
|
|
print(f"[{time.strftime('%H:%M:%S')}] [{i}/{len(eps)}] Ep {num}: transcribing", flush=True)
|
|
|
|
if need_transcript:
|
|
try:
|
|
do_transcribe(model, ep)
|
|
done_t += 1
|
|
# keep the searchable index fresh-ish
|
|
if done_t % 10 == 0:
|
|
import index as index_mod
|
|
print(f" [{time.strftime('%H:%M:%S')}] rebuilding search index...", flush=True)
|
|
index_mod.index_transcripts()
|
|
except Exception as ex:
|
|
print(f" [{time.strftime('%H:%M:%S')}] TRANSCRIBE FAILED, skipping episode: {ex}", flush=True)
|
|
traceback.print_exc()
|
|
continue
|
|
|
|
# analysis + fact-check (with hard timeout)
|
|
label = f"Ep {num}"
|
|
print(f" [{time.strftime('%H:%M:%S')}] analyzing {label} ...", flush=True)
|
|
try:
|
|
success, result, error = run_analysis_with_timeout(ep)
|
|
if success:
|
|
done_a += 1
|
|
print(f" [{time.strftime('%H:%M:%S')}] analyzed {label} (TI={result['transgression']['overall_ti']}, "
|
|
f"{len(result['talking_points'])} points, {len(result['notable_quotes'])} quotes)", flush=True)
|
|
print(f" [{time.strftime('%H:%M:%S')}] verified {label}", flush=True)
|
|
else:
|
|
print(f" [{time.strftime('%H:%M:%S')}] ANALYSIS FAILED {label}: {error}", flush=True)
|
|
except Exception as ex:
|
|
print(f" [{time.strftime('%H:%M:%S')}] ANALYZE FAILED {label}: {ex}", flush=True)
|
|
traceback.print_exc()
|
|
|
|
# pacing: keep the provider happy
|
|
print(f" [{time.strftime('%H:%M:%S')}] pacing pause {ANALYSIS_PAUSE}s ...", flush=True)
|
|
time.sleep(ANALYSIS_PAUSE)
|
|
|
|
if i % 10 == 0:
|
|
print(f" [{time.strftime('%H:%M:%S')}] rebuilding search index (periodic)...", flush=True)
|
|
import index as index_mod
|
|
index_mod.index_transcripts()
|
|
from aggregate import aggregate as _agg
|
|
_agg()
|
|
|
|
# Periodic encoding verification
|
|
if i % ENCODING_CHECK_INTERVAL == 0:
|
|
print(f" [{time.strftime('%H:%M:%S')}] verifying UTF-8 encoding...", flush=True)
|
|
errors, checked = verify_utf8_encoding()
|
|
if errors:
|
|
print(f" [WARNING] Found {errors} files with encoding issues!", flush=True)
|
|
else:
|
|
print(f" [OK] UTF-8 encoding verified for {checked} files", flush=True)
|
|
|
|
hrs = (time.time() - t_start) / 3600
|
|
rate_eps = i / hrs
|
|
remaining = (len(eps) - i) / max(rate_eps, 0.01)
|
|
print(f"[{time.strftime('%H:%M:%S')}] == progress: {i}/{len(eps)} eps "
|
|
f"({rate_eps:.1f} eps/hr, ~{remaining:.0f} hr left) ==", flush=True)
|
|
|
|
print(f"[{time.strftime('%H:%M:%S')}] final index rebuild...", flush=True)
|
|
import index as index_mod
|
|
index_mod.index_transcripts()
|
|
from aggregate import aggregate as _agg
|
|
_agg()
|
|
print(f"[{time.strftime('%H:%M:%S')}] PIPELINE COMPLETE: transcribed={done_t} analyzed={done_a}", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |