From 06724d7dc1c03943ffc4621c0265df79c29a0a04 Mon Sep 17 00:00:00 2001 From: n3wk Date: Tue, 25 Aug 2026 03:07:44 +0200 Subject: [PATCH] Initial commit: source code only --- .gitignore | 33 +++++ README.md | 166 +++++++++++++++++++++ aggregate.py | 123 ++++++++++++++++ analyze.py | 289 +++++++++++++++++++++++++++++++++++++ check_encoding.py | 11 ++ check_progress.py | 116 +++++++++++++++ download.py | 134 +++++++++++++++++ export_txt.py | 45 ++++++ facts.py | 30 ++++ feed.py | 127 ++++++++++++++++ fix_encoding.py | 98 +++++++++++++ fix_mojibake.py | 119 +++++++++++++++ index.py | 115 +++++++++++++++ lexicon.py | 131 +++++++++++++++++ peek.py | 19 +++ pipeline_all.py | 351 +++++++++++++++++++++++++++++++++++++++++++++ preprocess.py | 82 +++++++++++ requirements.txt | 2 + rubric.py | 80 +++++++++++ run_all.py | 50 +++++++ search.py | 136 ++++++++++++++++++ status.py | 14 ++ transcribe.py | 170 ++++++++++++++++++++++ verify.py | 177 +++++++++++++++++++++++ verify_encoding.py | 67 +++++++++ 25 files changed, 2685 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 aggregate.py create mode 100644 analyze.py create mode 100644 check_encoding.py create mode 100644 check_progress.py create mode 100644 download.py create mode 100644 export_txt.py create mode 100644 facts.py create mode 100644 feed.py create mode 100644 fix_encoding.py create mode 100644 fix_mojibake.py create mode 100644 index.py create mode 100644 lexicon.py create mode 100644 peek.py create mode 100644 pipeline_all.py create mode 100644 preprocess.py create mode 100644 requirements.txt create mode 100644 rubric.py create mode 100644 run_all.py create mode 100644 search.py create mode 100644 status.py create mode 100644 transcribe.py create mode 100644 verify.py create mode 100644 verify_encoding.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..10a5b8d --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +venv/ +env/ +.env + +# Logs +*.log + +# Data files (too large for git, share via other means) +audio/ +transcripts/ +data/analysis/ +data/index.db +data/index.db-wal +data/index.db-shm + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +# Local config +config.py \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..2905d22 --- /dev/null +++ b/README.md @@ -0,0 +1,166 @@ +# Dick Show Index + +A complete pipeline to download, transcribe, analyze, and search all episodes of **The Dick Show** podcast. + +## What This Does + +1. **Downloads** all episodes from the official RSS feed (Libsyn) +2. **Transcribes** audio locally using Whisper (faster-whisper, CPU/GPU) +3. **Analyzes** each episode with an LLM (via opencode) to extract: + - Talking points with timestamps + - Arguments, ideas, and notable quotes + - Transgression Index (TI) 0-10 scoring + - Speaker attribution and content warnings +4. **Fact-checks** analyses against a verified research dossier +5. **Indexes** everything in SQLite FTS5 for instant search +5. **Exports** a master JSON (`data/dickshow_analysis.json`) for web apps + +## Quick Start + +```bash +# 1. Clone and install +git clone https://kode.hannesahlin.se/n3wk/dickshow-index.git +cd dickshow-index +./install.sh + +# 2. (Optional) Authenticate opencode for higher rate limits +opencode auth login + +# 3. Run the Rekieta arc (28 episodes): +python pipeline_all.py --eps 84 87 89 93 96 121 124 152 166 176 198 209 213 275 285 299 321 344 352 368 376 412 416 424 425 435 439 455 + +# Or run the full backlog (520 episodes): +python pipeline_all.py +``` + +## What You Get + +After running, you'll have: + +``` +data/ +├── episodes.jsonl # Episode metadata (title, date, audio URL, duration) +├── index.db # SQLite FTS5 search index +├── dickshow_analysis.json # Master analysis JSON (for web apps) +└── analysis/ + ├── ep0084.json # Per-episode analysis + fact-check + └── ... + +transcripts/ +├── ep0084.json # Raw transcript with timestamps +└── ... +``` + +## Search Examples + +```bash +# Simple search +python search.py "isom trademark" + +# Filter by tag +python search.py maddox --tag lolsuit + +# Phonetic search (Whisper mangles names) +python search.py "raketa OR riketa OR rakita" --tag nick_rekieta + +# Date range + tag +python search.py maddox --from 2017-01-01 --to 2018-12-31 --tag lolsuit + +# Context around matches +python search.py "pull up" --tag eric_july --context 2 +``` + +Output includes **timestamps** for deep-linking: +``` +[Ep 373 | Mon, 28 Aug 2023] 00:25:32-00:25:35 #nick_rekieta + "I was on Nick's show, Busting Balls, Nick Riketa." + -> https://dick.show/episode-373/ + mp3: https://traffic.libsyn.com/.../The_Dick_Show_373.mp3 +``` + +## Configuration + +Set via environment variables or edit `config.py`: + +| Variable | Default | Description | +|----------|---------|-------------| +| `OPENCODE_ANALYZE_MODEL` | `opencode/nemotron-3-ultra-free` | LLM for analysis | +| `DS_ANALYZE_PAUSE` | `45` | Seconds between analysis calls | +| `DS_ANALYZE_TIMEOUT` | `600` | Analysis timeout (seconds) | +| `DS_MIN_FREE_RAM` | `1073741824` (1GB) | Pause if RAM below this | +| `DS_MODEL` | `small` | Whisper model: `tiny`, `base`, `small`, `medium`, `large-v3` | +| `DS_DEVICE` | `cpu` | `cpu` or `cuda` | +| `DS_COMPUTE` | `int8` | `int8`, `float16`, `float32` | +| `DS_KEEP_AUDIO` | `0` | Keep MP3s after transcription (`1` to keep) | +| `DS_ENCODING_CHECK` | `50` | Episodes between UTF-8 verification | + +## Encoding & Character Handling + +All JSON files are written as **UTF-8** with `ensure_ascii=False`. The RSS feed contains Unicode characters (em/en dashes, accented characters) which are preserved correctly in the JSON files. + +**If you see mojibake like `â€"` or `é`:** +- The JSON files are correctly stored as UTF-8 +- The issue is the *consumer* (text editor, web browser, terminal) reading the file with wrong encoding (Latin-1/Windows-1252) +- **Fix**: Ensure your tools/editor/browser use UTF-8 encoding + +**Verification & Repair:** +```bash +# Verify all JSON files are valid UTF-8 +python verify_encoding.py + +# Fix common mojibake patterns (run if data was corrupted) +python fix_mojibake.py --dry-run # preview fixes +python fix_mojibake.py # apply fixes + +# Check progress including encoding status +python check_progress.py +``` + +**For web apps displaying this data:** +- Set `Content-Type: application/json; charset=utf-8` +- Include `` in HTML +- Ensure database connections use UTF-8 + +## Resuming / Sharing Work + +The pipeline is **fully resumable**: +- Re-run the same command → skips completed episodes +- Only missing transcripts/analyses are processed +- Share `transcripts/`, `data/analysis/`, and `data/index.db` to collaborate + +## Requirements + +- **Python 3.10+** +- **ffmpeg** (for audio processing - usually auto-installed with faster-whisper) +- **opencode CLI** (installed by `./install.sh`) +- **~50 GB free disk** for full backlog (MP3s auto-deleted after transcription by default) + +## Architecture + +``` +download.py → fetches MP3s from Libsyn RSS (resumable, size-verified) +preprocess.py → converts MP3 → clean 16kHz mono WAV (handles Libsyn quirks) +transcribe.py → faster-whisper, chunked (10-min windows, no OOM) +analyze.py → opencode subprocess with timeout + retries +verify.py → second-pass fact-check against research dossier +index.py → SQLite FTS5 index with hashtags +search.py → CLI search with timestamps + context +pipeline_all.py → orchestrates everything with resume logic +``` + +## For Contributors + +The pipeline is designed for distributed workloads: +1. Each user runs a subset of episodes (`--eps 84 87 89 ...`) +2. Results are merged via `data/analysis/` JSON files and `index.db` +3. Master JSON (`data/dickshow_analysis.json`) aggregates everything + +## License + +Private research tool over public broadcasts. Content is adult/extreme — use responsibly. + +## Support + +- Issues: https://kode.hannesahlin.se/n3wk/dickshow-index/issues +- Pipeline logs: `rekieta_run.log`, `backlog_run.log` +- Analysis logs: `analyze_run.log`, `analyze_err.log` \ No newline at end of file diff --git a/aggregate.py b/aggregate.py new file mode 100644 index 0000000..922e029 --- /dev/null +++ b/aggregate.py @@ -0,0 +1,123 @@ +"""Merge per-episode analyses into ONE master JSON: +data/dickshow_analysis.json + +Contents: + meta counts, generated_at, rubric reference + episodes full per-episode analyses, sorted by ep_num + indexes derived views for fast lookup: + by_person, by_tag_topic, edginess_ranking, saga_map +""" +import json +import os +from datetime import datetime, timezone + +from config import DATA_DIR +from analyze import ANALYSIS_DIR, MASTER_JSON +from feed import load_episodes + + +def load_analyses() -> list[dict]: + out = [] + if not os.path.isdir(ANALYSIS_DIR): + return out + for f in sorted(os.listdir(ANALYSIS_DIR)): + if not f.endswith(".json"): + continue + try: + with open(os.path.join(ANALYSIS_DIR, f), encoding="utf-8") as fh: + out.append(json.load(fh)) + except (json.JSONDecodeError, OSError) as e: + print(f"warning: skipping bad analysis file {f}: {e}") + return out + + +def aggregate() -> None: + analyses = load_analyses() + eps_meta = {e["ep_num"]: e for e in load_episodes() if e["ep_num"]} + + # attach runtime from episode metadata when available + verification status + for a in analyses: + m = eps_meta.get(a.get("ep_num")) + if m and m.get("duration_sec"): + a["runtime_minutes"] = round(m["duration_sec"] / 60) + v = a.get("_verification") or {} + a["verified"] = v.get("verdict") if isinstance(v, dict) else None + + analyses.sort(key=lambda a: (a.get("ep_num") is None, + a.get("ep_num") or 0)) + + by_person: dict[str, list] = {} + by_topic: dict[str, list] = {} + quotes_ranked = [] + ti_scores = [] + for a in analyses: + epn = a.get("ep_num") + label = f"Ep {epn}" if epn else (a.get("title") or "?")[:20] + seen_p = set() + seen_t = set() + for tp in a.get("talking_points", []): + for p in tp.get("people", []): + key = p.lower().strip() + seen_p.add(key) + by_person.setdefault(key, []).append( + {"episode": epn, "topic": tp.get("topic"), + "ts": tp.get("start_ts")}) + topic = (tp.get("topic") or "").strip().lower() + if topic: + seen_t.add(topic) + for t in sorted(seen_t): + by_topic.setdefault(t, []).append(epn) + for q in a.get("notable_quotes", []): + quotes_ranked.append({ + "episode": epn, "ts": q.get("ts"), "ti": q.get("ti", 0), + "quote": q.get("quote"), "why": q.get("why"), + }) + tg = a.get("transgression", {}) + if isinstance(tg.get("overall_ti"), int): + ti_scores.append({"episode": epn, "overall_ti": tg["overall_ti"], + "sincerity": tg.get("sincerity"), + "targets": tg.get("targets", [])}) + + quotes_ranked.sort(key=lambda q: -q.get("ti", 0)) + ti_scores.sort(key=lambda s: -s.get("overall_ti", 0)) + + n_verified = sum(1 for a in analyses if a.get("verified") in ("pass", "fix")) + + master = { + "meta": { + "generated_at": datetime.now(timezone.utc).isoformat(), + "podcast": "The Dick Show", + "feed": "https://thedickshow.com/feed/podcast/", + "episodes_analyzed": len(analyses), + "episodes_factchecked": n_verified, + "total_runtime_minutes": sum(a.get("runtime_minutes") or 0 + for a in analyses), + "scale": "Transgression Index 0-10, see rubric.py / README", + "note": ("Companion file for routine opencode access. " + "Per-episode analyst files live in data/analysis/, " + "raw transcripts in transcripts/. Each episode carries " + "'verified': pass|fix|null from the dossier fact-check " + "pass (verify.py). Treat verified=null as unconfirmed."), + }, + "episodes": analyses, + "indexes": { + "by_person": {k: v for k, v in sorted(by_person.items())}, + "topics_by_episode": {k: sorted(set(x for x in v if x)) + for k, v in sorted(by_topic.items())}, + "edginess_ranking": { + "top_overall_ti": ti_scores[:25], + "top_quotes": quotes_ranked[:50], + }, + }, + } + + tmp = MASTER_JSON + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(master, f, ensure_ascii=False, indent=1) + os.replace(tmp, MASTER_JSON) + print(f"master -> {MASTER_JSON} " + f"({len(analyses)} episodes, {len(quotes_ranked)} ranked quotes)") + + +if __name__ == "__main__": + aggregate() diff --git a/analyze.py b/analyze.py new file mode 100644 index 0000000..5da3d8e --- /dev/null +++ b/analyze.py @@ -0,0 +1,289 @@ +"""Analyze episodes via the local `opencode run` CLI. + +For each transcript: + 1. export readable timestamped txt (export_txt.py) + 2. call: opencode run -f "" + 3. parse/validate the returned JSON (retry up to 3x on bad output) + 4. save data/analysis/epXXXX.json + +Then aggregate all analyses into one master file for routine opencode access: +data/dickshow_analysis.json (run aggregate.py, or it runs automatically + after each new analysis in this script) + +Resumable: existing per-episode analyses are skipped unless --force. +""" +import argparse +import json +import os +import re +import subprocess +import sys +import time +from datetime import datetime, timezone + +from config import DATA_DIR, BASE_DIR +from feed import load_episodes +from download import slug as ep_slug, audio_path +from transcribe import transcript_path as tpath +from export_txt import export_txt, txt_path +from rubric import RUBRIC, SCHEMA_EXAMPLE +from facts import CAST_SHEET + +ANALYSIS_DIR = os.path.join(DATA_DIR, "analysis") +MASTER_JSON = os.path.join(DATA_DIR, "dickshow_analysis.json") + +OPENCODE = os.environ.get( + "OPENCODE_BIN", os.path.expanduser(r"~\.opencode\bin\opencode.exe")) +# analyzer model override, e.g. opencode/x-preview-f-free +OPENCODE_MODEL = os.environ.get("OPENCODE_ANALYZE_MODEL", "").strip() +OPENCODE_TIMEOUT = int(os.environ.get("DS_ANALYZE_TIMEOUT", "900")) # 15 min + +PROMPT_TEMPLATE = """You are the show archivist for The Dick Show podcast. Attached is the FULL transcript of one episode (timestamped lines like [HH:MM:SS]). + +Your job: produce a structured analyst file for this episode. Read the ENTIRE transcript carefully, then reconstruct what was talked about, the arguments made, ideas/bits coined, and rate how edgy/transgressive the content is using the rubric below. The show contains extreme adult language; analyze it dispassionately like a content researcher - do not sanitize quotes, do not editorialize morally. + +IMPORTANT - speaker attribution rules: +{cast} +Base attributions on what voices actually say (self-identifications, direct address) - never on famous-name pattern matching. + +{rubric} + +OUTPUT RULES: +- Output ONLY one valid JSON object. No markdown fences, no commentary before or after. +- Match exactly these keys (extra keys not allowed): {schema} +- All timestamps are SECONDS (number), converted from the [HH:MM:SS] markers. +- talking_points: cover every distinct topic IN ORDER of appearance + (5-15 items; summaries max ~40 words each). +- notable_quotes: the most characteristic/extreme/funny verbatim lines (5-10 items). +- If a field would be empty, use [] or "". +- ep_num and title must be taken from the transcript header comments. + +Begin now.""" + +FENCES_RE = re.compile(r"^```(?:json)?\s*|\s*```$", re.MULTILINE) + + +def build_prompt() -> str: + return PROMPT_TEMPLATE.format(cast=CAST_SHEET.strip(), + rubric=RUBRIC.strip(), + schema=SCHEMA_EXAMPLE.strip()) + + +def _repair(text: str) -> str: + """Common LLM-JSON repairs: fences, trailing commas, stray text, missing commas.""" + text = FENCES_RE.sub("", text.strip()).strip() + start = text.find("{") + end = text.rfind("}") + if start > 0 or end < len(text) - 1: + if start != -1 and end > start: + text = text[start:end + 1] + # trailing commas before } or ] + text = re.sub(r",(\s*[}\]])", r"\1", text) + # missing commas between object members: "key": "value" "key2": "value2" -> add comma + text = re.sub(r'"\s+"', '", "', text) + # missing commas between array elements: "value1" "value2" -> add comma + text = re.sub(r'"\s*\n\s*"', '",\n"', text) + # missing commas after closing brace/bracket followed by opening brace/bracket + text = re.sub(r'(\]|\})\s*(\[|\{)', r'\1,\2', text) + return text + + +def strip_to_json(text: str): + # strict=False: tolerate raw newlines/tabs inside strings (common LLM slip) + return json.loads(_repair(text), strict=False) + + +REPAIR_PREAMBLE = ( + "IMPORTANT: Your reply must be ONE valid strict JSON object matching the " + "schema below. Double-quoted keys/strings, escape quotes inside strings " + "(\\\"), no trailing commas, no markdown fences, no commentary.\n\n" +) + + +def ensure_content_match(obj: dict, attach_txt: str) -> None: + """Guard against cross-episode contamination: the analysis must reference + words that actually occur in this episode's transcript.""" + import json as _json + with open(attach_txt, encoding="utf-8") as f: + lines = [ln for ln in f if ln.startswith("[")] + if not lines: + return + sample = lines[:40] + lines[len(lines)//2:len(lines)//2+40] + lines[-20:] + stop = {"there", "about", "would", "because", "really", "going", "gonna", + "know", "like", "that", "this", "with", "what", "when", "they", + "them", "then", "than", "have", "been", "were", "just", "yeah"} + words = {w.strip(".,!?\"':;()[]").lower() + for ln in sample for w in ln.split() + if len(w) > 4 and w.strip(".,!?\"':;()[]").lower() not in stop} + blob = _json.dumps(obj, ensure_ascii=False).lower() + hits = {w for w in words if w in blob} + if len(hits) < 5: + raise ValueError( + f"content mismatch: analysis shares only {len(hits)} vocabulary " + f"items with this transcript - likely wrong-session output") + + +def validate(obj: dict) -> None: + required_top = ["ep_num", "title", "one_line_summary", "guests", + "talking_points", "arguments", "ideas", "notable_quotes", + "transgression"] + missing = [k for k in required_top if k not in obj] + if missing: + raise ValueError(f"missing keys: {missing}") + tg = obj["transgression"] + for k in ["overall_ti", "profanity", "slur_usage", + "harassment_or_cruelty", "sincerity"]: + if k not in tg or not isinstance(tg[k], int) or not 0 <= tg[k] <= 10: + raise ValueError(f"transgression.{k} must be int 0-10") + for q in obj.get("notable_quotes", []): + if not isinstance(q.get("ti"), int) or not 0 <= q["ti"] <= 10: + raise ValueError("quote ti must be int 0-10") + + +def run_opencode(prompt: str, attach: str) -> str: + # NOTE: deliberately NO `-c`/`-s` session reuse: `run -c` continues the + # globally-last session, which under concurrent pipelines cross-pollinates + # analyses between episodes. Every call is a fresh session. + # --auto: headless runs must not stall on permission prompts (the model + # sometimes Reads attached files for more context); scope is our own data. + cmd = [OPENCODE, "run", "--auto", "-f", attach, + "--dir", BASE_DIR] + if OPENCODE_MODEL: + cmd += ["-m", OPENCODE_MODEL] + cmd.append(prompt) + proc = subprocess.run(cmd, capture_output=True, text=True, + encoding="utf-8", errors="replace", + cwd=BASE_DIR, timeout=OPENCODE_TIMEOUT) + out = proc.stdout or "" + err = proc.stderr or "" + combined = (out + err).lower() + if "rate limit" in combined or "429" in combined: + raise RateLimitError("provider rate limit exceeded") + if not out.strip(): + raise RuntimeError(f"opencode produced no stdout: {err[-400:]}") + return out + + +class RateLimitError(RuntimeError): + pass + + +# When the provider quota dries up, stop *attempting* analyses pipeline-wide +# for a while; transcription (local) keeps running. Probe again after cooldown. +_RATE_COOLDOWN = {"until": 0.0} +COOLDOWN_SECONDS = int(os.environ.get("DS_RATE_COOLDOWN", "900")) + + +def analyze_episode(ep: dict, force: bool = False, retries: int = 3) -> dict | None: + os.makedirs(ANALYSIS_DIR, exist_ok=True) + apath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json") + if os.path.exists(apath) and not force: + return None + if time.time() < _RATE_COOLDOWN["until"]: + print(f" (analysis cooldown {int(_RATE_COOLDOWN['until'] - time.time())}s " + f"- will catch up later)", flush=True) + return None + if not os.path.exists(tpath(ep)): + print(f" no transcript for {ep['title']}; skipping", flush=True) + return None + attach = export_txt(ep) + prompt = build_prompt() + last_err = None + # escalating waits; rate limits need minutes, not seconds + waits = [10, 60, 180, 420, 600, 900] + attempt = 0 + while attempt < len(waits): + attempt += 1 + t0 = time.time() + try: + raw = run_opencode( + (REPAIR_PREAMBLE + prompt) if attempt > 1 else prompt, attach) + obj = strip_to_json(raw) + validate(obj) + ensure_content_match(obj, attach) + obj["ep_num"] = ep["ep_num"] if ep["ep_num"] else obj.get("ep_num") + obj["title"] = obj.get("title") or ep["title"] + obj["pub_date"] = ep["pub_date"] + obj["_analyzed_at"] = datetime.now(timezone.utc).isoformat() + obj["_source_transcript"] = os.path.basename(tpath(ep)) + with open(apath + ".tmp", "w", encoding="utf-8") as f: + json.dump(obj, f, ensure_ascii=False, indent=1) + os.replace(apath + ".tmp", apath) + dt = time.time() - t0 + ti = obj["transgression"]["overall_ti"] + n_tp = len(obj.get("talking_points", [])) + n_q = len(obj.get("notable_quotes", [])) + print(f" OK {os.path.basename(apath)} in {dt:.0f}s " + f"(TI={ti}, {n_tp} points, {n_q} quotes)", flush=True) + return obj + except RateLimitError as e: + last_err = e + wait = waits[min(attempt, len(waits) - 1)] + print(f" rate-limited (attempt {attempt}); " + f"sleeping {wait}s ...", flush=True) + time.sleep(wait) + except Exception as e: # noqa: BLE001 + last_err = e + wait = waits[min(attempt - 1, len(waits) - 1)] + print(f" attempt {attempt} failed: {e}; retry in {wait}s", + flush=True) + time.sleep(wait) + print(f" GIVING UP on {ep['title']} after {attempt} attempts: " + f"{last_err}", flush=True) + if isinstance(last_err, RateLimitError): + _RATE_COOLDOWN["until"] = time.time() + COOLDOWN_SECONDS + print(f" entering analysis cooldown for {COOLDOWN_SECONDS}s " + f"(transcription continues; analyses catch up later)", + flush=True) + return None + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("eps", nargs="*", type=int) + ap.add_argument("--force", action="store_true") + ap.add_argument("--no-verify", action="store_true", + help="skip the dossier fact-check second pass") + args = ap.parse_args() + eps = load_episodes() + todo = [e for e in eps + if (not args.eps or e["ep_num"] in args.eps) + and os.path.exists(tpath(e))] + print(f"analyzing {len(todo)} episode(s) with {os.path.basename(OPENCODE)}", + flush=True) + + # lazy import: verify.py imports this module at its own top level + from verify import verify_episode + + done = 0 + for i, ep in enumerate(todo, 1): + label = f"Ep {ep['ep_num']}" if ep["ep_num"] else ep["title"][:40] + existing = os.path.exists(os.path.join( + ANALYSIS_DIR, f"{ep_slug(ep)}.json")) + if not existing: + print(f"[{i}/{len(todo)}] analyzing {label} ...", flush=True) + if analyze_episode(ep, force=args.force) is not None: + done += 1 + # ALWAYS fact-check fresh analyses against the dossier + if not args.no_verify: + print(f" fact-checking {label} ...", flush=True) + verify_episode(ep) + elif not args.no_verify: + # already analyzed -> run/refresh the dossier fact-check + vpath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json") + with open(vpath, encoding="utf-8") as f: + already = "_verification" in json.load(f) + if args.force or not already: + print(f"[{i}/{len(todo)}] verifying (existing) {label} ...", + flush=True) + verify_episode(ep) + # keep the master file fresh after every run + from aggregate import aggregate + aggregate() + print(f"analyzed={done} total_files=" + f"{len([f for f in os.listdir(ANALYSIS_DIR) if f.endswith('.json')])}", + flush=True) + + +if __name__ == "__main__": + main() diff --git a/check_encoding.py b/check_encoding.py new file mode 100644 index 0000000..ae37504 --- /dev/null +++ b/check_encoding.py @@ -0,0 +1,11 @@ +import json, glob + +for fname in glob.glob('transcripts/ep*.json'): + with open(fname, 'r', encoding='utf-8') as f: + data = json.load(f) + for seg in data['segments']: + text = seg['text'] + for c in text: + if ord(c) > 127: + print(f'{data["ep_num"]}: {repr(c)} U+{ord(c):04X} in: {seg["text"][:80]}') + break \ No newline at end of file diff --git a/check_progress.py b/check_progress.py new file mode 100644 index 0000000..21eaf1d --- /dev/null +++ b/check_progress.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Check which episodes are already transcribed/analyzed.""" + +import os +import json +from pathlib import Path + +TRANSCRIPT_DIR = "transcripts" +ANALYSIS_DIR = "data/analysis" + +def check_progress(): + print("=" * 60) + print("Dick Show Index - Progress Check") + print("=" * 60) + + # All episodes from the feed + all_episodes = set() + # Load from episodes.jsonl if available + if os.path.exists("data/episodes.jsonl"): + with open("data/episodes.jsonl", "r") as f: + for line in f: + ep = json.loads(line) + if ep.get("ep_num"): + all_episodes.add(ep["ep_num"]) + + # Check transcripts + transcript_files = set() + if os.path.exists("transcripts"): + for f in os.listdir("transcripts"): + if f.startswith("ep") and f.endswith(".json"): + try: + ep_num = int(f[2:6]) + transcript_files.add(ep_num) + except ValueError: + pass + + # Check analyses + analysis_files = set() + if os.path.exists("data/analysis"): + for f in os.listdir("data/analysis"): + if f.startswith("ep") and f.endswith(".json"): + try: + ep_num = int(f[2:6]) + analysis_files.add(ep_num) + except ValueError: + pass + + # Check verified analyses + verified = set() + if os.path.exists("data/analysis"): + for f in os.listdir("data/analysis"): + if f.startswith("ep") and f.endswith(".json"): + try: + ep_num = int(f[2:6]) + path = f"data/analysis/{f}" + with open(path, "r") as f: + data = json.load(f) + if data.get("_verification", {}).get("verdict") in ("pass", "fix"): + verified.add(ep_num) + except Exception: + pass + + all_eps = sorted(all_episodes) + if not all_eps: + all_eps = sorted(transcript_files | analysis_files) + + print(f"\n{'Ep':>4} {'Title':<45} {'Transcript':^10} {'Analysis':^10} {'Verified':^8}") + print("-" * 85) + + stats = {"total": 0, "transcript": 0, "analysis": 0, "verified": 0} + + for ep in all_eps: + has_t = ep in transcript_files + has_a = ep in analysis_files + has_v = ep in verified + + stats["total"] += 1 + if has_t: stats["transcript"] += 1 + if has_a: stats["analysis"] += 1 + if has_v: stats["verified"] += 1 + + t_status = "[OK]" if has_t else "[..]" + a_status = "[OK]" if has_a else "[..]" + v_status = "[OK]" if has_v else "[..]" + + # Try to get title + title = "" + try: + with open(f"transcripts/ep{ep:04d}.json", "r") as f: + data = json.load(f) + title = data.get("title", "")[:43] + except: + pass + + print(f"{ep:>4} {title:<45} {t_status:^10} {a_status:^10} {v_status:^8}") + + print("-" * 85) + print(f"Total episodes in feed: {stats['total']}") + print(f"Transcripts complete: {stats['transcript']}/{stats['total']}") + print(f"Analyses complete: {stats['analysis']}/{stats['total']}") + print(f"Verified: {stats['verified']}/{stats['total']}") + + # Missing + missing_t = [ep for ep in all_eps if ep not in transcript_files] + missing_a = [ep for ep in all_eps if ep not in analysis_files] + missing_v = [ep for ep in all_eps if ep not in verified] + + if missing_t: + print(f"\nMissing transcripts ({len(missing_t)}): {missing_t[:20]}{'...' if len(missing_t) > 20 else ''}") + if missing_a: + print(f"Missing analyses ({len(missing_a)}): {missing_a[:20]}{'...' if len(missing_a) > 20 else ''}") + if missing_v: + print(f"Unverified ({len(missing_v)}): {missing_v[:20]}{'...' if len(missing_v) > 20 else ''}") + +if __name__ == "__main__": + check_progress() \ No newline at end of file diff --git a/download.py b/download.py new file mode 100644 index 0000000..ffcfbe2 --- /dev/null +++ b/download.py @@ -0,0 +1,134 @@ +"""Resumable MP3 downloader. + +Downloads each episode's enclosure into audio/.mp3. +Skips files that already exist with the expected size (HTTP Range resume). +""" +import json +import os +import re +import sys +import time + +import requests + +from config import AUDIO_DIR, DOWNLOAD_TIMEOUT, CHUNK_SIZE +from feed import load_episodes + + +def slug(ep: dict) -> str: + if ep.get("ep_num"): + return f"ep{ep['ep_num']:04d}" + g = re.sub(r"[^a-zA-Z0-9-]", "", ep["guid"])[:24] + return f"bonus-{g}" + + +def audio_path(ep: dict) -> str: + return os.path.join(AUDIO_DIR, slug(ep) + ".mp3") + + +def expected_size(ep: dict) -> int: + return ep.get("audio_bytes") or 0 + + +def download_episode(ep: dict, force: bool = False) -> str: + """Download one episode. Returns local path. Raises on hard failure.""" + url = ep["audio_url"] + if not url: + raise ValueError(f"no audio_url for {ep['guid']}") + dest = audio_path(ep) + tmp = dest + ".part" + want = expected_size(ep) + + if os.path.exists(dest): + if force: + os.remove(dest) + else: + have = os.path.getsize(dest) + if want and have != want: + print(f" size mismatch for {dest} (have {have}, want {want}); " + f"redownloading", file=sys.stderr) + os.remove(dest) + else: + return dest + + headers = {"User-Agent": "dickshow-indexer/1.0"} + mode = "wb" + if os.path.exists(tmp): + have = os.path.getsize(tmp) + if want and have == want: + os.replace(tmp, dest) + return dest + if have > 0: + headers["Range"] = f"bytes={have}-" + mode = "ab" + + last_err = None + for attempt in range(1, 6): + try: + with requests.get(url, headers=headers, stream=True, + timeout=DOWNLOAD_TIMEOUT, allow_redirects=True) as r: + if r.status_code == 416: # range beyond EOF -> already complete + os.replace(tmp, dest) + return dest + r.raise_for_status() + total = int(r.headers.get("Content-Length", "0") or 0) + if mode == "ab" and r.status_code == 200: + # server ignored Range; restart cleanly + mode = "wb" + with open(tmp, mode) as f: + for chunk in r.iter_content(chunk_size=CHUNK_SIZE): + if chunk: + f.write(chunk) + have = os.path.getsize(tmp) + if want and have < want: + raise IOError(f"incomplete download {have}/{want}") + break + except Exception as e: # noqa: BLE001 + last_err = e + print(f" retry {attempt}: {e}", file=sys.stderr) + time.sleep(5 * attempt) + mode = "ab" + headers["Range"] = f"bytes={os.path.getsize(tmp)}-" \ + if os.path.exists(tmp) and os.path.getsize(tmp) > 0 else {} + else: + raise IOError(f"download failed for {url}: {last_err}") + + os.replace(tmp, dest) + return dest + + +def main() -> None: + from config import TRANSCRIPT_DIR + from transcribe import transcript_path + eps = load_episodes() + only = set() + for a in sys.argv[1:]: + only.add(int(a)) + ok = fail = skip = 0 + for ep in eps: + if ep["ep_num"] is not None and only and ep["ep_num"] not in only: + continue + if os.path.exists(transcript_path(ep)): + skip += 1 # already fully transcribed; audio not needed + continue + dest = audio_path(ep) + if os.path.exists(dest) and not (expected_size(ep) and + os.path.getsize(dest) != expected_size(ep)): + skip += 1 + continue + label = f"Ep {ep['ep_num']}" if ep["ep_num"] else ep["title"][:40] + try: + p = download_episode(ep) + mb = os.path.getsize(p) / 1e6 + print(f"{label}: {mb:.1f} MB") + ok += 1 + except Exception as e: # noqa: BLE001 + print(f"{label}: FAILED - {e}", file=sys.stderr) + fail += 1 + with open(os.path.join(AUDIO_DIR, ".progress.json"), "w") as f: + json.dump({"last_done": ep["guid"]}, f) + print(f"downloaded={ok} skipped={skip} failed={fail}") + + +if __name__ == "__main__": + main() diff --git a/export_txt.py b/export_txt.py new file mode 100644 index 0000000..1915ee5 --- /dev/null +++ b/export_txt.py @@ -0,0 +1,45 @@ +"""Convert transcript JSON into a readable timestamped txt for LLM analysis.""" +import json +import os +import sys + +from config import TRANSCRIPT_DIR +from feed import load_episodes +from download import slug as ep_slug + +TXT_DIR = os.path.join(TRANSCRIPT_DIR, "txt") + + +def txt_path(ep: dict) -> str: + return os.path.join(TXT_DIR, ep_slug(ep) + ".txt") + + +def export_txt(ep: dict) -> str: + tp = os.path.join(TRANSCRIPT_DIR, ep_slug(ep) + ".json") + with open(tp, encoding="utf-8") as f: + data = json.load(f) + os.makedirs(TXT_DIR, exist_ok=True) + out = txt_path(ep) + with open(out + ".tmp", "w", encoding="utf-8") as f: + f.write(f"# {data['title']}\n# published: {data['pub_date']}\n" + f"# episode: {data.get('ep_num')}\n\n") + for seg in data["segments"]: + s = int(seg["start"]) + stamp = f"{s // 3600:02d}:{(s % 3600) // 60:02d}:{s % 60:02d}" + f.write(f"[{stamp}] {seg['text']}\n") + os.replace(out + ".tmp", out) + return out + + +if __name__ == "__main__": + eps = load_episodes() + want = {int(a) for a in sys.argv[1:]} if len(sys.argv) > 1 else None + n = 0 + for ep in eps: + if want and (ep["ep_num"] not in want): + continue + src = os.path.join(TRANSCRIPT_DIR, ep_slug(ep) + ".json") + if os.path.exists(src): + print(export_txt(ep)) + n += 1 + print(f"exported {n}") diff --git a/facts.py b/facts.py new file mode 100644 index 0000000..55a5bba --- /dev/null +++ b/facts.py @@ -0,0 +1,30 @@ +"""Ground-truth facts used to keep episode analyses honest. + +The dossier file is maintained separately (research project); this module +points at it and carries a compact cast sheet for cheap prompt injection. +""" +import os + +from config import BASE_DIR + +DOSSIER_PATH = os.path.join(os.path.dirname(BASE_DIR), + "Dax_Herrera_Dick_Masterson_Profile.txt") + +# Verified roster with eras - injected into BOTH analyzer and verifier prompts. +CAST_SHEET = """VERIFIED CAST FACTS (from court records, WIPO filings, primary sources): +- DICK MASTERSON (real name Dax Herrera): host of The Dick Show since June 2016. +- MADDOX (George Ouzounian): former BPITU co-host (2014-2016). He and Dick have + been ADVERSARIES since Oct 2016 (the "LOLSUIT", dismissed 2018). Maddox has + NOT been a Dick Show cohost at ANY point and is long gone from Dick's life. + Any analysis attributing a modern-era (2017+) cohost seat to Maddox is WRONG. +- SEAN (a.k.a. Seanie): longtime Dick Show audio engineer/cohost voice, + present across essentially all episodes. +- VITO GESUALDI: game developer/comic creator; frequent Dick Show guest and + cohost of the BPITU REVIVAL (2025+). On modern-era episodes an unnamed second + voice is usually Sean or Vito - not Maddox. +- ASTERIOS KOKKINOS: recurring guest (2016 era onward). +- ERIC JULY / RIPPAVERSE saga: May 2023 onward (Isom review = Ep 361). +- Ethan Ralph, Josh Denny, Nick Rekieta, Null (Kiwi Farms): associates/guests + of the 2017-2023 era. +If a voice cannot be attributed confidently from the transcript itself, label +it "unknown voice" instead of guessing a famous name.""" diff --git a/feed.py b/feed.py new file mode 100644 index 0000000..354702a --- /dev/null +++ b/feed.py @@ -0,0 +1,127 @@ +"""Parse The Dick Show RSS feed into data/episodes.jsonl (one episode per line).""" +import json +import re +import sys +import time +import xml.etree.ElementTree as ET + +import requests + +from config import FEED_URL, EPISODES_JSONL, DOWNLOAD_TIMEOUT, MAX_FEED_RETRIES + +NS = { + "itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd", +} +EP_NUM_RE = re.compile(r"Episode\s+(\d+)", re.IGNORECASE) + + +def fetch_feed(url: str = FEED_URL) -> bytes: + last_err = None + for attempt in range(1, MAX_FEED_RETRIES + 1): + try: + r = requests.get(url, timeout=DOWNLOAD_TIMEOUT, + headers={"User-Agent": "dickshow-indexer/1.0"}) + r.raise_for_status() + return r.content + except Exception as e: # noqa: BLE001 + last_err = e + print(f"feed attempt {attempt}/{MAX_FEED_RETRIES} failed: {e}", + file=sys.stderr) + time.sleep(3 * attempt) + raise SystemExit(f"could not download feed: {last_err}") + + +def itunes_duration_seconds(text: str | None) -> int: + """Libsyn uses either HH:MM:SS or plain seconds.""" + if not text: + return 0 + text = text.strip() + if ":" in text: + parts = [int(p) for p in text.split(":")] + while len(parts) < 3: + parts.insert(0, 0) + h, m, s = parts + return h * 3600 + m * 60 + s + try: + return int(float(text)) + except ValueError: + return 0 + + +def parse_feed(xml_bytes: bytes) -> list[dict]: + root = ET.fromstring(xml_bytes) + channel = root.find("channel") + episodes = [] + for item in channel.findall("item"): + title_el = item.find("title") + title = (title_el.text or "").strip() if title_el is not None else "" + m = EP_NUM_RE.search(title) + ep_num = int(m.group(1)) if m else None + + guid_el = item.find("guid") + guid = (guid_el.text or "").strip() if guid_el is not None else "" + if not guid: + continue # unusable without stable id + + pub_el = item.find("pubDate") + pub_date = (pub_el.text or "").strip() if pub_el is not None else "" + + enc = item.find("enclosure") + audio_url, audio_type, audio_len = "", "", 0 + if enc is not None: + audio_url = enc.get("url", "").strip() + audio_type = enc.get("type", "") + try: + audio_len = int(enc.get("length", "0") or 0) + except ValueError: + audio_len = 0 + + dur_el = item.find("itunes:duration", NS) + duration_s = itunes_duration_seconds(dur_el.text if dur_el is not None else None) + + link_el = item.find("link") + page_link = (link_el.text or "").strip() if link_el is not None else "" + + episodes.append({ + "ep_num": ep_num, + "title": title, + "pub_date": pub_date, + "guid": guid, + "audio_url": audio_url, + "audio_type": audio_type, + "audio_bytes": audio_len, + "duration_sec": duration_s, + "page_link": page_link, + }) + # newest first from feed; store oldest-first for chronological processing + episodes.reverse() + return episodes + + +def write_episodes(episodes: list[dict]) -> None: + with open(EPISODES_JSONL, "w", encoding="utf-8") as f: + for ep in episodes: + f.write(json.dumps(ep, ensure_ascii=False) + "\n") + + +def load_episodes() -> list[dict]: + eps = [] + with open(EPISODES_JSONL, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + eps.append(json.loads(line)) + return eps + + +def main() -> None: + xml = fetch_feed() + episodes = parse_feed(xml) + write_episodes(episodes) + numbered = [e["ep_num"] for e in episodes if e["ep_num"]] + print(f"parsed {len(episodes)} episodes " + f"(numbered {min(numbered)}-{max(numbered)}) -> {EPISODES_JSONL}") + + +if __name__ == "__main__": + main() diff --git a/fix_encoding.py b/fix_encoding.py new file mode 100644 index 0000000..5cb5a71 --- /dev/null +++ b/fix_encoding.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Verify and repair UTF-8 encoding in all JSON files.""" + +import json +import sys +from pathlib import Path + +def check_file(path: Path) -> tuple[int, int]: + """Check a JSON file for encoding issues. Returns (errors_fixed, total_chars_checked).""" + try: + with open(path, 'r', encoding='utf-8') as f: + content = f.read() + except UnicodeDecodeError as e: + print(f" [ERROR] {path}: Not valid UTF-8 at position {e.start}: {e}") + return (0, 0) + + try: + data = json.loads(content) + except json.JSONDecodeError as e: + print(f" [ERROR] {path}: Invalid JSON at position {e.pos}: {e}") + return (0, 0) + + return (0, len(content)) + +def check_all_json_files(root: Path) -> dict: + """Scan all JSON files for encoding issues.""" + stats = {"files_checked": 0, "errors": 0, "total_bytes": 0} + + for path in root.rglob("*.json"): + if any(part.startswith('.') for part in path.parts): + continue + if path.name.startswith('.'): + continue + + errors, bytes_checked = check_file(path) + stats["files_checked"] += 1 + stats["errors"] += errors + stats["total_bytes"] += bytes_checked + + if stats["files_checked"] % 50 == 0: + print(f" Checked {stats['files_checked']} files...") + + return stats + +def fix_corrupted_unicode(text: str) -> str: + """Fix common UTF-8 mojibake patterns.""" + # Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252) + fixes = { + 'â€"': '—', # em dash + 'â€': '—', # em dash (truncated) + '’': "'", # right single quote + '“': '"', # left double quote + 'â€': '"', # right double quote + '•': '•', # bullet + '…': '…', # ellipsis + 'é': 'é', # é + 'è': 'è', # è + 'á': 'á', # á + 'í': 'í', # í + 'ó': 'ó', # ó + 'ú': 'ú', # ú + 'ñ': 'ñ', # ñ + 'ü': 'ü', # ü + 'ö': 'ö', # ö + 'ä': 'ä', # ä + 'Â': '', # stray  (common before accented chars) + } + + result = text + for bad, good in fixes.items(): + result = result.replace(bad, good) + return result + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Verify and repair UTF-8 encoding in JSON files") + parser.add_argument('path', nargs='?', default='.', help='Root directory to scan') + parser.add_argument('--fix', action='store_true', help='Attempt to fix mojibake') + parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed') + args = parser.parse_args() + + root = Path(args.path).resolve() + print(f"Scanning {root} for JSON files...") + + stats = check_all_json_files(Path('.')) + + print(f"\nResults:") + print(f" Files checked: {stats['files_checked']}") + print(f" Total bytes: {stats['total_bytes']:,}") + print(f" Encoding errors: {stats['errors']}") + + if stats['errors'] == 0: + print("All JSON files are valid UTF-8!") + else: + print(f"Found {stats['errors']} files with encoding issues") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/fix_mojibake.py b/fix_mojibake.py new file mode 100644 index 0000000..93e908c --- /dev/null +++ b/fix_mojibake.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Fix UTF-8 mojibake in JSON files.""" + +import json +import sys +from pathlib import Path + +# Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252) +MOJIBAKE_FIXES = { + 'â€"': '—', # em dash + 'â€': '—', # em dash (truncated) + '’': "'", # right single quote + '“': '"', # left double quote + 'â€': '"', # right double quote + '•': '•', # bullet + '…': '…', # ellipsis + 'é': 'é', # é + 'è': 'è', # è + 'á': 'á', # á + 'í': 'í', # í + 'ó': 'ó', # ó + 'ú': 'ú', # ú + 'ñ': 'ñ', # ñ + 'ü': 'ü', # ü + 'ö': 'ö', # ö + 'ä': 'ä', # ä + 'ü': 'ü', # ü + 'ö': 'ö', # ö + 'ä': 'ä', # ä + 'Â': '', # stray  (common before accented chars) +} + +def fix_mojibake(text: str) -> tuple[str, int]: + """Fix common UTF-8 mojibake patterns. Returns (fixed_text, num_fixes).""" + count = 0 + result = text + for bad, good in MOJIBAKE_FIXES.items(): + before = result.count(bad) + if before: + result = result.replace(bad, good) + count += before + return result, count + +def fix_json_file(path: Path, dry_run: bool = False) -> int: + """Fix mojibake in a JSON file. Returns number of fixes applied.""" + try: + with open(path, 'r', encoding='utf-8') as f: + content = f.read() + except UnicodeDecodeError as e: + print(f" [ERROR] {path}: Not valid UTF-8 at position {e.start}") + return 0 + + try: + data = json.loads(content) + except json.JSONDecodeError as e: + print(f" [ERROR] {path}: Invalid JSON at position {e.pos}") + return 0 + + # Recursively fix strings in JSON + def fix_obj(obj): + fixes = 0 + if isinstance(obj, str): + fixed, count = fix_mojibake(obj) + return fixed, count + elif isinstance(obj, dict): + new_dict = {} + for k, v in obj.items(): + fixed_k, c1 = fix_obj(k) + fixed_v, c2 = fix_obj(v) + fixes += c1 + c2 + new_dict[fixed_k] = fixed_v + return new_dict, fixes + elif isinstance(obj, list): + new_list = [] + for item in obj: + fixed, count = fix_obj(item) + fixes += count + new_list.append(fixed) + return new_list, fixes + else: + return obj, 0 + + fixed_data, fix_count = fix_obj(json.loads(json.dumps(data))) # Round-trip to ensure clean JSON + + if fix_count > 0 and not dry_run: + with open(path, 'w', encoding='utf-8') as f: + json.dump(fixed_data, f, ensure_ascii=False, indent=1) + print(f" Fixed {fix_count} mojibake in {path}") + + return fix_count + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Fix UTF-8 mojibake in JSON files") + parser.add_argument('path', nargs='?', default='.', help='Root directory to scan') + parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed') + args = parser.parse_args() + + root = Path('.').resolve() + print(f"Scanning {Path('.').resolve()} for JSON files...") + + total_fixes = 0 + files_fixed = 0 + + for path in Path('.').rglob("*.json"): + if any(part.startswith('.') for part in path.parts): + continue + if path.name.startswith('.'): + continue + + fixes = fix_json_file(path, dry_run=args.dry_run) + if fixes > 0: + files_fixed += 1 + print(f" Fixed {fixes} issues in {path}") + + print(f"\nDone. Fixed {files_fixed} files.") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/index.py b/index.py new file mode 100644 index 0000000..057103e --- /dev/null +++ b/index.py @@ -0,0 +1,115 @@ +"""Build the SQLite FTS5 index from transcripts + lexicon hashtags. + +Schema: + episodes(ep_num, guid, title, pub_date, audio_url, page_link, duration_sec, + model, n_segments, tags_json) + segments(ep_rowid -> episodes.rowid, seg_idx, start_sec, end_sec, text, + tags) [FTS5 external content] + +Search examples (via search.py): + python search.py "isom lawsuit" + python search.py maddox --tag lolsuit --from 2018-01-01 --to 2018-12-31 +""" +import json +import os +import sqlite3 + +from config import DB_PATH +from feed import load_episodes +from lexicon import build_lexicon, tags_for_text + +SCHEMA = """ +PRAGMA journal_mode=WAL; + +CREATE TABLE IF NOT EXISTS episodes ( + id INTEGER PRIMARY KEY, + ep_num INTEGER UNIQUE, + guid TEXT UNIQUE NOT NULL, + title TEXT, + pub_date TEXT, + audio_url TEXT, + page_link TEXT, + duration_sec REAL, + model TEXT, + n_segments INTEGER, + tags TEXT +); + +CREATE VIRTUAL TABLE IF NOT EXISTS segments USING fts5( + text, + tags, + episode_id UNINDEXED, + seg_idx UNINDEXED, + start_sec UNINDEXED, + end_sec UNINDEXED +); +""" + + +def open_db(path=DB_PATH) -> sqlite3.Connection: + con = sqlite3.connect(path) + con.executescript(SCHEMA) + return con + + +def index_transcripts() -> None: + from transcribe import transcript_path + lexicon = build_lexicon() + eps = load_episodes() + con = open_db() + cur = con.cursor() + + n_eps = n_segs = 0 + for ep in eps: + tp = transcript_path(ep) + if not os.path.exists(tp): + continue + with open(tp, encoding="utf-8") as f: + data = json.load(f) + + ep_tags = set() + for seg in data["segments"]: + ep_tags.update(tags_for_text(seg["text"], lexicon)) + + # upsert episode row + cur.execute( + """INSERT INTO episodes (ep_num, guid, title, pub_date, audio_url, + page_link, duration_sec, model, n_segments, tags) + VALUES (?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(guid) DO UPDATE SET + ep_num=excluded.ep_num, title=excluded.title, + pub_date=excluded.pub_date, audio_url=excluded.audio_url, + page_link=excluded.page_link, duration_sec=excluded.duration_sec, + model=excluded.model, n_segments=excluded.n_segments, + tags=excluded.tags""", + (data.get("ep_num"), data["guid"], data.get("title"), + data.get("pub_date"), data.get("audio_url"), data.get("page_link"), + data.get("duration_sec"), data.get("model"), + len(data["segments"]), json.dumps(sorted(ep_tags))), + ) + row_id = cur.execute("SELECT id FROM episodes WHERE guid=?", + (data["guid"],)).fetchone()[0] + + # rebuild this episode's segment rows (simplest correct incremental path) + cur.execute("DELETE FROM segments WHERE episode_id=?", (row_id,)) + cur.executemany( + "INSERT INTO segments (text, tags, episode_id, seg_idx, start_sec," + " end_sec) VALUES (?,?,?,?,?,?)", + [(seg["text"], + " ".join("#" + t for t in tags_for_text(seg["text"], lexicon)), + row_id, i, seg["start"], seg["end"]) + for i, seg in enumerate(data["segments"])], + ) + n_eps += 1 + n_segs += len(data["segments"]) + label = f"Ep {data.get('ep_num')}" if data.get("ep_num") else data.get("title", "")[:40] + print(f" indexed {label}: {len(data['segments'])} segs, " + f"tags: {' '.join('#' + t for t in sorted(ep_tags)) or '(none)'}") + + con.commit() + con.close() + print(f"indexed {n_eps} episodes / {n_segs} segments -> {DB_PATH}") + + +if __name__ == "__main__": + index_transcripts() diff --git a/lexicon.py b/lexicon.py new file mode 100644 index 0000000..b7e1c79 --- /dev/null +++ b/lexicon.py @@ -0,0 +1,131 @@ +"""Hashtag lexicon: maps aliases -> canonical tag. + +Seeded from the research dossier (people, feuds, events, recurring bits). +Extend freely; matching is case-insensitive whole-word on each transcript +segment, then aggregated per episode and per segment in the FTS index. +""" +import re + +PEOPLE = { + "maddox": ["maddox", "george ochotsk", "ouzounian"], + "asterios": ["asterios", "asterios kokkinos", "kokkinos"], + "vito": ["vito", "vito gesualdi", "gesualdi"], + "sean": ["sean", "seanie"], # Sean the audio engineer + "eric_july": ["eric july", "rippaverse", "rippa", "isom", "yaira", + "alphacore", "horseman", "backwordz"], + "ethan_van_sciver": ["van sciver", "van schiver", "evs", "cyberfrog", + "comicsgate", "trashcast"], + "razorfist": ["razorfist", "razor fist"], + "yellowflash": ["yellowflash", "yellow flash"], + "clippa": ["clippa", "young clippa", "riley"], + "kevin_landau": ["kevin landau", "landau"], + "nick_rekieta": ["rekieta", "rekieta law", "rackets", + # whisper phonetic variants observed in transcripts + "raketa", "riketa", "rikita", "rakiti", "rakeda", + "rikata", "rikade"], + "ethan_ralph": ["ralph", "the ralph retort", "killstream", "ralphretort"], + "null": ["joshua moon", "kiwi farms", "kiwifarms"], + "mumkey_jones": ["mumkey", "mumkey jones"], + "josh_denny": ["denny", "josh denny"], + "frank_hassle": ["hassle", "frank hassle"], + "destiny": ["destiny", "steven bonnell"], + "justin_whang": ["whang", "justin whang", "whang!"], + "digibro": ["digibro"], + "wauterboi": ["waut3rboi", "wauterboi"], + "andy_lee": ["andy lee", "callofthedeep"], + "cliff_campbell": ["cliff campbell"], + "john_morgan": ["john morgan"], + "mumkey": [], + "greg_boser": ["boser", "greg boser"], + "loren_baker": ["loren baker"], + "tuomas": ["tuomas", "holmburg", "collector crypt"], + "john_mcafee": ["mcafee"], + "bunty_king": ["bunty"], + "jf_gariepy": ["jf garpey", "gariepy", "jfg"], + "sargon": ["sargon"], + "boogie2988": ["boogie", "boogie2988"], + "kraut": ["kraut"], + "andy_warski": ["warski"], + "tony_heller": [], + "stefan_molyneux": ["molyneux"], + "lauren_southern": ["lauren southern", "southern"], + "gavin_mcinnes": ["gavin mcinnes", "mcinnes"], + "milo": ["milo yiannopoulos", "milo"], + "cuck_shack_cast": [], +} + +EVENTS = { + "lolsuit": ["lolsuit", "lol suit", "lawsuit", "complaint", "deposition", + "judge ramos", "greenberger", "subpoena", "dismissed with prejudice"], + "isom_lawsuit": ["international school of ministry", "trademark lawsuit", + "night letter", "cease and desist"], + "patreon_drama": ["patreon"], + "deplatforming": ["deplatform", "mastercard", "match list", "visa", + "stripe", "paypal", "banking", "backed.by", "backedby"], + "seanies": ["seanie", "seanies", "seany award"], + "rage_calendar": ["rage calendar", "advent calendar"], + "hot_goss": ["hot goss"], + "rape_list": ["rape list"], + "drunk_sober_high": ["drunk sober high"], + "dick_tips": ["dick tip", "dick tips"], + "voicemail": ["voicemail", "voice mail"], + "rage": ["are you a rage", "the rage"], + "isom_review": ["isom #1", "isom review", "grave digger", "tombstone"], + "charity_accusations": ["charity"], + "new_project2": ["new project 2", "np2"], + "web3": ["web3", "nft", "solana", "crypto", "token", "blockchain", + "wallet", "mint"], + "maddox_book_prank": ["fuck whales", "fuck whales"], + "best_debate": ["best debate in the universe"], + "biggest_problem": ["biggest problem in the universe", "tbpitu"], +} + +TOPICS = { + "politics": ["trump", "biden", "obama", "election", "senate", "congress", + "democrat", "republican", "libertarian"], + "gun_policy": ["second amendment", "nra", "gun control"], + "covid": ["covid", "pandemic", "mask", "vaccine", "lockdown", "fauci"], + "ukraine_war": ["ukraine", "zelensky", "putin", "russia"], + "israel": ["israel", "gaza", "hamas", "idf", "antisemitism"], + "ai_tech": ["artificial intelligence", " chatgpt", " openai", " ai ", + "machine learning"], + "gaming": ["video game", "playstation", "nintendo", "xbox", "steam"], + "movies_tv": ["movie", "netflix", "disney", "marvel", "star wars"], + "sports": ["nfl", "nba", "ufc", "boxing", "wnba"], + "relationships": ["girlfriend", "wife", "divorce", "dating", "marriage"], +} + + +def build_lexicon() -> list[tuple[str, str, "re.Pattern"]]: + """Returns [(category, canonical_tag, compiled_regex)].""" + import re + out = [] + for category, table in (("person", PEOPLE), ("event", EVENTS), + ("topic", TOPICS)): + for tag, aliases in table.items(): + parts = [] + for alias in aliases: + a = alias.strip() + if not a: + continue + esc = re.escape(a) + if a[:1].isalnum(): + esc = r"\b" + esc + if a[-1:].isalnum(): + esc = esc + r"\b" + parts.append(esc) + if not parts: + continue + rx = re.compile("|".join(parts), re.IGNORECASE) + out.append((category, tag, rx)) + return out + + +def tags_for_text(text: str, lexicon=None) -> list[str]: + if lexicon is None: + lexicon = build_lexicon() + found = [] + for _cat, tag, rx in lexicon: + if rx.search(text): + found.append(tag) + return found diff --git a/peek.py b/peek.py new file mode 100644 index 0000000..941e178 --- /dev/null +++ b/peek.py @@ -0,0 +1,19 @@ +"""Quick peek: grep finished transcripts for saga keywords.""" +import json +import sys + +KEYS = ["isom", "eric july", "rippa", "comic book", "yellowflash"] + +for f in sys.argv[1:] or ["transcripts/ep0361.json", "transcripts/ep0362.json"]: + with open(f, encoding="utf-8") as fh: + d = json.load(fh) + print("===", d["title"]) + n = 0 + for s in d["segments"]: + t = s["text"].lower() + if any(k in t for k in KEYS): + m, sec = divmod(int(s["start"]), 60) + print(f' [{m:02d}:{sec:02d}] {s["text"][:110]}') + n += 1 + if n >= 8: + break diff --git a/pipeline_all.py b/pipeline_all.py new file mode 100644 index 0000000..d9fda11 --- /dev/null +++ b/pipeline_all.py @@ -0,0 +1,351 @@ +"""Full-backlog driver: Ep 1 -> latest. One pass, fully resumable, verbose. + +Per episode: + 1. skip if transcript + verified analysis already exist + 2. download mp3 (skipped if present) + 3. transcribe (re-does partial transcripts, e.g. old smoke tests) + 4. analyze via opencode (fresh session, WITH HARD TIMEOUT) + dossier fact-check + 5. every 10 episodes: rebuild FTS index + master JSON + +Run: python pipeline_all.py [--eps 84 87 89 ...] [--start-ep N] [--dry-run] +Log: stdout (use with redirection) +""" +import argparse +import json +import os +import sys +import time +import traceback +import subprocess + +try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + sys.stderr.reconfigure(encoding="utf-8", errors="replace") +except Exception: + pass + +from config import TRANSCRIPT_DIR, KEEP_AUDIO +from feed import load_episodes +from download import audio_path, slug as ep_slug, download_episode +from preprocess import prepare_audio, prepared_path +from transcribe import transcript_path +from lexicon import build_lexicon, tags_for_text +from export_txt import export_txt + +# seconds between opencode calls - provider rate limits are real +ANALYSIS_PAUSE = int(os.environ.get("DS_ANALYZE_PAUSE", "45")) +# hard timeout for entire analysis (seconds) - opencode can hang +ANALYSIS_TIMEOUT = int(os.environ.get("DS_ANALYZE_TIMEOUT", "1800")) # 30 minutes + +# memory guard threshold (bytes) - pause if available RAM drops below this +MIN_FREE_RAM = int(os.environ.get("DS_MIN_FREE_RAM", str(1024 * 1024 * 1024))) # 1 GB + +# encoding validation interval (episodes) +ENCODING_CHECK_INTERVAL = int(os.environ.get("DS_ENCODING_CHECK", "50")) + + +def has_partial(transcript_file: str) -> bool: + try: + with open(transcript_file, encoding="utf-8") as f: + return json.load(f).get("partial_minutes") is not None + except (OSError, json.JSONDecodeError): + return False + + +def verify_utf8_encoding(root: Path = Path('.')) -> tuple[int, int]: + """Verify all JSON files are valid UTF-8. Returns (errors, files_checked).""" + errors = 0 + checked = 0 + for path in Path('.').rglob("*.json"): + if any(part.startswith('.') for part in path.parts): + continue + if path.name.startswith('.'): + continue + try: + with open(path, 'r', encoding='utf-8') as f: + json.load(path) + except (UnicodeDecodeError, json.JSONDecodeError) as e: + print(f" [ENCODING ERROR] {path}: {e}", flush=True) + errors += 1 + checked += 1 + return errors, checked + + +def has_partial(transcript_file: str) -> bool: + try: + with open(transcript_file, encoding="utf-8") as f: + return json.load(f).get("partial_minutes") is not None + except (OSError, json.JSONDecodeError): + return False + + +def check_memory() -> bool: + """Check if we have enough free RAM. Return True if OK, False if low.""" + try: + import psutil + avail = psutil.virtual_memory().available + if avail < MIN_FREE_RAM: + return False + except ImportError: + pass # psutil not installed, skip check + except Exception: + pass + return True + + +def wait_for_memory() -> None: + """Block until enough RAM is available.""" + print(f" [MEM] Low memory (< {MIN_FREE_RAM/1e9:.1f} GB free), waiting...", flush=True) + while not check_memory(): + time.sleep(30) + print(f" [MEM] Memory recovered, continuing...", flush=True) + + +def do_transcribe(model, ep: dict) -> None: + from transcribe import transcribe_episode + from preprocess import load_pcm + src = audio_path(ep) + if not os.path.exists(src): + print(f" [{time.strftime('%H:%M:%S')}] downloading {ep['audio_url'][-40:]} ...", flush=True) + download_episode(ep) + prep = prepared_path(ep) + ok = False + t0 = time.time() + try: + # Check memory before loading episode + wait_for_memory() + print(f" [{time.strftime('%H:%M:%S')}] transcribing Ep {ep['ep_num']} ({ep['title'][:50]})", flush=True) + result = transcribe_episode(model, ep, None) + outp = transcript_path(ep) + with open(outp + ".tmp", "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False) + os.replace(outp + ".tmp", outp) + dur = result["duration_sec"] or 1 + rate = dur / max(time.time() - t0, 0.001) + lex = build_lexicon() + tags = set() + for seg in result["segments"]: + tags.update(tags_for_text(seg["text"], lex)) + print(f" [{time.strftime('%H:%M:%S')}] TRANSCRIBED: {len(result['segments'])} segments, " + f"{dur/60:.0f} min, {rate:.1f}x realtime", flush=True) + print(f" TAGS: {' '.join(sorted('#' + t for t in tags)) or '(none)'}", flush=True) + ok = True + finally: + if os.path.exists(prep): + try: + os.remove(prep) + except OSError: + pass + if not KEEP_AUDIO and ok and os.path.exists(src): + try: + os.remove(src) + except OSError: + pass + + +def run_analysis_with_timeout(ep: dict) -> tuple: + """ + Run analysis + verification using opencode subprocess with hard timeout. + Returns (success: bool, result: dict | None, error: str | None) + """ + from analyze import ANALYSIS_DIR, OPENCODE, build_prompt, strip_to_json, validate + from verify import verify_episode + from download import slug as ep_slug + from export_txt import txt_path + from config import BASE_DIR + + label = f"Ep {ep['ep_num']}" + apath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json") + + # Check if already analyzed + if os.path.exists(apath): + with open(apath, encoding="utf-8") as f: + existing = json.load(f) + return (True, existing, None) + + # Build prompt + prompt = build_prompt() + attach = export_txt(ep) + + # Run analysis with timeout + rate-limit retries + cmd = [OPENCODE, "run", "--auto", "-f", attach, "--dir", BASE_DIR, prompt] + + # Rate limit retry waits (seconds) + rate_limit_waits = [10, 60, 180, 420, 600] + + for attempt, wait_time in enumerate(rate_limit_waits, 1): + try: + print(f" [{time.strftime('%H:%M:%S')}] running analysis (attempt {attempt}, timeout {ANALYSIS_TIMEOUT}s)...", flush=True) + proc = subprocess.run(cmd, capture_output=True, text=True, + encoding="utf-8", errors="replace", + cwd=BASE_DIR, timeout=ANALYSIS_TIMEOUT) + out = proc.stdout or "" + err = proc.stderr or "" + combined = (out + err).lower() + + if "rate limit" in combined or "429" in combined: + raise RuntimeError("provider rate limit exceeded") + + if not out.strip() and err: + raise RuntimeError(f"opencode produced no stdout: {err[-400:]}") + + # Parse result + from analyze import _repair + obj = json.loads(_repair(out), strict=False) + validate(obj) + + # Add metadata + obj["ep_num"] = ep["ep_num"] if ep["ep_num"] else obj.get("ep_num") + obj["title"] = obj.get("title") or ep["title"] + obj["pub_date"] = ep["pub_date"] + obj["_analyzed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + obj["_source_transcript"] = os.path.basename(transcript_path(ep)) + + # Save analysis + from analyze import ANALYSIS_DIR as ANALYSIS_DIR2 + from download import slug as ep_slug + apath = os.path.join(ANALYSIS_DIR2, f"{ep_slug(ep)}.json") + with open(apath + ".tmp", "w", encoding="utf-8") as f: + json.dump(obj, f, ensure_ascii=False, indent=1) + os.replace(apath + ".tmp", apath) + + # Now run verification + print(f" [{time.strftime('%H:%M:%S')}] fact-checking...", flush=True) + from verify import verify_episode + verify_episode(ep) + + return (True, obj, None) + + except RuntimeError as e: + if "rate limit" in str(e).lower(): + if attempt < len(rate_limit_waits): + wait = rate_limit_waits[attempt] + print(f" [{time.strftime('%H:%M:%S')}] rate limited (attempt {attempt}), sleeping {wait}s...", flush=True) + time.sleep(wait) + continue + else: + return (False, None, f"Rate limit exceeded after {attempt} attempts") + else: + return (False, None, str(e)) + except Exception as e: + return (False, None, str(e)) + + return (False, None, "Max retries exceeded") + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--eps", nargs="*", type=int, default=None, + help="Specific episode numbers to process (default: all from start-ep)") + ap.add_argument("--start-ep", type=int, default=0) + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + from faster_whisper import WhisperModel + from config import WHISPER_MODEL, DEVICE, COMPUTE_TYPE + from analyze import analyze_episode, OPENCODE + from verify import verify_episode + + eps = load_episodes() + if args.eps: + eps = [e for e in eps if e["ep_num"] in args.eps] + else: + eps = [e for e in eps if (e["ep_num"] or 0) >= args.start_ep] + + print(f"[{time.strftime('%H:%M:%S')}] pipeline over {len(eps)} episodes " + f"(model={WHISPER_MODEL}/{DEVICE}, analyzer={os.path.basename(OPENCODE)})", flush=True) + + if args.dry_run: + for e in eps[:20]: + tp = transcript_path(e) + t_exists = os.path.exists(tp) and not has_partial(tp) + a_exists = os.path.exists(os.path.join( + os.path.dirname(TRANSCRIPT_DIR), "analysis", f"{e['ep_num']:04d}.json")) if e["ep_num"] else False + print(f" ep {e['ep_num']}: transcript={'Y' if t_exists else 'N'} analysis={'Y' if a_exists else 'N'}", flush=True) + return + + model = None + done_t = done_a = 0 + t_start = time.time() + + for i, ep in enumerate(eps, 1): + num = ep["ep_num"] if ep["ep_num"] else ep["title"][:30] + tp = transcript_path(ep) + + need_transcript = (not os.path.exists(tp)) or has_partial(tp) + if need_transcript and model is None: + print(f"[{time.strftime('%H:%M:%S')}] loading whisper model '{WHISPER_MODEL}'...", flush=True) + model = WhisperModel(WHISPER_MODEL, device=DEVICE, compute_type=COMPUTE_TYPE) + + if has_partial(tp): + print(f"[{time.strftime('%H:%M:%S')}] [{i}/{len(eps)}] Ep {num}: redo partial transcript", flush=True) + os.remove(tp) + elif need_transcript: + print(f"[{time.strftime('%H:%M:%S')}] [{i}/{len(eps)}] Ep {num}: transcribing", flush=True) + + if need_transcript: + try: + do_transcribe(model, ep) + done_t += 1 + # keep the searchable index fresh-ish + if done_t % 10 == 0: + import index as index_mod + print(f" [{time.strftime('%H:%M:%S')}] rebuilding search index...", flush=True) + index_mod.index_transcripts() + except Exception as ex: + print(f" [{time.strftime('%H:%M:%S')}] TRANSCRIBE FAILED, skipping episode: {ex}", flush=True) + traceback.print_exc() + continue + + # analysis + fact-check (with hard timeout) + label = f"Ep {num}" + print(f" [{time.strftime('%H:%M:%S')}] analyzing {label} ...", flush=True) + try: + success, result, error = run_analysis_with_timeout(ep) + if success: + done_a += 1 + print(f" [{time.strftime('%H:%M:%S')}] analyzed {label} (TI={result['transgression']['overall_ti']}, " + f"{len(result['talking_points'])} points, {len(result['notable_quotes'])} quotes)", flush=True) + print(f" [{time.strftime('%H:%M:%S')}] verified {label}", flush=True) + else: + print(f" [{time.strftime('%H:%M:%S')}] ANALYSIS FAILED {label}: {error}", flush=True) + except Exception as ex: + print(f" [{time.strftime('%H:%M:%S')}] ANALYZE FAILED {label}: {ex}", flush=True) + traceback.print_exc() + + # pacing: keep the provider happy + print(f" [{time.strftime('%H:%M:%S')}] pacing pause {ANALYSIS_PAUSE}s ...", flush=True) + time.sleep(ANALYSIS_PAUSE) + + if i % 10 == 0: + print(f" [{time.strftime('%H:%M:%S')}] rebuilding search index (periodic)...", flush=True) + import index as index_mod + index_mod.index_transcripts() + from aggregate import aggregate as _agg + _agg() + + # Periodic encoding verification + if i % ENCODING_CHECK_INTERVAL == 0: + print(f" [{time.strftime('%H:%M:%S')}] verifying UTF-8 encoding...", flush=True) + errors, checked = verify_utf8_encoding() + if errors: + print(f" [WARNING] Found {errors} files with encoding issues!", flush=True) + else: + print(f" [OK] UTF-8 encoding verified for {checked} files", flush=True) + + hrs = (time.time() - t_start) / 3600 + rate_eps = i / hrs + remaining = (len(eps) - i) / max(rate_eps, 0.01) + print(f"[{time.strftime('%H:%M:%S')}] == progress: {i}/{len(eps)} eps " + f"({rate_eps:.1f} eps/hr, ~{remaining:.0f} hr left) ==", flush=True) + + print(f"[{time.strftime('%H:%M:%S')}] final index rebuild...", flush=True) + import index as index_mod + index_mod.index_transcripts() + from aggregate import aggregate as _agg + _agg() + print(f"[{time.strftime('%H:%M:%S')}] PIPELINE COMPLETE: transcribed={done_t} analyzed={done_a}", flush=True) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/preprocess.py b/preprocess.py new file mode 100644 index 0000000..c5811e5 --- /dev/null +++ b/preprocess.py @@ -0,0 +1,82 @@ +"""Robust audio preprocessing. + +Libsyn MP3s confuse faster-whisper's decoder on this box (bogus multi-channel +allocation). We therefore demux+decode ourselves with PyAV in bounded chunks, +downmix to mono, resample to 16 kHz s16, and write a plain WAV that Whisper +ingests instantly. + +Full-length prep of a 3 h episode takes ~1 min and ~340 MB; the file is +deleted by transcribe.py afterwards unless DS_PREP_KEEP=1. +""" +import os +import wave + +import av +import numpy as np + +from config import AUDIO_DIR +from download import slug as ep_slug + +PREP_DIR = os.path.join(AUDIO_DIR, "_prep") +TARGET_RATE = 16000 + + +def prepared_path(ep: dict) -> str: + return os.path.join(PREP_DIR, ep_slug(ep) + ".16k.wav") + + +def prepare_audio(src_mp3: str, dst_wav: str, + max_seconds: float | None = None) -> str: + os.makedirs(PREP_DIR, exist_ok=True) + if os.path.exists(dst_wav): + return dst_wav + + resampler = av.AudioResampler(format="s16", layout="mono", + rate=TARGET_RATE) + total_written = 0 + limit = int(max_seconds * TARGET_RATE) if max_seconds else None + + tmp = dst_wav + ".tmp" + with av.open(src_mp3, metadata_errors="ignore") as container: + # pick the first real audio stream explicitly + streams = [s for s in container.streams.audio] + if not streams: + raise ValueError(f"no audio stream in {src_mp3}") + stream = streams[0] + + import wave + wf = wave.open(tmp, "wb") + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(TARGET_RATE) + try: + for frame in container.decode(stream): + resampled = resampler.resample(frame) + for rf in resampled: + arr = rf.to_ndarray() + if arr.ndim == 2: # (1, samples) planar mono + arr = arr[0] + pcm = arr.astype(np.int16).tobytes() + n = len(pcm) // 2 + if limit and total_written + n > limit: + n = limit - total_written + pcm = pcm[: n * 2] + wf.writeframes(pcm) + total_written += n + if limit and total_written >= limit: + raise StopIteration + except StopIteration: + pass + finally: + wf.close() + + os.replace(tmp, dst_wav) + return dst_wav + + +def load_pcm(dst_wav: str) -> np.ndarray: + """Load a prepared 16 kHz s16 mono wav as float32 in [-1, 1].""" + with wave.open(dst_wav, "rb") as wf: + assert wf.getnchannels() == 1 and wf.getframerate() == TARGET_RATE + frames = wf.readframes(wf.getnframes()) + return np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..dc30adb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests>=2.31 +faster-whisper>=1.0.0 diff --git a/rubric.py b/rubric.py new file mode 100644 index 0000000..8427760 --- /dev/null +++ b/rubric.py @@ -0,0 +1,80 @@ +"""The Transgression Index (TI): an arbitrary-but-anchored 0-10 scale for how +edgy/transgressive content is. Deliberately opinionated anchors so scores are +comparable across episodes. Shared by analyze.py prompts and aggregate.py. +""" + +RUBRIC = """ +TRANSGRESSION INDEX (TI) RUBRIC - score each quote and each dimension 0-10: + +0 Wholesome/neutral: could air on network morning TV. +1 Mild crudeness: innuendo, toilet humor, light profanity. +2 Frequent profanity, explicit sexual talk, crude insults aimed at public figures. +3 Edgy shock humor: jokes about death/tragedy/disability; mean-spirited roasts. +4 Slurs used (quoted or reclaimed), graphic sexual content, sustained cruelty + toward an individual as a running bit. +5 Dehumanizing generalizations about groups (by race/sex/religion/etc.) played + for laughs; celebration of harassment campaigns. +6 Sincere demeaning rhetoric about protected groups (not clearly ironic); + praising doxxing/deplatforming of enemies; glorifying self-harm or violence + rhetorically. +7 Targeted abuse: encouraging audiences to swarm/threaten a named private + individual; credible threats framed as jokes. +8 Endorsement of political violence or extremism beyond rhetoric; material + support (fundraising) for violent/extremist causes. +9 Direct incitement to violence; non-consensual sexual content endorsement. +10 Planning/praising specific acts of terror or mass violence. + +Sub-dimensions (each 0-10, same anchors applied narrowly): +- profanity: raw frequency/intensity of swearing only. +- slur_usage: occurrence of slurs/bigoted epithets (quoted bits count half). +- harassment_or_cruelty: targeted attacks on named individuals. +- sincerity: how sincere vs comedic the worst content is (0 = pure bit, + 10 = stated sincerely). NOTE: this measures tone, not severity. + +overall_ti is NOT an average - it is the peak sustained level (>=60 seconds +or a defining moment of the episode). +""" + +SCHEMA_EXAMPLE = """{ + "ep_num": 361, + "title": "Episode 361 - Dick on Lizard Feet", + "pub_date": "2023-05-30", + "runtime_minutes": 147, + "one_line_summary": "...", + "guests": [{"name": "...", "role": "host/cohost/guest/caller", "notes": "..."}], + "talking_points": [ + {"topic": "short label", + "summary": "2-3 sentence account of what was said, including positions and punchlines", + "start_ts": 1234.5, + "people": ["maddox", "eric july"], + "controversy_ti": 4} + ], + "arguments": [ + {"claim": "what the host asserts", + "position": "pro/anti/mockery-of", + "target": "who or what it is against", + "quality": "strong|moderate|weak|trollish", + "evidence_used": "receipts, anecdotes, statistics cited"} + ], + "ideas": [ + {"concept": "coined term / recurring bit / theory introduced", + "explanation": "what it means", + "first_ts": 1234.5} + ], + "notable_quotes": [ + {"quote": "verbatim quote (may be lightly cleaned)", + "ts": 1234.5, + "ti": 5, + "why": "one clause explaining the score"} + ], + "transgression": { + "overall_ti": 5, + "profanity": 7, + "slur_usage": 3, + "harassment_or_cruelty": 4, + "sincerity": 2, + "targets": ["groups or individuals the edge is aimed at"], + "rationale": "2 sentences justifying overall_ti" + }, + "content_warnings": ["slurs", "sexual content"] +}""" diff --git a/run_all.py b/run_all.py new file mode 100644 index 0000000..8eae686 --- /dev/null +++ b/run_all.py @@ -0,0 +1,50 @@ +"""One-command pipeline: feed -> download -> transcribe -> index. + + python run_all.py # everything not yet done + python run_all.py --ep 361 # just one episode end-to-end (good first run) + python run_all.py --minutes 5 --ep 520 # 5-minute smoke test +""" +import argparse +import subprocess +import sys + + +def run(cmd: list[str]) -> int: + print(f"\n=== {' '.join(cmd)} ===\n", flush=True) + return subprocess.call([sys.executable] + cmd) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--ep", type=int, nargs="*", default=None) + ap.add_argument("--minutes", type=float, default=None) + ap.add_argument("--skip-download", action="store_true") + ap.add_argument("--skip-transcribe", action="store_true") + ap.add_argument("--force", action="store_true") + args = ap.parse_args() + + rc = run(["feed.py"]) + if rc: + return rc + + if not args.skip_download: + cmd = ["download.py"] + ([str(e) for e in args.ep] if args.ep else []) + if run(cmd): + print("download step reported failures; continuing", file=sys.stderr) + + if not args.skip_transcribe: + cmd = ["transcribe.py"] + if args.ep: + cmd += [str(e) for e in args.ep] + if args.minutes: + cmd += ["--minutes", str(args.minutes)] + if args.force: + cmd += ["--force"] + if run(cmd): + return 1 + + return run(["index.py"]) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/search.py b/search.py new file mode 100644 index 0000000..64a1e0d --- /dev/null +++ b/search.py @@ -0,0 +1,136 @@ +"""Search the transcript index. + +Usage: + python search.py "isom trademark" + python search.py "pull up" --tag eric_july + python search.py "judge ramos" --ep 103 --limit 5 + python search.py "rage" --from 2017-01-01 --to 2019-12-31 --report out.txt + +Output lines include EPISODE + TIMESTAMP so they can be deep-linked to audio +later: [Ep 373 | 2023-08-29] 01:12:45-01:13:02 "...transcript snippet..." +""" +import argparse +import datetime as dt +import json +import re +import sqlite3 +import sys + +from config import DB_PATH +from index import open_db + + +def ts(seconds: float) -> str: + s = int(seconds) + return f"{s // 3600:02d}:{(s % 3600) // 60:02d}:{s % 60:02d}" + + +def parse_date(s: str): + return dt.date.fromisoformat(s) + + +def pubdate_year(pub: str): + m = re.search(r"(\d{4})", pub or "") + return int(m.group(1)) if m else None + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("query", help="FTS5 query (supports \"quoted phrases\", AND/OR)") + ap.add_argument("--tag", action="append", default=[], + help="filter by hashtag (repeatable)") + ap.add_argument("--ep", type=int, action="append", default=[], + help="restrict to episode number(s)") + ap.add_argument("--from", dest="date_from", default=None) + ap.add_argument("--to", dest="date_to", default=None) + ap.add_argument("--limit", type=int, default=25) + ap.add_argument("--context", type=int, default=0, + help="include N neighbouring segments") + ap.add_argument("--report", default=None, + help="also write matches to this file") + args = ap.parse_args() + + con = open_db(DB_PATH) + q = args.query.strip() + + sql = """ + SELECT s.rowid, e.ep_num, e.guid, e.title, e.pub_date, e.audio_url, + e.page_link, s.seg_idx, s.start_sec, s.end_sec, s.text, s.tags + FROM segments s JOIN episodes e ON e.id = s.episode_id + WHERE segments MATCH ? + """ + params = [q] + + if args.tag: + for t in args.tag: + sql += " AND s.tags LIKE ?" + params.append(f"%#{t}%") + if args.ep: + sql += " AND e.ep_num IN (%s)" % ",".join("?" * len(args.ep)) + params += args.ep + if args.date_from: + y = parse_date(args.date_from).year + # crude pub-date year filter; feed dates are RFC2822 + sql += " AND CAST(substr(e.pub_date, length(e.pub_date)-3, 4) AS INTEGER) >= ?" + params.append(y) + if args.date_to: + y = parse_date(args.date_to).year + sql += " AND CAST(substr(e.pub_date, length(e.pub_date)-3, 4) AS INTEGER) <= ?" + params.append(y) + + sql += " ORDER BY e.pub_date LIMIT ?" + params.append(args.limit) + + rows = con.execute(sql, params).fetchall() + if not rows: + print("no matches") + return + + lines = [] + + def parse_tags(raw: str) -> list[str]: + raw = raw or "" + if not raw: + return [] + if raw.startswith("["): + try: + return json.loads(raw) + except ValueError: + return [] + return [t for t in raw.split() if t] + + for r in rows: + (_rid, ep_num, guid, title, pub, audio_url, page_link, seg_idx, + start, end, text, tags) = r + label = f"Ep {ep_num}" if ep_num else title[:30] + taglist = parse_tags(tags) + head = (f"[{label} | {pub}] {ts(start)}-{ts(end)} " + f"{' '.join(taglist)}") + body = f" \"{text}\"" + extra = "" + if args.context: + ctx = con.execute( + """SELECT start_sec, end_sec, text FROM segments + WHERE episode_id=(SELECT id FROM episodes WHERE guid=?) + AND seg_idx BETWEEN ? AND ? ORDER BY seg_idx""", + (guid, max(0, seg_idx - args.context), + seg_idx + args.context)).fetchall() + body = "\n".join( + f" [{ts(a)}-{ts(b)}] {t}" for a, b, t in ctx) + link = page_link or audio_url + extra = f" -> {link}\n mp3: {audio_url}" + lines.append(head + "\n" + body + "\n" + extra) + + out = "\n\n".join(lines) + print(out[:20000]) + print(f"\n({len(rows)} result(s), limit {args.limit})") + + if args.report: + with open(args.report, "a", encoding="utf-8") as f: + f.write(f"\n===== QUERY: {q} tags={args.tag} eps={args.ep} =====\n") + f.write(out + "\n") + print(f"appended to {args.report}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/status.py b/status.py new file mode 100644 index 0000000..84de827 --- /dev/null +++ b/status.py @@ -0,0 +1,14 @@ +"""Print master-file integrity summary.""" +import json + +m = json.load(open("data/dickshow_analysis.json", encoding="utf-8")) +meta = m["meta"] +print("meta:", {k: meta[k] for k in + ("episodes_analyzed", "episodes_factchecked", + "total_runtime_minutes")}) +for e in m["episodes"]: + v = e.get("_verification", {}) + n_issues = len(v.get("issues", [])) + print(f"Ep {e['ep_num']}: TI={e['transgression']['overall_ti']} " + f"verified={v.get('verdict')} issues_fixed={n_issues} " + f"points={len(e['talking_points'])}") diff --git a/transcribe.py b/transcribe.py new file mode 100644 index 0000000..ee2bf46 --- /dev/null +++ b/transcribe.py @@ -0,0 +1,170 @@ +"""Whisper transcription with segment timestamps. + +Reads audio/*.mp3, writes transcripts/.json: +{ + "guid": ..., "ep_num": ..., "title": ..., "pub_date": ..., + "audio_url": ..., "model": ..., "segments": [ + {"start": 12.34, "end": 18.90, "text": "..."}, ... + ] +} +Fully resumable: finished transcripts are skipped unless --force. +Use --minutes N to transcribe only the first N minutes (testing). +""" +import argparse +import json +import os +import sys +import time + +from config import (TRANSCRIPT_DIR, WHISPER_MODEL, DEVICE, COMPUTE_TYPE, + LANGUAGE, VAD_FILTER, BEAM_SIZE, WORD_TIMESTAMPS, + CONDITION_ON_PREVIOUS_TEXT, KEEP_AUDIO) +from feed import load_episodes +from download import audio_path, slug as ep_slug +from preprocess import prepare_audio, prepared_path, load_pcm +from lexicon import build_lexicon, tags_for_text + +# faster-whisper runs one giant FFT over the whole input; multi-hour episodes +# blow up RAM (~2.5 GB arrays). Transcribe in fixed windows instead. +CHUNK_SECONDS = 600 + + +def transcript_path(ep: dict) -> str: + return os.path.join(TRANSCRIPT_DIR, ep_slug(ep) + ".json") + + +def transcribe_episode(model, ep: dict, minutes: float | None) -> dict: + src = audio_path(ep) + prep = prepared_path(ep) + t0 = time.time() + prepare_audio(src, prep, max_seconds=minutes * 60 + 5 if minutes else None) + print(f" prepped audio in {time.time() - t0:.0f}s -> {os.path.basename(prep)}", + flush=True) + + pcm = load_pcm(prep) + total_samples = len(pcm) + if minutes: + total_samples = min(total_samples, int(minutes * 60 * 16000)) + duration = total_samples / 16000.0 + + segments = [] + n_chunks = (total_samples + CHUNK_SECONDS * 16000 - 1) // (CHUNK_SECONDS * 16000) + for ci in range(n_chunks): + a = ci * CHUNK_SECONDS * 16000 + b = min(a + CHUNK_SECONDS * 16000, total_samples) + offset = a / 16000.0 + chunk = pcm[a:b] + segs_iter, info = model.transcribe( + chunk, + language=LANGUAGE, + vad_filter=VAD_FILTER, + beam_size=BEAM_SIZE, + word_timestamps=WORD_TIMESTAMPS, + condition_on_previous_text=CONDITION_ON_PREVIOUS_TEXT, + ) + for seg in segs_iter: + text = seg.text.strip() + if not text: + continue + segments.append({ + "start": round(seg.start + offset, 2), + "end": round(seg.end + offset, 2), + "text": text, + }) + print(f" chunk {ci + 1}/{n_chunks} done " + f"(audio {(b - a) / 16000 / 60:.0f}m, " + f"{len(segments)} segs so far)", flush=True) + + return { + "guid": ep["guid"], + "ep_num": ep["ep_num"], + "title": ep["title"], + "pub_date": ep["pub_date"], + "audio_url": ep["audio_url"], + "page_link": ep.get("page_link", ""), + "duration_sec": duration, + "model": WHISPER_MODEL, + "language": LANGUAGE, + "partial_minutes": minutes, + "segments": segments, + } + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("eps", nargs="*", type=int, + help="episode numbers; default: all") + ap.add_argument("--minutes", type=float, default=None, + help="only transcribe first N minutes (testing)") + ap.add_argument("--force", action="store_true", + help="re-transcribe even if transcript exists") + args = ap.parse_args() + + from faster_whisper import WhisperModel + + eps = load_episodes() + todo = [] + for ep in eps: + if ep["ep_num"] is not None and args.eps and ep["ep_num"] not in args.eps: + continue + tp = transcript_path(ep) + if os.path.exists(tp) and not args.force and not args.minutes: + continue + if not os.path.exists(audio_path(ep)): + print(f"skip (no audio): {ep['title']}", file=sys.stderr) + continue + todo.append(ep) + + model = None # lazy-load after we know there's work + done = 0 + for i, ep in enumerate(todo, 1): + label = f"Ep {ep['ep_num']}" if ep["ep_num"] else ep["title"][:40] + if model is None: + print(f"loading whisper model '{WHISPER_MODEL}' " + f"(device={DEVICE}, compute={COMPUTE_TYPE}) ...") + model = WhisperModel(WHISPER_MODEL, device=DEVICE, + compute_type=COMPUTE_TYPE) + t0 = time.time() + src = audio_path(ep) + prep = prepared_path(ep) + ok = False + try: + result = transcribe_episode(model, ep, args.minutes) + outp = transcript_path(ep) + with open(outp + ".tmp", "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False) + os.replace(outp + ".tmp", outp) + dur = result["duration_sec"] or 1 + rate = dur / max(time.time() - t0, 0.001) + n_seg = len(result["segments"]) + lexicon = build_lexicon() + ep_tag_set: set[str] = set() + for seg in result["segments"]: + ep_tag_set.update(tags_for_text(seg["text"], lexicon)) + print(f"[{i}/{len(todo)}] {label}: {n_seg} segments, " + f"{dur/60:.0f} min audio, {rate:.1f}x realtime") + print(f" TAGS: {' '.join('#' + t for t in sorted(ep_tag_set)) or '(none)'}") + done += 1 + ok = True + except Exception as e: # noqa: BLE001 + print(f"{label}: TRANSCRIBE FAILED - {e}", file=sys.stderr) + continue + finally: + # reclaim disk: prepped wav always goes; mp3 unless KEEP_AUDIO + if os.path.exists(prep): + try: + os.remove(prep) + except OSError: + pass + if not KEEP_AUDIO and ok and not args.minutes: + try: + os.remove(src) + except OSError: + pass + + print(f"transcribed={done} skipped_existing=" + f"{len([e for e in eps if os.path.exists(transcript_path(e))])}") + + +if __name__ == "__main__": + main() diff --git a/verify.py b/verify.py new file mode 100644 index 0000000..5c85c21 --- /dev/null +++ b/verify.py @@ -0,0 +1,177 @@ +"""Second-pass fact-check: verify an episode analysis against the research +dossier (Dax_Herrera_Dick_Masterson_Profile.txt) via a second opencode run. + +Spots "strange assumptions" like attributing a 2023 cohost seat to Maddox. +Usage: + python verify.py # verify ALL existing analyses + python verify.py 361 373 # verify specific episodes +""" +import json +import os +import sys +import time + +# console-safe output (model errors may contain any unicode) +try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + sys.stderr.reconfigure(encoding="utf-8", errors="replace") +except Exception: + pass + +from config import BASE_DIR +from feed import load_episodes +from download import slug as ep_slug +from facts import DOSSIER_PATH, CAST_SHEET +from analyze import (ANALYSIS_DIR, OPENCODE, run_opencode, strip_to_json, + validate) +from export_txt import txt_path + +VERIFY_PROMPT = """You are the fact-checking archivist for The Dick Show project. Three files are attached: + + FILE 1: the episode transcript (timestamped lines [HH:MM:SS]). + FILE 2: a researched dossier about the host, Dax Herrera aka Dick Masterson (verified court records, timeline, cast roster, feuds). + FILE 3: a candidate ANALYSIS JSON of that episode produced by another analyst pass. + +Your job: cross-check FILE 3 against FILE 1 and FILE 2 for FACTUAL consistency only. Do not judge opinions, humor, or TI scores. Check specifically: +- Speaker/role attribution. {cast} +- Names, spellings, and who is who (e.g. Sean vs Vito vs Asterios vs Maddox). +- Timeline claims vs the dossier (what was happening when this episode aired; which feud is active). +- Events/people attributed to the wrong era or wrong person. +- Claims about guests that contradict the transcript itself. + +Output ONLY one valid JSON object, no fences, no commentary: +{{ + "verdict": "pass" or "fix", + "issues": [ + {{"field": "guests|talking_points|arguments|ideas|notable_quotes|transgression", + "claim": "...", "issue": "...", "correction": "..."}} + ], + "corrected_analysis": {{ ... full corrected analysis JSON, same schema as FILE 3, or null if verdict=pass ... }} +}} + +Rules: +- If everything checks out: verdict="pass", issues=[], corrected_analysis=null. +- If ANY factual issue affects content: verdict="fix" and corrected_analysis MUST contain the complete corrected JSON (not a diff). +- Keep all scores/timestamps unless they are part of an identified error. +Begin now.""" + + +def build_verify_prompt() -> str: + return VERIFY_PROMPT.replace("{cast}", CAST_SHEET.strip()) + + +def verify_episode(ep: dict, force: bool = False) -> dict | None: + apath = os.path.join(ANALYSIS_DIR, f"{ep_slug(ep)}.json") + if not os.path.exists(apath): + return None + with open(apath, encoding="utf-8") as f: + current = json.load(f) + + attach_txt = txt_path(ep) + if not os.path.exists(attach_txt): + from export_txt import export_txt + attach_txt = export_txt(ep) + if not os.path.exists(DOSSIER_PATH): + print(f" dossier missing at {DOSSIER_PATH}; cannot verify", + file=sys.stderr) + return None + + prompt = build_verify_prompt() + from analyze import RateLimitError + waits = [30, 120, 300, 600] + attempt = 0 + last_err = None + while attempt < len(waits): + attempt += 1 + try: + raw = run_opencode_multi(prompt, [attach_txt, DOSSIER_PATH, apath]) + verdict = strip_to_json(raw) + v = verdict.get("verdict") + if v not in ("pass", "fix"): + raise ValueError(f"bad verdict: {v!r}") + if v == "pass": + current.setdefault("_verification", { + "verdict": "pass", "checked_at": _now(), "issues": []}) + _save(apath, current) + print(f" verified OK (no issues)", flush=True) + return current + fixed = verdict.get("corrected_analysis") + if not isinstance(fixed, dict): + raise ValueError("verdict=fix but no corrected_analysis") + validate(fixed) + issues = verdict.get("issues", []) + fixed["_verification"] = {"verdict": "fix", "checked_at": _now(), + "issues": issues} + fixed["ep_num"] = ep["ep_num"] if ep["ep_num"] else fixed.get("ep_num") + _save(apath, fixed) + print(f" FIXED {len(issues)} issue(s):", flush=True) + for i in issues[:6]: + print(f" - [{i.get('field')}] {str(i.get('issue'))[:90]}", + flush=True) + return fixed + except RateLimitError as e: + last_err = e + wait = waits[min(attempt, len(waits) - 1)] + print(f" verify rate-limited (attempt {attempt}); " + f"sleeping {wait}s ...", flush=True) + time.sleep(wait) + except Exception as e: # noqa: BLE001 + last_err = e + wait = waits[min(attempt - 1, len(waits) - 1)] + print(f" verify attempt {attempt} failed: {e}; " + f"retry in {wait}s", flush=True) + time.sleep(wait) + print(f" verification FAILED for {ep['title']}: {last_err}", flush=True) + return None + + +def run_opencode_multi(prompt: str, attachments: list[str]) -> str: + from analyze import OPENCODE_MODEL + cmd = [OPENCODE, "run", "--auto"] + for a in attachments: + cmd += ["-f", a] + cmd += ["--dir", BASE_DIR] + if OPENCODE_MODEL: + cmd += ["-m", OPENCODE_MODEL] + cmd.append(prompt) + import subprocess + proc = subprocess.run(cmd, capture_output=True, text=True, + encoding="utf-8", errors="replace", + cwd=BASE_DIR, timeout=900) + out = proc.stdout or "" + if not out.strip() and proc.stderr: + raise RuntimeError(f"opencode produced no stdout: " + f"{proc.stderr[-400:]}") + return out + + +def _now() -> str: + from datetime import datetime, timezone + return datetime.now(timezone.utc).isoformat() + + +def _save(apath: str, obj: dict) -> None: + tmp = apath + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(obj, f, ensure_ascii=False, indent=1) + os.replace(tmp, apath) + + +def main() -> None: + args = [a for a in sys.argv[1:] if not a.startswith("-")] + eps = load_episodes() + todo = [e for e in eps + if os.path.exists(os.path.join(ANALYSIS_DIR, + f"{ep_slug(e)}.json")) + and (not args or e["ep_num"] in {int(a) for a in args})] + print(f"verifying {len(todo)} analysis file(s)", flush=True) + for i, ep in enumerate(todo, 1): + label = f"Ep {ep['ep_num']}" if ep['ep_num'] else ep['title'][:40] + print(f"[{i}/{len(todo)}] verifying {label} ...", flush=True) + verify_episode(ep) + from aggregate import aggregate + aggregate() + + +if __name__ == "__main__": + main() diff --git a/verify_encoding.py b/verify_encoding.py new file mode 100644 index 0000000..08cc787 --- /dev/null +++ b/verify_encoding.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Verify UTF-8 encoding in all JSON files.""" + +import json +import sys +from pathlib import Path + +def verify_utf8(path: Path) -> tuple[int, int]: + """Check a JSON file for valid UTF-8. Returns (errors, bytes_checked).""" + try: + with open(path, 'r', encoding='utf-8') as f: + content = f.read() + except UnicodeDecodeError as e: + print(f" [ERROR] {path}: Invalid UTF-8 at position {e.start}") + return (1, 0) + + try: + json.loads(content) + except json.JSONDecodeError as e: + print(f" [ERROR] {path}: Invalid JSON at position {e.pos}: {e}") + return (1, 0) + + return (0, len(content)) + +def main(path: str = '.') -> int: + import argparse + parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files") + parser.add_argument('path', nargs='?', default='.', help='Root directory to scan') + args = parser.parse_args() + + root = Path(args.path).resolve() + print(f"Scanning {root} for JSON files...") + + stats = {"files_checked": 0, "errors": 0, "total_bytes": 0} + + for path in root.rglob("*.json"): + if any(part.startswith('.') for part in path.parts): + continue + if path.name.startswith('.'): + continue + + errors, bytes_checked = verify_utf8(path) + stats["files_checked"] = stats.get("files_checked", 0) + 1 + stats["errors"] += errors + stats["total_bytes"] = stats.get("total_bytes", 0) + bytes_checked + + if stats["files_checked"] % 50 == 0: + print(f" Checked {stats['files_checked']} files...") + + print(f"\nResults:") + print(f" Files checked: {stats['files_checked']}") + print(f" Total bytes: {stats['total_bytes']:,}") + print(f" Encoding errors: {stats['errors']}") + + if stats['errors'] == 0: + print("All JSON files are valid UTF-8!") + return 0 + else: + print(f"Found {stats['errors']} files with encoding issues") + return 1 + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files") + parser.add_argument('path', nargs='?', default='.', help='Root directory to scan') + args = parser.parse_args() + sys.exit(main(args.path)) \ No newline at end of file