Initial commit: Dick Show Index pipeline with install script and docs

index.py

@@ -0,0 +1,115 @@
+"""Build the SQLite FTS5 index from transcripts + lexicon hashtags.
+
+Schema:
+  episodes(ep_num, guid, title, pub_date, audio_url, page_link, duration_sec,
+           model, n_segments, tags_json)
+  segments(ep_rowid -> episodes.rowid, seg_idx, start_sec, end_sec, text,
+           tags)  [FTS5 external content]
+
+Search examples (via search.py):
+  python search.py "isom lawsuit"
+  python search.py maddox --tag lolsuit --from 2018-01-01 --to 2018-12-31
+"""
+import json
+import os
+import sqlite3
+
+from config import DB_PATH
+from feed import load_episodes
+from lexicon import build_lexicon, tags_for_text
+
+SCHEMA = """
+PRAGMA journal_mode=WAL;
+
+CREATE TABLE IF NOT EXISTS episodes (
+  id INTEGER PRIMARY KEY,
+  ep_num INTEGER UNIQUE,
+  guid TEXT UNIQUE NOT NULL,
+  title TEXT,
+  pub_date TEXT,
+  audio_url TEXT,
+  page_link TEXT,
+  duration_sec REAL,
+  model TEXT,
+  n_segments INTEGER,
+  tags TEXT
+);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS segments USING fts5(
+  text,
+  tags,
+  episode_id UNINDEXED,
+  seg_idx UNINDEXED,
+  start_sec UNINDEXED,
+  end_sec UNINDEXED
+);
+"""
+
+
+def open_db(path=DB_PATH) -> sqlite3.Connection:
+    con = sqlite3.connect(path)
+    con.executescript(SCHEMA)
+    return con
+
+
+def index_transcripts() -> None:
+    from transcribe import transcript_path
+    lexicon = build_lexicon()
+    eps = load_episodes()
+    con = open_db()
+    cur = con.cursor()
+
+    n_eps = n_segs = 0
+    for ep in eps:
+        tp = transcript_path(ep)
+        if not os.path.exists(tp):
+            continue
+        with open(tp, encoding="utf-8") as f:
+            data = json.load(f)
+
+        ep_tags = set()
+        for seg in data["segments"]:
+            ep_tags.update(tags_for_text(seg["text"], lexicon))
+
+        # upsert episode row
+        cur.execute(
+            """INSERT INTO episodes (ep_num, guid, title, pub_date, audio_url,
+               page_link, duration_sec, model, n_segments, tags)
+               VALUES (?,?,?,?,?,?,?,?,?,?)
+               ON CONFLICT(guid) DO UPDATE SET
+                 ep_num=excluded.ep_num, title=excluded.title,
+                 pub_date=excluded.pub_date, audio_url=excluded.audio_url,
+                 page_link=excluded.page_link, duration_sec=excluded.duration_sec,
+                 model=excluded.model, n_segments=excluded.n_segments,
+                 tags=excluded.tags""",
+            (data.get("ep_num"), data["guid"], data.get("title"),
+             data.get("pub_date"), data.get("audio_url"), data.get("page_link"),
+             data.get("duration_sec"), data.get("model"),
+             len(data["segments"]), json.dumps(sorted(ep_tags))),
+        )
+        row_id = cur.execute("SELECT id FROM episodes WHERE guid=?",
+                             (data["guid"],)).fetchone()[0]
+
+        # rebuild this episode's segment rows (simplest correct incremental path)
+        cur.execute("DELETE FROM segments WHERE episode_id=?", (row_id,))
+        cur.executemany(
+            "INSERT INTO segments (text, tags, episode_id, seg_idx, start_sec,"
+            " end_sec) VALUES (?,?,?,?,?,?)",
+            [(seg["text"],
+              " ".join("#" + t for t in tags_for_text(seg["text"], lexicon)),
+              row_id, i, seg["start"], seg["end"])
+             for i, seg in enumerate(data["segments"])],
+        )
+        n_eps += 1
+        n_segs += len(data["segments"])
+        label = f"Ep {data.get('ep_num')}" if data.get("ep_num") else data.get("title", "")[:40]
+        print(f"  indexed {label}: {len(data['segments'])} segs, "
+              f"tags: {' '.join('#' + t for t in sorted(ep_tags)) or '(none)'}")
+
+    con.commit()
+    con.close()
+    print(f"indexed {n_eps} episodes / {n_segs} segments -> {DB_PATH}")
+
+
+if __name__ == "__main__":
+    index_transcripts()