dickshow-index/index.py

"""Build the SQLite FTS5 index from transcripts + lexicon hashtags.

Schema:
  episodes(ep_num, guid, title, pub_date, audio_url, page_link, duration_sec,
           model, n_segments, tags_json)
  segments(ep_rowid -> episodes.rowid, seg_idx, start_sec, end_sec, text,
           tags)  [FTS5 external content]

Search examples (via search.py):
  python search.py "isom lawsuit"
  python search.py maddox --tag lolsuit --from 2018-01-01 --to 2018-12-31
"""
import json
import os
import sqlite3

from config import DB_PATH
from feed import load_episodes
from lexicon import build_lexicon, tags_for_text

SCHEMA = """
PRAGMA journal_mode=WAL;

CREATE TABLE IF NOT EXISTS episodes (
  id INTEGER PRIMARY KEY,
  ep_num INTEGER UNIQUE,
  guid TEXT UNIQUE NOT NULL,
  title TEXT,
  pub_date TEXT,
  audio_url TEXT,
  page_link TEXT,
  duration_sec REAL,
  model TEXT,
  n_segments INTEGER,
  tags TEXT
);

CREATE VIRTUAL TABLE IF NOT EXISTS segments USING fts5(
  text,
  tags,
  episode_id UNINDEXED,
  seg_idx UNINDEXED,
  start_sec UNINDEXED,
  end_sec UNINDEXED
);
"""


def open_db(path=DB_PATH) -> sqlite3.Connection:
    con = sqlite3.connect(path)
    con.executescript(SCHEMA)
    return con


def index_transcripts() -> None:
    from transcribe import transcript_path
    lexicon = build_lexicon()
    eps = load_episodes()
    con = open_db()
    cur = con.cursor()

    n_eps = n_segs = 0
    for ep in eps:
        tp = transcript_path(ep)
        if not os.path.exists(tp):
            continue
        with open(tp, encoding="utf-8") as f:
            data = json.load(f)

        ep_tags = set()
        for seg in data["segments"]:
            ep_tags.update(tags_for_text(seg["text"], lexicon))

        # upsert episode row
        cur.execute(
            """INSERT INTO episodes (ep_num, guid, title, pub_date, audio_url,
               page_link, duration_sec, model, n_segments, tags)
               VALUES (?,?,?,?,?,?,?,?,?,?)
               ON CONFLICT(guid) DO UPDATE SET
                 ep_num=excluded.ep_num, title=excluded.title,
                 pub_date=excluded.pub_date, audio_url=excluded.audio_url,
                 page_link=excluded.page_link, duration_sec=excluded.duration_sec,
                 model=excluded.model, n_segments=excluded.n_segments,
                 tags=excluded.tags""",
            (data.get("ep_num"), data["guid"], data.get("title"),
             data.get("pub_date"), data.get("audio_url"), data.get("page_link"),
             data.get("duration_sec"), data.get("model"),
             len(data["segments"]), json.dumps(sorted(ep_tags))),
        )
        row_id = cur.execute("SELECT id FROM episodes WHERE guid=?",
                             (data["guid"],)).fetchone()[0]

        # rebuild this episode's segment rows (simplest correct incremental path)
        cur.execute("DELETE FROM segments WHERE episode_id=?", (row_id,))
        cur.executemany(
            "INSERT INTO segments (text, tags, episode_id, seg_idx, start_sec,"
            " end_sec) VALUES (?,?,?,?,?,?)",
            [(seg["text"],
              " ".join("#" + t for t in tags_for_text(seg["text"], lexicon)),
              row_id, i, seg["start"], seg["end"])
             for i, seg in enumerate(data["segments"])],
        )
        n_eps += 1
        n_segs += len(data["segments"])
        label = f"Ep {data.get('ep_num')}" if data.get("ep_num") else data.get("title", "")[:40]
        print(f"  indexed {label}: {len(data['segments'])} segs, "
              f"tags: {' '.join('#' + t for t in sorted(ep_tags)) or '(none)'}")

    con.commit()
    con.close()
    print(f"indexed {n_eps} episodes / {n_segs} segments -> {DB_PATH}")


if __name__ == "__main__":
    index_transcripts()