n3wk
116 lines
3.6 KiB
Python
116 lines
3.6 KiB
Python
"""Build the SQLite FTS5 index from transcripts + lexicon hashtags.
|
|
|
|
Schema:
|
|
episodes(ep_num, guid, title, pub_date, audio_url, page_link, duration_sec,
|
|
model, n_segments, tags_json)
|
|
segments(ep_rowid -> episodes.rowid, seg_idx, start_sec, end_sec, text,
|
|
tags) [FTS5 external content]
|
|
|
|
Search examples (via search.py):
|
|
python search.py "isom lawsuit"
|
|
python search.py maddox --tag lolsuit --from 2018-01-01 --to 2018-12-31
|
|
"""
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
|
|
from config import DB_PATH
|
|
from feed import load_episodes
|
|
from lexicon import build_lexicon, tags_for_text
|
|
|
|
SCHEMA = """
|
|
PRAGMA journal_mode=WAL;
|
|
|
|
CREATE TABLE IF NOT EXISTS episodes (
|
|
id INTEGER PRIMARY KEY,
|
|
ep_num INTEGER UNIQUE,
|
|
guid TEXT UNIQUE NOT NULL,
|
|
title TEXT,
|
|
pub_date TEXT,
|
|
audio_url TEXT,
|
|
page_link TEXT,
|
|
duration_sec REAL,
|
|
model TEXT,
|
|
n_segments INTEGER,
|
|
tags TEXT
|
|
);
|
|
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS segments USING fts5(
|
|
text,
|
|
tags,
|
|
episode_id UNINDEXED,
|
|
seg_idx UNINDEXED,
|
|
start_sec UNINDEXED,
|
|
end_sec UNINDEXED
|
|
);
|
|
"""
|
|
|
|
|
|
def open_db(path=DB_PATH) -> sqlite3.Connection:
|
|
con = sqlite3.connect(path)
|
|
con.executescript(SCHEMA)
|
|
return con
|
|
|
|
|
|
def index_transcripts() -> None:
|
|
from transcribe import transcript_path
|
|
lexicon = build_lexicon()
|
|
eps = load_episodes()
|
|
con = open_db()
|
|
cur = con.cursor()
|
|
|
|
n_eps = n_segs = 0
|
|
for ep in eps:
|
|
tp = transcript_path(ep)
|
|
if not os.path.exists(tp):
|
|
continue
|
|
with open(tp, encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
|
|
ep_tags = set()
|
|
for seg in data["segments"]:
|
|
ep_tags.update(tags_for_text(seg["text"], lexicon))
|
|
|
|
# upsert episode row
|
|
cur.execute(
|
|
"""INSERT INTO episodes (ep_num, guid, title, pub_date, audio_url,
|
|
page_link, duration_sec, model, n_segments, tags)
|
|
VALUES (?,?,?,?,?,?,?,?,?,?)
|
|
ON CONFLICT(guid) DO UPDATE SET
|
|
ep_num=excluded.ep_num, title=excluded.title,
|
|
pub_date=excluded.pub_date, audio_url=excluded.audio_url,
|
|
page_link=excluded.page_link, duration_sec=excluded.duration_sec,
|
|
model=excluded.model, n_segments=excluded.n_segments,
|
|
tags=excluded.tags""",
|
|
(data.get("ep_num"), data["guid"], data.get("title"),
|
|
data.get("pub_date"), data.get("audio_url"), data.get("page_link"),
|
|
data.get("duration_sec"), data.get("model"),
|
|
len(data["segments"]), json.dumps(sorted(ep_tags))),
|
|
)
|
|
row_id = cur.execute("SELECT id FROM episodes WHERE guid=?",
|
|
(data["guid"],)).fetchone()[0]
|
|
|
|
# rebuild this episode's segment rows (simplest correct incremental path)
|
|
cur.execute("DELETE FROM segments WHERE episode_id=?", (row_id,))
|
|
cur.executemany(
|
|
"INSERT INTO segments (text, tags, episode_id, seg_idx, start_sec,"
|
|
" end_sec) VALUES (?,?,?,?,?,?)",
|
|
[(seg["text"],
|
|
" ".join("#" + t for t in tags_for_text(seg["text"], lexicon)),
|
|
row_id, i, seg["start"], seg["end"])
|
|
for i, seg in enumerate(data["segments"])],
|
|
)
|
|
n_eps += 1
|
|
n_segs += len(data["segments"])
|
|
label = f"Ep {data.get('ep_num')}" if data.get("ep_num") else data.get("title", "")[:40]
|
|
print(f" indexed {label}: {len(data['segments'])} segs, "
|
|
f"tags: {' '.join('#' + t for t in sorted(ep_tags)) or '(none)'}")
|
|
|
|
con.commit()
|
|
con.close()
|
|
print(f"indexed {n_eps} episodes / {n_segs} segments -> {DB_PATH}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
index_transcripts()
|