Initial commit: Dick Show Index pipeline with install script and docs
This commit is contained in:
n3wk
+297
@@ -0,0 +1,297 @@
|
||||
#!/usr/bin/env bash
|
||||
# Dick Show Index - Installer
|
||||
# Installs all dependencies and sets up the pipeline for the Dick Show transcript index.
|
||||
# Usage: ./install.sh [--skip-opencode] [--skip-python-deps]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log_info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
log_success() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $*"; }
|
||||
|
||||
# Configuration
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PYTHON_MIN_VERSION="3.10"
|
||||
OPENCODE_BIN="${OPENCODE_BIN:-$HOME/.opencode/bin/opencode}"
|
||||
|
||||
# Flags
|
||||
SKIP_OPENCODE=false
|
||||
SKIP_PYTHON_DEPS=false
|
||||
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
--skip-opencode) SKIP_OPENCODE=true ;;
|
||||
--skip-python-deps) SKIP_PYTHON_DEPS=false ;;
|
||||
--help|-h)
|
||||
cat <<EOF
|
||||
Dick Show Index - Installer
|
||||
|
||||
Usage: ./install.sh [OPTIONS]
|
||||
|
||||
OPTIONS:
|
||||
--skip-opencode Skip opencode CLI installation
|
||||
--skip-python-deps Skip Python dependency installation
|
||||
--help, -h Show this help
|
||||
|
||||
Environment variables:
|
||||
OPENCODE_BIN Path to opencode binary (default: ~/.opencode/bin/opencode)
|
||||
OPENCODE_ANALYZE_MODEL Model to use for analysis (default: opencode/nemotron-3-ultra-free)
|
||||
DS_ANALYZE_PAUSE Seconds between analysis calls (default: 45)
|
||||
DS_ANALYZE_TIMEOUT Analysis timeout in seconds (default: 600)
|
||||
DS_MIN_FREE_RAM Minimum free RAM in bytes before pausing (default: 1GB)
|
||||
|
||||
Examples:
|
||||
./install.sh # Full install
|
||||
./install.sh --skip-opencode # Skip opencode install
|
||||
OPENCODE_ANALYZE_MODEL=opencode/x-preview-f-free ./install.sh
|
||||
EOF
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ============================================================================
|
||||
# Helper functions
|
||||
# ============================================================================
|
||||
|
||||
check_python() {
|
||||
log_info "Checking Python version..."
|
||||
if ! command -v python3 &> /dev/null; then
|
||||
log_error "Python 3 not found. Please install Python ${PYTHON_MIN_VERSION}+"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local version=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
|
||||
local major=$(echo $version | cut -d. -f1)
|
||||
local minor=$(echo $version | cut -d. -f2)
|
||||
|
||||
if [[ $major -lt 3 ]] || [[ $major -eq 3 && $minor -lt 10 ]]; then
|
||||
log_error "Python ${PYTHON_MIN_VERSION}+ required, found ${version}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_success "Python ${version} found"
|
||||
}
|
||||
|
||||
install_python_deps() {
|
||||
log_info "Installing Python dependencies..."
|
||||
if [[ -f "requirements.txt" ]]; then
|
||||
pip3 install --upgrade pip
|
||||
pip3 install -r requirements.txt
|
||||
log_success "Python dependencies installed"
|
||||
else
|
||||
log_warn "requirements.txt not found, skipping"
|
||||
fi
|
||||
}
|
||||
|
||||
install_opencode() {
|
||||
if [[ "$SKIP_OPENCODE" == "true" ]]; then
|
||||
log_info "Skipping opencode installation (--skip-opencode)"
|
||||
return
|
||||
fi
|
||||
|
||||
log_info "Installing opencode CLI..."
|
||||
|
||||
if command -v opencode &> /dev/null; then
|
||||
log_success "opencode already installed: $(opencode --version)"
|
||||
return
|
||||
fi
|
||||
|
||||
# Try official installer first
|
||||
if command -v curl &> /dev/null; then
|
||||
log_info "Downloading opencode installer..."
|
||||
if curl -fsSL https://opencode.ai/install.sh | bash; then
|
||||
log_success "opencode installed via official installer"
|
||||
return
|
||||
fi
|
||||
log_warn "Official installer failed, trying direct download..."
|
||||
fi
|
||||
|
||||
# Fallback: direct binary download from GitHub releases
|
||||
log_info "Downloading opencode binary from GitHub..."
|
||||
local arch=$(uname -m)
|
||||
local os=$(uname -s | tr '[:upper:]' '[:lower:]')
|
||||
|
||||
# Normalize arch
|
||||
case $arch in
|
||||
x86_64|amd64) arch="x64" ;;
|
||||
aarch64|arm64) arch="arm64" ;;
|
||||
esac
|
||||
|
||||
# Get latest release
|
||||
local release_url="https://api.github.com/repos/anomalyco/opencode/releases/latest"
|
||||
local download_url=$(curl -fsSL "$release_url" | grep "browser_download_url" | grep "${os}-${arch}" | head -1 | cut -d '"' -f 4)
|
||||
|
||||
if [[ -z "$download_url" ]]; then
|
||||
log_warn "Could not find binary for ${os}-${arch}, trying windows-x64.zip..."
|
||||
download_url=$(curl -fsSL "$release_url" | grep "browser_download_url" | grep "windows-x64.zip" | head -1 | cut -d '"' -f 4)
|
||||
fi
|
||||
|
||||
if [[ -z "$download_url" ]]; then
|
||||
log_error "Could not find opencode binary for your platform"
|
||||
log_info "Please install manually from: https://github.com/anomalyco/opencode/releases"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "Downloading from: $download_url"
|
||||
mkdir -p "$(dirname "$OPENCODE_BIN")"
|
||||
curl -fsSL "$download_url" -o /tmp/opencode.zip
|
||||
|
||||
if [[ "$download_url" == *.zip ]]; then
|
||||
unzip -o /tmp/opencode.zip -d "$(dirname "$OPENCODE_BIN")"
|
||||
else
|
||||
mv /tmp/opencode.zip "$OPENCODE_BIN"
|
||||
fi
|
||||
|
||||
chmod +x "$OPENCODE_BIN"
|
||||
|
||||
# Add to PATH if not already
|
||||
local bin_dir=$(dirname "$OPENCODE_BIN")
|
||||
if [[ ":$PATH:" != *":$bin_dir:"* ]]; then
|
||||
echo "export PATH=\"\$PATH:$bin_dir\"" >> ~/.bashrc
|
||||
log_info "Added $bin_dir to PATH in ~/.bashrc"
|
||||
fi
|
||||
|
||||
export PATH="$PATH:$bin_dir"
|
||||
log_success "opencode installed: $(opencode --version)"
|
||||
}
|
||||
|
||||
verify_opencode() {
|
||||
log_info "Verifying opencode installation..."
|
||||
if ! command -v opencode &> /dev/null; then
|
||||
log_error "opencode not found in PATH"
|
||||
log_info "Add $(dirname "$OPENCODE_BIN") to your PATH and restart your shell"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_success "opencode: $(opencode --version)"
|
||||
|
||||
# Check available models
|
||||
log_info "Available models:"
|
||||
opencode models 2>/dev/null | head -20 || log_warn "Could not list models (may need auth)"
|
||||
}
|
||||
|
||||
check_existing_work() {
|
||||
log_info "Checking for existing work..."
|
||||
|
||||
local transcript_count=0
|
||||
local analysis_count=0
|
||||
|
||||
if [[ -d "transcripts" ]]; then
|
||||
transcript_count=$(find transcripts -name "ep*.json" 2>/dev/null | wc -l)
|
||||
fi
|
||||
|
||||
if [[ -d "data/analysis" ]]; then
|
||||
analysis_count=$(find data/analysis -name "ep*.json" 2>/dev/null | wc -l)
|
||||
fi
|
||||
|
||||
log_info "Found $transcript_count existing transcripts"
|
||||
log_info "Found $analysis_count existing analyses"
|
||||
|
||||
if [[ $transcript_count -gt 0 || $analysis_count -gt 0 ]]; then
|
||||
log_info "Pipeline will resume from existing work (skips completed episodes)"
|
||||
fi
|
||||
}
|
||||
|
||||
create_dirs() {
|
||||
log_info "Creating directory structure..."
|
||||
mkdir -p audio transcripts data/analysis
|
||||
log_success "Directories created"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Main
|
||||
# ============================================================================
|
||||
|
||||
main() {
|
||||
echo "=========================================="
|
||||
echo " Dick Show Index - Installer"
|
||||
echo "=========================================="
|
||||
echo
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
log_info "Starting installation..."
|
||||
|
||||
check_python
|
||||
create_dirs
|
||||
|
||||
if [[ "$SKIP_PYTHON_DEPS" != "true" ]]; then
|
||||
install_python_deps
|
||||
fi
|
||||
|
||||
install_opencode
|
||||
verify_opencode
|
||||
check_existing_work
|
||||
|
||||
echo
|
||||
log_success "Installation complete!"
|
||||
echo
|
||||
cat <<EOF
|
||||
|
||||
==========================================
|
||||
Next Steps
|
||||
==========================================
|
||||
|
||||
1. (Optional) Configure opencode authentication:
|
||||
opencode auth login
|
||||
|
||||
2. Set analysis model (optional):
|
||||
export OPENCODE_ANALYZE_MODEL=opencode/nemotron-3-ultra-free
|
||||
# Or use your preferred model: opencode models
|
||||
|
||||
3. Run the pipeline:
|
||||
# Full Rekieta arc (28 episodes):
|
||||
python pipeline_all.py --eps 84 87 89 93 96 121 124 152 166 176 198 209 213 275 285 299 321 344 352 368 376 412 416 424 425 435 439 455
|
||||
|
||||
# Full backlog (all 520 episodes):
|
||||
python pipeline_all.py
|
||||
|
||||
# Dry run to see what would be processed:
|
||||
python pipeline_all.py --dry-run
|
||||
|
||||
4. Search the index:
|
||||
python search.py "isom trademark"
|
||||
python search.py maddox --tag lolsuit
|
||||
python search.py "raketa OR rekieta" --tag nick_rekieta
|
||||
|
||||
==========================================
|
||||
Environment Variables
|
||||
==========================================
|
||||
|
||||
OPENCODE_ANALYZE_MODEL Model for analysis (default: opencode/nemotron-3-ultra-free)
|
||||
DS_ANALYZE_PAUSE Seconds between analysis calls (default: 45)
|
||||
DS_ANALYZE_TIMEOUT Analysis timeout seconds (default: 600)
|
||||
DS_MIN_FREE_RAM Min free RAM before pausing (default: 1073741824 = 1GB)
|
||||
DS_MODEL Whisper model (default: small)
|
||||
DS_DEVICE Whisper device: cpu|cuda (default: cpu)
|
||||
DS_COMPUTE Whisper compute type: int8|float16 (default: int8)
|
||||
DS_KEEP_AUDIO Keep MP3s after transcription (default: 0)
|
||||
DS_KEEP_AUDIO=1 to keep
|
||||
|
||||
==========================================
|
||||
Resuming / Sharing Work
|
||||
==========================================
|
||||
|
||||
The pipeline is fully resumable - it automatically skips episodes that
|
||||
already have transcripts and verified analyses. Just re-run the same
|
||||
command and it will pick up where it left off.
|
||||
|
||||
To share progress with others:
|
||||
1. Commit the transcripts/, data/analysis/, and data/index.db files
|
||||
2. Others can pull and continue from where you left off
|
||||
|
||||
==========================================
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user