Add encoding validation, fix scripts, and encoding docs

README.md

@@ -92,6 +92,34 @@ Set via environment variables or edit `config.py`:
 | `DS_DEVICE` | `cpu` | `cpu` or `cuda` |
 | `DS_COMPUTE` | `int8` | `int8`, `float16`, `float32` |
 | `DS_KEEP_AUDIO` | `0` | Keep MP3s after transcription (`1` to keep) |
+| `DS_ENCODING_CHECK` | `50` | Episodes between UTF-8 verification |
+
+## Encoding & Character Handling
+
+All JSON files are written as **UTF-8** with `ensure_ascii=False`. The RSS feed contains Unicode characters (em/en dashes, accented characters) which are preserved correctly in the JSON files.
+
+**If you see mojibake like `â€"` or `é`:**
+- The JSON files are correctly stored as UTF-8
+- The issue is the *consumer* (text editor, web browser, terminal) reading the file with wrong encoding (Latin-1/Windows-1252)
+- **Fix**: Ensure your tools/editor/browser use UTF-8 encoding
+
+**Verification & Repair:**
+```bash
+# Verify all JSON files are valid UTF-8
+python verify_encoding.py
+
+# Fix common mojibake patterns (run if data was corrupted)
+python fix_mojibake.py --dry-run   # preview fixes
+python fix_mojibake.py              # apply fixes
+
+# Check progress including encoding status
+python check_progress.py
+```
+
+**For web apps displaying this data:**
+- Set `Content-Type: application/json; charset=utf-8`
+- Include `<meta charset="utf-8">` in HTML
+- Ensure database connections use UTF-8
 
 ## Resuming / Sharing Work
 

fix_encoding.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""Verify and repair UTF-8 encoding in all JSON files."""
+
+import json
+import sys
+from pathlib import Path
+
+def check_file(path: Path) -> tuple[int, int]:
+    """Check a JSON file for encoding issues. Returns (errors_fixed, total_chars_checked)."""
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except UnicodeDecodeError as e:
+        print(f"  [ERROR] {path}: Not valid UTF-8 at position {e.start}: {e}")
+        return (0, 0)
+    
+    try:
+        data = json.loads(content)
+    except json.JSONDecodeError as e:
+        print(f"  [ERROR] {path}: Invalid JSON at position {e.pos}: {e}")
+        return (0, 0)
+    
+    return (0, len(content))
+
+def check_all_json_files(root: Path) -> dict:
+    """Scan all JSON files for encoding issues."""
+    stats = {"files_checked": 0, "errors": 0, "total_bytes": 0}
+    
+    for path in root.rglob("*.json"):
+        if any(part.startswith('.') for part in path.parts):
+            continue
+        if path.name.startswith('.'):
+            continue
+            
+        errors, bytes_checked = check_file(path)
+        stats["files_checked"] += 1
+        stats["errors"] += errors
+        stats["total_bytes"] += bytes_checked
+        
+        if stats["files_checked"] % 50 == 0:
+            print(f"  Checked {stats['files_checked']} files...")
+    
+    return stats
+
+def fix_corrupted_unicode(text: str) -> str:
+    """Fix common UTF-8 mojibake patterns."""
+    # Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252)
+    fixes = {
+        'â€"': '—',    # em dash
+        'â€': '—',     # em dash (truncated)
+        '’': "'",    # right single quote
+        '“': '"',    # left double quote
+        'â€': '"',     # right double quote
+        '•': '•',    # bullet
+        '…': '…',    # ellipsis
+        'é': 'é',     # é
+        'è': 'è',     # è
+        'á': 'á',     # á
+        'í': 'í',     # í
+        'ó': 'ó',     # ó
+        'ú': 'ú',     # ú
+        'ñ': 'ñ',     # ñ
+        'ü': 'ü',     # ü
+        'ö': 'ö',     # ö
+        'ä': 'ä',     # ä
+        'Â': '',       # stray  (common before accented chars)
+    }
+    
+    result = text
+    for bad, good in fixes.items():
+        result = result.replace(bad, good)
+    return result
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Verify and repair UTF-8 encoding in JSON files")
+    parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
+    parser.add_argument('--fix', action='store_true', help='Attempt to fix mojibake')
+    parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed')
+    args = parser.parse_args()
+    
+    root = Path(args.path).resolve()
+    print(f"Scanning {root} for JSON files...")
+    
+    stats = check_all_json_files(Path('.'))
+    
+    print(f"\nResults:")
+    print(f"  Files checked: {stats['files_checked']}")
+    print(f"  Total bytes: {stats['total_bytes']:,}")
+    print(f"  Encoding errors: {stats['errors']}")
+    
+    if stats['errors'] == 0:
+        print("All JSON files are valid UTF-8!")
+    else:
+        print(f"Found {stats['errors']} files with encoding issues")
+
+if __name__ == '__main__':
+    main()

fix_mojibake.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Fix UTF-8 mojibake in JSON files."""
+
+import json
+import sys
+from pathlib import Path
+
+# Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252)
+MOJIBAKE_FIXES = {
+    'â€"': '—',    # em dash
+    'â€': '—',     # em dash (truncated)
+    '’': "'",    # right single quote
+    '“': '"',    # left double quote
+    'â€': '"',     # right double quote
+    '•': '•',    # bullet
+    '…': '…',    # ellipsis
+    'é': 'é',     # é
+    'è': 'è',     # è
+    'á': 'á',     # á
+    'í': 'í',     # í
+    'ó': 'ó',     # ó
+    'ú': 'ú',     # ú
+    'ñ': 'ñ',     # ñ
+    'ü': 'ü',     # ü
+    'ö': 'ö',     # ö
+    'ä': 'ä',     # ä
+    'ü': 'ü',     # ü
+    'ö': 'ö',     # ö
+    'ä': 'ä',     # ä
+    'Â': '',       # stray  (common before accented chars)
+}
+
+def fix_mojibake(text: str) -> tuple[str, int]:
+    """Fix common UTF-8 mojibake patterns. Returns (fixed_text, num_fixes)."""
+    count = 0
+    result = text
+    for bad, good in MOJIBAKE_FIXES.items():
+        before = result.count(bad)
+        if before:
+            result = result.replace(bad, good)
+            count += before
+    return result, count
+
+def fix_json_file(path: Path, dry_run: bool = False) -> int:
+    """Fix mojibake in a JSON file. Returns number of fixes applied."""
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except UnicodeDecodeError as e:
+        print(f"  [ERROR] {path}: Not valid UTF-8 at position {e.start}")
+        return 0
+    
+    try:
+        data = json.loads(content)
+    except json.JSONDecodeError as e:
+        print(f"  [ERROR] {path}: Invalid JSON at position {e.pos}")
+        return 0
+    
+    # Recursively fix strings in JSON
+    def fix_obj(obj):
+        fixes = 0
+        if isinstance(obj, str):
+            fixed, count = fix_mojibake(obj)
+            return fixed, count
+        elif isinstance(obj, dict):
+            new_dict = {}
+            for k, v in obj.items():
+                fixed_k, c1 = fix_obj(k)
+                fixed_v, c2 = fix_obj(v)
+                fixes += c1 + c2
+                new_dict[fixed_k] = fixed_v
+            return new_dict, fixes
+        elif isinstance(obj, list):
+            new_list = []
+            for item in obj:
+                fixed, count = fix_obj(item)
+                fixes += count
+                new_list.append(fixed)
+            return new_list, fixes
+        else:
+            return obj, 0
+    
+    fixed_data, fix_count = fix_obj(json.loads(json.dumps(data)))  # Round-trip to ensure clean JSON
+    
+    if fix_count > 0 and not dry_run:
+        with open(path, 'w', encoding='utf-8') as f:
+            json.dump(fixed_data, f, ensure_ascii=False, indent=1)
+        print(f"  Fixed {fix_count} mojibake in {path}")
+    
+    return fix_count
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Fix UTF-8 mojibake in JSON files")
+    parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
+    parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed')
+    args = parser.parse_args()
+    
+    root = Path('.').resolve()
+    print(f"Scanning {Path('.').resolve()} for JSON files...")
+    
+    total_fixes = 0
+    files_fixed = 0
+    
+    for path in Path('.').rglob("*.json"):
+        if any(part.startswith('.') for part in path.parts):
+            continue
+        if path.name.startswith('.'):
+            continue
+            
+        fixes = fix_json_file(path, dry_run=args.dry_run)
+        if fixes > 0:
+            files_fixed += 1
+            print(f"  Fixed {fixes} issues in {path}")
+    
+    print(f"\nDone. Fixed {files_fixed} files.")
+
+if __name__ == '__main__':
+    main()

pipeline_all.py

@@ -40,6 +40,36 @@ ANALYSIS_TIMEOUT = int(os.environ.get("DS_ANALYSIS_TIMEOUT", "600"))  # 10 minut
 # memory guard threshold (bytes) - pause if available RAM drops below this
 MIN_FREE_RAM = int(os.environ.get("DS_MIN_FREE_RAM", str(1024 * 1024 * 1024)))  # 1 GB
 
+# encoding validation interval (episodes)
+ENCODING_CHECK_INTERVAL = int(os.environ.get("DS_ENCODING_CHECK", "50"))
+
+
+def has_partial(transcript_file: str) -> bool:
+    try:
+        with open(transcript_file, encoding="utf-8") as f:
+            return json.load(f).get("partial_minutes") is not None
+    except (OSError, json.JSONDecodeError):
+        return False
+
+
+def verify_utf8_encoding(root: Path = Path('.')) -> tuple[int, int]:
+    """Verify all JSON files are valid UTF-8. Returns (errors, files_checked)."""
+    errors = 0
+    checked = 0
+    for path in Path('.').rglob("*.json"):
+        if any(part.startswith('.') for part in path.parts):
+            continue
+        if path.name.startswith('.'):
+            continue
+        try:
+            with open(path, 'r', encoding='utf-8') as f:
+                json.load(path)
+        except (UnicodeDecodeError, json.JSONDecodeError) as e:
+            print(f"  [ENCODING ERROR] {path}: {e}", flush=True)
+            errors += 1
+        checked += 1
+    return errors, checked
+
 
 def has_partial(transcript_file: str) -> bool:
     try:
@@ -293,6 +323,16 @@ def main() -> None:
             index_mod.index_transcripts()
             from aggregate import aggregate as _agg
             _agg()
+            
+            # Periodic encoding verification
+            if i % ENCODING_CHECK_INTERVAL == 0:
+                print(f"  [{time.strftime('%H:%M:%S')}] verifying UTF-8 encoding...", flush=True)
+                errors, checked = verify_utf8_encoding()
+                if errors:
+                    print(f"  [WARNING] Found {errors} files with encoding issues!", flush=True)
+                else:
+                    print(f"  [OK] UTF-8 encoding verified for {checked} files", flush=True)
+            
             hrs = (time.time() - t_start) / 3600
             rate_eps = i / hrs
             remaining = (len(eps) - i) / max(rate_eps, 0.01)

verify_encoding.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Verify UTF-8 encoding in all JSON files."""
+
+import json
+import sys
+from pathlib import Path
+
+def verify_utf8(path: Path) -> tuple[int, int]:
+    """Check a JSON file for valid UTF-8. Returns (errors, bytes_checked)."""
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except UnicodeDecodeError as e:
+        print(f"  [ERROR] {path}: Invalid UTF-8 at position {e.start}")
+        return (1, 0)
+    
+    try:
+        json.loads(content)
+    except json.JSONDecodeError as e:
+        print(f"  [ERROR] {path}: Invalid JSON at position {e.pos}: {e}")
+        return (1, 0)
+    
+    return (0, len(content))
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files")
+    parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
+    args = parser.parse_args()
+    
+    root = Path('.').resolve()
+    print(f"Scanning {root} for JSON files...")
+    
+    stats = {"files": 0, "errors": 0, "bytes": 0}
+    
+    for path in Path('.').rglob("*.json"):
+        if any(part.startswith('.') for part in path.parts):
+            continue
+        if path.name.startswith('.'):
+            continue
+            
+        errors, bytes_checked = verify_utf8(path)
+        stats["files_checked"] = stats.get("files_checked", 0) + 1
+        stats["errors"] += errors
+        stats["total_bytes"] = stats.get("total_bytes", 0) + bytes_checked
+        
+        if stats["files_checked"] % 50 == 0:
+            print(f"  Checked {stats['files_checked']} files...")
+    
+    print(f"\nResults:")
+    print(f"  Files checked: {stats['files_checked']}")
+    print(f"  Total bytes: {stats['total_bytes']:,}")
+    print(f"  Encoding errors: {stats['errors']}")
+    
+    if stats['errors'] == 0:
+        print("All JSON files are valid UTF-8!")
+        return 0
+    else:
+        print(f"Found {stats['errors']} files with encoding issues")
+        return 1
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files")
+    parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
+    args = parser.parse_args()
+    sys.exit(main(args.path))