Add encoding validation, fix scripts, and encoding docs

pipeline_all.py

@@ -40,6 +40,36 @@ ANALYSIS_TIMEOUT = int(os.environ.get("DS_ANALYSIS_TIMEOUT", "600"))  # 10 minut
 # memory guard threshold (bytes) - pause if available RAM drops below this
 MIN_FREE_RAM = int(os.environ.get("DS_MIN_FREE_RAM", str(1024 * 1024 * 1024)))  # 1 GB
 
+# encoding validation interval (episodes)
+ENCODING_CHECK_INTERVAL = int(os.environ.get("DS_ENCODING_CHECK", "50"))
+
+
+def has_partial(transcript_file: str) -> bool:
+    try:
+        with open(transcript_file, encoding="utf-8") as f:
+            return json.load(f).get("partial_minutes") is not None
+    except (OSError, json.JSONDecodeError):
+        return False
+
+
+def verify_utf8_encoding(root: Path = Path('.')) -> tuple[int, int]:
+    """Verify all JSON files are valid UTF-8. Returns (errors, files_checked)."""
+    errors = 0
+    checked = 0
+    for path in Path('.').rglob("*.json"):
+        if any(part.startswith('.') for part in path.parts):
+            continue
+        if path.name.startswith('.'):
+            continue
+        try:
+            with open(path, 'r', encoding='utf-8') as f:
+                json.load(path)
+        except (UnicodeDecodeError, json.JSONDecodeError) as e:
+            print(f"  [ENCODING ERROR] {path}: {e}", flush=True)
+            errors += 1
+        checked += 1
+    return errors, checked
+
 
 def has_partial(transcript_file: str) -> bool:
     try:
@@ -293,6 +323,16 @@ def main() -> None:
             index_mod.index_transcripts()
             from aggregate import aggregate as _agg
             _agg()
+            
+            # Periodic encoding verification
+            if i % ENCODING_CHECK_INTERVAL == 0:
+                print(f"  [{time.strftime('%H:%M:%S')}] verifying UTF-8 encoding...", flush=True)
+                errors, checked = verify_utf8_encoding()
+                if errors:
+                    print(f"  [WARNING] Found {errors} files with encoding issues!", flush=True)
+                else:
+                    print(f"  [OK] UTF-8 encoding verified for {checked} files", flush=True)
+            
             hrs = (time.time() - t_start) / 3600
             rate_eps = i / hrs
             remaining = (len(eps) - i) / max(rate_eps, 0.01)