Add encoding validation, fix scripts, and encoding docs

verify_encoding.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Verify UTF-8 encoding in all JSON files."""
+
+import json
+import sys
+from pathlib import Path
+
+def verify_utf8(path: Path) -> tuple[int, int]:
+    """Check a JSON file for valid UTF-8. Returns (errors, bytes_checked)."""
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except UnicodeDecodeError as e:
+        print(f"  [ERROR] {path}: Invalid UTF-8 at position {e.start}")
+        return (1, 0)
+    
+    try:
+        json.loads(content)
+    except json.JSONDecodeError as e:
+        print(f"  [ERROR] {path}: Invalid JSON at position {e.pos}: {e}")
+        return (1, 0)
+    
+    return (0, len(content))
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files")
+    parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
+    args = parser.parse_args()
+    
+    root = Path('.').resolve()
+    print(f"Scanning {root} for JSON files...")
+    
+    stats = {"files": 0, "errors": 0, "bytes": 0}
+    
+    for path in Path('.').rglob("*.json"):
+        if any(part.startswith('.') for part in path.parts):
+            continue
+        if path.name.startswith('.'):
+            continue
+            
+        errors, bytes_checked = verify_utf8(path)
+        stats["files_checked"] = stats.get("files_checked", 0) + 1
+        stats["errors"] += errors
+        stats["total_bytes"] = stats.get("total_bytes", 0) + bytes_checked
+        
+        if stats["files_checked"] % 50 == 0:
+            print(f"  Checked {stats['files_checked']} files...")
+    
+    print(f"\nResults:")
+    print(f"  Files checked: {stats['files_checked']}")
+    print(f"  Total bytes: {stats['total_bytes']:,}")
+    print(f"  Encoding errors: {stats['errors']}")
+    
+    if stats['errors'] == 0:
+        print("All JSON files are valid UTF-8!")
+        return 0
+    else:
+        print(f"Found {stats['errors']} files with encoding issues")
+        return 1
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files")
+    parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
+    args = parser.parse_args()
+    sys.exit(main(args.path))