Initial commit: source code only

fix_encoding.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""Verify and repair UTF-8 encoding in all JSON files."""
+
+import json
+import sys
+from pathlib import Path
+
+def check_file(path: Path) -> tuple[int, int]:
+    """Check a JSON file for encoding issues. Returns (errors_fixed, total_chars_checked)."""
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except UnicodeDecodeError as e:
+        print(f"  [ERROR] {path}: Not valid UTF-8 at position {e.start}: {e}")
+        return (0, 0)
+    
+    try:
+        data = json.loads(content)
+    except json.JSONDecodeError as e:
+        print(f"  [ERROR] {path}: Invalid JSON at position {e.pos}: {e}")
+        return (0, 0)
+    
+    return (0, len(content))
+
+def check_all_json_files(root: Path) -> dict:
+    """Scan all JSON files for encoding issues."""
+    stats = {"files_checked": 0, "errors": 0, "total_bytes": 0}
+    
+    for path in root.rglob("*.json"):
+        if any(part.startswith('.') for part in path.parts):
+            continue
+        if path.name.startswith('.'):
+            continue
+            
+        errors, bytes_checked = check_file(path)
+        stats["files_checked"] += 1
+        stats["errors"] += errors
+        stats["total_bytes"] += bytes_checked
+        
+        if stats["files_checked"] % 50 == 0:
+            print(f"  Checked {stats['files_checked']} files...")
+    
+    return stats
+
+def fix_corrupted_unicode(text: str) -> str:
+    """Fix common UTF-8 mojibake patterns."""
+    # Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252)
+    fixes = {
+        'â€"': '—',    # em dash
+        'â€': '—',     # em dash (truncated)
+        '’': "'",    # right single quote
+        '“': '"',    # left double quote
+        'â€': '"',     # right double quote
+        '•': '•',    # bullet
+        '…': '…',    # ellipsis
+        'é': 'é',     # é
+        'è': 'è',     # è
+        'á': 'á',     # á
+        'í': 'í',     # í
+        'ó': 'ó',     # ó
+        'ú': 'ú',     # ú
+        'ñ': 'ñ',     # ñ
+        'ü': 'ü',     # ü
+        'ö': 'ö',     # ö
+        'ä': 'ä',     # ä
+        'Â': '',       # stray  (common before accented chars)
+    }
+    
+    result = text
+    for bad, good in fixes.items():
+        result = result.replace(bad, good)
+    return result
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Verify and repair UTF-8 encoding in JSON files")
+    parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
+    parser.add_argument('--fix', action='store_true', help='Attempt to fix mojibake')
+    parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed')
+    args = parser.parse_args()
+    
+    root = Path(args.path).resolve()
+    print(f"Scanning {root} for JSON files...")
+    
+    stats = check_all_json_files(Path('.'))
+    
+    print(f"\nResults:")
+    print(f"  Files checked: {stats['files_checked']}")
+    print(f"  Total bytes: {stats['total_bytes']:,}")
+    print(f"  Encoding errors: {stats['errors']}")
+    
+    if stats['errors'] == 0:
+        print("All JSON files are valid UTF-8!")
+    else:
+        print(f"Found {stats['errors']} files with encoding issues")
+
+if __name__ == '__main__':
+    main()