Add encoding validation, fix scripts, and encoding docs

fix_mojibake.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Fix UTF-8 mojibake in JSON files."""
+
+import json
+import sys
+from pathlib import Path
+
+# Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252)
+MOJIBAKE_FIXES = {
+    'â€"': '—',    # em dash
+    'â€': '—',     # em dash (truncated)
+    '’': "'",    # right single quote
+    '“': '"',    # left double quote
+    'â€': '"',     # right double quote
+    '•': '•',    # bullet
+    '…': '…',    # ellipsis
+    'é': 'é',     # é
+    'è': 'è',     # è
+    'á': 'á',     # á
+    'í': 'í',     # í
+    'ó': 'ó',     # ó
+    'ú': 'ú',     # ú
+    'ñ': 'ñ',     # ñ
+    'ü': 'ü',     # ü
+    'ö': 'ö',     # ö
+    'ä': 'ä',     # ä
+    'ü': 'ü',     # ü
+    'ö': 'ö',     # ö
+    'ä': 'ä',     # ä
+    'Â': '',       # stray  (common before accented chars)
+}
+
+def fix_mojibake(text: str) -> tuple[str, int]:
+    """Fix common UTF-8 mojibake patterns. Returns (fixed_text, num_fixes)."""
+    count = 0
+    result = text
+    for bad, good in MOJIBAKE_FIXES.items():
+        before = result.count(bad)
+        if before:
+            result = result.replace(bad, good)
+            count += before
+    return result, count
+
+def fix_json_file(path: Path, dry_run: bool = False) -> int:
+    """Fix mojibake in a JSON file. Returns number of fixes applied."""
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except UnicodeDecodeError as e:
+        print(f"  [ERROR] {path}: Not valid UTF-8 at position {e.start}")
+        return 0
+    
+    try:
+        data = json.loads(content)
+    except json.JSONDecodeError as e:
+        print(f"  [ERROR] {path}: Invalid JSON at position {e.pos}")
+        return 0
+    
+    # Recursively fix strings in JSON
+    def fix_obj(obj):
+        fixes = 0
+        if isinstance(obj, str):
+            fixed, count = fix_mojibake(obj)
+            return fixed, count
+        elif isinstance(obj, dict):
+            new_dict = {}
+            for k, v in obj.items():
+                fixed_k, c1 = fix_obj(k)
+                fixed_v, c2 = fix_obj(v)
+                fixes += c1 + c2
+                new_dict[fixed_k] = fixed_v
+            return new_dict, fixes
+        elif isinstance(obj, list):
+            new_list = []
+            for item in obj:
+                fixed, count = fix_obj(item)
+                fixes += count
+                new_list.append(fixed)
+            return new_list, fixes
+        else:
+            return obj, 0
+    
+    fixed_data, fix_count = fix_obj(json.loads(json.dumps(data)))  # Round-trip to ensure clean JSON
+    
+    if fix_count > 0 and not dry_run:
+        with open(path, 'w', encoding='utf-8') as f:
+            json.dump(fixed_data, f, ensure_ascii=False, indent=1)
+        print(f"  Fixed {fix_count} mojibake in {path}")
+    
+    return fix_count
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Fix UTF-8 mojibake in JSON files")
+    parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
+    parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed')
+    args = parser.parse_args()
+    
+    root = Path('.').resolve()
+    print(f"Scanning {Path('.').resolve()} for JSON files...")
+    
+    total_fixes = 0
+    files_fixed = 0
+    
+    for path in Path('.').rglob("*.json"):
+        if any(part.startswith('.') for part in path.parts):
+            continue
+        if path.name.startswith('.'):
+            continue
+            
+        fixes = fix_json_file(path, dry_run=args.dry_run)
+        if fixes > 0:
+            files_fixed += 1
+            print(f"  Fixed {fixes} issues in {path}")
+    
+    print(f"\nDone. Fixed {files_fixed} files.")
+
+if __name__ == '__main__':
+    main()