#!/usr/bin/env python3 """Fix UTF-8 mojibake in JSON files.""" import json import sys from pathlib import Path # Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252) MOJIBAKE_FIXES = { 'â€"': '—', # em dash 'â€': '—', # em dash (truncated) '’': "'", # right single quote '“': '"', # left double quote 'â€': '"', # right double quote '•': '•', # bullet '…': '…', # ellipsis 'é': 'é', # é 'è': 'è', # è 'á': 'á', # á 'í': 'í', # í 'ó': 'ó', # ó 'ú': 'ú', # ú 'ñ': 'ñ', # ñ 'ü': 'ü', # ü 'ö': 'ö', # ö 'ä': 'ä', # ä 'ü': 'ü', # ü 'ö': 'ö', # ö 'ä': 'ä', # ä 'Â': '', # stray  (common before accented chars) } def fix_mojibake(text: str) -> tuple[str, int]: """Fix common UTF-8 mojibake patterns. Returns (fixed_text, num_fixes).""" count = 0 result = text for bad, good in MOJIBAKE_FIXES.items(): before = result.count(bad) if before: result = result.replace(bad, good) count += before return result, count def fix_json_file(path: Path, dry_run: bool = False) -> int: """Fix mojibake in a JSON file. Returns number of fixes applied.""" try: with open(path, 'r', encoding='utf-8') as f: content = f.read() except UnicodeDecodeError as e: print(f" [ERROR] {path}: Not valid UTF-8 at position {e.start}") return 0 try: data = json.loads(content) except json.JSONDecodeError as e: print(f" [ERROR] {path}: Invalid JSON at position {e.pos}") return 0 # Recursively fix strings in JSON def fix_obj(obj): fixes = 0 if isinstance(obj, str): fixed, count = fix_mojibake(obj) return fixed, count elif isinstance(obj, dict): new_dict = {} for k, v in obj.items(): fixed_k, c1 = fix_obj(k) fixed_v, c2 = fix_obj(v) fixes += c1 + c2 new_dict[fixed_k] = fixed_v return new_dict, fixes elif isinstance(obj, list): new_list = [] for item in obj: fixed, count = fix_obj(item) fixes += count new_list.append(fixed) return new_list, fixes else: return obj, 0 fixed_data, fix_count = fix_obj(json.loads(json.dumps(data))) # Round-trip to ensure clean JSON if fix_count > 0 and not dry_run: with open(path, 'w', encoding='utf-8') as f: json.dump(fixed_data, f, ensure_ascii=False, indent=1) print(f" Fixed {fix_count} mojibake in {path}") return fix_count def main(): import argparse parser = argparse.ArgumentParser(description="Fix UTF-8 mojibake in JSON files") parser.add_argument('path', nargs='?', default='.', help='Root directory to scan') parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed') args = parser.parse_args() root = Path('.').resolve() print(f"Scanning {Path('.').resolve()} for JSON files...") total_fixes = 0 files_fixed = 0 for path in Path('.').rglob("*.json"): if any(part.startswith('.') for part in path.parts): continue if path.name.startswith('.'): continue fixes = fix_json_file(path, dry_run=args.dry_run) if fixes > 0: files_fixed += 1 print(f" Fixed {fixes} issues in {path}") print(f"\nDone. Fixed {files_fixed} files.") if __name__ == '__main__': main()