n3wk
119 lines
3.8 KiB
Python
119 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Fix UTF-8 mojibake in JSON files."""
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252)
|
|
MOJIBAKE_FIXES = {
|
|
'â€"': '—', # em dash
|
|
'â€': '—', # em dash (truncated)
|
|
'’': "'", # right single quote
|
|
'“': '"', # left double quote
|
|
'â€': '"', # right double quote
|
|
'•': '•', # bullet
|
|
'…': '…', # ellipsis
|
|
'é': 'é', # é
|
|
'è': 'è', # è
|
|
'á': 'á', # á
|
|
'Ã': 'í', # í
|
|
'ó': 'ó', # ó
|
|
'ú': 'ú', # ú
|
|
'ñ': 'ñ', # ñ
|
|
'ü': 'ü', # ü
|
|
'ö': 'ö', # ö
|
|
'ä': 'ä', # ä
|
|
'ü': 'ü', # ü
|
|
'ö': 'ö', # ö
|
|
'ä': 'ä', # ä
|
|
'Â': '', # stray  (common before accented chars)
|
|
}
|
|
|
|
def fix_mojibake(text: str) -> tuple[str, int]:
|
|
"""Fix common UTF-8 mojibake patterns. Returns (fixed_text, num_fixes)."""
|
|
count = 0
|
|
result = text
|
|
for bad, good in MOJIBAKE_FIXES.items():
|
|
before = result.count(bad)
|
|
if before:
|
|
result = result.replace(bad, good)
|
|
count += before
|
|
return result, count
|
|
|
|
def fix_json_file(path: Path, dry_run: bool = False) -> int:
|
|
"""Fix mojibake in a JSON file. Returns number of fixes applied."""
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
except UnicodeDecodeError as e:
|
|
print(f" [ERROR] {path}: Not valid UTF-8 at position {e.start}")
|
|
return 0
|
|
|
|
try:
|
|
data = json.loads(content)
|
|
except json.JSONDecodeError as e:
|
|
print(f" [ERROR] {path}: Invalid JSON at position {e.pos}")
|
|
return 0
|
|
|
|
# Recursively fix strings in JSON
|
|
def fix_obj(obj):
|
|
fixes = 0
|
|
if isinstance(obj, str):
|
|
fixed, count = fix_mojibake(obj)
|
|
return fixed, count
|
|
elif isinstance(obj, dict):
|
|
new_dict = {}
|
|
for k, v in obj.items():
|
|
fixed_k, c1 = fix_obj(k)
|
|
fixed_v, c2 = fix_obj(v)
|
|
fixes += c1 + c2
|
|
new_dict[fixed_k] = fixed_v
|
|
return new_dict, fixes
|
|
elif isinstance(obj, list):
|
|
new_list = []
|
|
for item in obj:
|
|
fixed, count = fix_obj(item)
|
|
fixes += count
|
|
new_list.append(fixed)
|
|
return new_list, fixes
|
|
else:
|
|
return obj, 0
|
|
|
|
fixed_data, fix_count = fix_obj(json.loads(json.dumps(data))) # Round-trip to ensure clean JSON
|
|
|
|
if fix_count > 0 and not dry_run:
|
|
with open(path, 'w', encoding='utf-8') as f:
|
|
json.dump(fixed_data, f, ensure_ascii=False, indent=1)
|
|
print(f" Fixed {fix_count} mojibake in {path}")
|
|
|
|
return fix_count
|
|
|
|
def main():
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description="Fix UTF-8 mojibake in JSON files")
|
|
parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
|
|
parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed')
|
|
args = parser.parse_args()
|
|
|
|
root = Path('.').resolve()
|
|
print(f"Scanning {Path('.').resolve()} for JSON files...")
|
|
|
|
total_fixes = 0
|
|
files_fixed = 0
|
|
|
|
for path in Path('.').rglob("*.json"):
|
|
if any(part.startswith('.') for part in path.parts):
|
|
continue
|
|
if path.name.startswith('.'):
|
|
continue
|
|
|
|
fixes = fix_json_file(path, dry_run=args.dry_run)
|
|
if fixes > 0:
|
|
files_fixed += 1
|
|
print(f" Fixed {fixes} issues in {path}")
|
|
|
|
print(f"\nDone. Fixed {files_fixed} files.")
|
|
|
|
if __name__ == '__main__':
|
|
main() |