98 lines
3.3 KiB
Python
98 lines
3.3 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Verify and repair UTF-8 encoding in all JSON files."""
|
||
|
|
|
||
|
|
import json
|
||
|
|
import sys
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
def check_file(path: Path) -> tuple[int, int]:
|
||
|
|
"""Check a JSON file for encoding issues. Returns (errors_fixed, total_chars_checked)."""
|
||
|
|
try:
|
||
|
|
with open(path, 'r', encoding='utf-8') as f:
|
||
|
|
content = f.read()
|
||
|
|
except UnicodeDecodeError as e:
|
||
|
|
print(f" [ERROR] {path}: Not valid UTF-8 at position {e.start}: {e}")
|
||
|
|
return (0, 0)
|
||
|
|
|
||
|
|
try:
|
||
|
|
data = json.loads(content)
|
||
|
|
except json.JSONDecodeError as e:
|
||
|
|
print(f" [ERROR] {path}: Invalid JSON at position {e.pos}: {e}")
|
||
|
|
return (0, 0)
|
||
|
|
|
||
|
|
return (0, len(content))
|
||
|
|
|
||
|
|
def check_all_json_files(root: Path) -> dict:
|
||
|
|
"""Scan all JSON files for encoding issues."""
|
||
|
|
stats = {"files_checked": 0, "errors": 0, "total_bytes": 0}
|
||
|
|
|
||
|
|
for path in root.rglob("*.json"):
|
||
|
|
if any(part.startswith('.') for part in path.parts):
|
||
|
|
continue
|
||
|
|
if path.name.startswith('.'):
|
||
|
|
continue
|
||
|
|
|
||
|
|
errors, bytes_checked = check_file(path)
|
||
|
|
stats["files_checked"] += 1
|
||
|
|
stats["errors"] += errors
|
||
|
|
stats["total_bytes"] += bytes_checked
|
||
|
|
|
||
|
|
if stats["files_checked"] % 50 == 0:
|
||
|
|
print(f" Checked {stats['files_checked']} files...")
|
||
|
|
|
||
|
|
return stats
|
||
|
|
|
||
|
|
def fix_corrupted_unicode(text: str) -> str:
|
||
|
|
"""Fix common UTF-8 mojibake patterns."""
|
||
|
|
# Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252)
|
||
|
|
fixes = {
|
||
|
|
'â€"': '—', # em dash
|
||
|
|
'â€': '—', # em dash (truncated)
|
||
|
|
'’': "'", # right single quote
|
||
|
|
'“': '"', # left double quote
|
||
|
|
'â€': '"', # right double quote
|
||
|
|
'•': '•', # bullet
|
||
|
|
'…': '…', # ellipsis
|
||
|
|
'é': 'é', # é
|
||
|
|
'è': 'è', # è
|
||
|
|
'á': 'á', # á
|
||
|
|
'Ã': 'í', # í
|
||
|
|
'ó': 'ó', # ó
|
||
|
|
'ú': 'ú', # ú
|
||
|
|
'ñ': 'ñ', # ñ
|
||
|
|
'ü': 'ü', # ü
|
||
|
|
'ö': 'ö', # ö
|
||
|
|
'ä': 'ä', # ä
|
||
|
|
'Â': '', # stray  (common before accented chars)
|
||
|
|
}
|
||
|
|
|
||
|
|
result = text
|
||
|
|
for bad, good in fixes.items():
|
||
|
|
result = result.replace(bad, good)
|
||
|
|
return result
|
||
|
|
|
||
|
|
def main():
|
||
|
|
import argparse
|
||
|
|
parser = argparse.ArgumentParser(description="Verify and repair UTF-8 encoding in JSON files")
|
||
|
|
parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
|
||
|
|
parser.add_argument('--fix', action='store_true', help='Attempt to fix mojibake')
|
||
|
|
parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed')
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
root = Path(args.path).resolve()
|
||
|
|
print(f"Scanning {root} for JSON files...")
|
||
|
|
|
||
|
|
stats = check_all_json_files(Path('.'))
|
||
|
|
|
||
|
|
print(f"\nResults:")
|
||
|
|
print(f" Files checked: {stats['files_checked']}")
|
||
|
|
print(f" Total bytes: {stats['total_bytes']:,}")
|
||
|
|
print(f" Encoding errors: {stats['errors']}")
|
||
|
|
|
||
|
|
if stats['errors'] == 0:
|
||
|
|
print("All JSON files are valid UTF-8!")
|
||
|
|
else:
|
||
|
|
print(f"Found {stats['errors']} files with encoding issues")
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
main()
|