67 lines
2.2 KiB
Python
67 lines
2.2 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Verify UTF-8 encoding in all JSON files."""
|
||
|
|
|
||
|
|
import json
|
||
|
|
import sys
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
def verify_utf8(path: Path) -> tuple[int, int]:
|
||
|
|
"""Check a JSON file for valid UTF-8. Returns (errors, bytes_checked)."""
|
||
|
|
try:
|
||
|
|
with open(path, 'r', encoding='utf-8') as f:
|
||
|
|
content = f.read()
|
||
|
|
except UnicodeDecodeError as e:
|
||
|
|
print(f" [ERROR] {path}: Invalid UTF-8 at position {e.start}")
|
||
|
|
return (1, 0)
|
||
|
|
|
||
|
|
try:
|
||
|
|
json.loads(content)
|
||
|
|
except json.JSONDecodeError as e:
|
||
|
|
print(f" [ERROR] {path}: Invalid JSON at position {e.pos}: {e}")
|
||
|
|
return (1, 0)
|
||
|
|
|
||
|
|
return (0, len(content))
|
||
|
|
|
||
|
|
def main():
|
||
|
|
import argparse
|
||
|
|
parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files")
|
||
|
|
parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
root = Path('.').resolve()
|
||
|
|
print(f"Scanning {root} for JSON files...")
|
||
|
|
|
||
|
|
stats = {"files": 0, "errors": 0, "bytes": 0}
|
||
|
|
|
||
|
|
for path in Path('.').rglob("*.json"):
|
||
|
|
if any(part.startswith('.') for part in path.parts):
|
||
|
|
continue
|
||
|
|
if path.name.startswith('.'):
|
||
|
|
continue
|
||
|
|
|
||
|
|
errors, bytes_checked = verify_utf8(path)
|
||
|
|
stats["files_checked"] = stats.get("files_checked", 0) + 1
|
||
|
|
stats["errors"] += errors
|
||
|
|
stats["total_bytes"] = stats.get("total_bytes", 0) + bytes_checked
|
||
|
|
|
||
|
|
if stats["files_checked"] % 50 == 0:
|
||
|
|
print(f" Checked {stats['files_checked']} files...")
|
||
|
|
|
||
|
|
print(f"\nResults:")
|
||
|
|
print(f" Files checked: {stats['files_checked']}")
|
||
|
|
print(f" Total bytes: {stats['total_bytes']:,}")
|
||
|
|
print(f" Encoding errors: {stats['errors']}")
|
||
|
|
|
||
|
|
if stats['errors'] == 0:
|
||
|
|
print("All JSON files are valid UTF-8!")
|
||
|
|
return 0
|
||
|
|
else:
|
||
|
|
print(f"Found {stats['errors']} files with encoding issues")
|
||
|
|
return 1
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
import argparse
|
||
|
|
parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files")
|
||
|
|
parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
|
||
|
|
args = parser.parse_args()
|
||
|
|
sys.exit(main(args.path))
|