Add encoding validation, fix scripts, and encoding docs
This commit is contained in:
n3wk
@@ -92,6 +92,34 @@ Set via environment variables or edit `config.py`:
|
||||
| `DS_DEVICE` | `cpu` | `cpu` or `cuda` |
|
||||
| `DS_COMPUTE` | `int8` | `int8`, `float16`, `float32` |
|
||||
| `DS_KEEP_AUDIO` | `0` | Keep MP3s after transcription (`1` to keep) |
|
||||
| `DS_ENCODING_CHECK` | `50` | Episodes between UTF-8 verification |
|
||||
|
||||
## Encoding & Character Handling
|
||||
|
||||
All JSON files are written as **UTF-8** with `ensure_ascii=False`. The RSS feed contains Unicode characters (em/en dashes, accented characters) which are preserved correctly in the JSON files.
|
||||
|
||||
**If you see mojibake like `â€"` or `é`:**
|
||||
- The JSON files are correctly stored as UTF-8
|
||||
- The issue is the *consumer* (text editor, web browser, terminal) reading the file with wrong encoding (Latin-1/Windows-1252)
|
||||
- **Fix**: Ensure your tools/editor/browser use UTF-8 encoding
|
||||
|
||||
**Verification & Repair:**
|
||||
```bash
|
||||
# Verify all JSON files are valid UTF-8
|
||||
python verify_encoding.py
|
||||
|
||||
# Fix common mojibake patterns (run if data was corrupted)
|
||||
python fix_mojibake.py --dry-run # preview fixes
|
||||
python fix_mojibake.py # apply fixes
|
||||
|
||||
# Check progress including encoding status
|
||||
python check_progress.py
|
||||
```
|
||||
|
||||
**For web apps displaying this data:**
|
||||
- Set `Content-Type: application/json; charset=utf-8`
|
||||
- Include `<meta charset="utf-8">` in HTML
|
||||
- Ensure database connections use UTF-8
|
||||
|
||||
## Resuming / Sharing Work
|
||||
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify and repair UTF-8 encoding in all JSON files."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
def check_file(path: Path) -> tuple[int, int]:
|
||||
"""Check a JSON file for encoding issues. Returns (errors_fixed, total_chars_checked)."""
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
except UnicodeDecodeError as e:
|
||||
print(f" [ERROR] {path}: Not valid UTF-8 at position {e.start}: {e}")
|
||||
return (0, 0)
|
||||
|
||||
try:
|
||||
data = json.loads(content)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f" [ERROR] {path}: Invalid JSON at position {e.pos}: {e}")
|
||||
return (0, 0)
|
||||
|
||||
return (0, len(content))
|
||||
|
||||
def check_all_json_files(root: Path) -> dict:
|
||||
"""Scan all JSON files for encoding issues."""
|
||||
stats = {"files_checked": 0, "errors": 0, "total_bytes": 0}
|
||||
|
||||
for path in root.rglob("*.json"):
|
||||
if any(part.startswith('.') for part in path.parts):
|
||||
continue
|
||||
if path.name.startswith('.'):
|
||||
continue
|
||||
|
||||
errors, bytes_checked = check_file(path)
|
||||
stats["files_checked"] += 1
|
||||
stats["errors"] += errors
|
||||
stats["total_bytes"] += bytes_checked
|
||||
|
||||
if stats["files_checked"] % 50 == 0:
|
||||
print(f" Checked {stats['files_checked']} files...")
|
||||
|
||||
return stats
|
||||
|
||||
def fix_corrupted_unicode(text: str) -> str:
|
||||
"""Fix common UTF-8 mojibake patterns."""
|
||||
# Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252)
|
||||
fixes = {
|
||||
'â€"': '—', # em dash
|
||||
'â€': '—', # em dash (truncated)
|
||||
'’': "'", # right single quote
|
||||
'“': '"', # left double quote
|
||||
'â€': '"', # right double quote
|
||||
'•': '•', # bullet
|
||||
'…': '…', # ellipsis
|
||||
'é': 'é', # é
|
||||
'è': 'è', # è
|
||||
'á': 'á', # á
|
||||
'Ã': 'í', # í
|
||||
'ó': 'ó', # ó
|
||||
'ú': 'ú', # ú
|
||||
'ñ': 'ñ', # ñ
|
||||
'ü': 'ü', # ü
|
||||
'ö': 'ö', # ö
|
||||
'ä': 'ä', # ä
|
||||
'Â': '', # stray  (common before accented chars)
|
||||
}
|
||||
|
||||
result = text
|
||||
for bad, good in fixes.items():
|
||||
result = result.replace(bad, good)
|
||||
return result
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Verify and repair UTF-8 encoding in JSON files")
|
||||
parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
|
||||
parser.add_argument('--fix', action='store_true', help='Attempt to fix mojibake')
|
||||
parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed')
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path(args.path).resolve()
|
||||
print(f"Scanning {root} for JSON files...")
|
||||
|
||||
stats = check_all_json_files(Path('.'))
|
||||
|
||||
print(f"\nResults:")
|
||||
print(f" Files checked: {stats['files_checked']}")
|
||||
print(f" Total bytes: {stats['total_bytes']:,}")
|
||||
print(f" Encoding errors: {stats['errors']}")
|
||||
|
||||
if stats['errors'] == 0:
|
||||
print("All JSON files are valid UTF-8!")
|
||||
else:
|
||||
print(f"Found {stats['errors']} files with encoding issues")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix UTF-8 mojibake in JSON files."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252)
|
||||
MOJIBAKE_FIXES = {
|
||||
'â€"': '—', # em dash
|
||||
'â€': '—', # em dash (truncated)
|
||||
'’': "'", # right single quote
|
||||
'“': '"', # left double quote
|
||||
'â€': '"', # right double quote
|
||||
'•': '•', # bullet
|
||||
'…': '…', # ellipsis
|
||||
'é': 'é', # é
|
||||
'è': 'è', # è
|
||||
'á': 'á', # á
|
||||
'Ã': 'í', # í
|
||||
'ó': 'ó', # ó
|
||||
'ú': 'ú', # ú
|
||||
'ñ': 'ñ', # ñ
|
||||
'ü': 'ü', # ü
|
||||
'ö': 'ö', # ö
|
||||
'ä': 'ä', # ä
|
||||
'ü': 'ü', # ü
|
||||
'ö': 'ö', # ö
|
||||
'ä': 'ä', # ä
|
||||
'Â': '', # stray  (common before accented chars)
|
||||
}
|
||||
|
||||
def fix_mojibake(text: str) -> tuple[str, int]:
|
||||
"""Fix common UTF-8 mojibake patterns. Returns (fixed_text, num_fixes)."""
|
||||
count = 0
|
||||
result = text
|
||||
for bad, good in MOJIBAKE_FIXES.items():
|
||||
before = result.count(bad)
|
||||
if before:
|
||||
result = result.replace(bad, good)
|
||||
count += before
|
||||
return result, count
|
||||
|
||||
def fix_json_file(path: Path, dry_run: bool = False) -> int:
|
||||
"""Fix mojibake in a JSON file. Returns number of fixes applied."""
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
except UnicodeDecodeError as e:
|
||||
print(f" [ERROR] {path}: Not valid UTF-8 at position {e.start}")
|
||||
return 0
|
||||
|
||||
try:
|
||||
data = json.loads(content)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f" [ERROR] {path}: Invalid JSON at position {e.pos}")
|
||||
return 0
|
||||
|
||||
# Recursively fix strings in JSON
|
||||
def fix_obj(obj):
|
||||
fixes = 0
|
||||
if isinstance(obj, str):
|
||||
fixed, count = fix_mojibake(obj)
|
||||
return fixed, count
|
||||
elif isinstance(obj, dict):
|
||||
new_dict = {}
|
||||
for k, v in obj.items():
|
||||
fixed_k, c1 = fix_obj(k)
|
||||
fixed_v, c2 = fix_obj(v)
|
||||
fixes += c1 + c2
|
||||
new_dict[fixed_k] = fixed_v
|
||||
return new_dict, fixes
|
||||
elif isinstance(obj, list):
|
||||
new_list = []
|
||||
for item in obj:
|
||||
fixed, count = fix_obj(item)
|
||||
fixes += count
|
||||
new_list.append(fixed)
|
||||
return new_list, fixes
|
||||
else:
|
||||
return obj, 0
|
||||
|
||||
fixed_data, fix_count = fix_obj(json.loads(json.dumps(data))) # Round-trip to ensure clean JSON
|
||||
|
||||
if fix_count > 0 and not dry_run:
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
json.dump(fixed_data, f, ensure_ascii=False, indent=1)
|
||||
print(f" Fixed {fix_count} mojibake in {path}")
|
||||
|
||||
return fix_count
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Fix UTF-8 mojibake in JSON files")
|
||||
parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
|
||||
parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed')
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path('.').resolve()
|
||||
print(f"Scanning {Path('.').resolve()} for JSON files...")
|
||||
|
||||
total_fixes = 0
|
||||
files_fixed = 0
|
||||
|
||||
for path in Path('.').rglob("*.json"):
|
||||
if any(part.startswith('.') for part in path.parts):
|
||||
continue
|
||||
if path.name.startswith('.'):
|
||||
continue
|
||||
|
||||
fixes = fix_json_file(path, dry_run=args.dry_run)
|
||||
if fixes > 0:
|
||||
files_fixed += 1
|
||||
print(f" Fixed {fixes} issues in {path}")
|
||||
|
||||
print(f"\nDone. Fixed {files_fixed} files.")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -40,6 +40,36 @@ ANALYSIS_TIMEOUT = int(os.environ.get("DS_ANALYSIS_TIMEOUT", "600")) # 10 minut
|
||||
# memory guard threshold (bytes) - pause if available RAM drops below this
|
||||
MIN_FREE_RAM = int(os.environ.get("DS_MIN_FREE_RAM", str(1024 * 1024 * 1024))) # 1 GB
|
||||
|
||||
# encoding validation interval (episodes)
|
||||
ENCODING_CHECK_INTERVAL = int(os.environ.get("DS_ENCODING_CHECK", "50"))
|
||||
|
||||
|
||||
def has_partial(transcript_file: str) -> bool:
|
||||
try:
|
||||
with open(transcript_file, encoding="utf-8") as f:
|
||||
return json.load(f).get("partial_minutes") is not None
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return False
|
||||
|
||||
|
||||
def verify_utf8_encoding(root: Path = Path('.')) -> tuple[int, int]:
|
||||
"""Verify all JSON files are valid UTF-8. Returns (errors, files_checked)."""
|
||||
errors = 0
|
||||
checked = 0
|
||||
for path in Path('.').rglob("*.json"):
|
||||
if any(part.startswith('.') for part in path.parts):
|
||||
continue
|
||||
if path.name.startswith('.'):
|
||||
continue
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
json.load(path)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as e:
|
||||
print(f" [ENCODING ERROR] {path}: {e}", flush=True)
|
||||
errors += 1
|
||||
checked += 1
|
||||
return errors, checked
|
||||
|
||||
|
||||
def has_partial(transcript_file: str) -> bool:
|
||||
try:
|
||||
@@ -293,6 +323,16 @@ def main() -> None:
|
||||
index_mod.index_transcripts()
|
||||
from aggregate import aggregate as _agg
|
||||
_agg()
|
||||
|
||||
# Periodic encoding verification
|
||||
if i % ENCODING_CHECK_INTERVAL == 0:
|
||||
print(f" [{time.strftime('%H:%M:%S')}] verifying UTF-8 encoding...", flush=True)
|
||||
errors, checked = verify_utf8_encoding()
|
||||
if errors:
|
||||
print(f" [WARNING] Found {errors} files with encoding issues!", flush=True)
|
||||
else:
|
||||
print(f" [OK] UTF-8 encoding verified for {checked} files", flush=True)
|
||||
|
||||
hrs = (time.time() - t_start) / 3600
|
||||
rate_eps = i / hrs
|
||||
remaining = (len(eps) - i) / max(rate_eps, 0.01)
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify UTF-8 encoding in all JSON files."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
def verify_utf8(path: Path) -> tuple[int, int]:
|
||||
"""Check a JSON file for valid UTF-8. Returns (errors, bytes_checked)."""
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
except UnicodeDecodeError as e:
|
||||
print(f" [ERROR] {path}: Invalid UTF-8 at position {e.start}")
|
||||
return (1, 0)
|
||||
|
||||
try:
|
||||
json.loads(content)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f" [ERROR] {path}: Invalid JSON at position {e.pos}: {e}")
|
||||
return (1, 0)
|
||||
|
||||
return (0, len(content))
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files")
|
||||
parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path('.').resolve()
|
||||
print(f"Scanning {root} for JSON files...")
|
||||
|
||||
stats = {"files": 0, "errors": 0, "bytes": 0}
|
||||
|
||||
for path in Path('.').rglob("*.json"):
|
||||
if any(part.startswith('.') for part in path.parts):
|
||||
continue
|
||||
if path.name.startswith('.'):
|
||||
continue
|
||||
|
||||
errors, bytes_checked = verify_utf8(path)
|
||||
stats["files_checked"] = stats.get("files_checked", 0) + 1
|
||||
stats["errors"] += errors
|
||||
stats["total_bytes"] = stats.get("total_bytes", 0) + bytes_checked
|
||||
|
||||
if stats["files_checked"] % 50 == 0:
|
||||
print(f" Checked {stats['files_checked']} files...")
|
||||
|
||||
print(f"\nResults:")
|
||||
print(f" Files checked: {stats['files_checked']}")
|
||||
print(f" Total bytes: {stats['total_bytes']:,}")
|
||||
print(f" Encoding errors: {stats['errors']}")
|
||||
|
||||
if stats['errors'] == 0:
|
||||
print("All JSON files are valid UTF-8!")
|
||||
return 0
|
||||
else:
|
||||
print(f"Found {stats['errors']} files with encoding issues")
|
||||
return 1
|
||||
|
||||
if __name__ == '__main__':
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files")
|
||||
parser.add_argument('path', nargs='?', default='.', help='Root directory to scan')
|
||||
args = parser.parse_args()
|
||||
sys.exit(main(args.path))
|
||||
Reference in New Issue
Block a user