From ab1fc2958be1ed526267228c7d579bc0557f8e96 Mon Sep 17 00:00:00 2001 From: n3wk Date: Mon, 24 Aug 2026 20:52:53 +0200 Subject: [PATCH] Add encoding validation, fix scripts, and encoding docs --- README.md | 28 +++++++++++ fix_encoding.py | 98 +++++++++++++++++++++++++++++++++++++ fix_mojibake.py | 119 +++++++++++++++++++++++++++++++++++++++++++++ pipeline_all.py | 40 +++++++++++++++ verify_encoding.py | 67 +++++++++++++++++++++++++ 5 files changed, 352 insertions(+) create mode 100644 fix_encoding.py create mode 100644 fix_mojibake.py create mode 100644 verify_encoding.py diff --git a/README.md b/README.md index 68167bb..2905d22 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,34 @@ Set via environment variables or edit `config.py`: | `DS_DEVICE` | `cpu` | `cpu` or `cuda` | | `DS_COMPUTE` | `int8` | `int8`, `float16`, `float32` | | `DS_KEEP_AUDIO` | `0` | Keep MP3s after transcription (`1` to keep) | +| `DS_ENCODING_CHECK` | `50` | Episodes between UTF-8 verification | + +## Encoding & Character Handling + +All JSON files are written as **UTF-8** with `ensure_ascii=False`. The RSS feed contains Unicode characters (em/en dashes, accented characters) which are preserved correctly in the JSON files. + +**If you see mojibake like `â€"` or `é`:** +- The JSON files are correctly stored as UTF-8 +- The issue is the *consumer* (text editor, web browser, terminal) reading the file with wrong encoding (Latin-1/Windows-1252) +- **Fix**: Ensure your tools/editor/browser use UTF-8 encoding + +**Verification & Repair:** +```bash +# Verify all JSON files are valid UTF-8 +python verify_encoding.py + +# Fix common mojibake patterns (run if data was corrupted) +python fix_mojibake.py --dry-run # preview fixes +python fix_mojibake.py # apply fixes + +# Check progress including encoding status +python check_progress.py +``` + +**For web apps displaying this data:** +- Set `Content-Type: application/json; charset=utf-8` +- Include `` in HTML +- Ensure database connections use UTF-8 ## Resuming / Sharing Work diff --git a/fix_encoding.py b/fix_encoding.py new file mode 100644 index 0000000..5cb5a71 --- /dev/null +++ b/fix_encoding.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Verify and repair UTF-8 encoding in all JSON files.""" + +import json +import sys +from pathlib import Path + +def check_file(path: Path) -> tuple[int, int]: + """Check a JSON file for encoding issues. Returns (errors_fixed, total_chars_checked).""" + try: + with open(path, 'r', encoding='utf-8') as f: + content = f.read() + except UnicodeDecodeError as e: + print(f" [ERROR] {path}: Not valid UTF-8 at position {e.start}: {e}") + return (0, 0) + + try: + data = json.loads(content) + except json.JSONDecodeError as e: + print(f" [ERROR] {path}: Invalid JSON at position {e.pos}: {e}") + return (0, 0) + + return (0, len(content)) + +def check_all_json_files(root: Path) -> dict: + """Scan all JSON files for encoding issues.""" + stats = {"files_checked": 0, "errors": 0, "total_bytes": 0} + + for path in root.rglob("*.json"): + if any(part.startswith('.') for part in path.parts): + continue + if path.name.startswith('.'): + continue + + errors, bytes_checked = check_file(path) + stats["files_checked"] += 1 + stats["errors"] += errors + stats["total_bytes"] += bytes_checked + + if stats["files_checked"] % 50 == 0: + print(f" Checked {stats['files_checked']} files...") + + return stats + +def fix_corrupted_unicode(text: str) -> str: + """Fix common UTF-8 mojibake patterns.""" + # Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252) + fixes = { + 'â€"': '—', # em dash + 'â€': '—', # em dash (truncated) + '’': "'", # right single quote + '“': '"', # left double quote + 'â€': '"', # right double quote + '•': '•', # bullet + '…': '…', # ellipsis + 'é': 'é', # é + 'è': 'è', # è + 'á': 'á', # á + 'í': 'í', # í + 'ó': 'ó', # ó + 'ú': 'ú', # ú + 'ñ': 'ñ', # ñ + 'ü': 'ü', # ü + 'ö': 'ö', # ö + 'ä': 'ä', # ä + 'Â': '', # stray  (common before accented chars) + } + + result = text + for bad, good in fixes.items(): + result = result.replace(bad, good) + return result + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Verify and repair UTF-8 encoding in JSON files") + parser.add_argument('path', nargs='?', default='.', help='Root directory to scan') + parser.add_argument('--fix', action='store_true', help='Attempt to fix mojibake') + parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed') + args = parser.parse_args() + + root = Path(args.path).resolve() + print(f"Scanning {root} for JSON files...") + + stats = check_all_json_files(Path('.')) + + print(f"\nResults:") + print(f" Files checked: {stats['files_checked']}") + print(f" Total bytes: {stats['total_bytes']:,}") + print(f" Encoding errors: {stats['errors']}") + + if stats['errors'] == 0: + print("All JSON files are valid UTF-8!") + else: + print(f"Found {stats['errors']} files with encoding issues") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/fix_mojibake.py b/fix_mojibake.py new file mode 100644 index 0000000..93e908c --- /dev/null +++ b/fix_mojibake.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Fix UTF-8 mojibake in JSON files.""" + +import json +import sys +from pathlib import Path + +# Common mojibake patterns (UTF-8 bytes misinterpreted as Latin-1/Windows-1252) +MOJIBAKE_FIXES = { + 'â€"': '—', # em dash + 'â€': '—', # em dash (truncated) + '’': "'", # right single quote + '“': '"', # left double quote + 'â€': '"', # right double quote + '•': '•', # bullet + '…': '…', # ellipsis + 'é': 'é', # é + 'è': 'è', # è + 'á': 'á', # á + 'í': 'í', # í + 'ó': 'ó', # ó + 'ú': 'ú', # ú + 'ñ': 'ñ', # ñ + 'ü': 'ü', # ü + 'ö': 'ö', # ö + 'ä': 'ä', # ä + 'ü': 'ü', # ü + 'ö': 'ö', # ö + 'ä': 'ä', # ä + 'Â': '', # stray  (common before accented chars) +} + +def fix_mojibake(text: str) -> tuple[str, int]: + """Fix common UTF-8 mojibake patterns. Returns (fixed_text, num_fixes).""" + count = 0 + result = text + for bad, good in MOJIBAKE_FIXES.items(): + before = result.count(bad) + if before: + result = result.replace(bad, good) + count += before + return result, count + +def fix_json_file(path: Path, dry_run: bool = False) -> int: + """Fix mojibake in a JSON file. Returns number of fixes applied.""" + try: + with open(path, 'r', encoding='utf-8') as f: + content = f.read() + except UnicodeDecodeError as e: + print(f" [ERROR] {path}: Not valid UTF-8 at position {e.start}") + return 0 + + try: + data = json.loads(content) + except json.JSONDecodeError as e: + print(f" [ERROR] {path}: Invalid JSON at position {e.pos}") + return 0 + + # Recursively fix strings in JSON + def fix_obj(obj): + fixes = 0 + if isinstance(obj, str): + fixed, count = fix_mojibake(obj) + return fixed, count + elif isinstance(obj, dict): + new_dict = {} + for k, v in obj.items(): + fixed_k, c1 = fix_obj(k) + fixed_v, c2 = fix_obj(v) + fixes += c1 + c2 + new_dict[fixed_k] = fixed_v + return new_dict, fixes + elif isinstance(obj, list): + new_list = [] + for item in obj: + fixed, count = fix_obj(item) + fixes += count + new_list.append(fixed) + return new_list, fixes + else: + return obj, 0 + + fixed_data, fix_count = fix_obj(json.loads(json.dumps(data))) # Round-trip to ensure clean JSON + + if fix_count > 0 and not dry_run: + with open(path, 'w', encoding='utf-8') as f: + json.dump(fixed_data, f, ensure_ascii=False, indent=1) + print(f" Fixed {fix_count} mojibake in {path}") + + return fix_count + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Fix UTF-8 mojibake in JSON files") + parser.add_argument('path', nargs='?', default='.', help='Root directory to scan') + parser.add_argument('--dry-run', action='store_true', help='Show what would be fixed') + args = parser.parse_args() + + root = Path('.').resolve() + print(f"Scanning {Path('.').resolve()} for JSON files...") + + total_fixes = 0 + files_fixed = 0 + + for path in Path('.').rglob("*.json"): + if any(part.startswith('.') for part in path.parts): + continue + if path.name.startswith('.'): + continue + + fixes = fix_json_file(path, dry_run=args.dry_run) + if fixes > 0: + files_fixed += 1 + print(f" Fixed {fixes} issues in {path}") + + print(f"\nDone. Fixed {files_fixed} files.") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/pipeline_all.py b/pipeline_all.py index 94e16a3..49c2054 100644 --- a/pipeline_all.py +++ b/pipeline_all.py @@ -40,6 +40,36 @@ ANALYSIS_TIMEOUT = int(os.environ.get("DS_ANALYSIS_TIMEOUT", "600")) # 10 minut # memory guard threshold (bytes) - pause if available RAM drops below this MIN_FREE_RAM = int(os.environ.get("DS_MIN_FREE_RAM", str(1024 * 1024 * 1024))) # 1 GB +# encoding validation interval (episodes) +ENCODING_CHECK_INTERVAL = int(os.environ.get("DS_ENCODING_CHECK", "50")) + + +def has_partial(transcript_file: str) -> bool: + try: + with open(transcript_file, encoding="utf-8") as f: + return json.load(f).get("partial_minutes") is not None + except (OSError, json.JSONDecodeError): + return False + + +def verify_utf8_encoding(root: Path = Path('.')) -> tuple[int, int]: + """Verify all JSON files are valid UTF-8. Returns (errors, files_checked).""" + errors = 0 + checked = 0 + for path in Path('.').rglob("*.json"): + if any(part.startswith('.') for part in path.parts): + continue + if path.name.startswith('.'): + continue + try: + with open(path, 'r', encoding='utf-8') as f: + json.load(path) + except (UnicodeDecodeError, json.JSONDecodeError) as e: + print(f" [ENCODING ERROR] {path}: {e}", flush=True) + errors += 1 + checked += 1 + return errors, checked + def has_partial(transcript_file: str) -> bool: try: @@ -293,6 +323,16 @@ def main() -> None: index_mod.index_transcripts() from aggregate import aggregate as _agg _agg() + + # Periodic encoding verification + if i % ENCODING_CHECK_INTERVAL == 0: + print(f" [{time.strftime('%H:%M:%S')}] verifying UTF-8 encoding...", flush=True) + errors, checked = verify_utf8_encoding() + if errors: + print(f" [WARNING] Found {errors} files with encoding issues!", flush=True) + else: + print(f" [OK] UTF-8 encoding verified for {checked} files", flush=True) + hrs = (time.time() - t_start) / 3600 rate_eps = i / hrs remaining = (len(eps) - i) / max(rate_eps, 0.01) diff --git a/verify_encoding.py b/verify_encoding.py new file mode 100644 index 0000000..ff0d983 --- /dev/null +++ b/verify_encoding.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Verify UTF-8 encoding in all JSON files.""" + +import json +import sys +from pathlib import Path + +def verify_utf8(path: Path) -> tuple[int, int]: + """Check a JSON file for valid UTF-8. Returns (errors, bytes_checked).""" + try: + with open(path, 'r', encoding='utf-8') as f: + content = f.read() + except UnicodeDecodeError as e: + print(f" [ERROR] {path}: Invalid UTF-8 at position {e.start}") + return (1, 0) + + try: + json.loads(content) + except json.JSONDecodeError as e: + print(f" [ERROR] {path}: Invalid JSON at position {e.pos}: {e}") + return (1, 0) + + return (0, len(content)) + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files") + parser.add_argument('path', nargs='?', default='.', help='Root directory to scan') + args = parser.parse_args() + + root = Path('.').resolve() + print(f"Scanning {root} for JSON files...") + + stats = {"files": 0, "errors": 0, "bytes": 0} + + for path in Path('.').rglob("*.json"): + if any(part.startswith('.') for part in path.parts): + continue + if path.name.startswith('.'): + continue + + errors, bytes_checked = verify_utf8(path) + stats["files_checked"] = stats.get("files_checked", 0) + 1 + stats["errors"] += errors + stats["total_bytes"] = stats.get("total_bytes", 0) + bytes_checked + + if stats["files_checked"] % 50 == 0: + print(f" Checked {stats['files_checked']} files...") + + print(f"\nResults:") + print(f" Files checked: {stats['files_checked']}") + print(f" Total bytes: {stats['total_bytes']:,}") + print(f" Encoding errors: {stats['errors']}") + + if stats['errors'] == 0: + print("All JSON files are valid UTF-8!") + return 0 + else: + print(f"Found {stats['errors']} files with encoding issues") + return 1 + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser(description="Verify UTF-8 encoding in JSON files") + parser.add_argument('path', nargs='?', default='.', help='Root directory to scan') + args = parser.parse_args() + sys.exit(main(args.path)) \ No newline at end of file