From 37786612b772c3cf7c85ecd21210490fd22e4c29 Mon Sep 17 00:00:00 2001 From: Philip Guzman Date: Wed, 1 Jul 2026 08:13:20 -0700 Subject: [PATCH] Add read-only database V2 parser --- ROADMAP.md | 2 +- docs/design/database-v2-parser.md | 28 +++++++++++++ samples/small-library/generate.py | 14 +++++++ serato_doctor/cli.py | 9 +++++ serato_doctor/database_parser.py | 67 +++++++++++++++++++++++++++++++ serato_doctor/health.py | 25 +++++++++++- serato_doctor/models/__init__.py | 3 ++ serato_doctor/models/database.py | 22 ++++++++++ serato_doctor/models/health.py | 6 +++ serato_doctor/models/library.py | 8 +++- serato_doctor/web.py | 4 ++ serato_doctor/webui/index.html | 2 + tests/test_database_parser.py | 48 ++++++++++++++++++++++ tests/test_web.py | 3 ++ 14 files changed, 238 insertions(+), 3 deletions(-) create mode 100644 docs/design/database-v2-parser.md create mode 100644 serato_doctor/database_parser.py create mode 100644 serato_doctor/models/database.py create mode 100644 tests/test_database_parser.py diff --git a/ROADMAP.md b/ROADMAP.md index 4761c91..2556449 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -10,7 +10,7 @@ - [x] HTML health dashboard - [x] Test suite - [x] Sample library fixtures -- [ ] Database V2 read-only parser +- [x] Database V2 read-only parser - [x] Configuration - [x] Logging - [x] Matching engine diff --git a/docs/design/database-v2-parser.md b/docs/design/database-v2-parser.md new file mode 100644 index 0000000..280a3ea --- /dev/null +++ b/docs/design/database-v2-parser.md @@ -0,0 +1,28 @@ +# Database V2 Read-only Parser + +## Problem + +Crates and files do not explain every orange track in Serato. The legacy +`database V2` contains Serato's library-level track paths and metadata, so it must +be inspected independently from crate references. + +## Architecture + +The parser reads the file as a big-endian tag-length-value stream. Top-level +`otrk` records contain nested fields including `pfil` (path), `tsng` (title), +`tart` (artist), `talb` (album), and `tgen` (genre). Text is UTF-16 big-endian. + +Analysis reports total database entries, entries whose filenames occur in the +selected music scan, entries outside that scan, scanned tracks absent from the +database, and duplicate database paths. “Outside scan” is deliberately not called +missing because Serato databases can include samples and tracks from other roots. + +## Safety + +The parser calls only `read_bytes`; it never opens the database for writing. No +metadata values or personal paths are sent to logs or the dashboard. + +## Verification + +Synthetic TLV fixtures cover version, paths, metadata, incomplete records, and +health integration. The sample library includes a generated ten-entry database. diff --git a/samples/small-library/generate.py b/samples/small-library/generate.py index ae13268..160abd3 100644 --- a/samples/small-library/generate.py +++ b/samples/small-library/generate.py @@ -10,6 +10,10 @@ SAMPLE_ROOT = Path(__file__).parent SERATO_PATH_PREFIX = "Users/sample-user/OneDrive/Jukebox/" +def database_record(tag: bytes, payload: bytes) -> bytes: + return tag + len(payload).to_bytes(4, "big") + payload + + def load_manifest() -> dict: return json.loads((SAMPLE_ROOT / "manifest.json").read_text(encoding="utf-8")) @@ -51,6 +55,16 @@ def build_sample(output: Optional[Path] = None) -> Path: ) (crate_root / output_name).write_bytes(records.encode("utf-16-le")) + database_records = [ + database_record(b"vrsn", "2.0/Serato Doctor Fixture".encode("utf-16-be")) + ] + for relative_path in manifest["tracks"]: + fields = database_record( + b"pfil", f"{SERATO_PATH_PREFIX}{relative_path}".encode("utf-16-be") + ) + database_records.append(database_record(b"otrk", fields)) + (serato_root / "database V2").write_bytes(b"".join(database_records)) + return root diff --git a/serato_doctor/cli.py b/serato_doctor/cli.py index 435f37f..0b12c2e 100644 --- a/serato_doctor/cli.py +++ b/serato_doctor/cli.py @@ -3,6 +3,7 @@ import argparse from serato_doctor.config import ScanConfig from serato_doctor.crate_parser import load_library_crates +from serato_doctor.database_parser import parse_database from serato_doctor.health import analyze_health from serato_doctor.logging import configure_logging from serato_doctor.models.library import Library @@ -57,10 +58,13 @@ def main(): crates = load_library_crates(config.serato, config.reference_roots) filesystem = scan_filesystem(config.music) + database_path = config.serato / "database V2" + database = parse_database(database_path) if database_path.is_file() else None library = Library.from_crates( crates=crates, tracks=filesystem.tracks, broken_symlinks=filesystem.broken_symlinks, + database=database, ) results = library.reconcile_by_filename() missing_count = sum(1 for result in results if not result.exists_by_filename) @@ -99,6 +103,11 @@ def main(): print( f"Dynamic References Excluded: {health.dynamic_references_excluded}" ) + print(f"Database Entries: {health.database_entries}") + print(f"Database / Library Matches: {health.database_library_matches}") + print(f"Database Entries Outside Scan: {health.database_unmatched_entries}") + print(f"Tracks Missing From Database: {health.tracks_missing_from_database}") + print(f"Duplicate Database Paths: {health.duplicate_database_paths}") else: write_csv(results, config.out) write_missing_report(results, config.report) diff --git a/serato_doctor/database_parser.py b/serato_doctor/database_parser.py new file mode 100644 index 0000000..68ef22a --- /dev/null +++ b/serato_doctor/database_parser.py @@ -0,0 +1,67 @@ +from pathlib import Path +from typing import Dict, Iterator, Optional, Tuple + +from serato_doctor.models.database import DatabaseTrack, SeratoDatabase + + +TEXT_FIELDS = { + b"tsng": "title", + b"tart": "artist", + b"talb": "album", + b"tgen": "genre", +} + + +def iter_records(data: bytes) -> Iterator[Tuple[bytes, bytes]]: + """Yield complete big-endian tag-length-value records.""" + + offset = 0 + while offset + 8 <= len(data): + tag = data[offset : offset + 4] + length = int.from_bytes(data[offset + 4 : offset + 8], "big") + payload_start = offset + 8 + payload_end = payload_start + length + if payload_end > len(data): + break + yield tag, data[payload_start:payload_end] + offset = payload_end + + +def decode_text(payload: bytes) -> Optional[str]: + value = payload.decode("utf-16-be", errors="ignore").strip("\x00").strip() + return value or None + + +def normalize_database_path(value: str) -> Path: + if value.startswith(("Users/", "Volumes/")): + value = "/" + value + return Path(value) + + +def parse_track(payload: bytes) -> Optional[DatabaseTrack]: + fields: Dict[str, Optional[str]] = {} + path = None + for tag, value in iter_records(payload): + if tag == b"pfil": + decoded_path = decode_text(value) + if decoded_path: + path = normalize_database_path(decoded_path) + elif tag in TEXT_FIELDS: + fields[TEXT_FIELDS[tag]] = decode_text(value) + if path is None: + return None + return DatabaseTrack(path=path, filename=path.name, **fields) + + +def parse_database(database_path: Path) -> SeratoDatabase: + data = database_path.read_bytes() + version = None + tracks = [] + for tag, payload in iter_records(data): + if tag == b"vrsn": + version = decode_text(payload) + elif tag == b"otrk": + track = parse_track(payload) + if track is not None: + tracks.append(track) + return SeratoDatabase(database_path, version, tuple(tracks)) diff --git a/serato_doctor/health.py b/serato_doctor/health.py index 2cf4169..aaa5768 100644 --- a/serato_doctor/health.py +++ b/serato_doctor/health.py @@ -1,5 +1,7 @@ +from collections import Counter + from serato_doctor.duplicates import find_duplicate_groups -from serato_doctor.matching import MatchingEngine +from serato_doctor.matching import MatchingEngine, normalize from serato_doctor.models.crate import CrateKind from serato_doctor.models.duplicate import DuplicateKind from serato_doctor.models.health import HealthReport @@ -44,6 +46,13 @@ def analyze_health(library: Library) -> HealthReport: bool(matcher.candidates_for(result.reference)) for result in missing ) + database_tracks = library.database.tracks if library.database else () + database_names = {normalize(track.filename) for track in database_tracks} + library_names = {normalize(track.filename) for track in library.tracks} + database_path_counts = Counter( + normalize(str(track.path)) for track in database_tracks + ) + return HealthReport( score=score, total_references=len(library.references), @@ -82,4 +91,18 @@ def analyze_health(library: Library) -> HealthReport: for crate in library.crates if crate.kind is CrateKind.SMART ), + database_present=library.database is not None, + database_entries=len(database_tracks), + database_library_matches=sum( + normalize(track.filename) in library_names for track in database_tracks + ), + database_unmatched_entries=sum( + normalize(track.filename) not in library_names for track in database_tracks + ), + tracks_missing_from_database=sum( + normalize(track.filename) not in database_names for track in library.tracks + ), + duplicate_database_paths=sum( + count - 1 for count in database_path_counts.values() if count > 1 + ), ) diff --git a/serato_doctor/models/__init__.py b/serato_doctor/models/__init__.py index 7522360..1ba7c04 100644 --- a/serato_doctor/models/__init__.py +++ b/serato_doctor/models/__init__.py @@ -1,4 +1,5 @@ from serato_doctor.models.crate import Crate, CrateKind +from serato_doctor.models.database import DatabaseTrack, SeratoDatabase from serato_doctor.models.duplicate import DuplicateGroup, DuplicateKind from serato_doctor.models.filesystem import BrokenSymlink, FilesystemScan from serato_doctor.models.health import HealthReport @@ -10,6 +11,7 @@ from serato_doctor.models.track import DiskTrack __all__ = [ "Crate", "CrateKind", + "DatabaseTrack", "DiskTrack", "DuplicateGroup", "DuplicateKind", @@ -19,6 +21,7 @@ __all__ = [ "Library", "MatchEvidence", "ReferenceResult", + "SeratoDatabase", "TrackMatch", "TrackReference", ] diff --git a/serato_doctor/models/database.py b/serato_doctor/models/database.py new file mode 100644 index 0000000..a636da0 --- /dev/null +++ b/serato_doctor/models/database.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Optional, Tuple + + +@dataclass(frozen=True) +class DatabaseTrack: + """Read-only metadata extracted from one database V2 track record.""" + + path: Path + filename: str + title: Optional[str] = None + artist: Optional[str] = None + album: Optional[str] = None + genre: Optional[str] = None + + +@dataclass(frozen=True) +class SeratoDatabase: + path: Path + version: Optional[str] + tracks: Tuple[DatabaseTrack, ...] diff --git a/serato_doctor/models/health.py b/serato_doctor/models/health.py index 00fa3ab..bdb2bf7 100644 --- a/serato_doctor/models/health.py +++ b/serato_doctor/models/health.py @@ -25,6 +25,12 @@ class HealthReport: smart_crate_containers: int unknown_crates: int dynamic_references_excluded: int + database_present: bool + database_entries: int + database_library_matches: int + database_unmatched_entries: int + tracks_missing_from_database: int + duplicate_database_paths: int @property def score_basis(self) -> str: diff --git a/serato_doctor/models/library.py b/serato_doctor/models/library.py index 6cae56e..73ceb9c 100644 --- a/serato_doctor/models/library.py +++ b/serato_doctor/models/library.py @@ -1,7 +1,8 @@ from dataclasses import dataclass -from typing import Iterable, Tuple +from typing import Iterable, Optional, Tuple from serato_doctor.models.crate import Crate +from serato_doctor.models.database import SeratoDatabase from serato_doctor.models.filesystem import BrokenSymlink from serato_doctor.models.reference import ReferenceResult, TrackReference from serato_doctor.models.track import DiskTrack @@ -15,6 +16,7 @@ class Library: tracks: Tuple[DiskTrack, ...] crates: Tuple[Crate, ...] = () broken_symlinks: Tuple[BrokenSymlink, ...] = () + database: Optional[SeratoDatabase] = None @classmethod def build( @@ -22,11 +24,13 @@ class Library: references: Iterable[TrackReference], tracks: Iterable[DiskTrack], broken_symlinks: Iterable[BrokenSymlink] = (), + database: Optional[SeratoDatabase] = None, ) -> "Library": return cls( tuple(references), tuple(tracks), broken_symlinks=tuple(broken_symlinks), + database=database, ) @classmethod @@ -35,6 +39,7 @@ class Library: crates: Iterable[Crate], tracks: Iterable[DiskTrack], broken_symlinks: Iterable[BrokenSymlink] = (), + database: Optional[SeratoDatabase] = None, ) -> "Library": crate_tuple = tuple(crates) references = tuple( @@ -47,6 +52,7 @@ class Library: tuple(tracks), crate_tuple, tuple(broken_symlinks), + database, ) def reconcile_by_filename(self) -> Tuple[ReferenceResult, ...]: diff --git a/serato_doctor/web.py b/serato_doctor/web.py index 739939a..2eedfcc 100644 --- a/serato_doctor/web.py +++ b/serato_doctor/web.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Iterable from serato_doctor.crate_parser import load_library_crates +from serato_doctor.database_parser import parse_database from serato_doctor.health import analyze_health from serato_doctor.models.library import Library from serato_doctor.scanner import scan_filesystem @@ -33,10 +34,13 @@ def analyze_paths( crates = load_library_crates(serato, reference_roots) filesystem = scan_filesystem(music) + database_path = serato / "database V2" + database = parse_database(database_path) if database_path.is_file() else None library = Library.from_crates( crates, filesystem.tracks, filesystem.broken_symlinks, + database, ) report = analyze_health(library) result = asdict(report) diff --git a/serato_doctor/webui/index.html b/serato_doctor/webui/index.html index 96df962..94f3e48 100644 --- a/serato_doctor/webui/index.html +++ b/serato_doctor/webui/index.html @@ -77,6 +77,8 @@
≋

Duplicate filenames— exact groups · — extra files

⌁

Cloud conflicts— suspected groups · — extra files

↗

Broken symlinks— unresolved links

+
▦

Serato database V2— entries · — match scanned filenames · — scanned tracks absent

+
⌕

Database review— entries outside this music scan · — duplicate paths

diff --git a/tests/test_database_parser.py b/tests/test_database_parser.py new file mode 100644 index 0000000..523ddcd --- /dev/null +++ b/tests/test_database_parser.py @@ -0,0 +1,48 @@ +from pathlib import Path + +from serato_doctor.database_parser import iter_records, parse_database + + +def record(tag, payload): + return tag + len(payload).to_bytes(4, "big") + payload + + +def text_record(tag, value): + return record(tag, value.encode("utf-16-be")) + + +def test_parse_database_reads_track_paths_and_metadata(tmp_path): + track = b"".join( + [ + text_record(b"pfil", "Users/sample/Music/Track.mp3"), + text_record(b"tsng", "Track title"), + text_record(b"tart", "Test artist"), + text_record(b"talb", "Test album"), + text_record(b"tgen", "House"), + ] + ) + database_path = tmp_path / "database V2" + database_path.write_bytes( + text_record(b"vrsn", "2.0/Test Database") + record(b"otrk", track) + ) + + database = parse_database(database_path) + + assert database.version == "2.0/Test Database" + assert len(database.tracks) == 1 + parsed = database.tracks[0] + assert parsed.path == Path("/Users/sample/Music/Track.mp3") + assert parsed.filename == "Track.mp3" + assert parsed.title == "Track title" + assert parsed.artist == "Test artist" + assert parsed.album == "Test album" + assert parsed.genre == "House" + + +def test_iter_records_ignores_incomplete_trailing_record(): + complete = record(b"vrsn", "2.0".encode("utf-16-be")) + incomplete = b"otrk\x00\x00\x00\x10short" + + records = list(iter_records(complete + incomplete)) + + assert records == [(b"vrsn", "2.0".encode("utf-16-be"))] diff --git a/tests/test_web.py b/tests/test_web.py index 2205933..e923a62 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -21,6 +21,9 @@ def test_web_analysis_uses_production_health_pipeline(tmp_path): assert result["missing_references"] == 2 assert result["static_crates"] == 5 assert result["smart_crates"] == 2 + assert result["database_entries"] == 10 + assert result["database_library_matches"] == 10 + assert result["tracks_missing_from_database"] == 0 def test_web_analysis_rejects_missing_folders(tmp_path):