Merge feature/database-v2-parser into develop

ROADMAP.md

@@ -10,7 +10,7 @@
 - [x] HTML health dashboard
 - [x] Test suite
 - [x] Sample library fixtures
-- [ ] Database V2 read-only parser
+- [x] Database V2 read-only parser
 - [x] Configuration
 - [x] Logging
 - [x] Matching engine

docs/design/database-v2-parser.md

@@ -0,0 +1,28 @@
+# Database V2 Read-only Parser
+
+## Problem
+
+Crates and files do not explain every orange track in Serato. The legacy
+`database V2` contains Serato's library-level track paths and metadata, so it must
+be inspected independently from crate references.
+
+## Architecture
+
+The parser reads the file as a big-endian tag-length-value stream. Top-level
+`otrk` records contain nested fields including `pfil` (path), `tsng` (title),
+`tart` (artist), `talb` (album), and `tgen` (genre). Text is UTF-16 big-endian.
+
+Analysis reports total database entries, entries whose filenames occur in the
+selected music scan, entries outside that scan, scanned tracks absent from the
+database, and duplicate database paths. “Outside scan” is deliberately not called
+missing because Serato databases can include samples and tracks from other roots.
+
+## Safety
+
+The parser calls only `read_bytes`; it never opens the database for writing. No
+metadata values or personal paths are sent to logs or the dashboard.
+
+## Verification
+
+Synthetic TLV fixtures cover version, paths, metadata, incomplete records, and
+health integration. The sample library includes a generated ten-entry database.

samples/small-library/generate.py

@@ -10,6 +10,10 @@ SAMPLE_ROOT = Path(__file__).parent
 SERATO_PATH_PREFIX = "Users/sample-user/OneDrive/Jukebox/"
 
 
+def database_record(tag: bytes, payload: bytes) -> bytes:
+    return tag + len(payload).to_bytes(4, "big") + payload
+
+
 def load_manifest() -> dict:
     return json.loads((SAMPLE_ROOT / "manifest.json").read_text(encoding="utf-8"))
 
@@ -51,6 +55,16 @@ def build_sample(output: Optional[Path] = None) -> Path:
         )
         (crate_root / output_name).write_bytes(records.encode("utf-16-le"))
 
+    database_records = [
+        database_record(b"vrsn", "2.0/Serato Doctor Fixture".encode("utf-16-be"))
+    ]
+    for relative_path in manifest["tracks"]:
+        fields = database_record(
+            b"pfil", f"{SERATO_PATH_PREFIX}{relative_path}".encode("utf-16-be")
+        )
+        database_records.append(database_record(b"otrk", fields))
+    (serato_root / "database V2").write_bytes(b"".join(database_records))
+
     return root
 
 

serato_doctor/cli.py

@@ -3,6 +3,7 @@ import argparse
 
 from serato_doctor.config import ScanConfig
 from serato_doctor.crate_parser import load_library_crates
+from serato_doctor.database_parser import parse_database
 from serato_doctor.health import analyze_health
 from serato_doctor.logging import configure_logging
 from serato_doctor.models.library import Library
@@ -57,10 +58,13 @@ def main():
 
     crates = load_library_crates(config.serato, config.reference_roots)
     filesystem = scan_filesystem(config.music)
+    database_path = config.serato / "database V2"
+    database = parse_database(database_path) if database_path.is_file() else None
     library = Library.from_crates(
         crates=crates,
         tracks=filesystem.tracks,
         broken_symlinks=filesystem.broken_symlinks,
+        database=database,
     )
     results = library.reconcile_by_filename()
     missing_count = sum(1 for result in results if not result.exists_by_filename)
@@ -99,6 +103,11 @@ def main():
         print(
             f"Dynamic References Excluded: {health.dynamic_references_excluded}"
         )
+        print(f"Database Entries: {health.database_entries}")
+        print(f"Database / Library Matches: {health.database_library_matches}")
+        print(f"Database Entries Outside Scan: {health.database_unmatched_entries}")
+        print(f"Tracks Missing From Database: {health.tracks_missing_from_database}")
+        print(f"Duplicate Database Paths: {health.duplicate_database_paths}")
     else:
         write_csv(results, config.out)
         write_missing_report(results, config.report)

serato_doctor/database_parser.py

@@ -0,0 +1,67 @@
+from pathlib import Path
+from typing import Dict, Iterator, Optional, Tuple
+
+from serato_doctor.models.database import DatabaseTrack, SeratoDatabase
+
+
+TEXT_FIELDS = {
+    b"tsng": "title",
+    b"tart": "artist",
+    b"talb": "album",
+    b"tgen": "genre",
+}
+
+
+def iter_records(data: bytes) -> Iterator[Tuple[bytes, bytes]]:
+    """Yield complete big-endian tag-length-value records."""
+
+    offset = 0
+    while offset + 8 <= len(data):
+        tag = data[offset : offset + 4]
+        length = int.from_bytes(data[offset + 4 : offset + 8], "big")
+        payload_start = offset + 8
+        payload_end = payload_start + length
+        if payload_end > len(data):
+            break
+        yield tag, data[payload_start:payload_end]
+        offset = payload_end
+
+
+def decode_text(payload: bytes) -> Optional[str]:
+    value = payload.decode("utf-16-be", errors="ignore").strip("\x00").strip()
+    return value or None
+
+
+def normalize_database_path(value: str) -> Path:
+    if value.startswith(("Users/", "Volumes/")):
+        value = "/" + value
+    return Path(value)
+
+
+def parse_track(payload: bytes) -> Optional[DatabaseTrack]:
+    fields: Dict[str, Optional[str]] = {}
+    path = None
+    for tag, value in iter_records(payload):
+        if tag == b"pfil":
+            decoded_path = decode_text(value)
+            if decoded_path:
+                path = normalize_database_path(decoded_path)
+        elif tag in TEXT_FIELDS:
+            fields[TEXT_FIELDS[tag]] = decode_text(value)
+    if path is None:
+        return None
+    return DatabaseTrack(path=path, filename=path.name, **fields)
+
+
+def parse_database(database_path: Path) -> SeratoDatabase:
+    data = database_path.read_bytes()
+    version = None
+    tracks = []
+    for tag, payload in iter_records(data):
+        if tag == b"vrsn":
+            version = decode_text(payload)
+        elif tag == b"otrk":
+            track = parse_track(payload)
+            if track is not None:
+                tracks.append(track)
+    return SeratoDatabase(database_path, version, tuple(tracks))

serato_doctor/health.py

@@ -1,5 +1,7 @@
+from collections import Counter
+
 from serato_doctor.duplicates import find_duplicate_groups
-from serato_doctor.matching import MatchingEngine
+from serato_doctor.matching import MatchingEngine, normalize
 from serato_doctor.models.crate import CrateKind
 from serato_doctor.models.duplicate import DuplicateKind
 from serato_doctor.models.health import HealthReport
@@ -44,6 +46,13 @@ def analyze_health(library: Library) -> HealthReport:
         bool(matcher.candidates_for(result.reference)) for result in missing
     )
 
+    database_tracks = library.database.tracks if library.database else ()
+    database_names = {normalize(track.filename) for track in database_tracks}
+    library_names = {normalize(track.filename) for track in library.tracks}
+    database_path_counts = Counter(
+        normalize(str(track.path)) for track in database_tracks
+    )
+
     return HealthReport(
         score=score,
         total_references=len(library.references),
@@ -82,4 +91,18 @@ def analyze_health(library: Library) -> HealthReport:
             for crate in library.crates
             if crate.kind is CrateKind.SMART
         ),
+        database_present=library.database is not None,
+        database_entries=len(database_tracks),
+        database_library_matches=sum(
+            normalize(track.filename) in library_names for track in database_tracks
+        ),
+        database_unmatched_entries=sum(
+            normalize(track.filename) not in library_names for track in database_tracks
+        ),
+        tracks_missing_from_database=sum(
+            normalize(track.filename) not in database_names for track in library.tracks
+        ),
+        duplicate_database_paths=sum(
+            count - 1 for count in database_path_counts.values() if count > 1
+        ),
     )

serato_doctor/models/__init__.py

@@ -1,4 +1,5 @@
 from serato_doctor.models.crate import Crate, CrateKind
+from serato_doctor.models.database import DatabaseTrack, SeratoDatabase
 from serato_doctor.models.duplicate import DuplicateGroup, DuplicateKind
 from serato_doctor.models.filesystem import BrokenSymlink, FilesystemScan
 from serato_doctor.models.health import HealthReport
@@ -10,6 +11,7 @@ from serato_doctor.models.track import DiskTrack
 __all__ = [
     "Crate",
     "CrateKind",
+    "DatabaseTrack",
     "DiskTrack",
     "DuplicateGroup",
     "DuplicateKind",
@@ -19,6 +21,7 @@ __all__ = [
     "Library",
     "MatchEvidence",
     "ReferenceResult",
+    "SeratoDatabase",
     "TrackMatch",
     "TrackReference",
 ]

serato_doctor/models/database.py

@@ -0,0 +1,22 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional, Tuple
+
+
+@dataclass(frozen=True)
+class DatabaseTrack:
+    """Read-only metadata extracted from one database V2 track record."""
+
+    path: Path
+    filename: str
+    title: Optional[str] = None
+    artist: Optional[str] = None
+    album: Optional[str] = None
+    genre: Optional[str] = None
+
+
+@dataclass(frozen=True)
+class SeratoDatabase:
+    path: Path
+    version: Optional[str]
+    tracks: Tuple[DatabaseTrack, ...]

serato_doctor/models/health.py

@@ -25,6 +25,12 @@ class HealthReport:
     smart_crate_containers: int
     unknown_crates: int
     dynamic_references_excluded: int
+    database_present: bool
+    database_entries: int
+    database_library_matches: int
+    database_unmatched_entries: int
+    tracks_missing_from_database: int
+    duplicate_database_paths: int
 
     @property
     def score_basis(self) -> str:

serato_doctor/models/library.py

@@ -1,7 +1,8 @@
 from dataclasses import dataclass
-from typing import Iterable, Tuple
+from typing import Iterable, Optional, Tuple
 
 from serato_doctor.models.crate import Crate
+from serato_doctor.models.database import SeratoDatabase
 from serato_doctor.models.filesystem import BrokenSymlink
 from serato_doctor.models.reference import ReferenceResult, TrackReference
 from serato_doctor.models.track import DiskTrack
@@ -15,6 +16,7 @@ class Library:
     tracks: Tuple[DiskTrack, ...]
     crates: Tuple[Crate, ...] = ()
     broken_symlinks: Tuple[BrokenSymlink, ...] = ()
+    database: Optional[SeratoDatabase] = None
 
     @classmethod
     def build(
@@ -22,11 +24,13 @@ class Library:
         references: Iterable[TrackReference],
         tracks: Iterable[DiskTrack],
         broken_symlinks: Iterable[BrokenSymlink] = (),
+        database: Optional[SeratoDatabase] = None,
     ) -> "Library":
         return cls(
             tuple(references),
             tuple(tracks),
             broken_symlinks=tuple(broken_symlinks),
+            database=database,
         )
 
     @classmethod
@@ -35,6 +39,7 @@ class Library:
         crates: Iterable[Crate],
         tracks: Iterable[DiskTrack],
         broken_symlinks: Iterable[BrokenSymlink] = (),
+        database: Optional[SeratoDatabase] = None,
     ) -> "Library":
         crate_tuple = tuple(crates)
         references = tuple(
@@ -47,6 +52,7 @@ class Library:
             tuple(tracks),
             crate_tuple,
             tuple(broken_symlinks),
+            database,
         )
 
     def reconcile_by_filename(self) -> Tuple[ReferenceResult, ...]:

serato_doctor/web.py

@@ -7,6 +7,7 @@ from pathlib import Path
 from typing import Iterable
 
 from serato_doctor.crate_parser import load_library_crates
+from serato_doctor.database_parser import parse_database
 from serato_doctor.health import analyze_health
 from serato_doctor.models.library import Library
 from serato_doctor.scanner import scan_filesystem
@@ -33,10 +34,13 @@ def analyze_paths(
 
     crates = load_library_crates(serato, reference_roots)
     filesystem = scan_filesystem(music)
+    database_path = serato / "database V2"
+    database = parse_database(database_path) if database_path.is_file() else None
     library = Library.from_crates(
         crates,
         filesystem.tracks,
         filesystem.broken_symlinks,
+        database,
     )
     report = analyze_health(library)
     result = asdict(report)

serato_doctor/webui/index.html

@@ -77,6 +77,8 @@
             <div><span class="diag-icon amber">≋</span><p><strong>Duplicate filenames</strong><small><b data-field="duplicate_filename_groups">—</b> exact groups · <b data-field="duplicate_files">—</b> extra files</small></p></div>
             <div><span class="diag-icon blue">⌁</span><p><strong>Cloud conflicts</strong><small><b data-field="suspected_cloud_conflict_groups">—</b> suspected groups · <b data-field="suspected_cloud_conflict_files">—</b> extra files</small></p></div>
             <div><span class="diag-icon red">↗</span><p><strong>Broken symlinks</strong><small><b data-field="broken_symlinks">—</b> unresolved links</small></p></div>
+            <div><span class="diag-icon violet">▦</span><p><strong>Serato database V2</strong><small><b data-field="database_entries">—</b> entries · <b data-field="database_library_matches">—</b> match scanned filenames · <b data-field="tracks_missing_from_database">—</b> scanned tracks absent</small></p></div>
+            <div><span class="diag-icon amber">⌕</span><p><strong>Database review</strong><small><b data-field="database_unmatched_entries">—</b> entries outside this music scan · <b data-field="duplicate_database_paths">—</b> duplicate paths</small></p></div>
           </div>
         </div>
       </section>

tests/test_database_parser.py

@@ -0,0 +1,48 @@
+from pathlib import Path
+
+from serato_doctor.database_parser import iter_records, parse_database
+
+
+def record(tag, payload):
+    return tag + len(payload).to_bytes(4, "big") + payload
+
+
+def text_record(tag, value):
+    return record(tag, value.encode("utf-16-be"))
+
+
+def test_parse_database_reads_track_paths_and_metadata(tmp_path):
+    track = b"".join(
+        [
+            text_record(b"pfil", "Users/sample/Music/Track.mp3"),
+            text_record(b"tsng", "Track title"),
+            text_record(b"tart", "Test artist"),
+            text_record(b"talb", "Test album"),
+            text_record(b"tgen", "House"),
+        ]
+    )
+    database_path = tmp_path / "database V2"
+    database_path.write_bytes(
+        text_record(b"vrsn", "2.0/Test Database") + record(b"otrk", track)
+    )
+
+    database = parse_database(database_path)
+
+    assert database.version == "2.0/Test Database"
+    assert len(database.tracks) == 1
+    parsed = database.tracks[0]
+    assert parsed.path == Path("/Users/sample/Music/Track.mp3")
+    assert parsed.filename == "Track.mp3"
+    assert parsed.title == "Track title"
+    assert parsed.artist == "Test artist"
+    assert parsed.album == "Test album"
+    assert parsed.genre == "House"
+
+
+def test_iter_records_ignores_incomplete_trailing_record():
+    complete = record(b"vrsn", "2.0".encode("utf-16-be"))
+    incomplete = b"otrk\x00\x00\x00\x10short"
+
+    records = list(iter_records(complete + incomplete))
+
+    assert records == [(b"vrsn", "2.0".encode("utf-16-be"))]

tests/test_web.py

@@ -21,6 +21,9 @@ def test_web_analysis_uses_production_health_pipeline(tmp_path):
     assert result["missing_references"] == 2
     assert result["static_crates"] == 5
     assert result["smart_crates"] == 2
+    assert result["database_entries"] == 10
+    assert result["database_library_matches"] == 10
+    assert result["tracks_missing_from_database"] == 0
 
 
 def test_web_analysis_rejects_missing_folders(tmp_path):