Compare commits

..

5 Commits

Author SHA1 Message Date
Philip Guzman ce37b45058 Merge feature/database-v2-parser into develop 2026-07-01 08:30:53 -07:00
Philip Guzman 37786612b7 Add read-only database V2 parser 2026-07-01 08:13:20 -07:00
Philip Guzman 8ff4db4951 Merge feature/smart-crate-discovery into develop 2026-07-01 08:08:43 -07:00
Philip Guzman a384cdc88d Discover Serato smart crate definitions 2026-07-01 08:06:19 -07:00
Philip Guzman 71bca064ed Merge feature/web-interface into develop 2026-07-01 08:03:19 -07:00
21 changed files with 355 additions and 21 deletions
+1 -1
View File
@@ -10,7 +10,7 @@
- [x] HTML health dashboard
- [x] Test suite
- [x] Sample library fixtures
- [ ] Database V2 read-only parser
- [x] Database V2 read-only parser
- [x] Configuration
- [x] Logging
- [x] Matching engine
+7 -2
View File
@@ -9,10 +9,15 @@ broken manual references or given the same health-score weight.
## Architecture
Crates carry a `static`, `smart`, or `unknown` kind. Folder provenance is the
primary signal: Serato stores regular definitions in `Subcrates` and smart
definitions in `Smartcrates`. `Compatible by key.crate` is also treated as smart
primary signal: Serato stores regular `.crate` files in `Subcrates` and smart
`.scrate` definitions in `SmartCrates`. `Compatible by key.crate` is also treated as smart
when encountered in `Subcrates`, based on the original migration case study.
Smart crate names use `≫≫` to encode hierarchy. The model preserves those segments,
so `Compatible by key≫≫10A.scrate` has a parent of `Compatible by key` and a display
name of `10A`. Smart definitions and dynamic `.crate` containers are counted
separately.
The library loader reads both folders. Health analysis reports all references but
scores only non-smart references. Unknown crates remain scoreable so incomplete
classification cannot silently hide potential problems.
+28
View File
@@ -0,0 +1,28 @@
# Database V2 Read-only Parser
## Problem
Crates and files do not explain every orange track in Serato. The legacy
`database V2` contains Serato's library-level track paths and metadata, so it must
be inspected independently from crate references.
## Architecture
The parser reads the file as a big-endian tag-length-value stream. Top-level
`otrk` records contain nested fields including `pfil` (path), `tsng` (title),
`tart` (artist), `talb` (album), and `tgen` (genre). Text is UTF-16 big-endian.
Analysis reports total database entries, entries whose filenames occur in the
selected music scan, entries outside that scan, scanned tracks absent from the
database, and duplicate database paths. “Outside scan” is deliberately not called
missing because Serato databases can include samples and tracks from other roots.
## Safety
The parser calls only `read_bytes`; it never opens the database for writing. No
metadata values or personal paths are sent to logs or the dashboard.
## Verification
Synthetic TLV fixtures cover version, paths, metadata, incomplete records, and
health integration. The sample library includes a generated ten-entry database.
+19
View File
@@ -0,0 +1,19 @@
# Smart Crate Discovery Correction
## Problem
The first classifier searched `SmartCrates` for `*.crate`. Real Serato smart-crate
definitions use `.scrate`, so a library with 26 definitions displayed only the
single name-based `Compatible by key.crate` fallback.
## Correction
Library discovery now reads `.scrate` definitions case-insensitively from the
`SmartCrates` folder. Dynamic `.crate` containers remain excluded from manual
reference scoring but are reported separately. The `≫≫` filename separator is
preserved as smart-crate hierarchy metadata.
## Verification
Synthetic tests cover `.scrate` discovery, case-correct folder names, hierarchy,
definition/container counts, and the existing five-static/two-smart sample.
+30 -6
View File
@@ -10,6 +10,10 @@ SAMPLE_ROOT = Path(__file__).parent
SERATO_PATH_PREFIX = "Users/sample-user/OneDrive/Jukebox/"
def database_record(tag: bytes, payload: bytes) -> bytes:
return tag + len(payload).to_bytes(4, "big") + payload
def load_manifest() -> dict:
return json.loads((SAMPLE_ROOT / "manifest.json").read_text(encoding="utf-8"))
@@ -28,18 +32,38 @@ def build_sample(output: Optional[Path] = None) -> Path:
)
for crate in manifest["crates"]:
folder_name = "Smartcrates" if crate["type"] == "smart" else "Subcrates"
is_smart = crate["type"] == "smart"
folder_name = "SmartCrates" if is_smart else "Subcrates"
crate_root = serato_root / folder_name
crate_root.mkdir(parents=True, exist_ok=True)
other_folder = "Subcrates" if folder_name == "Smartcrates" else "Smartcrates"
stale_path = serato_root / other_folder / crate["name"]
if stale_path.exists():
stale_path.unlink()
output_name = (
Path(crate["name"]).with_suffix(".scrate").name
if is_smart
else crate["name"]
)
for other_folder in ("Subcrates", "Smartcrates", "SmartCrates"):
for stale_name in (
crate["name"],
Path(crate["name"]).with_suffix(".scrate").name,
):
stale_path = serato_root / other_folder / stale_name
if stale_path.exists() and stale_path != crate_root / output_name:
stale_path.unlink()
records = "".join(
f"{SERATO_PATH_PREFIX}{relative_path}otrk"
for relative_path in crate["references"]
)
(crate_root / crate["name"]).write_bytes(records.encode("utf-16-le"))
(crate_root / output_name).write_bytes(records.encode("utf-16-le"))
database_records = [
database_record(b"vrsn", "2.0/Serato Doctor Fixture".encode("utf-16-be"))
]
for relative_path in manifest["tracks"]:
fields = database_record(
b"pfil", f"{SERATO_PATH_PREFIX}{relative_path}".encode("utf-16-be")
)
database_records.append(database_record(b"otrk", fields))
(serato_root / "database V2").write_bytes(b"".join(database_records))
return root
+10
View File
@@ -3,6 +3,7 @@ import argparse
from serato_doctor.config import ScanConfig
from serato_doctor.crate_parser import load_library_crates
from serato_doctor.database_parser import parse_database
from serato_doctor.health import analyze_health
from serato_doctor.logging import configure_logging
from serato_doctor.models.library import Library
@@ -57,10 +58,13 @@ def main():
crates = load_library_crates(config.serato, config.reference_roots)
filesystem = scan_filesystem(config.music)
database_path = config.serato / "database V2"
database = parse_database(database_path) if database_path.is_file() else None
library = Library.from_crates(
crates=crates,
tracks=filesystem.tracks,
broken_symlinks=filesystem.broken_symlinks,
database=database,
)
results = library.reconcile_by_filename()
missing_count = sum(1 for result in results if not result.exists_by_filename)
@@ -95,9 +99,15 @@ def main():
print(f"Broken Symlinks: {health.broken_symlinks}")
print(f"Static Crates: {health.static_crates}")
print(f"Smart Crates: {health.smart_crates}")
print(f"Smart Crate Containers: {health.smart_crate_containers}")
print(
f"Dynamic References Excluded: {health.dynamic_references_excluded}"
)
print(f"Database Entries: {health.database_entries}")
print(f"Database / Library Matches: {health.database_library_matches}")
print(f"Database Entries Outside Scan: {health.database_unmatched_entries}")
print(f"Tracks Missing From Database: {health.tracks_missing_from_database}")
print(f"Duplicate Database Paths: {health.duplicate_database_paths}")
else:
write_csv(results, config.out)
write_missing_report(results, config.report)
+14 -4
View File
@@ -103,9 +103,19 @@ def load_library_crates(
serato_root: Path, reference_roots: Iterable[Path] = ()
) -> Tuple[Crate, ...]:
reference_roots = tuple(reference_roots)
if not serato_root.is_dir():
return ()
crates = []
for folder_name in ("Subcrates", "Smartcrates"):
folder = serato_root / folder_name
for crate_path in folder.rglob("*.crate"):
crates.append(load_crate(crate_path, reference_roots))
folder_patterns = {
"subcrates": ("*.crate",),
"smartcrates": ("*.scrate", "*.crate"),
}
for folder in serato_root.iterdir():
patterns = folder_patterns.get(folder.name.casefold())
if patterns is None or not folder.is_dir():
continue
for pattern in patterns:
for crate_path in folder.rglob(pattern):
crates.append(load_crate(crate_path, reference_roots))
return tuple(sorted(crates, key=lambda crate: str(crate.path)))
+67
View File
@@ -0,0 +1,67 @@
from pathlib import Path
from typing import Dict, Iterator, Optional, Tuple
from serato_doctor.models.database import DatabaseTrack, SeratoDatabase
TEXT_FIELDS = {
b"tsng": "title",
b"tart": "artist",
b"talb": "album",
b"tgen": "genre",
}
def iter_records(data: bytes) -> Iterator[Tuple[bytes, bytes]]:
"""Yield complete big-endian tag-length-value records."""
offset = 0
while offset + 8 <= len(data):
tag = data[offset : offset + 4]
length = int.from_bytes(data[offset + 4 : offset + 8], "big")
payload_start = offset + 8
payload_end = payload_start + length
if payload_end > len(data):
break
yield tag, data[payload_start:payload_end]
offset = payload_end
def decode_text(payload: bytes) -> Optional[str]:
value = payload.decode("utf-16-be", errors="ignore").strip("\x00").strip()
return value or None
def normalize_database_path(value: str) -> Path:
if value.startswith(("Users/", "Volumes/")):
value = "/" + value
return Path(value)
def parse_track(payload: bytes) -> Optional[DatabaseTrack]:
fields: Dict[str, Optional[str]] = {}
path = None
for tag, value in iter_records(payload):
if tag == b"pfil":
decoded_path = decode_text(value)
if decoded_path:
path = normalize_database_path(decoded_path)
elif tag in TEXT_FIELDS:
fields[TEXT_FIELDS[tag]] = decode_text(value)
if path is None:
return None
return DatabaseTrack(path=path, filename=path.name, **fields)
def parse_database(database_path: Path) -> SeratoDatabase:
data = database_path.read_bytes()
version = None
tracks = []
for tag, payload in iter_records(data):
if tag == b"vrsn":
version = decode_text(payload)
elif tag == b"otrk":
track = parse_track(payload)
if track is not None:
tracks.append(track)
return SeratoDatabase(database_path, version, tuple(tracks))
+30 -2
View File
@@ -1,5 +1,7 @@
from collections import Counter
from serato_doctor.duplicates import find_duplicate_groups
from serato_doctor.matching import MatchingEngine
from serato_doctor.matching import MatchingEngine, normalize
from serato_doctor.models.crate import CrateKind
from serato_doctor.models.duplicate import DuplicateKind
from serato_doctor.models.health import HealthReport
@@ -44,6 +46,13 @@ def analyze_health(library: Library) -> HealthReport:
bool(matcher.candidates_for(result.reference)) for result in missing
)
database_tracks = library.database.tracks if library.database else ()
database_names = {normalize(track.filename) for track in database_tracks}
library_names = {normalize(track.filename) for track in library.tracks}
database_path_counts = Counter(
normalize(str(track.path)) for track in database_tracks
)
return HealthReport(
score=score,
total_references=len(library.references),
@@ -67,7 +76,12 @@ def analyze_health(library: Library) -> HealthReport:
crate.kind is CrateKind.STATIC for crate in library.crates
),
smart_crates=sum(
crate.kind is CrateKind.SMART for crate in library.crates
crate.kind is CrateKind.SMART and crate.is_smart_definition
for crate in library.crates
),
smart_crate_containers=sum(
crate.kind is CrateKind.SMART and not crate.is_smart_definition
for crate in library.crates
),
unknown_crates=sum(
crate.kind is CrateKind.UNKNOWN for crate in library.crates
@@ -77,4 +91,18 @@ def analyze_health(library: Library) -> HealthReport:
for crate in library.crates
if crate.kind is CrateKind.SMART
),
database_present=library.database is not None,
database_entries=len(database_tracks),
database_library_matches=sum(
normalize(track.filename) in library_names for track in database_tracks
),
database_unmatched_entries=sum(
normalize(track.filename) not in library_names for track in database_tracks
),
tracks_missing_from_database=sum(
normalize(track.filename) not in database_names for track in library.tracks
),
duplicate_database_paths=sum(
count - 1 for count in database_path_counts.values() if count > 1
),
)
+3
View File
@@ -1,4 +1,5 @@
from serato_doctor.models.crate import Crate, CrateKind
from serato_doctor.models.database import DatabaseTrack, SeratoDatabase
from serato_doctor.models.duplicate import DuplicateGroup, DuplicateKind
from serato_doctor.models.filesystem import BrokenSymlink, FilesystemScan
from serato_doctor.models.health import HealthReport
@@ -10,6 +11,7 @@ from serato_doctor.models.track import DiskTrack
__all__ = [
"Crate",
"CrateKind",
"DatabaseTrack",
"DiskTrack",
"DuplicateGroup",
"DuplicateKind",
@@ -19,6 +21,7 @@ __all__ = [
"Library",
"MatchEvidence",
"ReferenceResult",
"SeratoDatabase",
"TrackMatch",
"TrackReference",
]
+12
View File
@@ -19,3 +19,15 @@ class Crate:
path: Path
references: Tuple[TrackReference, ...]
kind: CrateKind = CrateKind.UNKNOWN
@property
def hierarchy(self) -> Tuple[str, ...]:
return tuple(self.path.stem.split("≫≫"))
@property
def display_name(self) -> str:
return self.hierarchy[-1]
@property
def is_smart_definition(self) -> bool:
return self.path.suffix.casefold() == ".scrate"
+22
View File
@@ -0,0 +1,22 @@
from dataclasses import dataclass
from pathlib import Path
from typing import Optional, Tuple
@dataclass(frozen=True)
class DatabaseTrack:
"""Read-only metadata extracted from one database V2 track record."""
path: Path
filename: str
title: Optional[str] = None
artist: Optional[str] = None
album: Optional[str] = None
genre: Optional[str] = None
@dataclass(frozen=True)
class SeratoDatabase:
path: Path
version: Optional[str]
tracks: Tuple[DatabaseTrack, ...]
+7
View File
@@ -22,8 +22,15 @@ class HealthReport:
broken_symlinks: int
static_crates: int
smart_crates: int
smart_crate_containers: int
unknown_crates: int
dynamic_references_excluded: int
database_present: bool
database_entries: int
database_library_matches: int
database_unmatched_entries: int
tracks_missing_from_database: int
duplicate_database_paths: int
@property
def score_basis(self) -> str:
+7 -1
View File
@@ -1,7 +1,8 @@
from dataclasses import dataclass
from typing import Iterable, Tuple
from typing import Iterable, Optional, Tuple
from serato_doctor.models.crate import Crate
from serato_doctor.models.database import SeratoDatabase
from serato_doctor.models.filesystem import BrokenSymlink
from serato_doctor.models.reference import ReferenceResult, TrackReference
from serato_doctor.models.track import DiskTrack
@@ -15,6 +16,7 @@ class Library:
tracks: Tuple[DiskTrack, ...]
crates: Tuple[Crate, ...] = ()
broken_symlinks: Tuple[BrokenSymlink, ...] = ()
database: Optional[SeratoDatabase] = None
@classmethod
def build(
@@ -22,11 +24,13 @@ class Library:
references: Iterable[TrackReference],
tracks: Iterable[DiskTrack],
broken_symlinks: Iterable[BrokenSymlink] = (),
database: Optional[SeratoDatabase] = None,
) -> "Library":
return cls(
tuple(references),
tuple(tracks),
broken_symlinks=tuple(broken_symlinks),
database=database,
)
@classmethod
@@ -35,6 +39,7 @@ class Library:
crates: Iterable[Crate],
tracks: Iterable[DiskTrack],
broken_symlinks: Iterable[BrokenSymlink] = (),
database: Optional[SeratoDatabase] = None,
) -> "Library":
crate_tuple = tuple(crates)
references = tuple(
@@ -47,6 +52,7 @@ class Library:
tuple(tracks),
crate_tuple,
tuple(broken_symlinks),
database,
)
def reconcile_by_filename(self) -> Tuple[ReferenceResult, ...]:
+4
View File
@@ -7,6 +7,7 @@ from pathlib import Path
from typing import Iterable
from serato_doctor.crate_parser import load_library_crates
from serato_doctor.database_parser import parse_database
from serato_doctor.health import analyze_health
from serato_doctor.models.library import Library
from serato_doctor.scanner import scan_filesystem
@@ -33,10 +34,13 @@ def analyze_paths(
crates = load_library_crates(serato, reference_roots)
filesystem = scan_filesystem(music)
database_path = serato / "database V2"
database = parse_database(database_path) if database_path.is_file() else None
library = Library.from_crates(
crates,
filesystem.tracks,
filesystem.broken_symlinks,
database,
)
report = analyze_health(library)
result = asdict(report)
+3 -1
View File
@@ -73,10 +73,12 @@
<div id="diagnostics" class="diagnostics panel">
<div class="section-heading"><div><p class="eyebrow">Full picture</p><h2>Diagnostics</h2></div><span class="read-only-tag">No changes made</span></div>
<div class="diagnostic-list">
<div><span class="diag-icon violet"></span><p><strong>Crates</strong><small><b data-field="static_crates"></b> static · <b data-field="smart_crates"></b> smart · <b data-field="dynamic_references_excluded"></b> dynamic references excluded</small></p></div>
<div><span class="diag-icon violet"></span><p><strong>Crates</strong><small><b data-field="static_crates"></b> static · <b data-field="smart_crates"></b> smart · <b data-field="smart_crate_containers"></b> dynamic containers · <b data-field="dynamic_references_excluded"></b> references excluded</small></p></div>
<div><span class="diag-icon amber"></span><p><strong>Duplicate filenames</strong><small><b data-field="duplicate_filename_groups"></b> exact groups · <b data-field="duplicate_files"></b> extra files</small></p></div>
<div><span class="diag-icon blue"></span><p><strong>Cloud conflicts</strong><small><b data-field="suspected_cloud_conflict_groups"></b> suspected groups · <b data-field="suspected_cloud_conflict_files"></b> extra files</small></p></div>
<div><span class="diag-icon red"></span><p><strong>Broken symlinks</strong><small><b data-field="broken_symlinks"></b> unresolved links</small></p></div>
<div><span class="diag-icon violet"></span><p><strong>Serato database V2</strong><small><b data-field="database_entries"></b> entries · <b data-field="database_library_matches"></b> match scanned filenames · <b data-field="tracks_missing_from_database"></b> scanned tracks absent</small></p></div>
<div><span class="diag-icon amber"></span><p><strong>Database review</strong><small><b data-field="database_unmatched_entries"></b> entries outside this music scan · <b data-field="duplicate_database_paths"></b> duplicate paths</small></p></div>
</div>
</div>
</section>
+34 -1
View File
@@ -6,6 +6,7 @@ from serato_doctor.crate_parser import (
classify_crate,
clean_path,
load_crate,
load_library_crates,
parse_crate,
parse_crates,
path_markers,
@@ -92,7 +93,7 @@ def test_parse_crates_reuses_configured_roots_for_every_crate(tmp_path):
("relative_path", "expected"),
[
("_Serato_/Subcrates/House.crate", CrateKind.STATIC),
("_Serato_/Smartcrates/Warmup.crate", CrateKind.SMART),
("_Serato_/SmartCrates/Warmup.scrate", CrateKind.SMART),
("_Serato_/Subcrates/Compatible by key.crate", CrateKind.SMART),
("fixtures/Unknown.crate", CrateKind.UNKNOWN),
],
@@ -101,3 +102,35 @@ def test_classify_crate_uses_provenance_and_known_dynamic_name(
tmp_path, relative_path, expected
):
assert classify_crate(tmp_path / relative_path) is expected
def test_smart_crate_preserves_encoded_hierarchy(tmp_path):
crate_path = (
tmp_path / "_Serato_" / "SmartCrates" / "Compatible by key≫≫10A.scrate"
)
crate_path.parent.mkdir(parents=True)
crate_path.write_bytes(b"")
crate = load_crate(crate_path)
assert crate.kind is CrateKind.SMART
assert crate.hierarchy == ("Compatible by key", "10A")
assert crate.display_name == "10A"
assert crate.is_smart_definition
def test_load_library_crates_discovers_scrate_definitions(tmp_path):
smart_folder = tmp_path / "SmartCrates"
static_folder = tmp_path / "Subcrates"
smart_folder.mkdir()
static_folder.mkdir()
(smart_folder / "New EDM.scrate").write_bytes(b"")
(smart_folder / "Re-Drums.scrate").write_bytes(b"")
(static_folder / "House.crate").write_bytes(b"")
crates = load_library_crates(tmp_path)
assert [(crate.path.name, crate.kind) for crate in crates] == [
("New EDM.scrate", CrateKind.SMART),
("Re-Drums.scrate", CrateKind.SMART),
("House.crate", CrateKind.STATIC),
]
+48
View File
@@ -0,0 +1,48 @@
from pathlib import Path
from serato_doctor.database_parser import iter_records, parse_database
def record(tag, payload):
return tag + len(payload).to_bytes(4, "big") + payload
def text_record(tag, value):
return record(tag, value.encode("utf-16-be"))
def test_parse_database_reads_track_paths_and_metadata(tmp_path):
track = b"".join(
[
text_record(b"pfil", "Users/sample/Music/Track.mp3"),
text_record(b"tsng", "Track title"),
text_record(b"tart", "Test artist"),
text_record(b"talb", "Test album"),
text_record(b"tgen", "House"),
]
)
database_path = tmp_path / "database V2"
database_path.write_bytes(
text_record(b"vrsn", "2.0/Test Database") + record(b"otrk", track)
)
database = parse_database(database_path)
assert database.version == "2.0/Test Database"
assert len(database.tracks) == 1
parsed = database.tracks[0]
assert parsed.path == Path("/Users/sample/Music/Track.mp3")
assert parsed.filename == "Track.mp3"
assert parsed.title == "Track title"
assert parsed.artist == "Test artist"
assert parsed.album == "Test album"
assert parsed.genre == "House"
def test_iter_records_ignores_incomplete_trailing_record():
complete = record(b"vrsn", "2.0".encode("utf-16-be"))
incomplete = b"otrk\x00\x00\x00\x10short"
records = list(iter_records(complete + incomplete))
assert records == [(b"vrsn", "2.0".encode("utf-16-be"))]
+3 -2
View File
@@ -63,14 +63,14 @@ def test_empty_library_has_no_health_score():
def test_smart_crate_references_are_reported_but_not_scored():
static_reference = reference("Found.mp3")
smart_reference = TrackReference(
Path("Smartcrates/Dynamic.crate"),
Path("SmartCrates/Dynamic.scrate"),
Path("/old/House/Dynamic.mp3"),
"Dynamic.mp3",
)
crates = [
Crate(Path("Subcrates/Static.crate"), (static_reference,), CrateKind.STATIC),
Crate(
Path("Smartcrates/Dynamic.crate"),
Path("SmartCrates/Dynamic.scrate"),
(smart_reference,),
CrateKind.SMART,
),
@@ -85,6 +85,7 @@ def test_smart_crate_references_are_reported_but_not_scored():
assert report.missing_references == 0
assert report.static_crates == 1
assert report.smart_crates == 1
assert report.smart_crate_containers == 0
assert report.dynamic_references_excluded == 1
+3 -1
View File
@@ -24,7 +24,9 @@ def test_generated_sample_library_has_expected_scenario(tmp_path):
if not result.exists_by_filename
}
assert len(list((sample_root / "Serato").rglob("*.crate"))) == 7
crate_files = list((sample_root / "Serato").rglob("*.crate"))
smart_files = list((sample_root / "Serato").rglob("*.scrate"))
assert len(crate_files) + len(smart_files) == 7
assert len(library.references) == 13
assert len(tracks) == 10
assert missing == {"Missing.mp3", "Old Name.mp3"}
+3
View File
@@ -21,6 +21,9 @@ def test_web_analysis_uses_production_health_pipeline(tmp_path):
assert result["missing_references"] == 2
assert result["static_crates"] == 5
assert result["smart_crates"] == 2
assert result["database_entries"] == 10
assert result["database_library_matches"] == 10
assert result["tracks_missing_from_database"] == 0
def test_web_analysis_rejects_missing_folders(tmp_path):