Merge feature/crate-classification into develop

ROADMAP.md

@@ -23,7 +23,7 @@
 - [ ] Broken symlink detection
 - [ ] Orphaned audio detection
 - [ ] OneDrive rename detection
-- [ ] Crate classification: static vs smart/dynamic
+- [x] Crate classification: static vs smart/dynamic
 - [x] Library health score
 
 ## v0.3 — Safe Repair

docs/design/crate-classification.md

@@ -0,0 +1,33 @@
+# Crate Classification
+
+## Problem
+
+Static crates are manually maintained track lists. Smart crates are dynamic views
+generated from rules, so stale-looking entries in them should not be presented as
+broken manual references or given the same health-score weight.
+
+## Architecture
+
+Crates carry a `static`, `smart`, or `unknown` kind. Folder provenance is the
+primary signal: Serato stores regular definitions in `Subcrates` and smart
+definitions in `Smartcrates`. `Compatible by key.crate` is also treated as smart
+when encountered in `Subcrates`, based on the original migration case study.
+
+The library loader reads both folders. Health analysis reports all references but
+scores only non-smart references. Unknown crates remain scoreable so incomplete
+classification cannot silently hide potential problems.
+
+Serato documents the folder distinction in [What is in the _Serato_ folder?](https://support.serato.com/hc/en-us/articles/204022904-What-is-in-the-Serato-folder)
+and explains that smart crates are populated from rules in [Crates in Serato DJ](https://support.serato.com/hc/en-us/articles/227561407-Crates-in-Serato-DJ-Pro-Serato-DJ-Lite).
+
+## Edge Cases
+
+- The known `Compatible by key` dynamic crate in the `Subcrates` folder.
+- Crate fixtures outside a recognized Serato folder.
+- Libraries containing both static and smart references to the same track.
+- Dynamic references whose current materialized paths appear missing.
+
+## Verification
+
+Tests cover all three kinds, both Serato folders, the known dynamic fallback, the
+synthetic five-static/two-smart library, and exclusion from health scoring.

docs/design/health-engine.md

@@ -8,10 +8,11 @@ but an opaque blended score would imply confidence the current data cannot suppo
 ## Architecture
 
 The health engine produces an immutable report from the core `Library`. Its score
-is only the percentage of crate references resolved by exact filename. The report
-also exposes missing references, unique missing filenames, duplicate filename
-groups, extra duplicate files, unused tracks, and missing references with matching
-candidates.
+is only the percentage of non-dynamic crate references resolved by exact filename.
+Smart-crate references are counted but excluded because their contents are derived
+from rules. The report also exposes missing references, unique missing filenames,
+duplicate filename groups, extra duplicate files, unused tracks, and missing
+references with matching candidates.
 
 Duplicate, unused, and candidate counts are informational. They do not affect the
 score until the project has a documented and validated weighting policy. An empty

samples/small-library/generate.py

@@ -18,8 +18,7 @@ def build_sample(output: Optional[Path] = None) -> Path:
     root = output or SAMPLE_ROOT / "generated"
     manifest = load_manifest()
     music_root = root / "Music"
-    crate_root = root / "Serato" / "_Serato_" / "Subcrates"
-    crate_root.mkdir(parents=True, exist_ok=True)
+    serato_root = root / "Serato" / "_Serato_"
 
     for relative_path in manifest["tracks"]:
         track_path = music_root / relative_path
@@ -29,6 +28,13 @@ def build_sample(output: Optional[Path] = None) -> Path:
         )
 
     for crate in manifest["crates"]:
+        folder_name = "Smartcrates" if crate["type"] == "smart" else "Subcrates"
+        crate_root = serato_root / folder_name
+        crate_root.mkdir(parents=True, exist_ok=True)
+        other_folder = "Subcrates" if folder_name == "Smartcrates" else "Smartcrates"
+        stale_path = serato_root / other_folder / crate["name"]
+        if stale_path.exists():
+            stale_path.unlink()
         records = "".join(
             f"{SERATO_PATH_PREFIX}{relative_path}otrk"
             for relative_path in crate["references"]

serato_doctor/cli.py

@@ -2,7 +2,7 @@ from pathlib import Path
 import argparse
 
 from serato_doctor.config import ScanConfig
-from serato_doctor.crate_parser import parse_crates
+from serato_doctor.crate_parser import load_library_crates
 from serato_doctor.health import analyze_health
 from serato_doctor.logging import configure_logging
 from serato_doctor.models.library import Library
@@ -55,10 +55,9 @@ def main():
     logger.debug("Serato directory: %s", config.serato)
     logger.debug("Music directory: %s", config.music)
 
-    library = Library.build(
-        references=parse_crates(
-            config.serato / "Subcrates", config.reference_roots
-        ),
+    crates = load_library_crates(config.serato, config.reference_roots)
+    library = Library.from_crates(
+        crates=crates,
         tracks=scan_audio(config.music),
     )
     results = library.reconcile_by_filename()
@@ -75,6 +74,7 @@ def main():
         print(f"Score Basis: {health.score_basis}")
         print(f"Tracks: {health.disk_tracks}")
         print(f"Crate References: {health.total_references}")
+        print(f"References Scored: {health.scored_references}")
         print(f"Healthy References: {health.healthy_references}")
         print(f"Broken References: {health.missing_references}")
         print(f"Unique Missing Filenames: {health.unique_missing_filenames}")
@@ -82,6 +82,11 @@ def main():
         print(f"Duplicate Files: {health.duplicate_files}")
         print(f"Unused Tracks: {health.unused_tracks}")
         print(f"Suggested Matches: {health.suggested_matches}")
+        print(f"Static Crates: {health.static_crates}")
+        print(f"Smart Crates: {health.smart_crates}")
+        print(
+            f"Dynamic References Excluded: {health.dynamic_references_excluded}"
+        )
     else:
         write_csv(results, config.out)
         write_missing_report(results, config.report)

serato_doctor/crate_parser.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from typing import Iterable, Tuple
 
-from serato_doctor.models.crate import Crate
+from serato_doctor.models.crate import Crate, CrateKind
 from serato_doctor.models.reference import TrackReference
 
 
@@ -32,6 +32,17 @@ def path_markers(reference_roots: Iterable[Path]) -> Tuple[str, ...]:
     return configured or DEFAULT_PATH_MARKERS
 
 
+def classify_crate(crate_path: Path) -> CrateKind:
+    parent_names = {parent.name.casefold() for parent in crate_path.parents}
+    if "smartcrates" in parent_names:
+        return CrateKind.SMART
+    if crate_path.name.casefold() == "compatible by key.crate":
+        return CrateKind.SMART
+    if "subcrates" in parent_names:
+        return CrateKind.STATIC
+    return CrateKind.UNKNOWN
+
+
 def load_crate(crate_path: Path, reference_roots: Iterable[Path] = ()) -> Crate:
     text = read_crate_text(crate_path)
     refs = []
@@ -63,7 +74,11 @@ def load_crate(crate_path: Path, reference_roots: Iterable[Path] = ()) -> Crate:
                 )
             )
 
-    return Crate(path=crate_path, references=tuple(refs))
+    return Crate(
+        path=crate_path,
+        references=tuple(refs),
+        kind=classify_crate(crate_path),
+    )
 
 
 def parse_crate(
@@ -82,3 +97,15 @@ def parse_crates(
     for crate in root.rglob("*.crate"):
         refs.extend(parse_crate(crate, reference_roots))
     return refs
+
+
+def load_library_crates(
+    serato_root: Path, reference_roots: Iterable[Path] = ()
+) -> Tuple[Crate, ...]:
+    reference_roots = tuple(reference_roots)
+    crates = []
+    for folder_name in ("Subcrates", "Smartcrates"):
+        folder = serato_root / folder_name
+        for crate_path in folder.rglob("*.crate"):
+            crates.append(load_crate(crate_path, reference_roots))
+    return tuple(sorted(crates, key=lambda crate: str(crate.path)))

serato_doctor/health.py

@@ -1,6 +1,7 @@
 from collections import Counter
 
 from serato_doctor.matching import MatchingEngine
+from serato_doctor.models.crate import CrateKind
 from serato_doctor.models.health import HealthReport
 from serato_doctor.models.library import Library
 
@@ -8,7 +9,14 @@ from serato_doctor.models.library import Library
 def analyze_health(library: Library) -> HealthReport:
     """Calculate defensible health metrics without changing the library."""
 
-    results = library.reconcile_by_filename()
+    dynamic_sources = {
+        crate.path for crate in library.crates if crate.kind is CrateKind.SMART
+    }
+    results = tuple(
+        result
+        for result in library.reconcile_by_filename()
+        if result.reference.source not in dynamic_sources
+    )
     missing = [result for result in results if not result.exists_by_filename]
     healthy_count = len(results) - len(missing)
     score = (
@@ -29,7 +37,8 @@ def analyze_health(library: Library) -> HealthReport:
 
     return HealthReport(
         score=score,
-        total_references=len(results),
+        total_references=len(library.references),
+        scored_references=len(results),
         healthy_references=healthy_count,
         missing_references=len(missing),
         unique_missing_filenames=len(
@@ -40,4 +49,18 @@ def analyze_health(library: Library) -> HealthReport:
         duplicate_files=sum(count - 1 for count in duplicate_counts),
         unused_tracks=unused_count,
         suggested_matches=suggested_count,
+        static_crates=sum(
+            crate.kind is CrateKind.STATIC for crate in library.crates
+        ),
+        smart_crates=sum(
+            crate.kind is CrateKind.SMART for crate in library.crates
+        ),
+        unknown_crates=sum(
+            crate.kind is CrateKind.UNKNOWN for crate in library.crates
+        ),
+        dynamic_references_excluded=sum(
+            len(crate.references)
+            for crate in library.crates
+            if crate.kind is CrateKind.SMART
+        ),
     )

serato_doctor/models/__init__.py

@@ -1,4 +1,4 @@
-from serato_doctor.models.crate import Crate
+from serato_doctor.models.crate import Crate, CrateKind
 from serato_doctor.models.health import HealthReport
 from serato_doctor.models.library import Library
 from serato_doctor.models.match import MatchEvidence, TrackMatch
@@ -7,6 +7,7 @@ from serato_doctor.models.track import DiskTrack
 
 __all__ = [
     "Crate",
+    "CrateKind",
     "DiskTrack",
     "HealthReport",
     "Library",

serato_doctor/models/crate.py

@@ -1,13 +1,21 @@
 from dataclasses import dataclass
+from enum import Enum
 from pathlib import Path
 from typing import Tuple
 
 from serato_doctor.models.reference import TrackReference
 
 
+class CrateKind(str, Enum):
+    STATIC = "static"
+    SMART = "smart"
+    UNKNOWN = "unknown"
+
+
 @dataclass(frozen=True)
 class Crate:
     """A Serato crate and the track references parsed from it."""
 
     path: Path
     references: Tuple[TrackReference, ...]
+    kind: CrateKind = CrateKind.UNKNOWN

serato_doctor/models/health.py

@@ -8,6 +8,7 @@ class HealthReport:
 
     score: Optional[float]
     total_references: int
+    scored_references: int
     healthy_references: int
     missing_references: int
     unique_missing_filenames: int
@@ -16,7 +17,11 @@ class HealthReport:
     duplicate_files: int
     unused_tracks: int
     suggested_matches: int
+    static_crates: int
+    smart_crates: int
+    unknown_crates: int
+    dynamic_references_excluded: int
 
     @property
     def score_basis(self) -> str:
-        return "Resolved crate references / total crate references"
+        return "Resolved non-dynamic references / non-dynamic references scored"

serato_doctor/models/library.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from typing import Iterable, Tuple
 
+from serato_doctor.models.crate import Crate
 from serato_doctor.models.reference import ReferenceResult, TrackReference
 from serato_doctor.models.track import DiskTrack
 
@@ -11,6 +12,7 @@ class Library:
 
     references: Tuple[TrackReference, ...]
     tracks: Tuple[DiskTrack, ...]
+    crates: Tuple[Crate, ...] = ()
 
     @classmethod
     def build(
@@ -20,6 +22,18 @@ class Library:
     ) -> "Library":
         return cls(tuple(references), tuple(tracks))
 
+    @classmethod
+    def from_crates(
+        cls, crates: Iterable[Crate], tracks: Iterable[DiskTrack]
+    ) -> "Library":
+        crate_tuple = tuple(crates)
+        references = tuple(
+            reference
+            for crate in crate_tuple
+            for reference in crate.references
+        )
+        return cls(references, tuple(tracks), crate_tuple)
+
     def reconcile_by_filename(self) -> Tuple[ReferenceResult, ...]:
         disk_names = {track.filename for track in self.tracks}
         return tuple(

tests/test_cli.py

@@ -123,6 +123,7 @@ def test_analyze_prints_health_without_writing_reports(tmp_path, monkeypatch, ca
     assert "Overall Health: 50.0%" in output
     assert "Tracks: 1" in output
     assert "Crate References: 2" in output
+    assert "References Scored: 2" in output
     assert "Healthy References: 1" in output
     assert "Broken References: 1" in output
     assert "Unused Tracks: 0" in output

tests/test_crates.py

@@ -3,12 +3,14 @@ from pathlib import Path
 import pytest
 
 from serato_doctor.crate_parser import (
+    classify_crate,
     clean_path,
     load_crate,
     parse_crate,
     parse_crates,
     path_markers,
 )
+from serato_doctor.models.crate import CrateKind
 
 
 @pytest.mark.parametrize(
@@ -84,3 +86,18 @@ def test_parse_crates_reuses_configured_roots_for_every_crate(tmp_path):
         "First.mp3",
         "Second.mp3",
     }
+
+
+@pytest.mark.parametrize(
+    ("relative_path", "expected"),
+    [
+        ("_Serato_/Subcrates/House.crate", CrateKind.STATIC),
+        ("_Serato_/Smartcrates/Warmup.crate", CrateKind.SMART),
+        ("_Serato_/Subcrates/Compatible by key.crate", CrateKind.SMART),
+        ("fixtures/Unknown.crate", CrateKind.UNKNOWN),
+    ],
+)
+def test_classify_crate_uses_provenance_and_known_dynamic_name(
+    tmp_path, relative_path, expected
+):
+    assert classify_crate(tmp_path / relative_path) is expected

tests/test_health.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
 from serato_doctor.health import analyze_health
+from serato_doctor.models.crate import Crate, CrateKind
 from serato_doctor.models.library import Library
 from serato_doctor.models.reference import TrackReference
 from serato_doctor.models.track import DiskTrack
@@ -36,9 +37,10 @@ def test_health_report_exposes_each_metric():
 
     assert report.score == 33.3
     assert report.score_basis == (
-        "Resolved crate references / total crate references"
+        "Resolved non-dynamic references / non-dynamic references scored"
     )
     assert report.total_references == 3
+    assert report.scored_references == 3
     assert report.healthy_references == 1
     assert report.missing_references == 2
     assert report.unique_missing_filenames == 2
@@ -54,3 +56,32 @@ def test_empty_library_has_no_health_score():
 
     assert report.score is None
     assert report.total_references == 0
+    assert report.scored_references == 0
+
+
+def test_smart_crate_references_are_reported_but_not_scored():
+    static_reference = reference("Found.mp3")
+    smart_reference = TrackReference(
+        Path("Smartcrates/Dynamic.crate"),
+        Path("/old/House/Dynamic.mp3"),
+        "Dynamic.mp3",
+    )
+    crates = [
+        Crate(Path("Subcrates/Static.crate"), (static_reference,), CrateKind.STATIC),
+        Crate(
+            Path("Smartcrates/Dynamic.crate"),
+            (smart_reference,),
+            CrateKind.SMART,
+        ),
+    ]
+    library = Library.from_crates(crates, [track("Found.mp3")])
+
+    report = analyze_health(library)
+
+    assert report.score == 100.0
+    assert report.total_references == 2
+    assert report.scored_references == 1
+    assert report.missing_references == 0
+    assert report.static_crates == 1
+    assert report.smart_crates == 1
+    assert report.dynamic_references_excluded == 1

tests/test_sample_library.py

@@ -1,7 +1,8 @@
 import runpy
 from pathlib import Path
 
-from serato_doctor.crate_parser import parse_crates
+from serato_doctor.crate_parser import load_library_crates
+from serato_doctor.models.crate import CrateKind
 from serato_doctor.models.library import Library
 from serato_doctor.scanner import scan_audio
 
@@ -13,9 +14,10 @@ def test_generated_sample_library_has_expected_scenario(tmp_path):
     build_sample = runpy.run_path(str(generator_path))["build_sample"]
     sample_root = build_sample(tmp_path / "sample")
 
-    references = parse_crates(sample_root / "Serato" / "_Serato_" / "Subcrates")
+    crates = load_library_crates(sample_root / "Serato" / "_Serato_")
     tracks = scan_audio(sample_root / "Music")
-    results = Library.build(references, tracks).reconcile_by_filename()
+    library = Library.from_crates(crates, tracks)
+    results = library.reconcile_by_filename()
     missing = {
         result.reference.filename
         for result in results
@@ -23,6 +25,8 @@ def test_generated_sample_library_has_expected_scenario(tmp_path):
     }
 
     assert len(list((sample_root / "Serato").rglob("*.crate"))) == 7
-    assert len(references) == 13
+    assert len(library.references) == 13
     assert len(tracks) == 10
     assert missing == {"Missing.mp3", "Old Name.mp3"}
+    assert sum(crate.kind is CrateKind.STATIC for crate in crates) == 5
+    assert sum(crate.kind is CrateKind.SMART for crate in crates) == 2