From 6449c7de6ab860b8d3323781b129787d81e0943c Mon Sep 17 00:00:00 2001 From: Philip Guzman Date: Tue, 30 Jun 2026 18:52:16 -0700 Subject: [PATCH] Add duplicate filename diagnostics --- ROADMAP.md | 2 +- docs/design/duplicate-filenames.md | 32 +++++++++++++++++ serato_doctor/cli.py | 8 +++++ serato_doctor/duplicates.py | 41 +++++++++++++++++++++ serato_doctor/health.py | 25 +++++++++---- serato_doctor/models/__init__.py | 3 ++ serato_doctor/models/duplicate.py | 30 ++++++++++++++++ serato_doctor/models/health.py | 2 ++ tests/test_duplicates.py | 57 ++++++++++++++++++++++++++++++ tests/test_health.py | 19 ++++++++++ 10 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 docs/design/duplicate-filenames.md create mode 100644 serato_doctor/duplicates.py create mode 100644 serato_doctor/models/duplicate.py create mode 100644 tests/test_duplicates.py diff --git a/ROADMAP.md b/ROADMAP.md index 146f372..77225fa 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -18,7 +18,7 @@ ## v0.2 — Diagnostics - [x] `serato-doctor analyze` command -- [ ] Duplicate filename detection +- [x] Duplicate filename detection - [ ] Duplicate audio hash detection - [ ] Broken symlink detection - [ ] Orphaned audio detection diff --git a/docs/design/duplicate-filenames.md b/docs/design/duplicate-filenames.md new file mode 100644 index 0000000..c34d694 --- /dev/null +++ b/docs/design/duplicate-filenames.md @@ -0,0 +1,32 @@ +# Duplicate Filename Detection + +## Problem + +Two files with the same filename may be ordinary copies, while names such as +`Track.mp3` and `Track 2.mp3` may indicate a OneDrive conflict. These cases need +review, but neither is sufficient evidence for deletion. + +## Architecture + +The duplicate detector emits immutable groups of two kinds: + +- `exact_name` groups filenames after case and Unicode normalization. +- `cloud_conflict` groups names after additionally removing a trailing numeric + suffix from the stem. + +Groups contain every path, a stable comparison key, a display name, and the number +of files beyond the first. Health analysis reports exact and suspected-conflict +counts separately. Neither category changes the health score. + +## Edge Cases + +- Same filename in different folders. +- Case-only and Unicode representation differences. +- Multiple conflict suffixes such as `Track 2.mp3` and `Track 3.mp3`. +- Legitimate numbered titles, which remain explicitly labeled as suspected. +- A single file, which is never reported as a duplicate. + +## Verification + +Tests cover exact, normalized, conflict-family, unrelated, and deterministic-order +behavior, plus integration with health analysis. Detection is read-only. diff --git a/serato_doctor/cli.py b/serato_doctor/cli.py index d155295..4f82102 100644 --- a/serato_doctor/cli.py +++ b/serato_doctor/cli.py @@ -80,6 +80,14 @@ def main(): print(f"Unique Missing Filenames: {health.unique_missing_filenames}") print(f"Duplicate Filename Groups: {health.duplicate_filename_groups}") print(f"Duplicate Files: {health.duplicate_files}") + print( + "Suspected Cloud Conflict Groups: " + f"{health.suspected_cloud_conflict_groups}" + ) + print( + "Suspected Cloud Conflict Files: " + f"{health.suspected_cloud_conflict_files}" + ) print(f"Unused Tracks: {health.unused_tracks}") print(f"Suggested Matches: {health.suggested_matches}") print(f"Static Crates: {health.static_crates}") diff --git a/serato_doctor/duplicates.py b/serato_doctor/duplicates.py new file mode 100644 index 0000000..afdf6e9 --- /dev/null +++ b/serato_doctor/duplicates.py @@ -0,0 +1,41 @@ +from collections import defaultdict +from typing import DefaultDict, Iterable, List, Tuple + +from serato_doctor.matching import cloud_conflict_name, normalize +from serato_doctor.models.duplicate import DuplicateGroup, DuplicateKind +from serato_doctor.models.track import DiskTrack + + +def find_duplicate_groups( + tracks: Iterable[DiskTrack], +) -> Tuple[DuplicateGroup, ...]: + """Find exact-name duplicates and suspected numeric conflict copies.""" + + track_tuple = tuple(tracks) + by_name: DefaultDict[str, List[DiskTrack]] = defaultdict(list) + by_conflict_name: DefaultDict[str, List[DiskTrack]] = defaultdict(list) + + for track in track_tuple: + by_name[normalize(track.filename)].append(track) + by_conflict_name[cloud_conflict_name(track.filename)].append(track) + + groups = [] + for key, matches in by_name.items(): + if len(matches) > 1: + groups.append(_group(DuplicateKind.EXACT_NAME, key, matches)) + + for key, matches in by_conflict_name.items(): + distinct_names = {normalize(track.filename) for track in matches} + if len(distinct_names) > 1: + groups.append(_group(DuplicateKind.CLOUD_CONFLICT, key, matches)) + + return tuple( + sorted(groups, key=lambda group: (group.kind.value, group.comparison_key)) + ) + + +def _group( + kind: DuplicateKind, key: str, tracks: Iterable[DiskTrack] +) -> DuplicateGroup: + ordered = tuple(sorted(tracks, key=lambda track: str(track.path))) + return DuplicateGroup(kind, key, ordered) diff --git a/serato_doctor/health.py b/serato_doctor/health.py index 8f467df..be00534 100644 --- a/serato_doctor/health.py +++ b/serato_doctor/health.py @@ -1,7 +1,7 @@ -from collections import Counter - +from serato_doctor.duplicates import find_duplicate_groups from serato_doctor.matching import MatchingEngine from serato_doctor.models.crate import CrateKind +from serato_doctor.models.duplicate import DuplicateKind from serato_doctor.models.health import HealthReport from serato_doctor.models.library import Library @@ -23,8 +23,17 @@ def analyze_health(library: Library) -> HealthReport: round(healthy_count / len(results) * 100, 1) if results else None ) - disk_name_counts = Counter(track.filename for track in library.tracks) - duplicate_counts = [count for count in disk_name_counts.values() if count > 1] + duplicate_groups = find_duplicate_groups(library.tracks) + exact_duplicates = [ + group + for group in duplicate_groups + if group.kind is DuplicateKind.EXACT_NAME + ] + cloud_conflicts = [ + group + for group in duplicate_groups + if group.kind is DuplicateKind.CLOUD_CONFLICT + ] referenced_names = {reference.filename for reference in library.references} unused_count = sum( 1 for track in library.tracks if track.filename not in referenced_names @@ -45,8 +54,12 @@ def analyze_health(library: Library) -> HealthReport: {result.reference.filename for result in missing} ), disk_tracks=len(library.tracks), - duplicate_filename_groups=len(duplicate_counts), - duplicate_files=sum(count - 1 for count in duplicate_counts), + duplicate_filename_groups=len(exact_duplicates), + duplicate_files=sum(group.extra_files for group in exact_duplicates), + suspected_cloud_conflict_groups=len(cloud_conflicts), + suspected_cloud_conflict_files=sum( + group.extra_files for group in cloud_conflicts + ), unused_tracks=unused_count, suggested_matches=suggested_count, static_crates=sum( diff --git a/serato_doctor/models/__init__.py b/serato_doctor/models/__init__.py index 37991b1..1a2f647 100644 --- a/serato_doctor/models/__init__.py +++ b/serato_doctor/models/__init__.py @@ -1,4 +1,5 @@ from serato_doctor.models.crate import Crate, CrateKind +from serato_doctor.models.duplicate import DuplicateGroup, DuplicateKind from serato_doctor.models.health import HealthReport from serato_doctor.models.library import Library from serato_doctor.models.match import MatchEvidence, TrackMatch @@ -9,6 +10,8 @@ __all__ = [ "Crate", "CrateKind", "DiskTrack", + "DuplicateGroup", + "DuplicateKind", "HealthReport", "Library", "MatchEvidence", diff --git a/serato_doctor/models/duplicate.py b/serato_doctor/models/duplicate.py new file mode 100644 index 0000000..6a583ce --- /dev/null +++ b/serato_doctor/models/duplicate.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass +from enum import Enum +from typing import Tuple + +from serato_doctor.models.track import DiskTrack + + +class DuplicateKind(str, Enum): + EXACT_NAME = "exact_name" + CLOUD_CONFLICT = "cloud_conflict" + + +@dataclass(frozen=True) +class DuplicateGroup: + """A deterministic group of files that warrants duplicate review.""" + + kind: DuplicateKind + comparison_key: str + tracks: Tuple[DiskTrack, ...] + + @property + def extra_files(self) -> int: + return max(0, len(self.tracks) - 1) + + @property + def display_name(self) -> str: + return min( + (track.filename for track in self.tracks), + key=lambda name: (len(name), name.casefold()), + ) diff --git a/serato_doctor/models/health.py b/serato_doctor/models/health.py index 39b124c..e6c02cb 100644 --- a/serato_doctor/models/health.py +++ b/serato_doctor/models/health.py @@ -15,6 +15,8 @@ class HealthReport: disk_tracks: int duplicate_filename_groups: int duplicate_files: int + suspected_cloud_conflict_groups: int + suspected_cloud_conflict_files: int unused_tracks: int suggested_matches: int static_crates: int diff --git a/tests/test_duplicates.py b/tests/test_duplicates.py new file mode 100644 index 0000000..c821b37 --- /dev/null +++ b/tests/test_duplicates.py @@ -0,0 +1,57 @@ +from pathlib import Path + +from serato_doctor.duplicates import find_duplicate_groups +from serato_doctor.models.duplicate import DuplicateKind +from serato_doctor.models.track import DiskTrack + + +def track(path): + path = Path(path) + return DiskTrack(path, path.name, 100, path.suffix.lower()) + + +def test_exact_names_in_different_folders_form_a_group(): + groups = find_duplicate_groups( + [track("/A/Song.mp3"), track("/B/Song.mp3")] + ) + + assert len(groups) == 1 + assert groups[0].kind is DuplicateKind.EXACT_NAME + assert groups[0].display_name == "Song.mp3" + assert groups[0].extra_files == 1 + + +def test_case_only_difference_is_an_exact_name_duplicate(): + groups = find_duplicate_groups( + [track("/A/SONG.MP3"), track("/B/song.mp3")] + ) + + assert groups[0].kind is DuplicateKind.EXACT_NAME + + +def test_numeric_suffixes_form_a_suspected_cloud_conflict_group(): + groups = find_duplicate_groups( + [ + track("/A/Track.mp3"), + track("/B/Track 2.mp3"), + track("/C/Track 3.mp3"), + ] + ) + + assert len(groups) == 1 + assert groups[0].kind is DuplicateKind.CLOUD_CONFLICT + assert groups[0].display_name == "Track.mp3" + assert groups[0].extra_files == 2 + assert [item.path for item in groups[0].tracks] == [ + Path("/A/Track.mp3"), + Path("/B/Track 2.mp3"), + Path("/C/Track 3.mp3"), + ] + + +def test_unrelated_and_single_files_are_not_reported(): + groups = find_duplicate_groups( + [track("/A/First.mp3"), track("/B/Second.mp3")] + ) + + assert groups == () diff --git a/tests/test_health.py b/tests/test_health.py index 7147bcd..0511d8f 100644 --- a/tests/test_health.py +++ b/tests/test_health.py @@ -85,3 +85,22 @@ def test_smart_crate_references_are_reported_but_not_scored(): assert report.static_crates == 1 assert report.smart_crates == 1 assert report.dynamic_references_excluded == 1 + + +def test_health_reports_cloud_conflicts_separately_from_exact_duplicates(): + library = Library.build( + [], + [ + track("Track.mp3", "Original"), + track("Track 2.mp3", "Conflict"), + track("Copy.mp3", "First"), + track("Copy.mp3", "Second"), + ], + ) + + report = analyze_health(library) + + assert report.duplicate_filename_groups == 1 + assert report.duplicate_files == 1 + assert report.suspected_cloud_conflict_groups == 1 + assert report.suspected_cloud_conflict_files == 1