Compare commits

...

2 Commits

Author SHA1 Message Date
Philip Guzman 22c4f58646 Add defensible orphaned audio detection 2026-07-02 16:17:11 -07:00
Philip Guzman 4d04939846 Add duplicate audio hash detection 2026-07-01 16:36:21 -07:00
13 changed files with 319 additions and 15 deletions
+2 -2
View File
@@ -19,9 +19,9 @@
- [x] `serato-doctor analyze` command
- [x] Duplicate filename detection
- [ ] Duplicate audio hash detection
- [x] Duplicate audio hash detection
- [x] Broken symlink detection
- [ ] Orphaned audio detection
- [x] Orphaned audio detection
- [ ] OneDrive rename detection
- [x] Crate classification: static vs smart/dynamic
- [x] Library health score
+34
View File
@@ -0,0 +1,34 @@
# Duplicate audio hashing
## Problem
Matching filenames do not prove matching content. Two files can be different DJ
edits, masters, encodes, or entirely different tracks. Removing either without
stronger evidence is unsafe.
## Design
Serato Doctor computes SHA-256 only for the duplicate group currently being
reviewed. This avoids hashing an entire 22,000-track library during every scan.
The review queue labels a group as byte-for-byte identical or warns that its
files differ. File size and a short fingerprint remain available as supporting
detail.
The final batch dry-run recomputes every approved comparison server-side and
includes the status beside each keeper decision. The hash is evidence, not an
automatic repair decision; the DJ remains in control.
## Edge cases
- Identical audio stored under different filenames is recognized.
- Metadata changes inside an audio container produce a different byte hash and
therefore a conservative warning.
- Missing or unreadable files fail verification rather than being treated as
identical.
- Hashing is streamed in chunks rather than loading whole tracks into memory.
## Tests
- Equal byte content produces identical SHA-256 fingerprints.
- Different content is labeled different.
- Batch previews carry the server-computed confidence status.
+34
View File
@@ -0,0 +1,34 @@
# Orphaned audio detection
## Problem
“Not in a crate” does not mean unused. Many DJs keep active tracks in Serato's
main library without placing them in a crate. Treating those files as orphans
would create dangerous deletion pressure.
## Design
Serato Doctor now separates two read-only findings:
- **Not in crates**: scanned tracks whose normalized filename is absent from all
loaded crate references. These may still be ordinary Serato library tracks.
- **Possible orphan files**: tracks absent by normalized filename from both
loaded crates and database V2.
When database V2 is unavailable, orphan status is reported as not assessed—not
zero. Orphan candidates include preview and Finder controls but no delete or
repair action.
## Edge cases
- Filename comparison is case- and Unicode-normalized.
- A track in several folders may still require duplicate review separately.
- Smart-crate materialized references count as crate evidence.
- Missing database V2 prevents a defensible orphan conclusion.
## Tests
- Tracks known only to database V2 are excluded from orphan candidates.
- Tracks in crates are excluded even if absent from database V2.
- A track absent from both sources is counted once.
- No database produces a not-assessed result.
+33
View File
@@ -0,0 +1,33 @@
"""Targeted content hashing for duplicate candidates."""
import hashlib
from pathlib import Path
from typing import Iterable
def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(chunk_size), b""):
digest.update(chunk)
return digest.hexdigest()
def compare_audio_files(paths: Iterable[Path]) -> dict:
paths = tuple(paths)
if len(paths) < 2:
raise ValueError("At least two files are required for comparison")
fingerprints = []
for path in paths:
fingerprints.append(
{
"path": str(path),
"size": path.stat().st_size,
"sha256": sha256_file(path),
}
)
identical = len({item["sha256"] for item in fingerprints}) == 1
return {
"status": "identical" if identical else "different",
"files": fingerprints,
}
+15 -2
View File
@@ -36,9 +36,13 @@ def analyze_health(library: Library) -> HealthReport:
for group in duplicate_groups
if group.kind is DuplicateKind.CLOUD_CONFLICT
]
referenced_names = {reference.filename for reference in library.references}
referenced_names = {
normalize(reference.filename) for reference in library.references
}
unused_count = sum(
1 for track in library.tracks if track.filename not in referenced_names
1
for track in library.tracks
if normalize(track.filename) not in referenced_names
)
matcher = MatchingEngine(library.tracks)
@@ -73,6 +77,15 @@ def analyze_health(library: Library) -> HealthReport:
group.extra_files for group in cloud_conflicts
),
unused_tracks=unused_count,
orphan_candidates=(
sum(
normalize(track.filename) not in referenced_names
and normalize(track.filename) not in database_names
for track in library.tracks
)
if library.database
else None
),
suggested_matches=suggested_count,
broken_symlinks=len(library.broken_symlinks),
static_crates=sum(
+1
View File
@@ -18,6 +18,7 @@ class HealthReport:
suspected_cloud_conflict_groups: int
suspected_cloud_conflict_files: int
unused_tracks: int
orphan_candidates: Optional[int]
suggested_matches: int
broken_symlinks: int
static_crates: int
+62 -6
View File
@@ -18,6 +18,7 @@ from serato_doctor.crate_parser import load_library_crates
from serato_doctor.database_parser import parse_database
from serato_doctor.duplicates import find_duplicate_groups
from serato_doctor.health import analyze_health
from serato_doctor.hashing import compare_audio_files
from serato_doctor.matching import MatchingEngine, normalize
from serato_doctor.models.crate import CrateKind
from serato_doctor.models.duplicate import DuplicateKind
@@ -142,10 +143,24 @@ def diagnostic_details(library: Library, limit: int = DETAIL_LIMIT) -> dict:
database_filename_counts = Counter(
normalize(track.filename) for track in missing_database_tracks
)
referenced_names = {reference.filename for reference in library.references}
referenced_names = {
normalize(reference.filename) for reference in library.references
}
database_names = {normalize(track.filename) for track in database_tracks}
unused_tracks = [
track for track in library.tracks if track.filename not in referenced_names
track
for track in library.tracks
if normalize(track.filename) not in referenced_names
]
orphan_candidates = (
[
track
for track in unused_tracks
if normalize(track.filename) not in database_names
]
if library.database
else None
)
return {
"database_missing_tracks": {
@@ -245,10 +260,10 @@ def diagnostic_details(library: Library, limit: int = DETAIL_LIMIT) -> dict:
],
},
"unused_tracks": {
"title": "Unused tracks",
"title": "Not in crates",
"summary": (
"These scanned files were not referenced by any loaded crate. "
"That does not mean they should be deleted."
"These files are not in any loaded crate, but may still be "
"normal tracks in Serato's main library."
),
"total": len(unused_tracks),
"items": [
@@ -256,6 +271,24 @@ def diagnostic_details(library: Library, limit: int = DETAIL_LIMIT) -> dict:
for track in unused_tracks[:limit]
],
},
"orphan_candidates": {
"title": "Possible orphan files",
"summary": (
"These files were found in the music folder but not by filename "
"in loaded crates or Serato's database. Review only; this is "
"never an automatic deletion recommendation."
),
"total": len(orphan_candidates) if orphan_candidates is not None else None,
"assessed": orphan_candidates is not None,
"items": [
{
"filename": track.filename,
"path": _display_path(track.path),
**_file_preview(track.path),
}
for track in (orphan_candidates or ())[:limit]
],
},
}
@@ -357,6 +390,9 @@ def duplicate_repair_batch(
if not plans:
raise ValueError("Choose at least one duplicate group")
replaced = [str(path) for plan in plans for path in plan.replaced]
comparisons = [
compare_audio_files((plan.keeper,) + plan.replaced) for plan in plans
]
result = {
"choice_count": len(plans),
"replaced": replaced,
@@ -364,8 +400,9 @@ def duplicate_repair_batch(
{
"keeper": str(plan.keeper),
"replaced": [str(path) for path in plan.replaced],
"hash_status": comparison["status"],
}
for plan in plans
for plan, comparison in zip(plans, comparisons)
],
"metadata_backups": len(plans[0].metadata_files),
"strategy": "shortcut",
@@ -438,6 +475,7 @@ class SeratoDoctorHandler(BaseHTTPRequestHandler):
"/api/duplicates/apply",
"/api/duplicates/batch/preview",
"/api/duplicates/batch/apply",
"/api/duplicates/hash",
"/api/backups/restore",
"/api/backups",
"/api/reveal",
@@ -471,6 +509,24 @@ class SeratoDoctorHandler(BaseHTTPRequestHandler):
raise ValueError("That backup does not belong to this library")
restored = restore_backup(backup)
result = {"restored": [str(path) for path in restored]}
elif self.path == "/api/duplicates/hash":
tokens = payload["tokens"]
if not isinstance(tokens, list) or not 2 <= len(tokens) <= 10:
raise ValueError("Compare between 2 and 10 audio files")
comparison = compare_audio_files(
_verified_audio(token) for token in tokens
)
result = {
"status": comparison["status"],
"files": [
{
"path": item["path"],
"size": item["size"],
"fingerprint": item["sha256"][:12],
}
for item in comparison["files"]
],
}
elif self.path.startswith("/api/duplicates/batch/"):
raw_limit = payload.get("backup_limit", 10)
backup_limit = None if raw_limit is None else int(raw_limit)
+38 -3
View File
@@ -111,6 +111,11 @@ function detailLines(item) {
return lines.map((line) => `<li>${escapeHtml(line)}</li>`).join('');
}
function detailActions(item) {
if (!item.audio_url || !item.reveal_token) return '';
return `<div class="file-actions detail-actions"><button type="button" data-audio-url="${escapeHtml(item.audio_url)}" data-audio-name="${escapeHtml(item.path)}">▶ Play preview</button><button type="button" data-reveal-token="${escapeHtml(item.reveal_token)}">Show in Finder</button></div>`;
}
function renderDetail(key) {
const detail = latestAnalysis?.details?.[key];
if (!detail) return;
@@ -119,7 +124,7 @@ function renderDetail(key) {
trigger.classList.toggle('selected', trigger.dataset.detail === key);
});
drilldownTitle.textContent = detail.title;
drilldownCount.textContent = `${detail.total ?? 0} found`;
drilldownCount.textContent = detail.assessed === false ? 'Not assessed' : `${detail.total ?? 0} found`;
drilldownSummary.textContent = detail.summary;
repairPanel.hidden = true;
@@ -127,6 +132,10 @@ function renderDetail(key) {
previewedRepair = null;
latestBackup = null;
restoreRepairButton.hidden = true;
if (detail.assessed === false) {
drilldownList.innerHTML = '<div class="empty-detail">Seratos database V2 was not available, so orphan status cannot be assessed safely.</div>';
return;
}
if (!detail.items?.length) {
drilldownList.innerHTML = '<div class="empty-detail">Nothing to review here. Tiny victory parade, very tasteful.</div>';
return;
@@ -143,6 +152,7 @@ function renderDetail(key) {
<article class="detail-item">
<strong>${escapeHtml(item.filename || item.path || 'Untitled item')}</strong>
<ul>${detailLines(item)}</ul>
${detailActions(item)}
</article>
`).join('');
}
@@ -155,7 +165,7 @@ function renderReviewGroup() {
drilldownSummary.textContent = 'Listen to each candidate, choose the keeper, and well move to the next group. Skip anything uncertain.';
drilldownList.innerHTML = `
<article class="review-workspace">
<div class="review-heading"><div><span>Comparing now</span><strong>${escapeHtml(group.filename)}</strong></div><span>${reviewState.choices.size} selected</span></div>
<div class="review-heading"><div><span>Comparing now</span><strong>${escapeHtml(group.filename)}</strong></div><div class="review-signals"><span id="hash-confidence" class="hash-confidence checking">Checking file identity…</span><span>${reviewState.choices.size} selected</span></div></div>
<div class="review-candidates">${group.file_previews.map((file, index) => `
<section class="review-candidate ${chosen === file.path ? 'winner' : ''}">
<span class="candidate-number">Option ${index + 1}</span>
@@ -168,6 +178,31 @@ function renderReviewGroup() {
<div class="review-navigation"><button type="button" data-review-previous ${reviewState.index === 0 ? 'disabled' : ''}>← Previous</button><button type="button" data-review-skip>Skip for now</button></div>
</article>`;
updateBatchSummary();
loadHashConfidence(group, reviewState.index, reviewState.key);
}
async function loadHashConfidence(group, groupIndex, detailKey) {
const badge = document.querySelector('#hash-confidence');
if (group.hashComparison) {
showHashConfidence(badge, group.hashComparison);
return;
}
try {
const response = await fetch('/api/duplicates/hash', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({tokens: group.file_previews.map((file) => file.reveal_token)})});
const result = await response.json();
if (!response.ok) throw new Error(result.error || 'Identity check failed');
group.hashComparison = result;
if (reviewState?.index === groupIndex && reviewState?.key === detailKey) showHashConfidence(document.querySelector('#hash-confidence'), result);
} catch (error) {
if (reviewState?.index === groupIndex && reviewState?.key === detailKey && badge) { badge.className = 'hash-confidence unavailable'; badge.textContent = 'Identity not verified'; }
}
}
function showHashConfidence(badge, comparison) {
if (!badge) return;
badge.className = `hash-confidence ${comparison.status}`;
badge.textContent = comparison.status === 'identical' ? '✓ Byte-for-byte identical' : '⚠ Files differ — listen carefully';
badge.title = comparison.files.map((file) => `${formatBytes(file.size)} · ${file.fingerprint}`).join('\n');
}
function updateBatchSummary() {
@@ -200,7 +235,7 @@ previewRepairButton.addEventListener('click', async () => {
previewedRepair = await requestRepair('/api/duplicates/batch/preview');
batchPreviewSummary.textContent = `${previewedRepair.choice_count} keeper decision(s) · ${previewedRepair.replaced.length} duplicate file(s) consolidated`;
batchPreviewList.innerHTML = previewedRepair.decisions.map((decision, index) => `
<article class="preview-decision"><span>Decision ${index + 1}</span><div class="winner-path"><b>Keep</b><strong>${escapeHtml(decision.keeper.split('/').pop())}</strong><small>${escapeHtml(decision.keeper)}</small></div><div class="replaced-paths"><b>Replace with a shortcut</b>${decision.replaced.map((path) => `<small>${escapeHtml(path)}</small>`).join('')}</div></article>
<article class="preview-decision"><span>Decision ${index + 1}<em class="hash-confidence ${decision.hash_status}">${decision.hash_status === 'identical' ? 'Identical' : 'Files differ'}</em></span><div class="winner-path"><b>Keep</b><strong>${escapeHtml(decision.keeper.split('/').pop())}</strong><small>${escapeHtml(decision.keeper)}</small></div><div class="replaced-paths"><b>Replace with a shortcut</b>${decision.replaced.map((path) => `<small>${escapeHtml(path)}</small>`).join('')}</div></article>
`).join('');
batchPreviewSafety.textContent = `${previewedRepair.metadata_backups} Serato metadata file(s) and every replaced audio file will be backed up. Database V2 will not be modified.`;
repairPreview.innerHTML = `<strong>Preview approved</strong><p>${previewedRepair.choice_count} keeper decision(s) are ready for one backed-up apply.</p>`;
+3 -2
View File
@@ -69,7 +69,7 @@
<div class="metrics-grid">
<article class="metric panel"><button class="info-button" type="button" aria-label="About tracks" aria-expanded="false" data-info="Audio files found inside the music folder you selected. This is the collection Serato Doctor compared with your crates and database.">i</button><span>Tracks scanned</span><strong data-field="disk_tracks"></strong><small>audio files found</small></article>
<article class="metric panel warning drill-trigger" role="button" tabindex="0" data-detail="database_missing_tracks"><button class="info-button" type="button" aria-label="About missing tracks in Serato" aria-expanded="false" data-info="Tracks in Serato's database whose saved file location no longer exists. This should be close to the orange or unmapped track count you see in Serato.">i</button><span>Missing tracks in Serato</span><strong data-field="database_missing_paths"></strong><small><b data-field="database_missing_unique_filenames"></b> unique filenames · click for list</small></article>
<article class="metric panel drill-trigger" role="button" tabindex="0" data-detail="unused_tracks"><button class="info-button" type="button" aria-label="About unused tracks" aria-expanded="false" data-info="Files in the selected music folder whose filename is not used by any loaded crate. They may still be valid library tracks; this is informational, not a deletion recommendation.">i</button><span>Unused tracks</span><strong data-field="unused_tracks"></strong><small>not referenced by crates · click for list</small></article>
<article class="metric panel drill-trigger" role="button" tabindex="0" data-detail="orphan_candidates"><button class="info-button" type="button" aria-label="About possible orphan files" aria-expanded="false" data-info="Files found in the music folder but not by filename in loaded crates or Serato's database. They require review and are never automatic deletion recommendations.">i</button><span>Possible orphan files</span><strong data-field="orphan_candidates"></strong><small>outside crates and database · click for list</small></article>
<article class="metric panel drill-trigger" role="button" tabindex="0" data-detail="suggested_matches"><button class="info-button" type="button" aria-label="About suggested matches" aria-expanded="false" data-info="Missing crate entries with a filename-related candidate, such as an added OneDrive conflict number. Suggestions are evidence for review, never automatic repairs.">i</button><span>Suggested matches</span><strong data-field="suggested_matches"></strong><small>explainable candidates · click for list</small></article>
</div>
</div>
@@ -82,6 +82,7 @@
<div class="drill-trigger" role="button" tabindex="0" data-detail="cloud_conflicts"><span class="diag-icon blue"></span><p><strong>Possible cloud conflicts</strong><small><b data-field="suspected_cloud_conflict_groups"></b> groups · <b data-field="suspected_cloud_conflict_files"></b> extra files · click for list</small></p><button class="info-button" type="button" aria-label="About cloud conflicts" aria-expanded="false" data-info="Filename families such as Track.mp3 and Track 2.mp3. OneDrive often creates these during sync conflicts, but numbered song titles can also be legitimate.">i</button></div>
<div class="drill-trigger" role="button" tabindex="0" data-detail="broken_symlinks"><span class="diag-icon red"></span><p><strong>Broken shortcuts</strong><small><b data-field="broken_symlinks"></b> unresolved symbolic links · click for list</small></p><button class="info-button" type="button" aria-label="About broken shortcuts" aria-expanded="false" data-info="Shortcut-style symbolic links whose destination no longer exists. Serato Doctor reports them but never removes or recreates them automatically.">i</button></div>
<div class="drill-trigger" role="button" tabindex="0" data-detail="old_crate_references"><span class="diag-icon violet"></span><p><strong>Old crate references</strong><small><b data-field="missing_references"></b> appearances · <b data-field="unique_missing_filenames"></b> unique filenames · click for list</small></p><button class="info-button" type="button" aria-label="About old crate references" aria-expanded="false" data-info="Saved spots in regular crates whose exact filename was not found in the selected music folder. The same track can appear in several crates, so appearances are higher than unique filenames. These are separate from Serato's unmapped-track count.">i</button></div>
<div class="drill-trigger" role="button" tabindex="0" data-detail="unused_tracks"><span class="diag-icon blue"></span><p><strong>Not in crates</strong><small><b data-field="unused_tracks"></b> tracks still potentially in Serato · click for list</small></p><button class="info-button" type="button" aria-label="About tracks not in crates" aria-expanded="false" data-info="Tracks that do not appear in loaded crates. They may still be normal, active tracks in Serato's main library and should not be treated as orphans.">i</button></div>
<div><span class="diag-icon amber"></span><p><strong>Database coverage</strong><small><b data-field="database_entries"></b> Serato entries · <b data-field="tracks_missing_from_database"></b> scanned tracks absent</small></p><button class="info-button" type="button" aria-label="About database coverage" aria-expanded="false" data-info="Compares filenames in Serato's database with the selected music folder. A scanned track absent from the database may not have been imported, or may be represented under another filename.">i</button></div>
</div>
</div>
@@ -130,6 +131,6 @@
</dialog>
</main>
</div>
<script src="/app.js?v=6" defer></script>
<script src="/app.js?v=7" defer></script>
</body>
</html>
+40
View File
@@ -35,6 +35,46 @@
font-size: 14px;
}
.review-signals {
display: flex;
align-items: flex-end;
gap: 7px;
flex-direction: column;
}
.hash-confidence {
display: inline-block;
padding: 5px 8px;
border-radius: 999px;
font-size: 9px!important;
font-style: normal;
font-weight: 750;
letter-spacing: .02em!important;
text-transform: none!important;
}
.hash-confidence.checking,
.hash-confidence.unavailable {
color: var(--muted);
background: rgba(255, 255, 255, .05);
}
.hash-confidence.identical {
color: #8ce4b8;
background: rgba(84, 212, 154, .11);
}
.hash-confidence.different {
color: var(--amber);
background: rgba(245, 185, 76, .1);
}
.preview-decision > span .hash-confidence {
display: block;
width: max-content;
margin-top: 8px;
}
.review-candidates {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
+25
View File
@@ -0,0 +1,25 @@
from serato_doctor.hashing import compare_audio_files, sha256_file
def test_equal_files_have_the_same_hash(tmp_path):
first = tmp_path / "First.mp3"
second = tmp_path / "Second.mp3"
first.write_bytes(b"same audio bytes")
second.write_bytes(b"same audio bytes")
result = compare_audio_files((first, second))
assert result["status"] == "identical"
assert sha256_file(first) == sha256_file(second)
def test_different_files_are_not_reported_as_identical(tmp_path):
first = tmp_path / "First.mp3"
second = tmp_path / "Second.mp3"
first.write_bytes(b"version one")
second.write_bytes(b"version two")
result = compare_audio_files((first, second))
assert result["status"] == "different"
assert result["files"][0]["sha256"] != result["files"][1]["sha256"]
+28
View File
@@ -2,6 +2,7 @@ from pathlib import Path
from serato_doctor.health import analyze_health
from serato_doctor.models.crate import Crate, CrateKind
from serato_doctor.models.database import DatabaseTrack, SeratoDatabase
from serato_doctor.models.filesystem import BrokenSymlink
from serato_doctor.models.library import Library
from serato_doctor.models.reference import TrackReference
@@ -49,6 +50,7 @@ def test_health_report_exposes_each_metric():
assert report.duplicate_filename_groups == 1
assert report.duplicate_files == 1
assert report.unused_tracks == 3
assert report.orphan_candidates is None
assert report.suggested_matches == 1
@@ -60,6 +62,32 @@ def test_empty_library_has_no_health_score():
assert report.scored_references == 0
assert not report.database_present
assert report.tracks_missing_from_database == 0
assert report.orphan_candidates is None
def test_orphan_candidates_exclude_tracks_known_to_serato_or_crates():
tracks = [
track("Crated.mp3"),
track("LibraryOnly.mp3"),
track("OutsideEverything.mp3"),
]
database = SeratoDatabase(
Path("database V2"),
"test",
(
DatabaseTrack(
Path("/new/House/LibraryOnly.mp3"), "LibraryOnly.mp3"
),
),
)
library = Library.build(
[reference("Crated.mp3")], tracks, database=database
)
report = analyze_health(library)
assert report.unused_tracks == 2
assert report.orphan_candidates == 1
def test_smart_crate_references_are_reported_but_not_scored():
+4
View File
@@ -37,6 +37,8 @@ def test_web_analysis_uses_production_health_pipeline(tmp_path):
assert result["details"]["old_crate_references"]["items"][0]["crate"]
assert result["details"]["suggested_matches"]["total"] == 0
assert result["details"]["unused_tracks"]["total"] == 1
assert result["orphan_candidates"] == 0
assert result["details"]["orphan_candidates"]["assessed"] is True
def test_web_analysis_rejects_missing_folders(tmp_path):
@@ -58,6 +60,7 @@ def test_web_static_assets_are_declared_and_packaged():
html = (asset_root / "index.html").read_text(encoding="utf-8")
assert "Missing tracks in Serato" in html
assert "Old crate references" in html
assert "Possible orphan files" in html
assert "Choose a diagnostic" in html
assert "Backup recovery" in html
assert "Heres exactly what will happen" in html
@@ -143,5 +146,6 @@ def test_batch_preview_combines_approved_groups_without_changes(tmp_path):
assert len(result["replaced"]) == 2
assert len(result["decisions"]) == 2
assert result["decisions"][0]["keeper"].endswith("First.mp3")
assert result["decisions"][0]["hash_status"] == "different"
assert len(result["decisions"][0]["replaced"]) == 1
assert all(not Path(choice["group_files"][1]).is_symlink() for choice in choices)