From 4d04939846e5ae3613e3a0b8e2dbbd5df6bbc594 Mon Sep 17 00:00:00 2001 From: Philip Guzman Date: Wed, 1 Jul 2026 16:36:21 -0700 Subject: [PATCH] Add duplicate audio hash detection --- ROADMAP.md | 2 +- docs/design/duplicate-audio-hashing.md | 34 ++++++++++++++++++++++ serato_doctor/hashing.py | 33 +++++++++++++++++++++ serato_doctor/web.py | 26 ++++++++++++++++- serato_doctor/webui/app.js | 29 +++++++++++++++++-- serato_doctor/webui/layout-fixes.css | 40 ++++++++++++++++++++++++++ tests/test_hashing.py | 25 ++++++++++++++++ tests/test_web.py | 1 + 8 files changed, 186 insertions(+), 4 deletions(-) create mode 100644 docs/design/duplicate-audio-hashing.md create mode 100644 serato_doctor/hashing.py create mode 100644 tests/test_hashing.py diff --git a/ROADMAP.md b/ROADMAP.md index c6e9bc2..8f9f94d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -19,7 +19,7 @@ - [x] `serato-doctor analyze` command - [x] Duplicate filename detection -- [ ] Duplicate audio hash detection +- [x] Duplicate audio hash detection - [x] Broken symlink detection - [ ] Orphaned audio detection - [ ] OneDrive rename detection diff --git a/docs/design/duplicate-audio-hashing.md b/docs/design/duplicate-audio-hashing.md new file mode 100644 index 0000000..83d8101 --- /dev/null +++ b/docs/design/duplicate-audio-hashing.md @@ -0,0 +1,34 @@ +# Duplicate audio hashing + +## Problem + +Matching filenames do not prove matching content. Two files can be different DJ +edits, masters, encodes, or entirely different tracks. Removing either without +stronger evidence is unsafe. + +## Design + +Serato Doctor computes SHA-256 only for the duplicate group currently being +reviewed. This avoids hashing an entire 22,000-track library during every scan. +The review queue labels a group as byte-for-byte identical or warns that its +files differ. File size and a short fingerprint remain available as supporting +detail. + +The final batch dry-run recomputes every approved comparison server-side and +includes the status beside each keeper decision. The hash is evidence, not an +automatic repair decision; the DJ remains in control. + +## Edge cases + +- Identical audio stored under different filenames is recognized. +- Metadata changes inside an audio container produce a different byte hash and + therefore a conservative warning. +- Missing or unreadable files fail verification rather than being treated as + identical. +- Hashing is streamed in chunks rather than loading whole tracks into memory. + +## Tests + +- Equal byte content produces identical SHA-256 fingerprints. +- Different content is labeled different. +- Batch previews carry the server-computed confidence status. diff --git a/serato_doctor/hashing.py b/serato_doctor/hashing.py new file mode 100644 index 0000000..425b549 --- /dev/null +++ b/serato_doctor/hashing.py @@ -0,0 +1,33 @@ +"""Targeted content hashing for duplicate candidates.""" + +import hashlib +from pathlib import Path +from typing import Iterable + + +def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def compare_audio_files(paths: Iterable[Path]) -> dict: + paths = tuple(paths) + if len(paths) < 2: + raise ValueError("At least two files are required for comparison") + fingerprints = [] + for path in paths: + fingerprints.append( + { + "path": str(path), + "size": path.stat().st_size, + "sha256": sha256_file(path), + } + ) + identical = len({item["sha256"] for item in fingerprints}) == 1 + return { + "status": "identical" if identical else "different", + "files": fingerprints, + } diff --git a/serato_doctor/web.py b/serato_doctor/web.py index e71d7e9..d76799c 100644 --- a/serato_doctor/web.py +++ b/serato_doctor/web.py @@ -18,6 +18,7 @@ from serato_doctor.crate_parser import load_library_crates from serato_doctor.database_parser import parse_database from serato_doctor.duplicates import find_duplicate_groups from serato_doctor.health import analyze_health +from serato_doctor.hashing import compare_audio_files from serato_doctor.matching import MatchingEngine, normalize from serato_doctor.models.crate import CrateKind from serato_doctor.models.duplicate import DuplicateKind @@ -357,6 +358,9 @@ def duplicate_repair_batch( if not plans: raise ValueError("Choose at least one duplicate group") replaced = [str(path) for plan in plans for path in plan.replaced] + comparisons = [ + compare_audio_files((plan.keeper,) + plan.replaced) for plan in plans + ] result = { "choice_count": len(plans), "replaced": replaced, @@ -364,8 +368,9 @@ def duplicate_repair_batch( { "keeper": str(plan.keeper), "replaced": [str(path) for path in plan.replaced], + "hash_status": comparison["status"], } - for plan in plans + for plan, comparison in zip(plans, comparisons) ], "metadata_backups": len(plans[0].metadata_files), "strategy": "shortcut", @@ -438,6 +443,7 @@ class SeratoDoctorHandler(BaseHTTPRequestHandler): "/api/duplicates/apply", "/api/duplicates/batch/preview", "/api/duplicates/batch/apply", + "/api/duplicates/hash", "/api/backups/restore", "/api/backups", "/api/reveal", @@ -471,6 +477,24 @@ class SeratoDoctorHandler(BaseHTTPRequestHandler): raise ValueError("That backup does not belong to this library") restored = restore_backup(backup) result = {"restored": [str(path) for path in restored]} + elif self.path == "/api/duplicates/hash": + tokens = payload["tokens"] + if not isinstance(tokens, list) or not 2 <= len(tokens) <= 10: + raise ValueError("Compare between 2 and 10 audio files") + comparison = compare_audio_files( + _verified_audio(token) for token in tokens + ) + result = { + "status": comparison["status"], + "files": [ + { + "path": item["path"], + "size": item["size"], + "fingerprint": item["sha256"][:12], + } + for item in comparison["files"] + ], + } elif self.path.startswith("/api/duplicates/batch/"): raw_limit = payload.get("backup_limit", 10) backup_limit = None if raw_limit is None else int(raw_limit) diff --git a/serato_doctor/webui/app.js b/serato_doctor/webui/app.js index 06f1c85..d96f01d 100644 --- a/serato_doctor/webui/app.js +++ b/serato_doctor/webui/app.js @@ -155,7 +155,7 @@ function renderReviewGroup() { drilldownSummary.textContent = 'Listen to each candidate, choose the keeper, and we’ll move to the next group. Skip anything uncertain.'; drilldownList.innerHTML = `
-
Comparing now${escapeHtml(group.filename)}
${reviewState.choices.size} selected
+
Comparing now${escapeHtml(group.filename)}
Checking file identity…${reviewState.choices.size} selected
${group.file_previews.map((file, index) => `
Option ${index + 1} @@ -168,6 +168,31 @@ function renderReviewGroup() {
`; updateBatchSummary(); + loadHashConfidence(group, reviewState.index, reviewState.key); +} + +async function loadHashConfidence(group, groupIndex, detailKey) { + const badge = document.querySelector('#hash-confidence'); + if (group.hashComparison) { + showHashConfidence(badge, group.hashComparison); + return; + } + try { + const response = await fetch('/api/duplicates/hash', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({tokens: group.file_previews.map((file) => file.reveal_token)})}); + const result = await response.json(); + if (!response.ok) throw new Error(result.error || 'Identity check failed'); + group.hashComparison = result; + if (reviewState?.index === groupIndex && reviewState?.key === detailKey) showHashConfidence(document.querySelector('#hash-confidence'), result); + } catch (error) { + if (reviewState?.index === groupIndex && reviewState?.key === detailKey && badge) { badge.className = 'hash-confidence unavailable'; badge.textContent = 'Identity not verified'; } + } +} + +function showHashConfidence(badge, comparison) { + if (!badge) return; + badge.className = `hash-confidence ${comparison.status}`; + badge.textContent = comparison.status === 'identical' ? '✓ Byte-for-byte identical' : '⚠ Files differ — listen carefully'; + badge.title = comparison.files.map((file) => `${formatBytes(file.size)} · ${file.fingerprint}`).join('\n'); } function updateBatchSummary() { @@ -200,7 +225,7 @@ previewRepairButton.addEventListener('click', async () => { previewedRepair = await requestRepair('/api/duplicates/batch/preview'); batchPreviewSummary.textContent = `${previewedRepair.choice_count} keeper decision(s) · ${previewedRepair.replaced.length} duplicate file(s) consolidated`; batchPreviewList.innerHTML = previewedRepair.decisions.map((decision, index) => ` -
Decision ${index + 1}
Keep${escapeHtml(decision.keeper.split('/').pop())}${escapeHtml(decision.keeper)}
Replace with a shortcut${decision.replaced.map((path) => `${escapeHtml(path)}`).join('')}
+
Decision ${index + 1}${decision.hash_status === 'identical' ? 'Identical' : 'Files differ'}
Keep${escapeHtml(decision.keeper.split('/').pop())}${escapeHtml(decision.keeper)}
Replace with a shortcut${decision.replaced.map((path) => `${escapeHtml(path)}`).join('')}
`).join(''); batchPreviewSafety.textContent = `${previewedRepair.metadata_backups} Serato metadata file(s) and every replaced audio file will be backed up. Database V2 will not be modified.`; repairPreview.innerHTML = `Preview approved

${previewedRepair.choice_count} keeper decision(s) are ready for one backed-up apply.

`; diff --git a/serato_doctor/webui/layout-fixes.css b/serato_doctor/webui/layout-fixes.css index 6a33659..576c958 100644 --- a/serato_doctor/webui/layout-fixes.css +++ b/serato_doctor/webui/layout-fixes.css @@ -35,6 +35,46 @@ font-size: 14px; } +.review-signals { + display: flex; + align-items: flex-end; + gap: 7px; + flex-direction: column; +} + +.hash-confidence { + display: inline-block; + padding: 5px 8px; + border-radius: 999px; + font-size: 9px!important; + font-style: normal; + font-weight: 750; + letter-spacing: .02em!important; + text-transform: none!important; +} + +.hash-confidence.checking, +.hash-confidence.unavailable { + color: var(--muted); + background: rgba(255, 255, 255, .05); +} + +.hash-confidence.identical { + color: #8ce4b8; + background: rgba(84, 212, 154, .11); +} + +.hash-confidence.different { + color: var(--amber); + background: rgba(245, 185, 76, .1); +} + +.preview-decision > span .hash-confidence { + display: block; + width: max-content; + margin-top: 8px; +} + .review-candidates { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); diff --git a/tests/test_hashing.py b/tests/test_hashing.py new file mode 100644 index 0000000..7eba2f2 --- /dev/null +++ b/tests/test_hashing.py @@ -0,0 +1,25 @@ +from serato_doctor.hashing import compare_audio_files, sha256_file + + +def test_equal_files_have_the_same_hash(tmp_path): + first = tmp_path / "First.mp3" + second = tmp_path / "Second.mp3" + first.write_bytes(b"same audio bytes") + second.write_bytes(b"same audio bytes") + + result = compare_audio_files((first, second)) + + assert result["status"] == "identical" + assert sha256_file(first) == sha256_file(second) + + +def test_different_files_are_not_reported_as_identical(tmp_path): + first = tmp_path / "First.mp3" + second = tmp_path / "Second.mp3" + first.write_bytes(b"version one") + second.write_bytes(b"version two") + + result = compare_audio_files((first, second)) + + assert result["status"] == "different" + assert result["files"][0]["sha256"] != result["files"][1]["sha256"] diff --git a/tests/test_web.py b/tests/test_web.py index c7e3923..18b223a 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -143,5 +143,6 @@ def test_batch_preview_combines_approved_groups_without_changes(tmp_path): assert len(result["replaced"]) == 2 assert len(result["decisions"]) == 2 assert result["decisions"][0]["keeper"].endswith("First.mp3") + assert result["decisions"][0]["hash_status"] == "different" assert len(result["decisions"][0]["replaced"]) == 1 assert all(not Path(choice["group_files"][1]).is_symlink() for choice in choices)