Add duplicate audio hash detection
This commit is contained in:
Philip Guzman
@@ -0,0 +1,33 @@
|
||||
"""Targeted content hashing for duplicate candidates."""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(chunk_size), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def compare_audio_files(paths: Iterable[Path]) -> dict:
|
||||
paths = tuple(paths)
|
||||
if len(paths) < 2:
|
||||
raise ValueError("At least two files are required for comparison")
|
||||
fingerprints = []
|
||||
for path in paths:
|
||||
fingerprints.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"size": path.stat().st_size,
|
||||
"sha256": sha256_file(path),
|
||||
}
|
||||
)
|
||||
identical = len({item["sha256"] for item in fingerprints}) == 1
|
||||
return {
|
||||
"status": "identical" if identical else "different",
|
||||
"files": fingerprints,
|
||||
}
|
||||
+25
-1
@@ -18,6 +18,7 @@ from serato_doctor.crate_parser import load_library_crates
|
||||
from serato_doctor.database_parser import parse_database
|
||||
from serato_doctor.duplicates import find_duplicate_groups
|
||||
from serato_doctor.health import analyze_health
|
||||
from serato_doctor.hashing import compare_audio_files
|
||||
from serato_doctor.matching import MatchingEngine, normalize
|
||||
from serato_doctor.models.crate import CrateKind
|
||||
from serato_doctor.models.duplicate import DuplicateKind
|
||||
@@ -357,6 +358,9 @@ def duplicate_repair_batch(
|
||||
if not plans:
|
||||
raise ValueError("Choose at least one duplicate group")
|
||||
replaced = [str(path) for plan in plans for path in plan.replaced]
|
||||
comparisons = [
|
||||
compare_audio_files((plan.keeper,) + plan.replaced) for plan in plans
|
||||
]
|
||||
result = {
|
||||
"choice_count": len(plans),
|
||||
"replaced": replaced,
|
||||
@@ -364,8 +368,9 @@ def duplicate_repair_batch(
|
||||
{
|
||||
"keeper": str(plan.keeper),
|
||||
"replaced": [str(path) for path in plan.replaced],
|
||||
"hash_status": comparison["status"],
|
||||
}
|
||||
for plan in plans
|
||||
for plan, comparison in zip(plans, comparisons)
|
||||
],
|
||||
"metadata_backups": len(plans[0].metadata_files),
|
||||
"strategy": "shortcut",
|
||||
@@ -438,6 +443,7 @@ class SeratoDoctorHandler(BaseHTTPRequestHandler):
|
||||
"/api/duplicates/apply",
|
||||
"/api/duplicates/batch/preview",
|
||||
"/api/duplicates/batch/apply",
|
||||
"/api/duplicates/hash",
|
||||
"/api/backups/restore",
|
||||
"/api/backups",
|
||||
"/api/reveal",
|
||||
@@ -471,6 +477,24 @@ class SeratoDoctorHandler(BaseHTTPRequestHandler):
|
||||
raise ValueError("That backup does not belong to this library")
|
||||
restored = restore_backup(backup)
|
||||
result = {"restored": [str(path) for path in restored]}
|
||||
elif self.path == "/api/duplicates/hash":
|
||||
tokens = payload["tokens"]
|
||||
if not isinstance(tokens, list) or not 2 <= len(tokens) <= 10:
|
||||
raise ValueError("Compare between 2 and 10 audio files")
|
||||
comparison = compare_audio_files(
|
||||
_verified_audio(token) for token in tokens
|
||||
)
|
||||
result = {
|
||||
"status": comparison["status"],
|
||||
"files": [
|
||||
{
|
||||
"path": item["path"],
|
||||
"size": item["size"],
|
||||
"fingerprint": item["sha256"][:12],
|
||||
}
|
||||
for item in comparison["files"]
|
||||
],
|
||||
}
|
||||
elif self.path.startswith("/api/duplicates/batch/"):
|
||||
raw_limit = payload.get("backup_limit", 10)
|
||||
backup_limit = None if raw_limit is None else int(raw_limit)
|
||||
|
||||
@@ -155,7 +155,7 @@ function renderReviewGroup() {
|
||||
drilldownSummary.textContent = 'Listen to each candidate, choose the keeper, and we’ll move to the next group. Skip anything uncertain.';
|
||||
drilldownList.innerHTML = `
|
||||
<article class="review-workspace">
|
||||
<div class="review-heading"><div><span>Comparing now</span><strong>${escapeHtml(group.filename)}</strong></div><span>${reviewState.choices.size} selected</span></div>
|
||||
<div class="review-heading"><div><span>Comparing now</span><strong>${escapeHtml(group.filename)}</strong></div><div class="review-signals"><span id="hash-confidence" class="hash-confidence checking">Checking file identity…</span><span>${reviewState.choices.size} selected</span></div></div>
|
||||
<div class="review-candidates">${group.file_previews.map((file, index) => `
|
||||
<section class="review-candidate ${chosen === file.path ? 'winner' : ''}">
|
||||
<span class="candidate-number">Option ${index + 1}</span>
|
||||
@@ -168,6 +168,31 @@ function renderReviewGroup() {
|
||||
<div class="review-navigation"><button type="button" data-review-previous ${reviewState.index === 0 ? 'disabled' : ''}>← Previous</button><button type="button" data-review-skip>Skip for now</button></div>
|
||||
</article>`;
|
||||
updateBatchSummary();
|
||||
loadHashConfidence(group, reviewState.index, reviewState.key);
|
||||
}
|
||||
|
||||
async function loadHashConfidence(group, groupIndex, detailKey) {
|
||||
const badge = document.querySelector('#hash-confidence');
|
||||
if (group.hashComparison) {
|
||||
showHashConfidence(badge, group.hashComparison);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const response = await fetch('/api/duplicates/hash', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({tokens: group.file_previews.map((file) => file.reveal_token)})});
|
||||
const result = await response.json();
|
||||
if (!response.ok) throw new Error(result.error || 'Identity check failed');
|
||||
group.hashComparison = result;
|
||||
if (reviewState?.index === groupIndex && reviewState?.key === detailKey) showHashConfidence(document.querySelector('#hash-confidence'), result);
|
||||
} catch (error) {
|
||||
if (reviewState?.index === groupIndex && reviewState?.key === detailKey && badge) { badge.className = 'hash-confidence unavailable'; badge.textContent = 'Identity not verified'; }
|
||||
}
|
||||
}
|
||||
|
||||
function showHashConfidence(badge, comparison) {
|
||||
if (!badge) return;
|
||||
badge.className = `hash-confidence ${comparison.status}`;
|
||||
badge.textContent = comparison.status === 'identical' ? '✓ Byte-for-byte identical' : '⚠ Files differ — listen carefully';
|
||||
badge.title = comparison.files.map((file) => `${formatBytes(file.size)} · ${file.fingerprint}`).join('\n');
|
||||
}
|
||||
|
||||
function updateBatchSummary() {
|
||||
@@ -200,7 +225,7 @@ previewRepairButton.addEventListener('click', async () => {
|
||||
previewedRepair = await requestRepair('/api/duplicates/batch/preview');
|
||||
batchPreviewSummary.textContent = `${previewedRepair.choice_count} keeper decision(s) · ${previewedRepair.replaced.length} duplicate file(s) consolidated`;
|
||||
batchPreviewList.innerHTML = previewedRepair.decisions.map((decision, index) => `
|
||||
<article class="preview-decision"><span>Decision ${index + 1}</span><div class="winner-path"><b>Keep</b><strong>${escapeHtml(decision.keeper.split('/').pop())}</strong><small>${escapeHtml(decision.keeper)}</small></div><div class="replaced-paths"><b>Replace with a shortcut</b>${decision.replaced.map((path) => `<small>${escapeHtml(path)}</small>`).join('')}</div></article>
|
||||
<article class="preview-decision"><span>Decision ${index + 1}<em class="hash-confidence ${decision.hash_status}">${decision.hash_status === 'identical' ? 'Identical' : 'Files differ'}</em></span><div class="winner-path"><b>Keep</b><strong>${escapeHtml(decision.keeper.split('/').pop())}</strong><small>${escapeHtml(decision.keeper)}</small></div><div class="replaced-paths"><b>Replace with a shortcut</b>${decision.replaced.map((path) => `<small>${escapeHtml(path)}</small>`).join('')}</div></article>
|
||||
`).join('');
|
||||
batchPreviewSafety.textContent = `${previewedRepair.metadata_backups} Serato metadata file(s) and every replaced audio file will be backed up. Database V2 will not be modified.`;
|
||||
repairPreview.innerHTML = `<strong>Preview approved</strong><p>${previewedRepair.choice_count} keeper decision(s) are ready for one backed-up apply.</p>`;
|
||||
|
||||
@@ -35,6 +35,46 @@
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.review-signals {
|
||||
display: flex;
|
||||
align-items: flex-end;
|
||||
gap: 7px;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.hash-confidence {
|
||||
display: inline-block;
|
||||
padding: 5px 8px;
|
||||
border-radius: 999px;
|
||||
font-size: 9px!important;
|
||||
font-style: normal;
|
||||
font-weight: 750;
|
||||
letter-spacing: .02em!important;
|
||||
text-transform: none!important;
|
||||
}
|
||||
|
||||
.hash-confidence.checking,
|
||||
.hash-confidence.unavailable {
|
||||
color: var(--muted);
|
||||
background: rgba(255, 255, 255, .05);
|
||||
}
|
||||
|
||||
.hash-confidence.identical {
|
||||
color: #8ce4b8;
|
||||
background: rgba(84, 212, 154, .11);
|
||||
}
|
||||
|
||||
.hash-confidence.different {
|
||||
color: var(--amber);
|
||||
background: rgba(245, 185, 76, .1);
|
||||
}
|
||||
|
||||
.preview-decision > span .hash-confidence {
|
||||
display: block;
|
||||
width: max-content;
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.review-candidates {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
|
||||
Reference in New Issue
Block a user