Compare commits

..

1 Commits

Author SHA1 Message Date
Philip Guzman c4f1b4535d Add backup-first duplicate repair 2026-07-01 15:29:44 -07:00
9 changed files with 477 additions and 13 deletions
+39
View File
@@ -0,0 +1,39 @@
# Duplicate repair
## Problem
Duplicate and cloud-conflict files waste space, but deleting either path can
unmap tracks in crates or Serato's database. DJs need to choose the authoritative
copy and understand every change before it happens.
## Design
The web interface requires an analysis, an explicit keeper selection, and a
dry-run preview. Applying the plan first creates a timestamped backup beneath
`_Serato_/.serato-doctor-backups/`. The snapshot contains every replaced audio
file, loaded crate/smart-crate metadata, database V2, and a JSON restore manifest.
The non-kept audio path is then replaced with a symbolic link to the keeper.
This removes the extra audio payload while preserving every existing saved path.
Crate files and database V2 are never rewritten. The UI exposes immediate
restore using the manifest.
Users may retain all backups or set a positive rotation limit. Rotation occurs
only after a repair succeeds.
## Edge cases
- The duplicate group is rescanned and validated immediately before preview and
apply.
- Existing symlinks cannot be selected as disposable duplicate files.
- A partial failure restores already-changed files before reporting the error.
- Restore refuses to overwrite a real file.
- Healthy symlink aliases are excluded from future duplicate counts.
## Tests
- Backup creation includes audio and Serato metadata.
- The old path resolves to the selected keeper after repair.
- Restore returns the original file contents.
- Invalid keeper choices are rejected.
- Limited and unlimited retention behave deterministically.
+3 -1
View File
@@ -11,7 +11,9 @@ def find_duplicate_groups(
) -> Tuple[DuplicateGroup, ...]:
"""Find exact-name duplicates and suspected numeric conflict copies."""
track_tuple = tuple(tracks)
# Healthy symlinks preserve legacy Serato paths without consuming another
# copy of the audio, so they are aliases rather than duplicate files.
track_tuple = tuple(track for track in tracks if not track.path.is_symlink())
by_name: DefaultDict[str, List[DiskTrack]] = defaultdict(list)
by_conflict_name: DefaultDict[str, List[DiskTrack]] = defaultdict(list)
+160
View File
@@ -0,0 +1,160 @@
"""Backup-first duplicate consolidation.
Serato's binary metadata is deliberately not rewritten. Removed duplicate files
are replaced with symbolic links, so every existing path continues to resolve.
"""
import json
import shutil
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Iterable, Optional, Tuple
from uuid import uuid4
BACKUP_FOLDER = ".serato-doctor-backups"
@dataclass(frozen=True)
class DuplicateRepairPlan:
keeper: Path
replaced: Tuple[Path, ...]
metadata_files: Tuple[Path, ...]
@property
def changes(self) -> int:
return len(self.replaced)
@dataclass(frozen=True)
class RepairReceipt:
backup: Path
keeper: Path
replaced: Tuple[Path, ...]
def plan_duplicate_repair(
keeper: Path, duplicates: Iterable[Path], serato_root: Path
) -> DuplicateRepairPlan:
keeper = keeper.expanduser().resolve()
candidates = tuple(path.expanduser().resolve() for path in duplicates)
if keeper not in candidates:
raise ValueError("The file to keep must belong to this duplicate group")
if not keeper.is_file():
raise ValueError(f"The file to keep no longer exists: {keeper}")
replaced = tuple(path for path in candidates if path != keeper)
if not replaced:
raise ValueError("Choose a duplicate group containing at least two files")
if any(not path.is_file() or path.is_symlink() for path in replaced):
raise ValueError("A duplicate changed since the analysis; analyze again")
metadata = tuple(
sorted(
path
for path in serato_root.expanduser().resolve().rglob("*")
if path.is_file()
and BACKUP_FOLDER not in path.parts
and (
path.name == "database V2"
or path.suffix.casefold() in {".crate", ".scrate"}
)
)
)
return DuplicateRepairPlan(keeper, replaced, metadata)
def apply_duplicate_repair(
plan: DuplicateRepairPlan,
serato_root: Path,
backup_limit: Optional[int] = 10,
) -> RepairReceipt:
"""Create a complete rollback snapshot, then replace extras with symlinks."""
if backup_limit is not None and backup_limit < 1:
raise ValueError("Backup limit must be at least 1, or unlimited")
backup_root = serato_root.expanduser().resolve() / BACKUP_FOLDER
backup = backup_root / _backup_name()
files_root = backup / "files"
metadata_root = backup / "serato-metadata"
backup.mkdir(parents=True)
entries = []
try:
for path in plan.replaced:
destination = files_root / _safe_backup_path(path)
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(path, destination)
entries.append({"original": str(path), "backup": str(destination)})
for path in plan.metadata_files:
relative = path.relative_to(serato_root.expanduser().resolve())
destination = metadata_root / relative
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(path, destination)
manifest = {
"created_at": datetime.now(timezone.utc).isoformat(),
"keeper": str(plan.keeper),
"replaced": entries,
"strategy": "symlink",
}
(backup / "manifest.json").write_text(
json.dumps(manifest, indent=2), encoding="utf-8"
)
for path in plan.replaced:
path.unlink()
path.symlink_to(plan.keeper)
except Exception:
_rollback_entries(entries)
shutil.rmtree(backup, ignore_errors=True)
raise
rotate_backups(backup_root, backup_limit)
return RepairReceipt(backup, plan.keeper, plan.replaced)
def restore_backup(backup: Path) -> Tuple[Path, ...]:
manifest_path = backup.expanduser().resolve() / "manifest.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
restored = []
for entry in manifest["replaced"]:
original = Path(entry["original"])
saved = Path(entry["backup"])
if original.exists() and not original.is_symlink():
raise ValueError(f"Restore would overwrite a real file: {original}")
if original.is_symlink():
original.unlink()
original.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(saved, original)
restored.append(original)
return tuple(restored)
def rotate_backups(backup_root: Path, limit: Optional[int]) -> None:
if limit is None or not backup_root.is_dir():
return
backups = sorted(
(path for path in backup_root.iterdir() if (path / "manifest.json").is_file()),
key=lambda path: path.name,
reverse=True,
)
for expired in backups[limit:]:
shutil.rmtree(expired)
def _backup_name() -> str:
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
return f"{stamp}-{uuid4().hex[:8]}"
def _safe_backup_path(path: Path) -> Path:
anchorless = path.as_posix().lstrip("/").replace(":", "_")
return Path(anchorless)
def _rollback_entries(entries: Iterable[dict]) -> None:
for entry in entries:
original = Path(entry["original"])
saved = Path(entry["backup"])
if original.is_symlink():
original.unlink()
if not original.exists() and saved.is_file():
original.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(saved, original)
+83 -7
View File
@@ -5,7 +5,7 @@ from dataclasses import asdict
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from importlib import resources
from pathlib import Path
from typing import Iterable
from typing import Iterable, Optional
from serato_doctor.crate_parser import load_library_crates
from serato_doctor.database_parser import parse_database
@@ -16,6 +16,12 @@ from serato_doctor.models.crate import CrateKind
from serato_doctor.models.duplicate import DuplicateKind
from serato_doctor.models.library import Library
from serato_doctor.scanner import scan_filesystem
from serato_doctor.repair import (
BACKUP_FOLDER,
apply_duplicate_repair,
plan_duplicate_repair,
restore_backup,
)
MAX_REQUEST_BYTES = 64 * 1024
@@ -225,6 +231,44 @@ def analyze_paths(
return result
def duplicate_repair(
serato: Path,
music: Path,
keeper: Path,
group_files: Iterable[Path],
backup_limit: Optional[int],
apply: bool = False,
) -> dict:
"""Validate a current duplicate group and preview or apply consolidation."""
serato = serato.expanduser().resolve()
music = music.expanduser().resolve()
keeper = keeper.expanduser().resolve()
requested = tuple(path.expanduser().resolve() for path in group_files)
if not serato.is_dir() or not music.is_dir():
raise ValueError("Analyze the library again before repairing duplicates")
tracks = scan_filesystem(music).tracks
groups = find_duplicate_groups(tracks)
valid_groups = [
{track.path.resolve() for track in group.tracks} for group in groups
]
if set(requested) not in valid_groups:
raise ValueError("This duplicate group changed; analyze the library again")
plan = plan_duplicate_repair(keeper, requested, serato)
result = {
"keeper": str(plan.keeper),
"replaced": [str(path) for path in plan.replaced],
"metadata_backups": len(plan.metadata_files),
"strategy": "shortcut",
"database_v2_modified": False,
}
if apply:
receipt = apply_duplicate_repair(plan, serato, backup_limit)
result.update({"applied": True, "backup": str(receipt.backup)})
else:
result["applied"] = False
return result
class SeratoDoctorHandler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
asset = STATIC_FILES.get(self.path)
@@ -244,7 +288,13 @@ class SeratoDoctorHandler(BaseHTTPRequestHandler):
self.wfile.write(content)
def do_POST(self) -> None:
if self.path != "/api/analyze":
allowed = {
"/api/analyze",
"/api/duplicates/preview",
"/api/duplicates/apply",
"/api/backups/restore",
}
if self.path not in allowed:
self._json_response(404, {"error": "Not found"})
return
try:
@@ -254,11 +304,37 @@ class SeratoDoctorHandler(BaseHTTPRequestHandler):
payload = json.loads(self.rfile.read(length))
if not isinstance(payload, dict):
raise ValueError("Request body must be a JSON object")
roots = [Path(value) for value in payload.get("reference_roots", [])]
result = analyze_paths(
Path(payload["serato"]), Path(payload["music"]), roots
)
except (KeyError, TypeError, json.JSONDecodeError, ValueError) as error:
if self.path == "/api/analyze":
roots = [Path(value) for value in payload.get("reference_roots", [])]
result = analyze_paths(
Path(payload["serato"]), Path(payload["music"]), roots
)
elif self.path == "/api/backups/restore":
serato = Path(payload["serato"]).expanduser().resolve()
backup = Path(payload["backup"]).expanduser().resolve()
backup_root = (serato / BACKUP_FOLDER).resolve()
if backup.parent != backup_root:
raise ValueError("That backup does not belong to this library")
restored = restore_backup(backup)
result = {"restored": [str(path) for path in restored]}
else:
raw_limit = payload.get("backup_limit", 10)
backup_limit = None if raw_limit is None else int(raw_limit)
result = duplicate_repair(
Path(payload["serato"]),
Path(payload["music"]),
Path(payload["keeper"]),
(Path(value) for value in payload["group_files"]),
backup_limit,
apply=self.path.endswith("/apply"),
)
except (
KeyError,
TypeError,
OSError,
json.JSONDecodeError,
ValueError,
) as error:
self._json_response(400, {"error": str(error)})
return
self._json_response(200, result)
File diff suppressed because one or more lines are too long
+84 -2
View File
@@ -8,7 +8,17 @@ const drilldownTitle = document.querySelector('#drilldown-title');
const drilldownCount = document.querySelector('#drilldown-count');
const drilldownSummary = document.querySelector('#drilldown-summary');
const drilldownList = document.querySelector('#drilldown-list');
const repairPanel = document.querySelector('#duplicate-repair');
const repairChoice = document.querySelector('#repair-choice');
const repairPreview = document.querySelector('#repair-preview');
const repairMessage = document.querySelector('#repair-message');
const previewRepairButton = document.querySelector('#preview-repair');
const applyRepairButton = document.querySelector('#apply-repair');
const restoreRepairButton = document.querySelector('#restore-repair');
let latestAnalysis = null;
let selectedDuplicateGroup = null;
let previewedRepair = null;
let latestBackup = null;
function expandHome(path) {
return path.trim();
@@ -51,19 +61,91 @@ function renderDetail(key) {
drilldownCount.textContent = `${detail.total ?? 0} found`;
drilldownSummary.textContent = detail.summary;
repairPanel.hidden = true;
selectedDuplicateGroup = null;
previewedRepair = null;
latestBackup = null;
restoreRepairButton.hidden = true;
if (!detail.items?.length) {
drilldownList.innerHTML = '<div class="empty-detail">Nothing to review here. Tiny victory parade, very tasteful.</div>';
return;
}
drilldownList.innerHTML = detail.items.map((item) => `
<article class="detail-item">
drilldownList.innerHTML = detail.items.map((item, index) => `
<article class="detail-item${item.files ? ' selectable-duplicate' : ''}" ${item.files ? `data-duplicate-index="${index}" role="button" tabindex="0"` : ''}>
<strong>${escapeHtml(item.filename || item.path || 'Untitled item')}</strong>
<ul>${detailLines(item)}</ul>
${item.files ? '<small>Choose this group to review a safe cleanup →</small>' : ''}
</article>
`).join('');
}
function chooseDuplicate(detailKey, index) {
const group = latestAnalysis?.details?.[detailKey]?.items?.[index];
if (!group?.files) return;
selectedDuplicateGroup = group;
previewedRepair = null;
applyRepairButton.disabled = true;
repairPreview.hidden = true;
repairMessage.textContent = '';
repairChoice.innerHTML = group.files.map((file, fileIndex) => `
<label class="keeper-option"><input type="radio" name="keeper" value="${escapeHtml(file)}" ${fileIndex === 0 ? 'checked' : ''}><span><strong>${fileIndex === 0 ? 'Keep this file' : 'Keep instead'}</strong><small>${escapeHtml(file)}</small></span></label>
`).join('');
repairPanel.hidden = false;
repairPanel.scrollIntoView({behavior: 'smooth', block: 'start'});
}
function repairPayload() {
const keeper = document.querySelector('input[name="keeper"]:checked')?.value;
if (!selectedDuplicateGroup || !keeper) throw new Error('Choose a file to keep');
return {serato: expandHome(document.querySelector('#serato-path').value), music: expandHome(document.querySelector('#music-path').value), keeper, group_files: selectedDuplicateGroup.files, backup_limit: document.querySelector('#keep-all-backups').checked ? null : Number(document.querySelector('#backup-limit').value)};
}
async function requestRepair(endpoint) {
const response = await fetch(endpoint, {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify(repairPayload())});
const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Duplicate cleanup failed');
return data;
}
previewRepairButton.addEventListener('click', async () => {
repairMessage.textContent = 'Checking the plan…'; applyRepairButton.disabled = true;
try {
previewedRepair = await requestRepair('/api/duplicates/preview');
repairPreview.innerHTML = `<strong>Ready to protect and consolidate</strong><p>${previewedRepair.replaced.length} duplicate file(s) will be backed up, then replaced with shortcuts to the keeper. ${previewedRepair.metadata_backups} Serato metadata file(s) will also be copied into the rollback snapshot. Database V2 will not be changed.</p>`;
repairPreview.hidden = false; applyRepairButton.disabled = false;
repairMessage.textContent = 'Preview complete. Nothing has changed yet.';
} catch (error) { repairMessage.textContent = error.message; }
});
applyRepairButton.addEventListener('click', async () => {
if (!previewedRepair) return;
applyRepairButton.disabled = true; repairMessage.textContent = 'Creating the backup before making changes…';
try {
const result = await requestRepair('/api/duplicates/apply');
latestBackup = result.backup;
repairMessage.textContent = `Cleanup complete. Restore backup: ${result.backup}`;
previewRepairButton.disabled = true;
restoreRepairButton.hidden = false;
} catch (error) { repairMessage.textContent = error.message; applyRepairButton.disabled = false; }
});
restoreRepairButton.addEventListener('click', async () => {
if (!latestBackup) return;
restoreRepairButton.disabled = true; repairMessage.textContent = 'Restoring the duplicate files…';
try {
const response = await fetch('/api/backups/restore', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({serato: expandHome(document.querySelector('#serato-path').value), backup: latestBackup})});
const result = await response.json();
if (!response.ok) throw new Error(result.error || 'Restore failed');
repairMessage.textContent = `Restore complete. ${result.restored.length} original file(s) returned.`;
restoreRepairButton.hidden = true;
} catch (error) { repairMessage.textContent = error.message; restoreRepairButton.disabled = false; }
});
repairChoice.addEventListener('change', () => { previewedRepair = null; applyRepairButton.disabled = true; repairPreview.hidden = true; repairMessage.textContent = 'Keeper changed. Preview the plan again.'; });
drilldownList.addEventListener('click', (event) => { const item = event.target.closest('[data-duplicate-index]'); if (item) chooseDuplicate(document.querySelector('.drill-trigger.selected')?.dataset.detail, Number(item.dataset.duplicateIndex)); });
drilldownList.addEventListener('keydown', (event) => { if (event.key !== 'Enter' && event.key !== ' ') return; const item = event.target.closest('[data-duplicate-index]'); if (item) { event.preventDefault(); chooseDuplicate(document.querySelector('.drill-trigger.selected')?.dataset.detail, Number(item.dataset.duplicateIndex)); } });
function render(data) {
latestAnalysis = data;
document.querySelectorAll('[data-field]').forEach((element) => {
+18 -1
View File
@@ -23,7 +23,7 @@
</nav>
<div class="safety-card">
<span class="safety-icon"></span>
<div><strong>Read-only mode</strong><p>Your library will not be modified.</p></div>
<div><strong>Protected mode</strong><p>Analysis is read-only. Repairs require a preview and backup.</p></div>
</div>
<div class="sidebar-foot">Local interface · v0.1</div>
</aside>
@@ -88,6 +88,23 @@
<p id="drilldown-summary">Click a metric above to see example files and saved paths behind that number.</p>
<div id="drilldown-list" class="detail-list"></div>
</div>
<div id="duplicate-repair" class="repair-panel panel" hidden>
<div class="section-heading"><div><p class="eyebrow">Backup-first cleanup</p><h2>Safely consolidate duplicates</h2></div><span class="repair-tag">Preview required</span></div>
<p>Choose the real file to keep. Every other path will be backed up and replaced with a shortcut to it, so existing Serato crates remain mapped. Seratos database V2 is never edited.</p>
<div id="repair-choice" class="repair-choice"></div>
<div class="backup-options">
<label>Backups to keep <input id="backup-limit" type="number" min="1" value="10"></label>
<label class="check-label"><input id="keep-all-backups" type="checkbox"> Keep every backup</label>
</div>
<div id="repair-preview" class="repair-preview" hidden></div>
<div class="repair-actions">
<button id="preview-repair" type="button">Preview changes</button>
<button id="apply-repair" class="danger-action" type="button" disabled>Apply backed-up cleanup</button>
<button id="restore-repair" type="button" hidden>Restore this backup</button>
</div>
<div id="repair-message" class="repair-message" role="status"></div>
</div>
</section>
</main>
</div>
+67
View File
@@ -0,0 +1,67 @@
from pathlib import Path
import pytest
from serato_doctor.repair import (
BACKUP_FOLDER,
apply_duplicate_repair,
plan_duplicate_repair,
restore_backup,
rotate_backups,
)
def library(tmp_path):
serato = tmp_path / "_Serato_"
crate = serato / "Subcrates" / "House.crate"
crate.parent.mkdir(parents=True)
crate.write_bytes(b"crate-data")
(serato / "database V2").write_bytes(b"database-data")
music = tmp_path / "Music"
keeper = music / "Main" / "Track.mp3"
duplicate = music / "Old" / "Track.mp3"
keeper.parent.mkdir(parents=True)
duplicate.parent.mkdir(parents=True)
keeper.write_bytes(b"keeper")
duplicate.write_bytes(b"duplicate")
return serato, keeper, duplicate
def test_duplicate_repair_backs_up_then_preserves_old_path_as_link(tmp_path):
serato, keeper, duplicate = library(tmp_path)
plan = plan_duplicate_repair(keeper, (keeper, duplicate), serato)
receipt = apply_duplicate_repair(plan, serato, backup_limit=3)
assert duplicate.is_symlink()
assert duplicate.resolve() == keeper.resolve()
assert (receipt.backup / "manifest.json").is_file()
assert any((receipt.backup / "serato-metadata").rglob("House.crate"))
assert any((receipt.backup / "serato-metadata").rglob("database V2"))
restored = restore_backup(receipt.backup)
assert restored == (duplicate.resolve(),)
assert not duplicate.is_symlink()
assert duplicate.read_bytes() == b"duplicate"
def test_plan_rejects_keeper_outside_duplicate_group(tmp_path):
serato, keeper, duplicate = library(tmp_path)
outsider = tmp_path / "outsider.mp3"
outsider.write_bytes(b"other")
with pytest.raises(ValueError, match="must belong"):
plan_duplicate_repair(outsider, (keeper, duplicate), serato)
def test_backup_rotation_can_be_limited_or_unlimited(tmp_path):
root = tmp_path / BACKUP_FOLDER
for name in ("001", "002", "003"):
backup = root / name
backup.mkdir(parents=True)
(backup / "manifest.json").write_text("{}", encoding="utf-8")
rotate_backups(root, None)
assert len(list(root.iterdir())) == 3
rotate_backups(root, 2)
assert {path.name for path in root.iterdir()} == {"002", "003"}
+22 -1
View File
@@ -3,7 +3,7 @@ from pathlib import Path
import pytest
from serato_doctor.web import STATIC_FILES, analyze_paths
from serato_doctor.web import STATIC_FILES, analyze_paths, duplicate_repair
def test_web_analysis_uses_production_health_pipeline(tmp_path):
@@ -49,3 +49,24 @@ def test_web_static_assets_are_declared_and_packaged():
assert 'data-detail="database_missing_tracks"' in html
assert 'data-detail="old_crate_references"' in html
assert html.count('class="info-button"') >= 10
def test_duplicate_repair_preview_does_not_change_files(tmp_path):
serato = tmp_path / "_Serato_"
serato.mkdir()
music = tmp_path / "Music"
first = music / "A" / "Track.mp3"
second = music / "B" / "Track.mp3"
first.parent.mkdir(parents=True)
second.parent.mkdir(parents=True)
first.write_bytes(b"first")
second.write_bytes(b"second")
result = duplicate_repair(
serato, music, first, (first, second), backup_limit=10
)
assert result["applied"] is False
assert result["database_v2_modified"] is False
assert second.read_bytes() == b"second"
assert not second.is_symlink()