Library indexing and filesystem scanner with incremental rescans

2026-08-22 23:31:17 -05:00
parent d4ea7a4ca9
commit 49dbee617a
5 changed files with 915 additions and 0 deletions
@@ -0,0 +1,276 @@
+// Package scanner walks configured music folders, detects new/changed/
+// deleted/renamed files, and indexes them into the library via a bounded
+// worker pool.
+package scanner
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"git.noroutes.net/NoRoutes/vibetunes/internal/database"
+	"git.noroutes.net/NoRoutes/vibetunes/internal/library"
+	"git.noroutes.net/NoRoutes/vibetunes/internal/metadata"
+)
+
+// recognizedExtensions are the file extensions VibeSonic attempts to
+// probe as media. Anything else (art, .nfo, playlists, ...) is skipped
+// during a scan rather than passed to ffprobe.
+var recognizedExtensions = map[string]bool{
+	".mp3": true, ".flac": true, ".ogg": true, ".oga": true, ".opus": true,
+	".aac": true, ".m4a": true, ".m4b": true, ".alac": true,
+	".wav": true, ".aiff": true, ".aif": true, ".wma": true, ".mp4": true,
+}
+
+const maxWorkers = 4
+
+// Scanner indexes one or more music folders into the database.
+type Scanner struct {
+	DB     *sql.DB
+	Prober *metadata.Prober
+	Log    *slog.Logger
+
+	mu       sync.Mutex
+	scanning map[string]bool // music_folder_id -> in progress
+}
+
+func New(db *sql.DB, prober *metadata.Prober, log *slog.Logger) *Scanner {
+	return &Scanner{DB: db, Prober: prober, Log: log, scanning: make(map[string]bool)}
+}
+
+// ScanFolder walks a single music folder and indexes it incrementally:
+// unchanged files (same size+mtime) are only marked seen, changed files
+// are reprobed, moved files are matched by content hash and just get
+// their path updated, and files no longer found on disk are marked
+// missing rather than deleted outright.
+func (s *Scanner) ScanFolder(ctx context.Context, folderID string) error {
+	if !s.beginScan(folderID) {
+		return fmt.Errorf("scan already in progress for folder %s", folderID)
+	}
+	defer s.endScan(folderID)
+
+	folder, err := database.GetMusicFolder(s.DB, folderID)
+	if err != nil {
+		return err
+	}
+
+	startedAt := time.Now()
+	if err := database.StartScan(s.DB, folderID); err != nil {
+		return err
+	}
+
+	var seen, added, updated int
+	var mu sync.Mutex
+	sem := make(chan struct{}, maxWorkers)
+	var wg sync.WaitGroup
+	var firstErr error
+
+	walkErr := filepath.WalkDir(folder.Path, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			s.Log.Warn("scan: walk error", "path", path, "err", err)
+			return nil // keep scanning the rest of the tree
+		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		if d.IsDir() {
+			return nil
+		}
+		if !recognizedExtensions[strings.ToLower(filepath.Ext(path))] {
+			return nil
+		}
+		rel, err := filepath.Rel(folder.Path, path)
+		if err != nil {
+			return nil
+		}
+		rel = filepath.ToSlash(rel)
+
+		wg.Add(1)
+		sem <- struct{}{}
+		go func() {
+			defer wg.Done()
+			defer func() { <-sem }()
+
+			wasAdded, wasUpdated, err := s.indexOne(ctx, folder.ID, path, rel)
+			mu.Lock()
+			defer mu.Unlock()
+			seen++
+			if err != nil {
+				s.Log.Warn("scan: index failed", "path", rel, "err", err)
+			} else {
+				if wasAdded {
+					added++
+				} else if wasUpdated {
+					updated++
+				}
+			}
+			if seen%25 == 0 {
+				_ = database.UpdateScanProgress(s.DB, folder.ID, seen, added, updated)
+			}
+		}()
+		return nil
+	})
+	wg.Wait()
+	if walkErr != nil && firstErr == nil {
+		firstErr = walkErr
+	}
+
+	_ = database.UpdateScanProgress(s.DB, folder.ID, seen, added, updated)
+
+	removed, err := database.MarkFolderFilesMissingExcept(s.DB, folder.ID, startedAt)
+	if err != nil && firstErr == nil {
+		firstErr = err
+	}
+
+	errMsg := ""
+	if firstErr != nil {
+		errMsg = firstErr.Error()
+	}
+	if err := database.FinishScan(s.DB, folder.ID, int(removed), errMsg); err != nil {
+		return err
+	}
+	return firstErr
+}
+
+// indexOne probes and indexes a single file, returning whether it was
+// newly added or updated so the caller can track progress.
+func (s *Scanner) indexOne(ctx context.Context, folderID, absPath, relPath string) (added, updated bool, err error) {
+	info, err := os.Stat(absPath)
+	if err != nil {
+		return false, false, err
+	}
+	size := info.Size()
+	mtime := info.ModTime().Unix()
+	hash := fingerprint(size, mtime)
+
+	existing, err := database.FindMediaFileByPath(s.DB, folderID, relPath)
+	if err == nil {
+		if existing.SizeBytes == size && existing.MtimeUnix == mtime {
+			return false, false, database.TouchMediaFileScanned(s.DB, existing.ID)
+		}
+		// Changed: reprobe and update in place; the media_file (and
+		// therefore song) ID stays the same.
+		tags, perr := s.Prober.Probe(ctx, absPath)
+		if perr != nil {
+			return false, false, perr
+		}
+		if err := database.UpdateMediaFileScanned(s.DB, existing.ID, size, mtime, hash, tags.Format, tags.Kind, tags.DurationSeconds, tags.BitrateKbps, tags.SampleRateHz, tags.Channels); err != nil {
+			return false, false, err
+		}
+		mf, err := database.GetMediaFile(s.DB, existing.ID)
+		if err != nil {
+			return false, false, err
+		}
+		if _, err := library.IndexFile(s.DB, mf, tags); err != nil {
+			return false, false, err
+		}
+		return false, true, nil
+	}
+	if err != database.ErrNotFound {
+		return false, false, err
+	}
+
+	// Not found by path. Check whether this is a rename/move of a file we
+	// already indexed (same content fingerprint, different path).
+	if renamed, rerr := database.FindMediaFileByHash(s.DB, folderID, hash); rerr == nil {
+		if err := database.RenameMediaFile(s.DB, renamed.ID, relPath); err != nil {
+			return false, false, err
+		}
+		return false, true, nil
+	}
+
+	// Genuinely new file.
+	tags, err := s.Prober.Probe(ctx, absPath)
+	if err != nil {
+		return false, false, err
+	}
+	mf, err := database.CreateMediaFile(s.DB, folderID, relPath, size, mtime, hash, tags.Format, tags.Kind, tags.DurationSeconds, tags.BitrateKbps, tags.SampleRateHz, tags.Channels)
+	if err != nil {
+		return false, false, err
+	}
+	if _, err := library.IndexFile(s.DB, mf, tags); err != nil {
+		return false, false, err
+	}
+	return true, false, nil
+}
+
+// fingerprint is a cheap, fast rename/change detector based on size and
+// mtime rather than hashing file content (which would mean reading every
+// byte of every file on every scan).
+func fingerprint(size, mtime int64) string {
+	return fmt.Sprintf("%d-%d", size, mtime)
+}
+
+func (s *Scanner) beginScan(folderID string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.scanning[folderID] {
+		return false
+	}
+	s.scanning[folderID] = true
+	return true
+}
+
+func (s *Scanner) endScan(folderID string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	delete(s.scanning, folderID)
+}
+
+// IsScanning reports whether a scan of folderID is currently running.
+func (s *Scanner) IsScanning(folderID string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.scanning[folderID]
+}
+
+// ScanAllEnabled scans every enabled music folder sequentially (folders
+// run one at a time; files within a folder run concurrently via the
+// worker pool above).
+func (s *Scanner) ScanAllEnabled(ctx context.Context) error {
+	folders, err := database.ListMusicFolders(s.DB)
+	if err != nil {
+		return err
+	}
+	var firstErr error
+	for _, f := range folders {
+		if !f.Enabled {
+			continue
+		}
+		if err := s.ScanFolder(ctx, f.ID); err != nil && firstErr == nil {
+			firstErr = err
+		}
+	}
+	return firstErr
+}
+
+// RunScheduled runs ScanAllEnabled once at startup and then on the given
+// interval, until ctx is cancelled. interval <= 0 disables the periodic
+// rescan (startup scan still runs).
+func (s *Scanner) RunScheduled(ctx context.Context, interval time.Duration) {
+	if err := s.ScanAllEnabled(ctx); err != nil {
+		s.Log.Warn("startup scan finished with errors", "err", err)
+	}
+	if interval <= 0 {
+		return
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if err := s.ScanAllEnabled(ctx); err != nil {
+				s.Log.Warn("scheduled scan finished with errors", "err", err)
+			}
+		}
+	}
+}
@@ -0,0 +1,239 @@
+package scanner
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"git.noroutes.net/NoRoutes/vibetunes/internal/database"
+	"git.noroutes.net/NoRoutes/vibetunes/internal/metadata"
+)
+
+func testLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, nil))
+}
+
+func genFile(t *testing.T, dir, name, title, artist string) string {
+	t.Helper()
+	if _, err := exec.LookPath("ffmpeg"); err != nil {
+		t.Skip("ffmpeg not installed, skipping scanner integration test")
+	}
+	out := filepath.Join(dir, name)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "ffmpeg",
+		"-f", "lavfi", "-i", "sine=frequency=440:duration=1",
+		"-metadata", "title="+title,
+		"-metadata", "artist="+artist,
+		"-metadata", "album=Scan Test Album",
+		"-codec:a", "libmp3lame", "-b:a", "64k",
+		out,
+	)
+	if combined, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("ffmpeg failed: %v\n%s", err, combined)
+	}
+	return out
+}
+
+func newTestScanner(t *testing.T) (*Scanner, *database.MusicFolder, string) {
+	t.Helper()
+	if _, err := exec.LookPath("ffprobe"); err != nil {
+		t.Skip("ffprobe not installed, skipping scanner integration test")
+	}
+	dbPath := filepath.Join(t.TempDir(), "test.db")
+	db, err := database.Open(dbPath)
+	if err != nil {
+		t.Fatalf("database.Open: %v", err)
+	}
+	t.Cleanup(func() { db.Close() })
+
+	musicDir := t.TempDir()
+	folder, err := database.CreateMusicFolder(db, "Test Music", musicDir, "music")
+	if err != nil {
+		t.Fatalf("CreateMusicFolder: %v", err)
+	}
+
+	s := New(db, &metadata.Prober{}, testLogger())
+	return s, folder, musicDir
+}
+
+func TestScanFolderAddsNewFiles(t *testing.T) {
+	s, folder, dir := newTestScanner(t)
+	genFile(t, dir, "song1.mp3", "Song One", "Artist One")
+	genFile(t, dir, "song2.mp3", "Song Two", "Artist One")
+
+	if err := s.ScanFolder(context.Background(), folder.ID); err != nil {
+		t.Fatalf("ScanFolder: %v", err)
+	}
+
+	files, err := database.ListMediaFilesForFolder(s.DB, folder.ID)
+	if err != nil {
+		t.Fatalf("ListMediaFilesForFolder: %v", err)
+	}
+	if len(files) != 2 {
+		t.Fatalf("got %d media files, want 2", len(files))
+	}
+
+	state, err := database.GetScanState(s.DB, folder.ID)
+	if err != nil {
+		t.Fatalf("GetScanState: %v", err)
+	}
+	if state.FilesAdded != 2 {
+		t.Errorf("FilesAdded = %d, want 2", state.FilesAdded)
+	}
+	if state.Status != "idle" {
+		t.Errorf("Status = %q, want idle", state.Status)
+	}
+}
+
+func TestScanFolderIncrementalNoReprobe(t *testing.T) {
+	s, folder, dir := newTestScanner(t)
+	genFile(t, dir, "song1.mp3", "Song One", "Artist One")
+
+	if err := s.ScanFolder(context.Background(), folder.ID); err != nil {
+		t.Fatalf("ScanFolder (1st): %v", err)
+	}
+	files1, _ := database.ListMediaFilesForFolder(s.DB, folder.ID)
+	id1 := files1[0].ID
+
+	if err := s.ScanFolder(context.Background(), folder.ID); err != nil {
+		t.Fatalf("ScanFolder (2nd): %v", err)
+	}
+	files2, _ := database.ListMediaFilesForFolder(s.DB, folder.ID)
+	if len(files2) != 1 {
+		t.Fatalf("got %d media files after rescan, want 1 (no duplicates)", len(files2))
+	}
+	if files2[0].ID != id1 {
+		t.Errorf("media file ID changed across rescan: %s -> %s", id1, files2[0].ID)
+	}
+
+	state, err := database.GetScanState(s.DB, folder.ID)
+	if err != nil {
+		t.Fatalf("GetScanState: %v", err)
+	}
+	if state.FilesAdded != 0 || state.FilesUpdated != 0 {
+		t.Errorf("second scan should add/update nothing, got added=%d updated=%d", state.FilesAdded, state.FilesUpdated)
+	}
+}
+
+func TestScanFolderMarksDeletedFilesMissing(t *testing.T) {
+	s, folder, dir := newTestScanner(t)
+	path := genFile(t, dir, "song1.mp3", "Song One", "Artist One")
+
+	if err := s.ScanFolder(context.Background(), folder.ID); err != nil {
+		t.Fatalf("ScanFolder (1st): %v", err)
+	}
+	if err := os.Remove(path); err != nil {
+		t.Fatalf("os.Remove: %v", err)
+	}
+	if err := s.ScanFolder(context.Background(), folder.ID); err != nil {
+		t.Fatalf("ScanFolder (2nd): %v", err)
+	}
+
+	files, err := database.ListMediaFilesForFolder(s.DB, folder.ID)
+	if err != nil {
+		t.Fatalf("ListMediaFilesForFolder: %v", err)
+	}
+	if len(files) != 1 {
+		t.Fatalf("got %d media files, want 1 (row kept, not deleted)", len(files))
+	}
+	if !files[0].Missing {
+		t.Error("expected the removed file's row to be marked missing")
+	}
+
+	state, err := database.GetScanState(s.DB, folder.ID)
+	if err != nil {
+		t.Fatalf("GetScanState: %v", err)
+	}
+	if state.FilesRemoved != 1 {
+		t.Errorf("FilesRemoved = %d, want 1", state.FilesRemoved)
+	}
+}
+
+func TestScanFolderDetectsRename(t *testing.T) {
+	s, folder, dir := newTestScanner(t)
+	oldPath := genFile(t, dir, "old_name.mp3", "Song One", "Artist One")
+
+	if err := s.ScanFolder(context.Background(), folder.ID); err != nil {
+		t.Fatalf("ScanFolder (1st): %v", err)
+	}
+	files1, _ := database.ListMediaFilesForFolder(s.DB, folder.ID)
+	id1 := files1[0].ID
+
+	newPath := filepath.Join(dir, "new_name.mp3")
+	if err := os.Rename(oldPath, newPath); err != nil {
+		t.Fatalf("os.Rename: %v", err)
+	}
+	if err := s.ScanFolder(context.Background(), folder.ID); err != nil {
+		t.Fatalf("ScanFolder (2nd): %v", err)
+	}
+
+	files2, err := database.ListMediaFilesForFolder(s.DB, folder.ID)
+	if err != nil {
+		t.Fatalf("ListMediaFilesForFolder: %v", err)
+	}
+	if len(files2) != 1 {
+		t.Fatalf("got %d media files after rename, want 1 (matched by content, not duplicated)", len(files2))
+	}
+	if files2[0].ID != id1 {
+		t.Errorf("media file ID changed across rename: %s -> %s (should reuse the same row)", id1, files2[0].ID)
+	}
+	if files2[0].RelativePath != "new_name.mp3" {
+		t.Errorf("RelativePath = %q, want new_name.mp3", files2[0].RelativePath)
+	}
+	if files2[0].Missing {
+		t.Error("renamed file should not be marked missing")
+	}
+}
+
+func TestScanFolderIndexesLibraryTags(t *testing.T) {
+	s, folder, dir := newTestScanner(t)
+	genFile(t, dir, "song1.mp3", "Song One", "Shared Artist")
+	genFile(t, dir, "song2.mp3", "Song Two", "Shared Artist")
+
+	if err := s.ScanFolder(context.Background(), folder.ID); err != nil {
+		t.Fatalf("ScanFolder: %v", err)
+	}
+
+	artists, err := database.ListArtists(s.DB)
+	if err != nil {
+		t.Fatalf("ListArtists: %v", err)
+	}
+	if len(artists) != 1 {
+		t.Fatalf("artists = %v, want 1 shared artist reused across both songs", artists)
+	}
+
+	albums, err := database.ListAlbumsByArtist(s.DB, artists[0].ID)
+	if err != nil {
+		t.Fatalf("ListAlbumsByArtist: %v", err)
+	}
+	if len(albums) != 1 {
+		t.Fatalf("albums = %v, want 1", albums)
+	}
+	songs, err := database.ListSongsByAlbum(s.DB, albums[0].ID)
+	if err != nil {
+		t.Fatalf("ListSongsByAlbum: %v", err)
+	}
+	if len(songs) != 2 {
+		t.Fatalf("songs = %v, want 2", songs)
+	}
+}
+
+func TestScanFolderRejectsConcurrentScan(t *testing.T) {
+	s, folder, dir := newTestScanner(t)
+	genFile(t, dir, "song1.mp3", "Song One", "Artist One")
+
+	if !s.beginScan(folder.ID) {
+		t.Fatal("beginScan should succeed the first time")
+	}
+	defer s.endScan(folder.ID)
+
+	if err := s.ScanFolder(context.Background(), folder.ID); err == nil {
+		t.Fatal("expected error scanning a folder that is already being scanned")
+	}
+}