Library indexing and filesystem scanner with incremental rescans
This commit is contained in:
@@ -0,0 +1,276 @@
|
||||
// Package scanner walks configured music folders, detects new/changed/
|
||||
// deleted/renamed files, and indexes them into the library via a bounded
|
||||
// worker pool.
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/database"
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/library"
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/metadata"
|
||||
)
|
||||
|
||||
// recognizedExtensions are the file extensions VibeSonic attempts to
|
||||
// probe as media. Anything else (art, .nfo, playlists, ...) is skipped
|
||||
// during a scan rather than passed to ffprobe.
|
||||
var recognizedExtensions = map[string]bool{
|
||||
".mp3": true, ".flac": true, ".ogg": true, ".oga": true, ".opus": true,
|
||||
".aac": true, ".m4a": true, ".m4b": true, ".alac": true,
|
||||
".wav": true, ".aiff": true, ".aif": true, ".wma": true, ".mp4": true,
|
||||
}
|
||||
|
||||
const maxWorkers = 4
|
||||
|
||||
// Scanner indexes one or more music folders into the database.
|
||||
type Scanner struct {
|
||||
DB *sql.DB
|
||||
Prober *metadata.Prober
|
||||
Log *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
scanning map[string]bool // music_folder_id -> in progress
|
||||
}
|
||||
|
||||
func New(db *sql.DB, prober *metadata.Prober, log *slog.Logger) *Scanner {
|
||||
return &Scanner{DB: db, Prober: prober, Log: log, scanning: make(map[string]bool)}
|
||||
}
|
||||
|
||||
// ScanFolder walks a single music folder and indexes it incrementally:
|
||||
// unchanged files (same size+mtime) are only marked seen, changed files
|
||||
// are reprobed, moved files are matched by content hash and just get
|
||||
// their path updated, and files no longer found on disk are marked
|
||||
// missing rather than deleted outright.
|
||||
func (s *Scanner) ScanFolder(ctx context.Context, folderID string) error {
|
||||
if !s.beginScan(folderID) {
|
||||
return fmt.Errorf("scan already in progress for folder %s", folderID)
|
||||
}
|
||||
defer s.endScan(folderID)
|
||||
|
||||
folder, err := database.GetMusicFolder(s.DB, folderID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
startedAt := time.Now()
|
||||
if err := database.StartScan(s.DB, folderID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var seen, added, updated int
|
||||
var mu sync.Mutex
|
||||
sem := make(chan struct{}, maxWorkers)
|
||||
var wg sync.WaitGroup
|
||||
var firstErr error
|
||||
|
||||
walkErr := filepath.WalkDir(folder.Path, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
s.Log.Warn("scan: walk error", "path", path, "err", err)
|
||||
return nil // keep scanning the rest of the tree
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !recognizedExtensions[strings.ToLower(filepath.Ext(path))] {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(folder.Path, path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
|
||||
wasAdded, wasUpdated, err := s.indexOne(ctx, folder.ID, path, rel)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
seen++
|
||||
if err != nil {
|
||||
s.Log.Warn("scan: index failed", "path", rel, "err", err)
|
||||
} else {
|
||||
if wasAdded {
|
||||
added++
|
||||
} else if wasUpdated {
|
||||
updated++
|
||||
}
|
||||
}
|
||||
if seen%25 == 0 {
|
||||
_ = database.UpdateScanProgress(s.DB, folder.ID, seen, added, updated)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
})
|
||||
wg.Wait()
|
||||
if walkErr != nil && firstErr == nil {
|
||||
firstErr = walkErr
|
||||
}
|
||||
|
||||
_ = database.UpdateScanProgress(s.DB, folder.ID, seen, added, updated)
|
||||
|
||||
removed, err := database.MarkFolderFilesMissingExcept(s.DB, folder.ID, startedAt)
|
||||
if err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
|
||||
errMsg := ""
|
||||
if firstErr != nil {
|
||||
errMsg = firstErr.Error()
|
||||
}
|
||||
if err := database.FinishScan(s.DB, folder.ID, int(removed), errMsg); err != nil {
|
||||
return err
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// indexOne probes and indexes a single file, returning whether it was
|
||||
// newly added or updated so the caller can track progress.
|
||||
func (s *Scanner) indexOne(ctx context.Context, folderID, absPath, relPath string) (added, updated bool, err error) {
|
||||
info, err := os.Stat(absPath)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
size := info.Size()
|
||||
mtime := info.ModTime().Unix()
|
||||
hash := fingerprint(size, mtime)
|
||||
|
||||
existing, err := database.FindMediaFileByPath(s.DB, folderID, relPath)
|
||||
if err == nil {
|
||||
if existing.SizeBytes == size && existing.MtimeUnix == mtime {
|
||||
return false, false, database.TouchMediaFileScanned(s.DB, existing.ID)
|
||||
}
|
||||
// Changed: reprobe and update in place; the media_file (and
|
||||
// therefore song) ID stays the same.
|
||||
tags, perr := s.Prober.Probe(ctx, absPath)
|
||||
if perr != nil {
|
||||
return false, false, perr
|
||||
}
|
||||
if err := database.UpdateMediaFileScanned(s.DB, existing.ID, size, mtime, hash, tags.Format, tags.Kind, tags.DurationSeconds, tags.BitrateKbps, tags.SampleRateHz, tags.Channels); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
mf, err := database.GetMediaFile(s.DB, existing.ID)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if _, err := library.IndexFile(s.DB, mf, tags); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
return false, true, nil
|
||||
}
|
||||
if err != database.ErrNotFound {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
// Not found by path. Check whether this is a rename/move of a file we
|
||||
// already indexed (same content fingerprint, different path).
|
||||
if renamed, rerr := database.FindMediaFileByHash(s.DB, folderID, hash); rerr == nil {
|
||||
if err := database.RenameMediaFile(s.DB, renamed.ID, relPath); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
return false, true, nil
|
||||
}
|
||||
|
||||
// Genuinely new file.
|
||||
tags, err := s.Prober.Probe(ctx, absPath)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
mf, err := database.CreateMediaFile(s.DB, folderID, relPath, size, mtime, hash, tags.Format, tags.Kind, tags.DurationSeconds, tags.BitrateKbps, tags.SampleRateHz, tags.Channels)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if _, err := library.IndexFile(s.DB, mf, tags); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
return true, false, nil
|
||||
}
|
||||
|
||||
// fingerprint is a cheap, fast rename/change detector based on size and
|
||||
// mtime rather than hashing file content (which would mean reading every
|
||||
// byte of every file on every scan).
|
||||
func fingerprint(size, mtime int64) string {
|
||||
return fmt.Sprintf("%d-%d", size, mtime)
|
||||
}
|
||||
|
||||
func (s *Scanner) beginScan(folderID string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.scanning[folderID] {
|
||||
return false
|
||||
}
|
||||
s.scanning[folderID] = true
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Scanner) endScan(folderID string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.scanning, folderID)
|
||||
}
|
||||
|
||||
// IsScanning reports whether a scan of folderID is currently running.
|
||||
func (s *Scanner) IsScanning(folderID string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.scanning[folderID]
|
||||
}
|
||||
|
||||
// ScanAllEnabled scans every enabled music folder sequentially (folders
|
||||
// run one at a time; files within a folder run concurrently via the
|
||||
// worker pool above).
|
||||
func (s *Scanner) ScanAllEnabled(ctx context.Context) error {
|
||||
folders, err := database.ListMusicFolders(s.DB)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var firstErr error
|
||||
for _, f := range folders {
|
||||
if !f.Enabled {
|
||||
continue
|
||||
}
|
||||
if err := s.ScanFolder(ctx, f.ID); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// RunScheduled runs ScanAllEnabled once at startup and then on the given
|
||||
// interval, until ctx is cancelled. interval <= 0 disables the periodic
|
||||
// rescan (startup scan still runs).
|
||||
func (s *Scanner) RunScheduled(ctx context.Context, interval time.Duration) {
|
||||
if err := s.ScanAllEnabled(ctx); err != nil {
|
||||
s.Log.Warn("startup scan finished with errors", "err", err)
|
||||
}
|
||||
if interval <= 0 {
|
||||
return
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := s.ScanAllEnabled(ctx); err != nil {
|
||||
s.Log.Warn("scheduled scan finished with errors", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user