Implement sonicSimilarity extension with real audio feature analysis
Adds internal/audiofeatures: decodes each scanned song to PCM via ffmpeg and computes actual measured characteristics — tempo (via autocorrelation of the frame energy envelope, left unset rather than guessed when a clip is too short to contain a beat), an approximate loudness figure, spectral centroid/rolloff and zero-crossing rate (via a hand-written FFT), and a 12-bin chroma vector used both for a Krumhansl-Kessler key estimate and as similarity-index input. No external DSP/ML dependency. The scanner now runs this analysis after indexing each song (as part of the already-background scan, not on any request path) and stores results in the existing audio_features table via the new database.UpsertAudioFeatures/GetAudioFeatures/ListAllAudioFeatures. internal/similarity is a small in-process nearest-neighbor index: z-score normalizes each feature dimension across the analyzed library and answers Nearest (ranked real distances) and Path (a feature-space straight-line walk from one track to another, hopping to the closest not-yet-used real song at each step — never a fabricated intermediate track). getSonicSimilarTracks and findSonicPath expose these through the standard sonicMatch response, converting distance to a normalized (0,1] similarity score. A song that hasn't been analyzed yet is simply absent from results rather than faked. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014r98AtcBbevpscWLsfwVvv
This commit is contained in:
@@ -15,6 +15,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/audiofeatures"
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/coverart"
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/database"
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/library"
|
||||
@@ -35,9 +36,10 @@ const maxWorkers = 4
|
||||
|
||||
// Scanner indexes one or more music folders into the database.
|
||||
type Scanner struct {
|
||||
DB *sql.DB
|
||||
Prober *metadata.Prober
|
||||
Log *slog.Logger
|
||||
DB *sql.DB
|
||||
Prober *metadata.Prober
|
||||
Log *slog.Logger
|
||||
FFmpegPath string // for sonic-analysis audio decoding; "" defaults to "ffmpeg" on PATH
|
||||
|
||||
mu sync.Mutex
|
||||
scanning map[string]bool // music_folder_id -> in progress
|
||||
@@ -175,6 +177,7 @@ func (s *Scanner) indexOne(ctx context.Context, folderID, absPath, relPath strin
|
||||
return false, false, err
|
||||
}
|
||||
s.indexCoverArt(absPath, song, tags)
|
||||
s.analyzeAudio(ctx, absPath, song, tags)
|
||||
return false, true, nil
|
||||
}
|
||||
if err != database.ErrNotFound {
|
||||
@@ -204,9 +207,31 @@ func (s *Scanner) indexOne(ctx context.Context, folderID, absPath, relPath strin
|
||||
return false, false, err
|
||||
}
|
||||
s.indexCoverArt(absPath, song, tags)
|
||||
s.analyzeAudio(ctx, absPath, song, tags)
|
||||
return true, false, nil
|
||||
}
|
||||
|
||||
// analyzeAudio runs the sonic-feature analyzer over a scanned file and
|
||||
// stores the result, powering the sonicSimilarity extension. This runs
|
||||
// as part of the (already background, admin-triggered) scan itself
|
||||
// rather than on any request path. A decode/analysis failure is logged
|
||||
// and otherwise ignored: the song is still indexed and playable, it
|
||||
// just won't appear in sonic-similarity results until a future rescan
|
||||
// succeeds — never a reason to fail the whole scan.
|
||||
func (s *Scanner) analyzeAudio(ctx context.Context, absPath string, song *database.Song, tags *metadata.Tags) {
|
||||
if tags.Kind != "song" {
|
||||
return
|
||||
}
|
||||
features, err := audiofeatures.Analyze(ctx, s.FFmpegPath, absPath)
|
||||
if err != nil {
|
||||
s.Log.Warn("audio feature analysis failed", "err", err, "path", absPath)
|
||||
return
|
||||
}
|
||||
if err := database.UpsertAudioFeatures(s.DB, song.ID, features); err != nil {
|
||||
s.Log.Warn("storing audio features failed", "err", err, "path", absPath)
|
||||
}
|
||||
}
|
||||
|
||||
// indexCoverArt records where an album's cover art can be found: a
|
||||
// folder image next to the file if one exists, otherwise the file's own
|
||||
// embedded artwork (extracted lazily by the web layer on request, not
|
||||
|
||||
@@ -279,3 +279,40 @@ func TestScanFolderIndexesSidecarLRC(t *testing.T) {
|
||||
t.Fatalf("expected word-level cues from Enhanced LRC line, got %+v", synced.Lines[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanFolderAnalyzesAudioFeatures(t *testing.T) {
|
||||
s, folder, dir := newTestScanner(t)
|
||||
genFile(t, dir, "song1.mp3", "Song One", "Artist One")
|
||||
|
||||
if err := s.ScanFolder(context.Background(), folder.ID); err != nil {
|
||||
t.Fatalf("ScanFolder: %v", err)
|
||||
}
|
||||
|
||||
songs, err := database.ListSongs(s.DB, 10, 0)
|
||||
if err != nil || len(songs) != 1 {
|
||||
t.Fatalf("ListSongs: %v %v", songs, err)
|
||||
}
|
||||
|
||||
features, err := database.GetAudioFeatures(s.DB, songs[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudioFeatures: %v", err)
|
||||
}
|
||||
if features == nil {
|
||||
t.Fatal("expected audio features to be stored after scanning, got none")
|
||||
}
|
||||
// The fixture is a pure 440Hz sine tone (A4): its chroma should
|
||||
// actually peak at pitch class A (bin 9), a real measured property,
|
||||
// not a placeholder.
|
||||
if features.Extra.Chroma == ([12]float64{}) {
|
||||
t.Error("expected a non-zero chroma vector from real analysis")
|
||||
}
|
||||
best := 0
|
||||
for i := 1; i < 12; i++ {
|
||||
if features.Extra.Chroma[i] > features.Extra.Chroma[best] {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
if best != 9 { // A
|
||||
t.Errorf("chroma peak = bin %d, want 9 (A) for a 440Hz fixture", best)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user