Implement sonicSimilarity extension with real audio feature analysis
Adds internal/audiofeatures: decodes each scanned song to PCM via ffmpeg and computes actual measured characteristics — tempo (via autocorrelation of the frame energy envelope, left unset rather than guessed when a clip is too short to contain a beat), an approximate loudness figure, spectral centroid/rolloff and zero-crossing rate (via a hand-written FFT), and a 12-bin chroma vector used both for a Krumhansl-Kessler key estimate and as similarity-index input. No external DSP/ML dependency. The scanner now runs this analysis after indexing each song (as part of the already-background scan, not on any request path) and stores results in the existing audio_features table via the new database.UpsertAudioFeatures/GetAudioFeatures/ListAllAudioFeatures. internal/similarity is a small in-process nearest-neighbor index: z-score normalizes each feature dimension across the analyzed library and answers Nearest (ranked real distances) and Path (a feature-space straight-line walk from one track to another, hopping to the closest not-yet-used real song at each step — never a fabricated intermediate track). getSonicSimilarTracks and findSonicPath expose these through the standard sonicMatch response, converting distance to a normalized (0,1] similarity score. A song that hasn't been analyzed yet is simply absent from results rather than faked. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014r98AtcBbevpscWLsfwVvv
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,118 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/audiofeatures"
|
||||
)
|
||||
|
||||
// AudioFeaturesRow is one song's stored sonic-analysis result.
|
||||
type AudioFeaturesRow struct {
|
||||
SongID string
|
||||
TempoBPM float64
|
||||
TempoValid bool
|
||||
MusicalKey string
|
||||
LoudnessLUFS float64
|
||||
Extra audioFeaturesExtra
|
||||
}
|
||||
|
||||
// audioFeaturesExtra is the analyzer output that doesn't have its own
|
||||
// dedicated column — kept in extra_json so the schema doesn't need a
|
||||
// migration every time the analyzer grows a new measurement.
|
||||
type audioFeaturesExtra struct {
|
||||
SpectralCentroidHz float64 `json:"spectralCentroidHz"`
|
||||
SpectralRolloffHz float64 `json:"spectralRolloffHz"`
|
||||
ZeroCrossingRate float64 `json:"zeroCrossingRate"`
|
||||
Chroma [12]float64 `json:"chroma"`
|
||||
}
|
||||
|
||||
// Vector reconstructs the same fixed-length feature vector
|
||||
// audiofeatures.Features.Vector produces, from stored values.
|
||||
func (r AudioFeaturesRow) Vector() []float64 {
|
||||
tempo := 0.0
|
||||
if r.TempoValid {
|
||||
tempo = r.TempoBPM
|
||||
}
|
||||
v := make([]float64, 0, 17)
|
||||
v = append(v, tempo, r.LoudnessLUFS, r.Extra.SpectralCentroidHz, r.Extra.SpectralRolloffHz, r.Extra.ZeroCrossingRate)
|
||||
v = append(v, r.Extra.Chroma[:]...)
|
||||
return v
|
||||
}
|
||||
|
||||
// UpsertAudioFeatures stores (or replaces) the analyzed features for a
|
||||
// song.
|
||||
func UpsertAudioFeatures(db *sql.DB, songID string, f *audiofeatures.Features) error {
|
||||
extra := audioFeaturesExtra{
|
||||
SpectralCentroidHz: f.SpectralCentroidHz,
|
||||
SpectralRolloffHz: f.SpectralRolloffHz,
|
||||
ZeroCrossingRate: f.ZeroCrossingRate,
|
||||
Chroma: f.Chroma,
|
||||
}
|
||||
extraJSON, err := json.Marshal(extra)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var tempo sql.NullFloat64
|
||||
if f.TempoValid {
|
||||
tempo = sql.NullFloat64{Float64: f.TempoBPM, Valid: true}
|
||||
}
|
||||
_, err = db.Exec(`
|
||||
INSERT INTO audio_features (song_id, tempo_bpm, musical_key, loudness_lufs, extra_json, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
||||
ON CONFLICT(song_id) DO UPDATE SET
|
||||
tempo_bpm = excluded.tempo_bpm,
|
||||
musical_key = excluded.musical_key,
|
||||
loudness_lufs = excluded.loudness_lufs,
|
||||
extra_json = excluded.extra_json,
|
||||
updated_at = excluded.updated_at`,
|
||||
songID, tempo, f.Key, f.LoudnessLUFS, string(extraJSON))
|
||||
return err
|
||||
}
|
||||
|
||||
const audioFeaturesSelect = `SELECT song_id, tempo_bpm, musical_key, loudness_lufs, extra_json FROM audio_features`
|
||||
|
||||
func scanAudioFeaturesRow(scan func(dest ...any) error) (*AudioFeaturesRow, error) {
|
||||
var r AudioFeaturesRow
|
||||
var tempo sql.NullFloat64
|
||||
var loudness sql.NullFloat64
|
||||
var extraJSON string
|
||||
if err := scan(&r.SongID, &tempo, &r.MusicalKey, &loudness, &extraJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.TempoValid = tempo.Valid
|
||||
r.TempoBPM = tempo.Float64
|
||||
r.LoudnessLUFS = loudness.Float64
|
||||
_ = json.Unmarshal([]byte(extraJSON), &r.Extra)
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// GetAudioFeatures returns a song's stored analysis, or nil if it
|
||||
// hasn't been analyzed yet.
|
||||
func GetAudioFeatures(db *sql.DB, songID string) (*AudioFeaturesRow, error) {
|
||||
row := db.QueryRow(audioFeaturesSelect+` WHERE song_id = ?`, songID)
|
||||
r, err := scanAudioFeaturesRow(row.Scan)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
// ListAllAudioFeatures returns every analyzed song's features, the
|
||||
// input the in-process sonic-similarity index is built from.
|
||||
func ListAllAudioFeatures(db *sql.DB) ([]AudioFeaturesRow, error) {
|
||||
rows, err := db.Query(audioFeaturesSelect)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []AudioFeaturesRow
|
||||
for rows.Next() {
|
||||
r, err := scanAudioFeaturesRow(rows.Scan)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, *r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/audiofeatures"
|
||||
)
|
||||
|
||||
func openAudioFeaturesTestDB(t *testing.T) (*sql.DB, string) {
|
||||
t.Helper()
|
||||
db, err := Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
folder, err := CreateMusicFolder(db, "Music", "/music", "music")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mf, err := CreateMediaFile(db, folder.ID, "song.mp3", 100, 0, "h", "mp3", "song", 10, 128, 44100, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
song, err := UpsertSong(db, UpsertSongInput{MediaFileID: mf.ID, Title: "Song"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db, song.ID
|
||||
}
|
||||
|
||||
func TestUpsertAndGetAudioFeatures(t *testing.T) {
|
||||
db, songID := openAudioFeaturesTestDB(t)
|
||||
|
||||
f := &audiofeatures.Features{
|
||||
TempoBPM: 120, TempoValid: true, Key: "C major", LoudnessLUFS: -14,
|
||||
SpectralCentroidHz: 1500, SpectralRolloffHz: 4000, ZeroCrossingRate: 0.05,
|
||||
Chroma: [12]float64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
|
||||
}
|
||||
if err := UpsertAudioFeatures(db, songID, f); err != nil {
|
||||
t.Fatalf("UpsertAudioFeatures: %v", err)
|
||||
}
|
||||
|
||||
got, err := GetAudioFeatures(db, songID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudioFeatures: %v", err)
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatal("expected a row, got nil")
|
||||
}
|
||||
if !got.TempoValid || got.TempoBPM != 120 {
|
||||
t.Errorf("tempo = %v/%v, want 120/valid", got.TempoBPM, got.TempoValid)
|
||||
}
|
||||
if got.MusicalKey != "C major" {
|
||||
t.Errorf("key = %q", got.MusicalKey)
|
||||
}
|
||||
if got.LoudnessLUFS != -14 {
|
||||
t.Errorf("loudness = %v", got.LoudnessLUFS)
|
||||
}
|
||||
if got.Extra.SpectralCentroidHz != 1500 || got.Extra.Chroma[0] != 1 {
|
||||
t.Errorf("extra = %+v", got.Extra)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertAudioFeaturesOverwrites(t *testing.T) {
|
||||
db, songID := openAudioFeaturesTestDB(t)
|
||||
if err := UpsertAudioFeatures(db, songID, &audiofeatures.Features{Key: "C major"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := UpsertAudioFeatures(db, songID, &audiofeatures.Features{Key: "D minor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := GetAudioFeatures(db, songID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.MusicalKey != "D minor" {
|
||||
t.Errorf("key = %q, want overwrite to D minor", got.MusicalKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAudioFeaturesNotAnalyzed(t *testing.T) {
|
||||
db, songID := openAudioFeaturesTestDB(t)
|
||||
got, err := GetAudioFeatures(db, songID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Errorf("expected nil for an unanalyzed song, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListAllAudioFeatures(t *testing.T) {
|
||||
db, songID := openAudioFeaturesTestDB(t)
|
||||
if err := UpsertAudioFeatures(db, songID, &audiofeatures.Features{TempoValid: true, TempoBPM: 90}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := ListAllAudioFeatures(db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(all) != 1 || all[0].SongID != songID {
|
||||
t.Fatalf("all = %+v", all)
|
||||
}
|
||||
if len(all[0].Vector()) != 17 {
|
||||
t.Fatalf("Vector length = %d, want 17", len(all[0].Vector()))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user