Files
vibesonic/internal/database/audio_features.go
T
owenandClaude Sonnet 5 b6acd63c2d Implement sonicSimilarity extension with real audio feature analysis
Adds internal/audiofeatures: decodes each scanned song to PCM via
ffmpeg and computes actual measured characteristics — tempo (via
autocorrelation of the frame energy envelope, left unset rather than
guessed when a clip is too short to contain a beat), an approximate
loudness figure, spectral centroid/rolloff and zero-crossing rate (via
a hand-written FFT), and a 12-bin chroma vector used both for a
Krumhansl-Kessler key estimate and as similarity-index input. No
external DSP/ML dependency.

The scanner now runs this analysis after indexing each song (as part
of the already-background scan, not on any request path) and stores
results in the existing audio_features table via the new
database.UpsertAudioFeatures/GetAudioFeatures/ListAllAudioFeatures.

internal/similarity is a small in-process nearest-neighbor index:
z-score normalizes each feature dimension across the analyzed library
and answers Nearest (ranked real distances) and Path (a feature-space
straight-line walk from one track to another, hopping to the closest
not-yet-used real song at each step — never a fabricated intermediate
track).

getSonicSimilarTracks and findSonicPath expose these through the
standard sonicMatch response, converting distance to a normalized
(0,1] similarity score. A song that hasn't been analyzed yet is simply
absent from results rather than faked.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014r98AtcBbevpscWLsfwVvv
2026-08-23 08:30:20 -05:00

119 lines
3.6 KiB
Go

package database
import (
"database/sql"
"encoding/json"
"git.noroutes.net/NoRoutes/vibetunes/internal/audiofeatures"
)
// AudioFeaturesRow is one song's stored sonic-analysis result.
type AudioFeaturesRow struct {
SongID string
TempoBPM float64
TempoValid bool
MusicalKey string
LoudnessLUFS float64
Extra audioFeaturesExtra
}
// audioFeaturesExtra is the analyzer output that doesn't have its own
// dedicated column — kept in extra_json so the schema doesn't need a
// migration every time the analyzer grows a new measurement.
type audioFeaturesExtra struct {
SpectralCentroidHz float64 `json:"spectralCentroidHz"`
SpectralRolloffHz float64 `json:"spectralRolloffHz"`
ZeroCrossingRate float64 `json:"zeroCrossingRate"`
Chroma [12]float64 `json:"chroma"`
}
// Vector reconstructs the same fixed-length feature vector
// audiofeatures.Features.Vector produces, from stored values.
func (r AudioFeaturesRow) Vector() []float64 {
tempo := 0.0
if r.TempoValid {
tempo = r.TempoBPM
}
v := make([]float64, 0, 17)
v = append(v, tempo, r.LoudnessLUFS, r.Extra.SpectralCentroidHz, r.Extra.SpectralRolloffHz, r.Extra.ZeroCrossingRate)
v = append(v, r.Extra.Chroma[:]...)
return v
}
// UpsertAudioFeatures stores (or replaces) the analyzed features for a
// song.
func UpsertAudioFeatures(db *sql.DB, songID string, f *audiofeatures.Features) error {
extra := audioFeaturesExtra{
SpectralCentroidHz: f.SpectralCentroidHz,
SpectralRolloffHz: f.SpectralRolloffHz,
ZeroCrossingRate: f.ZeroCrossingRate,
Chroma: f.Chroma,
}
extraJSON, err := json.Marshal(extra)
if err != nil {
return err
}
var tempo sql.NullFloat64
if f.TempoValid {
tempo = sql.NullFloat64{Float64: f.TempoBPM, Valid: true}
}
_, err = db.Exec(`
INSERT INTO audio_features (song_id, tempo_bpm, musical_key, loudness_lufs, extra_json, updated_at)
VALUES (?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ','now'))
ON CONFLICT(song_id) DO UPDATE SET
tempo_bpm = excluded.tempo_bpm,
musical_key = excluded.musical_key,
loudness_lufs = excluded.loudness_lufs,
extra_json = excluded.extra_json,
updated_at = excluded.updated_at`,
songID, tempo, f.Key, f.LoudnessLUFS, string(extraJSON))
return err
}
const audioFeaturesSelect = `SELECT song_id, tempo_bpm, musical_key, loudness_lufs, extra_json FROM audio_features`
func scanAudioFeaturesRow(scan func(dest ...any) error) (*AudioFeaturesRow, error) {
var r AudioFeaturesRow
var tempo sql.NullFloat64
var loudness sql.NullFloat64
var extraJSON string
if err := scan(&r.SongID, &tempo, &r.MusicalKey, &loudness, &extraJSON); err != nil {
return nil, err
}
r.TempoValid = tempo.Valid
r.TempoBPM = tempo.Float64
r.LoudnessLUFS = loudness.Float64
_ = json.Unmarshal([]byte(extraJSON), &r.Extra)
return &r, nil
}
// GetAudioFeatures returns a song's stored analysis, or nil if it
// hasn't been analyzed yet.
func GetAudioFeatures(db *sql.DB, songID string) (*AudioFeaturesRow, error) {
row := db.QueryRow(audioFeaturesSelect+` WHERE song_id = ?`, songID)
r, err := scanAudioFeaturesRow(row.Scan)
if err == sql.ErrNoRows {
return nil, nil
}
return r, err
}
// ListAllAudioFeatures returns every analyzed song's features, the
// input the in-process sonic-similarity index is built from.
func ListAllAudioFeatures(db *sql.DB) ([]AudioFeaturesRow, error) {
rows, err := db.Query(audioFeaturesSelect)
if err != nil {
return nil, err
}
defer rows.Close()
var out []AudioFeaturesRow
for rows.Next() {
r, err := scanAudioFeaturesRow(rows.Scan)
if err != nil {
return nil, err
}
out = append(out, *r)
}
return out, rows.Err()
}