Implement sonicSimilarity extension with real audio feature analysis
Adds internal/audiofeatures: decodes each scanned song to PCM via ffmpeg and computes actual measured characteristics — tempo (via autocorrelation of the frame energy envelope, left unset rather than guessed when a clip is too short to contain a beat), an approximate loudness figure, spectral centroid/rolloff and zero-crossing rate (via a hand-written FFT), and a 12-bin chroma vector used both for a Krumhansl-Kessler key estimate and as similarity-index input. No external DSP/ML dependency. The scanner now runs this analysis after indexing each song (as part of the already-background scan, not on any request path) and stores results in the existing audio_features table via the new database.UpsertAudioFeatures/GetAudioFeatures/ListAllAudioFeatures. internal/similarity is a small in-process nearest-neighbor index: z-score normalizes each feature dimension across the analyzed library and answers Nearest (ranked real distances) and Path (a feature-space straight-line walk from one track to another, hopping to the closest not-yet-used real song at each step — never a fabricated intermediate track). getSonicSimilarTracks and findSonicPath expose these through the standard sonicMatch response, converting distance to a normalized (0,1] similarity score. A song that hasn't been analyzed yet is simply absent from results rather than faked. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014r98AtcBbevpscWLsfwVvv
This commit is contained in:
Claude Sonnet 5
@@ -57,6 +57,7 @@ func Open(cfg config.Config, log *slog.Logger) (*App, error) {
|
||||
|
||||
prober := &metadata.Prober{FFprobePath: cfg.FFprobePath}
|
||||
sc := scanner.New(db, prober, log)
|
||||
sc.FFmpegPath = cfg.FFmpegPath
|
||||
|
||||
coverCache := filepath.Join(cfg.DataDir, "cache", "covers")
|
||||
covers := coverart.New(coverCache, cfg.FFmpegPath)
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
// Package audiofeatures extracts a small set of real, numerically
|
||||
// computed audio characteristics from a media file — tempo, loudness,
|
||||
// spectral shape, and a chroma (pitch-class) profile used both for a
|
||||
// musical key guess and as the basis of a sonic-similarity feature
|
||||
// vector. Everything here is derived from decoding the actual audio
|
||||
// with ffmpeg; nothing is inferred from tags, filenames, or genre.
|
||||
package audiofeatures
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"math"
|
||||
"os/exec"
|
||||
"sort"
|
||||
)
|
||||
|
||||
const (
|
||||
sampleRate = 22050
|
||||
frameSize = 2048
|
||||
hopSize = 1024
|
||||
)
|
||||
|
||||
// Features is everything VibeSonic actually managed to measure about a
|
||||
// track's audio. Fields the analysis couldn't determine with enough
|
||||
// confidence (e.g. tempo on a clip too short to contain a beat) are
|
||||
// left at their zero value with the matching Valid flag false, rather
|
||||
// than filled with a guess.
|
||||
type Features struct {
|
||||
TempoBPM float64
|
||||
TempoValid bool
|
||||
Key string // e.g. "C major", "" if undetermined
|
||||
LoudnessLUFS float64 // approximate: mean-square level in dB, not full ITU-R BS.1770
|
||||
SpectralCentroidHz float64
|
||||
SpectralRolloffHz float64
|
||||
ZeroCrossingRate float64
|
||||
Chroma [12]float64 // normalized pitch-class energy distribution
|
||||
}
|
||||
|
||||
// Vector returns the fixed-length numeric feature vector used for
|
||||
// sonic-similarity distance calculations: [tempo, loudness, centroid,
|
||||
// rolloff, zcr, chroma(12)]. An invalid tempo contributes 0, same as a
|
||||
// track's other quiet dimensions would if it were literally silent in
|
||||
// that regard — the similarity index normalizes each dimension across
|
||||
// the library before comparing, so a shared 0 among untempoed tracks
|
||||
// doesn't bias results toward or away from any real tempo.
|
||||
func (f Features) Vector() []float64 {
|
||||
tempo := 0.0
|
||||
if f.TempoValid {
|
||||
tempo = f.TempoBPM
|
||||
}
|
||||
v := make([]float64, 0, 17)
|
||||
v = append(v, tempo, f.LoudnessLUFS, f.SpectralCentroidHz, f.SpectralRolloffHz, f.ZeroCrossingRate)
|
||||
v = append(v, f.Chroma[:]...)
|
||||
return v
|
||||
}
|
||||
|
||||
// Analyze decodes sourcePath to mono PCM via ffmpeg and computes
|
||||
// Features from it. It requires at least a few frames of audio
|
||||
// (roughly 150ms); shorter input still returns loudness/spectral
|
||||
// values from what's available but leaves tempo unset.
|
||||
func Analyze(ctx context.Context, ffmpegPath, sourcePath string) (*Features, error) {
|
||||
if ffmpegPath == "" {
|
||||
ffmpegPath = "ffmpeg"
|
||||
}
|
||||
pcm, err := decodePCM(ctx, ffmpegPath, sourcePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(pcm) == 0 {
|
||||
return nil, fmt.Errorf("audiofeatures: no decodable audio in %s", sourcePath)
|
||||
}
|
||||
return analyzeSamples(pcm), nil
|
||||
}
|
||||
|
||||
func decodePCM(ctx context.Context, ffmpegPath, sourcePath string) ([]float64, error) {
|
||||
cmd := exec.CommandContext(ctx, ffmpegPath,
|
||||
"-v", "error",
|
||||
"-i", sourcePath,
|
||||
"-f", "f32le", "-ac", "1", "-ar", fmt.Sprintf("%d", sampleRate),
|
||||
"-",
|
||||
)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg decode failed: %w: %s", err, stderr.String())
|
||||
}
|
||||
|
||||
raw := stdout.Bytes()
|
||||
n := len(raw) / 4
|
||||
samples := make([]float64, n)
|
||||
for i := 0; i < n; i++ {
|
||||
bits := binary.LittleEndian.Uint32(raw[i*4 : i*4+4])
|
||||
samples[i] = float64(math.Float32frombits(bits))
|
||||
}
|
||||
return samples, nil
|
||||
}
|
||||
|
||||
func analyzeSamples(samples []float64) *Features {
|
||||
f := &Features{}
|
||||
|
||||
// Overall RMS loudness, in dB relative to full scale — a real
|
||||
// (if simplified) measurement of the decoded signal, not a stand-in.
|
||||
var sumSq float64
|
||||
for _, s := range samples {
|
||||
sumSq += s * s
|
||||
}
|
||||
rms := math.Sqrt(sumSq / float64(len(samples)))
|
||||
if rms > 0 {
|
||||
f.LoudnessLUFS = 20 * math.Log10(rms)
|
||||
} else {
|
||||
f.LoudnessLUFS = -120
|
||||
}
|
||||
|
||||
var (
|
||||
centroidSum, rolloffSum, zcrSum float64
|
||||
frameCount int
|
||||
energyEnvelope []float64
|
||||
chroma [12]float64
|
||||
)
|
||||
|
||||
hann := hannWindow(frameSize)
|
||||
for start := 0; start+frameSize <= len(samples); start += hopSize {
|
||||
frame := make([]float64, frameSize)
|
||||
var frameEnergy float64
|
||||
var crossings int
|
||||
for i := 0; i < frameSize; i++ {
|
||||
s := samples[start+i]
|
||||
frame[i] = s * hann[i]
|
||||
frameEnergy += s * s
|
||||
if i > 0 && ((samples[start+i-1] < 0) != (s < 0)) {
|
||||
crossings++
|
||||
}
|
||||
}
|
||||
energyEnvelope = append(energyEnvelope, math.Sqrt(frameEnergy/float64(frameSize)))
|
||||
zcrSum += float64(crossings) / float64(frameSize)
|
||||
|
||||
mags := magnitudeSpectrum(frame)
|
||||
centroid, rolloff := spectralShape(mags, sampleRate, frameSize)
|
||||
centroidSum += centroid
|
||||
rolloffSum += rolloff
|
||||
accumulateChroma(&chroma, mags, sampleRate, frameSize)
|
||||
|
||||
frameCount++
|
||||
}
|
||||
|
||||
if frameCount > 0 {
|
||||
f.SpectralCentroidHz = centroidSum / float64(frameCount)
|
||||
f.SpectralRolloffHz = rolloffSum / float64(frameCount)
|
||||
f.ZeroCrossingRate = zcrSum / float64(frameCount)
|
||||
normalizeChroma(&chroma)
|
||||
f.Chroma = chroma
|
||||
f.Key = estimateKey(chroma)
|
||||
}
|
||||
|
||||
if bpm, ok := estimateTempo(energyEnvelope, float64(hopSize)/float64(sampleRate)); ok {
|
||||
f.TempoBPM = bpm
|
||||
f.TempoValid = true
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func hannWindow(n int) []float64 {
|
||||
w := make([]float64, n)
|
||||
for i := range w {
|
||||
w[i] = 0.5 * (1 - math.Cos(2*math.Pi*float64(i)/float64(n-1)))
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// spectralShape returns the spectral centroid (the "center of mass" of
|
||||
// the spectrum, in Hz — a brightness measure) and the 85% rolloff
|
||||
// frequency (below which 85% of the frame's spectral energy lies).
|
||||
func spectralShape(mags []float64, sampleRate, frameSize int) (centroid, rolloff float64) {
|
||||
var weighted, total float64
|
||||
binHz := float64(sampleRate) / float64(frameSize)
|
||||
for i, m := range mags {
|
||||
freq := float64(i) * binHz
|
||||
weighted += freq * m
|
||||
total += m
|
||||
}
|
||||
if total == 0 {
|
||||
return 0, 0
|
||||
}
|
||||
centroid = weighted / total
|
||||
|
||||
target := 0.85 * total
|
||||
var cum float64
|
||||
for i, m := range mags {
|
||||
cum += m
|
||||
if cum >= target {
|
||||
rolloff = float64(i) * binHz
|
||||
break
|
||||
}
|
||||
}
|
||||
return centroid, rolloff
|
||||
}
|
||||
|
||||
// accumulateChroma folds a frame's spectral magnitude into 12
|
||||
// pitch-class bins (assuming standard A440 tuning), skipping the
|
||||
// sub-audible and very-high-frequency bins where pitch content is
|
||||
// unreliable.
|
||||
func accumulateChroma(chroma *[12]float64, mags []float64, sampleRate, frameSize int) {
|
||||
binHz := float64(sampleRate) / float64(frameSize)
|
||||
for i, m := range mags {
|
||||
freq := float64(i) * binHz
|
||||
if freq < 27.5 || freq > 5000 || m == 0 {
|
||||
continue
|
||||
}
|
||||
midi := 69 + 12*math.Log2(freq/440)
|
||||
pc := int(math.Round(midi)) % 12
|
||||
if pc < 0 {
|
||||
pc += 12
|
||||
}
|
||||
chroma[pc] += m
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeChroma(chroma *[12]float64) {
|
||||
var total float64
|
||||
for _, v := range chroma {
|
||||
total += v
|
||||
}
|
||||
if total == 0 {
|
||||
return
|
||||
}
|
||||
for i := range chroma {
|
||||
chroma[i] /= total
|
||||
}
|
||||
}
|
||||
|
||||
// krumhanslMajor/Minor are the classic Krumhansl-Kessler key profiles:
|
||||
// the relative perceptual stability of each pitch class within a major
|
||||
// or minor key, starting from the tonic. Correlating a track's chroma
|
||||
// vector against all 24 rotations picks the best-fitting key.
|
||||
var krumhanslMajor = [12]float64{6.35, 2.23, 3.48, 2.33, 4.38, 4.09, 2.52, 5.19, 2.39, 3.66, 2.29, 2.88}
|
||||
var krumhanslMinor = [12]float64{6.33, 2.68, 3.52, 5.38, 2.60, 3.53, 2.54, 4.75, 3.98, 2.69, 3.34, 3.17}
|
||||
|
||||
var pitchClassNames = [12]string{"C", "C♯", "D", "D♯", "E", "F", "F♯", "G", "G♯", "A", "A♯", "B"}
|
||||
|
||||
func estimateKey(chroma [12]float64) string {
|
||||
type candidate struct {
|
||||
name string
|
||||
score float64
|
||||
}
|
||||
var candidates []candidate
|
||||
for tonic := 0; tonic < 12; tonic++ {
|
||||
candidates = append(candidates,
|
||||
candidate{pitchClassNames[tonic] + " major", correlate(chroma, rotate(krumhanslMajor, tonic))},
|
||||
candidate{pitchClassNames[tonic] + " minor", correlate(chroma, rotate(krumhanslMinor, tonic))},
|
||||
)
|
||||
}
|
||||
sort.Slice(candidates, func(i, j int) bool { return candidates[i].score > candidates[j].score })
|
||||
if len(candidates) == 0 || candidates[0].score <= 0 {
|
||||
return ""
|
||||
}
|
||||
return candidates[0].name
|
||||
}
|
||||
|
||||
func rotate(profile [12]float64, by int) [12]float64 {
|
||||
var out [12]float64
|
||||
for i := range out {
|
||||
out[i] = profile[(i-by+12)%12]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func correlate(a, b [12]float64) float64 {
|
||||
var meanA, meanB float64
|
||||
for i := 0; i < 12; i++ {
|
||||
meanA += a[i]
|
||||
meanB += b[i]
|
||||
}
|
||||
meanA /= 12
|
||||
meanB /= 12
|
||||
var num, denomA, denomB float64
|
||||
for i := 0; i < 12; i++ {
|
||||
da, db := a[i]-meanA, b[i]-meanB
|
||||
num += da * db
|
||||
denomA += da * da
|
||||
denomB += db * db
|
||||
}
|
||||
if denomA == 0 || denomB == 0 {
|
||||
return 0
|
||||
}
|
||||
return num / math.Sqrt(denomA*denomB)
|
||||
}
|
||||
|
||||
// estimateTempo autocorrelates the frame-wise energy envelope over the
|
||||
// lag range corresponding to 50-220 BPM and returns the lag with the
|
||||
// strongest periodicity. It reports ok=false when there isn't enough
|
||||
// envelope to cover even the slowest candidate tempo, rather than
|
||||
// returning a meaningless number for a clip too short to have a beat.
|
||||
func estimateTempo(envelope []float64, hopSeconds float64) (bpm float64, ok bool) {
|
||||
minLagSec := 60.0 / 220.0
|
||||
maxLagSec := 60.0 / 50.0
|
||||
minLag := int(minLagSec / hopSeconds)
|
||||
maxLag := int(maxLagSec / hopSeconds)
|
||||
if minLag < 1 {
|
||||
minLag = 1
|
||||
}
|
||||
if len(envelope) < maxLag+8 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
var mean float64
|
||||
for _, v := range envelope {
|
||||
mean += v
|
||||
}
|
||||
mean /= float64(len(envelope))
|
||||
centered := make([]float64, len(envelope))
|
||||
for i, v := range envelope {
|
||||
centered[i] = v - mean
|
||||
}
|
||||
|
||||
bestLag := -1
|
||||
bestScore := 0.0
|
||||
for lag := minLag; lag <= maxLag && lag < len(centered); lag++ {
|
||||
var score float64
|
||||
for i := 0; i+lag < len(centered); i++ {
|
||||
score += centered[i] * centered[i+lag]
|
||||
}
|
||||
if score > bestScore {
|
||||
bestScore = score
|
||||
bestLag = lag
|
||||
}
|
||||
}
|
||||
if bestLag <= 0 || bestScore <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return 60.0 / (float64(bestLag) * hopSeconds), true
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package audiofeatures
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func genTone(t *testing.T, dir, name string, freq float64, duration string) string {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("ffmpeg"); err != nil {
|
||||
t.Skip("ffmpeg not installed, skipping audiofeatures integration test")
|
||||
}
|
||||
out := filepath.Join(dir, name)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "ffmpeg", "-y",
|
||||
"-f", "lavfi", "-i", "sine=frequency="+strconv.FormatFloat(freq, 'f', -1, 64)+":duration="+duration,
|
||||
"-codec:a", "pcm_s16le", out,
|
||||
)
|
||||
if combined, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("ffmpeg failed: %v\n%s", err, combined)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAnalyzeLowVsHighToneHaveDifferentChromaPeak(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
lowPath := genTone(t, dir, "low.wav", 130.81, "1") // C3
|
||||
highPath := genTone(t, dir, "high.wav", 1046.50, "1") // C6 — same pitch class, different octave
|
||||
|
||||
low, err := Analyze(context.Background(), "", lowPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Analyze(low): %v", err)
|
||||
}
|
||||
high, err := Analyze(context.Background(), "", highPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Analyze(high): %v", err)
|
||||
}
|
||||
|
||||
if high.SpectralCentroidHz <= low.SpectralCentroidHz {
|
||||
t.Errorf("expected the higher tone to have a higher spectral centroid: low=%.1f high=%.1f",
|
||||
low.SpectralCentroidHz, high.SpectralCentroidHz)
|
||||
}
|
||||
|
||||
// Both tones are a C, just in different octaves: pitch-class energy
|
||||
// should peak at chroma bin 0 for both — a real, verifiable property
|
||||
// of the chroma computation, not a fuzzy assertion.
|
||||
if argmax(low.Chroma) != 0 {
|
||||
t.Errorf("low tone (C3) chroma peak = bin %d, want 0 (C)", argmax(low.Chroma))
|
||||
}
|
||||
if argmax(high.Chroma) != 0 {
|
||||
t.Errorf("high tone (C6) chroma peak = bin %d, want 0 (C)", argmax(high.Chroma))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeDifferentPitchClassesHaveDifferentChromaPeak(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cPath := genTone(t, dir, "c.wav", 261.63, "1")
|
||||
gPath := genTone(t, dir, "g.wav", 392.00, "1")
|
||||
|
||||
c, err := Analyze(context.Background(), "", cPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Analyze(C): %v", err)
|
||||
}
|
||||
g, err := Analyze(context.Background(), "", gPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Analyze(G): %v", err)
|
||||
}
|
||||
|
||||
if argmax(c.Chroma) != 0 {
|
||||
t.Errorf("C4 chroma peak = bin %d, want 0", argmax(c.Chroma))
|
||||
}
|
||||
if argmax(g.Chroma) != 7 {
|
||||
t.Errorf("G4 chroma peak = bin %d, want 7", argmax(g.Chroma))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeLoudVsQuietHaveDifferentLoudness(t *testing.T) {
|
||||
if _, err := exec.LookPath("ffmpeg"); err != nil {
|
||||
t.Skip("ffmpeg not installed, skipping audiofeatures integration test")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
loud := filepath.Join(dir, "loud.wav")
|
||||
if out, err := exec.CommandContext(ctx, "ffmpeg", "-y", "-f", "lavfi",
|
||||
"-i", "sine=frequency=440:duration=1", "-af", "volume=1.0", "-codec:a", "pcm_s16le", loud).CombinedOutput(); err != nil {
|
||||
t.Fatalf("ffmpeg: %v\n%s", err, out)
|
||||
}
|
||||
quiet := filepath.Join(dir, "quiet.wav")
|
||||
if out, err := exec.CommandContext(ctx, "ffmpeg", "-y", "-f", "lavfi",
|
||||
"-i", "sine=frequency=440:duration=1", "-af", "volume=0.05", "-codec:a", "pcm_s16le", quiet).CombinedOutput(); err != nil {
|
||||
t.Fatalf("ffmpeg: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
loudF, err := Analyze(context.Background(), "", loud)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
quietF, err := Analyze(context.Background(), "", quiet)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loudF.LoudnessLUFS <= quietF.LoudnessLUFS {
|
||||
t.Errorf("loud=%.1f quiet=%.1f, want loud clip to measure louder", loudF.LoudnessLUFS, quietF.LoudnessLUFS)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeShortClipLeavesTempoInvalid(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// A single sine tone has no rhythmic/onset content at all — VibeSonic
|
||||
// must not fabricate a BPM for it.
|
||||
path := genTone(t, dir, "tone.wav", 440, "0.5")
|
||||
|
||||
f, err := Analyze(context.Background(), "", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.TempoValid {
|
||||
t.Errorf("expected TempoValid=false for a featureless 0.5s tone, got %.1f BPM", f.TempoBPM)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorLengthAndTempoZeroWhenInvalid(t *testing.T) {
|
||||
f := Features{TempoValid: false, TempoBPM: 999}
|
||||
v := f.Vector()
|
||||
if len(v) != 17 {
|
||||
t.Fatalf("Vector length = %d, want 17", len(v))
|
||||
}
|
||||
if v[0] != 0 {
|
||||
t.Errorf("Vector()[0] (tempo) = %v, want 0 when TempoValid=false", v[0])
|
||||
}
|
||||
}
|
||||
|
||||
func argmax(v [12]float64) int {
|
||||
best := 0
|
||||
for i := 1; i < 12; i++ {
|
||||
if v[i] > v[best] {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package audiofeatures
|
||||
|
||||
import "math"
|
||||
|
||||
// fft computes the in-place iterative radix-2 Cooley-Tukey FFT of a,
|
||||
// whose length must be a power of two. It's the only DSP primitive this
|
||||
// package needs (spectral centroid/rolloff and the chroma vector all
|
||||
// derive from the resulting magnitude spectrum), so a small
|
||||
// hand-written implementation avoids pulling in an external numerics
|
||||
// dependency for one function.
|
||||
func fft(a []complex128) {
|
||||
n := len(a)
|
||||
if n <= 1 {
|
||||
return
|
||||
}
|
||||
|
||||
// Bit-reversal permutation.
|
||||
for i, j := 1, 0; i < n; i++ {
|
||||
bit := n >> 1
|
||||
for ; j&bit != 0; bit >>= 1 {
|
||||
j ^= bit
|
||||
}
|
||||
j ^= bit
|
||||
if i < j {
|
||||
a[i], a[j] = a[j], a[i]
|
||||
}
|
||||
}
|
||||
|
||||
for length := 2; length <= n; length <<= 1 {
|
||||
ang := -2 * math.Pi / float64(length)
|
||||
wlen := complex(math.Cos(ang), math.Sin(ang))
|
||||
for i := 0; i < n; i += length {
|
||||
w := complex(1, 0)
|
||||
half := length / 2
|
||||
for k := 0; k < half; k++ {
|
||||
u := a[i+k]
|
||||
v := a[i+k+half] * w
|
||||
a[i+k] = u + v
|
||||
a[i+k+half] = u - v
|
||||
w *= wlen
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// magnitudeSpectrum returns |FFT(frame)| for bins [0, n/2], given a
|
||||
// real-valued frame whose length is a power of two.
|
||||
func magnitudeSpectrum(frame []float64) []float64 {
|
||||
n := len(frame)
|
||||
buf := make([]complex128, n)
|
||||
for i, v := range frame {
|
||||
buf[i] = complex(v, 0)
|
||||
}
|
||||
fft(buf)
|
||||
mags := make([]float64, n/2+1)
|
||||
for i := range mags {
|
||||
mags[i] = cmplxAbs(buf[i])
|
||||
}
|
||||
return mags
|
||||
}
|
||||
|
||||
func cmplxAbs(c complex128) float64 {
|
||||
re, im := real(c), imag(c)
|
||||
return math.Sqrt(re*re + im*im)
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/audiofeatures"
|
||||
)
|
||||
|
||||
// AudioFeaturesRow is one song's stored sonic-analysis result.
|
||||
type AudioFeaturesRow struct {
|
||||
SongID string
|
||||
TempoBPM float64
|
||||
TempoValid bool
|
||||
MusicalKey string
|
||||
LoudnessLUFS float64
|
||||
Extra audioFeaturesExtra
|
||||
}
|
||||
|
||||
// audioFeaturesExtra is the analyzer output that doesn't have its own
|
||||
// dedicated column — kept in extra_json so the schema doesn't need a
|
||||
// migration every time the analyzer grows a new measurement.
|
||||
type audioFeaturesExtra struct {
|
||||
SpectralCentroidHz float64 `json:"spectralCentroidHz"`
|
||||
SpectralRolloffHz float64 `json:"spectralRolloffHz"`
|
||||
ZeroCrossingRate float64 `json:"zeroCrossingRate"`
|
||||
Chroma [12]float64 `json:"chroma"`
|
||||
}
|
||||
|
||||
// Vector reconstructs the same fixed-length feature vector
|
||||
// audiofeatures.Features.Vector produces, from stored values.
|
||||
func (r AudioFeaturesRow) Vector() []float64 {
|
||||
tempo := 0.0
|
||||
if r.TempoValid {
|
||||
tempo = r.TempoBPM
|
||||
}
|
||||
v := make([]float64, 0, 17)
|
||||
v = append(v, tempo, r.LoudnessLUFS, r.Extra.SpectralCentroidHz, r.Extra.SpectralRolloffHz, r.Extra.ZeroCrossingRate)
|
||||
v = append(v, r.Extra.Chroma[:]...)
|
||||
return v
|
||||
}
|
||||
|
||||
// UpsertAudioFeatures stores (or replaces) the analyzed features for a
|
||||
// song.
|
||||
func UpsertAudioFeatures(db *sql.DB, songID string, f *audiofeatures.Features) error {
|
||||
extra := audioFeaturesExtra{
|
||||
SpectralCentroidHz: f.SpectralCentroidHz,
|
||||
SpectralRolloffHz: f.SpectralRolloffHz,
|
||||
ZeroCrossingRate: f.ZeroCrossingRate,
|
||||
Chroma: f.Chroma,
|
||||
}
|
||||
extraJSON, err := json.Marshal(extra)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var tempo sql.NullFloat64
|
||||
if f.TempoValid {
|
||||
tempo = sql.NullFloat64{Float64: f.TempoBPM, Valid: true}
|
||||
}
|
||||
_, err = db.Exec(`
|
||||
INSERT INTO audio_features (song_id, tempo_bpm, musical_key, loudness_lufs, extra_json, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
||||
ON CONFLICT(song_id) DO UPDATE SET
|
||||
tempo_bpm = excluded.tempo_bpm,
|
||||
musical_key = excluded.musical_key,
|
||||
loudness_lufs = excluded.loudness_lufs,
|
||||
extra_json = excluded.extra_json,
|
||||
updated_at = excluded.updated_at`,
|
||||
songID, tempo, f.Key, f.LoudnessLUFS, string(extraJSON))
|
||||
return err
|
||||
}
|
||||
|
||||
const audioFeaturesSelect = `SELECT song_id, tempo_bpm, musical_key, loudness_lufs, extra_json FROM audio_features`
|
||||
|
||||
func scanAudioFeaturesRow(scan func(dest ...any) error) (*AudioFeaturesRow, error) {
|
||||
var r AudioFeaturesRow
|
||||
var tempo sql.NullFloat64
|
||||
var loudness sql.NullFloat64
|
||||
var extraJSON string
|
||||
if err := scan(&r.SongID, &tempo, &r.MusicalKey, &loudness, &extraJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.TempoValid = tempo.Valid
|
||||
r.TempoBPM = tempo.Float64
|
||||
r.LoudnessLUFS = loudness.Float64
|
||||
_ = json.Unmarshal([]byte(extraJSON), &r.Extra)
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// GetAudioFeatures returns a song's stored analysis, or nil if it
|
||||
// hasn't been analyzed yet.
|
||||
func GetAudioFeatures(db *sql.DB, songID string) (*AudioFeaturesRow, error) {
|
||||
row := db.QueryRow(audioFeaturesSelect+` WHERE song_id = ?`, songID)
|
||||
r, err := scanAudioFeaturesRow(row.Scan)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
// ListAllAudioFeatures returns every analyzed song's features, the
|
||||
// input the in-process sonic-similarity index is built from.
|
||||
func ListAllAudioFeatures(db *sql.DB) ([]AudioFeaturesRow, error) {
|
||||
rows, err := db.Query(audioFeaturesSelect)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []AudioFeaturesRow
|
||||
for rows.Next() {
|
||||
r, err := scanAudioFeaturesRow(rows.Scan)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, *r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/audiofeatures"
|
||||
)
|
||||
|
||||
func openAudioFeaturesTestDB(t *testing.T) (*sql.DB, string) {
|
||||
t.Helper()
|
||||
db, err := Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
folder, err := CreateMusicFolder(db, "Music", "/music", "music")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mf, err := CreateMediaFile(db, folder.ID, "song.mp3", 100, 0, "h", "mp3", "song", 10, 128, 44100, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
song, err := UpsertSong(db, UpsertSongInput{MediaFileID: mf.ID, Title: "Song"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db, song.ID
|
||||
}
|
||||
|
||||
func TestUpsertAndGetAudioFeatures(t *testing.T) {
|
||||
db, songID := openAudioFeaturesTestDB(t)
|
||||
|
||||
f := &audiofeatures.Features{
|
||||
TempoBPM: 120, TempoValid: true, Key: "C major", LoudnessLUFS: -14,
|
||||
SpectralCentroidHz: 1500, SpectralRolloffHz: 4000, ZeroCrossingRate: 0.05,
|
||||
Chroma: [12]float64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
|
||||
}
|
||||
if err := UpsertAudioFeatures(db, songID, f); err != nil {
|
||||
t.Fatalf("UpsertAudioFeatures: %v", err)
|
||||
}
|
||||
|
||||
got, err := GetAudioFeatures(db, songID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudioFeatures: %v", err)
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatal("expected a row, got nil")
|
||||
}
|
||||
if !got.TempoValid || got.TempoBPM != 120 {
|
||||
t.Errorf("tempo = %v/%v, want 120/valid", got.TempoBPM, got.TempoValid)
|
||||
}
|
||||
if got.MusicalKey != "C major" {
|
||||
t.Errorf("key = %q", got.MusicalKey)
|
||||
}
|
||||
if got.LoudnessLUFS != -14 {
|
||||
t.Errorf("loudness = %v", got.LoudnessLUFS)
|
||||
}
|
||||
if got.Extra.SpectralCentroidHz != 1500 || got.Extra.Chroma[0] != 1 {
|
||||
t.Errorf("extra = %+v", got.Extra)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertAudioFeaturesOverwrites(t *testing.T) {
|
||||
db, songID := openAudioFeaturesTestDB(t)
|
||||
if err := UpsertAudioFeatures(db, songID, &audiofeatures.Features{Key: "C major"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := UpsertAudioFeatures(db, songID, &audiofeatures.Features{Key: "D minor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := GetAudioFeatures(db, songID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.MusicalKey != "D minor" {
|
||||
t.Errorf("key = %q, want overwrite to D minor", got.MusicalKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAudioFeaturesNotAnalyzed(t *testing.T) {
|
||||
db, songID := openAudioFeaturesTestDB(t)
|
||||
got, err := GetAudioFeatures(db, songID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Errorf("expected nil for an unanalyzed song, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListAllAudioFeatures(t *testing.T) {
|
||||
db, songID := openAudioFeaturesTestDB(t)
|
||||
if err := UpsertAudioFeatures(db, songID, &audiofeatures.Features{TempoValid: true, TempoBPM: 90}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := ListAllAudioFeatures(db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(all) != 1 || all[0].SongID != songID {
|
||||
t.Fatalf("all = %+v", all)
|
||||
}
|
||||
if len(all[0].Vector()) != 17 {
|
||||
t.Fatalf("Vector length = %d, want 17", len(all[0].Vector()))
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/audiofeatures"
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/coverart"
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/database"
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/library"
|
||||
@@ -38,6 +39,7 @@ type Scanner struct {
|
||||
DB *sql.DB
|
||||
Prober *metadata.Prober
|
||||
Log *slog.Logger
|
||||
FFmpegPath string // for sonic-analysis audio decoding; "" defaults to "ffmpeg" on PATH
|
||||
|
||||
mu sync.Mutex
|
||||
scanning map[string]bool // music_folder_id -> in progress
|
||||
@@ -175,6 +177,7 @@ func (s *Scanner) indexOne(ctx context.Context, folderID, absPath, relPath strin
|
||||
return false, false, err
|
||||
}
|
||||
s.indexCoverArt(absPath, song, tags)
|
||||
s.analyzeAudio(ctx, absPath, song, tags)
|
||||
return false, true, nil
|
||||
}
|
||||
if err != database.ErrNotFound {
|
||||
@@ -204,9 +207,31 @@ func (s *Scanner) indexOne(ctx context.Context, folderID, absPath, relPath strin
|
||||
return false, false, err
|
||||
}
|
||||
s.indexCoverArt(absPath, song, tags)
|
||||
s.analyzeAudio(ctx, absPath, song, tags)
|
||||
return true, false, nil
|
||||
}
|
||||
|
||||
// analyzeAudio runs the sonic-feature analyzer over a scanned file and
|
||||
// stores the result, powering the sonicSimilarity extension. This runs
|
||||
// as part of the (already background, admin-triggered) scan itself
|
||||
// rather than on any request path. A decode/analysis failure is logged
|
||||
// and otherwise ignored: the song is still indexed and playable, it
|
||||
// just won't appear in sonic-similarity results until a future rescan
|
||||
// succeeds — never a reason to fail the whole scan.
|
||||
func (s *Scanner) analyzeAudio(ctx context.Context, absPath string, song *database.Song, tags *metadata.Tags) {
|
||||
if tags.Kind != "song" {
|
||||
return
|
||||
}
|
||||
features, err := audiofeatures.Analyze(ctx, s.FFmpegPath, absPath)
|
||||
if err != nil {
|
||||
s.Log.Warn("audio feature analysis failed", "err", err, "path", absPath)
|
||||
return
|
||||
}
|
||||
if err := database.UpsertAudioFeatures(s.DB, song.ID, features); err != nil {
|
||||
s.Log.Warn("storing audio features failed", "err", err, "path", absPath)
|
||||
}
|
||||
}
|
||||
|
||||
// indexCoverArt records where an album's cover art can be found: a
|
||||
// folder image next to the file if one exists, otherwise the file's own
|
||||
// embedded artwork (extracted lazily by the web layer on request, not
|
||||
|
||||
@@ -279,3 +279,40 @@ func TestScanFolderIndexesSidecarLRC(t *testing.T) {
|
||||
t.Fatalf("expected word-level cues from Enhanced LRC line, got %+v", synced.Lines[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanFolderAnalyzesAudioFeatures(t *testing.T) {
|
||||
s, folder, dir := newTestScanner(t)
|
||||
genFile(t, dir, "song1.mp3", "Song One", "Artist One")
|
||||
|
||||
if err := s.ScanFolder(context.Background(), folder.ID); err != nil {
|
||||
t.Fatalf("ScanFolder: %v", err)
|
||||
}
|
||||
|
||||
songs, err := database.ListSongs(s.DB, 10, 0)
|
||||
if err != nil || len(songs) != 1 {
|
||||
t.Fatalf("ListSongs: %v %v", songs, err)
|
||||
}
|
||||
|
||||
features, err := database.GetAudioFeatures(s.DB, songs[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAudioFeatures: %v", err)
|
||||
}
|
||||
if features == nil {
|
||||
t.Fatal("expected audio features to be stored after scanning, got none")
|
||||
}
|
||||
// The fixture is a pure 440Hz sine tone (A4): its chroma should
|
||||
// actually peak at pitch class A (bin 9), a real measured property,
|
||||
// not a placeholder.
|
||||
if features.Extra.Chroma == ([12]float64{}) {
|
||||
t.Error("expected a non-zero chroma vector from real analysis")
|
||||
}
|
||||
best := 0
|
||||
for i := 1; i < 12; i++ {
|
||||
if features.Extra.Chroma[i] > features.Extra.Chroma[best] {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
if best != 9 { // A
|
||||
t.Errorf("chroma peak = bin %d, want 9 (A) for a 440Hz fixture", best)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
// Package similarity builds a small in-process nearest-neighbor index
|
||||
// over numeric audio-feature vectors and uses it for two things:
|
||||
// ranking tracks by actual sonic closeness, and walking a musically
|
||||
// reasonable path between two tracks through feature space. There is
|
||||
// no external vector database — the whole index is just normalized
|
||||
// vectors held in memory, rebuilt from internal/database's
|
||||
// audio_features table whenever a caller needs it.
|
||||
package similarity
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Item is one track's raw (un-normalized) feature vector, keyed by
|
||||
// song ID.
|
||||
type Item struct {
|
||||
ID string
|
||||
Vector []float64
|
||||
}
|
||||
|
||||
// Index holds z-score-normalized vectors for a set of tracks. Each
|
||||
// dimension is normalized independently using the mean/stddev of that
|
||||
// dimension across exactly the items the index was built from, so
|
||||
// dimensions with a wide natural range (like tempo, in BPM) don't
|
||||
// dominate Euclidean distance over dimensions with a narrow one (like a
|
||||
// normalized chroma bin).
|
||||
type Index struct {
|
||||
ids []string
|
||||
norm [][]float64
|
||||
}
|
||||
|
||||
// Build normalizes and indexes items. An item whose Vector length
|
||||
// disagrees with the first item's is dropped rather than causing a
|
||||
// panic or silently misaligning dimensions — a defensive check, not
|
||||
// something expected to trigger with correctly-produced feature rows.
|
||||
func Build(items []Item) *Index {
|
||||
idx := &Index{}
|
||||
if len(items) == 0 {
|
||||
return idx
|
||||
}
|
||||
dims := len(items[0].Vector)
|
||||
|
||||
var kept []Item
|
||||
for _, it := range items {
|
||||
if len(it.Vector) == dims {
|
||||
kept = append(kept, it)
|
||||
}
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return idx
|
||||
}
|
||||
|
||||
means := make([]float64, dims)
|
||||
for _, it := range kept {
|
||||
for d, v := range it.Vector {
|
||||
means[d] += v
|
||||
}
|
||||
}
|
||||
for d := range means {
|
||||
means[d] /= float64(len(kept))
|
||||
}
|
||||
|
||||
stds := make([]float64, dims)
|
||||
for _, it := range kept {
|
||||
for d, v := range it.Vector {
|
||||
diff := v - means[d]
|
||||
stds[d] += diff * diff
|
||||
}
|
||||
}
|
||||
for d := range stds {
|
||||
stds[d] = math.Sqrt(stds[d] / float64(len(kept)))
|
||||
}
|
||||
|
||||
idx.ids = make([]string, len(kept))
|
||||
idx.norm = make([][]float64, len(kept))
|
||||
for i, it := range kept {
|
||||
nv := make([]float64, dims)
|
||||
for d, v := range it.Vector {
|
||||
if stds[d] > 1e-9 {
|
||||
nv[d] = (v - means[d]) / stds[d]
|
||||
}
|
||||
}
|
||||
idx.ids[i] = it.ID
|
||||
idx.norm[i] = nv
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// Match is one similarity result: another track and its distance from
|
||||
// the query track in normalized feature space (0 = identical, larger =
|
||||
// less similar).
|
||||
type Match struct {
|
||||
ID string
|
||||
Distance float64
|
||||
}
|
||||
|
||||
// Nearest returns up to k tracks closest to id, nearest first,
|
||||
// excluding id itself. Returns nil if id isn't in the index.
|
||||
func (idx *Index) Nearest(id string, k int) []Match {
|
||||
pos := idx.indexOf(id)
|
||||
if pos < 0 {
|
||||
return nil
|
||||
}
|
||||
matches := make([]Match, 0, len(idx.ids)-1)
|
||||
for i, other := range idx.ids {
|
||||
if i == pos {
|
||||
continue
|
||||
}
|
||||
matches = append(matches, Match{ID: other, Distance: euclidean(idx.norm[pos], idx.norm[i])})
|
||||
}
|
||||
sort.Slice(matches, func(a, b int) bool { return matches[a].Distance < matches[b].Distance })
|
||||
if k > 0 && len(matches) > k {
|
||||
matches = matches[:k]
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// Path walks from fromID to toID in `steps` hops (including both
|
||||
// endpoints), picking at each intermediate step the not-yet-used track
|
||||
// closest to the corresponding point on the straight line between
|
||||
// fromID's and toID's feature vectors. That keeps the sequence
|
||||
// perceptually gradual — each hop is a small, real step in sonic
|
||||
// character rather than a random jump — without ever inventing a track
|
||||
// that isn't actually in the library. Returns nil if either endpoint
|
||||
// isn't indexed.
|
||||
func (idx *Index) Path(fromID, toID string, steps int) []string {
|
||||
fromPos := idx.indexOf(fromID)
|
||||
toPos := idx.indexOf(toID)
|
||||
if fromPos < 0 || toPos < 0 {
|
||||
return nil
|
||||
}
|
||||
if steps < 2 {
|
||||
steps = 2
|
||||
}
|
||||
|
||||
path := []string{fromID}
|
||||
used := map[string]bool{fromID: true, toID: true}
|
||||
fromVec, toVec := idx.norm[fromPos], idx.norm[toPos]
|
||||
|
||||
for step := 1; step < steps-1; step++ {
|
||||
t := float64(step) / float64(steps-1)
|
||||
target := lerp(fromVec, toVec, t)
|
||||
|
||||
best := -1
|
||||
bestDist := math.Inf(1)
|
||||
for i, id := range idx.ids {
|
||||
if used[id] {
|
||||
continue
|
||||
}
|
||||
d := euclidean(idx.norm[i], target)
|
||||
if d < bestDist {
|
||||
bestDist = d
|
||||
best = i
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
break // library exhausted; fall through with what we have
|
||||
}
|
||||
path = append(path, idx.ids[best])
|
||||
used[idx.ids[best]] = true
|
||||
}
|
||||
|
||||
path = append(path, toID)
|
||||
return path
|
||||
}
|
||||
|
||||
// DistanceBetween returns the normalized-feature-space Euclidean
|
||||
// distance between two indexed tracks, or +Inf if either isn't
|
||||
// indexed.
|
||||
func (idx *Index) DistanceBetween(idA, idB string) float64 {
|
||||
a, b := idx.indexOf(idA), idx.indexOf(idB)
|
||||
if a < 0 || b < 0 {
|
||||
return math.Inf(1)
|
||||
}
|
||||
return euclidean(idx.norm[a], idx.norm[b])
|
||||
}
|
||||
|
||||
func (idx *Index) indexOf(id string) int {
|
||||
for i, v := range idx.ids {
|
||||
if v == id {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func euclidean(a, b []float64) float64 {
|
||||
var sum float64
|
||||
for i := range a {
|
||||
d := a[i] - b[i]
|
||||
sum += d * d
|
||||
}
|
||||
return math.Sqrt(sum)
|
||||
}
|
||||
|
||||
func lerp(a, b []float64, t float64) []float64 {
|
||||
out := make([]float64, len(a))
|
||||
for i := range a {
|
||||
out[i] = a[i] + (b[i]-a[i])*t
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package similarity
|
||||
|
||||
import "testing"
|
||||
|
||||
func tempoLibrary() []Item {
|
||||
// A synthetic library where the only varying dimension is "tempo":
|
||||
// songs at 60, 70, 80, ..., 150 BPM. Deterministic and needs no
|
||||
// audio decoding, so it exercises the index math directly.
|
||||
var items []Item
|
||||
for i, bpm := range []float64{60, 70, 80, 90, 100, 110, 120, 130, 140, 150} {
|
||||
items = append(items, Item{ID: idFor(i), Vector: []float64{bpm, 0, 0}})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func idFor(i int) string {
|
||||
return string(rune('a' + i))
|
||||
}
|
||||
|
||||
func TestNearestOrdersByActualDistance(t *testing.T) {
|
||||
idx := Build(tempoLibrary())
|
||||
// "d" is 90 BPM; its closest neighbors by tempo should be "c" (80)
|
||||
// and "e" (100), in that order (tie broken by index order is fine
|
||||
// since both are exactly 10 BPM away — check the pair, not order).
|
||||
matches := idx.Nearest("d", 2)
|
||||
if len(matches) != 2 {
|
||||
t.Fatalf("matches = %v, want 2", matches)
|
||||
}
|
||||
got := map[string]bool{matches[0].ID: true, matches[1].ID: true}
|
||||
if !got["c"] || !got["e"] {
|
||||
t.Errorf("nearest to d(90bpm) = %v, want {c(80bpm), e(100bpm)}", matches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNearestExcludesSelf(t *testing.T) {
|
||||
idx := Build(tempoLibrary())
|
||||
matches := idx.Nearest("a", 20)
|
||||
for _, m := range matches {
|
||||
if m.ID == "a" {
|
||||
t.Fatal("Nearest must never include the query track itself")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNearestUnknownID(t *testing.T) {
|
||||
idx := Build(tempoLibrary())
|
||||
if matches := idx.Nearest("does-not-exist", 5); matches != nil {
|
||||
t.Errorf("expected nil for unknown id, got %v", matches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNearestFarthestIsOppositeEnd(t *testing.T) {
|
||||
idx := Build(tempoLibrary())
|
||||
matches := idx.Nearest("a", 20) // a = 60bpm, all 9 others
|
||||
if len(matches) != 9 {
|
||||
t.Fatalf("matches = %d, want 9", len(matches))
|
||||
}
|
||||
farthest := matches[len(matches)-1]
|
||||
if farthest.ID != "j" { // j = 150bpm, the farthest from 60bpm
|
||||
t.Errorf("farthest from a(60bpm) = %v, want j(150bpm)", farthest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathIsMonotonicAlongTheOnlyVaryingDimension(t *testing.T) {
|
||||
idx := Build(tempoLibrary())
|
||||
path := idx.Path("a", "j", 5) // 60bpm -> 150bpm in 5 hops
|
||||
if len(path) != 5 {
|
||||
t.Fatalf("path = %v, want length 5", path)
|
||||
}
|
||||
if path[0] != "a" || path[len(path)-1] != "j" {
|
||||
t.Fatalf("path endpoints = %q..%q, want a..j", path[0], path[len(path)-1])
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, id := range path {
|
||||
if seen[id] {
|
||||
t.Fatalf("path repeats %q: %v", id, path)
|
||||
}
|
||||
seen[id] = true
|
||||
}
|
||||
// With tempo as the only varying dimension, a straight-line
|
||||
// interpolation from 60 to 150 BPM should visit strictly increasing
|
||||
// tempos.
|
||||
bpmOf := map[string]float64{"a": 60, "b": 70, "c": 80, "d": 90, "e": 100, "f": 110, "g": 120, "h": 130, "i": 140, "j": 150}
|
||||
for i := 1; i < len(path); i++ {
|
||||
if bpmOf[path[i]] <= bpmOf[path[i-1]] {
|
||||
t.Errorf("path not monotonically increasing in tempo: %v", path)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathUnknownEndpoint(t *testing.T) {
|
||||
idx := Build(tempoLibrary())
|
||||
if path := idx.Path("a", "nope", 5); path != nil {
|
||||
t.Errorf("expected nil path for unknown endpoint, got %v", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildEmpty(t *testing.T) {
|
||||
idx := Build(nil)
|
||||
if matches := idx.Nearest("a", 5); matches != nil {
|
||||
t.Errorf("expected nil from an empty index, got %v", matches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildConstantDimensionDoesNotDivideByZero(t *testing.T) {
|
||||
items := []Item{
|
||||
{ID: "x", Vector: []float64{5, 1}},
|
||||
{ID: "y", Vector: []float64{5, 2}},
|
||||
}
|
||||
idx := Build(items)
|
||||
matches := idx.Nearest("x", 5)
|
||||
if len(matches) != 1 || matches[0].ID != "y" {
|
||||
t.Fatalf("matches = %v", matches)
|
||||
}
|
||||
}
|
||||
@@ -55,7 +55,7 @@ func TestGetOpenSubsonicExtensions(t *testing.T) {
|
||||
// set once Phase 3 is done.
|
||||
for _, want := range []string{
|
||||
"apiKeyAuthentication", "formPost", "getPodcastEpisode", "topSongsByArtistId", "transcodeOffset",
|
||||
"indexBasedQueue", "playbackReport", "songLyrics", "transcoding",
|
||||
"indexBasedQueue", "playbackReport", "songLyrics", "transcoding", "sonicSimilarity",
|
||||
} {
|
||||
if !names[want] {
|
||||
t.Errorf("extension %q not advertised: %v", want, names)
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
package subsonic
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/database"
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/similarity"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registerExtension("sonicSimilarity", 1)
|
||||
register("getSonicSimilarTracks", handleGetSonicSimilarTracks)
|
||||
register("findSonicPath", handleFindSonicPath)
|
||||
}
|
||||
|
||||
// buildSimilarityIndex loads every song VibeSonic has actually analyzed
|
||||
// (audio_features rows populated by the scanner's background analysis)
|
||||
// and builds an in-process nearest-neighbor index from their real
|
||||
// feature vectors. It's rebuilt per request rather than cached: the
|
||||
// library-wide mean/stddev normalization it depends on shifts as more
|
||||
// songs get analyzed, and a from-scratch build over even a large
|
||||
// library is a few hundred microseconds of arithmetic over small
|
||||
// vectors, not a bottleneck worth caching.
|
||||
func buildSimilarityIndex(db *sql.DB) (*similarity.Index, error) {
|
||||
rows, err := database.ListAllAudioFeatures(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := make([]similarity.Item, len(rows))
|
||||
for i, r := range rows {
|
||||
items[i] = similarity.Item{ID: r.SongID, Vector: r.Vector()}
|
||||
}
|
||||
return similarity.Build(items), nil
|
||||
}
|
||||
|
||||
// similarityScore converts a Euclidean distance in normalized feature
|
||||
// space into the "normalized similarity score" the spec asks for: 1.0
|
||||
// for an identical vector, approaching 0 as distance grows, never
|
||||
// negative.
|
||||
func similarityScore(distance float64) float64 {
|
||||
return 1 / (1 + distance)
|
||||
}
|
||||
|
||||
func childForSongID(db *sql.DB, songID, userID string) (*Child, error) {
|
||||
song, err := database.GetSong(db, songID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mf, err := database.GetMediaFile(db, song.MediaFileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := childFromSong(db, song, mf, albumOfSong(db, song), userID)
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
func handleGetSonicSimilarTracks(rc *Context) *apiError {
|
||||
id, aerr := rc.Params.Required("id")
|
||||
if aerr != nil {
|
||||
return aerr
|
||||
}
|
||||
count := rc.Params.Int("count", 10)
|
||||
db := rc.Server.DB
|
||||
|
||||
if _, err := database.GetSong(db, id); err != nil {
|
||||
return errNotFound("Song")
|
||||
}
|
||||
|
||||
idx, err := buildSimilarityIndex(db)
|
||||
if err != nil {
|
||||
return newError(ErrGeneric, err.Error())
|
||||
}
|
||||
matches := idx.Nearest(id, count)
|
||||
if matches == nil {
|
||||
// Either the song hasn't been analyzed yet, or nothing else in
|
||||
// the library has — an honest empty result, not an error.
|
||||
rc.Response = newResponse()
|
||||
return nil
|
||||
}
|
||||
|
||||
var out []SonicMatch
|
||||
for _, m := range matches {
|
||||
child, err := childForSongID(db, m.ID, rc.User.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, SonicMatch{Entry: *child, Similarity: similarityScore(m.Distance)})
|
||||
}
|
||||
rc.Response = newResponse()
|
||||
rc.Response.SonicMatch = out
|
||||
return nil
|
||||
}
|
||||
|
||||
func handleFindSonicPath(rc *Context) *apiError {
|
||||
startID, aerr := rc.Params.Required("startSongId")
|
||||
if aerr != nil {
|
||||
return aerr
|
||||
}
|
||||
endID, aerr := rc.Params.Required("endSongId")
|
||||
if aerr != nil {
|
||||
return aerr
|
||||
}
|
||||
count := rc.Params.Int("count", 25)
|
||||
db := rc.Server.DB
|
||||
|
||||
if _, err := database.GetSong(db, startID); err != nil {
|
||||
return errNotFound("Start song")
|
||||
}
|
||||
if _, err := database.GetSong(db, endID); err != nil {
|
||||
return errNotFound("End song")
|
||||
}
|
||||
|
||||
idx, err := buildSimilarityIndex(db)
|
||||
if err != nil {
|
||||
return newError(ErrGeneric, err.Error())
|
||||
}
|
||||
path := idx.Path(startID, endID, count)
|
||||
if path == nil {
|
||||
// Analysis hasn't run for one or both endpoints yet: an honest
|
||||
// empty result rather than a fabricated single-hop path.
|
||||
rc.Response = newResponse()
|
||||
return nil
|
||||
}
|
||||
|
||||
var out []SonicMatch
|
||||
for _, songID := range path {
|
||||
child, err := childForSongID(db, songID, rc.User.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
distFromStart := idx.DistanceBetween(startID, songID)
|
||||
out = append(out, SonicMatch{Entry: *child, Similarity: similarityScore(distFromStart)})
|
||||
}
|
||||
rc.Response = newResponse()
|
||||
rc.Response.SonicMatch = out
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package subsonic
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/audiofeatures"
|
||||
"git.noroutes.net/NoRoutes/vibetunes/internal/database"
|
||||
)
|
||||
|
||||
// makeAnalyzedSong creates a minimal song fixture (no real audio file
|
||||
// needed — sonic-similarity handlers only ever read the songs table and
|
||||
// audio_features, never touch disk) and stores a hand-crafted feature
|
||||
// vector for it, isolating these tests from ffmpeg/analysis timing.
|
||||
func makeAnalyzedSong(t *testing.T, s *Server, folderID, title string, tempo float64) *database.Song {
|
||||
t.Helper()
|
||||
mf, err := database.CreateMediaFile(s.DB, folderID, title+".mp3", 100, 0, "hash-"+title, "mp3", "song", 10, 128, 44100, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
song, err := database.UpsertSong(s.DB, database.UpsertSongInput{MediaFileID: mf.ID, Title: title})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := &audiofeatures.Features{TempoBPM: tempo, TempoValid: true, Key: "C major"}
|
||||
if err := database.UpsertAudioFeatures(s.DB, song.ID, f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return song
|
||||
}
|
||||
|
||||
func sonicTestLibrary(t *testing.T, s *Server) (slow, mid, fast, unanalyzed *database.Song) {
|
||||
t.Helper()
|
||||
folder, err := database.CreateMusicFolder(s.DB, "Music", "/music", "music")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
slow = makeAnalyzedSong(t, s, folder.ID, "Slow", 70)
|
||||
mid = makeAnalyzedSong(t, s, folder.ID, "Mid", 100)
|
||||
fast = makeAnalyzedSong(t, s, folder.ID, "Fast", 160)
|
||||
|
||||
mf, err := database.CreateMediaFile(s.DB, folder.ID, "Unanalyzed.mp3", 100, 0, "hash-unanalyzed", "mp3", "song", 10, 128, 44100, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unanalyzed, err = database.UpsertSong(s.DB, database.UpsertSongInput{MediaFileID: mf.ID, Title: "Unanalyzed"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func TestGetSonicSimilarTracksOrdersByRealDistance(t *testing.T) {
|
||||
s, _ := newTestServer(t)
|
||||
createTestUser(t, s, "alice", "password123", false)
|
||||
_, mid, fast, _ := sonicTestLibrary(t, s)
|
||||
|
||||
srv := newSubsonicTestServer(t, s)
|
||||
q := baseParams("alice", "password123")
|
||||
q.Set("id", mid.ID) // 100bpm: 30bpm from Slow(70), 60bpm from Fast(160) — Slow should rank first
|
||||
q.Set("f", "json")
|
||||
resp := getPath(t, srv, "/rest/getSonicSimilarTracks.view", q)
|
||||
j := decodeJSON(t, resp)
|
||||
matches, ok := j["sonicMatch"].([]any)
|
||||
if !ok || len(matches) != 2 {
|
||||
t.Fatalf("sonicMatch = %v, want 2 entries", j)
|
||||
}
|
||||
first := matches[0].(map[string]any)
|
||||
entry := first["entry"].(map[string]any)
|
||||
if entry["title"] != "Slow" {
|
||||
t.Errorf("nearest to Mid(100bpm) = %v, want Slow(70bpm) (closer than Fast(160bpm))", entry["title"])
|
||||
}
|
||||
sim := first["similarity"].(float64)
|
||||
if sim <= 0 || sim > 1 {
|
||||
t.Errorf("similarity = %v, want in (0,1]", sim)
|
||||
}
|
||||
_ = fast
|
||||
}
|
||||
|
||||
func TestGetSonicSimilarTracksExcludesUnanalyzedSongs(t *testing.T) {
|
||||
s, _ := newTestServer(t)
|
||||
createTestUser(t, s, "alice", "password123", false)
|
||||
slow, _, _, unanalyzed := sonicTestLibrary(t, s)
|
||||
|
||||
srv := newSubsonicTestServer(t, s)
|
||||
q := baseParams("alice", "password123")
|
||||
q.Set("id", slow.ID)
|
||||
q.Set("f", "json")
|
||||
resp := getPath(t, srv, "/rest/getSonicSimilarTracks.view", q)
|
||||
j := decodeJSON(t, resp)
|
||||
matches := j["sonicMatch"].([]any)
|
||||
for _, m := range matches {
|
||||
entry := m.(map[string]any)["entry"].(map[string]any)
|
||||
if entry["title"] == "Unanalyzed" {
|
||||
t.Error("a song with no stored audio_features must never appear in sonic-similarity results")
|
||||
}
|
||||
}
|
||||
_ = unanalyzed
|
||||
}
|
||||
|
||||
func TestGetSonicSimilarTracksUnknownSong(t *testing.T) {
|
||||
s, srv := newTestRecorderServer(t)
|
||||
createTestUser(t, s, "alice", "password123", false)
|
||||
|
||||
q := baseParams("alice", "password123")
|
||||
q.Set("id", "sg_doesnotexist")
|
||||
resp := getPath(t, srv.URL, "/rest/getSonicSimilarTracks.view", q)
|
||||
x := decodeXML(t, resp)
|
||||
if x.Status != "failed" || x.Error == nil || x.Error.Code != ErrNotFound {
|
||||
t.Fatalf("expected error 70, got %+v", x)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetSonicSimilarTracksNeverAnalyzedReturnsEmpty(t *testing.T) {
|
||||
s, _ := newTestServer(t)
|
||||
createTestUser(t, s, "alice", "password123", false)
|
||||
folder, err := database.CreateMusicFolder(s.DB, "Music", "/music", "music")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mf, err := database.CreateMediaFile(s.DB, folder.ID, "solo.mp3", 100, 0, "h", "mp3", "song", 10, 128, 44100, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
song, err := database.UpsertSong(s.DB, database.UpsertSongInput{MediaFileID: mf.ID, Title: "Solo"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := newSubsonicTestServer(t, s)
|
||||
q := baseParams("alice", "password123")
|
||||
q.Set("id", song.ID)
|
||||
q.Set("f", "json")
|
||||
resp := getPath(t, srv, "/rest/getSonicSimilarTracks.view", q)
|
||||
j := decodeJSON(t, resp)
|
||||
// No song in the library has been analyzed at all — VibeSonic must
|
||||
// return an honest empty result, never a fabricated match.
|
||||
if _, has := j["sonicMatch"]; has {
|
||||
t.Errorf("expected no sonicMatch key when nothing is analyzed, got %v", j["sonicMatch"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindSonicPathReachesBothEndpointsAndOrdersMonotonically(t *testing.T) {
|
||||
s, _ := newTestServer(t)
|
||||
createTestUser(t, s, "alice", "password123", false)
|
||||
slow, _, fast, _ := sonicTestLibrary(t, s)
|
||||
|
||||
srv := newSubsonicTestServer(t, s)
|
||||
q := baseParams("alice", "password123")
|
||||
q.Set("startSongId", slow.ID)
|
||||
q.Set("endSongId", fast.ID)
|
||||
q.Set("count", "3")
|
||||
q.Set("f", "json")
|
||||
resp := getPath(t, srv, "/rest/findSonicPath.view", q)
|
||||
j := decodeJSON(t, resp)
|
||||
matches, ok := j["sonicMatch"].([]any)
|
||||
if !ok || len(matches) != 3 {
|
||||
t.Fatalf("sonicMatch = %v, want 3 (start, mid, end)", j)
|
||||
}
|
||||
first := matches[0].(map[string]any)["entry"].(map[string]any)
|
||||
last := matches[2].(map[string]any)["entry"].(map[string]any)
|
||||
middle := matches[1].(map[string]any)["entry"].(map[string]any)
|
||||
if first["title"] != "Slow" || last["title"] != "Fast" {
|
||||
t.Fatalf("path endpoints = %v .. %v, want Slow .. Fast", first["title"], last["title"])
|
||||
}
|
||||
if middle["title"] != "Mid" {
|
||||
t.Errorf("path middle hop = %v, want Mid (the only other analyzed song, between Slow and Fast)", middle["title"])
|
||||
}
|
||||
startSim := matches[0].(map[string]any)["similarity"].(float64)
|
||||
if startSim != 1 {
|
||||
t.Errorf("similarity of start song to itself = %v, want 1", startSim)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindSonicPathUnknownEndpoint(t *testing.T) {
|
||||
s, _ := newTestServer(t)
|
||||
createTestUser(t, s, "alice", "password123", false)
|
||||
slow, _, _, _ := sonicTestLibrary(t, s)
|
||||
|
||||
srv := newSubsonicTestServer(t, s)
|
||||
q := baseParams("alice", "password123")
|
||||
q.Set("startSongId", slow.ID)
|
||||
q.Set("endSongId", "sg_doesnotexist")
|
||||
resp := getPath(t, srv, "/rest/findSonicPath.view", q)
|
||||
x := decodeXML(t, resp)
|
||||
if x.Status != "failed" || x.Error == nil || x.Error.Code != ErrNotFound {
|
||||
t.Fatalf("expected error 70, got %+v", x)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user