very WIP - proof of concept for waveform seekbar

2025-07-23 22:10:33 -07:00
parent dfa493501d
commit ae7cb55c8c
5 changed files with 301 additions and 6 deletions
@@ -582,7 +582,11 @@ func (p *playbackEngine) cacheNextTracks() {
 				}
 			}
 		}
-		p.audiocache.CacheOnly(p.NowPlaying().Metadata().ID, fetch)
+		id := ""
+		if np := p.NowPlaying(); np != nil {
+			id = np.Metadata().ID
+		}
+		p.audiocache.CacheOnly(id, fetch)
 	}
 }

@@ -3,6 +3,7 @@ package backend
 import (
 	"context"
 	"fmt"
+	"image/color"
 	"log"
 	"math/rand"
 	"runtime"
@@ -10,6 +11,9 @@ import (
 	"sync"
 	"time"

+	"fyne.io/fyne/v2"
+	"fyne.io/fyne/v2/canvas"
+	"fyne.io/fyne/v2/theme"
 	"github.com/dweymouth/supersonic/backend/mediaprovider"
 	"github.com/dweymouth/supersonic/backend/player"
 	"github.com/dweymouth/supersonic/backend/player/dlna"
@@ -23,6 +27,7 @@ import (
 // intermediary between the frontend and various Player backends.
 type PlaybackManager struct {
 	engine   *playbackEngine
+	cache    *AudioCache
 	cmdQueue *playbackCommandQueue
 	cfg      *AppConfig

@@ -61,6 +66,7 @@ func NewPlaybackManager(
 		cfg:         appCfg,
 		autoplay:    playbackCfg.Autoplay,
 		localPlayer: p,
+		cache:       c,
 	}
 	pm.addOnTrackChangeHook()
 	go pm.runCmdQueue(ctx)
@@ -76,12 +82,50 @@ func (p *PlaybackManager) addOnTrackChangeHook() {
 		p.lastPlayTime = curTime
 	})

-	p.OnSongChange(func(mediaprovider.MediaItem, *mediaprovider.Track) {
+	p.OnSongChange(func(item mediaprovider.MediaItem, _ *mediaprovider.Track) {
 		// Autoplay if enabled and we are on the last track
 		if p.autoplay && p.NowPlayingIndex() == len(p.engine.playQueue)-1 {
 			p.enqueueAutoplayTracks()
 		}

+		// TODO: make more permanent
+		if p.cache != nil && item != nil {
+			go func() {
+				log.Println("begin generating waveform image")
+				if path := p.cache.PathForCachedFile(item.Metadata().ID); path != "" {
+					t := time.Now()
+					wd, err := GetWaveformDataForFile(context.Background(), path)
+					log.Printf("generate data took %0.3f milliseconds", float64(time.Since(t).Nanoseconds())/1000000)
+					if err != nil {
+						log.Println(err.Error())
+					} else {
+						log.Println("have waveform data")
+						im := NewWaveformImage()
+						var c color.Color
+						fyne.DoAndWait(func() {
+							c = fyne.CurrentApp().Settings().Theme().Color(
+								theme.ColorNamePrimary,
+								fyne.CurrentApp().Settings().ThemeVariant(),
+							)
+						})
+						log.Println("generating image...")
+						t := time.Now()
+						GenerateWaveformImage(wd, im, c)
+						log.Printf("generate image took %0.3f milliseconds", float64(time.Since(t).Nanoseconds())/1000000)
+						fyne.Do(func() {
+							w := fyne.CurrentApp().NewWindow("waveform")
+							w.SetPadded(false)
+							w.SetContent(canvas.NewImageFromImage(im))
+							w.Resize(fyne.NewSize(1024, 32))
+							w.Show()
+						})
+					}
+				} else {
+					log.Println("no cached file for waveform image")
+				}
+			}()
+		}
+
 		if runtime.GOOS != "windows" {
 			return
 		}
@@ -0,0 +1,235 @@
+package backend
+
+import (
+	"context"
+	"errors"
+	"image"
+	"image/color"
+	"io"
+	"math"
+	"os"
+	"path/filepath"
+
+	"github.com/go-audio/audio"
+	"github.com/go-audio/wav"
+	"github.com/supersonic-app/go-mpv"
+)
+
+type WaveformData struct {
+	Peak [1024]byte
+	RMS  [1024]byte
+}
+
+type WaveformImage = image.NRGBA
+
+func NewWaveformImage() *WaveformImage {
+	return image.NewNRGBA(image.Rect(0, 0, 1024, 32))
+}
+
+func GenerateWaveformImage(data *WaveformData, imgbuf *WaveformImage, c color.Color) {
+	centerY := imgbuf.Rect.Dy() / 2 // 16
+	top := centerY - 1
+	bottom := centerY
+
+	// Convert the input color to RGBA
+	r, g, b, _ := c.RGBA()
+	opaqueColor := color.NRGBA{
+		R: uint8(r >> 8),
+		G: uint8(g >> 8),
+		B: uint8(b >> 8),
+		A: 255,
+	}
+	translucentColor := color.NRGBA{
+		R: uint8(r >> 8),
+		G: uint8(g >> 8),
+		B: uint8(b >> 8),
+		A: 128, // 50% opacity
+	}
+
+	for x := 0; x < 1024; x++ {
+		rms := float64(data.RMS[x]) / 255.0
+		peak := float64(data.Peak[x]) / 255.0
+
+		rmsPixels := int(rms * 16)
+		peakPixels := int((peak - rms) * 16)
+
+		// Always draw at least 2 center pixels
+		setPixel(imgbuf, x, top, opaqueColor)
+		setPixel(imgbuf, x, bottom, opaqueColor)
+
+		// Draw RMS pixels (solid)
+		for i := 1; i <= rmsPixels; i++ {
+			setPixel(imgbuf, x, top-i, opaqueColor)
+			setPixel(imgbuf, x, bottom+i, opaqueColor)
+		}
+
+		// Draw Peak extension (translucent)
+		for i := 1; i <= peakPixels; i++ {
+			setPixel(imgbuf, x, top-rmsPixels-i, translucentColor)
+			setPixel(imgbuf, x, bottom+rmsPixels+i, translucentColor)
+		}
+	}
+}
+
+func GetWaveformDataForFile(ctx context.Context, fpath string) (*WaveformData, error) {
+	dir := filepath.Dir(fpath)
+	transcodeFile := filepath.Join(dir, filepath.Base(fpath)+"_waveform.wav")
+
+	err := convertToWav(ctx, fpath, transcodeFile)
+	if err != nil {
+		return nil, err
+	}
+
+	f, err := os.Open(transcodeFile)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	defer os.Remove(transcodeFile)
+
+	decoder := wav.NewDecoder(f)
+	if !decoder.IsValidFile() {
+		return nil, errors.New("invalid wav file")
+	}
+
+	dur, err := decoder.Duration()
+	if err != nil {
+		return nil, err
+	}
+
+	format := decoder.Format()
+
+	totalSamples := format.SampleRate * int(dur.Milliseconds()) / 1000
+	samplesPerChunk := totalSamples / 1024
+
+	if err := decoder.FwdToPCM(); err != nil {
+		return nil, err
+	}
+
+	buf := &audio.IntBuffer{Data: make([]int, 4096)}
+	data := &WaveformData{}
+	curChunk := 0
+	chunkSamples := make([]float32, 0, samplesPerChunk)
+	for {
+		n, err := decoder.PCMBuffer(buf)
+		if n == 0 || err == io.EOF {
+			break
+		}
+		if err != nil {
+			return data, err
+		}
+
+		// Process samples
+		for i := 0; i < n; i += format.NumChannels {
+			sum := 0
+			for c := 0; c < format.NumChannels; c++ {
+				sum += buf.Data[i+c]
+			}
+			avg := float64(sum) / float64(format.NumChannels)
+			// TODO: this assumes 16 bit
+			sample := float32(avg / float64(1<<15)) // Normalize to [-1, 1]
+			chunkSamples = append(chunkSamples, sample)
+
+			if len(chunkSamples) >= samplesPerChunk {
+				if curChunk < 1024 {
+					peak, rms := computePeakAndRMS(chunkSamples)
+					data.Peak[curChunk] = float32ToByte(peak)
+					data.RMS[curChunk] = float32ToByte(rms)
+				}
+				curChunk++
+				chunkSamples = chunkSamples[:0]
+				if curChunk >= 1024 {
+					break
+				}
+			}
+		}
+	}
+
+	// Optionally fill the last chunk if it's partially filled
+	if curChunk < 1024 && len(chunkSamples) > 0 {
+		peak, rms := computePeakAndRMS(chunkSamples)
+		data.Peak[curChunk] = float32ToByte(peak)
+		data.RMS[curChunk] = float32ToByte(rms)
+	}
+
+	return data, nil
+}
+
+func computePeakAndRMS(chunk []float32) (peak float32, rms float32) {
+	var sumSquares float64
+	peak = 0.0
+	for _, v := range chunk {
+		abs := float32(math.Abs(float64(v)))
+		if abs > peak {
+			peak = abs
+		}
+		sumSquares += float64(v * v)
+	}
+	rms = float32(math.Sqrt(sumSquares / float64(len(chunk))))
+	return
+}
+
+func float32ToByte(val float32) byte {
+	if val > 1.0 {
+		val = 1.0
+	}
+	if val < 0.0 {
+		val = 0.0
+	}
+	return byte(val * 255)
+}
+
+func convertToWav(ctx context.Context, inPath, outPath string) error {
+	m := mpv.Create()
+	m.SetOptionString("video", "no")
+	m.SetOptionString("audio-display", "no")
+	m.SetOptionString("terminal", "no")
+	m.SetOptionString("idle", "yes")
+	m.SetOptionString("ao-pcm-file", outPath)
+	m.SetOptionString("ao", "pcm")
+	m.SetOption("volume", mpv.FORMAT_INT64, 100)
+	// no need to preserve full sample resolution just for waveform image
+	// let's make less data to process and smaller on-disk file
+	m.SetOption("audio-samplerate", mpv.FORMAT_INT64, 22050)
+	m.SetOptionString("audio-channels", "mono")
+	m.SetOptionString("audio-format", "s16")
+	if err := m.Initialize(); err != nil {
+		return err
+	}
+
+	m.Command([]string{"loadfile", inPath, "replace"})
+
+	//log.Println("generating wav file from %s using MPV", inPath)
+	return mpvWaitForIdle(ctx, m)
+}
+
+func setPixel(img *image.NRGBA, x, y int, c color.NRGBA) {
+	if x < 0 || x >= img.Bounds().Dx() || y < 0 || y >= img.Bounds().Dy() {
+		return
+	}
+	offset := img.PixOffset(x, y)
+	img.Pix[offset+0] = c.R
+	img.Pix[offset+1] = c.G
+	img.Pix[offset+2] = c.B
+	img.Pix[offset+3] = c.A
+}
+
+func mpvWaitForIdle(ctx context.Context, m *mpv.Mpv) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			ia := m.GetPropertyString("idle-active")
+			if ia == "yes" || ia == "true" {
+				return nil
+			}
+			// use small timeout to allow detecting ctx expiry
+			// without too much delay
+			e := m.WaitEvent(0.1 /*timeout seconds*/)
+			if e.Event_Id == mpv.EVENT_IDLE {
+				return nil
+			}
+		}
+	}
+}
@@ -11,6 +11,8 @@ require (
 	github.com/dweymouth/fyne-advanced-list v0.0.0-20250211191927-58ea85eec72c
 	github.com/dweymouth/fyne-tooltip v0.3.0
 	github.com/dweymouth/go-jellyfin v0.0.0-20250716005557-0ea2becece1f
+	github.com/go-audio/audio v1.0.0
+	github.com/go-audio/wav v1.1.0
 	github.com/godbus/dbus/v5 v5.1.0
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-retryablehttp v0.7.7
@@ -39,6 +41,7 @@ require (
 	github.com/fyne-io/glfw-js v0.2.0 // indirect
 	github.com/fyne-io/image v0.1.1 // indirect
 	github.com/fyne-io/oksvg v0.1.0 // indirect
+	github.com/go-audio/riff v1.0.0 // indirect
 	github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71 // indirect
 	github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a // indirect
 	github.com/go-text/render v0.2.0 // indirect
@@ -64,3 +67,5 @@ require (
 )

 replace fyne.io/fyne/v2 v2.6.1 => github.com/dweymouth/fyne/v2 v2.3.0-rc1.0.20250712002006-5064d705dac4
+
+replace github.com/go-audio/wav v1.1.0 => github.com/dweymouth/go-wav v0.0.0-20250719173115-e60429a83eb0
@@ -23,12 +23,10 @@ github.com/dweymouth/fyne-tooltip v0.3.0 h1:NKCyTkh9NtvnTsiHtTOtaJzRDOFYP8AckQ2t
 github.com/dweymouth/fyne-tooltip v0.3.0/go.mod h1:m04ShLW/Tp6LXrNieTumApvNgo7YSB+wi+jZTN+kDBU=
 github.com/dweymouth/fyne/v2 v2.3.0-rc1.0.20250712002006-5064d705dac4 h1:Q3r94AcVL8yaF4Nrd3EQFKyLL3UN/zHPkj+My9B2vCQ=
 github.com/dweymouth/fyne/v2 v2.3.0-rc1.0.20250712002006-5064d705dac4/go.mod h1:YZt7SksjvrSNJCwbWFV32WON3mE1Sr7L41D29qMZ/lU=
-github.com/dweymouth/go-jellyfin v0.0.0-20250531151636-29591764f0a0 h1:9t1CR83uzn5Va4Nycwncmuw/NEAN64O4lf82VBPzdfE=
-github.com/dweymouth/go-jellyfin v0.0.0-20250531151636-29591764f0a0/go.mod h1:fcUagHBaQnt06GmBAllNE0J4O/7064zXRWdqnTTtVjI=
-github.com/dweymouth/go-jellyfin v0.0.0-20250715154414-d0a1630ee74f h1:7O7Cn17pwKHG7zPgNhxMabXJP1ZdmNVxCb4i8yJgVZo=
-github.com/dweymouth/go-jellyfin v0.0.0-20250715154414-d0a1630ee74f/go.mod h1:fcUagHBaQnt06GmBAllNE0J4O/7064zXRWdqnTTtVjI=
 github.com/dweymouth/go-jellyfin v0.0.0-20250716005557-0ea2becece1f h1:QsKPwFpTHuYHEEuhvp4VBClkHh00bNNgQ/2Ij1bkk8M=
 github.com/dweymouth/go-jellyfin v0.0.0-20250716005557-0ea2becece1f/go.mod h1:fcUagHBaQnt06GmBAllNE0J4O/7064zXRWdqnTTtVjI=
+github.com/dweymouth/go-wav v0.0.0-20250719173115-e60429a83eb0 h1:mYcctuWgVArHhSLJxndlUM43C3hoE18BLDBkXKM2tl0=
+github.com/dweymouth/go-wav v0.0.0-20250719173115-e60429a83eb0/go.mod h1:bp2870jtp/ixAJLIOdShBfl1WpyLGDZ57jnVWMgkgIc=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
@@ -47,6 +45,14 @@ github.com/fyne-io/image v0.1.1 h1:WH0z4H7qfvNUw5l4p3bC1q70sa5+YWVt6HCj7y4VNyA=
 github.com/fyne-io/image v0.1.1/go.mod h1:xrfYBh6yspc+KjkgdZU/ifUC9sPA5Iv7WYUBzQKK7JM=
 github.com/fyne-io/oksvg v0.1.0 h1:7EUKk3HV3Y2E+qypp3nWqMXD7mum0hCw2KEGhI1fnBw=
 github.com/fyne-io/oksvg v0.1.0/go.mod h1:dJ9oEkPiWhnTFNCmRgEze+YNprJF7YRbpjgpWS4kzoI=
+github.com/go-audio/aiff v0.0.0-20180403003018-6c3a8a6aff12/go.mod h1:AMSAp6W1zd0koOdX6QDgGIuBDTUvLa2SLQtm7d9eM3c=
+github.com/go-audio/aiff v1.0.0/go.mod h1:Kazp+9JR/Y1ITCXaDlO6OIIOrz6eGGAn+dGT04V4HPM=
+github.com/go-audio/audio v0.0.0-20180206231410-b697a35b5608/go.mod h1:6uAu0+H2lHkwdGsAY+j2wHPNPpPoeg5AaEFh9FlA+Zs=
+github.com/go-audio/audio v1.0.0 h1:zS9vebldgbQqktK4H0lUqWrG8P0NxCJVqcj7ZpNnwd4=
+github.com/go-audio/audio v1.0.0/go.mod h1:6uAu0+H2lHkwdGsAY+j2wHPNPpPoeg5AaEFh9FlA+Zs=
+github.com/go-audio/riff v1.0.0 h1:d8iCGbDvox9BfLagY94fBynxSPHO80LmZCaOsmKxokA=
+github.com/go-audio/riff v1.0.0/go.mod h1:l3cQwc85y79NQFCRB7TiPoNiaijp6q8Z0Uv38rVG498=
+github.com/go-audio/wav v0.0.0-20181013172942-de841e69b884/go.mod h1:UiqzUyfX0zs3pJ/DPyvS5v8sN6s5bXPUDDIVA5v8dks=
 github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71 h1:5BVwOaUSBTlVZowGO6VZGw2H/zl9nrd3eCZfYV+NfQA=
 github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71/go.mod h1:9YTyiznxEY1fVinfM7RvRcjRHbw2xLBJ3AAGIT0I4Nw=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a h1:vxnBhFDDT+xzxf1jTJKMKZw3H0swfWk9RpWbBbDK5+0=
@@ -87,6 +93,7 @@ github.com/koron/go-ssdp v0.0.5 h1:E1iSMxIs4WqxTbIBLtmNBeOOC+1sCIXQeqTWVnpmwhk=
 github.com/koron/go-ssdp v0.0.5/go.mod h1:Qm59B7hpKpDqfyRNWRNr00jGwLdXjDyZh6y7rH6VS0w=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mattetti/audio v0.0.0-20180912171649-01576cde1f21/go.mod h1:LlQmBGkOuV/SKzEDXBPKauvN2UqCgzXO2XjecTGj40s=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=