Restored recommendations based on acoustic similarities (using musicnn), fixes #301

This commit is contained in:
emeric
2026-06-02 08:32:43 +02:00
parent 1524106124
commit eb7f65878f
227 changed files with 10324 additions and 4673 deletions
+30
View File
@@ -2,6 +2,7 @@ pkg_check_modules(LIBAV IMPORTED_TARGET libavcodec libavutil libavformat libswre
pkg_check_modules(Taglib REQUIRED IMPORTED_TARGET taglib)
pkg_check_modules(PulseAudio IMPORTED_TARGET libpulse)
pkg_check_modules(ALSA IMPORTED_TARGET alsa)
pkg_check_modules(OnnxRuntime IMPORTED_TARGET libonnxruntime)
if (PulseAudio_FOUND OR ALSA_FOUND)
message(STATUS "Audio output available (PulseAudio=${PulseAudio_FOUND}, ALSA=${ALSA_FOUND})")
@@ -10,6 +11,7 @@ else()
endif()
add_library(lmsaudio STATIC
impl/features/MelFilterBank.cpp
impl/ffmpeg/AudioFile.cpp
impl/ffmpeg/AudioFileInfo.cpp
impl/ffmpeg/AudioFileInfoParser.cpp
@@ -19,6 +21,7 @@ add_library(lmsaudio STATIC
impl/ffmpeg/TagReader.cpp
impl/ffmpeg/Transcoder.cpp
impl/ffmpeg/Utils.cpp
impl/musicnn/MusicNNEmbeddings.cpp
impl/taglib/AudioFileInfo.cpp
impl/taglib/AudioFileInfoParser.cpp
impl/taglib/ImageReader.cpp
@@ -27,6 +30,7 @@ add_library(lmsaudio STATIC
impl/utils/PcmDecodeStreamer.cpp
impl/AudioFileInfoParser.cpp
impl/AudioOutput.cpp
impl/MusicNNEmbeddingExtractorCreator.cpp
impl/PcmTypes.cpp
impl/TagReader.cpp
)
@@ -37,6 +41,7 @@ target_include_directories(lmsaudio INTERFACE
target_include_directories(lmsaudio PRIVATE
include
impl
${AVCODEC_INCLUDE_DIR}
${AVFORMAT_INCLUDE_DIR}
${AVUTIL_INCLUDE_DIR}
@@ -47,6 +52,7 @@ target_link_libraries(lmsaudio PUBLIC
)
target_link_libraries(lmsaudio PRIVATE
lmsmath
PkgConfig::LIBAV
PkgConfig::Taglib
)
@@ -54,6 +60,12 @@ target_link_libraries(lmsaudio PRIVATE
target_compile_definitions(lmsaudio PRIVATE
$<$<BOOL:${PulseAudio_FOUND}>:LMS_HAVE_PULSEAUDIO>
$<$<BOOL:${ALSA_FOUND}>:LMS_HAVE_ALSA>
$<$<BOOL:${OnnxRuntime_FOUND}>:LMS_HAVE_ONNX_RUNTIME>
)
# Should be safe enough for what we're doing
target_compile_options(lmsaudio PRIVATE
$<$<NOT:$<CONFIG:Debug>>:-ffast-math>
)
if (PulseAudio_FOUND)
@@ -80,3 +92,21 @@ if (ALSA_FOUND)
)
endif()
if (OnnxRuntime_FOUND)
target_sources(lmsaudio PRIVATE
impl/musicnn/MusicNNEmbeddingExtractor.cpp
impl/musicnn/MusicNNModel.cpp
)
target_link_libraries(lmsaudio PRIVATE PkgConfig::OnnxRuntime)
message(STATUS "Using ONNX Runtime (${OnnxRuntime_VERSION})")
install(DIRECTORY models DESTINATION share/lms)
endif()
if(BUILD_TESTING)
add_subdirectory(test)
endif()
if (BUILD_BENCHMARKS)
add_subdirectory(bench)
endif()
+22
View File
@@ -0,0 +1,22 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include <benchmark/benchmark.h>
BENCHMARK_MAIN();
+18
View File
@@ -0,0 +1,18 @@
add_executable(bench-audio
Audio.cpp
)
if (OnnxRuntime_FOUND)
target_sources(bench-audio PRIVATE
MusicNNModel.cpp
)
endif()
target_include_directories(bench-audio PRIVATE
../impl
)
target_link_libraries(bench-audio PRIVATE
lmsaudio
benchmark
)
+68
View File
@@ -0,0 +1,68 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include <array>
#include <cstdlib>
#include <filesystem>
#include <random>
#include <benchmark/benchmark.h>
#include "musicnn/MusicNNModel.hpp"
namespace lms::audio::musicnn::benchmarks
{
namespace
{
std::filesystem::path getMusicNNModelPathFromEnv()
{
const char* p{ std::getenv("LMS_MUSICNN_MODEL") };
return p ? std::filesystem::path{ p } : std::filesystem::path{};
}
std::array<float, MusicNNModel::inputFrames * MusicNNModel::inputBands> makeRandomPatch()
{
std::minstd_rand rng{ 42 };
std::uniform_real_distribution<float> dist{ 0.F, 1.F };
std::array<float, MusicNNModel::inputFrames * MusicNNModel::inputBands> patch{};
for (float& v : patch)
v = dist(rng);
return patch;
}
} // namespace
static void BM_MusicNNModel_forward(benchmark::State& state)
{
const std::filesystem::path path{ getMusicNNModelPathFromEnv() };
if (path.empty())
{
state.SkipWithMessage("LMS_MUSICNN_MODEL not set");
return;
}
const MusicNNModel model{ path };
const auto patch{ makeRandomPatch() };
for (auto _ : state)
benchmark::DoNotOptimize(model.forward(patch));
}
BENCHMARK(BM_MusicNNModel_forward);
} // namespace lms::audio::musicnn::benchmarks
@@ -0,0 +1,70 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "audio/IMusicNNEmbeddingExtractor.hpp"
#include <array>
#include <fstream>
#include <span>
#include <string>
#include "core/XxHash3.hpp"
#if LMS_HAVE_ONNX_RUNTIME
#include "musicnn/MusicNNEmbeddingExtractor.hpp"
#endif
namespace lms::audio
{
bool canExtractMusicNNEmbeddings()
{
#if LMS_HAVE_ONNX_RUNTIME
return true;
#else
return false;
#endif
}
std::unique_ptr<IMusicNNEmbeddingExtractor> createMusicNNEmbeddingExtractor([[maybe_unused]] const std::filesystem::path& modelPath, std::size_t maxPatchCount)
{
#if LMS_HAVE_ONNX_RUNTIME
return std::make_unique<musicnn::MusicNNEmbeddingExtractor>(modelPath, maxPatchCount);
#else
return {};
#endif
}
std::string getMusicNNModelIdentifier(const std::filesystem::path& modelPath)
{
std::ifstream file{ modelPath, std::ios::binary };
if (!file)
return {};
core::XxHash3_64 hasher;
constexpr std::size_t readBufSize{ 65536 };
std::array<char, readBufSize> buf{};
while (file.read(buf.data(), buf.size()) || file.gcount() > 0)
hasher.update(std::as_bytes(std::span{ buf.data(), static_cast<std::size_t>(file.gcount()) }));
if (!file.eof())
return {};
return std::to_string(hasher.digest());
}
} // namespace lms::audio
+7
View File
@@ -38,4 +38,11 @@ namespace lms::audio
throw Exception{ "Unhandled sample type" };
}
namespace helpers
{
std::size_t sampleCountToByteCount(std::size_t sampleCount, PcmSampleType sampleType, unsigned channelCount)
{
return sampleCount * audio::getSampleSize(sampleType) * channelCount;
}
} // namespace helpers
} // namespace lms::audio
@@ -0,0 +1,143 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "MelFilterBank.hpp"
#include <algorithm>
#include <cassert>
#include <cmath>
#include <numeric>
#include "audio/Exception.hpp"
namespace lms::audio::features
{
float freqToMel(float freq)
{
return 2595.F * std::log10(1.0F + freq / 700.F);
}
float melToFreq(float mel)
{
return 700.F * (std::pow(10.F, mel / 2595.F) - 1.F);
}
MelFilterBank::MelFilterBank(std::vector<Filter>&& _filters, std::size_t binCount)
: _filters{ std::move(_filters) }
, _binCount{ binCount }
{
}
const MelFilterBank::Filter& MelFilterBank::getFilter(std::size_t m) const
{
return _filters.at(m);
}
std::size_t MelFilterBank::getFilterCount() const
{
return _filters.size();
}
std::size_t MelFilterBank::getBinCount() const
{
return _binCount;
}
float MelFilterBank::computeEnergy(std::size_t m, std::span<const float> input) const
{
if (input.size() != _binCount)
throw Exception{ "Input size must be equal to the number of bins" };
const auto& filter{ _filters.at(m) };
assert(filter.leftBinIndex + filter.weights.size() <= input.size());
float energy{};
std::size_t bin{ filter.leftBinIndex };
for (std::size_t i{}, n = filter.weights.size(); i < n; ++i, ++bin)
energy += input[bin] * filter.weights[i];
return energy;
}
MelFilterBank computeMelFilterBank(std::size_t nfft, std::size_t sampleRate, std::size_t filterCount, float fMin, float fMax)
{
const float nyquist{ sampleRate / 2.F };
const float effectiveFMin{ (fMin <= 0.F) ? 0.F : fMin };
const float effectiveFMax{ (fMax <= 0.F || fMax > nyquist) ? nyquist : fMax };
const float melMin{ freqToMel(effectiveFMin) };
const float melMax{ freqToMel(effectiveFMax) };
// 1. mel points
std::vector<float> melPoints(filterCount + 2);
for (std::size_t i{}; i < melPoints.size(); ++i)
melPoints[i] = melMin + i * (melMax - melMin) / (filterCount + 1);
// 2. mel -> Hz
std::vector<float> freqs(filterCount + 2);
std::transform(melPoints.begin(), melPoints.end(), freqs.begin(), melToFreq);
// 3. Hz -> bins
std::vector<size_t> bins(filterCount + 2);
for (std::size_t i{}; i < bins.size(); ++i)
bins[i] = static_cast<size_t>(std::floor(nfft * freqs[i] / sampleRate));
// 4. fix duplicates
for (std::size_t i{ 1 }; i < bins.size(); ++i)
{
if (bins[i] <= bins[i - 1])
bins[i] = bins[i - 1] + 1;
}
// 5. build filters
const std::size_t binCount{ nfft / 2 + 1 };
std::vector<MelFilterBank::Filter> filters{ filterCount };
for (std::size_t m{}; m < filterCount; ++m)
{
const std::size_t left{ bins[m] };
const std::size_t center{ bins[m + 1] };
const std::size_t right{ bins[m + 2] };
std::vector<float> filterWeights;
filterWeights.reserve(right - left);
// rising edge of the triangle
for (std::size_t k{ left }; k < center; ++k)
filterWeights.push_back(float(k - left) / (center - left));
// falling edge of the triangle
for (std::size_t k{ center }; k < right; ++k)
filterWeights.push_back(float(right - k) / (right - center));
assert(filterWeights.size() == right - left);
// normalize the filter to sum = 1
const float sum{ std::accumulate(filterWeights.begin(), filterWeights.end(), 0.F) };
if (sum > 0.F)
{
for (float& weight : filterWeights)
weight /= sum;
}
filters[m] = MelFilterBank::Filter{ std::move(filterWeights), left };
}
return MelFilterBank{ std::move(filters), binCount };
}
} // namespace lms::audio::features
@@ -0,0 +1,56 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <span>
#include <vector>
namespace lms::audio::features
{
float freqToMel(float freq);
float melToFreq(float mel);
struct MelFilterBank
{
struct Filter
{
std::vector<float> weights; // normalized so that the sum of weights equals 1.0
std::size_t leftBinIndex; // index of the leftmost FFT bin covered by the filter
};
// binCount is the number of FFT bins (nfft/2 + 1) that the filters can cover
MelFilterBank(std::vector<Filter>&& _filters, std::size_t binCount);
const Filter& getFilter(std::size_t m) const;
std::size_t getFilterCount() const;
std::size_t getBinCount() const;
float computeEnergy(std::size_t m, std::span<const float> input) const;
private:
const std::vector<Filter> _filters;
const std::size_t _binCount;
};
// Each filter covers a range of FFT bins and is normalized so that the sum of its weights equals 1.0
// Each filter stores only its non-zero triangular region (sparse representation)
// fMin/fMax: frequency range in Hz. Defaults (0.f, 0.f) span from 0 to Nyquist.
MelFilterBank computeMelFilterBank(size_t nfft, size_t sampleRate, size_t filterCount, float fMin = 0.F, float fMax = 0.F);
} // namespace lms::audio::features
+2 -2
View File
@@ -285,7 +285,7 @@ namespace lms::audio::ffmpeg
bool AudioFile::hasAttachedPictures() const
{
for (std::size_t i = 0; i < _context->nb_streams; ++i)
for (std::size_t i{}; i < _context->nb_streams; ++i)
{
if (_context->streams[i]->disposition & AV_DISPOSITION_ATTACHED_PIC)
return true;
@@ -304,7 +304,7 @@ namespace lms::audio::ffmpeg
{ AV_CODEC_ID_PPM, "image/x-portable-pixmap" },
};
for (std::size_t i = 0; i < _context->nb_streams; ++i)
for (std::size_t i{}; i < _context->nb_streams; ++i)
{
AVStream* avstream = _context->streams[i];
+15 -2
View File
@@ -127,6 +127,14 @@ namespace lms::audio::ffmpeg
}
}
{
_estimatedDuration = std::chrono::milliseconds{ _context->duration == AV_NOPTS_VALUE ? 0 : _context->duration / AV_TIME_BASE * 1'000 };
if (_estimatedDuration > offset)
_estimatedDuration = _estimatedDuration - std::chrono::duration_cast<std::chrono::milliseconds>(offset);
else
_estimatedDuration = {};
}
_decoderContext = AVCodecContextPtr{ ::avcodec_alloc_context3(decoder) };
if (!_decoderContext)
throw Exception{ "Cannot allocate decoder context" };
@@ -225,7 +233,7 @@ namespace lms::audio::ffmpeg
else
{
std::array<uint8_t*, AV_NUM_DATA_POINTERS> outData{};
for (size_t i = 0; i < outputChannelBuffers.size(); ++i)
for (std::size_t i{}; i < outputChannelBuffers.size(); ++i)
outData[i] = reinterpret_cast<uint8_t*>(outputChannelBuffers[i].data());
// Resample decoded audio
@@ -266,6 +274,11 @@ namespace lms::audio::ffmpeg
return _finished;
}
std::chrono::milliseconds PcmDecoder::getEstimatedDuration() const
{
return _estimatedDuration;
}
std::size_t PcmDecoder::computeSampleCountPerChannel(std::span<WritableBuffer> outputChannelBuffers) const
{
if (_parameters.planar)
@@ -343,7 +356,7 @@ namespace lms::audio::ffmpeg
std::size_t PcmDecoder::drainResampler(std::span<WritableBuffer> outputChannelBuffers, std::size_t maxSamplesPerChannel)
{
std::array<uint8_t*, AV_NUM_DATA_POINTERS> outData{};
for (size_t i = 0; i < outputChannelBuffers.size(); ++i)
for (std::size_t i{}; i < outputChannelBuffers.size(); ++i)
outData[i] = reinterpret_cast<uint8_t*>(outputChannelBuffers[i].data());
const int outSampleCount{ ::swr_convert(_resampleContext.get(),
@@ -40,6 +40,8 @@ namespace lms::audio::ffmpeg
std::size_t readSamples(std::span<WritableBuffer> outputChannelBuffers) override;
bool finished() const override;
std::chrono::milliseconds getEstimatedDuration() const override;
std::size_t computeSampleCountPerChannel(std::span<WritableBuffer> outputChannelBuffers) const;
void feedDecoder();
std::size_t drainResampler(std::span<WritableBuffer> outputChannelBuffers, std::size_t maxSamplesPerChannel);
@@ -52,6 +54,7 @@ namespace lms::audio::ffmpeg
bool _draining{};
AVFormatContextPtr _context;
std::chrono::milliseconds _estimatedDuration{};
int _inputStreamIndex{};
AVCodecContextPtr _decoderContext;
AVFramePtr _decodedFrame;
+1 -1
View File
@@ -62,7 +62,7 @@ namespace lms::audio::ffmpeg::utils
void avLogCallback(void*, int level, const char* fmt, va_list vl)
{
if (!core::Service<core::logging::ILogger>::get()->isSeverityActive(core::logging::Severity::DEBUG))
if (!core::Service<core::logging::ILogger>::get() || core::Service<core::logging::ILogger>::get()->isSeverityActive(core::logging::Severity::DEBUG))
return;
if (level > AV_LOG_WARNING)
@@ -0,0 +1,168 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "MusicNNEmbeddingExtractor.hpp"
#include <algorithm>
#include <array>
#include <cassert>
#include <cmath>
#include <numeric>
#include "audio/Exception.hpp"
#include "audio/IMusicNNEmbeddingExtractor.hpp"
#include "math/StatsAccumulator.hpp"
#include "musicnn/MusicNNModel.hpp"
namespace lms::audio::musicnn
{
namespace
{
constexpr float minMeaningfulPatchRms{ 0.003F };
template<typename FloatType>
FloatType computeRms(std::span<const FloatType> samples)
{
const FloatType sumSq{ std::transform_reduce(samples.begin(), samples.end(), FloatType{}, std::plus<>{}, [](FloatType s) { return s * s; }) };
return std::sqrt(sumSq / static_cast<FloatType>(samples.size()));
}
// We want something like this:
// gap patch(0) gap patch(1) gap patch(maxPatchCount) gap
std::size_t computePatchGap(std::size_t totalFrameCount, std::size_t patchFrameCount, std::size_t maxPatchCount)
{
assert(maxPatchCount > 0);
assert(patchFrameCount > 0);
const std::size_t patchCount{ std::min(maxPatchCount, totalFrameCount / patchFrameCount) };
if (patchCount == 0)
return 0;
return (totalFrameCount - patchCount * patchFrameCount) / (patchCount + 1);
}
} // namespace
// Accumulates one 187-frame MusicNN mel patch
class MusicNNEmbeddingExtractor::PatchAccumulator
{
public:
void addMelRow(std::span<const float, melBandCount> melRow, float frameRms)
{
assert(_frameCount < patchFrameCount);
const std::size_t offset{ _frameCount * melBandCount };
std::copy(melRow.begin(), melRow.end(), _melMatrix.begin() + static_cast<std::ptrdiff_t>(offset));
_rmsAccum += frameRms;
++_frameCount;
}
void reset() noexcept
{
_frameCount = {};
_rmsAccum = {};
}
[[nodiscard]] bool complete() const { return _frameCount == patchFrameCount; }
[[nodiscard]] bool meaningful() const
{
return (_frameCount > 0) && ((_rmsAccum / static_cast<float>(_frameCount)) >= minMeaningfulPatchRms);
}
[[nodiscard]] std::span<const float, patchFrameCount * melBandCount> data() const
{
return _melMatrix;
}
private:
std::size_t _frameCount{};
float _rmsAccum{};
std::array<float, patchFrameCount * melBandCount> _melMatrix;
};
MusicNNEmbeddingExtractor::MusicNNEmbeddingExtractor(const std::filesystem::path& modelPath, std::size_t maxPatchCount)
: _melFilterBank{ features::computeMelFilterBank(fftSize, sampleRate, melBandCount, melFMin, melFMax) }
, _model{ modelPath }
, _maxPatchCount{ maxPatchCount }
{
static_assert(MusicNNEmbeddingExtractor::windowSize == MusicNNEmbeddingExtractor::fftSize);
if (_maxPatchCount <= 0)
throw audio::Exception{ "MusicNN embedding extractor: max patch count must be > 0" };
}
IMusicNNEmbeddingExtractor::ExtractionResult MusicNNEmbeddingExtractor::extract(const std::filesystem::path& audioFile) const
{
auto frameDecoder{ std::make_unique<FrameDecoder>(audioFile,
PcmParameters{ .channelCount = 1,
.sampleRate = static_cast<unsigned>(sampleRate),
.sampleType = PcmSampleType::Float32,
.byteOrder = std::endian::native,
.planar = false },
frameHopSamples) };
std::array<float, melBandCount> logMelRow{};
std::array<math::StatsAccumulator<float>, decltype(_model)::outputSize> embeddingAccumulators;
ExtractionResult result;
const std::size_t estimatedFrameCount{ frameDecoder->getEstimatedFrameCount() };
// Fallback: use a gap of two patch lengths if the frame count is unknown
const std::size_t patchGapFrameCount{ estimatedFrameCount ? computePatchGap(frameDecoder->getEstimatedFrameCount(), patchFrameCount, _maxPatchCount) : (2 * patchFrameCount) };
const auto patchAccumulator{ std::make_unique<PatchAccumulator>() };
while (true)
{
patchAccumulator->reset();
const auto onFrame{ [&](const FrameDecoder::SpectralFrameView& frame) {
// MusicNN log compression: log10(10000 * mel + 1)
for (std::size_t m{}; m < melBandCount; ++m)
{
const float energy{ _melFilterBank.computeEnergy(m, std::span<const float>(frame.powerSpectrum)) };
logMelRow[m] = std::log10(10000.F * energy + 1.F);
}
const float rms{ computeRms(frame.rawSamples.subspan(0, frameHopSamples)) };
patchAccumulator->addMelRow(logMelRow, rms);
} };
if (patchGapFrameCount > 0 && frameDecoder->skipFrames(patchGapFrameCount) == 0)
break;
if (frameDecoder->decodeFrames(patchFrameCount, onFrame) < patchFrameCount)
break;
assert(patchAccumulator->complete());
if (!patchAccumulator->meaningful())
continue;
const auto embedding{ _model.forward(patchAccumulator->data()) };
for (std::size_t d{}; d < embedding.size(); ++d)
embeddingAccumulators[d].add(embedding[d]);
++result.patchCount;
}
if (result.patchCount > 0)
{
for (std::size_t d{}; d < decltype(_model)::outputSize; ++d)
result.embeddings.mean.values[d] = embeddingAccumulators[d].getMean();
}
return result;
}
} // namespace lms::audio::musicnn
@@ -0,0 +1,59 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "audio/IMusicNNEmbeddingExtractor.hpp"
#include "MusicNNModel.hpp"
#include "features/MelFilterBank.hpp"
#include "utils/PcmSpectralFrameDecoder.hpp"
namespace lms::audio::musicnn
{
class MusicNNEmbeddingExtractor : public IMusicNNEmbeddingExtractor
{
public:
MusicNNEmbeddingExtractor(const std::filesystem::path& modelPath, std::size_t maxPatchCount);
~MusicNNEmbeddingExtractor() override = default;
MusicNNEmbeddingExtractor(const MusicNNEmbeddingExtractor&) = delete;
MusicNNEmbeddingExtractor& operator=(const MusicNNEmbeddingExtractor&) = delete;
private:
[[nodiscard]] ExtractionResult extract(const std::filesystem::path& audioFile) const override;
// MusicNN signal processing constants (from musicnn/configuration.py and musicnn_torch.py)
static constexpr std::size_t sampleRate{ 16'000 };
static constexpr std::size_t windowSize{ 512 }; // 512-sample Hann window (32 ms)
static constexpr std::size_t fftSize{ 512 };
static constexpr std::size_t frameHopSamples{ 256 }; // 16 ms hop (matches FFT_HOP in musicnn)
static constexpr std::size_t melBandCount{ 96 };
static constexpr float melFMin{ 0.F };
static constexpr float melFMax{ 8'000.F };
static constexpr std::size_t patchFrameCount{ MusicNNModel::inputFrames }; // 187 frames = 3 s
class PatchAccumulator;
using FrameDecoder = PcmSpectralFrameDecoder<512, float>;
const features::MelFilterBank _melFilterBank;
const MusicNNModel _model;
const std::size_t _maxPatchCount;
};
} // namespace lms::audio::musicnn
@@ -0,0 +1,85 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "audio/MusicNNEmbeddings.hpp"
#include <bit>
#include <cstring>
#include <limits>
#include "audio/Exception.hpp"
namespace lms::audio
{
static_assert(sizeof(float) == 4);
static_assert(std::numeric_limits<float>::is_iec559);
namespace
{
constexpr uint32_t byteswap32(uint32_t x)
{
return (x >> 24) | ((x >> 8) & 0x0000FF00u) | ((x << 8) & 0x00FF0000u) | (x << 24);
}
void writeFloats(std::span<const float> data, std::span<std::byte> blob)
{
if (blob.size() < data.size() * sizeof(uint32_t))
throw Exception{ "Buffer too small to write MusicNN embeddings" };
for (std::size_t i{}; i < data.size(); ++i)
{
uint32_t bits{ std::bit_cast<uint32_t>(data[i]) };
if constexpr (std::endian::native == std::endian::little)
bits = byteswap32(bits);
std::memcpy(blob.data() + i * 4, &bits, 4);
}
}
void readFloats(std::span<const std::byte> blob, std::span<float> data)
{
if (blob.size() < data.size() * sizeof(uint32_t))
throw Exception{ "Buffer too small to read MusicNN embeddings" };
for (std::size_t i{}; i < data.size(); ++i)
{
uint32_t bits{};
std::memcpy(&bits, blob.data() + i * 4, 4);
if constexpr (std::endian::native == std::endian::little)
bits = byteswap32(bits);
data[i] = std::bit_cast<float>(bits);
}
}
} // anonymous namespace
void trackMusicNNEmbeddingsToBlob(const TrackMusicNNEmbeddings& embeddings, std::span<std::byte> buffer)
{
if (buffer.size() < sizeof(TrackMusicNNEmbeddings))
throw Exception{ "Buffer too small to write TrackMusicNNEmbeddings" };
writeFloats(embeddings.mean.values, buffer.subspan(0, MusicNNEmbedding::size * sizeof(float)));
}
void trackMusicNNEmbeddingsFromBlob(std::span<const std::byte> buffer, TrackMusicNNEmbeddings& embeddings)
{
if (buffer.size() < sizeof(TrackMusicNNEmbeddings))
throw Exception{ "Buffer too small to read TrackMusicNNEmbeddings" };
readFloats(buffer.subspan(0, MusicNNEmbedding::size * sizeof(float)), embeddings.mean.values);
}
} // namespace lms::audio
@@ -0,0 +1,111 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "MusicNNModel.hpp"
#include <string>
#include <onnxruntime_cxx_api.h>
#include "audio/Exception.hpp"
namespace lms::audio::musicnn
{
namespace
{
// Shape of the ONNX model's single input tensor: [batch=1, T=187, mel=96]
const std::array<std::int64_t, 3> inputShape{ 1,
static_cast<std::int64_t>(MusicNNModel::inputFrames),
static_cast<std::int64_t>(MusicNNModel::inputBands) };
// Shape of the ONNX model's single output tensor: [batch=1, embedding=200]
const std::array<std::int64_t, 2> outputShape{ 1,
static_cast<std::int64_t>(MusicNNModel::outputSize) };
constexpr const char* inputName{ "mel_patch" };
constexpr const char* outputName{ "embedding" };
} // namespace
struct MusicNNModel::Impl
{
Ort::Env env;
Ort::SessionOptions sessionOptions;
Ort::Session session;
Ort::MemoryInfo memoryInfo;
explicit Impl(const std::filesystem::path& onnxPath)
: env{ ORT_LOGGING_LEVEL_ERROR, "MusicNN" }
, session{ [&]() -> Ort::Session {
sessionOptions.SetIntraOpNumThreads(1);
sessionOptions.SetInterOpNumThreads(1);
return Ort::Session{ env, onnxPath.c_str(), sessionOptions };
}() }
, memoryInfo{ Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault) }
{
}
};
MusicNNModel::MusicNNModel(const std::filesystem::path& onnxPath)
{
try
{
_impl = std::make_unique<Impl>(onnxPath);
}
catch (const Ort::Exception& e)
{
throw audio::Exception{ std::string{ "Failed to load ONNX model '" } + onnxPath.string() + "': " + e.what() };
}
}
MusicNNModel::~MusicNNModel() = default;
std::array<float, MusicNNModel::outputSize> MusicNNModel::forward(
std::span<const float, inputFrames * inputBands> melPatch) const
{
Ort::Value inputTensor{ Ort::Value::CreateTensor<float>(
_impl->memoryInfo,
const_cast<float*>(melPatch.data()), // safe cast
melPatch.size(),
inputShape.data(),
inputShape.size()) };
std::array<float, outputSize> result{};
Ort::Value outputTensor{ Ort::Value::CreateTensor<float>(
_impl->memoryInfo,
result.data(),
result.size(),
outputShape.data(),
outputShape.size()) };
try
{
auto inputNames{ std::to_array({ inputName }) };
auto outputNames{ std::to_array({ outputName }) };
_impl->session.Run(Ort::RunOptions{ nullptr },
inputNames.data(), &inputTensor, 1,
outputNames.data(), &outputTensor, 1);
}
catch (const Ort::Exception& e)
{
throw audio::Exception{ std::string{ "ONNX inference failed: " } + e.what() };
}
return result;
}
} // namespace lms::audio::musicnn
@@ -0,0 +1,55 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <array>
#include <cstddef>
#include <filesystem>
#include <memory>
#include <span>
namespace lms::audio::musicnn
{
// The ONNX model must be exported with:
// input "mel_patch" shape [1, 1, 187, 96]
// output "embedding" shape [1, 200]
//
// Export script: tools/musicnn/export_onnx.py
class MusicNNModel
{
public:
explicit MusicNNModel(const std::filesystem::path& onnxPath);
~MusicNNModel();
MusicNNModel(const MusicNNModel&) = delete;
MusicNNModel& operator=(const MusicNNModel&) = delete;
static inline constexpr std::size_t inputFrames{ 187 };
static inline constexpr std::size_t inputBands{ 96 };
static inline constexpr std::size_t outputSize{ 200 };
[[nodiscard]] std::array<float, outputSize> forward(std::span<const float, inputFrames * inputBands> melPatch) const;
private:
// Pimpl: keep ORT headers out of translation units that include this header.
struct Impl;
std::unique_ptr<Impl> _impl;
};
} // namespace lms::audio::musicnn
@@ -20,6 +20,7 @@
#pragma once
#include <cstddef>
#include <span>
#include <vector>
#include <boost/asio/io_context.hpp>
@@ -0,0 +1,222 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <algorithm>
#include <array>
#include <bit>
#include <cassert>
#include <complex>
#include <concepts>
#include <cstddef>
#include <filesystem>
#include <functional>
#include <memory>
#include <vector>
#include "core/AlignedHeapArray.hpp"
#include "audio/IPcmDecoder.hpp"
#include "audio/PcmTypes.hpp"
#include "math/FFT.hpp"
#include "math/Window.hpp"
namespace lms::audio
{
// Stateful PCM frame decoder that applies a Hann window + FFT per frame.
template<std::size_t WindowSize, typename FloatType = float>
class PcmSpectralFrameDecoder
{
static_assert(std::has_single_bit(WindowSize), "WindowSize must be a power of two");
public:
using FFTPlan = math::FixedRealFFTPlan<WindowSize, FloatType>;
static constexpr std::size_t spectrumSize{ FFTPlan::getOutputSize() };
PcmSpectralFrameDecoder(const std::filesystem::path& audioFile, const PcmParameters& params, std::size_t hopSize)
: PcmSpectralFrameDecoder{ createPcmDecoder(audioFile, {}, params), hopSize }
{
}
~PcmSpectralFrameDecoder() = default;
explicit PcmSpectralFrameDecoder(std::unique_ptr<IPcmDecoder> decoder, std::size_t hopSize)
: _pcmParams{ decoder->getParameters() }
, _hopSize{ hopSize }
, _powerScale{ FloatType{ 1 } / (_window.energy() * static_cast<FloatType>(WindowSize)) }
, _decoder{ std::move(decoder) }
, _samplesBuffer(bufferFrameCount * _hopSize + WindowSize)
, _bufferedSampleCount{ WindowSize / 2 } // first analysis frame centered on sample 0, matching librosa center=True semantics.
{
assert(_hopSize > 0);
}
PcmSpectralFrameDecoder(const PcmSpectralFrameDecoder&) = delete;
PcmSpectralFrameDecoder& operator=(const PcmSpectralFrameDecoder&) = delete;
std::size_t hopSize() const noexcept { return _hopSize; }
const PcmParameters& pcmParameters() const noexcept { return _pcmParams; }
std::size_t getEstimatedFrameCount() const
{
const auto duration{ _decoder->getEstimatedDuration() };
if (duration <= std::chrono::milliseconds::zero())
return 0;
const auto totalSamples{ static_cast<std::size_t>((static_cast<std::uint64_t>(duration.count()) * _pcmParams.sampleRate) / 1'000) };
constexpr std::size_t halfWindow{ WindowSize / 2 };
if (totalSamples < halfWindow) // Not enough samples to produce even the first frame.
return 0;
return 1 + ((totalSamples - halfWindow) / _hopSize);
}
// Spectral data for a single frame.
struct SpectralFrameView
{
std::span<const FloatType, WindowSize> rawSamples;
std::span<const FloatType, spectrumSize> powerSpectrum;
};
// Decodes up to frameCount frames, invoking callback for each. May return fewer than
// frameCount at EOF. Returns 0 only if no frame at all could be decoded.
template<typename Callback>
requires std::invocable<Callback, const SpectralFrameView&>
std::size_t decodeFrames(std::size_t frameCount, Callback&& callback)
{
if (frameCount == 0)
return 0;
std::size_t decodedCount{};
while (decodedCount < frameCount)
{
if (!readAtLeastSamples(std::max(WindowSize, _hopSize)))
break;
const std::span<const FloatType, WindowSize> rawSamples{ _samplesBuffer.data(), WindowSize };
const std::span<FloatType, WindowSize> windowedFrame{ _windowedFrame.data(), WindowSize };
_window.apply(rawSamples, windowedFrame);
_fftPlan.apply(_windowedFrame, _fftOutput);
// Reuse _windowedFrame for power spectrum (spectrumSize <= WindowSize).
const std::span<FloatType, spectrumSize> powerBuffer{ _windowedFrame.data(), spectrumSize };
std::transform(_fftOutput.cbegin(), _fftOutput.cend(), powerBuffer.begin(),
[this](const std::complex<FloatType>& bin) {
return (bin.real() * bin.real() + bin.imag() * bin.imag()) * _powerScale;
});
const SpectralFrameView frame{ .rawSamples = rawSamples,
.powerSpectrum = std::span<const FloatType, spectrumSize>{ _windowedFrame.data(), spectrumSize } };
std::invoke(callback, frame);
consumeSamples(_hopSize);
++_currentFrameIndex;
++decodedCount;
}
return decodedCount;
}
// Skips exactly the next frameCount frames without computing FFT or invoking callbacks.
// Returns frameCount on success, 0 on EOF.
std::size_t skipFrames(std::size_t frameCount)
{
if (frameCount == 0)
return 0;
std::size_t skippedFrameCount{};
while (skippedFrameCount < frameCount)
{
if (!readAtLeastSamples(std::max(WindowSize, _hopSize)))
break;
consumeSamples(_hopSize);
++_currentFrameIndex;
++skippedFrameCount;
}
return skippedFrameCount;
}
[[nodiscard]] std::size_t currentFrameIndex() const noexcept { return _currentFrameIndex; }
private:
static constexpr std::size_t bufferFrameCount{ 20 };
bool readAtLeastSamples(std::size_t sampleCount)
{
if (sampleCount <= _bufferedSampleCount)
return true;
if (_samplesBuffer.size() < sampleCount)
_samplesBuffer.resize(sampleCount);
while ((_bufferedSampleCount < sampleCount) && !_endOfStream)
{
std::span<FloatType> dest{ _samplesBuffer.data() + _bufferedSampleCount, _samplesBuffer.size() - _bufferedSampleCount };
assert(!dest.empty());
if (dest.empty())
break;
std::array outputBuffers{ IPcmDecoder::WritableBuffer{ std::as_writable_bytes(dest) } };
const std::size_t samplesRead{ _decoder->readSamples(outputBuffers) };
if (samplesRead == 0)
{
_endOfStream = true;
break;
}
_bufferedSampleCount += samplesRead;
}
return _bufferedSampleCount >= sampleCount;
}
void consumeSamples(std::size_t samplesToDrop)
{
// TODO use a circular buffer and only compacts at the end of the buffer
assert(samplesToDrop <= _bufferedSampleCount);
const auto remaining{ _bufferedSampleCount - samplesToDrop };
if (remaining)
{
std::move(_samplesBuffer.begin() + samplesToDrop,
_samplesBuffer.begin() + _bufferedSampleCount,
_samplesBuffer.begin());
}
_bufferedSampleCount = remaining;
}
const PcmParameters _pcmParams;
const std::size_t _hopSize;
const math::HannWindow<WindowSize, FloatType> _window;
const FloatType _powerScale;
const FFTPlan _fftPlan{};
std::unique_ptr<IPcmDecoder> _decoder;
std::vector<FloatType> _samplesBuffer;
core::AlignedHeapArray<FloatType, FFTPlan::minBufferAlignment> _windowedFrame{ FFTPlan::getInputSize() };
core::AlignedHeapArray<std::complex<FloatType>, FFTPlan::minBufferAlignment> _fftOutput{ FFTPlan::getOutputSize() };
std::size_t _bufferedSampleCount{};
std::size_t _currentFrameIndex{};
bool _endOfStream{};
};
} // namespace lms::audio
@@ -0,0 +1,46 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <filesystem>
#include <memory>
#include "audio/MusicNNEmbeddings.hpp"
namespace lms::audio
{
class IMusicNNEmbeddingExtractor
{
public:
virtual ~IMusicNNEmbeddingExtractor() = default;
struct ExtractionResult
{
TrackMusicNNEmbeddings embeddings{};
std::size_t patchCount{};
};
[[nodiscard]] virtual ExtractionResult extract(const std::filesystem::path& audioFile) const = 0;
};
bool canExtractMusicNNEmbeddings();
std::unique_ptr<IMusicNNEmbeddingExtractor> createMusicNNEmbeddingExtractor(const std::filesystem::path& modelPath, std::size_t maxPatchCount);
std::string getMusicNNModelIdentifier(const std::filesystem::path& modelPath);
} // namespace lms::audio
@@ -43,8 +43,11 @@ namespace lms::audio
// Each buffer must be sized to hold an integer number of samples according to the requested sample type.
// For example, for Float32 planar output, each buffer size must be divisible by sizeof(float).
// The decoder will use the buffer sizes to determine the maximum number of samples it can write.
// The decoder will not try to fill in the whole supplied buffer
virtual std::size_t readSamples(std::span<WritableBuffer> outputChannelBuffers) = 0;
virtual bool finished() const = 0;
virtual std::chrono::milliseconds getEstimatedDuration() const = 0; // initial offset is taken into account, 0 if unknown
};
// Throw on error
@@ -0,0 +1,41 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <array>
#include <span>
namespace lms::audio
{
struct MusicNNEmbedding
{
static inline constexpr std::size_t size{ 200 };
std::array<float, size> values;
};
struct TrackMusicNNEmbeddings
{
MusicNNEmbedding mean;
};
// Buffer size must be at least sizeof(TrackMusicNNEmbeddings)
void trackMusicNNEmbeddingsToBlob(const TrackMusicNNEmbeddings& embeddings, std::span<std::byte> buffer);
void trackMusicNNEmbeddingsFromBlob(std::span<const std::byte> buffer, TrackMusicNNEmbeddings& embeddings);
} // namespace lms::audio
+18
View File
@@ -20,6 +20,7 @@
#pragma once
#include <bit>
#include <chrono>
namespace lms::audio
{
@@ -41,4 +42,21 @@ namespace lms::audio
std::endian byteOrder;
bool planar;
};
namespace helpers
{
template<typename Rep, typename Period>
std::size_t durationToSampleCount(std::chrono::duration<Rep, Period> duration, unsigned sampleRate)
{
return static_cast<std::size_t>(duration.count() * sampleRate * Period::num / Period::den);
}
template<typename Duration>
Duration sampleCountToDuration(std::size_t sampleCount, unsigned sampleRate)
{
return std::chrono::duration_cast<Duration>(std::chrono::duration<double>(sampleCount) / sampleRate);
}
std::size_t sampleCountToByteCount(std::size_t sampleCount, PcmSampleType sampleType, unsigned channelCount);
} // namespace helpers
} // namespace lms::audio
Binary file not shown.
+33
View File
@@ -0,0 +1,33 @@
include(GoogleTest)
add_executable(test-audio
MelFilterBank.cpp
MusicNNEmbeddings.cpp
PcmSpectralFrameDecoder.cpp
)
if (OnnxRuntime_FOUND)
target_sources(test-audio PRIVATE
MusicNNModel.cpp
)
endif()
target_include_directories(test-audio PRIVATE
../impl
)
target_link_libraries(test-audio PRIVATE
lmsaudio
lmsmath
GTest::GTest
GTest::gtest_main
)
target_compile_options(test-audio PRIVATE
$<$<NOT:$<CONFIG:Debug>>:-ffast-math>
)
if (NOT CMAKE_CROSSCOMPILING)
gtest_discover_tests(test-audio)
endif()
+203
View File
@@ -0,0 +1,203 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include <algorithm>
#include <limits>
#include <numeric>
#include <gtest/gtest.h>
#include "audio/Exception.hpp"
#include "features/MelFilterBank.hpp"
namespace lms::audio::features::tests
{
constexpr float epsilon{ 1e-5F };
constexpr std::size_t NFFT{ 2048 };
constexpr std::size_t sampleRate{ 22050 };
constexpr std::size_t filterCount{ 40 };
TEST(MelFilterBank, sizeCheck)
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
EXPECT_EQ(bank.getFilterCount(), filterCount);
EXPECT_EQ(bank.getBinCount(), NFFT / 2 + 1);
}
TEST(MelFilterBank, differentSampleRates)
{
for (const std::size_t sr : { std::size_t{ 8000 }, std::size_t{ 16000 }, std::size_t{ 44100 }, std::size_t{ 48000 } })
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sr, filterCount) };
EXPECT_EQ(bank.getFilterCount(), filterCount) << "sr=" << sr;
}
}
TEST(MelFilterBank, nonNegativeWeights)
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
{
const auto& filter = bank.getFilter(m);
for (float w : filter.weights)
EXPECT_GE(w, 0.F) << "Filter " << m << " has negative weight: " << w;
}
}
TEST(MelFilterBank, peaksAreNonZero)
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
{
const auto& filter{ bank.getFilter(m) };
const float maxVal{ *std::max_element(filter.weights.begin(), filter.weights.end()) };
EXPECT_GT(maxVal, 0.F) << "Filter " << m << " has zero peak value";
}
}
TEST(MelFilterBank, filtersAreUnitSum)
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
{
const auto& filter{ bank.getFilter(m) };
const float sum{ std::accumulate(filter.weights.begin(), filter.weights.end(), 0.F) };
EXPECT_NEAR(sum, 1.F, epsilon) << "Filter " << m << " sum = " << sum;
}
}
TEST(MelFilterBank, everyBinCovered)
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
const std::size_t binCount{ bank.getBinCount() };
std::vector<bool> covered(binCount, false);
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
{
const auto& filter{ bank.getFilter(m) };
std::size_t bin{ filter.leftBinIndex };
for (float w : filter.weights)
{
if (w > 0.F)
covered[bin] = true;
++bin;
}
}
// Find actual covered range
auto first{ std::find(covered.begin(), covered.end(), true) };
auto last{ std::find(covered.rbegin(), covered.rend(), true).base() };
ASSERT_NE(first, covered.end()); // sanity
for (auto it = first; it != last; ++it)
{
EXPECT_TRUE(*it) << "A bin in the covered range is not covered by any filter";
}
}
TEST(MelFilterBank, overlapAtMostTwo)
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
const std::size_t binCount{ bank.getBinCount() };
std::vector<std::size_t> overlap(binCount, 0);
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
{
const auto& filter{ bank.getFilter(m) };
std::size_t bin{ filter.leftBinIndex };
for (float w : filter.weights)
{
if (w > 0.F)
overlap[bin]++;
++bin;
}
}
for (std::size_t k{}; k < binCount; ++k)
{
EXPECT_LE(overlap[k], 2) << "Bin " << k << " is covered by " << overlap[k] << " filters";
}
}
TEST(MelFilterBank, flatSpectrumEnergySanity)
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
std::vector<float> flatSpectrum(bank.getBinCount(), 1.F);
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
{
const float energy{ bank.computeEnergy(m, flatSpectrum) };
EXPECT_GT(energy, 0.F) << "Filter " << m << " has zero energy for flat spectrum";
}
}
TEST(MelFilterBank, zeroFilterCount)
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, 0) };
EXPECT_EQ(bank.getFilterCount(), 0U);
EXPECT_EQ(bank.getBinCount(), NFFT / 2 + 1);
}
TEST(MelFilterBank, computeEnergyRejectsInvalidInputSize)
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
std::vector<float> invalidInput(bank.getBinCount() - 1, 1.F);
EXPECT_THROW(bank.computeEnergy(0, invalidInput), Exception);
}
TEST(MelFilterBank, zeroSpectrum)
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
std::vector<float> zeroSpectrum(bank.getBinCount(), 0.F);
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
{
const float energy{ bank.computeEnergy(m, zeroSpectrum) };
EXPECT_FLOAT_EQ(energy, 0.F) << "Filter " << m << " should have zero energy for zero spectrum";
}
}
TEST(MelFilterBank, largeSpectrumValues)
{
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
// Weights sum to 1.0 per filter, so energy = input * 1.0, no overflow risk at max/2
const float largeValue{ std::numeric_limits<float>::max() / 2.F };
std::vector<float> largeSpectrum(bank.getBinCount(), largeValue);
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
{
const float energy{ bank.computeEnergy(m, largeSpectrum) };
EXPECT_GT(energy, 0.F) << "Filter " << m << " energy is not positive for large spectrum";
}
}
} // namespace lms::audio::features::tests
+59
View File
@@ -0,0 +1,59 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include <numeric>
#include <vector>
#include <gtest/gtest.h>
#include "audio/Exception.hpp"
#include "audio/MusicNNEmbeddings.hpp"
namespace lms::audio::tests
{
TEST(MusicNNEmbeddings, blobRoundTrip)
{
TrackMusicNNEmbeddings original{};
std::iota(original.mean.values.begin(), original.mean.values.end(), 0.F);
std::vector<std::byte> blob(sizeof(TrackMusicNNEmbeddings));
trackMusicNNEmbeddingsToBlob(original, blob);
TrackMusicNNEmbeddings restored{};
trackMusicNNEmbeddingsFromBlob(blob, restored);
for (std::size_t i{}; i < MusicNNEmbedding::size; ++i)
EXPECT_FLOAT_EQ(restored.mean.values[i], original.mean.values[i]);
}
TEST(MusicNNEmbeddings, blobSizeTooSmallThrows)
{
TrackMusicNNEmbeddings embeddings{};
std::vector<std::byte> blob(sizeof(TrackMusicNNEmbeddings) - 1);
EXPECT_THROW(trackMusicNNEmbeddingsToBlob(embeddings, blob), Exception);
}
TEST(MusicNNEmbeddings, blobFromSizeTooSmallThrows)
{
std::vector<std::byte> blob(sizeof(TrackMusicNNEmbeddings) - 1, std::byte{});
TrackMusicNNEmbeddings embeddings{};
EXPECT_THROW(trackMusicNNEmbeddingsFromBlob(blob, embeddings), Exception);
}
} // namespace lms::audio::tests
+76
View File
@@ -0,0 +1,76 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include <array>
#include <cstdlib>
#include <filesystem>
#include <random>
#include <gtest/gtest.h>
#include "musicnn/MusicNNModel.hpp"
namespace lms::audio::musicnn::tests
{
namespace
{
std::filesystem::path getMusicNNModelPathFromEnv()
{
const char* p{ std::getenv("LMS_MUSICNN_MODEL") };
return p ? std::filesystem::path{ p } : std::filesystem::path{};
}
std::array<float, MusicNNModel::inputFrames * MusicNNModel::inputBands> makeRandomPatch()
{
std::minstd_rand rng{ 42 };
std::uniform_real_distribution<float> dist{ 0.F, 1.F };
std::array<float, MusicNNModel::inputFrames * MusicNNModel::inputBands> patch{};
for (float& v : patch)
v = dist(rng);
return patch;
}
} // namespace
TEST(MusicNNModel, CanConstruct)
{
const std::filesystem::path path{ getMusicNNModelPathFromEnv() };
if (path.empty())
GTEST_SKIP() << "LMS_MUSICNN_MODEL not set";
EXPECT_NO_THROW({ const MusicNNModel model{ path }; });
}
TEST(MusicNNModel, CanForward)
{
const std::filesystem::path path{ getMusicNNModelPathFromEnv() };
if (path.empty())
GTEST_SKIP() << "LMS_MUSICNN_MODEL not set";
const MusicNNModel model{ path };
const auto patch{ makeRandomPatch() };
EXPECT_NO_THROW({ [[maybe_unused]] const auto output{ model.forward(patch) }; });
}
} // namespace lms::audio::musicnn::tests
@@ -0,0 +1,201 @@
/*
* Copyright (C) 2026 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include <cstring>
#include <memory>
#include <random>
#include <span>
#include <vector>
#include <gtest/gtest.h>
#include "audio/IPcmDecoder.hpp"
#include "audio/PcmTypes.hpp"
#include "utils/PcmSpectralFrameDecoder.hpp"
namespace lms::audio::tests
{
namespace
{
// A mock IPcmDecoder that emits samples 0, 1, 2, 3, ... (as float) up to totalSamples,
class SequencePcmDecoder : public IPcmDecoder
{
public:
SequencePcmDecoder(std::size_t totalSampleCount)
: _totalSampleCount{ totalSampleCount }
{
}
const PcmParameters& getParameters() const override { return _params; }
std::size_t readSamples(std::span<WritableBuffer> outputChannelBuffers) override
{
assert(outputChannelBuffers.size() == 1); // only planar
if (_finished)
return 0;
auto& buf{ outputChannelBuffers[0] };
if (buf.size() == 0)
return 0;
assert(buf.size() % sizeof(float) == 0);
// not always writing up to what is requested
std::uniform_int_distribution dist{ std::size_t{ 1 }, buf.size() / sizeof(float) };
std::size_t sampleCountToWrite{ dist(_randomEngine) };
if (_currentSampleIndex + sampleCountToWrite > _totalSampleCount)
{
sampleCountToWrite = _totalSampleCount - _currentSampleIndex;
_finished = true;
}
float* dest{ reinterpret_cast<float*>(buf.data()) };
for (std::size_t i{}; i < sampleCountToWrite; ++i)
*(dest++) = static_cast<float>(_currentSampleIndex++);
return sampleCountToWrite;
}
bool finished() const override { return _finished; }
std::chrono::milliseconds getEstimatedDuration() const override
{
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::duration<float>{ static_cast<float>(_totalSampleCount) / static_cast<float>(_params.sampleRate) });
}
private:
const PcmParameters _params{
.channelCount = 1,
.sampleRate = 16000,
.sampleType = PcmSampleType::Float32,
.byteOrder = std::endian::native,
.planar = false,
};
std::minstd_rand _randomEngine{ 42 }; // fixed seed for reproducibility
std::size_t _totalSampleCount{};
std::size_t _currentSampleIndex{};
bool _finished{};
};
template<typename T, std::size_t N>
void expectSpanEq(std::span<const T, N> actual, const std::array<T, N>& expected)
{
for (std::size_t i{}; i < N; ++i)
EXPECT_FLOAT_EQ(actual[i], expected[i]) << "index=" << i;
}
constexpr std::size_t WindowSize{ 8 };
constexpr std::size_t HopSize{ 4 };
using FrameDecoder = PcmSpectralFrameDecoder<WindowSize, float>;
} // namespace
TEST(SequencePcmDecoder, basic)
{
constexpr std::size_t decoderTotalSampleCount{ 32 };
SequencePcmDecoder decoder{ decoderTotalSampleCount };
std::size_t totalSampleReadCount{};
while (true)
{
std::array<float, 16> buffer{};
std::array outputBuffers{ IPcmDecoder::WritableBuffer{ std::as_writable_bytes(std::span{ buffer }) } };
const std::size_t sampleReadCount{ decoder.readSamples(outputBuffers) };
if (sampleReadCount == 0)
break;
for (std::size_t i{}; i < sampleReadCount; ++i)
EXPECT_FLOAT_EQ(buffer[i], totalSampleReadCount + i);
totalSampleReadCount += sampleReadCount;
}
EXPECT_EQ(totalSampleReadCount, decoderTotalSampleCount);
}
TEST(PcmSpectralFrameDecoder, firstFrameIsCenteredOnSample0)
{
FrameDecoder frameDecoder{ std::make_unique<SequencePcmDecoder>(32), HopSize };
using Frame = std::array<float, WindowSize>;
std::vector<Frame> frames;
EXPECT_EQ(frameDecoder.currentFrameIndex(), 0);
const std::size_t decoded{ frameDecoder.decodeFrames(1,
[&](const FrameDecoder::SpectralFrameView& frame) {
auto& newFrame{ frames.emplace_back() };
std::copy(std::cbegin(frame.rawSamples), std::cend(frame.rawSamples), std::begin(newFrame));
}) };
ASSERT_EQ(decoded, 1);
ASSERT_EQ(frames.size(), 1);
EXPECT_EQ(frameDecoder.currentFrameIndex(), 1);
expectSpanEq<float, WindowSize>(frames[0], { 0.F, 0.F, 0.F, 0.F, 0.F, 1.F, 2.F, 3.F });
}
TEST(PcmSpectralFrameDecoder, framesAdvanceByHopSize)
{
FrameDecoder frameDecoder{ std::make_unique<SequencePcmDecoder>(16), HopSize };
using Frame = std::array<float, WindowSize>;
std::vector<Frame> frames;
const std::size_t decoded{ frameDecoder.decodeFrames(2,
[&](const FrameDecoder::SpectralFrameView& frame) {
auto& newFrame{ frames.emplace_back() };
std::copy(std::cbegin(frame.rawSamples), std::cend(frame.rawSamples), std::begin(newFrame));
}) };
ASSERT_EQ(decoded, 2);
ASSERT_EQ(frames.size(), 2);
EXPECT_EQ(frameDecoder.currentFrameIndex(), 2);
expectSpanEq<float, WindowSize>(
frames[0],
{ 0.F, 0.F, 0.F, 0.F, 0.F, 1.F, 2.F, 3.F });
expectSpanEq<float, WindowSize>(
frames[1],
{ 0.F, 1.F, 2.F, 3.F, 4.F, 5.F, 6.F, 7.F });
}
TEST(PcmSpectralFrameDecoder, skipFramesAdvancesState)
{
FrameDecoder frameDecoder{ std::make_unique<SequencePcmDecoder>(32), HopSize };
using Frame = std::array<float, WindowSize>;
std::vector<Frame> frames;
EXPECT_EQ(frameDecoder.currentFrameIndex(), 0);
ASSERT_EQ(frameDecoder.skipFrames(2), 2);
EXPECT_EQ(frameDecoder.currentFrameIndex(), 2);
Frame lastFrame{};
ASSERT_EQ(frameDecoder.decodeFrames(1,
[&](const FrameDecoder::SpectralFrameView& frame) {
std::copy(std::cbegin(frame.rawSamples), std::cend(frame.rawSamples), std::begin(lastFrame));
}),
1);
expectSpanEq<float, WindowSize>(
lastFrame,
{ 4.F, 5.F, 6.F, 7.F, 8.F, 9.F, 10.F, 11.F });
EXPECT_EQ(frameDecoder.currentFrameIndex(), 3);
}
} // namespace lms::audio::tests