Restored recommendations based on acoustic similarities (using musicnn), fixes #301
This commit is contained in:
@@ -2,6 +2,7 @@ pkg_check_modules(LIBAV IMPORTED_TARGET libavcodec libavutil libavformat libswre
|
||||
pkg_check_modules(Taglib REQUIRED IMPORTED_TARGET taglib)
|
||||
pkg_check_modules(PulseAudio IMPORTED_TARGET libpulse)
|
||||
pkg_check_modules(ALSA IMPORTED_TARGET alsa)
|
||||
pkg_check_modules(OnnxRuntime IMPORTED_TARGET libonnxruntime)
|
||||
|
||||
if (PulseAudio_FOUND OR ALSA_FOUND)
|
||||
message(STATUS "Audio output available (PulseAudio=${PulseAudio_FOUND}, ALSA=${ALSA_FOUND})")
|
||||
@@ -10,6 +11,7 @@ else()
|
||||
endif()
|
||||
|
||||
add_library(lmsaudio STATIC
|
||||
impl/features/MelFilterBank.cpp
|
||||
impl/ffmpeg/AudioFile.cpp
|
||||
impl/ffmpeg/AudioFileInfo.cpp
|
||||
impl/ffmpeg/AudioFileInfoParser.cpp
|
||||
@@ -19,6 +21,7 @@ add_library(lmsaudio STATIC
|
||||
impl/ffmpeg/TagReader.cpp
|
||||
impl/ffmpeg/Transcoder.cpp
|
||||
impl/ffmpeg/Utils.cpp
|
||||
impl/musicnn/MusicNNEmbeddings.cpp
|
||||
impl/taglib/AudioFileInfo.cpp
|
||||
impl/taglib/AudioFileInfoParser.cpp
|
||||
impl/taglib/ImageReader.cpp
|
||||
@@ -27,6 +30,7 @@ add_library(lmsaudio STATIC
|
||||
impl/utils/PcmDecodeStreamer.cpp
|
||||
impl/AudioFileInfoParser.cpp
|
||||
impl/AudioOutput.cpp
|
||||
impl/MusicNNEmbeddingExtractorCreator.cpp
|
||||
impl/PcmTypes.cpp
|
||||
impl/TagReader.cpp
|
||||
)
|
||||
@@ -37,6 +41,7 @@ target_include_directories(lmsaudio INTERFACE
|
||||
|
||||
target_include_directories(lmsaudio PRIVATE
|
||||
include
|
||||
impl
|
||||
${AVCODEC_INCLUDE_DIR}
|
||||
${AVFORMAT_INCLUDE_DIR}
|
||||
${AVUTIL_INCLUDE_DIR}
|
||||
@@ -47,6 +52,7 @@ target_link_libraries(lmsaudio PUBLIC
|
||||
)
|
||||
|
||||
target_link_libraries(lmsaudio PRIVATE
|
||||
lmsmath
|
||||
PkgConfig::LIBAV
|
||||
PkgConfig::Taglib
|
||||
)
|
||||
@@ -54,6 +60,12 @@ target_link_libraries(lmsaudio PRIVATE
|
||||
target_compile_definitions(lmsaudio PRIVATE
|
||||
$<$<BOOL:${PulseAudio_FOUND}>:LMS_HAVE_PULSEAUDIO>
|
||||
$<$<BOOL:${ALSA_FOUND}>:LMS_HAVE_ALSA>
|
||||
$<$<BOOL:${OnnxRuntime_FOUND}>:LMS_HAVE_ONNX_RUNTIME>
|
||||
)
|
||||
|
||||
# Should be safe enough for what we're doing
|
||||
target_compile_options(lmsaudio PRIVATE
|
||||
$<$<NOT:$<CONFIG:Debug>>:-ffast-math>
|
||||
)
|
||||
|
||||
if (PulseAudio_FOUND)
|
||||
@@ -80,3 +92,21 @@ if (ALSA_FOUND)
|
||||
)
|
||||
endif()
|
||||
|
||||
if (OnnxRuntime_FOUND)
|
||||
target_sources(lmsaudio PRIVATE
|
||||
impl/musicnn/MusicNNEmbeddingExtractor.cpp
|
||||
impl/musicnn/MusicNNModel.cpp
|
||||
)
|
||||
target_link_libraries(lmsaudio PRIVATE PkgConfig::OnnxRuntime)
|
||||
message(STATUS "Using ONNX Runtime (${OnnxRuntime_VERSION})")
|
||||
|
||||
install(DIRECTORY models DESTINATION share/lms)
|
||||
endif()
|
||||
|
||||
if(BUILD_TESTING)
|
||||
add_subdirectory(test)
|
||||
endif()
|
||||
|
||||
if (BUILD_BENCHMARKS)
|
||||
add_subdirectory(bench)
|
||||
endif()
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <benchmark/benchmark.h>
|
||||
|
||||
BENCHMARK_MAIN();
|
||||
@@ -0,0 +1,18 @@
|
||||
add_executable(bench-audio
|
||||
Audio.cpp
|
||||
)
|
||||
|
||||
if (OnnxRuntime_FOUND)
|
||||
target_sources(bench-audio PRIVATE
|
||||
MusicNNModel.cpp
|
||||
)
|
||||
endif()
|
||||
|
||||
target_include_directories(bench-audio PRIVATE
|
||||
../impl
|
||||
)
|
||||
|
||||
target_link_libraries(bench-audio PRIVATE
|
||||
lmsaudio
|
||||
benchmark
|
||||
)
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <array>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <random>
|
||||
|
||||
#include <benchmark/benchmark.h>
|
||||
|
||||
#include "musicnn/MusicNNModel.hpp"
|
||||
|
||||
namespace lms::audio::musicnn::benchmarks
|
||||
{
|
||||
namespace
|
||||
{
|
||||
std::filesystem::path getMusicNNModelPathFromEnv()
|
||||
{
|
||||
const char* p{ std::getenv("LMS_MUSICNN_MODEL") };
|
||||
return p ? std::filesystem::path{ p } : std::filesystem::path{};
|
||||
}
|
||||
|
||||
std::array<float, MusicNNModel::inputFrames * MusicNNModel::inputBands> makeRandomPatch()
|
||||
{
|
||||
std::minstd_rand rng{ 42 };
|
||||
std::uniform_real_distribution<float> dist{ 0.F, 1.F };
|
||||
std::array<float, MusicNNModel::inputFrames * MusicNNModel::inputBands> patch{};
|
||||
for (float& v : patch)
|
||||
v = dist(rng);
|
||||
return patch;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
static void BM_MusicNNModel_forward(benchmark::State& state)
|
||||
{
|
||||
const std::filesystem::path path{ getMusicNNModelPathFromEnv() };
|
||||
if (path.empty())
|
||||
{
|
||||
state.SkipWithMessage("LMS_MUSICNN_MODEL not set");
|
||||
return;
|
||||
}
|
||||
|
||||
const MusicNNModel model{ path };
|
||||
const auto patch{ makeRandomPatch() };
|
||||
|
||||
for (auto _ : state)
|
||||
benchmark::DoNotOptimize(model.forward(patch));
|
||||
}
|
||||
|
||||
BENCHMARK(BM_MusicNNModel_forward);
|
||||
|
||||
} // namespace lms::audio::musicnn::benchmarks
|
||||
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "audio/IMusicNNEmbeddingExtractor.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <fstream>
|
||||
#include <span>
|
||||
#include <string>
|
||||
|
||||
#include "core/XxHash3.hpp"
|
||||
|
||||
#if LMS_HAVE_ONNX_RUNTIME
|
||||
#include "musicnn/MusicNNEmbeddingExtractor.hpp"
|
||||
#endif
|
||||
|
||||
namespace lms::audio
|
||||
{
|
||||
bool canExtractMusicNNEmbeddings()
|
||||
{
|
||||
#if LMS_HAVE_ONNX_RUNTIME
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
std::unique_ptr<IMusicNNEmbeddingExtractor> createMusicNNEmbeddingExtractor([[maybe_unused]] const std::filesystem::path& modelPath, std::size_t maxPatchCount)
|
||||
{
|
||||
#if LMS_HAVE_ONNX_RUNTIME
|
||||
return std::make_unique<musicnn::MusicNNEmbeddingExtractor>(modelPath, maxPatchCount);
|
||||
#else
|
||||
return {};
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string getMusicNNModelIdentifier(const std::filesystem::path& modelPath)
|
||||
{
|
||||
std::ifstream file{ modelPath, std::ios::binary };
|
||||
if (!file)
|
||||
return {};
|
||||
|
||||
core::XxHash3_64 hasher;
|
||||
constexpr std::size_t readBufSize{ 65536 };
|
||||
std::array<char, readBufSize> buf{};
|
||||
while (file.read(buf.data(), buf.size()) || file.gcount() > 0)
|
||||
hasher.update(std::as_bytes(std::span{ buf.data(), static_cast<std::size_t>(file.gcount()) }));
|
||||
|
||||
if (!file.eof())
|
||||
return {};
|
||||
|
||||
return std::to_string(hasher.digest());
|
||||
}
|
||||
} // namespace lms::audio
|
||||
@@ -38,4 +38,11 @@ namespace lms::audio
|
||||
throw Exception{ "Unhandled sample type" };
|
||||
}
|
||||
|
||||
namespace helpers
|
||||
{
|
||||
std::size_t sampleCountToByteCount(std::size_t sampleCount, PcmSampleType sampleType, unsigned channelCount)
|
||||
{
|
||||
return sampleCount * audio::getSampleSize(sampleType) * channelCount;
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace lms::audio
|
||||
@@ -0,0 +1,143 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "MelFilterBank.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <numeric>
|
||||
|
||||
#include "audio/Exception.hpp"
|
||||
|
||||
namespace lms::audio::features
|
||||
{
|
||||
float freqToMel(float freq)
|
||||
{
|
||||
return 2595.F * std::log10(1.0F + freq / 700.F);
|
||||
}
|
||||
|
||||
float melToFreq(float mel)
|
||||
{
|
||||
return 700.F * (std::pow(10.F, mel / 2595.F) - 1.F);
|
||||
}
|
||||
|
||||
MelFilterBank::MelFilterBank(std::vector<Filter>&& _filters, std::size_t binCount)
|
||||
: _filters{ std::move(_filters) }
|
||||
, _binCount{ binCount }
|
||||
{
|
||||
}
|
||||
|
||||
const MelFilterBank::Filter& MelFilterBank::getFilter(std::size_t m) const
|
||||
{
|
||||
return _filters.at(m);
|
||||
}
|
||||
|
||||
std::size_t MelFilterBank::getFilterCount() const
|
||||
{
|
||||
return _filters.size();
|
||||
}
|
||||
|
||||
std::size_t MelFilterBank::getBinCount() const
|
||||
{
|
||||
return _binCount;
|
||||
}
|
||||
|
||||
float MelFilterBank::computeEnergy(std::size_t m, std::span<const float> input) const
|
||||
{
|
||||
if (input.size() != _binCount)
|
||||
throw Exception{ "Input size must be equal to the number of bins" };
|
||||
|
||||
const auto& filter{ _filters.at(m) };
|
||||
assert(filter.leftBinIndex + filter.weights.size() <= input.size());
|
||||
|
||||
float energy{};
|
||||
std::size_t bin{ filter.leftBinIndex };
|
||||
for (std::size_t i{}, n = filter.weights.size(); i < n; ++i, ++bin)
|
||||
energy += input[bin] * filter.weights[i];
|
||||
|
||||
return energy;
|
||||
}
|
||||
|
||||
MelFilterBank computeMelFilterBank(std::size_t nfft, std::size_t sampleRate, std::size_t filterCount, float fMin, float fMax)
|
||||
{
|
||||
const float nyquist{ sampleRate / 2.F };
|
||||
const float effectiveFMin{ (fMin <= 0.F) ? 0.F : fMin };
|
||||
const float effectiveFMax{ (fMax <= 0.F || fMax > nyquist) ? nyquist : fMax };
|
||||
const float melMin{ freqToMel(effectiveFMin) };
|
||||
const float melMax{ freqToMel(effectiveFMax) };
|
||||
|
||||
// 1. mel points
|
||||
std::vector<float> melPoints(filterCount + 2);
|
||||
for (std::size_t i{}; i < melPoints.size(); ++i)
|
||||
melPoints[i] = melMin + i * (melMax - melMin) / (filterCount + 1);
|
||||
|
||||
// 2. mel -> Hz
|
||||
std::vector<float> freqs(filterCount + 2);
|
||||
std::transform(melPoints.begin(), melPoints.end(), freqs.begin(), melToFreq);
|
||||
|
||||
// 3. Hz -> bins
|
||||
std::vector<size_t> bins(filterCount + 2);
|
||||
for (std::size_t i{}; i < bins.size(); ++i)
|
||||
bins[i] = static_cast<size_t>(std::floor(nfft * freqs[i] / sampleRate));
|
||||
|
||||
// 4. fix duplicates
|
||||
for (std::size_t i{ 1 }; i < bins.size(); ++i)
|
||||
{
|
||||
if (bins[i] <= bins[i - 1])
|
||||
bins[i] = bins[i - 1] + 1;
|
||||
}
|
||||
|
||||
// 5. build filters
|
||||
const std::size_t binCount{ nfft / 2 + 1 };
|
||||
std::vector<MelFilterBank::Filter> filters{ filterCount };
|
||||
|
||||
for (std::size_t m{}; m < filterCount; ++m)
|
||||
{
|
||||
const std::size_t left{ bins[m] };
|
||||
const std::size_t center{ bins[m + 1] };
|
||||
const std::size_t right{ bins[m + 2] };
|
||||
|
||||
std::vector<float> filterWeights;
|
||||
filterWeights.reserve(right - left);
|
||||
|
||||
// rising edge of the triangle
|
||||
for (std::size_t k{ left }; k < center; ++k)
|
||||
filterWeights.push_back(float(k - left) / (center - left));
|
||||
|
||||
// falling edge of the triangle
|
||||
for (std::size_t k{ center }; k < right; ++k)
|
||||
filterWeights.push_back(float(right - k) / (right - center));
|
||||
|
||||
assert(filterWeights.size() == right - left);
|
||||
|
||||
// normalize the filter to sum = 1
|
||||
const float sum{ std::accumulate(filterWeights.begin(), filterWeights.end(), 0.F) };
|
||||
if (sum > 0.F)
|
||||
{
|
||||
for (float& weight : filterWeights)
|
||||
weight /= sum;
|
||||
}
|
||||
|
||||
filters[m] = MelFilterBank::Filter{ std::move(filterWeights), left };
|
||||
}
|
||||
|
||||
return MelFilterBank{ std::move(filters), binCount };
|
||||
}
|
||||
} // namespace lms::audio::features
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
namespace lms::audio::features
|
||||
{
|
||||
float freqToMel(float freq);
|
||||
float melToFreq(float mel);
|
||||
|
||||
struct MelFilterBank
|
||||
{
|
||||
struct Filter
|
||||
{
|
||||
std::vector<float> weights; // normalized so that the sum of weights equals 1.0
|
||||
std::size_t leftBinIndex; // index of the leftmost FFT bin covered by the filter
|
||||
};
|
||||
|
||||
// binCount is the number of FFT bins (nfft/2 + 1) that the filters can cover
|
||||
MelFilterBank(std::vector<Filter>&& _filters, std::size_t binCount);
|
||||
|
||||
const Filter& getFilter(std::size_t m) const;
|
||||
std::size_t getFilterCount() const;
|
||||
std::size_t getBinCount() const;
|
||||
|
||||
float computeEnergy(std::size_t m, std::span<const float> input) const;
|
||||
|
||||
private:
|
||||
const std::vector<Filter> _filters;
|
||||
const std::size_t _binCount;
|
||||
};
|
||||
|
||||
// Each filter covers a range of FFT bins and is normalized so that the sum of its weights equals 1.0
|
||||
// Each filter stores only its non-zero triangular region (sparse representation)
|
||||
// fMin/fMax: frequency range in Hz. Defaults (0.f, 0.f) span from 0 to Nyquist.
|
||||
MelFilterBank computeMelFilterBank(size_t nfft, size_t sampleRate, size_t filterCount, float fMin = 0.F, float fMax = 0.F);
|
||||
} // namespace lms::audio::features
|
||||
@@ -285,7 +285,7 @@ namespace lms::audio::ffmpeg
|
||||
|
||||
bool AudioFile::hasAttachedPictures() const
|
||||
{
|
||||
for (std::size_t i = 0; i < _context->nb_streams; ++i)
|
||||
for (std::size_t i{}; i < _context->nb_streams; ++i)
|
||||
{
|
||||
if (_context->streams[i]->disposition & AV_DISPOSITION_ATTACHED_PIC)
|
||||
return true;
|
||||
@@ -304,7 +304,7 @@ namespace lms::audio::ffmpeg
|
||||
{ AV_CODEC_ID_PPM, "image/x-portable-pixmap" },
|
||||
};
|
||||
|
||||
for (std::size_t i = 0; i < _context->nb_streams; ++i)
|
||||
for (std::size_t i{}; i < _context->nb_streams; ++i)
|
||||
{
|
||||
AVStream* avstream = _context->streams[i];
|
||||
|
||||
|
||||
@@ -127,6 +127,14 @@ namespace lms::audio::ffmpeg
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
_estimatedDuration = std::chrono::milliseconds{ _context->duration == AV_NOPTS_VALUE ? 0 : _context->duration / AV_TIME_BASE * 1'000 };
|
||||
if (_estimatedDuration > offset)
|
||||
_estimatedDuration = _estimatedDuration - std::chrono::duration_cast<std::chrono::milliseconds>(offset);
|
||||
else
|
||||
_estimatedDuration = {};
|
||||
}
|
||||
|
||||
_decoderContext = AVCodecContextPtr{ ::avcodec_alloc_context3(decoder) };
|
||||
if (!_decoderContext)
|
||||
throw Exception{ "Cannot allocate decoder context" };
|
||||
@@ -225,7 +233,7 @@ namespace lms::audio::ffmpeg
|
||||
else
|
||||
{
|
||||
std::array<uint8_t*, AV_NUM_DATA_POINTERS> outData{};
|
||||
for (size_t i = 0; i < outputChannelBuffers.size(); ++i)
|
||||
for (std::size_t i{}; i < outputChannelBuffers.size(); ++i)
|
||||
outData[i] = reinterpret_cast<uint8_t*>(outputChannelBuffers[i].data());
|
||||
|
||||
// Resample decoded audio
|
||||
@@ -266,6 +274,11 @@ namespace lms::audio::ffmpeg
|
||||
return _finished;
|
||||
}
|
||||
|
||||
std::chrono::milliseconds PcmDecoder::getEstimatedDuration() const
|
||||
{
|
||||
return _estimatedDuration;
|
||||
}
|
||||
|
||||
std::size_t PcmDecoder::computeSampleCountPerChannel(std::span<WritableBuffer> outputChannelBuffers) const
|
||||
{
|
||||
if (_parameters.planar)
|
||||
@@ -343,7 +356,7 @@ namespace lms::audio::ffmpeg
|
||||
std::size_t PcmDecoder::drainResampler(std::span<WritableBuffer> outputChannelBuffers, std::size_t maxSamplesPerChannel)
|
||||
{
|
||||
std::array<uint8_t*, AV_NUM_DATA_POINTERS> outData{};
|
||||
for (size_t i = 0; i < outputChannelBuffers.size(); ++i)
|
||||
for (std::size_t i{}; i < outputChannelBuffers.size(); ++i)
|
||||
outData[i] = reinterpret_cast<uint8_t*>(outputChannelBuffers[i].data());
|
||||
|
||||
const int outSampleCount{ ::swr_convert(_resampleContext.get(),
|
||||
|
||||
@@ -40,6 +40,8 @@ namespace lms::audio::ffmpeg
|
||||
std::size_t readSamples(std::span<WritableBuffer> outputChannelBuffers) override;
|
||||
bool finished() const override;
|
||||
|
||||
std::chrono::milliseconds getEstimatedDuration() const override;
|
||||
|
||||
std::size_t computeSampleCountPerChannel(std::span<WritableBuffer> outputChannelBuffers) const;
|
||||
void feedDecoder();
|
||||
std::size_t drainResampler(std::span<WritableBuffer> outputChannelBuffers, std::size_t maxSamplesPerChannel);
|
||||
@@ -52,6 +54,7 @@ namespace lms::audio::ffmpeg
|
||||
bool _draining{};
|
||||
|
||||
AVFormatContextPtr _context;
|
||||
std::chrono::milliseconds _estimatedDuration{};
|
||||
int _inputStreamIndex{};
|
||||
AVCodecContextPtr _decoderContext;
|
||||
AVFramePtr _decodedFrame;
|
||||
|
||||
@@ -62,7 +62,7 @@ namespace lms::audio::ffmpeg::utils
|
||||
|
||||
void avLogCallback(void*, int level, const char* fmt, va_list vl)
|
||||
{
|
||||
if (!core::Service<core::logging::ILogger>::get()->isSeverityActive(core::logging::Severity::DEBUG))
|
||||
if (!core::Service<core::logging::ILogger>::get() || core::Service<core::logging::ILogger>::get()->isSeverityActive(core::logging::Severity::DEBUG))
|
||||
return;
|
||||
|
||||
if (level > AV_LOG_WARNING)
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "MusicNNEmbeddingExtractor.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <numeric>
|
||||
|
||||
#include "audio/Exception.hpp"
|
||||
#include "audio/IMusicNNEmbeddingExtractor.hpp"
|
||||
#include "math/StatsAccumulator.hpp"
|
||||
#include "musicnn/MusicNNModel.hpp"
|
||||
|
||||
namespace lms::audio::musicnn
|
||||
{
|
||||
namespace
|
||||
{
|
||||
constexpr float minMeaningfulPatchRms{ 0.003F };
|
||||
|
||||
template<typename FloatType>
|
||||
FloatType computeRms(std::span<const FloatType> samples)
|
||||
{
|
||||
const FloatType sumSq{ std::transform_reduce(samples.begin(), samples.end(), FloatType{}, std::plus<>{}, [](FloatType s) { return s * s; }) };
|
||||
return std::sqrt(sumSq / static_cast<FloatType>(samples.size()));
|
||||
}
|
||||
|
||||
// We want something like this:
|
||||
// gap patch(0) gap patch(1) gap patch(maxPatchCount) gap
|
||||
std::size_t computePatchGap(std::size_t totalFrameCount, std::size_t patchFrameCount, std::size_t maxPatchCount)
|
||||
{
|
||||
assert(maxPatchCount > 0);
|
||||
assert(patchFrameCount > 0);
|
||||
|
||||
const std::size_t patchCount{ std::min(maxPatchCount, totalFrameCount / patchFrameCount) };
|
||||
if (patchCount == 0)
|
||||
return 0;
|
||||
|
||||
return (totalFrameCount - patchCount * patchFrameCount) / (patchCount + 1);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// Accumulates one 187-frame MusicNN mel patch
|
||||
class MusicNNEmbeddingExtractor::PatchAccumulator
|
||||
{
|
||||
public:
|
||||
void addMelRow(std::span<const float, melBandCount> melRow, float frameRms)
|
||||
{
|
||||
assert(_frameCount < patchFrameCount);
|
||||
const std::size_t offset{ _frameCount * melBandCount };
|
||||
std::copy(melRow.begin(), melRow.end(), _melMatrix.begin() + static_cast<std::ptrdiff_t>(offset));
|
||||
_rmsAccum += frameRms;
|
||||
++_frameCount;
|
||||
}
|
||||
|
||||
void reset() noexcept
|
||||
{
|
||||
_frameCount = {};
|
||||
_rmsAccum = {};
|
||||
}
|
||||
|
||||
[[nodiscard]] bool complete() const { return _frameCount == patchFrameCount; }
|
||||
|
||||
[[nodiscard]] bool meaningful() const
|
||||
{
|
||||
return (_frameCount > 0) && ((_rmsAccum / static_cast<float>(_frameCount)) >= minMeaningfulPatchRms);
|
||||
}
|
||||
|
||||
[[nodiscard]] std::span<const float, patchFrameCount * melBandCount> data() const
|
||||
{
|
||||
return _melMatrix;
|
||||
}
|
||||
|
||||
private:
|
||||
std::size_t _frameCount{};
|
||||
float _rmsAccum{};
|
||||
std::array<float, patchFrameCount * melBandCount> _melMatrix;
|
||||
};
|
||||
|
||||
MusicNNEmbeddingExtractor::MusicNNEmbeddingExtractor(const std::filesystem::path& modelPath, std::size_t maxPatchCount)
|
||||
: _melFilterBank{ features::computeMelFilterBank(fftSize, sampleRate, melBandCount, melFMin, melFMax) }
|
||||
, _model{ modelPath }
|
||||
, _maxPatchCount{ maxPatchCount }
|
||||
{
|
||||
static_assert(MusicNNEmbeddingExtractor::windowSize == MusicNNEmbeddingExtractor::fftSize);
|
||||
if (_maxPatchCount <= 0)
|
||||
throw audio::Exception{ "MusicNN embedding extractor: max patch count must be > 0" };
|
||||
}
|
||||
|
||||
IMusicNNEmbeddingExtractor::ExtractionResult MusicNNEmbeddingExtractor::extract(const std::filesystem::path& audioFile) const
|
||||
{
|
||||
auto frameDecoder{ std::make_unique<FrameDecoder>(audioFile,
|
||||
PcmParameters{ .channelCount = 1,
|
||||
.sampleRate = static_cast<unsigned>(sampleRate),
|
||||
.sampleType = PcmSampleType::Float32,
|
||||
.byteOrder = std::endian::native,
|
||||
.planar = false },
|
||||
frameHopSamples) };
|
||||
|
||||
std::array<float, melBandCount> logMelRow{};
|
||||
std::array<math::StatsAccumulator<float>, decltype(_model)::outputSize> embeddingAccumulators;
|
||||
ExtractionResult result;
|
||||
const std::size_t estimatedFrameCount{ frameDecoder->getEstimatedFrameCount() };
|
||||
|
||||
// Fallback: use a gap of two patch lengths if the frame count is unknown
|
||||
const std::size_t patchGapFrameCount{ estimatedFrameCount ? computePatchGap(frameDecoder->getEstimatedFrameCount(), patchFrameCount, _maxPatchCount) : (2 * patchFrameCount) };
|
||||
const auto patchAccumulator{ std::make_unique<PatchAccumulator>() };
|
||||
|
||||
while (true)
|
||||
{
|
||||
patchAccumulator->reset();
|
||||
|
||||
const auto onFrame{ [&](const FrameDecoder::SpectralFrameView& frame) {
|
||||
// MusicNN log compression: log10(10000 * mel + 1)
|
||||
for (std::size_t m{}; m < melBandCount; ++m)
|
||||
{
|
||||
const float energy{ _melFilterBank.computeEnergy(m, std::span<const float>(frame.powerSpectrum)) };
|
||||
logMelRow[m] = std::log10(10000.F * energy + 1.F);
|
||||
}
|
||||
|
||||
const float rms{ computeRms(frame.rawSamples.subspan(0, frameHopSamples)) };
|
||||
patchAccumulator->addMelRow(logMelRow, rms);
|
||||
} };
|
||||
|
||||
if (patchGapFrameCount > 0 && frameDecoder->skipFrames(patchGapFrameCount) == 0)
|
||||
break;
|
||||
|
||||
if (frameDecoder->decodeFrames(patchFrameCount, onFrame) < patchFrameCount)
|
||||
break;
|
||||
|
||||
assert(patchAccumulator->complete());
|
||||
|
||||
if (!patchAccumulator->meaningful())
|
||||
continue;
|
||||
|
||||
const auto embedding{ _model.forward(patchAccumulator->data()) };
|
||||
for (std::size_t d{}; d < embedding.size(); ++d)
|
||||
embeddingAccumulators[d].add(embedding[d]);
|
||||
++result.patchCount;
|
||||
}
|
||||
|
||||
if (result.patchCount > 0)
|
||||
{
|
||||
for (std::size_t d{}; d < decltype(_model)::outputSize; ++d)
|
||||
result.embeddings.mean.values[d] = embeddingAccumulators[d].getMean();
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
} // namespace lms::audio::musicnn
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "audio/IMusicNNEmbeddingExtractor.hpp"
|
||||
|
||||
#include "MusicNNModel.hpp"
|
||||
#include "features/MelFilterBank.hpp"
|
||||
#include "utils/PcmSpectralFrameDecoder.hpp"
|
||||
|
||||
namespace lms::audio::musicnn
|
||||
{
|
||||
class MusicNNEmbeddingExtractor : public IMusicNNEmbeddingExtractor
|
||||
{
|
||||
public:
|
||||
MusicNNEmbeddingExtractor(const std::filesystem::path& modelPath, std::size_t maxPatchCount);
|
||||
~MusicNNEmbeddingExtractor() override = default;
|
||||
|
||||
MusicNNEmbeddingExtractor(const MusicNNEmbeddingExtractor&) = delete;
|
||||
MusicNNEmbeddingExtractor& operator=(const MusicNNEmbeddingExtractor&) = delete;
|
||||
|
||||
private:
|
||||
[[nodiscard]] ExtractionResult extract(const std::filesystem::path& audioFile) const override;
|
||||
|
||||
// MusicNN signal processing constants (from musicnn/configuration.py and musicnn_torch.py)
|
||||
static constexpr std::size_t sampleRate{ 16'000 };
|
||||
static constexpr std::size_t windowSize{ 512 }; // 512-sample Hann window (32 ms)
|
||||
static constexpr std::size_t fftSize{ 512 };
|
||||
static constexpr std::size_t frameHopSamples{ 256 }; // 16 ms hop (matches FFT_HOP in musicnn)
|
||||
static constexpr std::size_t melBandCount{ 96 };
|
||||
static constexpr float melFMin{ 0.F };
|
||||
static constexpr float melFMax{ 8'000.F };
|
||||
static constexpr std::size_t patchFrameCount{ MusicNNModel::inputFrames }; // 187 frames = 3 s
|
||||
|
||||
class PatchAccumulator;
|
||||
|
||||
using FrameDecoder = PcmSpectralFrameDecoder<512, float>;
|
||||
const features::MelFilterBank _melFilterBank;
|
||||
const MusicNNModel _model;
|
||||
const std::size_t _maxPatchCount;
|
||||
};
|
||||
} // namespace lms::audio::musicnn
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "audio/MusicNNEmbeddings.hpp"
|
||||
|
||||
#include <bit>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
|
||||
#include "audio/Exception.hpp"
|
||||
|
||||
namespace lms::audio
|
||||
{
|
||||
static_assert(sizeof(float) == 4);
|
||||
static_assert(std::numeric_limits<float>::is_iec559);
|
||||
|
||||
namespace
|
||||
{
|
||||
constexpr uint32_t byteswap32(uint32_t x)
|
||||
{
|
||||
return (x >> 24) | ((x >> 8) & 0x0000FF00u) | ((x << 8) & 0x00FF0000u) | (x << 24);
|
||||
}
|
||||
|
||||
void writeFloats(std::span<const float> data, std::span<std::byte> blob)
|
||||
{
|
||||
if (blob.size() < data.size() * sizeof(uint32_t))
|
||||
throw Exception{ "Buffer too small to write MusicNN embeddings" };
|
||||
|
||||
for (std::size_t i{}; i < data.size(); ++i)
|
||||
{
|
||||
uint32_t bits{ std::bit_cast<uint32_t>(data[i]) };
|
||||
if constexpr (std::endian::native == std::endian::little)
|
||||
bits = byteswap32(bits);
|
||||
std::memcpy(blob.data() + i * 4, &bits, 4);
|
||||
}
|
||||
}
|
||||
|
||||
void readFloats(std::span<const std::byte> blob, std::span<float> data)
|
||||
{
|
||||
if (blob.size() < data.size() * sizeof(uint32_t))
|
||||
throw Exception{ "Buffer too small to read MusicNN embeddings" };
|
||||
|
||||
for (std::size_t i{}; i < data.size(); ++i)
|
||||
{
|
||||
uint32_t bits{};
|
||||
std::memcpy(&bits, blob.data() + i * 4, 4);
|
||||
if constexpr (std::endian::native == std::endian::little)
|
||||
bits = byteswap32(bits);
|
||||
data[i] = std::bit_cast<float>(bits);
|
||||
}
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
void trackMusicNNEmbeddingsToBlob(const TrackMusicNNEmbeddings& embeddings, std::span<std::byte> buffer)
|
||||
{
|
||||
if (buffer.size() < sizeof(TrackMusicNNEmbeddings))
|
||||
throw Exception{ "Buffer too small to write TrackMusicNNEmbeddings" };
|
||||
|
||||
writeFloats(embeddings.mean.values, buffer.subspan(0, MusicNNEmbedding::size * sizeof(float)));
|
||||
}
|
||||
|
||||
void trackMusicNNEmbeddingsFromBlob(std::span<const std::byte> buffer, TrackMusicNNEmbeddings& embeddings)
|
||||
{
|
||||
if (buffer.size() < sizeof(TrackMusicNNEmbeddings))
|
||||
throw Exception{ "Buffer too small to read TrackMusicNNEmbeddings" };
|
||||
|
||||
readFloats(buffer.subspan(0, MusicNNEmbedding::size * sizeof(float)), embeddings.mean.values);
|
||||
}
|
||||
} // namespace lms::audio
|
||||
@@ -0,0 +1,111 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "MusicNNModel.hpp"
|
||||
|
||||
#include <string>
|
||||
|
||||
#include <onnxruntime_cxx_api.h>
|
||||
|
||||
#include "audio/Exception.hpp"
|
||||
|
||||
namespace lms::audio::musicnn
|
||||
{
|
||||
namespace
|
||||
{
|
||||
// Shape of the ONNX model's single input tensor: [batch=1, T=187, mel=96]
|
||||
const std::array<std::int64_t, 3> inputShape{ 1,
|
||||
static_cast<std::int64_t>(MusicNNModel::inputFrames),
|
||||
static_cast<std::int64_t>(MusicNNModel::inputBands) };
|
||||
|
||||
// Shape of the ONNX model's single output tensor: [batch=1, embedding=200]
|
||||
const std::array<std::int64_t, 2> outputShape{ 1,
|
||||
static_cast<std::int64_t>(MusicNNModel::outputSize) };
|
||||
|
||||
constexpr const char* inputName{ "mel_patch" };
|
||||
constexpr const char* outputName{ "embedding" };
|
||||
} // namespace
|
||||
|
||||
struct MusicNNModel::Impl
|
||||
{
|
||||
Ort::Env env;
|
||||
Ort::SessionOptions sessionOptions;
|
||||
Ort::Session session;
|
||||
Ort::MemoryInfo memoryInfo;
|
||||
|
||||
explicit Impl(const std::filesystem::path& onnxPath)
|
||||
: env{ ORT_LOGGING_LEVEL_ERROR, "MusicNN" }
|
||||
, session{ [&]() -> Ort::Session {
|
||||
sessionOptions.SetIntraOpNumThreads(1);
|
||||
sessionOptions.SetInterOpNumThreads(1);
|
||||
return Ort::Session{ env, onnxPath.c_str(), sessionOptions };
|
||||
}() }
|
||||
, memoryInfo{ Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault) }
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
MusicNNModel::MusicNNModel(const std::filesystem::path& onnxPath)
|
||||
{
|
||||
try
|
||||
{
|
||||
_impl = std::make_unique<Impl>(onnxPath);
|
||||
}
|
||||
catch (const Ort::Exception& e)
|
||||
{
|
||||
throw audio::Exception{ std::string{ "Failed to load ONNX model '" } + onnxPath.string() + "': " + e.what() };
|
||||
}
|
||||
}
|
||||
|
||||
MusicNNModel::~MusicNNModel() = default;
|
||||
|
||||
std::array<float, MusicNNModel::outputSize> MusicNNModel::forward(
|
||||
std::span<const float, inputFrames * inputBands> melPatch) const
|
||||
{
|
||||
Ort::Value inputTensor{ Ort::Value::CreateTensor<float>(
|
||||
_impl->memoryInfo,
|
||||
const_cast<float*>(melPatch.data()), // safe cast
|
||||
melPatch.size(),
|
||||
inputShape.data(),
|
||||
inputShape.size()) };
|
||||
|
||||
std::array<float, outputSize> result{};
|
||||
Ort::Value outputTensor{ Ort::Value::CreateTensor<float>(
|
||||
_impl->memoryInfo,
|
||||
result.data(),
|
||||
result.size(),
|
||||
outputShape.data(),
|
||||
outputShape.size()) };
|
||||
|
||||
try
|
||||
{
|
||||
auto inputNames{ std::to_array({ inputName }) };
|
||||
auto outputNames{ std::to_array({ outputName }) };
|
||||
_impl->session.Run(Ort::RunOptions{ nullptr },
|
||||
inputNames.data(), &inputTensor, 1,
|
||||
outputNames.data(), &outputTensor, 1);
|
||||
}
|
||||
catch (const Ort::Exception& e)
|
||||
{
|
||||
throw audio::Exception{ std::string{ "ONNX inference failed: " } + e.what() };
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
} // namespace lms::audio::musicnn
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <span>
|
||||
|
||||
namespace lms::audio::musicnn
|
||||
{
|
||||
// The ONNX model must be exported with:
|
||||
// input "mel_patch" shape [1, 1, 187, 96]
|
||||
// output "embedding" shape [1, 200]
|
||||
//
|
||||
// Export script: tools/musicnn/export_onnx.py
|
||||
class MusicNNModel
|
||||
{
|
||||
public:
|
||||
explicit MusicNNModel(const std::filesystem::path& onnxPath);
|
||||
~MusicNNModel();
|
||||
|
||||
MusicNNModel(const MusicNNModel&) = delete;
|
||||
MusicNNModel& operator=(const MusicNNModel&) = delete;
|
||||
|
||||
static inline constexpr std::size_t inputFrames{ 187 };
|
||||
static inline constexpr std::size_t inputBands{ 96 };
|
||||
static inline constexpr std::size_t outputSize{ 200 };
|
||||
|
||||
[[nodiscard]] std::array<float, outputSize> forward(std::span<const float, inputFrames * inputBands> melPatch) const;
|
||||
|
||||
private:
|
||||
// Pimpl: keep ORT headers out of translation units that include this header.
|
||||
struct Impl;
|
||||
std::unique_ptr<Impl> _impl;
|
||||
};
|
||||
} // namespace lms::audio::musicnn
|
||||
@@ -20,6 +20,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/asio/io_context.hpp>
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <bit>
|
||||
#include <cassert>
|
||||
#include <complex>
|
||||
#include <concepts>
|
||||
#include <cstddef>
|
||||
#include <filesystem>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "core/AlignedHeapArray.hpp"
|
||||
|
||||
#include "audio/IPcmDecoder.hpp"
|
||||
#include "audio/PcmTypes.hpp"
|
||||
#include "math/FFT.hpp"
|
||||
#include "math/Window.hpp"
|
||||
|
||||
namespace lms::audio
|
||||
{
|
||||
// Stateful PCM frame decoder that applies a Hann window + FFT per frame.
|
||||
template<std::size_t WindowSize, typename FloatType = float>
|
||||
class PcmSpectralFrameDecoder
|
||||
{
|
||||
static_assert(std::has_single_bit(WindowSize), "WindowSize must be a power of two");
|
||||
|
||||
public:
|
||||
using FFTPlan = math::FixedRealFFTPlan<WindowSize, FloatType>;
|
||||
static constexpr std::size_t spectrumSize{ FFTPlan::getOutputSize() };
|
||||
|
||||
PcmSpectralFrameDecoder(const std::filesystem::path& audioFile, const PcmParameters& params, std::size_t hopSize)
|
||||
: PcmSpectralFrameDecoder{ createPcmDecoder(audioFile, {}, params), hopSize }
|
||||
{
|
||||
}
|
||||
~PcmSpectralFrameDecoder() = default;
|
||||
|
||||
explicit PcmSpectralFrameDecoder(std::unique_ptr<IPcmDecoder> decoder, std::size_t hopSize)
|
||||
: _pcmParams{ decoder->getParameters() }
|
||||
, _hopSize{ hopSize }
|
||||
, _powerScale{ FloatType{ 1 } / (_window.energy() * static_cast<FloatType>(WindowSize)) }
|
||||
, _decoder{ std::move(decoder) }
|
||||
, _samplesBuffer(bufferFrameCount * _hopSize + WindowSize)
|
||||
, _bufferedSampleCount{ WindowSize / 2 } // first analysis frame centered on sample 0, matching librosa center=True semantics.
|
||||
{
|
||||
assert(_hopSize > 0);
|
||||
}
|
||||
|
||||
PcmSpectralFrameDecoder(const PcmSpectralFrameDecoder&) = delete;
|
||||
PcmSpectralFrameDecoder& operator=(const PcmSpectralFrameDecoder&) = delete;
|
||||
|
||||
std::size_t hopSize() const noexcept { return _hopSize; }
|
||||
const PcmParameters& pcmParameters() const noexcept { return _pcmParams; }
|
||||
|
||||
std::size_t getEstimatedFrameCount() const
|
||||
{
|
||||
const auto duration{ _decoder->getEstimatedDuration() };
|
||||
if (duration <= std::chrono::milliseconds::zero())
|
||||
return 0;
|
||||
|
||||
const auto totalSamples{ static_cast<std::size_t>((static_cast<std::uint64_t>(duration.count()) * _pcmParams.sampleRate) / 1'000) };
|
||||
|
||||
constexpr std::size_t halfWindow{ WindowSize / 2 };
|
||||
if (totalSamples < halfWindow) // Not enough samples to produce even the first frame.
|
||||
return 0;
|
||||
|
||||
return 1 + ((totalSamples - halfWindow) / _hopSize);
|
||||
}
|
||||
|
||||
// Spectral data for a single frame.
|
||||
struct SpectralFrameView
|
||||
{
|
||||
std::span<const FloatType, WindowSize> rawSamples;
|
||||
std::span<const FloatType, spectrumSize> powerSpectrum;
|
||||
};
|
||||
|
||||
// Decodes up to frameCount frames, invoking callback for each. May return fewer than
|
||||
// frameCount at EOF. Returns 0 only if no frame at all could be decoded.
|
||||
template<typename Callback>
|
||||
requires std::invocable<Callback, const SpectralFrameView&>
|
||||
std::size_t decodeFrames(std::size_t frameCount, Callback&& callback)
|
||||
{
|
||||
if (frameCount == 0)
|
||||
return 0;
|
||||
|
||||
std::size_t decodedCount{};
|
||||
while (decodedCount < frameCount)
|
||||
{
|
||||
if (!readAtLeastSamples(std::max(WindowSize, _hopSize)))
|
||||
break;
|
||||
|
||||
const std::span<const FloatType, WindowSize> rawSamples{ _samplesBuffer.data(), WindowSize };
|
||||
const std::span<FloatType, WindowSize> windowedFrame{ _windowedFrame.data(), WindowSize };
|
||||
_window.apply(rawSamples, windowedFrame);
|
||||
|
||||
_fftPlan.apply(_windowedFrame, _fftOutput);
|
||||
|
||||
// Reuse _windowedFrame for power spectrum (spectrumSize <= WindowSize).
|
||||
const std::span<FloatType, spectrumSize> powerBuffer{ _windowedFrame.data(), spectrumSize };
|
||||
std::transform(_fftOutput.cbegin(), _fftOutput.cend(), powerBuffer.begin(),
|
||||
[this](const std::complex<FloatType>& bin) {
|
||||
return (bin.real() * bin.real() + bin.imag() * bin.imag()) * _powerScale;
|
||||
});
|
||||
|
||||
const SpectralFrameView frame{ .rawSamples = rawSamples,
|
||||
.powerSpectrum = std::span<const FloatType, spectrumSize>{ _windowedFrame.data(), spectrumSize } };
|
||||
std::invoke(callback, frame);
|
||||
|
||||
consumeSamples(_hopSize);
|
||||
++_currentFrameIndex;
|
||||
++decodedCount;
|
||||
}
|
||||
|
||||
return decodedCount;
|
||||
}
|
||||
|
||||
// Skips exactly the next frameCount frames without computing FFT or invoking callbacks.
|
||||
// Returns frameCount on success, 0 on EOF.
|
||||
std::size_t skipFrames(std::size_t frameCount)
|
||||
{
|
||||
if (frameCount == 0)
|
||||
return 0;
|
||||
|
||||
std::size_t skippedFrameCount{};
|
||||
while (skippedFrameCount < frameCount)
|
||||
{
|
||||
if (!readAtLeastSamples(std::max(WindowSize, _hopSize)))
|
||||
break;
|
||||
|
||||
consumeSamples(_hopSize);
|
||||
++_currentFrameIndex;
|
||||
++skippedFrameCount;
|
||||
}
|
||||
|
||||
return skippedFrameCount;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::size_t currentFrameIndex() const noexcept { return _currentFrameIndex; }
|
||||
|
||||
private:
|
||||
static constexpr std::size_t bufferFrameCount{ 20 };
|
||||
|
||||
bool readAtLeastSamples(std::size_t sampleCount)
|
||||
{
|
||||
if (sampleCount <= _bufferedSampleCount)
|
||||
return true;
|
||||
|
||||
if (_samplesBuffer.size() < sampleCount)
|
||||
_samplesBuffer.resize(sampleCount);
|
||||
|
||||
while ((_bufferedSampleCount < sampleCount) && !_endOfStream)
|
||||
{
|
||||
std::span<FloatType> dest{ _samplesBuffer.data() + _bufferedSampleCount, _samplesBuffer.size() - _bufferedSampleCount };
|
||||
assert(!dest.empty());
|
||||
if (dest.empty())
|
||||
break;
|
||||
|
||||
std::array outputBuffers{ IPcmDecoder::WritableBuffer{ std::as_writable_bytes(dest) } };
|
||||
const std::size_t samplesRead{ _decoder->readSamples(outputBuffers) };
|
||||
if (samplesRead == 0)
|
||||
{
|
||||
_endOfStream = true;
|
||||
break;
|
||||
}
|
||||
|
||||
_bufferedSampleCount += samplesRead;
|
||||
}
|
||||
|
||||
return _bufferedSampleCount >= sampleCount;
|
||||
}
|
||||
|
||||
void consumeSamples(std::size_t samplesToDrop)
|
||||
{
|
||||
// TODO use a circular buffer and only compacts at the end of the buffer
|
||||
assert(samplesToDrop <= _bufferedSampleCount);
|
||||
|
||||
const auto remaining{ _bufferedSampleCount - samplesToDrop };
|
||||
if (remaining)
|
||||
{
|
||||
std::move(_samplesBuffer.begin() + samplesToDrop,
|
||||
_samplesBuffer.begin() + _bufferedSampleCount,
|
||||
_samplesBuffer.begin());
|
||||
}
|
||||
|
||||
_bufferedSampleCount = remaining;
|
||||
}
|
||||
|
||||
const PcmParameters _pcmParams;
|
||||
const std::size_t _hopSize;
|
||||
const math::HannWindow<WindowSize, FloatType> _window;
|
||||
const FloatType _powerScale;
|
||||
const FFTPlan _fftPlan{};
|
||||
std::unique_ptr<IPcmDecoder> _decoder;
|
||||
std::vector<FloatType> _samplesBuffer;
|
||||
core::AlignedHeapArray<FloatType, FFTPlan::minBufferAlignment> _windowedFrame{ FFTPlan::getInputSize() };
|
||||
core::AlignedHeapArray<std::complex<FloatType>, FFTPlan::minBufferAlignment> _fftOutput{ FFTPlan::getOutputSize() };
|
||||
std::size_t _bufferedSampleCount{};
|
||||
std::size_t _currentFrameIndex{};
|
||||
bool _endOfStream{};
|
||||
};
|
||||
} // namespace lms::audio
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
|
||||
#include "audio/MusicNNEmbeddings.hpp"
|
||||
|
||||
namespace lms::audio
|
||||
{
|
||||
class IMusicNNEmbeddingExtractor
|
||||
{
|
||||
public:
|
||||
virtual ~IMusicNNEmbeddingExtractor() = default;
|
||||
|
||||
struct ExtractionResult
|
||||
{
|
||||
TrackMusicNNEmbeddings embeddings{};
|
||||
std::size_t patchCount{};
|
||||
};
|
||||
|
||||
[[nodiscard]] virtual ExtractionResult extract(const std::filesystem::path& audioFile) const = 0;
|
||||
};
|
||||
|
||||
bool canExtractMusicNNEmbeddings();
|
||||
std::unique_ptr<IMusicNNEmbeddingExtractor> createMusicNNEmbeddingExtractor(const std::filesystem::path& modelPath, std::size_t maxPatchCount);
|
||||
std::string getMusicNNModelIdentifier(const std::filesystem::path& modelPath);
|
||||
} // namespace lms::audio
|
||||
@@ -43,8 +43,11 @@ namespace lms::audio
|
||||
// Each buffer must be sized to hold an integer number of samples according to the requested sample type.
|
||||
// For example, for Float32 planar output, each buffer size must be divisible by sizeof(float).
|
||||
// The decoder will use the buffer sizes to determine the maximum number of samples it can write.
|
||||
// The decoder will not try to fill in the whole supplied buffer
|
||||
virtual std::size_t readSamples(std::span<WritableBuffer> outputChannelBuffers) = 0;
|
||||
virtual bool finished() const = 0;
|
||||
|
||||
virtual std::chrono::milliseconds getEstimatedDuration() const = 0; // initial offset is taken into account, 0 if unknown
|
||||
};
|
||||
|
||||
// Throw on error
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <span>
|
||||
|
||||
namespace lms::audio
|
||||
{
|
||||
struct MusicNNEmbedding
|
||||
{
|
||||
static inline constexpr std::size_t size{ 200 };
|
||||
std::array<float, size> values;
|
||||
};
|
||||
|
||||
struct TrackMusicNNEmbeddings
|
||||
{
|
||||
MusicNNEmbedding mean;
|
||||
};
|
||||
|
||||
// Buffer size must be at least sizeof(TrackMusicNNEmbeddings)
|
||||
void trackMusicNNEmbeddingsToBlob(const TrackMusicNNEmbeddings& embeddings, std::span<std::byte> buffer);
|
||||
void trackMusicNNEmbeddingsFromBlob(std::span<const std::byte> buffer, TrackMusicNNEmbeddings& embeddings);
|
||||
} // namespace lms::audio
|
||||
@@ -20,6 +20,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <bit>
|
||||
#include <chrono>
|
||||
|
||||
namespace lms::audio
|
||||
{
|
||||
@@ -41,4 +42,21 @@ namespace lms::audio
|
||||
std::endian byteOrder;
|
||||
bool planar;
|
||||
};
|
||||
|
||||
namespace helpers
|
||||
{
|
||||
template<typename Rep, typename Period>
|
||||
std::size_t durationToSampleCount(std::chrono::duration<Rep, Period> duration, unsigned sampleRate)
|
||||
{
|
||||
return static_cast<std::size_t>(duration.count() * sampleRate * Period::num / Period::den);
|
||||
}
|
||||
|
||||
template<typename Duration>
|
||||
Duration sampleCountToDuration(std::size_t sampleCount, unsigned sampleRate)
|
||||
{
|
||||
return std::chrono::duration_cast<Duration>(std::chrono::duration<double>(sampleCount) / sampleRate);
|
||||
}
|
||||
|
||||
std::size_t sampleCountToByteCount(std::size_t sampleCount, PcmSampleType sampleType, unsigned channelCount);
|
||||
} // namespace helpers
|
||||
} // namespace lms::audio
|
||||
Binary file not shown.
@@ -0,0 +1,33 @@
|
||||
include(GoogleTest)
|
||||
|
||||
add_executable(test-audio
|
||||
MelFilterBank.cpp
|
||||
MusicNNEmbeddings.cpp
|
||||
PcmSpectralFrameDecoder.cpp
|
||||
)
|
||||
|
||||
if (OnnxRuntime_FOUND)
|
||||
target_sources(test-audio PRIVATE
|
||||
MusicNNModel.cpp
|
||||
)
|
||||
endif()
|
||||
|
||||
target_include_directories(test-audio PRIVATE
|
||||
../impl
|
||||
)
|
||||
|
||||
target_link_libraries(test-audio PRIVATE
|
||||
lmsaudio
|
||||
lmsmath
|
||||
GTest::GTest
|
||||
GTest::gtest_main
|
||||
)
|
||||
|
||||
target_compile_options(test-audio PRIVATE
|
||||
$<$<NOT:$<CONFIG:Debug>>:-ffast-math>
|
||||
)
|
||||
|
||||
if (NOT CMAKE_CROSSCOMPILING)
|
||||
gtest_discover_tests(test-audio)
|
||||
endif()
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <numeric>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "audio/Exception.hpp"
|
||||
|
||||
#include "features/MelFilterBank.hpp"
|
||||
|
||||
namespace lms::audio::features::tests
|
||||
{
|
||||
constexpr float epsilon{ 1e-5F };
|
||||
|
||||
constexpr std::size_t NFFT{ 2048 };
|
||||
constexpr std::size_t sampleRate{ 22050 };
|
||||
constexpr std::size_t filterCount{ 40 };
|
||||
|
||||
TEST(MelFilterBank, sizeCheck)
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
|
||||
EXPECT_EQ(bank.getFilterCount(), filterCount);
|
||||
EXPECT_EQ(bank.getBinCount(), NFFT / 2 + 1);
|
||||
}
|
||||
|
||||
TEST(MelFilterBank, differentSampleRates)
|
||||
{
|
||||
for (const std::size_t sr : { std::size_t{ 8000 }, std::size_t{ 16000 }, std::size_t{ 44100 }, std::size_t{ 48000 } })
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sr, filterCount) };
|
||||
EXPECT_EQ(bank.getFilterCount(), filterCount) << "sr=" << sr;
|
||||
}
|
||||
}
|
||||
|
||||
TEST(MelFilterBank, nonNegativeWeights)
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
|
||||
|
||||
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
|
||||
{
|
||||
const auto& filter = bank.getFilter(m);
|
||||
for (float w : filter.weights)
|
||||
EXPECT_GE(w, 0.F) << "Filter " << m << " has negative weight: " << w;
|
||||
}
|
||||
}
|
||||
|
||||
TEST(MelFilterBank, peaksAreNonZero)
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
|
||||
|
||||
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
|
||||
{
|
||||
const auto& filter{ bank.getFilter(m) };
|
||||
const float maxVal{ *std::max_element(filter.weights.begin(), filter.weights.end()) };
|
||||
|
||||
EXPECT_GT(maxVal, 0.F) << "Filter " << m << " has zero peak value";
|
||||
}
|
||||
}
|
||||
|
||||
TEST(MelFilterBank, filtersAreUnitSum)
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
|
||||
|
||||
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
|
||||
{
|
||||
const auto& filter{ bank.getFilter(m) };
|
||||
const float sum{ std::accumulate(filter.weights.begin(), filter.weights.end(), 0.F) };
|
||||
|
||||
EXPECT_NEAR(sum, 1.F, epsilon) << "Filter " << m << " sum = " << sum;
|
||||
}
|
||||
}
|
||||
|
||||
TEST(MelFilterBank, everyBinCovered)
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
|
||||
const std::size_t binCount{ bank.getBinCount() };
|
||||
|
||||
std::vector<bool> covered(binCount, false);
|
||||
|
||||
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
|
||||
{
|
||||
const auto& filter{ bank.getFilter(m) };
|
||||
|
||||
std::size_t bin{ filter.leftBinIndex };
|
||||
for (float w : filter.weights)
|
||||
{
|
||||
if (w > 0.F)
|
||||
covered[bin] = true;
|
||||
|
||||
++bin;
|
||||
}
|
||||
}
|
||||
|
||||
// Find actual covered range
|
||||
auto first{ std::find(covered.begin(), covered.end(), true) };
|
||||
auto last{ std::find(covered.rbegin(), covered.rend(), true).base() };
|
||||
|
||||
ASSERT_NE(first, covered.end()); // sanity
|
||||
|
||||
for (auto it = first; it != last; ++it)
|
||||
{
|
||||
EXPECT_TRUE(*it) << "A bin in the covered range is not covered by any filter";
|
||||
}
|
||||
}
|
||||
|
||||
TEST(MelFilterBank, overlapAtMostTwo)
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
|
||||
const std::size_t binCount{ bank.getBinCount() };
|
||||
|
||||
std::vector<std::size_t> overlap(binCount, 0);
|
||||
|
||||
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
|
||||
{
|
||||
const auto& filter{ bank.getFilter(m) };
|
||||
|
||||
std::size_t bin{ filter.leftBinIndex };
|
||||
for (float w : filter.weights)
|
||||
{
|
||||
if (w > 0.F)
|
||||
overlap[bin]++;
|
||||
++bin;
|
||||
}
|
||||
}
|
||||
|
||||
for (std::size_t k{}; k < binCount; ++k)
|
||||
{
|
||||
EXPECT_LE(overlap[k], 2) << "Bin " << k << " is covered by " << overlap[k] << " filters";
|
||||
}
|
||||
}
|
||||
|
||||
TEST(MelFilterBank, flatSpectrumEnergySanity)
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
|
||||
std::vector<float> flatSpectrum(bank.getBinCount(), 1.F);
|
||||
|
||||
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
|
||||
{
|
||||
const float energy{ bank.computeEnergy(m, flatSpectrum) };
|
||||
EXPECT_GT(energy, 0.F) << "Filter " << m << " has zero energy for flat spectrum";
|
||||
}
|
||||
}
|
||||
TEST(MelFilterBank, zeroFilterCount)
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, 0) };
|
||||
|
||||
EXPECT_EQ(bank.getFilterCount(), 0U);
|
||||
EXPECT_EQ(bank.getBinCount(), NFFT / 2 + 1);
|
||||
}
|
||||
|
||||
TEST(MelFilterBank, computeEnergyRejectsInvalidInputSize)
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
|
||||
std::vector<float> invalidInput(bank.getBinCount() - 1, 1.F);
|
||||
|
||||
EXPECT_THROW(bank.computeEnergy(0, invalidInput), Exception);
|
||||
}
|
||||
|
||||
TEST(MelFilterBank, zeroSpectrum)
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
|
||||
std::vector<float> zeroSpectrum(bank.getBinCount(), 0.F);
|
||||
|
||||
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
|
||||
{
|
||||
const float energy{ bank.computeEnergy(m, zeroSpectrum) };
|
||||
EXPECT_FLOAT_EQ(energy, 0.F) << "Filter " << m << " should have zero energy for zero spectrum";
|
||||
}
|
||||
}
|
||||
|
||||
TEST(MelFilterBank, largeSpectrumValues)
|
||||
{
|
||||
const MelFilterBank bank{ computeMelFilterBank(NFFT, sampleRate, filterCount) };
|
||||
// Weights sum to 1.0 per filter, so energy = input * 1.0, no overflow risk at max/2
|
||||
const float largeValue{ std::numeric_limits<float>::max() / 2.F };
|
||||
std::vector<float> largeSpectrum(bank.getBinCount(), largeValue);
|
||||
|
||||
for (std::size_t m{}; m < bank.getFilterCount(); ++m)
|
||||
{
|
||||
const float energy{ bank.computeEnergy(m, largeSpectrum) };
|
||||
EXPECT_GT(energy, 0.F) << "Filter " << m << " energy is not positive for large spectrum";
|
||||
}
|
||||
}
|
||||
} // namespace lms::audio::features::tests
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <numeric>
|
||||
#include <vector>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "audio/Exception.hpp"
|
||||
#include "audio/MusicNNEmbeddings.hpp"
|
||||
|
||||
namespace lms::audio::tests
|
||||
{
|
||||
TEST(MusicNNEmbeddings, blobRoundTrip)
|
||||
{
|
||||
TrackMusicNNEmbeddings original{};
|
||||
|
||||
std::iota(original.mean.values.begin(), original.mean.values.end(), 0.F);
|
||||
|
||||
std::vector<std::byte> blob(sizeof(TrackMusicNNEmbeddings));
|
||||
trackMusicNNEmbeddingsToBlob(original, blob);
|
||||
|
||||
TrackMusicNNEmbeddings restored{};
|
||||
trackMusicNNEmbeddingsFromBlob(blob, restored);
|
||||
|
||||
for (std::size_t i{}; i < MusicNNEmbedding::size; ++i)
|
||||
EXPECT_FLOAT_EQ(restored.mean.values[i], original.mean.values[i]);
|
||||
}
|
||||
|
||||
TEST(MusicNNEmbeddings, blobSizeTooSmallThrows)
|
||||
{
|
||||
TrackMusicNNEmbeddings embeddings{};
|
||||
std::vector<std::byte> blob(sizeof(TrackMusicNNEmbeddings) - 1);
|
||||
EXPECT_THROW(trackMusicNNEmbeddingsToBlob(embeddings, blob), Exception);
|
||||
}
|
||||
|
||||
TEST(MusicNNEmbeddings, blobFromSizeTooSmallThrows)
|
||||
{
|
||||
std::vector<std::byte> blob(sizeof(TrackMusicNNEmbeddings) - 1, std::byte{});
|
||||
TrackMusicNNEmbeddings embeddings{};
|
||||
EXPECT_THROW(trackMusicNNEmbeddingsFromBlob(blob, embeddings), Exception);
|
||||
}
|
||||
} // namespace lms::audio::tests
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <array>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <random>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "musicnn/MusicNNModel.hpp"
|
||||
|
||||
namespace lms::audio::musicnn::tests
|
||||
{
|
||||
namespace
|
||||
{
|
||||
std::filesystem::path getMusicNNModelPathFromEnv()
|
||||
{
|
||||
const char* p{ std::getenv("LMS_MUSICNN_MODEL") };
|
||||
return p ? std::filesystem::path{ p } : std::filesystem::path{};
|
||||
}
|
||||
|
||||
std::array<float, MusicNNModel::inputFrames * MusicNNModel::inputBands> makeRandomPatch()
|
||||
{
|
||||
std::minstd_rand rng{ 42 };
|
||||
std::uniform_real_distribution<float> dist{ 0.F, 1.F };
|
||||
|
||||
std::array<float, MusicNNModel::inputFrames * MusicNNModel::inputBands> patch{};
|
||||
|
||||
for (float& v : patch)
|
||||
v = dist(rng);
|
||||
|
||||
return patch;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(MusicNNModel, CanConstruct)
|
||||
{
|
||||
const std::filesystem::path path{ getMusicNNModelPathFromEnv() };
|
||||
|
||||
if (path.empty())
|
||||
GTEST_SKIP() << "LMS_MUSICNN_MODEL not set";
|
||||
|
||||
EXPECT_NO_THROW({ const MusicNNModel model{ path }; });
|
||||
}
|
||||
|
||||
TEST(MusicNNModel, CanForward)
|
||||
{
|
||||
const std::filesystem::path path{ getMusicNNModelPathFromEnv() };
|
||||
|
||||
if (path.empty())
|
||||
GTEST_SKIP() << "LMS_MUSICNN_MODEL not set";
|
||||
|
||||
const MusicNNModel model{ path };
|
||||
const auto patch{ makeRandomPatch() };
|
||||
|
||||
EXPECT_NO_THROW({ [[maybe_unused]] const auto output{ model.forward(patch) }; });
|
||||
}
|
||||
|
||||
} // namespace lms::audio::musicnn::tests
|
||||
@@ -0,0 +1,201 @@
|
||||
/*
|
||||
* Copyright (C) 2026 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "audio/IPcmDecoder.hpp"
|
||||
#include "audio/PcmTypes.hpp"
|
||||
|
||||
#include "utils/PcmSpectralFrameDecoder.hpp"
|
||||
|
||||
namespace lms::audio::tests
|
||||
{
|
||||
namespace
|
||||
{
|
||||
// A mock IPcmDecoder that emits samples 0, 1, 2, 3, ... (as float) up to totalSamples,
|
||||
class SequencePcmDecoder : public IPcmDecoder
|
||||
{
|
||||
public:
|
||||
SequencePcmDecoder(std::size_t totalSampleCount)
|
||||
: _totalSampleCount{ totalSampleCount }
|
||||
{
|
||||
}
|
||||
|
||||
const PcmParameters& getParameters() const override { return _params; }
|
||||
|
||||
std::size_t readSamples(std::span<WritableBuffer> outputChannelBuffers) override
|
||||
{
|
||||
assert(outputChannelBuffers.size() == 1); // only planar
|
||||
if (_finished)
|
||||
return 0;
|
||||
|
||||
auto& buf{ outputChannelBuffers[0] };
|
||||
if (buf.size() == 0)
|
||||
return 0;
|
||||
|
||||
assert(buf.size() % sizeof(float) == 0);
|
||||
|
||||
// not always writing up to what is requested
|
||||
std::uniform_int_distribution dist{ std::size_t{ 1 }, buf.size() / sizeof(float) };
|
||||
std::size_t sampleCountToWrite{ dist(_randomEngine) };
|
||||
if (_currentSampleIndex + sampleCountToWrite > _totalSampleCount)
|
||||
{
|
||||
sampleCountToWrite = _totalSampleCount - _currentSampleIndex;
|
||||
_finished = true;
|
||||
}
|
||||
|
||||
float* dest{ reinterpret_cast<float*>(buf.data()) };
|
||||
for (std::size_t i{}; i < sampleCountToWrite; ++i)
|
||||
*(dest++) = static_cast<float>(_currentSampleIndex++);
|
||||
|
||||
return sampleCountToWrite;
|
||||
}
|
||||
|
||||
bool finished() const override { return _finished; }
|
||||
|
||||
std::chrono::milliseconds getEstimatedDuration() const override
|
||||
{
|
||||
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::duration<float>{ static_cast<float>(_totalSampleCount) / static_cast<float>(_params.sampleRate) });
|
||||
}
|
||||
|
||||
private:
|
||||
const PcmParameters _params{
|
||||
.channelCount = 1,
|
||||
.sampleRate = 16000,
|
||||
.sampleType = PcmSampleType::Float32,
|
||||
.byteOrder = std::endian::native,
|
||||
.planar = false,
|
||||
};
|
||||
|
||||
std::minstd_rand _randomEngine{ 42 }; // fixed seed for reproducibility
|
||||
std::size_t _totalSampleCount{};
|
||||
std::size_t _currentSampleIndex{};
|
||||
bool _finished{};
|
||||
};
|
||||
|
||||
template<typename T, std::size_t N>
|
||||
void expectSpanEq(std::span<const T, N> actual, const std::array<T, N>& expected)
|
||||
{
|
||||
for (std::size_t i{}; i < N; ++i)
|
||||
EXPECT_FLOAT_EQ(actual[i], expected[i]) << "index=" << i;
|
||||
}
|
||||
|
||||
constexpr std::size_t WindowSize{ 8 };
|
||||
constexpr std::size_t HopSize{ 4 };
|
||||
using FrameDecoder = PcmSpectralFrameDecoder<WindowSize, float>;
|
||||
} // namespace
|
||||
|
||||
TEST(SequencePcmDecoder, basic)
|
||||
{
|
||||
constexpr std::size_t decoderTotalSampleCount{ 32 };
|
||||
SequencePcmDecoder decoder{ decoderTotalSampleCount };
|
||||
|
||||
std::size_t totalSampleReadCount{};
|
||||
while (true)
|
||||
{
|
||||
std::array<float, 16> buffer{};
|
||||
std::array outputBuffers{ IPcmDecoder::WritableBuffer{ std::as_writable_bytes(std::span{ buffer }) } };
|
||||
const std::size_t sampleReadCount{ decoder.readSamples(outputBuffers) };
|
||||
if (sampleReadCount == 0)
|
||||
break;
|
||||
|
||||
for (std::size_t i{}; i < sampleReadCount; ++i)
|
||||
EXPECT_FLOAT_EQ(buffer[i], totalSampleReadCount + i);
|
||||
|
||||
totalSampleReadCount += sampleReadCount;
|
||||
}
|
||||
|
||||
EXPECT_EQ(totalSampleReadCount, decoderTotalSampleCount);
|
||||
}
|
||||
|
||||
TEST(PcmSpectralFrameDecoder, firstFrameIsCenteredOnSample0)
|
||||
{
|
||||
FrameDecoder frameDecoder{ std::make_unique<SequencePcmDecoder>(32), HopSize };
|
||||
using Frame = std::array<float, WindowSize>;
|
||||
std::vector<Frame> frames;
|
||||
|
||||
EXPECT_EQ(frameDecoder.currentFrameIndex(), 0);
|
||||
const std::size_t decoded{ frameDecoder.decodeFrames(1,
|
||||
[&](const FrameDecoder::SpectralFrameView& frame) {
|
||||
auto& newFrame{ frames.emplace_back() };
|
||||
std::copy(std::cbegin(frame.rawSamples), std::cend(frame.rawSamples), std::begin(newFrame));
|
||||
}) };
|
||||
|
||||
ASSERT_EQ(decoded, 1);
|
||||
ASSERT_EQ(frames.size(), 1);
|
||||
EXPECT_EQ(frameDecoder.currentFrameIndex(), 1);
|
||||
expectSpanEq<float, WindowSize>(frames[0], { 0.F, 0.F, 0.F, 0.F, 0.F, 1.F, 2.F, 3.F });
|
||||
}
|
||||
|
||||
TEST(PcmSpectralFrameDecoder, framesAdvanceByHopSize)
|
||||
{
|
||||
FrameDecoder frameDecoder{ std::make_unique<SequencePcmDecoder>(16), HopSize };
|
||||
using Frame = std::array<float, WindowSize>;
|
||||
std::vector<Frame> frames;
|
||||
|
||||
const std::size_t decoded{ frameDecoder.decodeFrames(2,
|
||||
[&](const FrameDecoder::SpectralFrameView& frame) {
|
||||
auto& newFrame{ frames.emplace_back() };
|
||||
std::copy(std::cbegin(frame.rawSamples), std::cend(frame.rawSamples), std::begin(newFrame));
|
||||
}) };
|
||||
|
||||
ASSERT_EQ(decoded, 2);
|
||||
ASSERT_EQ(frames.size(), 2);
|
||||
EXPECT_EQ(frameDecoder.currentFrameIndex(), 2);
|
||||
|
||||
expectSpanEq<float, WindowSize>(
|
||||
frames[0],
|
||||
{ 0.F, 0.F, 0.F, 0.F, 0.F, 1.F, 2.F, 3.F });
|
||||
|
||||
expectSpanEq<float, WindowSize>(
|
||||
frames[1],
|
||||
{ 0.F, 1.F, 2.F, 3.F, 4.F, 5.F, 6.F, 7.F });
|
||||
}
|
||||
|
||||
TEST(PcmSpectralFrameDecoder, skipFramesAdvancesState)
|
||||
{
|
||||
FrameDecoder frameDecoder{ std::make_unique<SequencePcmDecoder>(32), HopSize };
|
||||
using Frame = std::array<float, WindowSize>;
|
||||
std::vector<Frame> frames;
|
||||
|
||||
EXPECT_EQ(frameDecoder.currentFrameIndex(), 0);
|
||||
ASSERT_EQ(frameDecoder.skipFrames(2), 2);
|
||||
EXPECT_EQ(frameDecoder.currentFrameIndex(), 2);
|
||||
|
||||
Frame lastFrame{};
|
||||
|
||||
ASSERT_EQ(frameDecoder.decodeFrames(1,
|
||||
[&](const FrameDecoder::SpectralFrameView& frame) {
|
||||
std::copy(std::cbegin(frame.rawSamples), std::cend(frame.rawSamples), std::begin(lastFrame));
|
||||
}),
|
||||
1);
|
||||
|
||||
expectSpanEq<float, WindowSize>(
|
||||
lastFrame,
|
||||
{ 4.F, 5.F, 6.F, 7.F, 8.F, 9.F, 10.F, 11.F });
|
||||
|
||||
EXPECT_EQ(frameDecoder.currentFrameIndex(), 3);
|
||||
}
|
||||
} // namespace lms::audio::tests
|
||||
Reference in New Issue
Block a user