owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' for release 3.7.0

Browse Source
This commit is contained in:
emeric
2020-02-21 15:45:06 +01:00
parent 8f6f044533 d02a7ca86b
commit 2e051e1fc5
223 changed files with 3065 additions and 2383 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+2 -4
View File
@@ -16,15 +16,13 @@ matrix:
- gcc
before_install:
- eval "${MATRIX_EVAL}"
- sudo apt-get install build-essential autoconf automake cmake libboost-all-dev libconfig++-dev libavcodec-dev libavutil-dev libavformat-dev ffmpeg libmagick++-dev libpstreams-dev libconfig++-dev libpstreams-dev libtag1-dev
- sudo apt-get install build-essential cmake libboost-all-dev libconfig++-dev libavcodec-dev libavutil-dev libavformat-dev ffmpeg libmagick++-dev libpstreams-dev libconfig++-dev libpstreams-dev libtag1-dev
- git clone https://github.com/emweb/wt.git wt
- pushd wt;
- git checkout 4.1.0
- cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr && sudo make install
- popd
script: autoreconf -vfi && CXXFLAGS="-Wall -Wextra -Werror" ./configure --enable-tools && make distcheck
script: cmake -DCMAKE_BUILD_TYPE=Release . && make && make test
env:
global:
- MAKEFLAGS="-j 2"
after_failure:
- cat config.log
CMakeLists.txt
+18
View File
@@ -0,0 +1,18 @@
cmake_minimum_required(VERSION 3.10)
project(lms)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED True)
include(CTest)
find_package(PkgConfig)
pkg_check_modules(IMAGEMAGICKXX REQUIRED ImageMagick++)
add_subdirectory(src)
install(DIRECTORY approot DESTINATION share/lms)
install(DIRECTORY docroot DESTINATION share/lms)
install(FILES systemd/default.service DESTINATION share/lms)
install(FILES conf/lms.conf DESTINATION share/lms)
Makefile.am
-56
View File
@@ -1,56 +0,0 @@
AUTOMAKE_OPTIONS = dist-bzip2 no-dist-gzip
SUBDIRS = test tools src
lms_docrootdir=$(pkgdatadir)/docroot
lms_approotdir=$(pkgdatadir)/approot
lms_cssdir=$(lms_docrootdir)/css
lms_jsdir=$(lms_docrootdir)/js
lms_imagesdir=$(lms_approotdir)/images
dist_pkgdata_DATA = \
systemd/default.service \
conf/lms.conf
dist_lms_css_DATA = \
docroot/css/lms.css
dist_lms_js_DATA = \
docroot/js/bootstrap-notify.js \
docroot/js/jquery-1.10.2.min.js \
docroot/js/mediaplayer.js
dist_lms_images_DATA = \
approot/images/unknown-cover.jpg \
approot/images/unknown-artist.jpg
dist_lms_approot_DATA = \
approot/admin-database.xml \
approot/admin-user.xml \
approot/admin-users.xml \
approot/admin-initwizard.xml \
approot/artist.xml \
approot/artistinfo.xml \
approot/artistlink.xml \
approot/artists.xml \
approot/artistsinfo.xml \
approot/error.xml \
approot/explore.xml \
approot/login.xml \
approot/mediaplayer.xml \
approot/messages.xml \
approot/messages_fr.xml \
approot/playhistory.xml \
approot/playqueue.xml \
approot/release.xml \
approot/releaseinfo.xml \
approot/releaselink.xml \
approot/releases.xml \
approot/releasesinfo.xml \
approot/settings.xml \
approot/templates.xml \
approot/tracks.xml \
approot/tracksinfo.xml
README.md
+18 -19
View File
@@ -17,28 +17,33 @@ A [demo](http://lms.demo.poupon.io) instance is available, with the following li
* Audio transcode for maximum interoperability and low bandwith requirements
* Persistent play queue across sessions
* Subsonic API
* Album artist
* Compilation support
* Multi-value tags: artists, genres, ...
* Custom tags (ex: _mood_, _genre_, _albummood_, _albumgrouping_, ...)
* MusicBrainzID support to handle duplicated artist and release names
* Playlists, (only using Subsonic API for now)
* Starred Album/Artist/Tracks (only using Subsonic API for now)
* _Systemd_ integration
* Subsonic-only features:
* Playlists
* Starred Album/Artist/Tracks
* Bookmarks
## Recommendation engine
## Music discovery
_LMS_ provides several ways to help you find the music you like:
* Tag-based filters (ex: _Rock_, _Metal_ and _Aggressive_, _Electronic_ and _Relaxed_, ...)
* Recommendations for similar artists and albums
* Radio mode
* Radio mode, based on what is in the current playqueue
* Searches in album, artist and track names
* Most played/Recently added music
The recommendation engine makes use of [Self-Organizing Maps](https://en.wikipedia.org/wiki/Self-organizing_map).</br>
__Notes__:
* constructing the map requires significant computation time on large collections (ex: half an hour for 40k tracks)
* audio data is pulled from [AcousticBrainz](https://acousticbrainz.org/). Therefore your music files must contain the [MusicBrainz Identifier](https://musicbrainz.org/doc/MusicBrainz_Identifier) for the recommendation engine to work properly (otherwise, only tag-based recommendations are provided)
* to use the _self-organizing map_ based engine, you have to enable it first in the settings panel.
The recommendation engine uses two different sources:
1. Tags that are present in the audio files
2. Acoustic similarities of the audio files, using a trained [Self-Organizing Map](https://en.wikipedia.org/wiki/Self-organizing_map)
__Notes on the self-organizing map__:
* training the map requires significant computation time on large collections (ex: half an hour for 40k tracks)
* audio acoustic data is pulled from [AcousticBrainz](https://acousticbrainz.org/). Therefore your audio files _must_ contain the [MusicBrainz Identifier](https://musicbrainz.org/doc/MusicBrainz_Identifier).
* to enable the audio similarity source, you have to enable it first in the settings panel.
## Subsonic API
The API version implemented is 1.12.0 and has been tested on _Android_ using the official application, _Ultrasonic_ and _DSub_.
@@ -80,7 +85,7 @@ __Notes__:
* a C++17 compiler is needed
* ffmpeg version 4 minimum is required
```sh
apt-get install g++ autoconf automake libboost-filesystem-dev libboost-system-dev libavutil-dev libavformat-dev libmagick++-dev libpstreams-dev libconfig++-dev libpstreams-dev ffmpeg libtag1-dev
apt-get install g++ cmake libboost-system-dev libavutil-dev libavformat-dev libmagick++-dev libconfig++-dev libpstreams-dev ffmpeg libtag1-dev
```
You also need _Wt4_, which is not packaged yet on _Debian_. See [installation instructions](https://www.webtoolkit.eu/wt/doc/reference/html/InstallationUnix.html).</br>
@@ -92,16 +97,10 @@ Get the latest stable release and build it:
```sh
git clone https://github.com/epoupon/lms.git lms
cd lms
autoreconf -vfi
mkdir build
cd build
../configure --prefix=/usr
cmake .. -DCMAKE_BUILD_TYPE=Release
```
configure will report any missing library.
__Note__: in order to customize the installation directories, you can use the following options of the `configure` script:
* _--prefix_ (defaults to `/usr/local`).
* _--bindir_ (defaults to `$PREFIX/bin`).
__Note__: in order to customize the installation directory, you can use the _-DCMAKE_INSTALL_PREFIX_ option (defaults to `/usr/local`).
```sh
make
approot/admin-database.xml
+4 -4
View File
@@ -47,14 +47,14 @@
<div class="form-horizontal">
<div class="form-group">
<label class="control-label col-sm-2" for="${id:similarity-engine-type}">
${tr:Lms.Admin.Database.similarity-engine-type}
<label class="control-label col-sm-2" for="${id:recommendation-engine-type}">
${tr:Lms.Admin.Database.recommendation-engine-type}
</label>
<div class="col-sm-5">
${similarity-engine-type}
${recommendation-engine-type}
</div>
<div class="help-block col-sm-5">
${similarity-engine-type-info}
${recommendation-engine-type-info}
</div>
</div>
approot/messages.xml
+3 -3
View File
@@ -39,9 +39,9 @@
<message id="Lms.Admin.Database.monthly">Monthly</message>
<message id="Lms.Admin.Database.never">Never</message>
<message id="Lms.Admin.Database.path">Media root directory</message>
<message id="Lms.Admin.Database.similarity-engine-type">Recommendation engine</message>
<message id="Lms.Admin.Database.similarity-engine-type.clusters">Tags based</message>
<message id="Lms.Admin.Database.similarity-engine-type.features">Audio analysis based</message>
<message id="Lms.Admin.Database.recommendation-engine-type">Recommendation engine</message>
<message id="Lms.Admin.Database.recommendation-engine-type.clusters">Tags based</message>
<message id="Lms.Admin.Database.recommendation-engine-type.features">Audio analysis based</message>
<message id="Lms.Admin.Database.scan-complete">Scan complete: {1} total files, {2} additions, {3} updates, {4} deletions, {5} duplicates, {6} errors</message>
<message id="Lms.Admin.Database.scan-launched">Scan launched!</message>
<message id="Lms.Admin.Database.scan-options">Scan options</message>
approot/messages_fr.xml
+3 -3
View File
@@ -39,9 +39,9 @@
<message id="Lms.Admin.Database.monthly">Tous les mois</message>
<message id="Lms.Admin.Database.never">Jamais</message>
<message id="Lms.Admin.Database.path">Dossier racine des fichiers de musique</message>
<message id="Lms.Admin.Database.similarity-engine-type">Moteur de recommandation</message>
<message id="Lms.Admin.Database.similarity-engine-type.clusters">Basé sur les tags</message>
<message id="Lms.Admin.Database.similarity-engine-type.features">Basé sur l'analyse audio</message>
<message id="Lms.Admin.Database.recommendation-engine-type">Moteur de recommandation</message>
<message id="Lms.Admin.Database.recommendation-engine-type.clusters">Basé sur les tags</message>
<message id="Lms.Admin.Database.recommendation-engine-type.features">Basé sur l'analyse audio</message>
<message id="Lms.Admin.Database.scan-complete">Scan terminé : {1} fichiers, {2} ajouts, {3} mises à jour, {4} suppressions, {5} duplicatas, {6} erreurs</message>
<message id="Lms.Admin.Database.scan-launched">Scan lancé !</message>
<message id="Lms.Admin.Database.scan-options">Options </message>
configure.ac
-94
View File
@@ -1,94 +0,0 @@
AC_PREREQ(2.59)
AC_INIT(lms, 3.6.3, test@test)
AM_INIT_AUTOMAKE
AC_CONFIG_HEADER(src/config/config.h)
AC_LANG_CPLUSPLUS
AC_PROG_CXX
PKG_CHECK_MODULES(IMAGEMAGICKXX, "ImageMagick++", [ HAVE_IMAGEMAGICKXX=yes ], [ ])
if test -n "$HAVE_IMAGEMAGICKXX"; then
MAGICKXX_CFLAGS="$IMAGEMAGICKXX_CFLAGS"
MAGICKXX_LIBS="$IMAGEMAGICKXX_LIBS"
fi
AC_SUBST(MAGICKXX_CFLAGS)
AC_SUBST(MAGICKXX_LIBS)
AC_CHECK_HEADERS([Wt/WApplication.h pstreams/pstream.h boost/asio.hpp],
[],
[AC_MSG_ERROR([Header not found or unusable !])])
AC_CHECK_LIB([stdc++fs],
[main],
,
[AC_MSG_ERROR([lib filesystem not found!])])
AC_CHECK_LIB([pthread],
[pthread_rwlock_unlock],
,
[AC_MSG_ERROR([libpthread not found!])])
AC_CHECK_LIB([boost_system],
[main],
,
[AC_MSG_ERROR([libboost_system not found!])])
AC_CHECK_LIB([wt],
[main],
,
[AC_MSG_ERROR([libwt not found!])])
AC_CHECK_LIB([wtdbo],
[main],
,
[AC_MSG_ERROR([libwtdbo not found!])])
AC_CHECK_LIB([wtdbosqlite3],
[main],
,
[AC_MSG_ERROR([libwtdbosqlite3 not found!])])
AC_CHECK_LIB([wthttp],
[main],
,
[AC_MSG_ERROR([libwthttp not found!])])
AC_CHECK_LIB([avutil],
[av_free],
,
[AC_MSG_ERROR([libavutil not found!])])
AC_CHECK_LIB([avformat],
[av_read_frame],
,
[AC_MSG_ERROR([libavformat not found!])])
AC_CHECK_LIB( [tag],
[main],
,
[AC_MSG_ERROR([libtag not found!])])
AC_CHECK_LIB( [config++],
[main],
,
[AC_MSG_ERROR([libconfig++ not found!])])
AC_CONFIG_FILES([Makefile
src/Makefile
test/Makefile
tools/Makefile
tools/similarity/Makefile
tools/similarity-parameters/Makefile
tools/metadata/Makefile])
AC_ARG_ENABLE([tools],
[AC_HELP_STRING([--enable-tools], [Build the tools])],
[:],
[enable_tools=no])
AM_CONDITIONAL([BUILD_TOOLS], [test "$enable_tools" = "yes"])
AC_OUTPUT
docker/Dockerfile
+5 -7
View File
@@ -7,7 +7,7 @@ ARG FFMPEG_VERSION=4.1.4
ARG WT_VERSION=4.2.0
ARG IMAGEMAGICK6_VERSION=6.9.10-71
ARG PSTREAMS_VERSION=1.0.1
ARG LMS_VERSION=3.5.0
ARG LMS_VERSION=v3.6.3
ARG PREFIX="/tmp/install"
@@ -153,23 +153,21 @@ RUN \
# LMS
RUN \
DIR=/tmp/lms && mkdir -p ${DIR} && cd ${DIR} && \
curl -sLO https://github.com/epoupon/lms/archive/v${LMS_VERSION}.tar.gz && \
tar -x --strip-components=1 -f v${LMS_VERSION}.tar.gz
curl -sL https://github.com/epoupon/lms/archive/${LMS_VERSION}.tar.gz -o ${LMS_VERSION}.tar.gz && \
tar -x --strip-components=1 -f ${LMS_VERSION}.tar.gz
RUN \
DIR=/tmp/lms && mkdir -p ${DIR} && cd ${DIR} && \
autoreconf -vfi && \
PKG_CONFIG_PATH=/tmp/install/lib/pkgconfig CXXFLAGS="-O2 -I${PREFIX}/include" LDFLAGS="-L${PREFIX}/lib -Wl,--rpath-link=${PREFIX}/lib" ./configure --prefix=${PREFIX} && \
PKG_CONFIG_PATH=/tmp/install/lib/pkgconfig CXXFLAGS="-I${PREFIX}/include -L${PREFIX}/lib -Wl,--rpath-link=${PREFIX}/lib" cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} && \
make && \
make install && \
make distclean && \
mkdir -p ${PREFIX}/etc/ && \
cp conf/lms.conf ${PREFIX}/etc
# Now copy all the stuff installed in a new folder (/tmp/fakeroot/)
RUN \
mkdir -p /tmp/fakeroot/bin && \
for bin in ${PREFIX}/bin/ffmpeg ${PREFIX}/bin/lms; \
for bin in ${PREFIX}/bin/ffmpeg ${PREFIX}/bin/lms*; \
do \
strip --strip-all $bin && \
cp $bin /tmp/fakeroot/bin/; \
src/CMakeLists.txt
+11
View File
@@ -0,0 +1,11 @@
add_compile_options(-Wall -Wextra -pedantic)
add_subdirectory(libs)
add_subdirectory(lms)
add_subdirectory(tools)
if(BUILD_TESTING)
add_subdirectory(test)
endif()
src/Makefile.am
-172
View File
@@ -1,172 +0,0 @@
bin_PROGRAMS = lms
lms_SOURCES = \
$(srcdir)/api/subsonic/SubsonicId.cpp \
$(srcdir)/api/subsonic/SubsonicId.hpp \
$(srcdir)/api/subsonic/SubsonicResource.cpp \
$(srcdir)/api/subsonic/SubsonicResource.hpp \
$(srcdir)/api/subsonic/SubsonicResponse.cpp \
$(srcdir)/api/subsonic/SubsonicResponse.hpp \
$(srcdir)/auth/AuthTokenService.cpp \
$(srcdir)/auth/AuthTokenService.hpp \
$(srcdir)/auth/PasswordService.cpp \
$(srcdir)/auth/PasswordService.hpp \
$(srcdir)/auth/LoginThrottler.cpp \
$(srcdir)/auth/LoginThrottler.hpp \
$(srcdir)/av/AvInfo.cpp \
$(srcdir)/av/AvInfo.hpp \
$(srcdir)/av/AvTranscoder.cpp \
$(srcdir)/av/AvTranscoder.hpp \
$(srcdir)/av/AvTypes.cpp \
$(srcdir)/av/AvTypes.hpp \
$(srcdir)/cover/CoverArtGrabber.cpp \
$(srcdir)/cover/CoverArtGrabber.hpp \
$(srcdir)/database/Artist.cpp \
$(srcdir)/database/Artist.hpp \
$(srcdir)/database/Cluster.cpp \
$(srcdir)/database/Cluster.hpp \
$(srcdir)/database/Db.cpp \
$(srcdir)/database/Db.hpp \
$(srcdir)/database/TrackArtistLink.cpp \
$(srcdir)/database/TrackArtistLink.hpp \
$(srcdir)/database/TrackFeatures.cpp \
$(srcdir)/database/TrackFeatures.hpp \
$(srcdir)/database/TrackList.cpp \
$(srcdir)/database/TrackList.hpp \
$(srcdir)/database/Types.hpp \
$(srcdir)/database/Release.cpp \
$(srcdir)/database/Release.hpp \
$(srcdir)/database/ScanSettings.cpp \
$(srcdir)/database/ScanSettings.hpp \
$(srcdir)/database/Session.cpp \
$(srcdir)/database/Session.hpp \
$(srcdir)/database/SessionPool.cpp \
$(srcdir)/database/SessionPool.hpp \
$(srcdir)/database/SqlQuery.cpp \
$(srcdir)/database/SqlQuery.hpp \
$(srcdir)/database/Track.cpp \
$(srcdir)/database/Track.hpp \
$(srcdir)/database/TrackBookmark.cpp \
$(srcdir)/database/TrackBookmark.hpp \
$(srcdir)/database/User.cpp \
$(srcdir)/database/User.hpp \
$(srcdir)/image/Image.cpp \
$(srcdir)/image/Image.hpp \
$(srcdir)/main/main.cpp \
$(srcdir)/metadata/AvFormat.cpp \
$(srcdir)/metadata/AvFormat.hpp \
$(srcdir)/metadata/MetaData.hpp \
$(srcdir)/metadata/TagLibParser.cpp \
$(srcdir)/metadata/TagLibParser.hpp \
$(srcdir)/scanner/MediaScanner.cpp \
$(srcdir)/scanner/MediaScanner.hpp \
$(srcdir)/scanner/MediaScannerStats.cpp \
$(srcdir)/scanner/MediaScannerStats.hpp \
$(srcdir)/scanner/MediaScannerAddon.hpp \
$(srcdir)/similarity/SimilaritySearcher.cpp \
$(srcdir)/similarity/SimilaritySearcher.hpp \
$(srcdir)/similarity/cluster/SimilarityClusterSearcher.cpp \
$(srcdir)/similarity/cluster/SimilarityClusterSearcher.hpp \
$(srcdir)/similarity/features/AcousticBrainzUtils.cpp \
$(srcdir)/similarity/features/AcousticBrainzUtils.hpp \
$(srcdir)/similarity/features/SimilarityFeaturesCache.cpp \
$(srcdir)/similarity/features/SimilarityFeaturesCache.hpp \
$(srcdir)/similarity/features/SimilarityFeaturesDefs.cpp \
$(srcdir)/similarity/features/SimilarityFeaturesDefs.hpp \
$(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.cpp \
$(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.hpp \
$(srcdir)/similarity/features/SimilarityFeaturesSearcher.cpp \
$(srcdir)/similarity/features/SimilarityFeaturesSearcher.hpp \
$(srcdir)/similarity/features/som/DataNormalizer.cpp \
$(srcdir)/similarity/features/som/DataNormalizer.hpp \
$(srcdir)/similarity/features/som/InputVector.hpp \
$(srcdir)/similarity/features/som/Matrix.hpp \
$(srcdir)/similarity/features/som/Network.cpp \
$(srcdir)/similarity/features/som/Network.hpp \
$(srcdir)/ui/Auth.cpp \
$(srcdir)/ui/Auth.hpp \
$(srcdir)/ui/LmsApplication.cpp \
$(srcdir)/ui/LmsApplication.hpp \
$(srcdir)/ui/LmsApplicationException.hpp \
$(srcdir)/ui/LmsApplicationGroup.cpp \
$(srcdir)/ui/LmsApplicationGroup.hpp \
$(srcdir)/ui/MediaPlayer.cpp \
$(srcdir)/ui/MediaPlayer.hpp \
$(srcdir)/ui/PlayQueueView.cpp \
$(srcdir)/ui/PlayQueueView.hpp \
$(srcdir)/ui/PlayHistoryView.cpp \
$(srcdir)/ui/PlayHistoryView.hpp \
$(srcdir)/ui/SettingsView.cpp \
$(srcdir)/ui/SettingsView.hpp \
$(srcdir)/ui/TrackStringUtils.cpp \
$(srcdir)/ui/TrackStringUtils.hpp \
$(srcdir)/ui/admin/DatabaseSettingsView.cpp \
$(srcdir)/ui/admin/DatabaseSettingsView.hpp \
$(srcdir)/ui/admin/DatabaseStatus.cpp \
$(srcdir)/ui/admin/DatabaseStatus.hpp \
$(srcdir)/ui/admin/InitWizardView.cpp \
$(srcdir)/ui/admin/InitWizardView.hpp \
$(srcdir)/ui/admin/UserView.cpp \
$(srcdir)/ui/admin/UserView.hpp \
$(srcdir)/ui/admin/UsersView.cpp \
$(srcdir)/ui/admin/UsersView.hpp \
$(srcdir)/ui/common/Validators.cpp \
$(srcdir)/ui/common/Validators.hpp \
$(srcdir)/ui/common/ValueStringModel.hpp \
$(srcdir)/ui/explore/ArtistInfoView.cpp \
$(srcdir)/ui/explore/ArtistInfoView.hpp \
$(srcdir)/ui/explore/ArtistLink.cpp \
$(srcdir)/ui/explore/ArtistLink.hpp \
$(srcdir)/ui/explore/ArtistsInfoView.cpp \
$(srcdir)/ui/explore/ArtistsInfoView.hpp \
$(srcdir)/ui/explore/ArtistView.cpp \
$(srcdir)/ui/explore/ArtistView.hpp \
$(srcdir)/ui/explore/ArtistsView.cpp \
$(srcdir)/ui/explore/ArtistsView.hpp \
$(srcdir)/ui/explore/Explore.cpp \
$(srcdir)/ui/explore/Explore.hpp \
$(srcdir)/ui/explore/Filters.cpp \
$(srcdir)/ui/explore/Filters.hpp \
$(srcdir)/ui/explore/ReleaseInfoView.cpp \
$(srcdir)/ui/explore/ReleaseInfoView.hpp \
$(srcdir)/ui/explore/ReleaseLink.cpp \
$(srcdir)/ui/explore/ReleaseLink.hpp \
$(srcdir)/ui/explore/ReleasesInfoView.cpp \
$(srcdir)/ui/explore/ReleasesInfoView.hpp \
$(srcdir)/ui/explore/ReleasesView.cpp \
$(srcdir)/ui/explore/ReleasesView.hpp \
$(srcdir)/ui/explore/ReleaseView.cpp \
$(srcdir)/ui/explore/ReleaseView.hpp \
$(srcdir)/ui/explore/TracksInfoView.cpp \
$(srcdir)/ui/explore/TracksInfoView.hpp \
$(srcdir)/ui/explore/TracksView.cpp \
$(srcdir)/ui/explore/TracksView.hpp \
$(srcdir)/ui/resource/ImageResource.cpp \
$(srcdir)/ui/resource/ImageResource.hpp \
$(srcdir)/ui/resource/AudioResource.cpp \
$(srcdir)/ui/resource/AudioResource.hpp \
$(srcdir)/utils/Config.cpp \
$(srcdir)/utils/Config.hpp \
$(srcdir)/utils/Exception.hpp \
$(srcdir)/utils/Logger.cpp \
$(srcdir)/utils/Logger.hpp \
$(srcdir)/utils/NetAddress.cpp \
$(srcdir)/utils/NetAddress.hpp \
$(srcdir)/utils/Path.cpp \
$(srcdir)/utils/Path.hpp \
$(srcdir)/utils/Random.cpp \
$(srcdir)/utils/Random.hpp \
$(srcdir)/utils/Service.hpp \
$(srcdir)/utils/StreamLogger.cpp \
$(srcdir)/utils/StreamLogger.hpp \
$(srcdir)/utils/String.cpp \
$(srcdir)/utils/String.hpp \
$(srcdir)/utils/Utils.hpp \
$(srcdir)/utils/UUID.cpp \
$(srcdir)/utils/UUID.hpp \
$(srcdir)/utils/WtLogger.cpp \
$(srcdir)/utils/WtLogger.hpp
lms_CXXFLAGS=-std=c++17 -I$(srcdir)/ui $(MAGICKXX_CFLAGS) -D_REENTRANT
lms_LDADD=$(MAGICKXX_LIBS)
src/config/.empty
View File
src/cover/CoverArtGrabber.hpp
-76
View File
@@ -1,76 +0,0 @@
/*
* Copyright (C) 2015 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <filesystem>
#include <map>
#include <mutex>
#include <optional>
#include <vector>
#include "database/Types.hpp"
#include "image/Image.hpp"
namespace Database {
class Session;
}
namespace CoverArt {
class Grabber
{
public:
Grabber();
Grabber(const Grabber&) = delete;
Grabber& operator=(const Grabber&) = delete;
Grabber(Grabber&&) = delete;
Grabber& operator=(Grabber&&) = delete;
void setDefaultCover(const std::filesystem::path& defaultCoverPath);
std::vector<uint8_t> getFromTrack(Database::Session& dbSession, Database::IdType trackId, Image::Format format, std::size_t size);
std::vector<uint8_t> getFromRelease(Database::Session& dbSession, Database::IdType releaseId, Image::Format format, std::size_t size);
private:
Image::Image getFromTrack(Database::Session& dbSession, Database::IdType trackId, std::size_t size);
Image::Image getFromRelease(Database::Session& dbSession, Database::IdType releaseId, std::size_t size);
std::optional<Image::Image> getFromTrack(const std::filesystem::path& path) const;
std::vector<std::filesystem::path> getCoverPaths(const std::filesystem::path& directoryPath) const;
std::optional<Image::Image> getFromDirectory(const std::filesystem::path& path) const;
Image::Image getDefaultCover(std::size_t size);
Image::Image _defaultCover;
std::mutex _mutex;
std::map<std::size_t /* size */, Image::Image> _defaultCovers;
static inline const std::vector<std::filesystem::path> _fileExtensions {".jpg", ".jpeg", ".png", ".bmp"}; // TODO parametrize
static inline const std::size_t _maxFileSize {10000000};
static inline const std::vector<std::filesystem::path> _preferredFileNames {"cover", "front"}; // TODO parametrize
};
} // namespace CoverArt
src/libs/CMakeLists.txt
+13
View File
@@ -0,0 +1,13 @@
add_subdirectory(auth)
add_subdirectory(av)
add_subdirectory(cover)
add_subdirectory(database)
add_subdirectory(metadata)
add_subdirectory(recommendation)
add_subdirectory(scanner)
add_subdirectory(som)
add_subdirectory(subsonic)
add_subdirectory(utils)
src/libs/auth/CMakeLists.txt
+28
View File
@@ -0,0 +1,28 @@
add_library(lmsauth SHARED
impl/AuthTokenService.cpp
impl/PasswordService.cpp
impl/LoginThrottler.cpp
)
target_include_directories(lmsauth INTERFACE
include
)
target_include_directories(lmsauth PRIVATE
include/
)
target_link_libraries(lmsauth PRIVATE
lmsutils
lmsdatabase
)
target_link_libraries(lmsauth PUBLIC
pthread
boost_system
wt
)
install(TARGETS lmsauth DESTINATION lib)
src/auth/AuthTokenService.cpp → src/libs/auth/impl/AuthTokenService.cpp
+6
View File
@@ -26,11 +26,17 @@
#include <Wt/WRandom.h>
#include "database/Session.hpp"
#include "database/User.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
namespace Auth {
std::unique_ptr<IAuthTokenService> createAuthTokenService(std::size_t maxThrottlerEntries)
{
return std::make_unique<AuthTokenService>(maxThrottlerEntries);
}
static const Wt::Auth::SHA1HashFunction sha1Function;
AuthTokenService::AuthTokenService(std::size_t maxThrottlerEntries)
src/auth/AuthTokenService.hpp → src/libs/auth/impl/AuthTokenService.hpp
+4 -31
View File
@@ -21,14 +21,9 @@
#pragma once
#include <optional>
#include <string>
#include <boost/asio/ip/address.hpp>
#include "auth/IAuthTokenService.hpp"
#include "LoginThrottler.hpp"
#include "database/User.hpp"
#include "database/Types.hpp"
namespace Database
{
@@ -38,7 +33,7 @@ namespace Database
namespace Auth {
class AuthTokenService
class AuthTokenService : public IAuthTokenService
{
public:
@@ -52,30 +47,8 @@ namespace Auth {
AuthTokenService(AuthTokenService&&) = delete;
AuthTokenService& operator=(AuthTokenService&&) = delete;
// Auth Token services
struct AuthTokenProcessResult
{
enum class State
{
Found,
Throttled,
NotFound,
};
struct AuthTokenInfo
{
Database::IdType userId;
Wt::WDateTime expiry;
};
State state {State::NotFound};
std::optional<AuthTokenInfo> authTokenInfo {};
};
// Removed if found
AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue);
std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry);
AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue) override;
std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry) override;
private:
src/auth/LoginThrottler.cpp → src/libs/auth/impl/LoginThrottler.cpp
View File
src/auth/LoginThrottler.hpp → src/libs/auth/impl/LoginThrottler.hpp
View File
src/auth/PasswordService.cpp → src/libs/auth/impl/PasswordService.cpp
+5
View File
@@ -31,6 +31,11 @@
namespace Auth {
std::unique_ptr<IPasswordService> createPasswordService(std::size_t maxThrottlerEntries)
{
return std::make_unique<PasswordService>(maxThrottlerEntries);
}
PasswordService::PasswordService(std::size_t maxThrottlerEntries)
: _loginThrottler{maxThrottlerEntries}
{
src/auth/PasswordService.hpp → src/libs/auth/impl/PasswordService.hpp
+6 -41
View File
@@ -17,17 +17,12 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
/* This file contains some classes in order to get info from file using the libavconv */
#pragma once
#include <string>
#include <boost/asio/ip/address.hpp>
#include <shared_mutex>
#include "auth/IPasswordService.hpp"
#include "LoginThrottler.hpp"
#include "database/User.hpp"
#include "database/Types.hpp"
namespace Database
{
@@ -37,7 +32,7 @@ namespace Database
namespace Auth {
class PasswordService
class PasswordService : public IPasswordService
{
public:
@@ -53,39 +48,9 @@ namespace Auth {
// Password services
enum class PasswordCheckResult
{
Match,
Mismatch,
Throttled,
};
PasswordCheckResult checkUserPassword(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& loginName, const std::string& password);
Database::User::PasswordHash hashPassword(const std::string& password) const;
bool evaluatePasswordStrength(const std::string& loginName, const std::string& password) const;
// Auth Token services
struct AuthTokenProcessResult
{
enum class State
{
Found,
Throttled,
NotFound,
};
struct AuthTokenInfo
{
Database::IdType userId;
Wt::WDateTime expiry;
};
State state;
std::optional<AuthTokenInfo> authTokenInfo;
};
// Removed if found
AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue);
std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry);
PasswordCheckResult checkUserPassword(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& loginName, const std::string& password) override;
Database::User::PasswordHash hashPassword(const std::string& password) const override;
bool evaluatePasswordStrength(const std::string& loginName, const std::string& password) const override;
private:
src/libs/auth/include/auth/IAuthTokenService.hpp
+71
View File
@@ -0,0 +1,71 @@
/*
* Copyright (C) 2019 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
/* This file contains some classes in order to get info from file using the libavconv */
#pragma once
#include <optional>
#include <string>
#include <boost/asio/ip/address.hpp>
#include <Wt/WDateTime.h>
#include "database/Types.hpp"
namespace Database
{
class Session;
}
namespace Auth {
class IAuthTokenService
{
public:
// Auth Token services
struct AuthTokenProcessResult
{
enum class State
{
Found,
Throttled,
NotFound,
};
struct AuthTokenInfo
{
Database::IdType userId;
Wt::WDateTime expiry;
};
State state {State::NotFound};
std::optional<AuthTokenInfo> authTokenInfo {};
};
// Removed if found
virtual AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue) = 0;
virtual std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry) = 0;
};
std::unique_ptr<IAuthTokenService> createAuthTokenService(std::size_t maxThrottlerEntryCount);
}
src/libs/auth/include/auth/IPasswordService.hpp
+59
View File
@@ -0,0 +1,59 @@
/*
* Copyright (C) 2019 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <optional>
#include <string>
#include <boost/asio/ip/address.hpp>
#include "database/User.hpp"
#include "database/Types.hpp"
namespace Database
{
class Session;
}
namespace Auth {
class IPasswordService
{
public:
virtual ~IPasswordService() = default;
// Password services
enum class PasswordCheckResult
{
Match,
Mismatch,
Throttled,
};
virtual PasswordCheckResult checkUserPassword(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& loginName, const std::string& password) = 0;
virtual Database::User::PasswordHash hashPassword(const std::string& password) const = 0;
virtual bool evaluatePasswordStrength(const std::string& loginName, const std::string& password) const = 0;
};
std::unique_ptr<IPasswordService> createPasswordService(std::size_t maxThrottlerEntryCount);
}
src/libs/av/CMakeLists.txt
+24
View File
@@ -0,0 +1,24 @@
add_library(lmsav SHARED
impl/AvInfo.cpp
impl/AvTranscoder.cpp
impl/AvTypes.cpp
)
target_include_directories(lmsav INTERFACE
include
)
target_include_directories(lmsav PRIVATE
include/
)
# TODO make these private
target_link_libraries(lmsav PUBLIC
lmsutils
avformat
avutil
)
install(TARGETS lmsav DESTINATION lib)
src/av/AvInfo.cpp → src/libs/av/impl/AvInfo.cpp
+3 -5
View File
@@ -17,9 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "AvInfo.hpp"
#include <boost/algorithm/string.hpp>
#include "av/AvInfo.hpp"
#include <array>
@@ -92,7 +90,7 @@ getMetaDataFromDictionnary(AVDictionary* dictionnary, std::map<std::string, std:
AVDictionaryEntry *tag = NULL;
while ((tag = av_dict_get(dictionnary, "", tag, AV_DICT_IGNORE_SUFFIX)))
{
res[boost::to_upper_copy<std::string>(tag->key)] = tag->value;
res[StringUtils::stringToUpper(tag->key)] = tag->value;
}
}
@@ -141,7 +139,7 @@ MediaFile::getStreamInfo() const
if (avstream->codecpar->codec_type != AVMEDIA_TYPE_AUDIO)
continue;
res.push_back( {.id = i, .bitrate = static_cast<std::size_t>(avstream->codecpar->bit_rate)} );
res.push_back( {i, static_cast<std::size_t>(avstream->codecpar->bit_rate)} );
}
return res;
src/av/AvTranscoder.cpp → src/libs/av/impl/AvTranscoder.cpp
+4 -4
View File
@@ -17,13 +17,13 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "AvTranscoder.hpp"
#include "av/AvTranscoder.hpp"
#include <atomic>
#include <mutex>
#include "AvInfo.hpp"
#include "utils/Config.hpp"
#include "av/AvInfo.hpp"
#include "utils/IConfig.hpp"
#include "utils/Path.hpp"
#include "utils/Logger.hpp"
#include "utils/Service.hpp"
@@ -38,7 +38,7 @@ static std::filesystem::path ffmpegPath;
void
Transcoder::init()
{
ffmpegPath = ServiceProvider<Config>::get()->getPath("ffmpeg-file", "/usr/bin/ffmpeg");
ffmpegPath = ServiceProvider<IConfig>::get()->getPath("ffmpeg-file", "/usr/bin/ffmpeg");
if (!std::filesystem::exists(ffmpegPath))
throw LmsException {"File '" + ffmpegPath.string() + "' does not exist!"};
}
src/av/AvTypes.cpp → src/libs/av/impl/AvTypes.cpp
+2 -4
View File
@@ -17,9 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "AvTypes.hpp"
#include <map>
#include "av/AvTypes.hpp"
namespace Av {
@@ -34,7 +32,7 @@ const char* encodingToMimetype(Encoding encoding)
case Encoding::WEBM_VORBIS: return "audio/webm";
}
throw AvException("Invalid encoding");
throw AvException {"Invalid encoding"};
}
}
src/av/AvInfo.hpp → src/libs/av/include/av/AvInfo.hpp
View File
src/av/AvTranscoder.hpp → src/libs/av/include/av/AvTranscoder.hpp
View File
src/av/AvTypes.hpp → src/libs/av/include/av/AvTypes.hpp
View File
src/libs/cover/CMakeLists.txt
+27
View File
@@ -0,0 +1,27 @@
add_library(lmscover SHARED
impl/CoverArtGrabber.cpp
impl/Image.cpp
)
target_include_directories(lmscover INTERFACE
include
)
target_include_directories(lmscover PRIVATE
include
${IMAGEMAGICKXX_INCLUDE_DIRS}
)
target_compile_options(lmscover PRIVATE
${IMAGEMAGICKXX_CFLAGS_OTHER}
)
target_link_libraries(lmscover PRIVATE
lmsav
lmsdatabase
${IMAGEMAGICKXX_LIBRARIES}
)
install(TARGETS lmscover DESTINATION lib)
src/cover/CoverArtGrabber.cpp → src/libs/cover/impl/CoverArtGrabber.cpp
+33 -22
View File
@@ -39,8 +39,19 @@ isFileSupported(const std::filesystem::path& file, const std::vector<std::filesy
namespace CoverArt {
Grabber::Grabber()
std::unique_ptr<IGrabber> createGrabber(const std::filesystem::path& execPath)
{
return std::make_unique<Grabber>(execPath);
}
Grabber::Grabber(const std::filesystem::path& execPath)
{
init(execPath);
}
Grabber::~Grabber()
{
deinit();
}
void
@@ -50,7 +61,7 @@ Grabber::setDefaultCover(const std::filesystem::path& p)
throw LmsException("Cannot read default cover file '" + p.string() + "'");
}
Image::Image
Image
Grabber::getDefaultCover(std::size_t size)
{
LMS_LOG(COVER, DEBUG) << "Getting a default cover using size = " << size;
@@ -59,12 +70,12 @@ Grabber::getDefaultCover(std::size_t size)
auto it = _defaultCovers.find(size);
if (it == _defaultCovers.end())
{
Image::Image cover = _defaultCover;
Image cover = _defaultCover;
LMS_LOG(COVER, DEBUG) << "default cover size = " << cover.getSize().width << " x " << cover.getSize().height;
LMS_LOG(COVER, DEBUG) << "Scaling cover to size = " << size;
cover.scale(Image::Geometry{size, size});
cover.scale(Geometry{size, size});
LMS_LOG(COVER, DEBUG) << "Scaling DONE";
auto res = _defaultCovers.insert(std::make_pair(size, cover));
assert(res.second);
@@ -74,14 +85,14 @@ Grabber::getDefaultCover(std::size_t size)
return it->second;
}
static std::optional<Image::Image>
static std::optional<Image>
getFromAvMediaFile(const Av::MediaFile& input)
{
std::vector<Image::Image> res;
std::vector<Image> res;
for (auto& picture : input.getAttachedPictures(2))
{
Image::Image image;
Image image;
if (image.load(picture.data))
return image;
@@ -93,12 +104,12 @@ getFromAvMediaFile(const Av::MediaFile& input)
return std::nullopt;
}
std::optional<Image::Image>
std::optional<Image>
Grabber::getFromDirectory(const std::filesystem::path& p) const
{
for (auto coverPath : getCoverPaths(p))
{
Image::Image image;
Image image;
if (image.load(coverPath))
return image;
@@ -143,7 +154,7 @@ Grabber::getCoverPaths(const std::filesystem::path& directoryPath) const
return res;
}
std::optional<Image::Image>
std::optional<Image>
Grabber::getFromTrack(const std::filesystem::path& p) const
{
try
@@ -159,12 +170,12 @@ Grabber::getFromTrack(const std::filesystem::path& p) const
}
}
Image::Image
Image
Grabber::getFromTrack(Database::Session& dbSession, Database::IdType trackId, std::size_t size)
{
using namespace Database;
std::optional<Image::Image> cover;
std::optional<Image> cover;
bool hasCover {};
bool isMultiDisc {};
@@ -200,16 +211,16 @@ Grabber::getFromTrack(Database::Session& dbSession, Database::IdType trackId, st
if (!cover)
cover = getDefaultCover(size);
else
cover->scale(Image::Geometry {size, size});
cover->scale(Geometry {size, size});
return *cover;
}
Image::Image
Image
Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, std::size_t size)
{
std::optional<Image::Image> cover;
std::optional<Image> cover;
std::optional<Database::IdType> trackId;
{
@@ -230,26 +241,26 @@ Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId,
if (!cover)
cover = getDefaultCover(size);
else
cover->scale(Image::Geometry {size, size});
cover->scale(Geometry {size, size});
return *cover;
}
std::vector<uint8_t>
Grabber::getFromTrack(Database::Session& session, Database::IdType trackId, Image::Format format, std::size_t size)
Grabber::getFromTrack(Database::Session& session, Database::IdType trackId, Format format, std::size_t width)
{
const Image::Image cover {getFromTrack(session, trackId, size)};
const Image cover {getFromTrack(session, trackId, width)};
assert(format == Image::Format::JPEG);
assert(format == Format::JPEG);
return cover.save(format);
}
std::vector<uint8_t>
Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, Image::Format format, std::size_t size)
Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, Format format, std::size_t width)
{
const Image::Image cover {getFromRelease(session, releaseId, size)};
const Image cover {getFromRelease(session, releaseId, width)};
assert(format == Image::Format::JPEG);
assert(format == Format::JPEG);
return cover.save(format);
}
src/libs/cover/impl/CoverArtGrabber.hpp
+77
View File
@@ -0,0 +1,77 @@
/*
* Copyright (C) 2015 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <filesystem>
#include <map>
#include <mutex>
#include <optional>
#include <vector>
#include "cover/ICoverArtGrabber.hpp"
#include "database/Types.hpp"
#include "Image.hpp"
namespace Database
{
class Session;
}
namespace CoverArt
{
class Grabber : public IGrabber
{
public:
Grabber(const std::filesystem::path& execPath);
~Grabber();
Grabber(const Grabber&) = delete;
Grabber& operator=(const Grabber&) = delete;
Grabber(Grabber&&) = delete;
Grabber& operator=(Grabber&&) = delete;
void setDefaultCover(const std::filesystem::path& defaultCoverPath) override;
std::vector<uint8_t> getFromTrack(Database::Session& dbSession, Database::IdType trackId, Format format, std::size_t width) override;
std::vector<uint8_t> getFromRelease(Database::Session& dbSession, Database::IdType releaseId, Format format, std::size_t width) override;
private:
Image getFromTrack(Database::Session& dbSession, Database::IdType trackId, std::size_t size);
Image getFromRelease(Database::Session& dbSession, Database::IdType releaseId, std::size_t size);
std::optional<Image> getFromTrack(const std::filesystem::path& path) const;
std::vector<std::filesystem::path> getCoverPaths(const std::filesystem::path& directoryPath) const;
std::optional<Image> getFromDirectory(const std::filesystem::path& path) const;
Image getDefaultCover(std::size_t size);
Image _defaultCover;
std::mutex _mutex;
std::map<std::size_t /* size */, Image> _defaultCovers;
static inline const std::vector<std::filesystem::path> _fileExtensions {".jpg", ".jpeg", ".png", ".bmp"}; // TODO parametrize
static inline const std::size_t _maxFileSize {10000000};
static inline const std::vector<std::filesystem::path> _preferredFileNames {"cover", "front"}; // TODO parametrize
};
} // namespace CoverArt
src/image/Image.cpp → src/libs/cover/impl/Image.cpp
+22 -12
View File
@@ -21,10 +21,23 @@
#include "utils/Logger.hpp"
namespace Image {
namespace CoverArt {
void
init(const std::filesystem::path& path)
{
Magick::InitializeMagick(path.string().c_str());
}
void
deinit()
{
MagickCore::MagickCoreTerminus();
}
static
std::string format_to_magick(Format format)
std::string
formatToMagick(Format format)
{
switch (format)
{
@@ -34,7 +47,8 @@ std::string format_to_magick(Format format)
return "JPEG";
}
std::string format_to_mimeType(Format format)
std::string
formatToMimeType(Format format)
{
switch (format)
{
@@ -44,18 +58,13 @@ std::string format_to_mimeType(Format format)
return "application/octet-stream";
}
void
init(const char *path)
{
Magick::InitializeMagick(path);
}
bool
Image::load(const std::vector<unsigned char>& rawData)
{
try
{
Magick::Blob blob(&rawData[0], rawData.size());
Magick::Blob blob {&rawData[0], rawData.size()};
_image.read(blob);
return true;
@@ -116,9 +125,9 @@ Image::save(Format format) const
try
{
Magick::Image outputImage(_image);
Magick::Image outputImage {_image};
outputImage.magick( format_to_magick(format));
outputImage.magick(formatToMagick(format));
Magick::Blob blob;
outputImage.write(&blob);
@@ -135,4 +144,5 @@ Image::save(Format format) const
}
}
} // namespace Image
} // namespace CoverArt
src/image/Image.hpp → src/libs/cover/impl/Image.hpp
+20 -30
View File
@@ -24,44 +24,34 @@
#include <Magick++.h>
namespace Image
#include "cover/CoverArt.hpp"
namespace CoverArt
{
enum class Format
{
JPEG,
};
void init(const std::filesystem::path& path);
void deinit();
std::string format_to_mimeType(Format format);
class Image
{
public:
void init(const char *path);
// input
bool load(const std::vector<unsigned char>& rawData);
bool load(const std::filesystem::path& p);
struct Geometry
{
std::size_t width;
std::size_t height;
};
Geometry getSize() const;
class Image
{
public:
// Operations
bool scale(Geometry geometry);
// input
bool load(const std::vector<unsigned char>& rawData);
bool load(const std::filesystem::path& p);
// output
std::vector<uint8_t> save(Format format) const;
Geometry getSize() const;
// Operations
bool scale(Geometry geometry);
// output
std::vector<uint8_t> save(Format format) const;
private:
Magick::Image _image;
};
private:
Magick::Image _image;
};
} // namespace Image
} // namespace CoverArt
src/scanner/MediaScannerAddon.hpp → src/libs/cover/include/cover/CoverArt.hpp
+15 -16
View File
@@ -1,5 +1,6 @@
/*
* Copyright (C) 2018 Emeric Poupon
* Copyright (C) 2015 Emeric Poupon
*
* This file is part of LMS.
*
@@ -19,23 +20,21 @@
#pragma once
#include "database/Types.hpp"
#include <string>
namespace Scanner {
class MediaScannerAddon
namespace CoverArt
{
public:
virtual void refreshSettings() = 0;
virtual void requestStop() = 0;
virtual void preScanComplete() = 0;
enum class Format
{
JPEG,
};
std::string formatToMimeType(Format format);
virtual void trackAdded(Database::IdType trackId) = 0;
virtual void trackToRemove(Database::IdType trackId) = 0;
virtual void trackUpdated(Database::IdType trackId) = 0;
};
} // ns Scanner
struct Geometry
{
std::size_t width;
std::size_t height;
};
}
src/similarity/cluster/SimilarityClusterSearcher.hpp → src/libs/cover/include/cover/ICoverArtGrabber.hpp
+17 -9
View File
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018 Emeric Poupon
* Copyright (C) 2015 Emeric Poupon
*
* This file is part of LMS.
*
@@ -19,22 +19,30 @@
#pragma once
#include <set>
#include <filesystem>
#include <vector>
#include "database/Types.hpp"
#include "cover/CoverArt.hpp"
namespace Database {
class Session;
}
namespace Similarity {
namespace CoverArt {
namespace ClusterSearcher
class IGrabber
{
std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount);
std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::set<Database::IdType>& tracksId, std::size_t maxCount);
std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount);
std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount);
public:
virtual ~IGrabber() = default;
virtual void setDefaultCover(const std::filesystem::path& defaultCoverPath) = 0;
virtual std::vector<uint8_t> getFromTrack(Database::Session& dbSession, Database::IdType trackId, Format format, std::size_t width) = 0;
virtual std::vector<uint8_t> getFromRelease(Database::Session& dbSession, Database::IdType releaseId, Format format, std::size_t width) = 0;
};
} // namespace Similarity
std::unique_ptr<IGrabber> createGrabber(const std::filesystem::path& execPath);
} // namespace CoverArt
src/libs/database/CMakeLists.txt
+37
View File
@@ -0,0 +1,37 @@
add_library(lmsdatabase SHARED
impl/Artist.cpp
impl/Cluster.cpp
impl/Db.cpp
impl/TrackArtistLink.cpp
impl/TrackFeatures.cpp
impl/TrackList.cpp
impl/Release.cpp
impl/ScanSettings.cpp
impl/Session.cpp
impl/SessionPool.cpp
impl/SqlQuery.cpp
impl/Track.cpp
impl/TrackBookmark.cpp
impl/User.cpp
)
target_include_directories(lmsdatabase INTERFACE
include
)
target_include_directories(lmsdatabase PRIVATE
include
)
target_link_libraries(lmsdatabase PRIVATE
wtdbosqlite3
)
target_link_libraries(lmsdatabase PUBLIC
lmsutils
wtdbo
)
install(TARGETS lmsdatabase DESTINATION lib)
src/database/Artist.cpp → src/libs/database/impl/Artist.cpp
+22 -7
View File
@@ -16,18 +16,18 @@
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Artist.hpp"
#include "database/Artist.hpp"
#include <Wt/Dbo/WtSqlTraits.h>
#include "database/Cluster.hpp"
#include "database/Release.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "database/User.hpp"
#include "utils/Logger.hpp"
#include "Cluster.hpp"
#include "Release.hpp"
#include "SqlQuery.hpp"
#include "Session.hpp"
#include "Track.hpp"
#include "User.hpp"
namespace Database
{
@@ -104,6 +104,21 @@ Artist::getAllOrphans(Session& session)
return std::vector<pointer>(res.begin(), res.end());
}
std::vector<IdType>
Artist::getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit)
{
session.checkSharedLocked();
Wt::Dbo::collection<IdType> res = session.getDboSession().query<IdType>
("SELECT DISTINCT a.id FROM artist a"
" INNER JOIN track t ON t.id = t_a_l.track_id INNER JOIN track_artist_link t_a_l ON t_a_l.artist_id = a.id"
" INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
.limit(limit ? static_cast<int>(*limit) : -1);
return std::vector<IdType>(res.begin(), res.end());
}
static
Wt::Dbo::Query<Artist::pointer>
getQuery(Session& session,
src/database/Cluster.cpp → src/libs/database/impl/Cluster.cpp
+6 -6
View File
@@ -17,14 +17,14 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Cluster.hpp"
#include "database/Cluster.hpp"
#include "Artist.hpp"
#include "Release.hpp"
#include "ScanSettings.hpp"
#include "Session.hpp"
#include "database/Artist.hpp"
#include "database/Release.hpp"
#include "database/ScanSettings.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "SqlQuery.hpp"
#include "Track.hpp"
namespace Database {
src/database/Db.cpp → src/libs/database/impl/Db.cpp
+2 -2
View File
@@ -17,13 +17,13 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Db.hpp"
#include "database/Db.hpp"
#include <Wt/Dbo/FixedSqlConnectionPool.h>
#include <Wt/Dbo/backend/Sqlite3.h>
#include "database/User.hpp"
#include "utils/Logger.hpp"
#include "User.hpp"
namespace Database {
src/database/Release.cpp → src/libs/database/impl/Release.cpp
+21 -6
View File
@@ -17,16 +17,16 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Release.hpp"
#include "database/Release.hpp"
#include "utils/Logger.hpp"
#include "Artist.hpp"
#include "Cluster.hpp"
#include "Session.hpp"
#include "database/Artist.hpp"
#include "database/Cluster.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "database/User.hpp"
#include "SqlQuery.hpp"
#include "Track.hpp"
#include "User.hpp"
namespace Database
{
@@ -252,6 +252,21 @@ Release::getByFilter(Session& session,
return res;
}
std::vector<IdType>
Release::getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit)
{
session.checkSharedLocked();
Wt::Dbo::collection<IdType> res = session.getDboSession().query<IdType>
("SELECT DISTINCT r.id FROM release r"
" INNER JOIN track t ON t.release_id = r.id"
" INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
.limit(limit ? static_cast<int>(*limit) : -1);
return std::vector<IdType>(res.begin(), res.end());
}
std::optional<std::size_t>
Release::getTotalTrackNumber(void) const
{
src/database/ScanSettings.cpp → src/libs/database/impl/ScanSettings.cpp
+6 -5
View File
@@ -17,15 +17,16 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanSettings.hpp"
#include "database/ScanSettings.hpp"
#include <Wt/Dbo/WtSqlTraits.h>
#include "utils/Path.hpp"
#include "utils/Logger.hpp"
#include "utils/String.hpp"
#include "Cluster.hpp"
#include "Session.hpp"
#include "database/Cluster.hpp"
#include "database/Session.hpp"
namespace {
@@ -62,11 +63,11 @@ ScanSettings::get(Session& session)
return session.getDboSession().find<ScanSettings>();
}
std::set<std::filesystem::path>
std::unordered_set<std::filesystem::path>
ScanSettings::getAudioFileExtensions() const
{
auto extensions = StringUtils::splitString(_audioFileExtensions, " ");
return std::set<std::filesystem::path>(std::cbegin(extensions), std::cend(extensions));
return std::unordered_set<std::filesystem::path>(std::cbegin(extensions), std::cend(extensions));
}
void
src/database/Session.cpp → src/libs/database/impl/Session.cpp
+13 -13
View File
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Session.hpp"
#include "database/Session.hpp"
#include <map>
#include <mutex>
@@ -26,17 +26,17 @@
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "Artist.hpp"
#include "Cluster.hpp"
#include "Db.hpp"
#include "Release.hpp"
#include "ScanSettings.hpp"
#include "Track.hpp"
#include "TrackBookmark.hpp"
#include "TrackArtistLink.hpp"
#include "TrackList.hpp"
#include "TrackFeatures.hpp"
#include "User.hpp"
#include "database/Artist.hpp"
#include "database/Cluster.hpp"
#include "database/Db.hpp"
#include "database/Release.hpp"
#include "database/ScanSettings.hpp"
#include "database/Track.hpp"
#include "database/TrackBookmark.hpp"
#include "database/TrackArtistLink.hpp"
#include "database/TrackList.hpp"
#include "database/TrackFeatures.hpp"
#include "database/User.hpp"
namespace Database {
@@ -118,7 +118,7 @@ Session::doDatabaseMigrationIfNeeded()
{
_session.execute("DROP TABLE similarity_settings");
_session.execute("DROP TABLE similarity_settings_feature");
_session.execute("ALTER TABLE scan_settings ADD similarity_engine_type INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast<int>(ScanSettings::SimilarityEngineType::Clusters)) + ")");
_session.execute("ALTER TABLE scan_settings ADD similarity_engine_type INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast<int>(ScanSettings::RecommendationEngineType::Clusters)) + ")");
}
else if (version == 8)
{
src/database/SessionPool.cpp → src/libs/database/impl/SessionPool.cpp
+3 -3
View File
@@ -17,13 +17,13 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "SessionPool.hpp"
#include "database/SessionPool.hpp"
#include "database/Session.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "Session.hpp"
namespace Database {
SessionPool::SessionPool(Db& database, std::size_t maxSessionCount)
src/database/SqlQuery.cpp → src/libs/database/impl/SqlQuery.cpp
View File
src/database/SqlQuery.hpp → src/libs/database/impl/SqlQuery.hpp
View File
src/database/Track.cpp → src/libs/database/impl/Track.cpp
+20 -7
View File
@@ -17,17 +17,17 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Track.hpp"
#include "database/Track.hpp"
#include <Wt/Dbo/WtSqlTraits.h>
#include "database/Artist.hpp"
#include "database/Cluster.hpp"
#include "database/Release.hpp"
#include "database/TrackFeatures.hpp"
#include "database/Session.hpp"
#include "utils/Logger.hpp"
#include "Artist.hpp"
#include "Cluster.hpp"
#include "Release.hpp"
#include "TrackFeatures.hpp"
#include "Session.hpp"
#include "SqlQuery.hpp"
namespace Database {
@@ -163,6 +163,19 @@ Track::getAllIdsWithFeatures(Session& session, std::optional<std::size_t> limit)
return std::vector<IdType>(res.begin(), res.end());
}
std::vector<IdType>
Track::getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit)
{
session.checkSharedLocked();
Wt::Dbo::collection<IdType> res = session.getDboSession().query<IdType>
("SELECT DISTINCT t.id FROM track t"
" INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
.limit(limit ? static_cast<int>(*limit) : -1);
return std::vector<IdType>(res.begin(), res.end());
}
std::vector<Cluster::pointer>
Track::getClusters(void) const
{
@@ -263,7 +276,7 @@ Track::getByFilter(Session& session,
std::vector<Track::pointer>
Track::getSimilarTracks(Session& session,
const std::set<IdType>& tracks,
const std::unordered_set<IdType>& tracks,
std::optional<std::size_t> offset,
std::optional<std::size_t> size)
{
src/database/TrackArtistLink.cpp → src/libs/database/impl/TrackArtistLink.cpp
+4 -4
View File
@@ -17,11 +17,11 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "TrackArtistLink.hpp"
#include "database/TrackArtistLink.hpp"
#include "Artist.hpp"
#include "Session.hpp"
#include "Track.hpp"
#include "database/Artist.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
namespace Database {
src/database/TrackBookmark.cpp → src/libs/database/impl/TrackBookmark.cpp
+4 -4
View File
@@ -17,11 +17,11 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "TrackBookmark.hpp"
#include "database/TrackBookmark.hpp"
#include "Session.hpp"
#include "Track.hpp"
#include "User.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "database/User.hpp"
namespace Database {
src/database/TrackFeatures.cpp → src/libs/database/impl/TrackFeatures.cpp
+3 -3
View File
@@ -17,14 +17,14 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "TrackFeatures.hpp"
#include "database/TrackFeatures.hpp"
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "utils/Logger.hpp"
#include "Session.hpp"
#include "Track.hpp"
namespace Database {
src/database/TrackList.cpp → src/libs/database/impl/TrackList.cpp
+7 -8
View File
@@ -16,19 +16,18 @@
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "TrackList.hpp"
#include "database/TrackList.hpp"
#include <cassert>
#include <random>
#include "utils/Logger.hpp"
#include "Artist.hpp"
#include "Cluster.hpp"
#include "Release.hpp"
#include "Session.hpp"
#include "User.hpp"
#include "Track.hpp"
#include "database/Artist.hpp"
#include "database/Cluster.hpp"
#include "database/Release.hpp"
#include "database/Session.hpp"
#include "database/User.hpp"
#include "database/Track.hpp"
namespace Database {
src/database/User.cpp → src/libs/database/impl/User.cpp
+6 -6
View File
@@ -17,14 +17,14 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "User.hpp"
#include "database/User.hpp"
#include "database/Artist.hpp"
#include "database/Release.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "database/TrackList.hpp"
#include "utils/Logger.hpp"
#include "Artist.hpp"
#include "Release.hpp"
#include "Session.hpp"
#include "Track.hpp"
#include "TrackList.hpp"
namespace Database {
src/database/Artist.hpp → src/libs/database/include/database/Artist.hpp
+1
View File
@@ -68,6 +68,7 @@ class Artist : public Wt::Dbo::Dbo<Artist>
static std::vector<IdType> getAllIds(Session& session);
static std::vector<pointer> getAllOrphans(Session& session); // No track related
static std::vector<pointer> getLastAdded(Session& session, Wt::WDateTime after, std::optional<std::size_t> size = {});
static std::vector<IdType> getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit = {});
// Accessors
const std::string& getName(void) const { return _name; }
src/database/Cluster.hpp → src/libs/database/include/database/Cluster.hpp
View File
src/database/Db.hpp → src/libs/database/include/database/Db.hpp
View File
src/database/Release.hpp → src/libs/database/include/database/Release.hpp
+1
View File
@@ -66,6 +66,7 @@ class Release : public Wt::Dbo::Dbo<Release>
std::optional<std::size_t> offset,
std::optional<std::size_t> size,
bool& moreExpected);
static std::vector<IdType> getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit = {});
std::vector<Wt::Dbo::ptr<Track>> getTracks(const std::set<IdType>& clusters = std::set<IdType>()) const;
std::size_t getTracksCount() const;
src/database/ScanSettings.hpp → src/libs/database/include/database/ScanSettings.hpp
+9 -7
View File
@@ -19,11 +19,13 @@
#pragma once
#include <filesystem>
#include <unordered_set>
#include <Wt/Dbo/Dbo.h>
#include <Wt/WTime.h>
#include "utils/Path.hpp"
namespace Database {
class ClusterType;
@@ -43,7 +45,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
};
// Do not modify values (just add)
enum class SimilarityEngineType
enum class RecommendationEngineType
{
Clusters = 0,
Features,
@@ -59,8 +61,8 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
Wt::WTime getUpdateStartTime() const { return _startTime; }
UpdatePeriod getUpdatePeriod() const { return _updatePeriod; }
std::vector<Wt::Dbo::ptr<ClusterType>> getClusterTypes() const;
std::set<std::filesystem::path> getAudioFileExtensions() const;
SimilarityEngineType getSimilarityEngineType() const { return _similarityEngineType; }
std::unordered_set<std::filesystem::path> getAudioFileExtensions() const;
RecommendationEngineType getRecommendationEngineType() const { return _recommendationEngineType; }
// Setters
void addAudioFileExtension(const std::filesystem::path& ext);
@@ -68,7 +70,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
void setUpdateStartTime(Wt::WTime t) { _startTime = t; }
void setUpdatePeriod(UpdatePeriod p) { _updatePeriod = p; }
void setClusterTypes(Session& session, const std::set<std::string>& clusterTypeNames);
void setSimilarityEngineType(SimilarityEngineType type) { _similarityEngineType = type; }
void setRecommendationEngineType(RecommendationEngineType type) { _recommendationEngineType = type; }
void incScanVersion();
template<class Action>
@@ -79,7 +81,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
Wt::Dbo::field(a, _startTime, "start_time");
Wt::Dbo::field(a, _updatePeriod, "update_period");
Wt::Dbo::field(a, _audioFileExtensions, "audio_file_extensions");
Wt::Dbo::field(a, _similarityEngineType,"similarity_engine_type");
Wt::Dbo::field(a, _recommendationEngineType,"similarity_engine_type");
Wt::Dbo::hasMany(a, _clusterTypes, Wt::Dbo::ManyToOne, "scan_settings");
}
@@ -89,7 +91,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
std::string _mediaDirectory;
Wt::WTime _startTime = Wt::WTime {0,0,0};
UpdatePeriod _updatePeriod {UpdatePeriod::Never};
SimilarityEngineType _similarityEngineType {SimilarityEngineType::Clusters};
RecommendationEngineType _recommendationEngineType {RecommendationEngineType::Clusters};
std::string _audioFileExtensions {".alac .mp3 .ogg .oga .aac .m4a .m4b .flac .wav .wma .aif .aiff .ape .mpc .shn .opus"};
Wt::Dbo::collection<Wt::Dbo::ptr<ClusterType>> _clusterTypes;
};
src/database/Session.hpp → src/libs/database/include/database/Session.hpp
+2 -2
View File
@@ -67,8 +67,8 @@ class Session
Session& operator=(const Session&) = delete;
Session& operator=(Session&&) = delete;
UniqueTransaction createUniqueTransaction();
SharedTransaction createSharedTransaction();
[[nodiscard]] UniqueTransaction createUniqueTransaction();
[[nodiscard]] SharedTransaction createSharedTransaction();
void checkUniqueLocked();
void checkSharedLocked();
src/database/SessionPool.hpp → src/libs/database/include/database/SessionPool.hpp
View File
src/database/Track.hpp → src/libs/database/include/database/Track.hpp
+4 -2
View File
@@ -22,8 +22,9 @@
#include <chrono>
#include <filesystem>
#include <optional>
#include <vector>
#include <string>
#include <unordered_set>
#include <vector>
#include <Wt/Dbo/Dbo.h>
#include <Wt/WDateTime.h>
@@ -58,7 +59,7 @@ class Track : public Wt::Dbo::Dbo<Track>
static pointer getById(Session& session, IdType id);
static pointer getByMBID(Session& session, const UUID& MBID);
static std::vector<pointer> getSimilarTracks(Session& session,
const std::set<IdType>& trackIds,
const std::unordered_set<IdType>& trackIds,
std::optional<std::size_t> offset = {},
std::optional<std::size_t> size = {});
static std::vector<pointer> getByClusters(Session& session,
@@ -78,6 +79,7 @@ class Track : public Wt::Dbo::Dbo<Track>
static std::vector<pointer> getLastAdded(Session& session, const Wt::WDateTime& after, std::optional<std::size_t> size = 1);
static std::vector<pointer> getAllWithMBIDAndMissingFeatures(Session& session);
static std::vector<IdType> getAllIdsWithFeatures(Session& session, std::optional<std::size_t> limit = {});
static std::vector<IdType> getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit = {});
// Create utility
static pointer create(Session& session, const std::filesystem::path& p);
src/database/TrackArtistLink.hpp → src/libs/database/include/database/TrackArtistLink.hpp
View File
src/database/TrackBookmark.hpp → src/libs/database/include/database/TrackBookmark.hpp
View File
src/database/TrackFeatures.hpp → src/libs/database/include/database/TrackFeatures.hpp
View File
src/database/TrackList.hpp → src/libs/database/include/database/TrackList.hpp
View File
src/database/Types.hpp → src/libs/database/include/database/Types.hpp
View File
src/database/User.hpp → src/libs/database/include/database/User.hpp
View File
src/libs/metadata/CMakeLists.txt
+25
View File
@@ -0,0 +1,25 @@
add_library(lmsmetadata SHARED
impl/AvFormatParser.cpp
impl/TagLibParser.cpp
)
target_include_directories(lmsmetadata INTERFACE
include
)
target_include_directories(lmsmetadata PRIVATE
include
)
target_link_libraries(lmsmetadata PRIVATE
lmsav
tag
)
target_link_libraries(lmsmetadata PUBLIC
lmsutils
)
install(TARGETS lmsmetadata DESTINATION lib)
src/metadata/AvFormat.cpp → src/libs/metadata/impl/AvFormatParser.cpp
+3 -4
View File
@@ -17,13 +17,12 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "AvFormat.hpp"
#include "metadata/AvFormatParser.hpp"
#include <algorithm>
#include <iostream>
#include "av/AvInfo.hpp"
#include "utils/Logger.hpp"
#include "utils/String.hpp"
@@ -128,7 +127,7 @@ getArtists(const MetadataMap& metadataMap)
}
std::optional<Track>
AvFormat::parse(const std::filesystem::path& p, bool debug)
AvFormatParser::parse(const std::filesystem::path& p, bool debug)
{
Track track;
@@ -142,7 +141,7 @@ AvFormat::parse(const std::filesystem::path& p, bool debug)
for (auto stream : mediaFile.getStreamInfo())
{
MetaData::AudioStream audioStream {.bitRate = static_cast<unsigned>(stream.bitrate)};
MetaData::AudioStream audioStream {static_cast<unsigned>(stream.bitrate)};
track.audioStreams.emplace_back(audioStream);
}
}
src/metadata/TagLibParser.cpp → src/libs/metadata/impl/TagLibParser.cpp
+2 -2
View File
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "TagLibParser.hpp"
#include "metadata/TagLibParser.hpp"
#include <taglib/asffile.h>
#include <taglib/id3v2tag.h>
@@ -180,7 +180,7 @@ TagLibParser::parse(const std::filesystem::path& p, bool debug)
track.duration = std::chrono::milliseconds {properties->length() * 1000};
MetaData::AudioStream audioStream {.bitRate = static_cast<unsigned>(properties->bitrate() * 1000)};
MetaData::AudioStream audioStream {static_cast<unsigned>(properties->bitrate() * 1000)};
track.audioStreams = {std::move(audioStream)};
}
src/metadata/AvFormat.hpp → src/libs/metadata/include/metadata/AvFormatParser.hpp
+2 -2
View File
@@ -19,13 +19,13 @@
#pragma once
#include "MetaData.hpp"
#include "metadata/IParser.hpp"
namespace MetaData
{
// Parse that makes use of AvFormat
class AvFormat : public Parser
class AvFormatParser : public IParser
{
public:
std::optional<Track> parse(const std::filesystem::path& p, bool debug = false) override;
src/metadata/MetaData.hpp → src/libs/metadata/include/metadata/IParser.hpp
+1 -2
View File
@@ -26,7 +26,6 @@
#include <set>
#include <vector>
//#include "utils/Utils.hpp"
#include "utils/UUID.hpp"
namespace MetaData
@@ -73,7 +72,7 @@ namespace MetaData
std::string copyrightURL;
};
class Parser
class IParser
{
public:
virtual std::optional<Track> parse(const std::filesystem::path& p, bool debug = false) = 0;
src/metadata/TagLibParser.hpp → src/libs/metadata/include/metadata/TagLibParser.hpp
+2 -2
View File
@@ -19,13 +19,13 @@
#pragma once
#include "MetaData.hpp"
#include "metadata/IParser.hpp"
namespace MetaData
{
// Parse that makes use of AvFormat
class TagLibParser : public Parser
class TagLibParser : public IParser
{
public:
std::optional<Track> parse(const std::filesystem::path& p, bool debug = false) override;
src/libs/recommendation/CMakeLists.txt
+25
View File
@@ -0,0 +1,25 @@
add_library(lmsrecommendation SHARED
impl/clusters/ClustersClassifier.cpp
impl/features/FeaturesClassifierCache.cpp
impl/features/FeaturesClassifier.cpp
impl/features/FeaturesDefs.cpp
impl/Engine.cpp
)
target_include_directories(lmsrecommendation INTERFACE
include
)
target_include_directories(lmsrecommendation PRIVATE
include
)
target_link_libraries(lmsrecommendation PRIVATE
lmsdatabase
lmssom
wt
)
install(TARGETS lmsrecommendation DESTINATION lib)
src/libs/recommendation/impl/Engine.cpp
+278
View File
@@ -0,0 +1,278 @@
/*
* Copyright (C) 2019 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Engine.hpp"
#include "recommendation/ClustersClassifierCreator.hpp"
#include "recommendation/FeaturesClassifierCreator.hpp"
#include "database/ScanSettings.hpp"
#include "database/TrackList.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
namespace Recommendation {
std::unique_ptr<IEngine>
createEngine(Database::Db& db)
{
return std::make_unique<Engine>(db);
}
Engine::Engine(Database::Db& db)
: _dbSession {db}
{
}
void
Engine::start()
{
assert(!_running);
_running = true;
requestReloadInternal(false);
_ioService.start();
}
void
Engine::stop()
{
assert(_running);
_running = false;
cancelPendingClassifiers();
_ioService.stop();
}
void
Engine::requestReload()
{
requestReloadInternal(true);
}
void
Engine::requestReloadInternal(bool databaseChanged)
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Reload requested...";
_ioService.post([=]()
{
reload(databaseChanged);
});
}
std::vector<Database::IdType>
Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount)
{
std::shared_lock lock {_classifiersMutex};
std::vector<Database::IdType> res;
for (const auto& classifierName : _classifierPriorities)
{
auto itClassifier {_classifiers.find(classifierName)};
if (itClassifier == std::cend(_classifiers))
continue;
res = itClassifier->second->getSimilarTracksFromTrackList(session, trackListId, maxCount);
if (!res.empty())
break;
}
return res;
}
std::vector<Database::IdType>
Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<Database::IdType>& trackIds, std::size_t maxCount)
{
std::shared_lock lock {_classifiersMutex};
std::vector<Database::IdType> res;
for (const auto& classifierName : _classifierPriorities)
{
auto itClassifier {_classifiers.find(classifierName)};
if (itClassifier == std::cend(_classifiers))
continue;
res = itClassifier->second->getSimilarTracks(dbSession, trackIds, maxCount);
if (!res.empty())
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using classifier '" << classifierName << "'";
break;
}
}
return res;
}
std::vector<Database::IdType>
Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount)
{
std::shared_lock lock {_classifiersMutex};
std::vector<Database::IdType> res;
for (const auto& classifierName : _classifierPriorities)
{
auto itClassifier {_classifiers.find(classifierName)};
if (itClassifier == std::cend(_classifiers))
continue;
res = itClassifier->second->getSimilarReleases(dbSession, releaseId, maxCount);
if (!res.empty())
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using classifier '" << classifierName << "'";
break;
}
}
return res;
}
std::vector<Database::IdType>
Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount)
{
std::shared_lock lock {_classifiersMutex};
std::vector<Database::IdType> res;
for (const auto& classifierName : _classifierPriorities)
{
auto itClassifier {_classifiers.find(classifierName)};
if (itClassifier == std::cend(_classifiers))
continue;
res = itClassifier->second->getSimilarArtists(dbSession, artistId, maxCount);
if (!res.empty())
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using classifier '" << classifierName << "'";
return res;
}
}
return res;
}
void
Engine::reload(bool databaseChanged)
{
using namespace Database;
LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines...";
const ScanSettings::RecommendationEngineType engineType {[&]()
{
auto transaction {_dbSession.createSharedTransaction()};
return ScanSettings::get(_dbSession)->getRecommendationEngineType();
}()};
clearClassifiers();
switch (engineType)
{
case ScanSettings::RecommendationEngineType::Features:
{
auto clustersClassifier {createClustersClassifier()};
auto featuresClassifier {createFeaturesClassifier()};
setClassifierPriorities({featuresClassifier->getName(), clustersClassifier->getName()});
initAndAddClassifier(std::move(clustersClassifier), databaseChanged); // init first since faster
initAndAddClassifier(std::move(featuresClassifier), databaseChanged);
break;
}
case ScanSettings::RecommendationEngineType::Clusters:
auto clustersClassifier {createClustersClassifier()};
setClassifierPriorities({clustersClassifier->getName()});
initAndAddClassifier(std::move(clustersClassifier), databaseChanged);
break;
}
LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines reloaded!";
_sigReloaded.emit();
}
void
Engine::setClassifierPriorities(std::initializer_list<std::string_view> classifierPriorities)
{
std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
_classifierPriorities.clear();
std::transform(std::cbegin(classifierPriorities), std::cend(classifierPriorities), std::back_inserter(_classifierPriorities), [](std::string_view name) { return std::string {name}; });
}
void
Engine::clearClassifiers()
{
std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
_classifiers.clear();
}
void
Engine::initAndAddClassifier(std::unique_ptr<IClassifier> classifier, bool databaseChanged)
{
PendingClassifierHandler pendingClassifier {*this, *classifier.get()};
LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "'...";
bool res {classifier->init(_dbSession, databaseChanged)};
LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "': " << (res ? "SUCCESS" : "FAILURE");
if (res)
{
std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
_classifiers.emplace(classifier->getName(), std::move(classifier));
}
}
void
Engine::cancelPendingClassifiers()
{
std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
for (IClassifier* classifier : _pendingClassifiers)
classifier->requestCancelInit();
}
void
Engine::addPendingClassifier(IClassifier& classifier)
{
std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
_pendingClassifiers.insert(&classifier);
}
void
Engine::removePendingClassifier(IClassifier& classifier)
{
std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
_pendingClassifiers.erase(&classifier);
}
} // ns Similarity
src/libs/recommendation/impl/Engine.hpp
+94
View File
@@ -0,0 +1,94 @@
/*
* Copyright (C) 2019 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <map>
#include <shared_mutex>
#include <vector>
#include <Wt/WIOService.h>
#include "database/Session.hpp"
#include "recommendation/IEngine.hpp"
#include "recommendation/IClassifier.hpp"
namespace Recommendation
{
class Engine : public IEngine
{
public:
Engine(Database::Db& db);
private:
void start() override;
void stop() override;
void requestReload() override;
Wt::Signal<>& reloaded() override { return _sigReloaded; }
std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override;
std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) override;
std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override;
std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override;
void requestReloadInternal(bool databaseChanged);
void reload(bool databaseChanged);
void setClassifierPriorities(std::initializer_list<std::string_view> classifierNames);
void clearClassifiers();
void initAndAddClassifier(std::unique_ptr<IClassifier> classifier, bool databaseChanged);
class PendingClassifierHandler
{
public:
PendingClassifierHandler(Engine& engine, IClassifier& classifier) : _engine {engine}, _classifier {classifier}
{
_engine.addPendingClassifier(_classifier);
}
~PendingClassifierHandler()
{
_engine.removePendingClassifier(_classifier);
}
private:
Engine& _engine;
IClassifier& _classifier;
};
void cancelPendingClassifiers();
void addPendingClassifier(IClassifier& classifier);
void removePendingClassifier(IClassifier& classifier);
bool _running {};
Wt::WIOService _ioService;
Database::Session _dbSession;
Wt::Signal<> _sigReloaded;
std::shared_mutex _classifiersMutex;
std::map<std::string, std::unique_ptr<IClassifier>> _classifiers;
std::vector<std::string> _classifierPriorities; // ordered by priority
std::unordered_set<IClassifier*> _pendingClassifiers;
};
} // ns Recommendation
src/similarity/cluster/SimilarityClusterSearcher.cpp → src/libs/recommendation/impl/clusters/ClustersClassifier.cpp
+12 -12
View File
@@ -17,10 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "SimilarityClusterSearcher.hpp"
#include <random>
#include <chrono>
#include "ClustersClassifier.hpp"
#include "database/Artist.hpp"
#include "database/Cluster.hpp"
@@ -29,11 +26,15 @@
#include "database/Track.hpp"
#include "database/TrackList.hpp"
namespace Similarity {
namespace ClusterSearcher {
namespace Recommendation {
std::unique_ptr<IClassifier> createClustersClassifier()
{
return std::make_unique<ClusterClassifier>();
}
std::vector<Database::IdType>
getSimilarTracks(Database::Session& dbSession, const std::set<Database::IdType>& trackIds, std::size_t maxCount)
ClusterClassifier::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<Database::IdType>& trackIds, std::size_t maxCount) const
{
auto transaction {dbSession.createSharedTransaction()};
@@ -46,7 +47,7 @@ getSimilarTracks(Database::Session& dbSession, const std::set<Database::IdType>&
}
std::vector<Database::IdType>
getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount)
ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const
{
std::vector<Database::IdType> res;
@@ -65,7 +66,7 @@ getSimilarTracksFromTrackList(Database::Session& session, Database::IdType track
}
std::vector<Database::IdType>
getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount)
ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) const
{
std::vector<Database::IdType> res;
@@ -83,7 +84,7 @@ getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std
}
std::vector<Database::IdType>
getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount)
ClusterClassifier::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) const
{
std::vector<Database::IdType> res;
@@ -100,5 +101,4 @@ getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::
return res;
}
} // namespace ClusterSearcher
} // namespace Similarity
} // namespace Recommendation
src/libs/recommendation/impl/clusters/ClustersClassifier.hpp
+52
View File
@@ -0,0 +1,52 @@
/*
* Copyright (C) 2018 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "recommendation/IClassifier.hpp"
namespace Recommendation
{
class ClusterClassifier : public IClassifier
{
public:
ClusterClassifier() = default;
ClusterClassifier(const ClusterClassifier&) = delete;
ClusterClassifier(ClusterClassifier&&) = delete;
ClusterClassifier& operator=(const ClusterClassifier&) = delete;
ClusterClassifier& operator=(ClusterClassifier&&) = delete;
private:
std::string_view getName() const { return "Clusters"; }
bool init(Database::Session&, bool) override {return true;}
void requestCancelInit() override {}
std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override;
std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override;
};
} // namespace Recommendation
src/libs/recommendation/impl/features/FeaturesClassifier.cpp
+487
View File
@@ -0,0 +1,487 @@
/*
* Copyright (C) 2018 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "FeaturesClassifier.hpp"
#include <numeric>
#include "database/Artist.hpp"
#include "database/Release.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "database/TrackFeatures.hpp"
#include "database/TrackList.hpp"
#include "som/DataNormalizer.hpp"
#include "utils/Logger.hpp"
#include "utils/Random.hpp"
namespace Recommendation {
std::unique_ptr<IClassifier> createFeaturesClassifier()
{
return std::make_unique<FeaturesClassifier>();
}
const FeatureSettingsMap&
FeaturesClassifier::getDefaultTrainFeatureSettings()
{
static const FeatureSettingsMap defaultTrainFeatureSettings
{
{ "lowlevel.spectral_energyband_high.mean", {1}},
{ "lowlevel.spectral_rolloff.median", {1}},
{ "lowlevel.spectral_contrast_valleys.var", {1}},
{ "lowlevel.erbbands.mean", {1}},
{ "lowlevel.gfcc.mean", {1}},
};
return defaultTrainFeatureSettings;
}
static
std::optional<FeatureValuesMap>
getTrackFeatureValues(FeaturesClassifier::FeaturesFetchFunc func, Database::IdType trackId, const std::unordered_set<FeatureName>& featureNames)
{
return func(trackId, featureNames);
}
static
std::optional<FeatureValuesMap>
getTrackFeatureValuesFromDb(Database::Session& session, Database::IdType trackId, const std::unordered_set<FeatureName>& featureNames)
{
auto func = [&](Database::IdType trackId, const std::unordered_set<FeatureName>& featureNames)
{
std::optional<FeatureValuesMap> res;
auto transaction {session.createSharedTransaction()};
Database::Track::pointer track {Database::Track::getById(session, trackId)};
if (!track)
return res;
res = track->getTrackFeatures()->getFeatureValuesMap(featureNames);
if (res->empty())
res.reset();
return res;
};
return getTrackFeatureValues(func, trackId, featureNames);
}
static
std::optional<SOM::InputVector>
convertFeatureValuesMapToInputVector(const FeatureValuesMap& featureValuesMap, std::size_t nbDimensions)
{
std::size_t i {};
std::optional<SOM::InputVector> res {SOM::InputVector {nbDimensions}};
for (const auto& [featureName, values] : featureValuesMap)
{
if (values.size() != getFeatureDef(featureName).nbDimensions)
{
LMS_LOG(RECOMMENDATION, WARNING) << "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << values.size();
res.reset();
break;
}
for (double val : values)
(*res)[i++] = val;
}
return res;
}
static
SOM::InputVector
getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t nbDimensions)
{
SOM::InputVector weights {nbDimensions};
std::size_t index {};
for (const auto& [featureName, featureSettings] : featureSettingsMap)
{
const std::size_t featureNbDimensions {getFeatureDef(featureName).nbDimensions};
for (std::size_t i {}; i < featureNbDimensions; ++i)
weights[index++] = (1. / featureNbDimensions * featureSettings.weight);
}
assert(index == nbDimensions);
return weights;
}
bool
FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSettings& trainSettings)
{
LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier...";
std::unordered_set<FeatureName> featureNames;
std::transform(std::cbegin(trainSettings.featureSettingsMap), std::cend(trainSettings.featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)),
[](const auto& itFeatureSetting) { return itFeatureSetting.first; });
const std::size_t nbDimensions {std::accumulate(std::cbegin(featureNames), std::cend(featureNames), std::size_t {0},
[](std::size_t sum, const FeatureName& featureName) { return sum + getFeatureDef(featureName).nbDimensions; })};
LMS_LOG(RECOMMENDATION, DEBUG) << "Features dimension = " << nbDimensions;
std::vector<Database::IdType> trackIds;
{
auto transaction {session.createSharedTransaction()};
LMS_LOG(RECOMMENDATION, DEBUG) << "Getting Tracks with features...";
trackIds = Database::Track::getAllIdsWithFeatures(session);
LMS_LOG(RECOMMENDATION, DEBUG) << "Getting Tracks with features DONE (found " << trackIds.size() << " tracks)";
}
std::vector<SOM::InputVector> samples;
std::vector<Database::IdType> samplesTrackIds;
samples.reserve(trackIds.size());
samplesTrackIds.reserve(trackIds.size());
LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features...";
for (Database::IdType trackId : trackIds)
{
if (_initCancelled)
return false;
std::optional<FeatureValuesMap> featureValuesMap;
if (_featuresFetchFunc)
featureValuesMap = getTrackFeatureValues(_featuresFetchFunc, trackId, featureNames);
else
featureValuesMap = getTrackFeatureValuesFromDb(session, trackId, featureNames);
if (!featureValuesMap)
continue;
std::optional<SOM::InputVector> inputVector {convertFeatureValuesMapToInputVector(*featureValuesMap, nbDimensions)};
if (!inputVector)
continue;
samples.emplace_back(std::move(*inputVector));
samplesTrackIds.emplace_back(trackId);
}
LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features DONE";
if (samples.empty())
{
LMS_LOG(RECOMMENDATION, INFO) << "Nothing to classify!";
return false;
}
LMS_LOG(RECOMMENDATION, DEBUG) << "Normalizing data...";
SOM::DataNormalizer dataNormalizer {nbDimensions};
dataNormalizer.computeNormalizationFactors(samples);
for (auto& sample : samples)
dataNormalizer.normalizeData(sample);
const SOM::Coordinate size {static_cast<SOM::Coordinate>(std::sqrt(samples.size() / trainSettings.sampleCountPerNeuron))};
LMS_LOG(RECOMMENDATION, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network";
SOM::Network network {size, size, nbDimensions};
SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)};
network.setDataWeights(weights);
auto progressIndicator{[](const auto& iter)
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount;
}};
LMS_LOG(RECOMMENDATION, DEBUG) << "Training network...";
network.train(samples, trainSettings.iterationCount, progressIndicator);
LMS_LOG(RECOMMENDATION, DEBUG) << "Training network DONE";
if (_initCancelled)
return false;
LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks...";
ObjectPositions trackPositions;
for (std::size_t i {}; i < samples.size(); ++i)
{
if (_initCancelled)
return false;
const SOM::Position position {network.getClosestRefVectorPosition(samples[i])};
trackPositions[samplesTrackIds[i]].insert(position);
}
LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks DONE";
return init(session, std::move(network), std::move(trackPositions));
}
bool
FeaturesClassifier::initFromCache(Database::Session& session, const FeaturesClassifierCache& cache)
{
LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier from cache...";
return init(session, std::move(cache._network), cache._trackPositions);
}
std::vector<Database::IdType>
FeaturesClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) const
{
const std::unordered_set<Database::IdType> trackIds {[&]() -> std::unordered_set<Database::IdType>
{
auto transaction {session.createSharedTransaction()};
const Database::TrackList::pointer trackList {Database::TrackList::getById(session, trackListId)};
if (trackList)
{
const std::vector<Database::IdType> orderedTrackIds {trackList->getTrackIds()};
return std::unordered_set<Database::IdType> {std::cbegin(orderedTrackIds), std::cend(orderedTrackIds)};
}
return {};
}()};
return getSimilarTracks(session, trackIds, maxCount);
}
std::vector<Database::IdType>
FeaturesClassifier::getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksIds, std::size_t maxCount) const
{
std::vector<Database::IdType> similarTrackIds {getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount)};
if (!similarTrackIds.empty())
{
// Report only existing ids
auto transaction {session.createSharedTransaction()};
similarTrackIds.erase(std::remove_if(std::begin(similarTrackIds), std::end(similarTrackIds),
[&](Database::IdType trackId) { return Database::Track::getById(session, trackId) == Database::Track::pointer {}; }),
std::cend(similarTrackIds));
}
return similarTrackIds;
}
std::vector<Database::IdType>
FeaturesClassifier::getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const
{
std::vector<Database::IdType> similarReleaseIds {getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount)};
if (!similarReleaseIds.empty())
{
// Report only existing ids
auto transaction {session.createSharedTransaction()};
similarReleaseIds.erase(std::remove_if(std::begin(similarReleaseIds), std::end(similarReleaseIds),
[&](Database::IdType releaseId) { return Database::Release::getById(session, releaseId) == Database::Release::pointer {}; }),
std::cend(similarReleaseIds));
}
return similarReleaseIds;
}
std::vector<Database::IdType>
FeaturesClassifier::getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const
{
std::vector<Database::IdType> similarArtistIds {getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount)};
if (!similarArtistIds.empty())
{
// Report only existing ids
auto transaction {session.createSharedTransaction()};
similarArtistIds.erase(std::remove_if(std::begin(similarArtistIds), std::end(similarArtistIds),
[&](Database::IdType artistId) { return Database::Artist::getById(session, artistId) == Database::Artist::pointer {}; }),
std::cend(similarArtistIds));
}
return similarArtistIds;
}
FeaturesClassifierCache
FeaturesClassifier::toCache() const
{
return FeaturesClassifierCache {*_network, _trackPositions};
}
bool
FeaturesClassifier::init(Database::Session& session, bool databaseChanged)
{
if (databaseChanged)
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Database changed: invidating cache";
FeaturesClassifierCache::invalidate();
}
std::optional<FeaturesClassifierCache> cache {FeaturesClassifierCache::read()};
if (cache)
return initFromCache(session, *cache);
TrainSettings trainSettings;
trainSettings.featureSettingsMap = getDefaultTrainFeatureSettings();
bool res {initFromTraining(session, trainSettings)};
if (res)
toCache().write();
return res;
}
void
FeaturesClassifier::requestCancelInit()
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Requesting init cancellation";
_initCancelled = true;
}
bool
FeaturesClassifier::init(Database::Session& session,
SOM::Network network,
const ObjectPositions& tracksPosition)
{
_networkRefVectorsDistanceMedian = network.computeRefVectorsDistanceMedian();
LMS_LOG(RECOMMENDATION, DEBUG) << "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian;
const SOM::Coordinate width {network.getWidth()};
const SOM::Coordinate height {network.getHeight()};
_artistsMap = MatrixOfObjects {width, height};
_releasesMap = MatrixOfObjects {width, height};
_tracksMap = MatrixOfObjects {width, height};
LMS_LOG(RECOMMENDATION, DEBUG) << "Constructing maps...";
for (auto itTrackCoord : tracksPosition)
{
if (_initCancelled)
return false;
auto transaction {session.createSharedTransaction()};
Database::IdType trackId {itTrackCoord.first};
const std::unordered_set<SOM::Position>& positionSet {itTrackCoord.second};
const Database::Track::pointer track {Database::Track::getById(session, trackId)};
if (!track)
continue;
for (const SOM::Position& position : positionSet)
{
_tracksMap[position].insert(trackId);
_trackPositions[trackId].insert(position);
if (track->getRelease())
{
_releasePositions[track->getRelease().id()].insert(position);
_releasesMap[position].insert(track->getRelease().id());
}
for (const auto& artist : track->getArtists())
{
_artistPositions[artist.id()].insert(position);
_artistsMap[position].insert(artist.id());
}
}
}
_network = std::make_unique<SOM::Network>(std::move(network));
LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully initialized!";
return true;
}
std::unordered_set<SOM::Position>
FeaturesClassifier::getMatchingRefVectorsPosition(const std::unordered_set<Database::IdType>& ids, const ObjectPositions& objectPositions)
{
std::unordered_set<SOM::Position> res;
if (ids.empty())
return res;
for (auto id : ids)
{
auto it = objectPositions.find(id);
if (it == objectPositions.end())
continue;
for (const auto& position : it->second)
res.insert(position);
}
return res;
}
std::unordered_set<Database::IdType>
FeaturesClassifier::getObjectsIds(const std::unordered_set<SOM::Position>& positionSet, const MatrixOfObjects& objectsMap)
{
std::unordered_set<Database::IdType> res;
for (const auto& position : positionSet)
{
for (auto id : objectsMap.get(position))
res.insert(id);
}
return res;
}
std::vector<Database::IdType>
FeaturesClassifier::getSimilarObjects(const std::unordered_set<Database::IdType>& ids,
const MatrixOfObjects& objectsMap,
const ObjectPositions& objectPosition,
std::size_t maxCount) const
{
std::vector<Database::IdType> res;
std::unordered_set<SOM::Position> searchedRefVectorsPosition {getMatchingRefVectorsPosition(ids, objectPosition)};
if (searchedRefVectorsPosition.empty())
return res;
while (1)
{
std::unordered_set<Database::IdType> closestObjectIds {getObjectsIds(searchedRefVectorsPosition, objectsMap)};
// Remove objects that are already in input or already reported
for (auto id : ids)
closestObjectIds.erase(id);
{
std::vector<Database::IdType> objectIdsToAdd {std::cbegin(closestObjectIds), std::cend(closestObjectIds)};
Random::shuffleContainer(objectIdsToAdd );
std::copy(std::cbegin(objectIdsToAdd), std::cend(objectIdsToAdd), std::back_inserter(res));
}
if (res.size() > maxCount)
res.resize(maxCount);
if (res.size() == maxCount)
break;
// If there is not enough objects, try again with closest neighbour until there is too much distance
const std::optional<SOM::Position> closestRefVectorPosition {_network->getClosestRefVectorPosition(searchedRefVectorsPosition, _networkRefVectorsDistanceMedian * 0.75)};
if (!closestRefVectorPosition)
break;
searchedRefVectorsPosition.insert(closestRefVectorPosition.value());
}
return res;
}
} // ns Recommendation
src/libs/recommendation/impl/features/FeaturesClassifier.hpp
+112
View File
@@ -0,0 +1,112 @@
/*
* Copyright (C) 2018 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <unordered_map>
#include <optional>
#include <string>
#include "recommendation/IClassifier.hpp"
#include "som/DataNormalizer.hpp"
#include "som/Network.hpp"
#include "FeaturesClassifierCache.hpp"
#include "FeaturesDefs.hpp"
namespace Database
{
class Session;
}
namespace Recommendation {
using FeatureWeight = double;
class FeaturesClassifier : public IClassifier
{
public:
FeaturesClassifier() = default;
FeaturesClassifier(const FeaturesClassifier&) = delete;
FeaturesClassifier(FeaturesClassifier&&) = delete;
FeaturesClassifier& operator=(const FeaturesClassifier&) = delete;
FeaturesClassifier& operator=(FeaturesClassifier&&) = delete;
using FeaturesFetchFunc = std::function<std::optional<std::unordered_map<std::string, std::vector<double>>>(Database::IdType /*trackId*/, const std::unordered_set<std::string>& /*features*/)>;
// Default is to retrieve the features from the database (may be slow).
// Use this only if you want to train different searchers with some cached data
static void setFeaturesFetchFunc(FeaturesFetchFunc func) { _featuresFetchFunc = func; }
static const FeatureSettingsMap& getDefaultTrainFeatureSettings();
private:
std::string_view getName() const { return "Features"; }
bool init(Database::Session& session, bool databaseChanged) override;
void requestCancelInit() override;
std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const;
std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const;
bool initFromCache(Database::Session& session, const FeaturesClassifierCache& cache);
// Use training (may be very slow)
struct TrainSettings
{
std::size_t iterationCount {10};
float sampleCountPerNeuron {4};
FeatureSettingsMap featureSettingsMap;
};
bool initFromTraining(Database::Session& session, const TrainSettings& trainSettings);
using ObjectPositions = std::unordered_map<Database::IdType, std::unordered_set<SOM::Position>>;
using MatrixOfObjects = SOM::Matrix<std::unordered_set<Database::IdType>>;
bool init(Database::Session& session,
SOM::Network network,
const ObjectPositions& tracksPosition);
FeaturesClassifierCache toCache() const;
static std::unordered_set<SOM::Position> getMatchingRefVectorsPosition(const std::unordered_set<Database::IdType>& ids, const ObjectPositions& objectPositions);
static std::unordered_set<Database::IdType> getObjectsIds(const std::unordered_set<SOM::Position>& positionSet, const MatrixOfObjects& objectsMap);
std::vector<Database::IdType> getSimilarObjects(const std::unordered_set<Database::IdType>& ids,
const SOM::Matrix<std::unordered_set<Database::IdType>>& objectsMap,
const ObjectPositions& objectPosition,
std::size_t maxCount) const;
bool _initCancelled {};
std::unique_ptr<SOM::Network> _network;
double _networkRefVectorsDistanceMedian {};
MatrixOfObjects _artistsMap;
ObjectPositions _artistPositions;
MatrixOfObjects _releasesMap;
ObjectPositions _releasePositions;
MatrixOfObjects _tracksMap;
ObjectPositions _trackPositions;
static inline FeaturesFetchFunc _featuresFetchFunc;
};
} // ns Recommendation
src/similarity/features/SimilarityFeaturesCache.cpp → src/libs/recommendation/impl/features/FeaturesClassifierCache.cpp
+29 -35
View File
@@ -17,28 +17,28 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "SimilarityFeaturesCache.hpp"
#include "FeaturesClassifierCache.hpp"
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/xml_parser.hpp>
#include "utils/Config.hpp"
#include "utils/IConfig.hpp"
#include "utils/Logger.hpp"
#include "utils/Service.hpp"
namespace Similarity {
namespace Recommendation {
static
std::filesystem::path getCacheDirectory()
{
return ServiceProvider<Config>::get()->getPath("working-dir") / "cache" / "features";
return ServiceProvider<IConfig>::get()->getPath("working-dir") / "cache" / "features";
}
static std::filesystem::path getCacheNetworkFilePath()
{
return getCacheDirectory() / "network";
};
}
static std::filesystem::path getCacheTrackPositionsFilePath()
{
@@ -79,26 +79,25 @@ networkToCacheFile(const SOM::Network& network, std::filesystem::path path)
boost::property_tree::write_xml(path.string(), root);
LMS_LOG(SIMILARITY, DEBUG) << "Created network cache";
LMS_LOG(RECOMMENDATION, DEBUG) << "Created network cache";
return true;
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(SIMILARITY, ERROR) << "Cannot create network cache: " << error.what();
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot create network cache: " << error.what();
return false;
}
}
static
std::optional<SOM::Network>
createNetworkFromCacheFile(const std::filesystem::path& path)
FeaturesClassifierCache::createNetworkFromCacheFile(const std::filesystem::path& path)
{
if (!std::filesystem::exists(path))
return std::nullopt;
try
{
LMS_LOG(SIMILARITY, INFO) << "Reading network from cache...";
LMS_LOG(RECOMMENDATION, INFO) << "Reading network from cache...";
boost::property_tree::ptree root;
@@ -132,20 +131,19 @@ createNetworkFromCacheFile(const std::filesystem::path& path)
res.setRefVector({x, y}, refVector);
}
LMS_LOG(SIMILARITY, INFO) << "Successfully read network from cache";
LMS_LOG(RECOMMENDATION, INFO) << "Successfully read network from cache";
return res;
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(SIMILARITY, ERROR) << "Cannot read network cache: " << error.what();
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot read network cache: " << error.what();
return std::nullopt;
}
}
static
bool
objectPositionToCacheFile(const std::map<Database::IdType, std::set<SOM::Position>>& objectsPosition, std::filesystem::path path)
FeaturesClassifierCache::objectPositionToCacheFile(const ObjectPositions& objectsPosition, const std::filesystem::path& path)
{
try
{
@@ -174,24 +172,23 @@ objectPositionToCacheFile(const std::map<Database::IdType, std::set<SOM::Positio
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(SIMILARITY, ERROR) << "Cannot cache object position: " << error.what();
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot cache object position: " << error.what();
return false;
}
}
static
std::optional<std::map<Database::IdType, std::set<SOM::Position>>>
createObjectPositionsFromCacheFile(std::filesystem::path path)
std::optional<FeaturesClassifierCache::ObjectPositions>
FeaturesClassifierCache::createObjectPositionsFromCacheFile(const std::filesystem::path& path)
{
try
{
LMS_LOG(SIMILARITY, INFO) << "Reading object position from cache...";
LMS_LOG(RECOMMENDATION, INFO) << "Reading object position from cache...";
boost::property_tree::ptree root;
boost::property_tree::read_xml(path.string(), root);
std::map<Database::IdType, std::set<SOM::Position>> res;
ObjectPositions res;
for (const auto& object : root.get_child("objects"))
{
@@ -205,44 +202,42 @@ createObjectPositionsFromCacheFile(std::filesystem::path path)
}
}
LMS_LOG(SIMILARITY, INFO) << "Successfully read object position from cache";
LMS_LOG(RECOMMENDATION, INFO) << "Successfully read object position from cache";
return res;
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(SIMILARITY, ERROR) << "Cannot create object position from cache file: " << error.what();
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot create object position from cache file: " << error.what();
return std::nullopt;
}
}
void
FeaturesCache::invalidate()
FeaturesClassifierCache::invalidate()
{
std::filesystem::remove(getCacheNetworkFilePath());
std::filesystem::remove(getCacheTrackPositionsFilePath());
}
std::optional<FeaturesCache>
FeaturesCache::read()
std::optional<FeaturesClassifierCache>
FeaturesClassifierCache::read()
{
std::optional<FeaturesCache> res;
auto network{createNetworkFromCacheFile(getCacheNetworkFilePath())};
if (!network)
return res;
return std::nullopt;
auto trackPositions{createObjectPositionsFromCacheFile(getCacheTrackPositionsFilePath())};
if (!trackPositions)
return res;
return std::nullopt;
return FeaturesCache{std::move(*network), std::move(*trackPositions)};
return FeaturesClassifierCache {std::move(*network), std::move(*trackPositions)};
}
void
FeaturesCache::write()
FeaturesClassifierCache::write() const
{
std::filesystem::create_directories(ServiceProvider<Config>::get()->getPath("working-dir") / "cache" / "features");
std::filesystem::create_directories(ServiceProvider<IConfig>::get()->getPath("working-dir") / "cache" / "features");
if (!networkToCacheFile(_network, getCacheNetworkFilePath())
|| !objectPositionToCacheFile(_trackPositions, getCacheTrackPositionsFilePath()))
@@ -251,11 +246,10 @@ FeaturesCache::write()
}
}
FeaturesCache::FeaturesCache(SOM::Network network, ObjectPositions trackPositions)
FeaturesClassifierCache::FeaturesClassifierCache(SOM::Network network, ObjectPositions trackPositions)
: _network {std::move(network)},
_trackPositions {std::move(trackPositions)}
{
}
} // namespace Similarity
} // namespace Recommendation
src/libs/recommendation/impl/features/FeaturesClassifierCache.hpp
+54
View File
@@ -0,0 +1,54 @@
/*
* Copyright (C) 2018 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <filesystem>
#include <unordered_map>
#include <unordered_set>
#include "database/Types.hpp"
#include "som/Network.hpp"
namespace Recommendation {
class FeaturesClassifierCache
{
public:
static void invalidate();
static std::optional<FeaturesClassifierCache> read();
void write() const;
private:
using ObjectPositions = std::unordered_map<Database::IdType, std::unordered_set<SOM::Position>>;
FeaturesClassifierCache(SOM::Network network, ObjectPositions trackPositions);
static std::optional<SOM::Network> createNetworkFromCacheFile(const std::filesystem::path& path);
static std::optional<ObjectPositions> createObjectPositionsFromCacheFile(const std::filesystem::path& path);
static bool objectPositionToCacheFile(const ObjectPositions& objectsPosition, const std::filesystem::path& path);
friend class FeaturesClassifier;
SOM::Network _network;
ObjectPositions _trackPositions;
};
} // namespace Recommendation
src/similarity/features/SimilarityFeaturesDefs.cpp → src/libs/recommendation/impl/features/FeaturesDefs.cpp
+3 -3
View File
@@ -17,14 +17,14 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "SimilarityFeaturesDefs.hpp"
#include "FeaturesDefs.hpp"
#include <algorithm>
#include <iterator>
#include "utils/Exception.hpp"
namespace Similarity {
namespace Recommendation {
static const std::unordered_map<FeatureName, FeatureDef> featureDefinitions
{
@@ -398,5 +398,5 @@ getFeatureNames()
return res;
}
} // namespace Similarity
} // namespace Recommendation
src/similarity/features/SimilarityFeaturesDefs.hpp → src/libs/recommendation/impl/features/FeaturesDefs.hpp
+2 -2
View File
@@ -24,7 +24,7 @@
#include <unordered_set>
#include <vector>
namespace Similarity {
namespace Recommendation {
using FeatureName = std::string;
using FeatureNames = std::unordered_set<FeatureName>;
@@ -46,4 +46,4 @@ struct FeatureSettings
};
using FeatureSettingsMap = std::unordered_map<FeatureName, FeatureSettings>;
} // namespace Similarity
} // namespace Recommendation
src/similarity/features/SimilarityFeaturesCache.hpp → src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp
+6 -27
View File
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018 Emeric Poupon
* Copyright (C) 2020 Emeric Poupon
*
* This file is part of LMS.
*
@@ -19,33 +19,12 @@
#pragma once
#include <map>
#include <optional>
#include <set>
#include <memory>
#include "database/Types.hpp"
#include "som/Network.hpp"
namespace Similarity {
class FeaturesCache
namespace Recommendation
{
public:
class IClassifier;
static void invalidate();
std::unique_ptr<IClassifier> createClustersClassifier();
}
static std::optional<FeaturesCache> read();
void write();
private:
using ObjectPositions = std::map<Database::IdType, std::set<SOM::Position>>;
FeaturesCache(SOM::Network network, ObjectPositions trackPositions);
friend class FeaturesSearcher;
SOM::Network _network;
ObjectPositions _trackPositions;
};
} // namespace Similarity
src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp
+29
View File
@@ -0,0 +1,29 @@
/*
* Copyright (C) 2020 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <memory>
#include "recommendation/IClassifier.hpp"
namespace Recommendation
{
std::unique_ptr<IClassifier> createFeaturesClassifier();
}
src/libs/recommendation/include/recommendation/IClassifier.hpp
+52
View File
@@ -0,0 +1,52 @@
/*
* Copyright (C) 2020 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <string_view>
#include <unordered_set>
#include <vector>
#include "database/Types.hpp"
namespace Database
{
class Session;
}
namespace Recommendation
{
class IClassifier
{
public:
virtual ~IClassifier() = default;
virtual std::string_view getName() const = 0;
virtual bool init(Database::Session& session, bool databaseChanged) = 0;
virtual void requestCancelInit() = 0;
virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0;
virtual std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const = 0;
virtual std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const = 0;
virtual std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const = 0;
};
} // ns Recommendation
src/libs/recommendation/include/recommendation/IEngine.hpp
+59
View File
@@ -0,0 +1,59 @@
/*
* Copyright (C) 2019 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <functional>
#include <vector>
#include <unordered_set>
#include <Wt/WSignal.h>
#include "database/Types.hpp"
namespace Database
{
class Db;
class Session;
}
namespace Recommendation
{
class IEngine
{
public:
virtual ~IEngine() = default;
virtual void start() = 0;
virtual void stop() = 0;
virtual void requestReload() = 0;
virtual Wt::Signal<>& reloaded() = 0;
// Closest results first
virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0;
virtual std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) = 0;
virtual std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) = 0;
virtual std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0;
};
std::unique_ptr<IEngine> createEngine(Database::Db& db);
} // ns Recommendation
src/libs/scanner/CMakeLists.txt
+27
View File
@@ -0,0 +1,27 @@
add_library(lmsscanner SHARED
impl/AcousticBrainzUtils.cpp
impl/MediaScanner.cpp
impl/MediaScannerStats.cpp
)
target_include_directories(lmsscanner INTERFACE
include
)
target_include_directories(lmsscanner PRIVATE
include
)
target_link_libraries(lmsscanner PRIVATE
lmsdatabase
lmsmetadata
lmsutils
)
target_link_libraries(lmsscanner PUBLIC
wt
)
install(TARGETS lmsscanner DESTINATION lib)
src/similarity/features/AcousticBrainzUtils.cpp → src/libs/scanner/impl/AcousticBrainzUtils.cpp
+6 -5
View File
@@ -25,9 +25,10 @@
#include <Wt/WIOService.h>
#include <Wt/Http/Client.h>
#include "utils/Config.hpp"
#include "utils/IConfig.hpp"
#include "utils/Logger.hpp"
#include "utils/Service.hpp"
#include "utils/UUID.hpp"
namespace AcousticBrainz
@@ -39,7 +40,7 @@ getJsonData(const UUID& mbid)
{
static const std::string defaultAPIURL = "https://acousticbrainz.org/api/v1/";
const std::string url {ServiceProvider<Config>::get()->getString("acousticbrainz-api-url", defaultAPIURL) + std::string {mbid.getAsString()} + "/low-level"};
const std::string url {ServiceProvider<IConfig>::get()->getString("acousticbrainz-api-url", defaultAPIURL) + std::string {mbid.getAsString()} + "/low-level"};
boost::asio::io_service ioService;
@@ -50,7 +51,7 @@ getJsonData(const UUID& mbid)
if (!client.get(url))
{
LMS_LOG(SIMILARITY, ERROR) << "Cannot perform a GET request to url '" << url << "'";
LMS_LOG(DBUPDATER, ERROR) << "Cannot perform a GET request to url '" << url << "'";
return {};
}
@@ -59,13 +60,13 @@ getJsonData(const UUID& mbid)
{
if (ec)
{
LMS_LOG(SIMILARITY, ERROR) << "GET request to url '" << url << "' failed: " << ec.message();
LMS_LOG(DBUPDATER, ERROR) << "GET request to url '" << url << "' failed: " << ec.message();
return;
}
if (msg.status() != 200)
{
LMS_LOG(SIMILARITY, ERROR) << "GET request to url '" << url << "' failed: status = " << msg.status() << ", body = " << msg.body();
LMS_LOG(DBUPDATER, ERROR) << "GET request to url '" << url << "' failed: status = " << msg.status() << ", body = " << msg.body();
return;
}
src/similarity/features/AcousticBrainzUtils.hpp → src/libs/scanner/impl/AcousticBrainzUtils.hpp
+1 -1
View File
@@ -21,7 +21,7 @@
#include <string>
#include "utils/UUID.hpp"
class UUID;
namespace AcousticBrainz
{
src/scanner/MediaScanner.cpp → src/libs/scanner/impl/MediaScanner.cpp
+131 -78
View File
@@ -23,15 +23,18 @@
#include <Wt/WLocalDateTime.h>
#include "cover/CoverArtGrabber.hpp"
#include "database/Artist.hpp"
#include "database/Cluster.hpp"
#include "database/Release.hpp"
#include "database/ScanSettings.hpp"
#include "database/Track.hpp"
#include "database/TrackFeatures.hpp"
#include "metadata/TagLibParser.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/Path.hpp"
#include "utils/UUID.hpp"
#include "AcousticBrainzUtils.hpp"
using namespace Database;
@@ -60,7 +63,7 @@ getNextFirstOfMonth(Wt::WDate current)
}
bool
isFileSupported(const std::filesystem::path& file, const std::set<std::filesystem::path>& extensions)
isFileSupported(const std::filesystem::path& file, const std::unordered_set<std::filesystem::path>& extensions)
{
return (extensions.find(file.extension()) != extensions.end());
}
@@ -206,18 +209,27 @@ getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames)
namespace Scanner {
std::unique_ptr<IMediaScanner>
createMediaScanner(Database::Db& db)
{
return std::make_unique<MediaScanner>(db);
}
MediaScanner::MediaScanner(Database::Db& db)
: _dbSession {db}
{
// For now, always use TagLib
_metadataParser = std::make_unique<MetaData::TagLibParser>();
_ioService.setThreadCount(1);
refreshScanSettings();
}
void
MediaScanner::setAddon(MediaScannerAddon& addon)
MediaScanner::~MediaScanner()
{
_addons.push_back(&addon);
if (_running)
stop();
}
void
@@ -242,9 +254,6 @@ MediaScanner::stop(void)
{
_running = false;
for (auto& addon : _addons)
addon->requestStop();
_scheduleTimer.cancel();
_ioService.stop();
@@ -290,7 +299,7 @@ MediaScanner::scheduleNextScan()
refreshScanSettings();
Wt::WDateTime now {Wt::WLocalDateTime::currentServerDateTime().toUTC()};
const Wt::WDateTime now {Wt::WLocalDateTime::currentServerDateTime().toUTC()};
Wt::WDate nextScanDate;
switch (_updatePeriod)
@@ -341,33 +350,21 @@ MediaScanner::scheduleNextScan()
void
MediaScanner::countAllFiles(ScanStats& stats)
{
std::error_code ec;
stats.filesToScan = 0;
std::filesystem::recursive_directory_iterator itPath {_mediaDirectory, std::filesystem::directory_options::follow_directory_symlink, ec};
if (ec)
exploreFilesRecursive(_mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
{
LMS_LOG(DBUPDATER, ERROR) << "Cannot iterate over '" << _mediaDirectory.string() << "': " << ec.message();
return;
}
if (!_running)
return false;
std::filesystem::recursive_directory_iterator itEnd;
while (_running && itPath != itEnd)
{
const std::filesystem::path& path {*itPath};
if (!ec)
if (!ec && isFileSupported(path, _fileExtensions))
{
if (std::filesystem::is_regular_file(path) && isFileSupported(path, _fileExtensions))
stats.filesToScan ++;
if (stats.filesToScan % 250 == 0)
notifyInProgressIfNeeded(stats);
stats.filesToScan++;
notifyInProgressIfNeeded(stats);
}
itPath.increment(ec);
}
return true;
});
}
void
@@ -428,13 +425,14 @@ MediaScanner::scan(boost::system::error_code err)
if (_running)
checkDuplicatedAudioFiles(stats);
LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), duplicates = " << stats.duplicates.size();
// Now update all the track features if needed
fetchTrackFeatures(stats);
if (_running)
{
for (auto& addon : _addons)
addon->preScanComplete();
}
LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << "/" << stats.featuresToFetch <<", duplicates = " << stats.duplicates.size();
LMS_LOG(DBUPDATER, INFO) << "Optimizing db...";
_dbSession.optimize();
LMS_LOG(DBUPDATER, INFO) << "Optimize db done!";
if (_running)
{
@@ -457,41 +455,104 @@ MediaScanner::scan(boost::system::error_code err)
_curState = State::NotScheduled;
_inProgressScanStats.reset();
}
}
LMS_LOG(DBUPDATER, INFO) << "Optimizing db...";
_dbSession.optimize();
LMS_LOG(DBUPDATER, INFO) << "Optimize db done!";
bool
MediaScanner::fetchTrackFeatures(Database::IdType trackId, const UUID& MBID)
{
std::map<std::string, double> features;
LMS_LOG(DBUPDATER, INFO) << "Fetching low level features for track '" << MBID.getAsString() << "'";
const std::string data {AcousticBrainz::extractLowLevelFeatures(MBID)};
if (data.empty())
{
LMS_LOG(DBUPDATER, ERROR) << "Track " << trackId << ", MBID = '" << MBID.getAsString() << "': cannot extract features using AcousticBrainz";
return false;
}
{
auto uniqueTransaction {_dbSession.createUniqueTransaction()};
Wt::Dbo::ptr<Database::Track> track {Database::Track::getById(_dbSession, trackId)};
if (!track)
return false;
Database::TrackFeatures::create(_dbSession, track, data);
}
return true;
}
void
MediaScanner::fetchTrackFeatures(ScanStats& stats)
{
if (_recommendationEngineType != ScanSettings::RecommendationEngineType::Features)
return;
LMS_LOG(DBUPDATER, INFO) << "Fetching missing track features...";
struct TrackInfo
{
Database::IdType id;
UUID mbid;
};
const auto tracksToFetch {[&]()
{
std::vector<TrackInfo> res;
auto transaction {_dbSession.createSharedTransaction()};
auto tracks {Database::Track::getAllWithMBIDAndMissingFeatures(_dbSession)};
for (const auto& track : tracks)
res.emplace_back(TrackInfo {track.id(), *track->getMBID()});
return res;
}()};
stats.featuresToFetch = tracksToFetch.size();
LMS_LOG(DBUPDATER, INFO) << "Found " << tracksToFetch.size() << " track(s) to fetch!";
for (const TrackInfo& trackToFetch : tracksToFetch)
{
if (!_running)
return;
if (fetchTrackFeatures(trackToFetch.id, trackToFetch.mbid))
stats.featuresFetched++;
}
LMS_LOG(DBUPDATER, INFO) << "Track features fetched!";
}
void
MediaScanner::refreshScanSettings()
{
{
auto transaction {_dbSession.createSharedTransaction()};
auto transaction {_dbSession.createSharedTransaction()};
ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)};
ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)};
LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion();
LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion();
_scanVersion = scanSettings->getScanVersion();
_startTime = scanSettings->getUpdateStartTime();
_updatePeriod = scanSettings->getUpdatePeriod();
_scanVersion = scanSettings->getScanVersion();
_startTime = scanSettings->getUpdateStartTime();
_updatePeriod = scanSettings->getUpdatePeriod();
_fileExtensions = scanSettings->getAudioFileExtensions();
_mediaDirectory = scanSettings->getMediaDirectory();
_fileExtensions = scanSettings->getAudioFileExtensions();
_mediaDirectory = scanSettings->getMediaDirectory();
_recommendationEngineType = scanSettings->getRecommendationEngineType();
auto clusterTypes = scanSettings->getClusterTypes();
std::set<std::string> clusterTypeNames;
auto clusterTypes = scanSettings->getClusterTypes();
std::set<std::string> clusterTypeNames;
std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes),
std::inserter(clusterTypeNames, clusterTypeNames.begin()),
[](ClusterType::pointer clusterType) { return clusterType->getName(); });
std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes),
std::inserter(clusterTypeNames, clusterTypeNames.begin()),
[](ClusterType::pointer clusterType) { return clusterType->getName(); });
_metadataParser.setClusterTypeNames(clusterTypeNames);
}
_metadataParser->setClusterTypeNames(clusterTypeNames);
for (auto& addon : _addons)
addon->refreshSettings();
}
void
@@ -519,8 +580,6 @@ MediaScanner::notifyInProgressIfNeeded(const ScanStats& stats)
void
MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats)
{
notifyInProgressIfNeeded(stats);
Wt::WDateTime lastWriteTime;
try
{
@@ -548,7 +607,7 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S
}
}
std::optional<MetaData::Track> trackInfo {_metadataParser.parse(file)};
std::optional<MetaData::Track> trackInfo {_metadataParser->parse(file)};
if (!trackInfo)
{
stats.errors.emplace_back(file, ScanErrorType::CannotParseFile);
@@ -666,6 +725,7 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S
track.modify()->setYear(*trackInfo->originalYear);
track.modify()->setMBID(trackInfo->musicBrainzRecordID);
track.modify()->setFeatures({}); // TODO: only if MBID changed?
track.modify()->setHasCover(trackInfo->hasCover);
track.modify()->setCopyright(trackInfo->copyright);
track.modify()->setCopyrightURL(trackInfo->copyrightURL);
@@ -674,41 +734,32 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S
void
MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats)
{
std::error_code ec;
std::filesystem::recursive_directory_iterator itPath {_mediaDirectory, std::filesystem::directory_options::follow_directory_symlink, ec};
if (ec)
exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
{
LMS_LOG(DBUPDATER, ERROR) << "Cannot iterate over '" << mediaDirectory.string() << "': " << ec.message();
stats.errors.emplace_back(ScanError {mediaDirectory, ScanErrorType::CannotReadFile, ec.message()});
return;
}
std::filesystem::recursive_directory_iterator itEnd;
while (_running && itPath != itEnd)
{
const std::filesystem::path& path {*itPath};
if (!_running)
return false;
if (ec)
{
LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message();
stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()});
}
else if (std::filesystem::is_regular_file(path))
else if (isFileSupported(path, _fileExtensions))
{
if (isFileSupported(path, _fileExtensions))
scanAudioFile(path, forceScan, stats );
scanAudioFile(path, forceScan, stats );
notifyInProgressIfNeeded(stats);
}
itPath.increment(ec);
}
return true;
});
notifyInProgress(stats);
}
// Check if a file exists and is still in a media directory
static bool
checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory, const std::set<std::filesystem::path>& extensions)
checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory, const std::unordered_set<std::filesystem::path>& extensions)
{
try
{
@@ -769,6 +820,8 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
stats.deletions++;
}
}
notifyInProgressIfNeeded(stats);
}
}
src/scanner/MediaScanner.hpp → src/libs/scanner/impl/MediaScanner.hpp
+27 -42
View File
@@ -29,56 +29,39 @@
#include <boost/asio/system_timer.hpp>
#include "database/Types.hpp"
#include "database/ScanSettings.hpp"
#include "database/Session.hpp"
#include "metadata/TagLibParser.hpp"
#include "metadata/IParser.hpp"
#include "scanner/IMediaScanner.hpp"
#include "MediaScannerAddon.hpp"
#include "MediaScannerStats.hpp"
class UUID;
namespace Scanner {
class MediaScanner
class MediaScanner : public IMediaScanner
{
public:
MediaScanner(Database::Db& db);
~MediaScanner();
void setAddon(MediaScannerAddon& addon);
MediaScanner(const MediaScanner&) = delete;
MediaScanner(MediaScanner&&) = delete;
MediaScanner& operator=(const MediaScanner&) = delete;
MediaScanner& operator=(MediaScanner&&) = delete;
void start();
void stop();
void restart();
void start() override;
void stop() override;
void restart() override;
// Async requests
void requestImmediateScan();
void requestReschedule();
void requestImmediateScan() override;
void requestReschedule() override ;
Status getStatus() override;
enum class State
{
NotScheduled,
Scheduled,
InProgress,
};
struct Status
{
State currentState {State::NotScheduled};
Wt::WDateTime nextScheduledScan;
std::optional<ScanStats> lastCompleteScanStats;
std::optional<ScanProgressStats> inProgressScanStats;
};
Status getStatus();
// Called just after scan complete
Wt::Signal<>& scanComplete() { return _sigScanComplete; }
// Called during scan in progress
Wt::Signal<ScanProgressStats>& scanInProgress() { return _sigScanInProgress; }
// Called after a schedule
Wt::Signal<Wt::WDateTime>& scheduled() { return _sigScheduled; }
Wt::Signal<>& scanComplete() override { return _sigScanComplete; }
Wt::Signal<ScanProgressStats>& scanInProgress() override { return _sigScanInProgress; }
Wt::Signal<Wt::WDateTime>& scheduled() override { return _sigScheduled; }
private:
@@ -90,6 +73,8 @@ class MediaScanner
void scan(boost::system::error_code ec);
void scanMediaDirectory( const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats);
bool fetchTrackFeatures(Database::IdType trackId, const UUID& MBID);
void fetchTrackFeatures(ScanStats& stats);
// Helpers
void refreshScanSettings();
@@ -103,7 +88,7 @@ class MediaScanner
void notifyInProgressIfNeeded(const ScanStats& stats);
void notifyInProgress(const ScanStats& stats);
bool _running {false};
bool _running {};
Wt::WIOService _ioService;
boost::asio::system_timer _scheduleTimer {_ioService};
Wt::Signal<> _sigScanComplete;
@@ -111,8 +96,7 @@ class MediaScanner
std::chrono::system_clock::time_point _lastScanInProgressEmit {};
Wt::Signal<Wt::WDateTime> _sigScheduled;
Database::Session _dbSession;
MetaData::TagLibParser _metadataParser;
std::vector<MediaScannerAddon*> _addons;
std::unique_ptr<MetaData::IParser> _metadataParser;
std::mutex _statusMutex;
State _curState {State::NotScheduled};
@@ -121,11 +105,12 @@ class MediaScanner
Wt::WDateTime _nextScheduledScan;
// Current scan settings
std::size_t _scanVersion {};
Wt::WTime _startTime;
std::size_t _scanVersion {};
Wt::WTime _startTime;
Database::ScanSettings::UpdatePeriod _updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
std::set<std::filesystem::path> _fileExtensions;
std::unordered_set<std::filesystem::path> _fileExtensions;
std::filesystem::path _mediaDirectory;
Database::ScanSettings::RecommendationEngineType _recommendationEngineType;
}; // class MediaScanner
src/scanner/MediaScannerStats.cpp → src/libs/scanner/impl/MediaScannerStats.cpp
+1 -1
View File
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "MediaScannerStats.hpp"
#include "scanner/MediaScannerStats.hpp"
namespace Scanner {
src/libs/scanner/include/scanner/IMediaScanner.hpp
+82
View File
@@ -0,0 +1,82 @@
/*
* Copyright (C) 2013 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <optional>
#include <Wt/WDateTime.h>
#include <Wt/WSignal.h>
#include "MediaScannerStats.hpp"
namespace Database
{
class Db;
}
namespace Scanner {
class IMediaScanner
{
public:
virtual ~IMediaScanner() = default;
virtual void start() = 0;
virtual void stop() = 0;
virtual void restart() = 0;
// Async requests
virtual void requestImmediateScan() = 0;
virtual void requestReschedule() = 0;
enum class State
{
NotScheduled,
Scheduled,
InProgress,
};
struct Status
{
State currentState {State::NotScheduled};
Wt::WDateTime nextScheduledScan;
std::optional<ScanStats> lastCompleteScanStats;
std::optional<ScanProgressStats> inProgressScanStats;
};
virtual Status getStatus() = 0;
// Called just after scan complete
virtual Wt::Signal<>& scanComplete() = 0;
// Called during scan in progress
virtual Wt::Signal<ScanProgressStats>& scanInProgress() = 0;
// Called after a schedule
virtual Wt::Signal<Wt::WDateTime>& scheduled() = 0;
};
std::unique_ptr<IMediaScanner> createMediaScanner(Database::Db& db);
} // Scanner
src/scanner/MediaScannerStats.hpp → src/libs/scanner/include/scanner/MediaScannerStats.hpp
+4 -1
View File
@@ -77,10 +77,13 @@ namespace Scanner {
std::size_t skips {}; // no change since last scan
std::size_t scans {}; // actually scanned filed
std::size_t additions {}; // Added in DB
std::size_t additions {}; // added in DB
std::size_t deletions {}; // removed from DB
std::size_t updates {}; // updated file in DB
std::size_t featuresFetched {}; // features fetched in DB
std::size_t featuresToFetch {}; // features to be fetched in DB
std::vector<ScanError> errors;
std::vector<ScanDuplicate> duplicates;
src/libs/som/CMakeLists.txt
+20
View File
@@ -0,0 +1,20 @@
add_library(lmssom STATIC
impl/DataNormalizer.cpp
impl/Network.cpp
)
target_include_directories(lmssom INTERFACE
include
)
target_include_directories(lmssom PRIVATE
include
)
target_link_libraries(lmssom PUBLIC
lmsutils
)
set_property(TARGET lmssom PROPERTY POSITION_INDEPENDENT_CODE ON)
src/similarity/features/som/DataNormalizer.cpp → src/libs/som/impl/DataNormalizer.cpp
+1 -1
View File
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "DataNormalizer.hpp"
#include "som/DataNormalizer.hpp"
#include <algorithm>
#include <numeric>
src/similarity/features/som/Network.cpp → src/libs/som/impl/Network.cpp
+3 -3
View File
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Network.hpp"
#include "som/Network.hpp"
#include <algorithm>
#include <chrono>
@@ -201,9 +201,9 @@ Network::getClosestRefVectorPosition(const InputVector& data, InputVector::Dista
}
std::optional<Position>
Network::getClosestRefVectorPosition(const std::set<Position>& refVectorsPosition, InputVector::Distance maxDistance) const
Network::getClosestRefVectorPosition(const std::unordered_set<Position>& refVectorsPosition, InputVector::Distance maxDistance) const
{
std::set<Position> neighboursPosition;
std::unordered_set<Position> neighboursPosition;
for (const Position& refVectorPosition : refVectorsPosition)
{
if (refVectorPosition.y > 0)
src/similarity/features/som/DataNormalizer.hpp → src/libs/som/include/som/DataNormalizer.hpp
View File

Some files were not shown because too many files have changed in this diff Show More