diff --git a/.travis.yml b/.travis.yml index f53ac887..f2f798b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,15 +16,13 @@ matrix: - gcc before_install: - eval "${MATRIX_EVAL}" -- sudo apt-get install build-essential autoconf automake cmake libboost-all-dev libconfig++-dev libavcodec-dev libavutil-dev libavformat-dev ffmpeg libmagick++-dev libpstreams-dev libconfig++-dev libpstreams-dev libtag1-dev +- sudo apt-get install build-essential cmake libboost-all-dev libconfig++-dev libavcodec-dev libavutil-dev libavformat-dev ffmpeg libmagick++-dev libpstreams-dev libconfig++-dev libpstreams-dev libtag1-dev - git clone https://github.com/emweb/wt.git wt - pushd wt; - git checkout 4.1.0 - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr && sudo make install - popd -script: autoreconf -vfi && CXXFLAGS="-Wall -Wextra -Werror" ./configure --enable-tools && make distcheck +script: cmake -DCMAKE_BUILD_TYPE=Release . && make && make test env: global: - MAKEFLAGS="-j 2" -after_failure: - - cat config.log diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..516595ee --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.10) + +project(lms) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +include(CTest) +find_package(PkgConfig) +pkg_check_modules(IMAGEMAGICKXX REQUIRED ImageMagick++) + +add_subdirectory(src) + +install(DIRECTORY approot DESTINATION share/lms) +install(DIRECTORY docroot DESTINATION share/lms) +install(FILES systemd/default.service DESTINATION share/lms) +install(FILES conf/lms.conf DESTINATION share/lms) + diff --git a/Makefile.am b/Makefile.am deleted file mode 100644 index a82bd6d3..00000000 --- a/Makefile.am +++ /dev/null @@ -1,56 +0,0 @@ - -AUTOMAKE_OPTIONS = dist-bzip2 no-dist-gzip - -SUBDIRS = test tools src - -lms_docrootdir=$(pkgdatadir)/docroot -lms_approotdir=$(pkgdatadir)/approot - -lms_cssdir=$(lms_docrootdir)/css -lms_jsdir=$(lms_docrootdir)/js -lms_imagesdir=$(lms_approotdir)/images - -dist_pkgdata_DATA = \ - systemd/default.service \ - conf/lms.conf - -dist_lms_css_DATA = \ - docroot/css/lms.css - -dist_lms_js_DATA = \ - docroot/js/bootstrap-notify.js \ - docroot/js/jquery-1.10.2.min.js \ - docroot/js/mediaplayer.js - -dist_lms_images_DATA = \ - approot/images/unknown-cover.jpg \ - approot/images/unknown-artist.jpg - -dist_lms_approot_DATA = \ - approot/admin-database.xml \ - approot/admin-user.xml \ - approot/admin-users.xml \ - approot/admin-initwizard.xml \ - approot/artist.xml \ - approot/artistinfo.xml \ - approot/artistlink.xml \ - approot/artists.xml \ - approot/artistsinfo.xml \ - approot/error.xml \ - approot/explore.xml \ - approot/login.xml \ - approot/mediaplayer.xml \ - approot/messages.xml \ - approot/messages_fr.xml \ - approot/playhistory.xml \ - approot/playqueue.xml \ - approot/release.xml \ - approot/releaseinfo.xml \ - approot/releaselink.xml \ - approot/releases.xml \ - approot/releasesinfo.xml \ - approot/settings.xml \ - approot/templates.xml \ - approot/tracks.xml \ - approot/tracksinfo.xml - diff --git a/README.md b/README.md index 6e07b808..25b903a4 100644 --- a/README.md +++ b/README.md @@ -17,28 +17,33 @@ A [demo](http://lms.demo.poupon.io) instance is available, with the following li * Audio transcode for maximum interoperability and low bandwith requirements * Persistent play queue across sessions * Subsonic API -* Album artist +* Compilation support * Multi-value tags: artists, genres, ... * Custom tags (ex: _mood_, _genre_, _albummood_, _albumgrouping_, ...) * MusicBrainzID support to handle duplicated artist and release names -* Playlists, (only using Subsonic API for now) -* Starred Album/Artist/Tracks (only using Subsonic API for now) * _Systemd_ integration +* Subsonic-only features: + * Playlists + * Starred Album/Artist/Tracks + * Bookmarks -## Recommendation engine +## Music discovery _LMS_ provides several ways to help you find the music you like: * Tag-based filters (ex: _Rock_, _Metal_ and _Aggressive_, _Electronic_ and _Relaxed_, ...) * Recommendations for similar artists and albums -* Radio mode +* Radio mode, based on what is in the current playqueue * Searches in album, artist and track names * Most played/Recently added music -The recommendation engine makes use of [Self-Organizing Maps](https://en.wikipedia.org/wiki/Self-organizing_map).
-__Notes__: -* constructing the map requires significant computation time on large collections (ex: half an hour for 40k tracks) -* audio data is pulled from [AcousticBrainz](https://acousticbrainz.org/). Therefore your music files must contain the [MusicBrainz Identifier](https://musicbrainz.org/doc/MusicBrainz_Identifier) for the recommendation engine to work properly (otherwise, only tag-based recommendations are provided) -* to use the _self-organizing map_ based engine, you have to enable it first in the settings panel. +The recommendation engine uses two different sources: +1. Tags that are present in the audio files +2. Acoustic similarities of the audio files, using a trained [Self-Organizing Map](https://en.wikipedia.org/wiki/Self-organizing_map) + +__Notes on the self-organizing map__: +* training the map requires significant computation time on large collections (ex: half an hour for 40k tracks) +* audio acoustic data is pulled from [AcousticBrainz](https://acousticbrainz.org/). Therefore your audio files _must_ contain the [MusicBrainz Identifier](https://musicbrainz.org/doc/MusicBrainz_Identifier). +* to enable the audio similarity source, you have to enable it first in the settings panel. ## Subsonic API The API version implemented is 1.12.0 and has been tested on _Android_ using the official application, _Ultrasonic_ and _DSub_. @@ -80,7 +85,7 @@ __Notes__: * a C++17 compiler is needed * ffmpeg version 4 minimum is required ```sh -apt-get install g++ autoconf automake libboost-filesystem-dev libboost-system-dev libavutil-dev libavformat-dev libmagick++-dev libpstreams-dev libconfig++-dev libpstreams-dev ffmpeg libtag1-dev +apt-get install g++ cmake libboost-system-dev libavutil-dev libavformat-dev libmagick++-dev libconfig++-dev libpstreams-dev ffmpeg libtag1-dev ``` You also need _Wt4_, which is not packaged yet on _Debian_. See [installation instructions](https://www.webtoolkit.eu/wt/doc/reference/html/InstallationUnix.html).
@@ -92,16 +97,10 @@ Get the latest stable release and build it: ```sh git clone https://github.com/epoupon/lms.git lms cd lms -autoreconf -vfi mkdir build -cd build -../configure --prefix=/usr +cmake .. -DCMAKE_BUILD_TYPE=Release ``` -configure will report any missing library. - -__Note__: in order to customize the installation directories, you can use the following options of the `configure` script: -* _--prefix_ (defaults to `/usr/local`). -* _--bindir_ (defaults to `$PREFIX/bin`). +__Note__: in order to customize the installation directory, you can use the _-DCMAKE_INSTALL_PREFIX_ option (defaults to `/usr/local`). ```sh make diff --git a/approot/admin-database.xml b/approot/admin-database.xml index e5c8ec6f..6e2b08ec 100644 --- a/approot/admin-database.xml +++ b/approot/admin-database.xml @@ -47,14 +47,14 @@
-
diff --git a/approot/messages.xml b/approot/messages.xml index 17d5ded8..20677bc8 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -39,9 +39,9 @@ Monthly Never Media root directory -Recommendation engine -Tags based -Audio analysis based +Recommendation engine +Tags based +Audio analysis based Scan complete: {1} total files, {2} additions, {3} updates, {4} deletions, {5} duplicates, {6} errors Scan launched! Scan options diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index 2d61dc1d..934cd400 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -39,9 +39,9 @@ Tous les mois Jamais Dossier racine des fichiers de musique -Moteur de recommandation -Basé sur les tags -Basé sur l'analyse audio +Moteur de recommandation +Basé sur les tags +Basé sur l'analyse audio Scan terminé : {1} fichiers, {2} ajouts, {3} mises à jour, {4} suppressions, {5} duplicatas, {6} erreurs Scan lancé ! Options diff --git a/configure.ac b/configure.ac deleted file mode 100644 index ac5aaf5e..00000000 --- a/configure.ac +++ /dev/null @@ -1,94 +0,0 @@ -AC_PREREQ(2.59) -AC_INIT(lms, 3.6.3, test@test) -AM_INIT_AUTOMAKE -AC_CONFIG_HEADER(src/config/config.h) - -AC_LANG_CPLUSPLUS - -AC_PROG_CXX - -PKG_CHECK_MODULES(IMAGEMAGICKXX, "ImageMagick++", [ HAVE_IMAGEMAGICKXX=yes ], [ ]) -if test -n "$HAVE_IMAGEMAGICKXX"; then - MAGICKXX_CFLAGS="$IMAGEMAGICKXX_CFLAGS" - MAGICKXX_LIBS="$IMAGEMAGICKXX_LIBS" -fi -AC_SUBST(MAGICKXX_CFLAGS) -AC_SUBST(MAGICKXX_LIBS) - -AC_CHECK_HEADERS([Wt/WApplication.h pstreams/pstream.h boost/asio.hpp], - [], - [AC_MSG_ERROR([Header not found or unusable !])]) - -AC_CHECK_LIB([stdc++fs], - [main], - , - [AC_MSG_ERROR([lib filesystem not found!])]) - -AC_CHECK_LIB([pthread], - [pthread_rwlock_unlock], - , - [AC_MSG_ERROR([libpthread not found!])]) - -AC_CHECK_LIB([boost_system], - [main], - , - [AC_MSG_ERROR([libboost_system not found!])]) - -AC_CHECK_LIB([wt], - [main], - , - [AC_MSG_ERROR([libwt not found!])]) - -AC_CHECK_LIB([wtdbo], - [main], - , - [AC_MSG_ERROR([libwtdbo not found!])]) - -AC_CHECK_LIB([wtdbosqlite3], - [main], - , - [AC_MSG_ERROR([libwtdbosqlite3 not found!])]) - -AC_CHECK_LIB([wthttp], - [main], - , - [AC_MSG_ERROR([libwthttp not found!])]) - - -AC_CHECK_LIB([avutil], - [av_free], - , - [AC_MSG_ERROR([libavutil not found!])]) - -AC_CHECK_LIB([avformat], - [av_read_frame], - , - [AC_MSG_ERROR([libavformat not found!])]) - -AC_CHECK_LIB( [tag], - [main], - , - [AC_MSG_ERROR([libtag not found!])]) - -AC_CHECK_LIB( [config++], - [main], - , - [AC_MSG_ERROR([libconfig++ not found!])]) - -AC_CONFIG_FILES([Makefile - src/Makefile - test/Makefile - tools/Makefile - tools/similarity/Makefile - tools/similarity-parameters/Makefile - tools/metadata/Makefile]) - -AC_ARG_ENABLE([tools], - [AC_HELP_STRING([--enable-tools], [Build the tools])], - [:], - [enable_tools=no]) - -AM_CONDITIONAL([BUILD_TOOLS], [test "$enable_tools" = "yes"]) - -AC_OUTPUT - diff --git a/docker/Dockerfile b/docker/Dockerfile index 2a94c55d..4ac83cd4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,7 +7,7 @@ ARG FFMPEG_VERSION=4.1.4 ARG WT_VERSION=4.2.0 ARG IMAGEMAGICK6_VERSION=6.9.10-71 ARG PSTREAMS_VERSION=1.0.1 -ARG LMS_VERSION=3.5.0 +ARG LMS_VERSION=v3.6.3 ARG PREFIX="/tmp/install" @@ -153,23 +153,21 @@ RUN \ # LMS RUN \ DIR=/tmp/lms && mkdir -p ${DIR} && cd ${DIR} && \ - curl -sLO https://github.com/epoupon/lms/archive/v${LMS_VERSION}.tar.gz && \ - tar -x --strip-components=1 -f v${LMS_VERSION}.tar.gz + curl -sL https://github.com/epoupon/lms/archive/${LMS_VERSION}.tar.gz -o ${LMS_VERSION}.tar.gz && \ + tar -x --strip-components=1 -f ${LMS_VERSION}.tar.gz RUN \ DIR=/tmp/lms && mkdir -p ${DIR} && cd ${DIR} && \ - autoreconf -vfi && \ - PKG_CONFIG_PATH=/tmp/install/lib/pkgconfig CXXFLAGS="-O2 -I${PREFIX}/include" LDFLAGS="-L${PREFIX}/lib -Wl,--rpath-link=${PREFIX}/lib" ./configure --prefix=${PREFIX} && \ + PKG_CONFIG_PATH=/tmp/install/lib/pkgconfig CXXFLAGS="-I${PREFIX}/include -L${PREFIX}/lib -Wl,--rpath-link=${PREFIX}/lib" cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} && \ make && \ make install && \ - make distclean && \ mkdir -p ${PREFIX}/etc/ && \ cp conf/lms.conf ${PREFIX}/etc # Now copy all the stuff installed in a new folder (/tmp/fakeroot/) RUN \ mkdir -p /tmp/fakeroot/bin && \ - for bin in ${PREFIX}/bin/ffmpeg ${PREFIX}/bin/lms; \ + for bin in ${PREFIX}/bin/ffmpeg ${PREFIX}/bin/lms*; \ do \ strip --strip-all $bin && \ cp $bin /tmp/fakeroot/bin/; \ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 00000000..c796f0e9 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,11 @@ +add_compile_options(-Wall -Wextra -pedantic) + +add_subdirectory(libs) +add_subdirectory(lms) +add_subdirectory(tools) + +if(BUILD_TESTING) + add_subdirectory(test) +endif() + + diff --git a/src/Makefile.am b/src/Makefile.am deleted file mode 100644 index 4fd65f58..00000000 --- a/src/Makefile.am +++ /dev/null @@ -1,172 +0,0 @@ -bin_PROGRAMS = lms - -lms_SOURCES = \ - $(srcdir)/api/subsonic/SubsonicId.cpp \ - $(srcdir)/api/subsonic/SubsonicId.hpp \ - $(srcdir)/api/subsonic/SubsonicResource.cpp \ - $(srcdir)/api/subsonic/SubsonicResource.hpp \ - $(srcdir)/api/subsonic/SubsonicResponse.cpp \ - $(srcdir)/api/subsonic/SubsonicResponse.hpp \ - $(srcdir)/auth/AuthTokenService.cpp \ - $(srcdir)/auth/AuthTokenService.hpp \ - $(srcdir)/auth/PasswordService.cpp \ - $(srcdir)/auth/PasswordService.hpp \ - $(srcdir)/auth/LoginThrottler.cpp \ - $(srcdir)/auth/LoginThrottler.hpp \ - $(srcdir)/av/AvInfo.cpp \ - $(srcdir)/av/AvInfo.hpp \ - $(srcdir)/av/AvTranscoder.cpp \ - $(srcdir)/av/AvTranscoder.hpp \ - $(srcdir)/av/AvTypes.cpp \ - $(srcdir)/av/AvTypes.hpp \ - $(srcdir)/cover/CoverArtGrabber.cpp \ - $(srcdir)/cover/CoverArtGrabber.hpp \ - $(srcdir)/database/Artist.cpp \ - $(srcdir)/database/Artist.hpp \ - $(srcdir)/database/Cluster.cpp \ - $(srcdir)/database/Cluster.hpp \ - $(srcdir)/database/Db.cpp \ - $(srcdir)/database/Db.hpp \ - $(srcdir)/database/TrackArtistLink.cpp \ - $(srcdir)/database/TrackArtistLink.hpp \ - $(srcdir)/database/TrackFeatures.cpp \ - $(srcdir)/database/TrackFeatures.hpp \ - $(srcdir)/database/TrackList.cpp \ - $(srcdir)/database/TrackList.hpp \ - $(srcdir)/database/Types.hpp \ - $(srcdir)/database/Release.cpp \ - $(srcdir)/database/Release.hpp \ - $(srcdir)/database/ScanSettings.cpp \ - $(srcdir)/database/ScanSettings.hpp \ - $(srcdir)/database/Session.cpp \ - $(srcdir)/database/Session.hpp \ - $(srcdir)/database/SessionPool.cpp \ - $(srcdir)/database/SessionPool.hpp \ - $(srcdir)/database/SqlQuery.cpp \ - $(srcdir)/database/SqlQuery.hpp \ - $(srcdir)/database/Track.cpp \ - $(srcdir)/database/Track.hpp \ - $(srcdir)/database/TrackBookmark.cpp \ - $(srcdir)/database/TrackBookmark.hpp \ - $(srcdir)/database/User.cpp \ - $(srcdir)/database/User.hpp \ - $(srcdir)/image/Image.cpp \ - $(srcdir)/image/Image.hpp \ - $(srcdir)/main/main.cpp \ - $(srcdir)/metadata/AvFormat.cpp \ - $(srcdir)/metadata/AvFormat.hpp \ - $(srcdir)/metadata/MetaData.hpp \ - $(srcdir)/metadata/TagLibParser.cpp \ - $(srcdir)/metadata/TagLibParser.hpp \ - $(srcdir)/scanner/MediaScanner.cpp \ - $(srcdir)/scanner/MediaScanner.hpp \ - $(srcdir)/scanner/MediaScannerStats.cpp \ - $(srcdir)/scanner/MediaScannerStats.hpp \ - $(srcdir)/scanner/MediaScannerAddon.hpp \ - $(srcdir)/similarity/SimilaritySearcher.cpp \ - $(srcdir)/similarity/SimilaritySearcher.hpp \ - $(srcdir)/similarity/cluster/SimilarityClusterSearcher.cpp \ - $(srcdir)/similarity/cluster/SimilarityClusterSearcher.hpp \ - $(srcdir)/similarity/features/AcousticBrainzUtils.cpp \ - $(srcdir)/similarity/features/AcousticBrainzUtils.hpp \ - $(srcdir)/similarity/features/SimilarityFeaturesCache.cpp \ - $(srcdir)/similarity/features/SimilarityFeaturesCache.hpp \ - $(srcdir)/similarity/features/SimilarityFeaturesDefs.cpp \ - $(srcdir)/similarity/features/SimilarityFeaturesDefs.hpp \ - $(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.cpp \ - $(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.hpp \ - $(srcdir)/similarity/features/SimilarityFeaturesSearcher.cpp \ - $(srcdir)/similarity/features/SimilarityFeaturesSearcher.hpp \ - $(srcdir)/similarity/features/som/DataNormalizer.cpp \ - $(srcdir)/similarity/features/som/DataNormalizer.hpp \ - $(srcdir)/similarity/features/som/InputVector.hpp \ - $(srcdir)/similarity/features/som/Matrix.hpp \ - $(srcdir)/similarity/features/som/Network.cpp \ - $(srcdir)/similarity/features/som/Network.hpp \ - $(srcdir)/ui/Auth.cpp \ - $(srcdir)/ui/Auth.hpp \ - $(srcdir)/ui/LmsApplication.cpp \ - $(srcdir)/ui/LmsApplication.hpp \ - $(srcdir)/ui/LmsApplicationException.hpp \ - $(srcdir)/ui/LmsApplicationGroup.cpp \ - $(srcdir)/ui/LmsApplicationGroup.hpp \ - $(srcdir)/ui/MediaPlayer.cpp \ - $(srcdir)/ui/MediaPlayer.hpp \ - $(srcdir)/ui/PlayQueueView.cpp \ - $(srcdir)/ui/PlayQueueView.hpp \ - $(srcdir)/ui/PlayHistoryView.cpp \ - $(srcdir)/ui/PlayHistoryView.hpp \ - $(srcdir)/ui/SettingsView.cpp \ - $(srcdir)/ui/SettingsView.hpp \ - $(srcdir)/ui/TrackStringUtils.cpp \ - $(srcdir)/ui/TrackStringUtils.hpp \ - $(srcdir)/ui/admin/DatabaseSettingsView.cpp \ - $(srcdir)/ui/admin/DatabaseSettingsView.hpp \ - $(srcdir)/ui/admin/DatabaseStatus.cpp \ - $(srcdir)/ui/admin/DatabaseStatus.hpp \ - $(srcdir)/ui/admin/InitWizardView.cpp \ - $(srcdir)/ui/admin/InitWizardView.hpp \ - $(srcdir)/ui/admin/UserView.cpp \ - $(srcdir)/ui/admin/UserView.hpp \ - $(srcdir)/ui/admin/UsersView.cpp \ - $(srcdir)/ui/admin/UsersView.hpp \ - $(srcdir)/ui/common/Validators.cpp \ - $(srcdir)/ui/common/Validators.hpp \ - $(srcdir)/ui/common/ValueStringModel.hpp \ - $(srcdir)/ui/explore/ArtistInfoView.cpp \ - $(srcdir)/ui/explore/ArtistInfoView.hpp \ - $(srcdir)/ui/explore/ArtistLink.cpp \ - $(srcdir)/ui/explore/ArtistLink.hpp \ - $(srcdir)/ui/explore/ArtistsInfoView.cpp \ - $(srcdir)/ui/explore/ArtistsInfoView.hpp \ - $(srcdir)/ui/explore/ArtistView.cpp \ - $(srcdir)/ui/explore/ArtistView.hpp \ - $(srcdir)/ui/explore/ArtistsView.cpp \ - $(srcdir)/ui/explore/ArtistsView.hpp \ - $(srcdir)/ui/explore/Explore.cpp \ - $(srcdir)/ui/explore/Explore.hpp \ - $(srcdir)/ui/explore/Filters.cpp \ - $(srcdir)/ui/explore/Filters.hpp \ - $(srcdir)/ui/explore/ReleaseInfoView.cpp \ - $(srcdir)/ui/explore/ReleaseInfoView.hpp \ - $(srcdir)/ui/explore/ReleaseLink.cpp \ - $(srcdir)/ui/explore/ReleaseLink.hpp \ - $(srcdir)/ui/explore/ReleasesInfoView.cpp \ - $(srcdir)/ui/explore/ReleasesInfoView.hpp \ - $(srcdir)/ui/explore/ReleasesView.cpp \ - $(srcdir)/ui/explore/ReleasesView.hpp \ - $(srcdir)/ui/explore/ReleaseView.cpp \ - $(srcdir)/ui/explore/ReleaseView.hpp \ - $(srcdir)/ui/explore/TracksInfoView.cpp \ - $(srcdir)/ui/explore/TracksInfoView.hpp \ - $(srcdir)/ui/explore/TracksView.cpp \ - $(srcdir)/ui/explore/TracksView.hpp \ - $(srcdir)/ui/resource/ImageResource.cpp \ - $(srcdir)/ui/resource/ImageResource.hpp \ - $(srcdir)/ui/resource/AudioResource.cpp \ - $(srcdir)/ui/resource/AudioResource.hpp \ - $(srcdir)/utils/Config.cpp \ - $(srcdir)/utils/Config.hpp \ - $(srcdir)/utils/Exception.hpp \ - $(srcdir)/utils/Logger.cpp \ - $(srcdir)/utils/Logger.hpp \ - $(srcdir)/utils/NetAddress.cpp \ - $(srcdir)/utils/NetAddress.hpp \ - $(srcdir)/utils/Path.cpp \ - $(srcdir)/utils/Path.hpp \ - $(srcdir)/utils/Random.cpp \ - $(srcdir)/utils/Random.hpp \ - $(srcdir)/utils/Service.hpp \ - $(srcdir)/utils/StreamLogger.cpp \ - $(srcdir)/utils/StreamLogger.hpp \ - $(srcdir)/utils/String.cpp \ - $(srcdir)/utils/String.hpp \ - $(srcdir)/utils/Utils.hpp \ - $(srcdir)/utils/UUID.cpp \ - $(srcdir)/utils/UUID.hpp \ - $(srcdir)/utils/WtLogger.cpp \ - $(srcdir)/utils/WtLogger.hpp - -lms_CXXFLAGS=-std=c++17 -I$(srcdir)/ui $(MAGICKXX_CFLAGS) -D_REENTRANT -lms_LDADD=$(MAGICKXX_LIBS) - diff --git a/src/config/.empty b/src/config/.empty deleted file mode 100644 index e69de29b..00000000 diff --git a/src/cover/CoverArtGrabber.hpp b/src/cover/CoverArtGrabber.hpp deleted file mode 100644 index 07b4cc32..00000000 --- a/src/cover/CoverArtGrabber.hpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2015 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include -#include -#include -#include -#include - -#include "database/Types.hpp" - -#include "image/Image.hpp" - -namespace Database { - class Session; -} - -namespace CoverArt { - -class Grabber -{ - public: - Grabber(); - Grabber(const Grabber&) = delete; - Grabber& operator=(const Grabber&) = delete; - Grabber(Grabber&&) = delete; - Grabber& operator=(Grabber&&) = delete; - - void setDefaultCover(const std::filesystem::path& defaultCoverPath); - - std::vector getFromTrack(Database::Session& dbSession, Database::IdType trackId, Image::Format format, std::size_t size); - std::vector getFromRelease(Database::Session& dbSession, Database::IdType releaseId, Image::Format format, std::size_t size); - - private: - - Image::Image getFromTrack(Database::Session& dbSession, Database::IdType trackId, std::size_t size); - Image::Image getFromRelease(Database::Session& dbSession, Database::IdType releaseId, std::size_t size); - - std::optional getFromTrack(const std::filesystem::path& path) const; - std::vector getCoverPaths(const std::filesystem::path& directoryPath) const; - std::optional getFromDirectory(const std::filesystem::path& path) const; - - Image::Image getDefaultCover(std::size_t size); - - Image::Image _defaultCover; - - std::mutex _mutex; - std::map _defaultCovers; - - static inline const std::vector _fileExtensions {".jpg", ".jpeg", ".png", ".bmp"}; // TODO parametrize - - static inline const std::size_t _maxFileSize {10000000}; - - static inline const std::vector _preferredFileNames {"cover", "front"}; // TODO parametrize -}; - -} // namespace CoverArt - diff --git a/src/libs/CMakeLists.txt b/src/libs/CMakeLists.txt new file mode 100644 index 00000000..7302b579 --- /dev/null +++ b/src/libs/CMakeLists.txt @@ -0,0 +1,13 @@ + +add_subdirectory(auth) +add_subdirectory(av) +add_subdirectory(cover) +add_subdirectory(database) +add_subdirectory(metadata) +add_subdirectory(recommendation) +add_subdirectory(scanner) +add_subdirectory(som) +add_subdirectory(subsonic) +add_subdirectory(utils) + + diff --git a/src/libs/auth/CMakeLists.txt b/src/libs/auth/CMakeLists.txt new file mode 100644 index 00000000..9be20d0c --- /dev/null +++ b/src/libs/auth/CMakeLists.txt @@ -0,0 +1,28 @@ + +add_library(lmsauth SHARED + impl/AuthTokenService.cpp + impl/PasswordService.cpp + impl/LoginThrottler.cpp + ) + +target_include_directories(lmsauth INTERFACE + include + ) + +target_include_directories(lmsauth PRIVATE + include/ + ) + +target_link_libraries(lmsauth PRIVATE + lmsutils + lmsdatabase + ) + +target_link_libraries(lmsauth PUBLIC + pthread + boost_system + wt + ) + +install(TARGETS lmsauth DESTINATION lib) + diff --git a/src/auth/AuthTokenService.cpp b/src/libs/auth/impl/AuthTokenService.cpp similarity index 95% rename from src/auth/AuthTokenService.cpp rename to src/libs/auth/impl/AuthTokenService.cpp index 14cb933c..ba37b35c 100644 --- a/src/auth/AuthTokenService.cpp +++ b/src/libs/auth/impl/AuthTokenService.cpp @@ -26,11 +26,17 @@ #include #include "database/Session.hpp" +#include "database/User.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" namespace Auth { +std::unique_ptr createAuthTokenService(std::size_t maxThrottlerEntries) +{ + return std::make_unique(maxThrottlerEntries); +} + static const Wt::Auth::SHA1HashFunction sha1Function; AuthTokenService::AuthTokenService(std::size_t maxThrottlerEntries) diff --git a/src/auth/AuthTokenService.hpp b/src/libs/auth/impl/AuthTokenService.hpp similarity index 72% rename from src/auth/AuthTokenService.hpp rename to src/libs/auth/impl/AuthTokenService.hpp index b44ae3e7..36a9721a 100644 --- a/src/auth/AuthTokenService.hpp +++ b/src/libs/auth/impl/AuthTokenService.hpp @@ -21,14 +21,9 @@ #pragma once -#include -#include - -#include +#include "auth/IAuthTokenService.hpp" #include "LoginThrottler.hpp" -#include "database/User.hpp" -#include "database/Types.hpp" namespace Database { @@ -38,7 +33,7 @@ namespace Database namespace Auth { - class AuthTokenService + class AuthTokenService : public IAuthTokenService { public: @@ -52,30 +47,8 @@ namespace Auth { AuthTokenService(AuthTokenService&&) = delete; AuthTokenService& operator=(AuthTokenService&&) = delete; - - // Auth Token services - struct AuthTokenProcessResult - { - enum class State - { - Found, - Throttled, - NotFound, - }; - - struct AuthTokenInfo - { - Database::IdType userId; - Wt::WDateTime expiry; - }; - - State state {State::NotFound}; - std::optional authTokenInfo {}; - }; - - // Removed if found - AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue); - std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry); + AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue) override; + std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry) override; private: diff --git a/src/auth/LoginThrottler.cpp b/src/libs/auth/impl/LoginThrottler.cpp similarity index 100% rename from src/auth/LoginThrottler.cpp rename to src/libs/auth/impl/LoginThrottler.cpp diff --git a/src/auth/LoginThrottler.hpp b/src/libs/auth/impl/LoginThrottler.hpp similarity index 100% rename from src/auth/LoginThrottler.hpp rename to src/libs/auth/impl/LoginThrottler.hpp diff --git a/src/auth/PasswordService.cpp b/src/libs/auth/impl/PasswordService.cpp similarity index 95% rename from src/auth/PasswordService.cpp rename to src/libs/auth/impl/PasswordService.cpp index 1c592f45..9c17de2b 100644 --- a/src/auth/PasswordService.cpp +++ b/src/libs/auth/impl/PasswordService.cpp @@ -31,6 +31,11 @@ namespace Auth { +std::unique_ptr createPasswordService(std::size_t maxThrottlerEntries) +{ + return std::make_unique(maxThrottlerEntries); +} + PasswordService::PasswordService(std::size_t maxThrottlerEntries) : _loginThrottler{maxThrottlerEntries} { diff --git a/src/auth/PasswordService.hpp b/src/libs/auth/impl/PasswordService.hpp similarity index 60% rename from src/auth/PasswordService.hpp rename to src/libs/auth/impl/PasswordService.hpp index 303615be..99377e8b 100644 --- a/src/auth/PasswordService.hpp +++ b/src/libs/auth/impl/PasswordService.hpp @@ -17,17 +17,12 @@ * along with LMS. If not, see . */ -/* This file contains some classes in order to get info from file using the libavconv */ - #pragma once -#include - -#include +#include +#include "auth/IPasswordService.hpp" #include "LoginThrottler.hpp" -#include "database/User.hpp" -#include "database/Types.hpp" namespace Database { @@ -37,7 +32,7 @@ namespace Database namespace Auth { - class PasswordService + class PasswordService : public IPasswordService { public: @@ -53,39 +48,9 @@ namespace Auth { // Password services - enum class PasswordCheckResult - { - Match, - Mismatch, - Throttled, - }; - PasswordCheckResult checkUserPassword(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& loginName, const std::string& password); - Database::User::PasswordHash hashPassword(const std::string& password) const; - bool evaluatePasswordStrength(const std::string& loginName, const std::string& password) const; - - // Auth Token services - struct AuthTokenProcessResult - { - enum class State - { - Found, - Throttled, - NotFound, - }; - - struct AuthTokenInfo - { - Database::IdType userId; - Wt::WDateTime expiry; - }; - - State state; - std::optional authTokenInfo; - }; - - // Removed if found - AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue); - std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry); + PasswordCheckResult checkUserPassword(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& loginName, const std::string& password) override; + Database::User::PasswordHash hashPassword(const std::string& password) const override; + bool evaluatePasswordStrength(const std::string& loginName, const std::string& password) const override; private: diff --git a/src/libs/auth/include/auth/IAuthTokenService.hpp b/src/libs/auth/include/auth/IAuthTokenService.hpp new file mode 100644 index 00000000..f85148dd --- /dev/null +++ b/src/libs/auth/include/auth/IAuthTokenService.hpp @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +/* This file contains some classes in order to get info from file using the libavconv */ + +#pragma once + +#include +#include +#include +#include + +#include "database/Types.hpp" + +namespace Database +{ + class Session; +} + + +namespace Auth { + + class IAuthTokenService + { + public: + + // Auth Token services + struct AuthTokenProcessResult + { + enum class State + { + Found, + Throttled, + NotFound, + }; + + struct AuthTokenInfo + { + Database::IdType userId; + Wt::WDateTime expiry; + }; + + State state {State::NotFound}; + std::optional authTokenInfo {}; + }; + + // Removed if found + virtual AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue) = 0; + virtual std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry) = 0; + }; + + std::unique_ptr createAuthTokenService(std::size_t maxThrottlerEntryCount); + +} + diff --git a/src/libs/auth/include/auth/IPasswordService.hpp b/src/libs/auth/include/auth/IPasswordService.hpp new file mode 100644 index 00000000..fb5f9551 --- /dev/null +++ b/src/libs/auth/include/auth/IPasswordService.hpp @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +#include + +#include "database/User.hpp" +#include "database/Types.hpp" + +namespace Database +{ + class Session; +} + + +namespace Auth { + + class IPasswordService + { + public: + + virtual ~IPasswordService() = default; + + // Password services + enum class PasswordCheckResult + { + Match, + Mismatch, + Throttled, + }; + virtual PasswordCheckResult checkUserPassword(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& loginName, const std::string& password) = 0; + virtual Database::User::PasswordHash hashPassword(const std::string& password) const = 0; + virtual bool evaluatePasswordStrength(const std::string& loginName, const std::string& password) const = 0; + }; + + std::unique_ptr createPasswordService(std::size_t maxThrottlerEntryCount); + +} + diff --git a/src/libs/av/CMakeLists.txt b/src/libs/av/CMakeLists.txt new file mode 100644 index 00000000..c2d03eb6 --- /dev/null +++ b/src/libs/av/CMakeLists.txt @@ -0,0 +1,24 @@ + +add_library(lmsav SHARED + impl/AvInfo.cpp + impl/AvTranscoder.cpp + impl/AvTypes.cpp + ) + +target_include_directories(lmsav INTERFACE + include + ) + +target_include_directories(lmsav PRIVATE + include/ + ) + +# TODO make these private +target_link_libraries(lmsav PUBLIC + lmsutils + avformat + avutil + ) + +install(TARGETS lmsav DESTINATION lib) + diff --git a/src/av/AvInfo.cpp b/src/libs/av/impl/AvInfo.cpp similarity index 96% rename from src/av/AvInfo.cpp rename to src/libs/av/impl/AvInfo.cpp index 2f7d9ee8..e9b1bd8c 100644 --- a/src/av/AvInfo.cpp +++ b/src/libs/av/impl/AvInfo.cpp @@ -17,9 +17,7 @@ * along with LMS. If not, see . */ -#include "AvInfo.hpp" - -#include +#include "av/AvInfo.hpp" #include @@ -92,7 +90,7 @@ getMetaDataFromDictionnary(AVDictionary* dictionnary, std::map(tag->key)] = tag->value; + res[StringUtils::stringToUpper(tag->key)] = tag->value; } } @@ -141,7 +139,7 @@ MediaFile::getStreamInfo() const if (avstream->codecpar->codec_type != AVMEDIA_TYPE_AUDIO) continue; - res.push_back( {.id = i, .bitrate = static_cast(avstream->codecpar->bit_rate)} ); + res.push_back( {i, static_cast(avstream->codecpar->bit_rate)} ); } return res; diff --git a/src/av/AvTranscoder.cpp b/src/libs/av/impl/AvTranscoder.cpp similarity index 97% rename from src/av/AvTranscoder.cpp rename to src/libs/av/impl/AvTranscoder.cpp index 1dac0797..f98b3771 100644 --- a/src/av/AvTranscoder.cpp +++ b/src/libs/av/impl/AvTranscoder.cpp @@ -17,13 +17,13 @@ * along with LMS. If not, see . */ -#include "AvTranscoder.hpp" +#include "av/AvTranscoder.hpp" #include #include -#include "AvInfo.hpp" -#include "utils/Config.hpp" +#include "av/AvInfo.hpp" +#include "utils/IConfig.hpp" #include "utils/Path.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -38,7 +38,7 @@ static std::filesystem::path ffmpegPath; void Transcoder::init() { - ffmpegPath = ServiceProvider::get()->getPath("ffmpeg-file", "/usr/bin/ffmpeg"); + ffmpegPath = ServiceProvider::get()->getPath("ffmpeg-file", "/usr/bin/ffmpeg"); if (!std::filesystem::exists(ffmpegPath)) throw LmsException {"File '" + ffmpegPath.string() + "' does not exist!"}; } diff --git a/src/av/AvTypes.cpp b/src/libs/av/impl/AvTypes.cpp similarity index 93% rename from src/av/AvTypes.cpp rename to src/libs/av/impl/AvTypes.cpp index 9ad5bd5d..7e556294 100644 --- a/src/av/AvTypes.cpp +++ b/src/libs/av/impl/AvTypes.cpp @@ -17,9 +17,7 @@ * along with LMS. If not, see . */ -#include "AvTypes.hpp" - -#include +#include "av/AvTypes.hpp" namespace Av { @@ -34,7 +32,7 @@ const char* encodingToMimetype(Encoding encoding) case Encoding::WEBM_VORBIS: return "audio/webm"; } - throw AvException("Invalid encoding"); + throw AvException {"Invalid encoding"}; } } diff --git a/src/av/AvInfo.hpp b/src/libs/av/include/av/AvInfo.hpp similarity index 100% rename from src/av/AvInfo.hpp rename to src/libs/av/include/av/AvInfo.hpp diff --git a/src/av/AvTranscoder.hpp b/src/libs/av/include/av/AvTranscoder.hpp similarity index 100% rename from src/av/AvTranscoder.hpp rename to src/libs/av/include/av/AvTranscoder.hpp diff --git a/src/av/AvTypes.hpp b/src/libs/av/include/av/AvTypes.hpp similarity index 100% rename from src/av/AvTypes.hpp rename to src/libs/av/include/av/AvTypes.hpp diff --git a/src/libs/cover/CMakeLists.txt b/src/libs/cover/CMakeLists.txt new file mode 100644 index 00000000..62eb5d43 --- /dev/null +++ b/src/libs/cover/CMakeLists.txt @@ -0,0 +1,27 @@ + +add_library(lmscover SHARED + impl/CoverArtGrabber.cpp + impl/Image.cpp + ) + +target_include_directories(lmscover INTERFACE + include + ) + +target_include_directories(lmscover PRIVATE + include + ${IMAGEMAGICKXX_INCLUDE_DIRS} + ) + +target_compile_options(lmscover PRIVATE + ${IMAGEMAGICKXX_CFLAGS_OTHER} + ) + +target_link_libraries(lmscover PRIVATE + lmsav + lmsdatabase + ${IMAGEMAGICKXX_LIBRARIES} + ) + +install(TARGETS lmscover DESTINATION lib) + diff --git a/src/cover/CoverArtGrabber.cpp b/src/libs/cover/impl/CoverArtGrabber.cpp similarity index 86% rename from src/cover/CoverArtGrabber.cpp rename to src/libs/cover/impl/CoverArtGrabber.cpp index 8809a6c1..110b80d1 100644 --- a/src/cover/CoverArtGrabber.cpp +++ b/src/libs/cover/impl/CoverArtGrabber.cpp @@ -39,8 +39,19 @@ isFileSupported(const std::filesystem::path& file, const std::vector createGrabber(const std::filesystem::path& execPath) { + return std::make_unique(execPath); +} + +Grabber::Grabber(const std::filesystem::path& execPath) +{ + init(execPath); +} + +Grabber::~Grabber() +{ + deinit(); } void @@ -50,7 +61,7 @@ Grabber::setDefaultCover(const std::filesystem::path& p) throw LmsException("Cannot read default cover file '" + p.string() + "'"); } -Image::Image +Image Grabber::getDefaultCover(std::size_t size) { LMS_LOG(COVER, DEBUG) << "Getting a default cover using size = " << size; @@ -59,12 +70,12 @@ Grabber::getDefaultCover(std::size_t size) auto it = _defaultCovers.find(size); if (it == _defaultCovers.end()) { - Image::Image cover = _defaultCover; + Image cover = _defaultCover; LMS_LOG(COVER, DEBUG) << "default cover size = " << cover.getSize().width << " x " << cover.getSize().height; LMS_LOG(COVER, DEBUG) << "Scaling cover to size = " << size; - cover.scale(Image::Geometry{size, size}); + cover.scale(Geometry{size, size}); LMS_LOG(COVER, DEBUG) << "Scaling DONE"; auto res = _defaultCovers.insert(std::make_pair(size, cover)); assert(res.second); @@ -74,14 +85,14 @@ Grabber::getDefaultCover(std::size_t size) return it->second; } -static std::optional +static std::optional getFromAvMediaFile(const Av::MediaFile& input) { - std::vector res; + std::vector res; for (auto& picture : input.getAttachedPictures(2)) { - Image::Image image; + Image image; if (image.load(picture.data)) return image; @@ -93,12 +104,12 @@ getFromAvMediaFile(const Av::MediaFile& input) return std::nullopt; } -std::optional +std::optional Grabber::getFromDirectory(const std::filesystem::path& p) const { for (auto coverPath : getCoverPaths(p)) { - Image::Image image; + Image image; if (image.load(coverPath)) return image; @@ -143,7 +154,7 @@ Grabber::getCoverPaths(const std::filesystem::path& directoryPath) const return res; } -std::optional +std::optional Grabber::getFromTrack(const std::filesystem::path& p) const { try @@ -159,12 +170,12 @@ Grabber::getFromTrack(const std::filesystem::path& p) const } } -Image::Image +Image Grabber::getFromTrack(Database::Session& dbSession, Database::IdType trackId, std::size_t size) { using namespace Database; - std::optional cover; + std::optional cover; bool hasCover {}; bool isMultiDisc {}; @@ -200,16 +211,16 @@ Grabber::getFromTrack(Database::Session& dbSession, Database::IdType trackId, st if (!cover) cover = getDefaultCover(size); else - cover->scale(Image::Geometry {size, size}); + cover->scale(Geometry {size, size}); return *cover; } -Image::Image +Image Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, std::size_t size) { - std::optional cover; + std::optional cover; std::optional trackId; { @@ -230,26 +241,26 @@ Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, if (!cover) cover = getDefaultCover(size); else - cover->scale(Image::Geometry {size, size}); + cover->scale(Geometry {size, size}); return *cover; } std::vector -Grabber::getFromTrack(Database::Session& session, Database::IdType trackId, Image::Format format, std::size_t size) +Grabber::getFromTrack(Database::Session& session, Database::IdType trackId, Format format, std::size_t width) { - const Image::Image cover {getFromTrack(session, trackId, size)}; + const Image cover {getFromTrack(session, trackId, width)}; - assert(format == Image::Format::JPEG); + assert(format == Format::JPEG); return cover.save(format); } std::vector -Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, Image::Format format, std::size_t size) +Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, Format format, std::size_t width) { - const Image::Image cover {getFromRelease(session, releaseId, size)}; + const Image cover {getFromRelease(session, releaseId, width)}; - assert(format == Image::Format::JPEG); + assert(format == Format::JPEG); return cover.save(format); } diff --git a/src/libs/cover/impl/CoverArtGrabber.hpp b/src/libs/cover/impl/CoverArtGrabber.hpp new file mode 100644 index 00000000..bc921355 --- /dev/null +++ b/src/libs/cover/impl/CoverArtGrabber.hpp @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2015 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "cover/ICoverArtGrabber.hpp" +#include "database/Types.hpp" +#include "Image.hpp" + +namespace Database +{ + class Session; +} + +namespace CoverArt +{ + + class Grabber : public IGrabber + { + public: + Grabber(const std::filesystem::path& execPath); + ~Grabber(); + + Grabber(const Grabber&) = delete; + Grabber& operator=(const Grabber&) = delete; + Grabber(Grabber&&) = delete; + Grabber& operator=(Grabber&&) = delete; + + void setDefaultCover(const std::filesystem::path& defaultCoverPath) override; + + std::vector getFromTrack(Database::Session& dbSession, Database::IdType trackId, Format format, std::size_t width) override; + std::vector getFromRelease(Database::Session& dbSession, Database::IdType releaseId, Format format, std::size_t width) override; + + private: + + Image getFromTrack(Database::Session& dbSession, Database::IdType trackId, std::size_t size); + Image getFromRelease(Database::Session& dbSession, Database::IdType releaseId, std::size_t size); + + std::optional getFromTrack(const std::filesystem::path& path) const; + std::vector getCoverPaths(const std::filesystem::path& directoryPath) const; + std::optional getFromDirectory(const std::filesystem::path& path) const; + Image getDefaultCover(std::size_t size); + + Image _defaultCover; + + std::mutex _mutex; + std::map _defaultCovers; + + static inline const std::vector _fileExtensions {".jpg", ".jpeg", ".png", ".bmp"}; // TODO parametrize + static inline const std::size_t _maxFileSize {10000000}; + static inline const std::vector _preferredFileNames {"cover", "front"}; // TODO parametrize + }; + +} // namespace CoverArt + diff --git a/src/image/Image.cpp b/src/libs/cover/impl/Image.cpp similarity index 85% rename from src/image/Image.cpp rename to src/libs/cover/impl/Image.cpp index 2d50bebb..a39c0061 100644 --- a/src/image/Image.cpp +++ b/src/libs/cover/impl/Image.cpp @@ -21,10 +21,23 @@ #include "utils/Logger.hpp" -namespace Image { +namespace CoverArt { + +void +init(const std::filesystem::path& path) +{ + Magick::InitializeMagick(path.string().c_str()); +} + +void +deinit() +{ + MagickCore::MagickCoreTerminus(); +} static -std::string format_to_magick(Format format) +std::string +formatToMagick(Format format) { switch (format) { @@ -34,7 +47,8 @@ std::string format_to_magick(Format format) return "JPEG"; } -std::string format_to_mimeType(Format format) +std::string +formatToMimeType(Format format) { switch (format) { @@ -44,18 +58,13 @@ std::string format_to_mimeType(Format format) return "application/octet-stream"; } -void -init(const char *path) -{ - Magick::InitializeMagick(path); -} bool Image::load(const std::vector& rawData) { try { - Magick::Blob blob(&rawData[0], rawData.size()); + Magick::Blob blob {&rawData[0], rawData.size()}; _image.read(blob); return true; @@ -116,9 +125,9 @@ Image::save(Format format) const try { - Magick::Image outputImage(_image); + Magick::Image outputImage {_image}; - outputImage.magick( format_to_magick(format)); + outputImage.magick(formatToMagick(format)); Magick::Blob blob; outputImage.write(&blob); @@ -135,4 +144,5 @@ Image::save(Format format) const } } -} // namespace Image +} // namespace CoverArt + diff --git a/src/image/Image.hpp b/src/libs/cover/impl/Image.hpp similarity index 60% rename from src/image/Image.hpp rename to src/libs/cover/impl/Image.hpp index 40bc2d6b..6932825f 100644 --- a/src/image/Image.hpp +++ b/src/libs/cover/impl/Image.hpp @@ -24,44 +24,34 @@ #include -namespace Image +#include "cover/CoverArt.hpp" + +namespace CoverArt { -enum class Format -{ - JPEG, -}; + void init(const std::filesystem::path& path); + void deinit(); -std::string format_to_mimeType(Format format); + class Image + { + public: -void init(const char *path); + // input + bool load(const std::vector& rawData); + bool load(const std::filesystem::path& p); -struct Geometry -{ - std::size_t width; - std::size_t height; -}; + Geometry getSize() const; -class Image -{ - public: + // Operations + bool scale(Geometry geometry); - // input - bool load(const std::vector& rawData); - bool load(const std::filesystem::path& p); + // output + std::vector save(Format format) const; - Geometry getSize() const; - - // Operations - bool scale(Geometry geometry); - - // output - std::vector save(Format format) const; - - private: - Magick::Image _image; -}; + private: + Magick::Image _image; + }; -} // namespace Image +} // namespace CoverArt diff --git a/src/scanner/MediaScannerAddon.hpp b/src/libs/cover/include/cover/CoverArt.hpp similarity index 61% rename from src/scanner/MediaScannerAddon.hpp rename to src/libs/cover/include/cover/CoverArt.hpp index fdfd9577..db392e6f 100644 --- a/src/scanner/MediaScannerAddon.hpp +++ b/src/libs/cover/include/cover/CoverArt.hpp @@ -1,5 +1,6 @@ + /* - * Copyright (C) 2018 Emeric Poupon + * Copyright (C) 2015 Emeric Poupon * * This file is part of LMS. * @@ -19,23 +20,21 @@ #pragma once -#include "database/Types.hpp" +#include -namespace Scanner { - -class MediaScannerAddon +namespace CoverArt { - public: - virtual void refreshSettings() = 0; - virtual void requestStop() = 0; - virtual void preScanComplete() = 0; + enum class Format + { + JPEG, + }; + std::string formatToMimeType(Format format); - virtual void trackAdded(Database::IdType trackId) = 0; - virtual void trackToRemove(Database::IdType trackId) = 0; - virtual void trackUpdated(Database::IdType trackId) = 0; - -}; - -} // ns Scanner + struct Geometry + { + std::size_t width; + std::size_t height; + }; +} diff --git a/src/similarity/cluster/SimilarityClusterSearcher.hpp b/src/libs/cover/include/cover/ICoverArtGrabber.hpp similarity index 53% rename from src/similarity/cluster/SimilarityClusterSearcher.hpp rename to src/libs/cover/include/cover/ICoverArtGrabber.hpp index 90ef0db8..c62e3955 100644 --- a/src/similarity/cluster/SimilarityClusterSearcher.hpp +++ b/src/libs/cover/include/cover/ICoverArtGrabber.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018 Emeric Poupon + * Copyright (C) 2015 Emeric Poupon * * This file is part of LMS. * @@ -19,22 +19,30 @@ #pragma once -#include +#include +#include #include "database/Types.hpp" +#include "cover/CoverArt.hpp" namespace Database { class Session; } -namespace Similarity { +namespace CoverArt { -namespace ClusterSearcher +class IGrabber { - std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount); - std::vector getSimilarTracks(Database::Session& session, const std::set& tracksId, std::size_t maxCount); - std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount); - std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount); + public: + virtual ~IGrabber() = default; + + virtual void setDefaultCover(const std::filesystem::path& defaultCoverPath) = 0; + + virtual std::vector getFromTrack(Database::Session& dbSession, Database::IdType trackId, Format format, std::size_t width) = 0; + virtual std::vector getFromRelease(Database::Session& dbSession, Database::IdType releaseId, Format format, std::size_t width) = 0; }; -} // namespace Similarity +std::unique_ptr createGrabber(const std::filesystem::path& execPath); + +} // namespace CoverArt + diff --git a/src/libs/database/CMakeLists.txt b/src/libs/database/CMakeLists.txt new file mode 100644 index 00000000..638fb2e2 --- /dev/null +++ b/src/libs/database/CMakeLists.txt @@ -0,0 +1,37 @@ + +add_library(lmsdatabase SHARED + impl/Artist.cpp + impl/Cluster.cpp + impl/Db.cpp + impl/TrackArtistLink.cpp + impl/TrackFeatures.cpp + impl/TrackList.cpp + impl/Release.cpp + impl/ScanSettings.cpp + impl/Session.cpp + impl/SessionPool.cpp + impl/SqlQuery.cpp + impl/Track.cpp + impl/TrackBookmark.cpp + impl/User.cpp + ) + +target_include_directories(lmsdatabase INTERFACE + include + ) + +target_include_directories(lmsdatabase PRIVATE + include + ) + +target_link_libraries(lmsdatabase PRIVATE + wtdbosqlite3 + ) + +target_link_libraries(lmsdatabase PUBLIC + lmsutils + wtdbo + ) + +install(TARGETS lmsdatabase DESTINATION lib) + diff --git a/src/database/Artist.cpp b/src/libs/database/impl/Artist.cpp similarity index 94% rename from src/database/Artist.cpp rename to src/libs/database/impl/Artist.cpp index 88ee6cb1..b5c6d221 100644 --- a/src/database/Artist.cpp +++ b/src/libs/database/impl/Artist.cpp @@ -16,18 +16,18 @@ * You should have received a copy of the GNU General Public License * along with LMS. If not, see . */ -#include "Artist.hpp" +#include "database/Artist.hpp" #include +#include "database/Cluster.hpp" +#include "database/Release.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "database/User.hpp" #include "utils/Logger.hpp" - -#include "Cluster.hpp" -#include "Release.hpp" #include "SqlQuery.hpp" -#include "Session.hpp" -#include "Track.hpp" -#include "User.hpp" + namespace Database { @@ -104,6 +104,21 @@ Artist::getAllOrphans(Session& session) return std::vector(res.begin(), res.end()); } +std::vector +Artist::getAllIdsWithClusters(Session& session, std::optional limit) +{ + session.checkSharedLocked(); + + Wt::Dbo::collection res = session.getDboSession().query + ("SELECT DISTINCT a.id FROM artist a" + " INNER JOIN track t ON t.id = t_a_l.track_id INNER JOIN track_artist_link t_a_l ON t_a_l.artist_id = a.id" + " INNER JOIN track_cluster t_c ON t_c.track_id = t.id") + .limit(limit ? static_cast(*limit) : -1); + + return std::vector(res.begin(), res.end()); +} + + static Wt::Dbo::Query getQuery(Session& session, diff --git a/src/database/Cluster.cpp b/src/libs/database/impl/Cluster.cpp similarity index 96% rename from src/database/Cluster.cpp rename to src/libs/database/impl/Cluster.cpp index ddfafb43..66e95c12 100644 --- a/src/database/Cluster.cpp +++ b/src/libs/database/impl/Cluster.cpp @@ -17,14 +17,14 @@ * along with LMS. If not, see . */ -#include "Cluster.hpp" +#include "database/Cluster.hpp" -#include "Artist.hpp" -#include "Release.hpp" -#include "ScanSettings.hpp" -#include "Session.hpp" +#include "database/Artist.hpp" +#include "database/Release.hpp" +#include "database/ScanSettings.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" #include "SqlQuery.hpp" -#include "Track.hpp" namespace Database { diff --git a/src/database/Db.cpp b/src/libs/database/impl/Db.cpp similarity index 96% rename from src/database/Db.cpp rename to src/libs/database/impl/Db.cpp index 23edca35..87bdf5d7 100644 --- a/src/database/Db.cpp +++ b/src/libs/database/impl/Db.cpp @@ -17,13 +17,13 @@ * along with LMS. If not, see . */ -#include "Db.hpp" +#include "database/Db.hpp" #include #include +#include "database/User.hpp" #include "utils/Logger.hpp" -#include "User.hpp" namespace Database { diff --git a/src/database/Release.cpp b/src/libs/database/impl/Release.cpp similarity index 95% rename from src/database/Release.cpp rename to src/libs/database/impl/Release.cpp index a4857f15..7fdd639a 100644 --- a/src/database/Release.cpp +++ b/src/libs/database/impl/Release.cpp @@ -17,16 +17,16 @@ * along with LMS. If not, see . */ -#include "Release.hpp" +#include "database/Release.hpp" #include "utils/Logger.hpp" -#include "Artist.hpp" -#include "Cluster.hpp" -#include "Session.hpp" +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "database/User.hpp" #include "SqlQuery.hpp" -#include "Track.hpp" -#include "User.hpp" namespace Database { @@ -252,6 +252,21 @@ Release::getByFilter(Session& session, return res; } +std::vector +Release::getAllIdsWithClusters(Session& session, std::optional limit) +{ + session.checkSharedLocked(); + + Wt::Dbo::collection res = session.getDboSession().query + ("SELECT DISTINCT r.id FROM release r" + " INNER JOIN track t ON t.release_id = r.id" + " INNER JOIN track_cluster t_c ON t_c.track_id = t.id") + .limit(limit ? static_cast(*limit) : -1); + + return std::vector(res.begin(), res.end()); +} + + std::optional Release::getTotalTrackNumber(void) const { diff --git a/src/database/ScanSettings.cpp b/src/libs/database/impl/ScanSettings.cpp similarity index 92% rename from src/database/ScanSettings.cpp rename to src/libs/database/impl/ScanSettings.cpp index 1c024353..3d9a1a86 100644 --- a/src/database/ScanSettings.cpp +++ b/src/libs/database/impl/ScanSettings.cpp @@ -17,15 +17,16 @@ * along with LMS. If not, see . */ -#include "ScanSettings.hpp" +#include "database/ScanSettings.hpp" #include +#include "utils/Path.hpp" #include "utils/Logger.hpp" #include "utils/String.hpp" -#include "Cluster.hpp" -#include "Session.hpp" +#include "database/Cluster.hpp" +#include "database/Session.hpp" namespace { @@ -62,11 +63,11 @@ ScanSettings::get(Session& session) return session.getDboSession().find(); } -std::set +std::unordered_set ScanSettings::getAudioFileExtensions() const { auto extensions = StringUtils::splitString(_audioFileExtensions, " "); - return std::set(std::cbegin(extensions), std::cend(extensions)); + return std::unordered_set(std::cbegin(extensions), std::cend(extensions)); } void diff --git a/src/database/Session.cpp b/src/libs/database/impl/Session.cpp similarity index 95% rename from src/database/Session.cpp rename to src/libs/database/impl/Session.cpp index d96d8448..35eeb7df 100644 --- a/src/database/Session.cpp +++ b/src/libs/database/impl/Session.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "Session.hpp" +#include "database/Session.hpp" #include #include @@ -26,17 +26,17 @@ #include "utils/Exception.hpp" #include "utils/Logger.hpp" -#include "Artist.hpp" -#include "Cluster.hpp" -#include "Db.hpp" -#include "Release.hpp" -#include "ScanSettings.hpp" -#include "Track.hpp" -#include "TrackBookmark.hpp" -#include "TrackArtistLink.hpp" -#include "TrackList.hpp" -#include "TrackFeatures.hpp" -#include "User.hpp" +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Db.hpp" +#include "database/Release.hpp" +#include "database/ScanSettings.hpp" +#include "database/Track.hpp" +#include "database/TrackBookmark.hpp" +#include "database/TrackArtistLink.hpp" +#include "database/TrackList.hpp" +#include "database/TrackFeatures.hpp" +#include "database/User.hpp" namespace Database { @@ -118,7 +118,7 @@ Session::doDatabaseMigrationIfNeeded() { _session.execute("DROP TABLE similarity_settings"); _session.execute("DROP TABLE similarity_settings_feature"); - _session.execute("ALTER TABLE scan_settings ADD similarity_engine_type INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(ScanSettings::SimilarityEngineType::Clusters)) + ")"); + _session.execute("ALTER TABLE scan_settings ADD similarity_engine_type INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(ScanSettings::RecommendationEngineType::Clusters)) + ")"); } else if (version == 8) { diff --git a/src/database/SessionPool.cpp b/src/libs/database/impl/SessionPool.cpp similarity index 96% rename from src/database/SessionPool.cpp rename to src/libs/database/impl/SessionPool.cpp index f6526aca..ff6b1111 100644 --- a/src/database/SessionPool.cpp +++ b/src/libs/database/impl/SessionPool.cpp @@ -17,13 +17,13 @@ * along with LMS. If not, see . */ -#include "SessionPool.hpp" +#include "database/SessionPool.hpp" + +#include "database/Session.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" -#include "Session.hpp" - namespace Database { SessionPool::SessionPool(Db& database, std::size_t maxSessionCount) diff --git a/src/database/SqlQuery.cpp b/src/libs/database/impl/SqlQuery.cpp similarity index 100% rename from src/database/SqlQuery.cpp rename to src/libs/database/impl/SqlQuery.cpp diff --git a/src/database/SqlQuery.hpp b/src/libs/database/impl/SqlQuery.hpp similarity index 100% rename from src/database/SqlQuery.hpp rename to src/libs/database/impl/SqlQuery.hpp diff --git a/src/database/Track.cpp b/src/libs/database/impl/Track.cpp similarity index 95% rename from src/database/Track.cpp rename to src/libs/database/impl/Track.cpp index 65503bbf..2acff171 100644 --- a/src/database/Track.cpp +++ b/src/libs/database/impl/Track.cpp @@ -17,17 +17,17 @@ * along with LMS. If not, see . */ -#include "Track.hpp" +#include "database/Track.hpp" #include +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Release.hpp" +#include "database/TrackFeatures.hpp" +#include "database/Session.hpp" #include "utils/Logger.hpp" -#include "Artist.hpp" -#include "Cluster.hpp" -#include "Release.hpp" -#include "TrackFeatures.hpp" -#include "Session.hpp" #include "SqlQuery.hpp" namespace Database { @@ -163,6 +163,19 @@ Track::getAllIdsWithFeatures(Session& session, std::optional limit) return std::vector(res.begin(), res.end()); } +std::vector +Track::getAllIdsWithClusters(Session& session, std::optional limit) +{ + session.checkSharedLocked(); + + Wt::Dbo::collection res = session.getDboSession().query + ("SELECT DISTINCT t.id FROM track t" + " INNER JOIN track_cluster t_c ON t_c.track_id = t.id") + .limit(limit ? static_cast(*limit) : -1); + + return std::vector(res.begin(), res.end()); +} + std::vector Track::getClusters(void) const { @@ -263,7 +276,7 @@ Track::getByFilter(Session& session, std::vector Track::getSimilarTracks(Session& session, - const std::set& tracks, + const std::unordered_set& tracks, std::optional offset, std::optional size) { diff --git a/src/database/TrackArtistLink.cpp b/src/libs/database/impl/TrackArtistLink.cpp similarity index 90% rename from src/database/TrackArtistLink.cpp rename to src/libs/database/impl/TrackArtistLink.cpp index 64ce9281..57526eab 100644 --- a/src/database/TrackArtistLink.cpp +++ b/src/libs/database/impl/TrackArtistLink.cpp @@ -17,11 +17,11 @@ * along with LMS. If not, see . */ -#include "TrackArtistLink.hpp" +#include "database/TrackArtistLink.hpp" -#include "Artist.hpp" -#include "Session.hpp" -#include "Track.hpp" +#include "database/Artist.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" namespace Database { diff --git a/src/database/TrackBookmark.cpp b/src/libs/database/impl/TrackBookmark.cpp similarity index 94% rename from src/database/TrackBookmark.cpp rename to src/libs/database/impl/TrackBookmark.cpp index 1307f0a7..2a0c4bef 100644 --- a/src/database/TrackBookmark.cpp +++ b/src/libs/database/impl/TrackBookmark.cpp @@ -17,11 +17,11 @@ * along with LMS. If not, see . */ -#include "TrackBookmark.hpp" +#include "database/TrackBookmark.hpp" -#include "Session.hpp" -#include "Track.hpp" -#include "User.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "database/User.hpp" namespace Database { diff --git a/src/database/TrackFeatures.cpp b/src/libs/database/impl/TrackFeatures.cpp similarity index 95% rename from src/database/TrackFeatures.cpp rename to src/libs/database/impl/TrackFeatures.cpp index 6a812bc9..6fb3c963 100644 --- a/src/database/TrackFeatures.cpp +++ b/src/libs/database/impl/TrackFeatures.cpp @@ -17,14 +17,14 @@ * along with LMS. If not, see . */ -#include "TrackFeatures.hpp" +#include "database/TrackFeatures.hpp" #include #include +#include "database/Session.hpp" +#include "database/Track.hpp" #include "utils/Logger.hpp" -#include "Session.hpp" -#include "Track.hpp" namespace Database { diff --git a/src/database/TrackList.cpp b/src/libs/database/impl/TrackList.cpp similarity index 97% rename from src/database/TrackList.cpp rename to src/libs/database/impl/TrackList.cpp index d63312ca..a7fc145c 100644 --- a/src/database/TrackList.cpp +++ b/src/libs/database/impl/TrackList.cpp @@ -16,19 +16,18 @@ * You should have received a copy of the GNU General Public License * along with LMS. If not, see . */ -#include "TrackList.hpp" +#include "database/TrackList.hpp" #include -#include #include "utils/Logger.hpp" -#include "Artist.hpp" -#include "Cluster.hpp" -#include "Release.hpp" -#include "Session.hpp" -#include "User.hpp" -#include "Track.hpp" +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Release.hpp" +#include "database/Session.hpp" +#include "database/User.hpp" +#include "database/Track.hpp" namespace Database { diff --git a/src/database/User.cpp b/src/libs/database/impl/User.cpp similarity index 97% rename from src/database/User.cpp rename to src/libs/database/impl/User.cpp index 2f0e52ef..6b980130 100644 --- a/src/database/User.cpp +++ b/src/libs/database/impl/User.cpp @@ -17,14 +17,14 @@ * along with LMS. If not, see . */ -#include "User.hpp" +#include "database/User.hpp" +#include "database/Artist.hpp" +#include "database/Release.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "database/TrackList.hpp" #include "utils/Logger.hpp" -#include "Artist.hpp" -#include "Release.hpp" -#include "Session.hpp" -#include "Track.hpp" -#include "TrackList.hpp" namespace Database { diff --git a/src/database/Artist.hpp b/src/libs/database/include/database/Artist.hpp similarity index 97% rename from src/database/Artist.hpp rename to src/libs/database/include/database/Artist.hpp index 83b51ef2..4d515c7f 100644 --- a/src/database/Artist.hpp +++ b/src/libs/database/include/database/Artist.hpp @@ -68,6 +68,7 @@ class Artist : public Wt::Dbo::Dbo static std::vector getAllIds(Session& session); static std::vector getAllOrphans(Session& session); // No track related static std::vector getLastAdded(Session& session, Wt::WDateTime after, std::optional size = {}); + static std::vector getAllIdsWithClusters(Session& session, std::optional limit = {}); // Accessors const std::string& getName(void) const { return _name; } diff --git a/src/database/Cluster.hpp b/src/libs/database/include/database/Cluster.hpp similarity index 100% rename from src/database/Cluster.hpp rename to src/libs/database/include/database/Cluster.hpp diff --git a/src/database/Db.hpp b/src/libs/database/include/database/Db.hpp similarity index 100% rename from src/database/Db.hpp rename to src/libs/database/include/database/Db.hpp diff --git a/src/database/Release.hpp b/src/libs/database/include/database/Release.hpp similarity index 98% rename from src/database/Release.hpp rename to src/libs/database/include/database/Release.hpp index 76446bc6..6360b90e 100644 --- a/src/database/Release.hpp +++ b/src/libs/database/include/database/Release.hpp @@ -66,6 +66,7 @@ class Release : public Wt::Dbo::Dbo std::optional offset, std::optional size, bool& moreExpected); + static std::vector getAllIdsWithClusters(Session& session, std::optional limit = {}); std::vector> getTracks(const std::set& clusters = std::set()) const; std::size_t getTracksCount() const; diff --git a/src/database/ScanSettings.hpp b/src/libs/database/include/database/ScanSettings.hpp similarity index 83% rename from src/database/ScanSettings.hpp rename to src/libs/database/include/database/ScanSettings.hpp index 0ce37a49..0a2ee450 100644 --- a/src/database/ScanSettings.hpp +++ b/src/libs/database/include/database/ScanSettings.hpp @@ -19,11 +19,13 @@ #pragma once -#include +#include #include #include +#include "utils/Path.hpp" + namespace Database { class ClusterType; @@ -43,7 +45,7 @@ class ScanSettings : public Wt::Dbo::Dbo }; // Do not modify values (just add) - enum class SimilarityEngineType + enum class RecommendationEngineType { Clusters = 0, Features, @@ -59,8 +61,8 @@ class ScanSettings : public Wt::Dbo::Dbo Wt::WTime getUpdateStartTime() const { return _startTime; } UpdatePeriod getUpdatePeriod() const { return _updatePeriod; } std::vector> getClusterTypes() const; - std::set getAudioFileExtensions() const; - SimilarityEngineType getSimilarityEngineType() const { return _similarityEngineType; } + std::unordered_set getAudioFileExtensions() const; + RecommendationEngineType getRecommendationEngineType() const { return _recommendationEngineType; } // Setters void addAudioFileExtension(const std::filesystem::path& ext); @@ -68,7 +70,7 @@ class ScanSettings : public Wt::Dbo::Dbo void setUpdateStartTime(Wt::WTime t) { _startTime = t; } void setUpdatePeriod(UpdatePeriod p) { _updatePeriod = p; } void setClusterTypes(Session& session, const std::set& clusterTypeNames); - void setSimilarityEngineType(SimilarityEngineType type) { _similarityEngineType = type; } + void setRecommendationEngineType(RecommendationEngineType type) { _recommendationEngineType = type; } void incScanVersion(); template @@ -79,7 +81,7 @@ class ScanSettings : public Wt::Dbo::Dbo Wt::Dbo::field(a, _startTime, "start_time"); Wt::Dbo::field(a, _updatePeriod, "update_period"); Wt::Dbo::field(a, _audioFileExtensions, "audio_file_extensions"); - Wt::Dbo::field(a, _similarityEngineType,"similarity_engine_type"); + Wt::Dbo::field(a, _recommendationEngineType,"similarity_engine_type"); Wt::Dbo::hasMany(a, _clusterTypes, Wt::Dbo::ManyToOne, "scan_settings"); } @@ -89,7 +91,7 @@ class ScanSettings : public Wt::Dbo::Dbo std::string _mediaDirectory; Wt::WTime _startTime = Wt::WTime {0,0,0}; UpdatePeriod _updatePeriod {UpdatePeriod::Never}; - SimilarityEngineType _similarityEngineType {SimilarityEngineType::Clusters}; + RecommendationEngineType _recommendationEngineType {RecommendationEngineType::Clusters}; std::string _audioFileExtensions {".alac .mp3 .ogg .oga .aac .m4a .m4b .flac .wav .wma .aif .aiff .ape .mpc .shn .opus"}; Wt::Dbo::collection> _clusterTypes; }; diff --git a/src/database/Session.hpp b/src/libs/database/include/database/Session.hpp similarity index 94% rename from src/database/Session.hpp rename to src/libs/database/include/database/Session.hpp index 427fc6fd..303501bd 100644 --- a/src/database/Session.hpp +++ b/src/libs/database/include/database/Session.hpp @@ -67,8 +67,8 @@ class Session Session& operator=(const Session&) = delete; Session& operator=(Session&&) = delete; - UniqueTransaction createUniqueTransaction(); - SharedTransaction createSharedTransaction(); + [[nodiscard]] UniqueTransaction createUniqueTransaction(); + [[nodiscard]] SharedTransaction createSharedTransaction(); void checkUniqueLocked(); void checkSharedLocked(); diff --git a/src/database/SessionPool.hpp b/src/libs/database/include/database/SessionPool.hpp similarity index 100% rename from src/database/SessionPool.hpp rename to src/libs/database/include/database/SessionPool.hpp diff --git a/src/database/Track.hpp b/src/libs/database/include/database/Track.hpp similarity index 97% rename from src/database/Track.hpp rename to src/libs/database/include/database/Track.hpp index d3bcd824..24113f21 100644 --- a/src/database/Track.hpp +++ b/src/libs/database/include/database/Track.hpp @@ -22,8 +22,9 @@ #include #include #include -#include #include +#include +#include #include #include @@ -58,7 +59,7 @@ class Track : public Wt::Dbo::Dbo static pointer getById(Session& session, IdType id); static pointer getByMBID(Session& session, const UUID& MBID); static std::vector getSimilarTracks(Session& session, - const std::set& trackIds, + const std::unordered_set& trackIds, std::optional offset = {}, std::optional size = {}); static std::vector getByClusters(Session& session, @@ -78,6 +79,7 @@ class Track : public Wt::Dbo::Dbo static std::vector getLastAdded(Session& session, const Wt::WDateTime& after, std::optional size = 1); static std::vector getAllWithMBIDAndMissingFeatures(Session& session); static std::vector getAllIdsWithFeatures(Session& session, std::optional limit = {}); + static std::vector getAllIdsWithClusters(Session& session, std::optional limit = {}); // Create utility static pointer create(Session& session, const std::filesystem::path& p); diff --git a/src/database/TrackArtistLink.hpp b/src/libs/database/include/database/TrackArtistLink.hpp similarity index 100% rename from src/database/TrackArtistLink.hpp rename to src/libs/database/include/database/TrackArtistLink.hpp diff --git a/src/database/TrackBookmark.hpp b/src/libs/database/include/database/TrackBookmark.hpp similarity index 100% rename from src/database/TrackBookmark.hpp rename to src/libs/database/include/database/TrackBookmark.hpp diff --git a/src/database/TrackFeatures.hpp b/src/libs/database/include/database/TrackFeatures.hpp similarity index 100% rename from src/database/TrackFeatures.hpp rename to src/libs/database/include/database/TrackFeatures.hpp diff --git a/src/database/TrackList.hpp b/src/libs/database/include/database/TrackList.hpp similarity index 100% rename from src/database/TrackList.hpp rename to src/libs/database/include/database/TrackList.hpp diff --git a/src/database/Types.hpp b/src/libs/database/include/database/Types.hpp similarity index 100% rename from src/database/Types.hpp rename to src/libs/database/include/database/Types.hpp diff --git a/src/database/User.hpp b/src/libs/database/include/database/User.hpp similarity index 100% rename from src/database/User.hpp rename to src/libs/database/include/database/User.hpp diff --git a/src/libs/metadata/CMakeLists.txt b/src/libs/metadata/CMakeLists.txt new file mode 100644 index 00000000..cc6a1dbb --- /dev/null +++ b/src/libs/metadata/CMakeLists.txt @@ -0,0 +1,25 @@ + +add_library(lmsmetadata SHARED + impl/AvFormatParser.cpp + impl/TagLibParser.cpp + ) + +target_include_directories(lmsmetadata INTERFACE + include + ) + +target_include_directories(lmsmetadata PRIVATE + include + ) + +target_link_libraries(lmsmetadata PRIVATE + lmsav + tag + ) + +target_link_libraries(lmsmetadata PUBLIC + lmsutils + ) + +install(TARGETS lmsmetadata DESTINATION lib) + diff --git a/src/metadata/AvFormat.cpp b/src/libs/metadata/impl/AvFormatParser.cpp similarity index 97% rename from src/metadata/AvFormat.cpp rename to src/libs/metadata/impl/AvFormatParser.cpp index 38790589..7dfd8af4 100644 --- a/src/metadata/AvFormat.cpp +++ b/src/libs/metadata/impl/AvFormatParser.cpp @@ -17,13 +17,12 @@ * along with LMS. If not, see . */ -#include "AvFormat.hpp" +#include "metadata/AvFormatParser.hpp" #include #include #include "av/AvInfo.hpp" - #include "utils/Logger.hpp" #include "utils/String.hpp" @@ -128,7 +127,7 @@ getArtists(const MetadataMap& metadataMap) } std::optional -AvFormat::parse(const std::filesystem::path& p, bool debug) +AvFormatParser::parse(const std::filesystem::path& p, bool debug) { Track track; @@ -142,7 +141,7 @@ AvFormat::parse(const std::filesystem::path& p, bool debug) for (auto stream : mediaFile.getStreamInfo()) { - MetaData::AudioStream audioStream {.bitRate = static_cast(stream.bitrate)}; + MetaData::AudioStream audioStream {static_cast(stream.bitrate)}; track.audioStreams.emplace_back(audioStream); } } diff --git a/src/metadata/TagLibParser.cpp b/src/libs/metadata/impl/TagLibParser.cpp similarity index 98% rename from src/metadata/TagLibParser.cpp rename to src/libs/metadata/impl/TagLibParser.cpp index 5f430e14..78949ce8 100644 --- a/src/metadata/TagLibParser.cpp +++ b/src/libs/metadata/impl/TagLibParser.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "TagLibParser.hpp" +#include "metadata/TagLibParser.hpp" #include #include @@ -180,7 +180,7 @@ TagLibParser::parse(const std::filesystem::path& p, bool debug) track.duration = std::chrono::milliseconds {properties->length() * 1000}; - MetaData::AudioStream audioStream {.bitRate = static_cast(properties->bitrate() * 1000)}; + MetaData::AudioStream audioStream {static_cast(properties->bitrate() * 1000)}; track.audioStreams = {std::move(audioStream)}; } diff --git a/src/metadata/AvFormat.hpp b/src/libs/metadata/include/metadata/AvFormatParser.hpp similarity index 92% rename from src/metadata/AvFormat.hpp rename to src/libs/metadata/include/metadata/AvFormatParser.hpp index 28c25a8e..009d4c2e 100644 --- a/src/metadata/AvFormat.hpp +++ b/src/libs/metadata/include/metadata/AvFormatParser.hpp @@ -19,13 +19,13 @@ #pragma once -#include "MetaData.hpp" +#include "metadata/IParser.hpp" namespace MetaData { // Parse that makes use of AvFormat -class AvFormat : public Parser +class AvFormatParser : public IParser { public: std::optional parse(const std::filesystem::path& p, bool debug = false) override; diff --git a/src/metadata/MetaData.hpp b/src/libs/metadata/include/metadata/IParser.hpp similarity index 98% rename from src/metadata/MetaData.hpp rename to src/libs/metadata/include/metadata/IParser.hpp index b4f652ad..4a8b5362 100644 --- a/src/metadata/MetaData.hpp +++ b/src/libs/metadata/include/metadata/IParser.hpp @@ -26,7 +26,6 @@ #include #include -//#include "utils/Utils.hpp" #include "utils/UUID.hpp" namespace MetaData @@ -73,7 +72,7 @@ namespace MetaData std::string copyrightURL; }; - class Parser + class IParser { public: virtual std::optional parse(const std::filesystem::path& p, bool debug = false) = 0; diff --git a/src/metadata/TagLibParser.hpp b/src/libs/metadata/include/metadata/TagLibParser.hpp similarity index 92% rename from src/metadata/TagLibParser.hpp rename to src/libs/metadata/include/metadata/TagLibParser.hpp index 5dad7a3f..1c055b76 100644 --- a/src/metadata/TagLibParser.hpp +++ b/src/libs/metadata/include/metadata/TagLibParser.hpp @@ -19,13 +19,13 @@ #pragma once -#include "MetaData.hpp" +#include "metadata/IParser.hpp" namespace MetaData { // Parse that makes use of AvFormat -class TagLibParser : public Parser +class TagLibParser : public IParser { public: std::optional parse(const std::filesystem::path& p, bool debug = false) override; diff --git a/src/libs/recommendation/CMakeLists.txt b/src/libs/recommendation/CMakeLists.txt new file mode 100644 index 00000000..b7a52f12 --- /dev/null +++ b/src/libs/recommendation/CMakeLists.txt @@ -0,0 +1,25 @@ + +add_library(lmsrecommendation SHARED + impl/clusters/ClustersClassifier.cpp + impl/features/FeaturesClassifierCache.cpp + impl/features/FeaturesClassifier.cpp + impl/features/FeaturesDefs.cpp + impl/Engine.cpp + ) + +target_include_directories(lmsrecommendation INTERFACE + include + ) + +target_include_directories(lmsrecommendation PRIVATE + include + ) + +target_link_libraries(lmsrecommendation PRIVATE + lmsdatabase + lmssom + wt + ) + +install(TARGETS lmsrecommendation DESTINATION lib) + diff --git a/src/libs/recommendation/impl/Engine.cpp b/src/libs/recommendation/impl/Engine.cpp new file mode 100644 index 00000000..072c3d48 --- /dev/null +++ b/src/libs/recommendation/impl/Engine.cpp @@ -0,0 +1,278 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "Engine.hpp" + +#include "recommendation/ClustersClassifierCreator.hpp" +#include "recommendation/FeaturesClassifierCreator.hpp" + +#include "database/ScanSettings.hpp" +#include "database/TrackList.hpp" +#include "utils/Exception.hpp" +#include "utils/Logger.hpp" + +namespace Recommendation { + +std::unique_ptr +createEngine(Database::Db& db) +{ + return std::make_unique(db); +} + +Engine::Engine(Database::Db& db) +: _dbSession {db} +{ +} + +void +Engine::start() +{ + assert(!_running); + _running = true; + + requestReloadInternal(false); + + _ioService.start(); +} + +void +Engine::stop() +{ + assert(_running); + _running = false; + + cancelPendingClassifiers(); + + _ioService.stop(); +} + +void +Engine::requestReload() +{ + requestReloadInternal(true); +} + +void +Engine::requestReloadInternal(bool databaseChanged) +{ + LMS_LOG(RECOMMENDATION, DEBUG) << "Reload requested..."; + + _ioService.post([=]() + { + reload(databaseChanged); + }); +} + +std::vector +Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) +{ + std::shared_lock lock {_classifiersMutex}; + + std::vector res; + + for (const auto& classifierName : _classifierPriorities) + { + auto itClassifier {_classifiers.find(classifierName)}; + if (itClassifier == std::cend(_classifiers)) + continue; + + res = itClassifier->second->getSimilarTracksFromTrackList(session, trackListId, maxCount); + if (!res.empty()) + break; + } + + return res; +} + +std::vector +Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set& trackIds, std::size_t maxCount) +{ + std::shared_lock lock {_classifiersMutex}; + + std::vector res; + + for (const auto& classifierName : _classifierPriorities) + { + auto itClassifier {_classifiers.find(classifierName)}; + if (itClassifier == std::cend(_classifiers)) + continue; + + res = itClassifier->second->getSimilarTracks(dbSession, trackIds, maxCount); + if (!res.empty()) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using classifier '" << classifierName << "'"; + break; + } + } + + return res; +} + +std::vector +Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) +{ + std::shared_lock lock {_classifiersMutex}; + + std::vector res; + + for (const auto& classifierName : _classifierPriorities) + { + auto itClassifier {_classifiers.find(classifierName)}; + if (itClassifier == std::cend(_classifiers)) + continue; + + res = itClassifier->second->getSimilarReleases(dbSession, releaseId, maxCount); + if (!res.empty()) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using classifier '" << classifierName << "'"; + break; + } + } + + return res; +} + +std::vector +Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) +{ + std::shared_lock lock {_classifiersMutex}; + + std::vector res; + + for (const auto& classifierName : _classifierPriorities) + { + auto itClassifier {_classifiers.find(classifierName)}; + if (itClassifier == std::cend(_classifiers)) + continue; + + res = itClassifier->second->getSimilarArtists(dbSession, artistId, maxCount); + if (!res.empty()) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using classifier '" << classifierName << "'"; + return res; + } + } + + return res; +} + +void +Engine::reload(bool databaseChanged) +{ + using namespace Database; + + LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines..."; + + const ScanSettings::RecommendationEngineType engineType {[&]() + { + auto transaction {_dbSession.createSharedTransaction()}; + + return ScanSettings::get(_dbSession)->getRecommendationEngineType(); + }()}; + + clearClassifiers(); + + switch (engineType) + { + case ScanSettings::RecommendationEngineType::Features: + { + auto clustersClassifier {createClustersClassifier()}; + auto featuresClassifier {createFeaturesClassifier()}; + + setClassifierPriorities({featuresClassifier->getName(), clustersClassifier->getName()}); + + initAndAddClassifier(std::move(clustersClassifier), databaseChanged); // init first since faster + initAndAddClassifier(std::move(featuresClassifier), databaseChanged); + break; + } + + case ScanSettings::RecommendationEngineType::Clusters: + auto clustersClassifier {createClustersClassifier()}; + + setClassifierPriorities({clustersClassifier->getName()}); + + initAndAddClassifier(std::move(clustersClassifier), databaseChanged); + break; + } + + LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines reloaded!"; + + _sigReloaded.emit(); +} + +void +Engine::setClassifierPriorities(std::initializer_list classifierPriorities) +{ + std::unique_lock lock {_classifiersMutex}; + + _classifierPriorities.clear(); + std::transform(std::cbegin(classifierPriorities), std::cend(classifierPriorities), std::back_inserter(_classifierPriorities), [](std::string_view name) { return std::string {name}; }); +} + +void +Engine::clearClassifiers() +{ + std::unique_lock lock {_classifiersMutex}; + + _classifiers.clear(); +} + +void +Engine::initAndAddClassifier(std::unique_ptr classifier, bool databaseChanged) +{ + PendingClassifierHandler pendingClassifier {*this, *classifier.get()}; + + LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "'..."; + bool res {classifier->init(_dbSession, databaseChanged)}; + LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "': " << (res ? "SUCCESS" : "FAILURE"); + + if (res) + { + std::unique_lock lock {_classifiersMutex}; + + _classifiers.emplace(classifier->getName(), std::move(classifier)); + } +} + +void +Engine::cancelPendingClassifiers() +{ + std::unique_lock lock {_classifiersMutex}; + + for (IClassifier* classifier : _pendingClassifiers) + classifier->requestCancelInit(); +} + +void +Engine::addPendingClassifier(IClassifier& classifier) +{ + std::unique_lock lock {_classifiersMutex}; + + _pendingClassifiers.insert(&classifier); +} + +void +Engine::removePendingClassifier(IClassifier& classifier) +{ + std::unique_lock lock {_classifiersMutex}; + + _pendingClassifiers.erase(&classifier); +} + +} // ns Similarity diff --git a/src/libs/recommendation/impl/Engine.hpp b/src/libs/recommendation/impl/Engine.hpp new file mode 100644 index 00000000..4fab63f1 --- /dev/null +++ b/src/libs/recommendation/impl/Engine.hpp @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include + +#include "database/Session.hpp" +#include "recommendation/IEngine.hpp" +#include "recommendation/IClassifier.hpp" + +namespace Recommendation +{ + class Engine : public IEngine + { + public: + Engine(Database::Db& db); + + private: + + void start() override; + void stop() override; + + void requestReload() override; + Wt::Signal<>& reloaded() override { return _sigReloaded; } + + std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override; + std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) override; + std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override; + std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override; + + + void requestReloadInternal(bool databaseChanged); + void reload(bool databaseChanged); + + void setClassifierPriorities(std::initializer_list classifierNames); + void clearClassifiers(); + void initAndAddClassifier(std::unique_ptr classifier, bool databaseChanged); + + class PendingClassifierHandler + { + public: + PendingClassifierHandler(Engine& engine, IClassifier& classifier) : _engine {engine}, _classifier {classifier} + { + _engine.addPendingClassifier(_classifier); + } + + ~PendingClassifierHandler() + { + _engine.removePendingClassifier(_classifier); + } + + private: + Engine& _engine; + IClassifier& _classifier; + }; + + void cancelPendingClassifiers(); + void addPendingClassifier(IClassifier& classifier); + void removePendingClassifier(IClassifier& classifier); + + bool _running {}; + Wt::WIOService _ioService; + Database::Session _dbSession; + Wt::Signal<> _sigReloaded; + + std::shared_mutex _classifiersMutex; + std::map> _classifiers; + std::vector _classifierPriorities; // ordered by priority + std::unordered_set _pendingClassifiers; + }; + +} // ns Recommendation + diff --git a/src/similarity/cluster/SimilarityClusterSearcher.cpp b/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp similarity index 79% rename from src/similarity/cluster/SimilarityClusterSearcher.cpp rename to src/libs/recommendation/impl/clusters/ClustersClassifier.cpp index 037dd3c4..b433d833 100644 --- a/src/similarity/cluster/SimilarityClusterSearcher.cpp +++ b/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp @@ -17,10 +17,7 @@ * along with LMS. If not, see . */ -#include "SimilarityClusterSearcher.hpp" - -#include -#include +#include "ClustersClassifier.hpp" #include "database/Artist.hpp" #include "database/Cluster.hpp" @@ -29,11 +26,15 @@ #include "database/Track.hpp" #include "database/TrackList.hpp" -namespace Similarity { -namespace ClusterSearcher { +namespace Recommendation { + +std::unique_ptr createClustersClassifier() +{ + return std::make_unique(); +} std::vector -getSimilarTracks(Database::Session& dbSession, const std::set& trackIds, std::size_t maxCount) +ClusterClassifier::getSimilarTracks(Database::Session& dbSession, const std::unordered_set& trackIds, std::size_t maxCount) const { auto transaction {dbSession.createSharedTransaction()}; @@ -46,7 +47,7 @@ getSimilarTracks(Database::Session& dbSession, const std::set& } std::vector -getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) +ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const { std::vector res; @@ -65,7 +66,7 @@ getSimilarTracksFromTrackList(Database::Session& session, Database::IdType track } std::vector -getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) +ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) const { std::vector res; @@ -83,7 +84,7 @@ getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std } std::vector -getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) +ClusterClassifier::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) const { std::vector res; @@ -100,5 +101,4 @@ getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std:: return res; } -} // namespace ClusterSearcher -} // namespace Similarity +} // namespace Recommendation diff --git a/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp b/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp new file mode 100644 index 00000000..4619d5dc --- /dev/null +++ b/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "recommendation/IClassifier.hpp" + + +namespace Recommendation +{ + + class ClusterClassifier : public IClassifier + { + public: + ClusterClassifier() = default; + ClusterClassifier(const ClusterClassifier&) = delete; + ClusterClassifier(ClusterClassifier&&) = delete; + ClusterClassifier& operator=(const ClusterClassifier&) = delete; + ClusterClassifier& operator=(ClusterClassifier&&) = delete; + + private: + + std::string_view getName() const { return "Clusters"; } + + bool init(Database::Session&, bool) override {return true;} + void requestCancelInit() override {} + + std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override; + std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const override; + std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override; + std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override; + +}; + +} // namespace Recommendation + diff --git a/src/libs/recommendation/impl/features/FeaturesClassifier.cpp b/src/libs/recommendation/impl/features/FeaturesClassifier.cpp new file mode 100644 index 00000000..d3b13d5d --- /dev/null +++ b/src/libs/recommendation/impl/features/FeaturesClassifier.cpp @@ -0,0 +1,487 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "FeaturesClassifier.hpp" + +#include + +#include "database/Artist.hpp" +#include "database/Release.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "database/TrackFeatures.hpp" +#include "database/TrackList.hpp" +#include "som/DataNormalizer.hpp" +#include "utils/Logger.hpp" +#include "utils/Random.hpp" + + +namespace Recommendation { + +std::unique_ptr createFeaturesClassifier() +{ + return std::make_unique(); +} + +const FeatureSettingsMap& +FeaturesClassifier::getDefaultTrainFeatureSettings() +{ + static const FeatureSettingsMap defaultTrainFeatureSettings + { + { "lowlevel.spectral_energyband_high.mean", {1}}, + { "lowlevel.spectral_rolloff.median", {1}}, + { "lowlevel.spectral_contrast_valleys.var", {1}}, + { "lowlevel.erbbands.mean", {1}}, + { "lowlevel.gfcc.mean", {1}}, + }; + + return defaultTrainFeatureSettings; +} + +static +std::optional +getTrackFeatureValues(FeaturesClassifier::FeaturesFetchFunc func, Database::IdType trackId, const std::unordered_set& featureNames) +{ + return func(trackId, featureNames); +} + +static +std::optional +getTrackFeatureValuesFromDb(Database::Session& session, Database::IdType trackId, const std::unordered_set& featureNames) +{ + auto func = [&](Database::IdType trackId, const std::unordered_set& featureNames) + { + std::optional res; + + auto transaction {session.createSharedTransaction()}; + + Database::Track::pointer track {Database::Track::getById(session, trackId)}; + if (!track) + return res; + + res = track->getTrackFeatures()->getFeatureValuesMap(featureNames); + if (res->empty()) + res.reset(); + + return res; + }; + + return getTrackFeatureValues(func, trackId, featureNames); +} + +static +std::optional +convertFeatureValuesMapToInputVector(const FeatureValuesMap& featureValuesMap, std::size_t nbDimensions) +{ + std::size_t i {}; + std::optional res {SOM::InputVector {nbDimensions}}; + for (const auto& [featureName, values] : featureValuesMap) + { + if (values.size() != getFeatureDef(featureName).nbDimensions) + { + LMS_LOG(RECOMMENDATION, WARNING) << "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << values.size(); + res.reset(); + break; + } + + for (double val : values) + (*res)[i++] = val; + } + + return res; +} + +static +SOM::InputVector +getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t nbDimensions) +{ + SOM::InputVector weights {nbDimensions}; + std::size_t index {}; + for (const auto& [featureName, featureSettings] : featureSettingsMap) + { + const std::size_t featureNbDimensions {getFeatureDef(featureName).nbDimensions}; + + for (std::size_t i {}; i < featureNbDimensions; ++i) + weights[index++] = (1. / featureNbDimensions * featureSettings.weight); + } + + assert(index == nbDimensions); + + return weights; +} + +bool +FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSettings& trainSettings) +{ + LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier..."; + + std::unordered_set featureNames; + std::transform(std::cbegin(trainSettings.featureSettingsMap), std::cend(trainSettings.featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)), + [](const auto& itFeatureSetting) { return itFeatureSetting.first; }); + + const std::size_t nbDimensions {std::accumulate(std::cbegin(featureNames), std::cend(featureNames), std::size_t {0}, + [](std::size_t sum, const FeatureName& featureName) { return sum + getFeatureDef(featureName).nbDimensions; })}; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Features dimension = " << nbDimensions; + + std::vector trackIds; + { + auto transaction {session.createSharedTransaction()}; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Getting Tracks with features..."; + trackIds = Database::Track::getAllIdsWithFeatures(session); + LMS_LOG(RECOMMENDATION, DEBUG) << "Getting Tracks with features DONE (found " << trackIds.size() << " tracks)"; + } + + std::vector samples; + std::vector samplesTrackIds; + + samples.reserve(trackIds.size()); + samplesTrackIds.reserve(trackIds.size()); + + LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features..."; + for (Database::IdType trackId : trackIds) + { + if (_initCancelled) + return false; + + std::optional featureValuesMap; + + if (_featuresFetchFunc) + featureValuesMap = getTrackFeatureValues(_featuresFetchFunc, trackId, featureNames); + else + featureValuesMap = getTrackFeatureValuesFromDb(session, trackId, featureNames); + + if (!featureValuesMap) + continue; + + std::optional inputVector {convertFeatureValuesMapToInputVector(*featureValuesMap, nbDimensions)}; + if (!inputVector) + continue; + + samples.emplace_back(std::move(*inputVector)); + samplesTrackIds.emplace_back(trackId); + } + LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features DONE"; + + if (samples.empty()) + { + LMS_LOG(RECOMMENDATION, INFO) << "Nothing to classify!"; + return false; + } + + LMS_LOG(RECOMMENDATION, DEBUG) << "Normalizing data..."; + SOM::DataNormalizer dataNormalizer {nbDimensions}; + + dataNormalizer.computeNormalizationFactors(samples); + for (auto& sample : samples) + dataNormalizer.normalizeData(sample); + + const SOM::Coordinate size {static_cast(std::sqrt(samples.size() / trainSettings.sampleCountPerNeuron))}; + LMS_LOG(RECOMMENDATION, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network"; + + SOM::Network network {size, size, nbDimensions}; + + SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)}; + network.setDataWeights(weights); + + auto progressIndicator{[](const auto& iter) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount; + }}; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Training network..."; + network.train(samples, trainSettings.iterationCount, progressIndicator); + LMS_LOG(RECOMMENDATION, DEBUG) << "Training network DONE"; + + if (_initCancelled) + return false; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks..."; + ObjectPositions trackPositions; + for (std::size_t i {}; i < samples.size(); ++i) + { + if (_initCancelled) + return false; + + const SOM::Position position {network.getClosestRefVectorPosition(samples[i])}; + + trackPositions[samplesTrackIds[i]].insert(position); + } + + LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks DONE"; + + return init(session, std::move(network), std::move(trackPositions)); +} + +bool +FeaturesClassifier::initFromCache(Database::Session& session, const FeaturesClassifierCache& cache) +{ + LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier from cache..."; + + return init(session, std::move(cache._network), cache._trackPositions); +} + +std::vector +FeaturesClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) const +{ + const std::unordered_set trackIds {[&]() -> std::unordered_set + { + auto transaction {session.createSharedTransaction()}; + + const Database::TrackList::pointer trackList {Database::TrackList::getById(session, trackListId)}; + if (trackList) + { + const std::vector orderedTrackIds {trackList->getTrackIds()}; + return std::unordered_set {std::cbegin(orderedTrackIds), std::cend(orderedTrackIds)}; + } + + return {}; + }()}; + + return getSimilarTracks(session, trackIds, maxCount); +} + +std::vector +FeaturesClassifier::getSimilarTracks(Database::Session& session, const std::unordered_set& tracksIds, std::size_t maxCount) const +{ + std::vector similarTrackIds {getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount)}; + + if (!similarTrackIds.empty()) + { + // Report only existing ids + auto transaction {session.createSharedTransaction()}; + + similarTrackIds.erase(std::remove_if(std::begin(similarTrackIds), std::end(similarTrackIds), + [&](Database::IdType trackId) { return Database::Track::getById(session, trackId) == Database::Track::pointer {}; }), + std::cend(similarTrackIds)); + } + + return similarTrackIds; +} + +std::vector +FeaturesClassifier::getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const +{ + std::vector similarReleaseIds {getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount)}; + + if (!similarReleaseIds.empty()) + { + // Report only existing ids + auto transaction {session.createSharedTransaction()}; + + similarReleaseIds.erase(std::remove_if(std::begin(similarReleaseIds), std::end(similarReleaseIds), + [&](Database::IdType releaseId) { return Database::Release::getById(session, releaseId) == Database::Release::pointer {}; }), + std::cend(similarReleaseIds)); + } + + return similarReleaseIds; +} + +std::vector +FeaturesClassifier::getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const +{ + std::vector similarArtistIds {getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount)}; + + if (!similarArtistIds.empty()) + { + // Report only existing ids + auto transaction {session.createSharedTransaction()}; + + similarArtistIds.erase(std::remove_if(std::begin(similarArtistIds), std::end(similarArtistIds), + [&](Database::IdType artistId) { return Database::Artist::getById(session, artistId) == Database::Artist::pointer {}; }), + std::cend(similarArtistIds)); + } + + return similarArtistIds; +} + +FeaturesClassifierCache +FeaturesClassifier::toCache() const +{ + return FeaturesClassifierCache {*_network, _trackPositions}; +} + +bool +FeaturesClassifier::init(Database::Session& session, bool databaseChanged) +{ + if (databaseChanged) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Database changed: invidating cache"; + FeaturesClassifierCache::invalidate(); + } + + std::optional cache {FeaturesClassifierCache::read()}; + if (cache) + return initFromCache(session, *cache); + + TrainSettings trainSettings; + trainSettings.featureSettingsMap = getDefaultTrainFeatureSettings(); + + bool res {initFromTraining(session, trainSettings)}; + if (res) + toCache().write(); + + return res; +} + +void +FeaturesClassifier::requestCancelInit() +{ + LMS_LOG(RECOMMENDATION, DEBUG) << "Requesting init cancellation"; + _initCancelled = true; +} + +bool +FeaturesClassifier::init(Database::Session& session, + SOM::Network network, + const ObjectPositions& tracksPosition) +{ + _networkRefVectorsDistanceMedian = network.computeRefVectorsDistanceMedian(); + LMS_LOG(RECOMMENDATION, DEBUG) << "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian; + + const SOM::Coordinate width {network.getWidth()}; + const SOM::Coordinate height {network.getHeight()}; + + _artistsMap = MatrixOfObjects {width, height}; + _releasesMap = MatrixOfObjects {width, height}; + _tracksMap = MatrixOfObjects {width, height}; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Constructing maps..."; + + for (auto itTrackCoord : tracksPosition) + { + if (_initCancelled) + return false; + + auto transaction {session.createSharedTransaction()}; + + Database::IdType trackId {itTrackCoord.first}; + const std::unordered_set& positionSet {itTrackCoord.second}; + + const Database::Track::pointer track {Database::Track::getById(session, trackId)}; + if (!track) + continue; + + for (const SOM::Position& position : positionSet) + { + _tracksMap[position].insert(trackId); + _trackPositions[trackId].insert(position); + + if (track->getRelease()) + { + _releasePositions[track->getRelease().id()].insert(position); + _releasesMap[position].insert(track->getRelease().id()); + } + for (const auto& artist : track->getArtists()) + { + _artistPositions[artist.id()].insert(position); + _artistsMap[position].insert(artist.id()); + } + } + } + + _network = std::make_unique(std::move(network)); + + LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully initialized!"; + + return true; +} + +std::unordered_set +FeaturesClassifier::getMatchingRefVectorsPosition(const std::unordered_set& ids, const ObjectPositions& objectPositions) +{ + std::unordered_set res; + + if (ids.empty()) + return res; + + for (auto id : ids) + { + auto it = objectPositions.find(id); + if (it == objectPositions.end()) + continue; + + for (const auto& position : it->second) + res.insert(position); + } + + return res; +} + +std::unordered_set +FeaturesClassifier::getObjectsIds(const std::unordered_set& positionSet, const MatrixOfObjects& objectsMap) +{ + std::unordered_set res; + + for (const auto& position : positionSet) + { + for (auto id : objectsMap.get(position)) + res.insert(id); + } + + return res; +} + +std::vector +FeaturesClassifier::getSimilarObjects(const std::unordered_set& ids, + const MatrixOfObjects& objectsMap, + const ObjectPositions& objectPosition, + std::size_t maxCount) const +{ + std::vector res; + + std::unordered_set searchedRefVectorsPosition {getMatchingRefVectorsPosition(ids, objectPosition)}; + if (searchedRefVectorsPosition.empty()) + return res; + + while (1) + { + std::unordered_set closestObjectIds {getObjectsIds(searchedRefVectorsPosition, objectsMap)}; + + // Remove objects that are already in input or already reported + for (auto id : ids) + closestObjectIds.erase(id); + + { + std::vector objectIdsToAdd {std::cbegin(closestObjectIds), std::cend(closestObjectIds)}; + Random::shuffleContainer(objectIdsToAdd ); + std::copy(std::cbegin(objectIdsToAdd), std::cend(objectIdsToAdd), std::back_inserter(res)); + } + + if (res.size() > maxCount) + res.resize(maxCount); + + if (res.size() == maxCount) + break; + + // If there is not enough objects, try again with closest neighbour until there is too much distance + const std::optional closestRefVectorPosition {_network->getClosestRefVectorPosition(searchedRefVectorsPosition, _networkRefVectorsDistanceMedian * 0.75)}; + if (!closestRefVectorPosition) + break; + + searchedRefVectorsPosition.insert(closestRefVectorPosition.value()); + } + + return res; +} + + + +} // ns Recommendation diff --git a/src/libs/recommendation/impl/features/FeaturesClassifier.hpp b/src/libs/recommendation/impl/features/FeaturesClassifier.hpp new file mode 100644 index 00000000..ecf3b947 --- /dev/null +++ b/src/libs/recommendation/impl/features/FeaturesClassifier.hpp @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include "recommendation/IClassifier.hpp" +#include "som/DataNormalizer.hpp" +#include "som/Network.hpp" +#include "FeaturesClassifierCache.hpp" +#include "FeaturesDefs.hpp" + +namespace Database +{ + class Session; +} + +namespace Recommendation { + +using FeatureWeight = double; + +class FeaturesClassifier : public IClassifier +{ + public: + FeaturesClassifier() = default; + FeaturesClassifier(const FeaturesClassifier&) = delete; + FeaturesClassifier(FeaturesClassifier&&) = delete; + FeaturesClassifier& operator=(const FeaturesClassifier&) = delete; + FeaturesClassifier& operator=(FeaturesClassifier&&) = delete; + + using FeaturesFetchFunc = std::function>>(Database::IdType /*trackId*/, const std::unordered_set& /*features*/)>; + // Default is to retrieve the features from the database (may be slow). + // Use this only if you want to train different searchers with some cached data + static void setFeaturesFetchFunc(FeaturesFetchFunc func) { _featuresFetchFunc = func; } + + static const FeatureSettingsMap& getDefaultTrainFeatureSettings(); + private: + + std::string_view getName() const { return "Features"; } + + bool init(Database::Session& session, bool databaseChanged) override; + void requestCancelInit() override; + + std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override; + std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const override; + std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const; + std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const; + + bool initFromCache(Database::Session& session, const FeaturesClassifierCache& cache); + + // Use training (may be very slow) + struct TrainSettings + { + std::size_t iterationCount {10}; + float sampleCountPerNeuron {4}; + FeatureSettingsMap featureSettingsMap; + }; + bool initFromTraining(Database::Session& session, const TrainSettings& trainSettings); + + using ObjectPositions = std::unordered_map>; + using MatrixOfObjects = SOM::Matrix>; + + bool init(Database::Session& session, + SOM::Network network, + const ObjectPositions& tracksPosition); + + FeaturesClassifierCache toCache() const; + + static std::unordered_set getMatchingRefVectorsPosition(const std::unordered_set& ids, const ObjectPositions& objectPositions); + static std::unordered_set getObjectsIds(const std::unordered_set& positionSet, const MatrixOfObjects& objectsMap); + + std::vector getSimilarObjects(const std::unordered_set& ids, + const SOM::Matrix>& objectsMap, + const ObjectPositions& objectPosition, + std::size_t maxCount) const; + + bool _initCancelled {}; + std::unique_ptr _network; + double _networkRefVectorsDistanceMedian {}; + + MatrixOfObjects _artistsMap; + ObjectPositions _artistPositions; + + MatrixOfObjects _releasesMap; + ObjectPositions _releasePositions; + + MatrixOfObjects _tracksMap; + ObjectPositions _trackPositions; + + static inline FeaturesFetchFunc _featuresFetchFunc; +}; + +} // ns Recommendation diff --git a/src/similarity/features/SimilarityFeaturesCache.cpp b/src/libs/recommendation/impl/features/FeaturesClassifierCache.cpp similarity index 73% rename from src/similarity/features/SimilarityFeaturesCache.cpp rename to src/libs/recommendation/impl/features/FeaturesClassifierCache.cpp index 1f81c010..fc80098d 100644 --- a/src/similarity/features/SimilarityFeaturesCache.cpp +++ b/src/libs/recommendation/impl/features/FeaturesClassifierCache.cpp @@ -17,28 +17,28 @@ * along with LMS. If not, see . */ -#include "SimilarityFeaturesCache.hpp" +#include "FeaturesClassifierCache.hpp" #include #include -#include "utils/Config.hpp" +#include "utils/IConfig.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" -namespace Similarity { +namespace Recommendation { static std::filesystem::path getCacheDirectory() { - return ServiceProvider::get()->getPath("working-dir") / "cache" / "features"; + return ServiceProvider::get()->getPath("working-dir") / "cache" / "features"; } static std::filesystem::path getCacheNetworkFilePath() { return getCacheDirectory() / "network"; -}; +} static std::filesystem::path getCacheTrackPositionsFilePath() { @@ -79,26 +79,25 @@ networkToCacheFile(const SOM::Network& network, std::filesystem::path path) boost::property_tree::write_xml(path.string(), root); - LMS_LOG(SIMILARITY, DEBUG) << "Created network cache"; + LMS_LOG(RECOMMENDATION, DEBUG) << "Created network cache"; return true; } catch (boost::property_tree::ptree_error& error) { - LMS_LOG(SIMILARITY, ERROR) << "Cannot create network cache: " << error.what(); + LMS_LOG(RECOMMENDATION, ERROR) << "Cannot create network cache: " << error.what(); return false; } } -static std::optional -createNetworkFromCacheFile(const std::filesystem::path& path) +FeaturesClassifierCache::createNetworkFromCacheFile(const std::filesystem::path& path) { if (!std::filesystem::exists(path)) return std::nullopt; try { - LMS_LOG(SIMILARITY, INFO) << "Reading network from cache..."; + LMS_LOG(RECOMMENDATION, INFO) << "Reading network from cache..."; boost::property_tree::ptree root; @@ -132,20 +131,19 @@ createNetworkFromCacheFile(const std::filesystem::path& path) res.setRefVector({x, y}, refVector); } - LMS_LOG(SIMILARITY, INFO) << "Successfully read network from cache"; + LMS_LOG(RECOMMENDATION, INFO) << "Successfully read network from cache"; return res; } catch (boost::property_tree::ptree_error& error) { - LMS_LOG(SIMILARITY, ERROR) << "Cannot read network cache: " << error.what(); + LMS_LOG(RECOMMENDATION, ERROR) << "Cannot read network cache: " << error.what(); return std::nullopt; } } -static bool -objectPositionToCacheFile(const std::map>& objectsPosition, std::filesystem::path path) +FeaturesClassifierCache::objectPositionToCacheFile(const ObjectPositions& objectsPosition, const std::filesystem::path& path) { try { @@ -174,24 +172,23 @@ objectPositionToCacheFile(const std::map>> -createObjectPositionsFromCacheFile(std::filesystem::path path) +std::optional +FeaturesClassifierCache::createObjectPositionsFromCacheFile(const std::filesystem::path& path) { try { - LMS_LOG(SIMILARITY, INFO) << "Reading object position from cache..."; + LMS_LOG(RECOMMENDATION, INFO) << "Reading object position from cache..."; boost::property_tree::ptree root; boost::property_tree::read_xml(path.string(), root); - std::map> res; + ObjectPositions res; for (const auto& object : root.get_child("objects")) { @@ -205,44 +202,42 @@ createObjectPositionsFromCacheFile(std::filesystem::path path) } } - LMS_LOG(SIMILARITY, INFO) << "Successfully read object position from cache"; + LMS_LOG(RECOMMENDATION, INFO) << "Successfully read object position from cache"; return res; } catch (boost::property_tree::ptree_error& error) { - LMS_LOG(SIMILARITY, ERROR) << "Cannot create object position from cache file: " << error.what(); + LMS_LOG(RECOMMENDATION, ERROR) << "Cannot create object position from cache file: " << error.what(); return std::nullopt; } } void -FeaturesCache::invalidate() +FeaturesClassifierCache::invalidate() { std::filesystem::remove(getCacheNetworkFilePath()); std::filesystem::remove(getCacheTrackPositionsFilePath()); } -std::optional -FeaturesCache::read() +std::optional +FeaturesClassifierCache::read() { - std::optional res; - auto network{createNetworkFromCacheFile(getCacheNetworkFilePath())}; if (!network) - return res; + return std::nullopt; auto trackPositions{createObjectPositionsFromCacheFile(getCacheTrackPositionsFilePath())}; if (!trackPositions) - return res; + return std::nullopt; - return FeaturesCache{std::move(*network), std::move(*trackPositions)}; + return FeaturesClassifierCache {std::move(*network), std::move(*trackPositions)}; } void -FeaturesCache::write() +FeaturesClassifierCache::write() const { - std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir") / "cache" / "features"); + std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir") / "cache" / "features"); if (!networkToCacheFile(_network, getCacheNetworkFilePath()) || !objectPositionToCacheFile(_trackPositions, getCacheTrackPositionsFilePath())) @@ -251,11 +246,10 @@ FeaturesCache::write() } } -FeaturesCache::FeaturesCache(SOM::Network network, ObjectPositions trackPositions) +FeaturesClassifierCache::FeaturesClassifierCache(SOM::Network network, ObjectPositions trackPositions) : _network {std::move(network)}, _trackPositions {std::move(trackPositions)} { } - -} // namespace Similarity +} // namespace Recommendation diff --git a/src/libs/recommendation/impl/features/FeaturesClassifierCache.hpp b/src/libs/recommendation/impl/features/FeaturesClassifierCache.hpp new file mode 100644 index 00000000..25ddaeff --- /dev/null +++ b/src/libs/recommendation/impl/features/FeaturesClassifierCache.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include "database/Types.hpp" +#include "som/Network.hpp" + +namespace Recommendation { + +class FeaturesClassifierCache +{ + public: + static void invalidate(); + + static std::optional read(); + void write() const; + + private: + using ObjectPositions = std::unordered_map>; + + FeaturesClassifierCache(SOM::Network network, ObjectPositions trackPositions); + + static std::optional createNetworkFromCacheFile(const std::filesystem::path& path); + static std::optional createObjectPositionsFromCacheFile(const std::filesystem::path& path); + static bool objectPositionToCacheFile(const ObjectPositions& objectsPosition, const std::filesystem::path& path); + + friend class FeaturesClassifier; + + SOM::Network _network; + ObjectPositions _trackPositions; +}; + +} // namespace Recommendation diff --git a/src/similarity/features/SimilarityFeaturesDefs.cpp b/src/libs/recommendation/impl/features/FeaturesDefs.cpp similarity index 99% rename from src/similarity/features/SimilarityFeaturesDefs.cpp rename to src/libs/recommendation/impl/features/FeaturesDefs.cpp index a0aecc4d..b3de7dcb 100644 --- a/src/similarity/features/SimilarityFeaturesDefs.cpp +++ b/src/libs/recommendation/impl/features/FeaturesDefs.cpp @@ -17,14 +17,14 @@ * along with LMS. If not, see . */ -#include "SimilarityFeaturesDefs.hpp" +#include "FeaturesDefs.hpp" #include #include #include "utils/Exception.hpp" -namespace Similarity { +namespace Recommendation { static const std::unordered_map featureDefinitions { @@ -398,5 +398,5 @@ getFeatureNames() return res; } -} // namespace Similarity +} // namespace Recommendation diff --git a/src/similarity/features/SimilarityFeaturesDefs.hpp b/src/libs/recommendation/impl/features/FeaturesDefs.hpp similarity index 95% rename from src/similarity/features/SimilarityFeaturesDefs.hpp rename to src/libs/recommendation/impl/features/FeaturesDefs.hpp index 58eed273..d6e1d759 100644 --- a/src/similarity/features/SimilarityFeaturesDefs.hpp +++ b/src/libs/recommendation/impl/features/FeaturesDefs.hpp @@ -24,7 +24,7 @@ #include #include -namespace Similarity { +namespace Recommendation { using FeatureName = std::string; using FeatureNames = std::unordered_set; @@ -46,4 +46,4 @@ struct FeatureSettings }; using FeatureSettingsMap = std::unordered_map; -} // namespace Similarity +} // namespace Recommendation diff --git a/src/similarity/features/SimilarityFeaturesCache.hpp b/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp similarity index 54% rename from src/similarity/features/SimilarityFeaturesCache.hpp rename to src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp index a12334f3..c9527ef8 100644 --- a/src/similarity/features/SimilarityFeaturesCache.hpp +++ b/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018 Emeric Poupon + * Copyright (C) 2020 Emeric Poupon * * This file is part of LMS. * @@ -19,33 +19,12 @@ #pragma once -#include -#include -#include +#include -#include "database/Types.hpp" -#include "som/Network.hpp" - -namespace Similarity { - -class FeaturesCache +namespace Recommendation { - public: + class IClassifier; - static void invalidate(); + std::unique_ptr createClustersClassifier(); +} - static std::optional read(); - void write(); - - private: - using ObjectPositions = std::map>; - - FeaturesCache(SOM::Network network, ObjectPositions trackPositions); - - friend class FeaturesSearcher; - - SOM::Network _network; - ObjectPositions _trackPositions; -}; - -} // namespace Similarity diff --git a/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp b/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp new file mode 100644 index 00000000..28a079d7 --- /dev/null +++ b/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2020 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include "recommendation/IClassifier.hpp" + +namespace Recommendation +{ + std::unique_ptr createFeaturesClassifier(); +} + diff --git a/src/libs/recommendation/include/recommendation/IClassifier.hpp b/src/libs/recommendation/include/recommendation/IClassifier.hpp new file mode 100644 index 00000000..44fa47d5 --- /dev/null +++ b/src/libs/recommendation/include/recommendation/IClassifier.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2020 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include "database/Types.hpp" + +namespace Database +{ + class Session; +} + +namespace Recommendation +{ + + class IClassifier + { + public: + virtual ~IClassifier() = default; + + virtual std::string_view getName() const = 0; + + virtual bool init(Database::Session& session, bool databaseChanged) = 0; + virtual void requestCancelInit() = 0; + + virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0; + virtual std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const = 0; + virtual std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const = 0; + virtual std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const = 0; + }; + +} // ns Recommendation diff --git a/src/libs/recommendation/include/recommendation/IEngine.hpp b/src/libs/recommendation/include/recommendation/IEngine.hpp new file mode 100644 index 00000000..37f7cb8c --- /dev/null +++ b/src/libs/recommendation/include/recommendation/IEngine.hpp @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include + +#include "database/Types.hpp" + +namespace Database +{ + class Db; + class Session; +} + +namespace Recommendation +{ + class IEngine + { + public: + virtual ~IEngine() = default; + + virtual void start() = 0; + virtual void stop() = 0; + + virtual void requestReload() = 0; + virtual Wt::Signal<>& reloaded() = 0; + + // Closest results first + virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0; + virtual std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) = 0; + virtual std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) = 0; + virtual std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0; + }; + + std::unique_ptr createEngine(Database::Db& db); + +} // ns Recommendation + diff --git a/src/libs/scanner/CMakeLists.txt b/src/libs/scanner/CMakeLists.txt new file mode 100644 index 00000000..57fae63a --- /dev/null +++ b/src/libs/scanner/CMakeLists.txt @@ -0,0 +1,27 @@ + +add_library(lmsscanner SHARED + impl/AcousticBrainzUtils.cpp + impl/MediaScanner.cpp + impl/MediaScannerStats.cpp + ) + +target_include_directories(lmsscanner INTERFACE + include + ) + +target_include_directories(lmsscanner PRIVATE + include + ) + +target_link_libraries(lmsscanner PRIVATE + lmsdatabase + lmsmetadata + lmsutils + ) + +target_link_libraries(lmsscanner PUBLIC + wt + ) + +install(TARGETS lmsscanner DESTINATION lib) + diff --git a/src/similarity/features/AcousticBrainzUtils.cpp b/src/libs/scanner/impl/AcousticBrainzUtils.cpp similarity index 76% rename from src/similarity/features/AcousticBrainzUtils.cpp rename to src/libs/scanner/impl/AcousticBrainzUtils.cpp index 7e460782..970e04da 100644 --- a/src/similarity/features/AcousticBrainzUtils.cpp +++ b/src/libs/scanner/impl/AcousticBrainzUtils.cpp @@ -25,9 +25,10 @@ #include #include -#include "utils/Config.hpp" +#include "utils/IConfig.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" +#include "utils/UUID.hpp" namespace AcousticBrainz @@ -39,7 +40,7 @@ getJsonData(const UUID& mbid) { static const std::string defaultAPIURL = "https://acousticbrainz.org/api/v1/"; - const std::string url {ServiceProvider::get()->getString("acousticbrainz-api-url", defaultAPIURL) + std::string {mbid.getAsString()} + "/low-level"}; + const std::string url {ServiceProvider::get()->getString("acousticbrainz-api-url", defaultAPIURL) + std::string {mbid.getAsString()} + "/low-level"}; boost::asio::io_service ioService; @@ -50,7 +51,7 @@ getJsonData(const UUID& mbid) if (!client.get(url)) { - LMS_LOG(SIMILARITY, ERROR) << "Cannot perform a GET request to url '" << url << "'"; + LMS_LOG(DBUPDATER, ERROR) << "Cannot perform a GET request to url '" << url << "'"; return {}; } @@ -59,13 +60,13 @@ getJsonData(const UUID& mbid) { if (ec) { - LMS_LOG(SIMILARITY, ERROR) << "GET request to url '" << url << "' failed: " << ec.message(); + LMS_LOG(DBUPDATER, ERROR) << "GET request to url '" << url << "' failed: " << ec.message(); return; } if (msg.status() != 200) { - LMS_LOG(SIMILARITY, ERROR) << "GET request to url '" << url << "' failed: status = " << msg.status() << ", body = " << msg.body(); + LMS_LOG(DBUPDATER, ERROR) << "GET request to url '" << url << "' failed: status = " << msg.status() << ", body = " << msg.body(); return; } diff --git a/src/similarity/features/AcousticBrainzUtils.hpp b/src/libs/scanner/impl/AcousticBrainzUtils.hpp similarity index 96% rename from src/similarity/features/AcousticBrainzUtils.hpp rename to src/libs/scanner/impl/AcousticBrainzUtils.hpp index 70d8fdf6..cdd73d77 100644 --- a/src/similarity/features/AcousticBrainzUtils.hpp +++ b/src/libs/scanner/impl/AcousticBrainzUtils.hpp @@ -21,7 +21,7 @@ #include -#include "utils/UUID.hpp" +class UUID; namespace AcousticBrainz { diff --git a/src/scanner/MediaScanner.cpp b/src/libs/scanner/impl/MediaScanner.cpp similarity index 81% rename from src/scanner/MediaScanner.cpp rename to src/libs/scanner/impl/MediaScanner.cpp index b7c09754..e70b1c43 100644 --- a/src/scanner/MediaScanner.cpp +++ b/src/libs/scanner/impl/MediaScanner.cpp @@ -23,15 +23,18 @@ #include -#include "cover/CoverArtGrabber.hpp" #include "database/Artist.hpp" #include "database/Cluster.hpp" #include "database/Release.hpp" #include "database/ScanSettings.hpp" #include "database/Track.hpp" +#include "database/TrackFeatures.hpp" +#include "metadata/TagLibParser.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" #include "utils/Path.hpp" +#include "utils/UUID.hpp" +#include "AcousticBrainzUtils.hpp" using namespace Database; @@ -60,7 +63,7 @@ getNextFirstOfMonth(Wt::WDate current) } bool -isFileSupported(const std::filesystem::path& file, const std::set& extensions) +isFileSupported(const std::filesystem::path& file, const std::unordered_set& extensions) { return (extensions.find(file.extension()) != extensions.end()); } @@ -206,18 +209,27 @@ getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames) namespace Scanner { +std::unique_ptr +createMediaScanner(Database::Db& db) +{ + return std::make_unique(db); +} + MediaScanner::MediaScanner(Database::Db& db) : _dbSession {db} { + // For now, always use TagLib + _metadataParser = std::make_unique(); + _ioService.setThreadCount(1); refreshScanSettings(); } -void -MediaScanner::setAddon(MediaScannerAddon& addon) +MediaScanner::~MediaScanner() { - _addons.push_back(&addon); + if (_running) + stop(); } void @@ -242,9 +254,6 @@ MediaScanner::stop(void) { _running = false; - for (auto& addon : _addons) - addon->requestStop(); - _scheduleTimer.cancel(); _ioService.stop(); @@ -290,7 +299,7 @@ MediaScanner::scheduleNextScan() refreshScanSettings(); - Wt::WDateTime now {Wt::WLocalDateTime::currentServerDateTime().toUTC()}; + const Wt::WDateTime now {Wt::WLocalDateTime::currentServerDateTime().toUTC()}; Wt::WDate nextScanDate; switch (_updatePeriod) @@ -341,33 +350,21 @@ MediaScanner::scheduleNextScan() void MediaScanner::countAllFiles(ScanStats& stats) { - std::error_code ec; - stats.filesToScan = 0; - std::filesystem::recursive_directory_iterator itPath {_mediaDirectory, std::filesystem::directory_options::follow_directory_symlink, ec}; - if (ec) + exploreFilesRecursive(_mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path) { - LMS_LOG(DBUPDATER, ERROR) << "Cannot iterate over '" << _mediaDirectory.string() << "': " << ec.message(); - return; - } + if (!_running) + return false; - std::filesystem::recursive_directory_iterator itEnd; - while (_running && itPath != itEnd) - { - const std::filesystem::path& path {*itPath}; - - if (!ec) + if (!ec && isFileSupported(path, _fileExtensions)) { - if (std::filesystem::is_regular_file(path) && isFileSupported(path, _fileExtensions)) - stats.filesToScan ++; - - if (stats.filesToScan % 250 == 0) - notifyInProgressIfNeeded(stats); + stats.filesToScan++; + notifyInProgressIfNeeded(stats); } - itPath.increment(ec); - } + return true; + }); } void @@ -428,13 +425,14 @@ MediaScanner::scan(boost::system::error_code err) if (_running) checkDuplicatedAudioFiles(stats); - LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), duplicates = " << stats.duplicates.size(); + // Now update all the track features if needed + fetchTrackFeatures(stats); - if (_running) - { - for (auto& addon : _addons) - addon->preScanComplete(); - } + LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << "/" << stats.featuresToFetch <<", duplicates = " << stats.duplicates.size(); + + LMS_LOG(DBUPDATER, INFO) << "Optimizing db..."; + _dbSession.optimize(); + LMS_LOG(DBUPDATER, INFO) << "Optimize db done!"; if (_running) { @@ -457,41 +455,104 @@ MediaScanner::scan(boost::system::error_code err) _curState = State::NotScheduled; _inProgressScanStats.reset(); } +} - LMS_LOG(DBUPDATER, INFO) << "Optimizing db..."; - _dbSession.optimize(); - LMS_LOG(DBUPDATER, INFO) << "Optimize db done!"; +bool +MediaScanner::fetchTrackFeatures(Database::IdType trackId, const UUID& MBID) +{ + std::map features; + + LMS_LOG(DBUPDATER, INFO) << "Fetching low level features for track '" << MBID.getAsString() << "'"; + const std::string data {AcousticBrainz::extractLowLevelFeatures(MBID)}; + if (data.empty()) + { + LMS_LOG(DBUPDATER, ERROR) << "Track " << trackId << ", MBID = '" << MBID.getAsString() << "': cannot extract features using AcousticBrainz"; + return false; + } + + { + auto uniqueTransaction {_dbSession.createUniqueTransaction()}; + + Wt::Dbo::ptr track {Database::Track::getById(_dbSession, trackId)}; + if (!track) + return false; + + Database::TrackFeatures::create(_dbSession, track, data); + } + + return true; +} + +void +MediaScanner::fetchTrackFeatures(ScanStats& stats) +{ + if (_recommendationEngineType != ScanSettings::RecommendationEngineType::Features) + return; + + LMS_LOG(DBUPDATER, INFO) << "Fetching missing track features..."; + + struct TrackInfo + { + Database::IdType id; + UUID mbid; + }; + + const auto tracksToFetch {[&]() + { + std::vector res; + + auto transaction {_dbSession.createSharedTransaction()}; + + auto tracks {Database::Track::getAllWithMBIDAndMissingFeatures(_dbSession)}; + for (const auto& track : tracks) + res.emplace_back(TrackInfo {track.id(), *track->getMBID()}); + + return res; + }()}; + + stats.featuresToFetch = tracksToFetch.size(); + + LMS_LOG(DBUPDATER, INFO) << "Found " << tracksToFetch.size() << " track(s) to fetch!"; + + for (const TrackInfo& trackToFetch : tracksToFetch) + { + if (!_running) + return; + + if (fetchTrackFeatures(trackToFetch.id, trackToFetch.mbid)) + stats.featuresFetched++; + + } + + LMS_LOG(DBUPDATER, INFO) << "Track features fetched!"; } void MediaScanner::refreshScanSettings() { - { - auto transaction {_dbSession.createSharedTransaction()}; + auto transaction {_dbSession.createSharedTransaction()}; - ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)}; + ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)}; - LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion(); + LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion(); - _scanVersion = scanSettings->getScanVersion(); - _startTime = scanSettings->getUpdateStartTime(); - _updatePeriod = scanSettings->getUpdatePeriod(); + _scanVersion = scanSettings->getScanVersion(); + _startTime = scanSettings->getUpdateStartTime(); + _updatePeriod = scanSettings->getUpdatePeriod(); - _fileExtensions = scanSettings->getAudioFileExtensions(); - _mediaDirectory = scanSettings->getMediaDirectory(); + _fileExtensions = scanSettings->getAudioFileExtensions(); + _mediaDirectory = scanSettings->getMediaDirectory(); + _recommendationEngineType = scanSettings->getRecommendationEngineType(); - auto clusterTypes = scanSettings->getClusterTypes(); - std::set clusterTypeNames; + auto clusterTypes = scanSettings->getClusterTypes(); + std::set clusterTypeNames; - std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes), - std::inserter(clusterTypeNames, clusterTypeNames.begin()), - [](ClusterType::pointer clusterType) { return clusterType->getName(); }); + std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes), + std::inserter(clusterTypeNames, clusterTypeNames.begin()), + [](ClusterType::pointer clusterType) { return clusterType->getName(); }); - _metadataParser.setClusterTypeNames(clusterTypeNames); - } + _metadataParser->setClusterTypeNames(clusterTypeNames); - for (auto& addon : _addons) - addon->refreshSettings(); } void @@ -519,8 +580,6 @@ MediaScanner::notifyInProgressIfNeeded(const ScanStats& stats) void MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats) { - notifyInProgressIfNeeded(stats); - Wt::WDateTime lastWriteTime; try { @@ -548,7 +607,7 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S } } - std::optional trackInfo {_metadataParser.parse(file)}; + std::optional trackInfo {_metadataParser->parse(file)}; if (!trackInfo) { stats.errors.emplace_back(file, ScanErrorType::CannotParseFile); @@ -666,6 +725,7 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S track.modify()->setYear(*trackInfo->originalYear); track.modify()->setMBID(trackInfo->musicBrainzRecordID); + track.modify()->setFeatures({}); // TODO: only if MBID changed? track.modify()->setHasCover(trackInfo->hasCover); track.modify()->setCopyright(trackInfo->copyright); track.modify()->setCopyrightURL(trackInfo->copyrightURL); @@ -674,41 +734,32 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S void MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats) { - std::error_code ec; - - std::filesystem::recursive_directory_iterator itPath {_mediaDirectory, std::filesystem::directory_options::follow_directory_symlink, ec}; - if (ec) + exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path) { - LMS_LOG(DBUPDATER, ERROR) << "Cannot iterate over '" << mediaDirectory.string() << "': " << ec.message(); - stats.errors.emplace_back(ScanError {mediaDirectory, ScanErrorType::CannotReadFile, ec.message()}); - return; - } - - std::filesystem::recursive_directory_iterator itEnd; - while (_running && itPath != itEnd) - { - const std::filesystem::path& path {*itPath}; + if (!_running) + return false; if (ec) { LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message(); stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()}); } - else if (std::filesystem::is_regular_file(path)) + else if (isFileSupported(path, _fileExtensions)) { - if (isFileSupported(path, _fileExtensions)) - scanAudioFile(path, forceScan, stats ); + scanAudioFile(path, forceScan, stats ); + + notifyInProgressIfNeeded(stats); } - itPath.increment(ec); - } + return true; + }); notifyInProgress(stats); } // Check if a file exists and is still in a media directory static bool -checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory, const std::set& extensions) +checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory, const std::unordered_set& extensions) { try { @@ -769,6 +820,8 @@ MediaScanner::removeMissingTracks(ScanStats& stats) stats.deletions++; } } + + notifyInProgressIfNeeded(stats); } } diff --git a/src/scanner/MediaScanner.hpp b/src/libs/scanner/impl/MediaScanner.hpp similarity index 68% rename from src/scanner/MediaScanner.hpp rename to src/libs/scanner/impl/MediaScanner.hpp index 0ad1586a..a04ceb39 100644 --- a/src/scanner/MediaScanner.hpp +++ b/src/libs/scanner/impl/MediaScanner.hpp @@ -29,56 +29,39 @@ #include +#include "database/Types.hpp" #include "database/ScanSettings.hpp" #include "database/Session.hpp" -#include "metadata/TagLibParser.hpp" +#include "metadata/IParser.hpp" +#include "scanner/IMediaScanner.hpp" -#include "MediaScannerAddon.hpp" -#include "MediaScannerStats.hpp" +class UUID; namespace Scanner { -class MediaScanner +class MediaScanner : public IMediaScanner { public: MediaScanner(Database::Db& db); + ~MediaScanner(); - void setAddon(MediaScannerAddon& addon); + MediaScanner(const MediaScanner&) = delete; + MediaScanner(MediaScanner&&) = delete; + MediaScanner& operator=(const MediaScanner&) = delete; + MediaScanner& operator=(MediaScanner&&) = delete; - void start(); - void stop(); - void restart(); + void start() override; + void stop() override; + void restart() override; - // Async requests - void requestImmediateScan(); - void requestReschedule(); + void requestImmediateScan() override; + void requestReschedule() override ; + Status getStatus() override; - enum class State - { - NotScheduled, - Scheduled, - InProgress, - }; - - struct Status - { - State currentState {State::NotScheduled}; - Wt::WDateTime nextScheduledScan; - std::optional lastCompleteScanStats; - std::optional inProgressScanStats; - }; - - Status getStatus(); - - // Called just after scan complete - Wt::Signal<>& scanComplete() { return _sigScanComplete; } - - // Called during scan in progress - Wt::Signal& scanInProgress() { return _sigScanInProgress; } - - // Called after a schedule - Wt::Signal& scheduled() { return _sigScheduled; } + Wt::Signal<>& scanComplete() override { return _sigScanComplete; } + Wt::Signal& scanInProgress() override { return _sigScanInProgress; } + Wt::Signal& scheduled() override { return _sigScheduled; } private: @@ -90,6 +73,8 @@ class MediaScanner void scan(boost::system::error_code ec); void scanMediaDirectory( const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats); + bool fetchTrackFeatures(Database::IdType trackId, const UUID& MBID); + void fetchTrackFeatures(ScanStats& stats); // Helpers void refreshScanSettings(); @@ -103,7 +88,7 @@ class MediaScanner void notifyInProgressIfNeeded(const ScanStats& stats); void notifyInProgress(const ScanStats& stats); - bool _running {false}; + bool _running {}; Wt::WIOService _ioService; boost::asio::system_timer _scheduleTimer {_ioService}; Wt::Signal<> _sigScanComplete; @@ -111,8 +96,7 @@ class MediaScanner std::chrono::system_clock::time_point _lastScanInProgressEmit {}; Wt::Signal _sigScheduled; Database::Session _dbSession; - MetaData::TagLibParser _metadataParser; - std::vector _addons; + std::unique_ptr _metadataParser; std::mutex _statusMutex; State _curState {State::NotScheduled}; @@ -121,11 +105,12 @@ class MediaScanner Wt::WDateTime _nextScheduledScan; // Current scan settings - std::size_t _scanVersion {}; - Wt::WTime _startTime; + std::size_t _scanVersion {}; + Wt::WTime _startTime; Database::ScanSettings::UpdatePeriod _updatePeriod {Database::ScanSettings::UpdatePeriod::Never}; - std::set _fileExtensions; + std::unordered_set _fileExtensions; std::filesystem::path _mediaDirectory; + Database::ScanSettings::RecommendationEngineType _recommendationEngineType; }; // class MediaScanner diff --git a/src/scanner/MediaScannerStats.cpp b/src/libs/scanner/impl/MediaScannerStats.cpp similarity index 97% rename from src/scanner/MediaScannerStats.cpp rename to src/libs/scanner/impl/MediaScannerStats.cpp index f2f991f5..a2a984da 100644 --- a/src/scanner/MediaScannerStats.cpp +++ b/src/libs/scanner/impl/MediaScannerStats.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "MediaScannerStats.hpp" +#include "scanner/MediaScannerStats.hpp" namespace Scanner { diff --git a/src/libs/scanner/include/scanner/IMediaScanner.hpp b/src/libs/scanner/include/scanner/IMediaScanner.hpp new file mode 100644 index 00000000..e50be005 --- /dev/null +++ b/src/libs/scanner/include/scanner/IMediaScanner.hpp @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2013 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +#include +#include + +#include "MediaScannerStats.hpp" + +namespace Database +{ + class Db; +} + +namespace Scanner { + +class IMediaScanner +{ + public: + virtual ~IMediaScanner() = default; + + virtual void start() = 0; + virtual void stop() = 0; + virtual void restart() = 0; + + // Async requests + virtual void requestImmediateScan() = 0; + virtual void requestReschedule() = 0; + + + enum class State + { + NotScheduled, + Scheduled, + InProgress, + }; + + struct Status + { + State currentState {State::NotScheduled}; + Wt::WDateTime nextScheduledScan; + std::optional lastCompleteScanStats; + std::optional inProgressScanStats; + }; + + virtual Status getStatus() = 0; + + // Called just after scan complete + virtual Wt::Signal<>& scanComplete() = 0; + + // Called during scan in progress + virtual Wt::Signal& scanInProgress() = 0; + + // Called after a schedule + virtual Wt::Signal& scheduled() = 0; + +}; + +std::unique_ptr createMediaScanner(Database::Db& db); + + +} // Scanner + diff --git a/src/scanner/MediaScannerStats.hpp b/src/libs/scanner/include/scanner/MediaScannerStats.hpp similarity index 92% rename from src/scanner/MediaScannerStats.hpp rename to src/libs/scanner/include/scanner/MediaScannerStats.hpp index 6f4e514d..99c898b6 100644 --- a/src/scanner/MediaScannerStats.hpp +++ b/src/libs/scanner/include/scanner/MediaScannerStats.hpp @@ -77,10 +77,13 @@ namespace Scanner { std::size_t skips {}; // no change since last scan std::size_t scans {}; // actually scanned filed - std::size_t additions {}; // Added in DB + std::size_t additions {}; // added in DB std::size_t deletions {}; // removed from DB std::size_t updates {}; // updated file in DB + std::size_t featuresFetched {}; // features fetched in DB + std::size_t featuresToFetch {}; // features to be fetched in DB + std::vector errors; std::vector duplicates; diff --git a/src/libs/som/CMakeLists.txt b/src/libs/som/CMakeLists.txt new file mode 100644 index 00000000..2e19b9d2 --- /dev/null +++ b/src/libs/som/CMakeLists.txt @@ -0,0 +1,20 @@ + +add_library(lmssom STATIC + impl/DataNormalizer.cpp + impl/Network.cpp + ) + +target_include_directories(lmssom INTERFACE + include + ) + +target_include_directories(lmssom PRIVATE + include + ) + +target_link_libraries(lmssom PUBLIC + lmsutils + ) + +set_property(TARGET lmssom PROPERTY POSITION_INDEPENDENT_CODE ON) + diff --git a/src/similarity/features/som/DataNormalizer.cpp b/src/libs/som/impl/DataNormalizer.cpp similarity index 98% rename from src/similarity/features/som/DataNormalizer.cpp rename to src/libs/som/impl/DataNormalizer.cpp index d6498fcf..a4c39351 100644 --- a/src/similarity/features/som/DataNormalizer.cpp +++ b/src/libs/som/impl/DataNormalizer.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "DataNormalizer.hpp" +#include "som/DataNormalizer.hpp" #include #include diff --git a/src/similarity/features/som/Network.cpp b/src/libs/som/impl/Network.cpp similarity index 97% rename from src/similarity/features/som/Network.cpp rename to src/libs/som/impl/Network.cpp index 0431c966..75e60fce 100644 --- a/src/similarity/features/som/Network.cpp +++ b/src/libs/som/impl/Network.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "Network.hpp" +#include "som/Network.hpp" #include #include @@ -201,9 +201,9 @@ Network::getClosestRefVectorPosition(const InputVector& data, InputVector::Dista } std::optional -Network::getClosestRefVectorPosition(const std::set& refVectorsPosition, InputVector::Distance maxDistance) const +Network::getClosestRefVectorPosition(const std::unordered_set& refVectorsPosition, InputVector::Distance maxDistance) const { - std::set neighboursPosition; + std::unordered_set neighboursPosition; for (const Position& refVectorPosition : refVectorsPosition) { if (refVectorPosition.y > 0) diff --git a/src/similarity/features/som/DataNormalizer.hpp b/src/libs/som/include/som/DataNormalizer.hpp similarity index 100% rename from src/similarity/features/som/DataNormalizer.hpp rename to src/libs/som/include/som/DataNormalizer.hpp diff --git a/src/similarity/features/som/InputVector.hpp b/src/libs/som/include/som/InputVector.hpp similarity index 98% rename from src/similarity/features/som/InputVector.hpp rename to src/libs/som/include/som/InputVector.hpp index 68d5834e..50737fec 100644 --- a/src/similarity/features/som/InputVector.hpp +++ b/src/libs/som/include/som/InputVector.hpp @@ -23,13 +23,15 @@ #include #include +#include "utils/Exception.hpp" + namespace SOM { class Exception : public LmsException { public: - Exception(const std::string& msg) : LmsException(msg) {} + using LmsException::LmsException; }; class InputVector diff --git a/src/similarity/features/som/Matrix.hpp b/src/libs/som/include/som/Matrix.hpp similarity index 90% rename from src/similarity/features/som/Matrix.hpp rename to src/libs/som/include/som/Matrix.hpp index 32df1bff..2688d729 100644 --- a/src/similarity/features/som/Matrix.hpp +++ b/src/libs/som/include/som/Matrix.hpp @@ -21,7 +21,7 @@ #include #include -#include +#include #include namespace SOM @@ -116,3 +116,20 @@ class Matrix }; } // ns SOM + +namespace std { + +template<> +class hash +{ + public: + size_t operator()(const SOM::Position& s) const + { + size_t h1 = std::hash()(s.x); + size_t h2 = std::hash()(s.y); + return h1 ^ (h2 << 1); + } +}; + +} // ns std + diff --git a/src/similarity/features/som/Network.hpp b/src/libs/som/include/som/Network.hpp similarity index 97% rename from src/similarity/features/som/Network.hpp rename to src/libs/som/include/som/Network.hpp index cf64a83c..f3ba2df1 100644 --- a/src/similarity/features/som/Network.hpp +++ b/src/libs/som/include/som/Network.hpp @@ -20,12 +20,11 @@ #pragma once #include -#include +#include #include #include #include -#include "utils/Exception.hpp" #include "InputVector.hpp" #include "Matrix.hpp" @@ -71,7 +70,7 @@ class Network Position getClosestRefVectorPosition(const InputVector& data) const; std::optional getClosestRefVectorPosition(const InputVector& data, InputVector::Distance maxDistance) const; - std::optional getClosestRefVectorPosition(const std::set& refVectorsPosition, InputVector::Distance maxDistance) const; + std::optional getClosestRefVectorPosition(const std::unordered_set& refVectorsPosition, InputVector::Distance maxDistance) const; InputVector::Distance getRefVectorsDistance(const Position& position1, const Position& position2) const; diff --git a/src/libs/subsonic/CMakeLists.txt b/src/libs/subsonic/CMakeLists.txt new file mode 100644 index 00000000..0531b7bf --- /dev/null +++ b/src/libs/subsonic/CMakeLists.txt @@ -0,0 +1,30 @@ + +add_library(lmssubsonic SHARED + impl/SubsonicId.cpp + impl/SubsonicResource.cpp + impl/SubsonicResponse.cpp + ) + +target_include_directories(lmssubsonic INTERFACE + include + ) + +target_include_directories(lmssubsonic PRIVATE + include + ) + +target_link_libraries(lmssubsonic PRIVATE + lmsauth + lmsav + lmscover + lmsrecommendation + lmsutils + ) + +target_link_libraries(lmssubsonic PUBLIC + lmsdatabase + wt + ) + +install(TARGETS lmssubsonic DESTINATION lib) + diff --git a/src/api/subsonic/SubsonicId.cpp b/src/libs/subsonic/impl/SubsonicId.cpp similarity index 100% rename from src/api/subsonic/SubsonicId.cpp rename to src/libs/subsonic/impl/SubsonicId.cpp diff --git a/src/api/subsonic/SubsonicId.hpp b/src/libs/subsonic/impl/SubsonicId.hpp similarity index 100% rename from src/api/subsonic/SubsonicId.hpp rename to src/libs/subsonic/impl/SubsonicId.hpp diff --git a/src/api/subsonic/SubsonicResource.cpp b/src/libs/subsonic/impl/SubsonicResource.cpp similarity index 97% rename from src/api/subsonic/SubsonicResource.cpp rename to src/libs/subsonic/impl/SubsonicResource.cpp index a52f5d22..056005c5 100644 --- a/src/api/subsonic/SubsonicResource.cpp +++ b/src/libs/subsonic/impl/SubsonicResource.cpp @@ -16,7 +16,7 @@ * You should have received a copy of the GNU General Public License * along with LMS. If not, see . */ -#include "SubsonicResource.hpp" +#include "subsonic/SubsonicResource.hpp" #include #include @@ -26,9 +26,9 @@ #include -#include "auth/PasswordService.hpp" +#include "auth/IPasswordService.hpp" #include "av/AvTranscoder.hpp" -#include "cover/CoverArtGrabber.hpp" +#include "cover/ICoverArtGrabber.hpp" #include "database/Artist.hpp" #include "database/Cluster.hpp" #include "database/Db.hpp" @@ -38,7 +38,7 @@ #include "database/TrackBookmark.hpp" #include "database/TrackList.hpp" #include "database/User.hpp" -#include "similarity/SimilaritySearcher.hpp" +#include "recommendation/IEngine.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" #include "utils/String.hpp" @@ -578,10 +578,10 @@ handleChangePassword(RequestContext& context) std::string username {getMandatoryParameterAs(context.parameters, "username")}; std::string password {decodePasswordIfNeeded(getMandatoryParameterAs(context.parameters, "password"))}; - if (!ServiceProvider::get()->evaluatePasswordStrength(username, password)) + if (!ServiceProvider::get()->evaluatePasswordStrength(username, password)) throw PasswordTooWeakGenericError {}; - const User::PasswordHash hash {ServiceProvider::get()->hashPassword(password)}; + const User::PasswordHash hash {ServiceProvider::get()->hashPassword(password)}; auto transaction {context.dbSession.createUniqueTransaction()}; @@ -660,10 +660,10 @@ handleCreateUserRequest(RequestContext& context) std::string password {decodePasswordIfNeeded(getMandatoryParameterAs(context.parameters, "password"))}; // Just ignore all the other fields as we don't handle them - if (!ServiceProvider::get()->evaluatePasswordStrength(username, password)) + if (!ServiceProvider::get()->evaluatePasswordStrength(username, password)) throw PasswordTooWeakGenericError {}; - const User::PasswordHash hash {ServiceProvider::get()->hashPassword(password)}; + const User::PasswordHash hash {ServiceProvider::get()->hashPassword(password)}; auto transaction {context.dbSession.createUniqueTransaction()}; @@ -944,7 +944,7 @@ handleGetArtistInfoRequestCommon(RequestContext& context, bool id3) artistInfoNode.createChild("musicBrainzId").setValue(artistMBID->getAsString()); } - auto similarArtistsId {ServiceProvider::get()->getSimilarArtists(context.dbSession, id.value, count)}; + auto similarArtistsId {ServiceProvider::get()->getSimilarArtists(context.dbSession, id.value, count)}; { auto transaction {context.dbSession.createSharedTransaction()}; @@ -1140,7 +1140,7 @@ handleGetSimilarSongsRequestCommon(RequestContext& context, bool id3) // Optional params std::size_t count {getParameterAs(context.parameters, "count").value_or(50)}; - auto similarArtistsId {ServiceProvider::get()->getSimilarArtists(context.dbSession, id.value, 5)}; + auto similarArtistsId {ServiceProvider::get()->getSimilarArtists(context.dbSession, id.value, 5)}; auto transaction {context.dbSession.createSharedTransaction()}; @@ -1588,10 +1588,10 @@ handleUpdateUserRequest(RequestContext& context) if (password) { *password = decodePasswordIfNeeded(*password); - if (!ServiceProvider::get()->evaluatePasswordStrength(username, *password)) + if (!ServiceProvider::get()->evaluatePasswordStrength(username, *password)) throw PasswordTooWeakGenericError {}; - hash = ServiceProvider::get()->hashPassword(*password); + hash = ServiceProvider::get()->hashPassword(*password); } auto transaction {context.dbSession.createUniqueTransaction()}; @@ -1895,16 +1895,16 @@ handleGetCoverArt(RequestContext& context, Wt::Http::ResponseContinuation*) switch (id.type) { case Id::Type::Track: - res.data = ServiceProvider::get()->getFromTrack(context.dbSession, id.value, Image::Format::JPEG, size); + res.data = ServiceProvider::get()->getFromTrack(context.dbSession, id.value, CoverArt::Format::JPEG, size); break; case Id::Type::Release: - res.data = ServiceProvider::get()->getFromRelease(context.dbSession, id.value, Image::Format::JPEG, size); + res.data = ServiceProvider::get()->getFromRelease(context.dbSession, id.value, CoverArt::Format::JPEG, size); break; default: throw BadParameterGenericError {"id"}; } - res.mimeType = Image::format_to_mimeType(Image::Format::JPEG); + res.mimeType = CoverArt::formatToMimeType(CoverArt::Format::JPEG); return res; } @@ -2056,19 +2056,19 @@ SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Resp SessionPool::ScopedSession dbSession {_sessionPool}; - switch (ServiceProvider::get()->checkUserPassword(dbSession.get(), + switch (ServiceProvider::get()->checkUserPassword(dbSession.get(), boost::asio::ip::address::from_string(request.clientAddress()), clientInfo.user, clientInfo.password)) { - case Auth::PasswordService::PasswordCheckResult::Match: + case Auth::IPasswordService::PasswordCheckResult::Match: break; - case Auth::PasswordService::PasswordCheckResult::Mismatch: + case Auth::IPasswordService::PasswordCheckResult::Mismatch: throw WrongUsernameOrPasswordError {}; - case Auth::PasswordService::PasswordCheckResult::Throttled: + case Auth::IPasswordService::PasswordCheckResult::Throttled: throw LoginThrottledGenericError {}; } - RequestContext requestContext {.parameters = parameters, .dbSession = dbSession.get(), .userName = clientInfo.user}; + RequestContext requestContext {parameters, dbSession.get(), clientInfo.user}; auto itEntryPoint {requestEntryPoints.find(requestPath)}; if (itEntryPoint != requestEntryPoints.end()) diff --git a/src/api/subsonic/SubsonicResponse.cpp b/src/libs/subsonic/impl/SubsonicResponse.cpp similarity index 100% rename from src/api/subsonic/SubsonicResponse.cpp rename to src/libs/subsonic/impl/SubsonicResponse.cpp diff --git a/src/api/subsonic/SubsonicResponse.hpp b/src/libs/subsonic/impl/SubsonicResponse.hpp similarity index 100% rename from src/api/subsonic/SubsonicResponse.hpp rename to src/libs/subsonic/impl/SubsonicResponse.hpp diff --git a/src/api/subsonic/SubsonicResource.hpp b/src/libs/subsonic/include/subsonic/SubsonicResource.hpp similarity index 100% rename from src/api/subsonic/SubsonicResource.hpp rename to src/libs/subsonic/include/subsonic/SubsonicResource.hpp diff --git a/src/libs/utils/CMakeLists.txt b/src/libs/utils/CMakeLists.txt new file mode 100644 index 00000000..c468be2f --- /dev/null +++ b/src/libs/utils/CMakeLists.txt @@ -0,0 +1,33 @@ + +add_library(lmsutils SHARED + impl/Config.cpp + impl/Logger.cpp + impl/NetAddress.cpp + impl/Path.cpp + impl/Random.cpp + impl/StreamLogger.cpp + impl/String.cpp + impl/UUID.cpp + impl/WtLogger.cpp + ) + +target_include_directories(lmsutils INTERFACE + include + ) + +target_include_directories(lmsutils PRIVATE + include + ) + +target_link_libraries(lmsutils PRIVATE + config++ + ) + +target_link_libraries(lmsutils PUBLIC + boost_system + stdc++fs + wt + ) + +install(TARGETS lmsutils DESTINATION lib) + diff --git a/src/utils/Config.cpp b/src/libs/utils/impl/Config.cpp similarity index 73% rename from src/utils/Config.cpp rename to src/libs/utils/impl/Config.cpp index 40855b64..ff8c4846 100644 --- a/src/utils/Config.cpp +++ b/src/libs/utils/impl/Config.cpp @@ -19,16 +19,36 @@ #include "Config.hpp" +#include "utils/Exception.hpp" #include "utils/Logger.hpp" +std::unique_ptr createConfig(const std::filesystem::path& p) +{ + return std::make_unique(p); +} Config::Config(const std::filesystem::path& p) { - _config.readFile(p.string().c_str()); + try + { + _config.readFile(p.string().c_str()); + } + catch( libconfig::FileIOException& e) + { + throw LmsException {"Cannot open config file '" + p.string() + "'"}; + } + catch( libconfig::ParseException& e) + { + throw LmsException {"Cannot parse config file '" + p.string() + "', line = " + std::to_string(e.getLine()) + ", error = '" + e.getError() + "'"}; + } + catch (libconfig::ConfigException& e) + { + throw LmsException {"Cannot open config file '" + p.string() + "': " + e.what()}; + } } std::string -Config::getString(const std::string& setting, const std::string& def, const std::set& allowedValues) +Config::getString(const std::string& setting, const std::string& def, const std::unordered_set& allowedValues) { try { std::string res {(const char*)_config.lookup(setting)}; diff --git a/src/utils/Config.hpp b/src/libs/utils/impl/Config.hpp similarity index 77% rename from src/utils/Config.hpp rename to src/libs/utils/impl/Config.hpp index 0b248e20..e1e9600e 100644 --- a/src/utils/Config.hpp +++ b/src/libs/utils/impl/Config.hpp @@ -18,13 +18,12 @@ */ #pragma once -#include -#include +#include "utils/IConfig.hpp" #include // Used to get config values from configuration files -class Config final +class Config final : public IConfig { public: Config(const std::filesystem::path& p); @@ -36,11 +35,11 @@ class Config final Config& operator=(Config&&) = delete; // Default values are returned in case of setting not found - std::string getString(const std::string& setting, const std::string& def = "", const std::set& allowedValues = {}); - std::filesystem::path getPath(const std::string& setting, const std::filesystem::path& def = std::filesystem::path()); - unsigned long getULong(const std::string& setting, unsigned long def = 0); - long getLong(const std::string& setting, long def = 0); - bool getBool(const std::string& setting, bool def = false); + std::string getString(const std::string& setting, const std::string& def = "", const std::unordered_set& allowedValues = {}) override; + std::filesystem::path getPath(const std::string& setting, const std::filesystem::path& def = std::filesystem::path()) override; + unsigned long getULong(const std::string& setting, unsigned long def = 0) override; + long getLong(const std::string& setting, long def = 0) override; + bool getBool(const std::string& setting, bool def = false) override; private: diff --git a/src/utils/Logger.cpp b/src/libs/utils/impl/Logger.cpp similarity index 95% rename from src/utils/Logger.cpp rename to src/libs/utils/impl/Logger.cpp index 79bfafe9..602fdbc3 100644 --- a/src/utils/Logger.cpp +++ b/src/libs/utils/impl/Logger.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "Logger.hpp" +#include "utils/Logger.hpp" const char* getModuleName(Module mod) { @@ -34,7 +34,7 @@ const char* getModuleName(Module mod) case Module::METADATA: return "METADATA"; case Module::REMOTE: return "REMOTE"; case Module::SERVICE: return "SERVICE"; - case Module::SIMILARITY: return "SIMILARITY"; + case Module::RECOMMENDATION: return "RECOMMENDATION"; case Module::TRANSCODE: return "TRANSCODE"; case Module::UI: return "UI"; } diff --git a/src/utils/NetAddress.cpp b/src/libs/utils/impl/NetAddress.cpp similarity index 97% rename from src/utils/NetAddress.cpp rename to src/libs/utils/impl/NetAddress.cpp index 723de502..cede3029 100644 --- a/src/utils/NetAddress.cpp +++ b/src/libs/utils/impl/NetAddress.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "NetAddress.hpp" +#include "utils/NetAddress.hpp" namespace std { diff --git a/src/utils/Path.cpp b/src/libs/utils/impl/Path.cpp similarity index 71% rename from src/utils/Path.cpp rename to src/libs/utils/impl/Path.cpp index dd2c4b21..0bfc807b 100644 --- a/src/utils/Path.cpp +++ b/src/libs/utils/impl/Path.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "Path.hpp" +#include "utils/Path.hpp" #include #include @@ -87,3 +87,46 @@ getLastWriteTime(const std::filesystem::path& file) return Wt::WDateTime::fromTime_t(sb.st_mtime); } +void +exploreFilesRecursive(const std::filesystem::path& directory, std::function cb) +{ + std::error_code ec; + std::filesystem::directory_iterator itPath {directory, std::filesystem::directory_options::follow_directory_symlink, ec}; + + if (ec) + { + cb(ec, directory); + return; + } + + std::filesystem::directory_iterator itEnd; + while (itPath != itEnd) + { + bool continueExploring {true}; + + if (ec) + { + continueExploring = cb(ec, *itPath); + } + else + { + if (std::filesystem::is_regular_file(*itPath, ec)) + { + continueExploring = cb(ec, *itPath); + } + else if (std::filesystem::is_directory(*itPath, ec)) + { + if (!ec) + exploreFilesRecursive(*itPath, cb); + else + continueExploring = cb(ec, *itPath); + } + } + + if (!continueExploring) + break; + + itPath.increment(ec); + } +} + diff --git a/src/utils/Random.cpp b/src/libs/utils/impl/Random.cpp similarity index 96% rename from src/utils/Random.cpp rename to src/libs/utils/impl/Random.cpp index c7114ddf..eb081ae0 100644 --- a/src/utils/Random.cpp +++ b/src/libs/utils/impl/Random.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "Random.hpp" +#include "utils/Random.hpp" namespace Random { diff --git a/src/utils/StreamLogger.cpp b/src/libs/utils/impl/StreamLogger.cpp similarity index 96% rename from src/utils/StreamLogger.cpp rename to src/libs/utils/impl/StreamLogger.cpp index f4fc9cf5..94434332 100644 --- a/src/utils/StreamLogger.cpp +++ b/src/libs/utils/impl/StreamLogger.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "StreamLogger.hpp" +#include "utils/StreamLogger.hpp" StreamLogger::StreamLogger(std::ostream& os) : _os {os} diff --git a/src/utils/String.cpp b/src/libs/utils/impl/String.cpp similarity index 97% rename from src/utils/String.cpp rename to src/libs/utils/impl/String.cpp index 299c177e..be6a79a1 100644 --- a/src/utils/String.cpp +++ b/src/libs/utils/impl/String.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "String.hpp" +#include "utils/String.hpp" #include @@ -103,6 +103,12 @@ stringToLower(const std::string& str) return boost::algorithm::to_lower_copy(str); } +std::string +stringToUpper(const std::string& str) +{ + return boost::to_upper_copy(str); +} + std::string bufferToString(const std::vector& data) { diff --git a/src/utils/UUID.cpp b/src/libs/utils/impl/UUID.cpp similarity index 97% rename from src/utils/UUID.cpp rename to src/libs/utils/impl/UUID.cpp index 9edbdad4..f95209e2 100644 --- a/src/utils/UUID.cpp +++ b/src/libs/utils/impl/UUID.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "UUID.hpp" +#include "utils/UUID.hpp" #include diff --git a/src/utils/WtLogger.cpp b/src/libs/utils/impl/WtLogger.cpp similarity index 94% rename from src/utils/WtLogger.cpp rename to src/libs/utils/impl/WtLogger.cpp index bb1359b0..126af923 100644 --- a/src/utils/WtLogger.cpp +++ b/src/libs/utils/impl/WtLogger.cpp @@ -17,12 +17,12 @@ * along with LMS. If not, see . */ -#include "WtLogger.hpp" +#include "utils/WtLogger.hpp" #include #include -#include "Logger.hpp" +#include "utils/Logger.hpp" void WtLogger::processLog(const Log& log) diff --git a/src/utils/Exception.hpp b/src/libs/utils/include/utils/Exception.hpp similarity index 100% rename from src/utils/Exception.hpp rename to src/libs/utils/include/utils/Exception.hpp diff --git a/src/libs/utils/include/utils/IConfig.hpp b/src/libs/utils/include/utils/IConfig.hpp new file mode 100644 index 00000000..6b62169b --- /dev/null +++ b/src/libs/utils/include/utils/IConfig.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2016 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ +#pragma once + +#include +#include +#include + +// Used to get config values from configuration files +class IConfig +{ + public: + + // Default values are returned in case of setting not found + virtual std::string getString(const std::string& setting, const std::string& def = "", const std::unordered_set& allowedValues = {}) = 0; + virtual std::filesystem::path getPath(const std::string& setting, const std::filesystem::path& def = std::filesystem::path()) = 0; + virtual unsigned long getULong(const std::string& setting, unsigned long def = 0) = 0; + virtual long getLong(const std::string& setting, long def = 0) = 0; + virtual bool getBool(const std::string& setting, bool def = false) = 0; +}; + + +std::unique_ptr createConfig(const std::filesystem::path& p); + diff --git a/src/utils/Logger.hpp b/src/libs/utils/include/utils/Logger.hpp similarity index 98% rename from src/utils/Logger.hpp rename to src/libs/utils/include/utils/Logger.hpp index 9597aa5c..9f5b2ec5 100644 --- a/src/utils/Logger.hpp +++ b/src/libs/utils/include/utils/Logger.hpp @@ -46,7 +46,7 @@ enum class Module METADATA, REMOTE, SERVICE, - SIMILARITY, + RECOMMENDATION, TRANSCODE, UI, }; diff --git a/src/utils/NetAddress.hpp b/src/libs/utils/include/utils/NetAddress.hpp similarity index 100% rename from src/utils/NetAddress.hpp rename to src/libs/utils/include/utils/NetAddress.hpp diff --git a/src/utils/Path.hpp b/src/libs/utils/include/utils/Path.hpp similarity index 76% rename from src/utils/Path.hpp rename to src/libs/utils/include/utils/Path.hpp index f7e82893..573375e5 100644 --- a/src/utils/Path.hpp +++ b/src/libs/utils/include/utils/Path.hpp @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -34,3 +35,13 @@ bool ensureDirectory(const std::filesystem::path& dir); // Get the last write time since Epoch Wt::WDateTime getLastWriteTime(const std::filesystem::path& dir); +void exploreFilesRecursive(const std::filesystem::path& directory, std::function cb); + +namespace std +{ + template <> + struct hash + { + inline std::size_t operator()(const std::filesystem::path &path) const { return hash_value(path); } + }; +} diff --git a/src/utils/Random.hpp b/src/libs/utils/include/utils/Random.hpp similarity index 100% rename from src/utils/Random.hpp rename to src/libs/utils/include/utils/Random.hpp diff --git a/src/libs/utils/include/utils/Semaphore.hpp b/src/libs/utils/include/utils/Semaphore.hpp new file mode 100644 index 00000000..7b82879d --- /dev/null +++ b/src/libs/utils/include/utils/Semaphore.hpp @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2020 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +class Semaphore +{ + public: + Semaphore() = default; + Semaphore(const Semaphore&) = delete; + Semaphore(Semaphore&&) = delete; + Semaphore& operator=(const Semaphore&) = delete; + Semaphore& operator=(Semaphore&&) = delete; + + void notify() + { + std::unique_lock lock {_mutex}; + + _count++; + _cv.notify_one(); + } + + void wait() + { + std::unique_lock lock(_mutex); + + while (_count == 0) + _cv.wait(lock); + + _count--; + } + + private: + std::mutex _mutex; + std::condition_variable _cv; + unsigned _count {}; +}; + + diff --git a/src/utils/Service.hpp b/src/libs/utils/include/utils/Service.hpp similarity index 92% rename from src/utils/Service.hpp rename to src/libs/utils/include/utils/Service.hpp index a18363ce..c53a8c81 100644 --- a/src/utils/Service.hpp +++ b/src/libs/utils/include/utils/Service.hpp @@ -46,7 +46,14 @@ class ServiceProvider return *get(); } - static void assign(std::unique_ptr service) { _service = std::move(service); } + static + Class& + assign(std::unique_ptr service) + { + _service = std::move(service); + return *get(); + } + static void clear() { _service.reset(); } static Class* get() { return _service.get(); } diff --git a/src/utils/StreamLogger.hpp b/src/libs/utils/include/utils/StreamLogger.hpp similarity index 100% rename from src/utils/StreamLogger.hpp rename to src/libs/utils/include/utils/StreamLogger.hpp diff --git a/src/utils/String.hpp b/src/libs/utils/include/utils/String.hpp similarity index 97% rename from src/utils/String.hpp rename to src/libs/utils/include/utils/String.hpp index 763249d1..e62edb91 100644 --- a/src/utils/String.hpp +++ b/src/libs/utils/include/utils/String.hpp @@ -26,6 +26,7 @@ namespace StringUtils { + std::vector splitString(const std::string& string, const std::string& separators); @@ -41,6 +42,9 @@ stringTrimEnd(const std::string& str, const std::string& whitespaces = " \t"); std::string stringToLower(const std::string& str); +std::string +stringToUpper(const std::string& str); + std::string bufferToString(const std::vector& data); diff --git a/src/utils/UUID.hpp b/src/libs/utils/include/utils/UUID.hpp similarity index 100% rename from src/utils/UUID.hpp rename to src/libs/utils/include/utils/UUID.hpp diff --git a/src/utils/Utils.hpp b/src/libs/utils/include/utils/Utils.hpp similarity index 100% rename from src/utils/Utils.hpp rename to src/libs/utils/include/utils/Utils.hpp diff --git a/src/utils/WtLogger.hpp b/src/libs/utils/include/utils/WtLogger.hpp similarity index 100% rename from src/utils/WtLogger.hpp rename to src/libs/utils/include/utils/WtLogger.hpp diff --git a/src/lms/CMakeLists.txt b/src/lms/CMakeLists.txt new file mode 100644 index 00000000..d422a93b --- /dev/null +++ b/src/lms/CMakeLists.txt @@ -0,0 +1,54 @@ + +add_executable(lms + main.cpp + ui/Auth.cpp + ui/LmsApplication.cpp + ui/LmsApplicationGroup.cpp + ui/MediaPlayer.cpp + ui/PlayQueueView.cpp + ui/PlayHistoryView.cpp + ui/SettingsView.cpp + ui/TrackStringUtils.cpp + ui/admin/DatabaseSettingsView.cpp + ui/admin/DatabaseStatus.cpp + ui/admin/InitWizardView.cpp + ui/admin/UserView.cpp + ui/admin/UsersView.cpp + ui/common/Validators.cpp + ui/explore/ArtistInfoView.cpp + ui/explore/ArtistLink.cpp + ui/explore/ArtistsInfoView.cpp + ui/explore/ArtistView.cpp + ui/explore/ArtistsView.cpp + ui/explore/Explore.cpp + ui/explore/Filters.cpp + ui/explore/ReleaseInfoView.cpp + ui/explore/ReleaseLink.cpp + ui/explore/ReleasesInfoView.cpp + ui/explore/ReleasesView.cpp + ui/explore/ReleaseView.cpp + ui/explore/TracksInfoView.cpp + ui/explore/TracksView.cpp + ui/resource/ImageResource.cpp + ui/resource/AudioResource.cpp + ) + +target_include_directories(lms PRIVATE + ui/ + ) + +target_link_libraries(lms PRIVATE + lmsav + lmsauth + lmscover + lmsdatabase + lmsrecommendation + lmsscanner + lmssubsonic + lmsutils + wt + wthttp + ) + +install(TARGETS lms DESTINATION bin) + diff --git a/src/main/main.cpp b/src/lms/main.cpp similarity index 56% rename from src/main/main.cpp rename to src/lms/main.cpp index 805de6c0..54fe076c 100644 --- a/src/main/main.cpp +++ b/src/lms/main.cpp @@ -22,19 +22,17 @@ #include #include -#include "api/subsonic/SubsonicResource.hpp" +#include "auth/IAuthTokenService.hpp" +#include "auth/IPasswordService.hpp" #include "av/AvInfo.hpp" #include "av/AvTranscoder.hpp" -#include "auth/AuthTokenService.hpp" -#include "auth/PasswordService.hpp" -#include "cover/CoverArtGrabber.hpp" +#include "cover/ICoverArtGrabber.hpp" #include "database/Db.hpp" -#include "image/Image.hpp" -#include "scanner/MediaScanner.hpp" -#include "similarity/features/SimilarityFeaturesScannerAddon.hpp" -#include "similarity/SimilaritySearcher.hpp" +#include "scanner/IMediaScanner.hpp" +#include "recommendation/IEngine.hpp" +#include "subsonic/SubsonicResource.hpp" #include "ui/LmsApplication.hpp" -#include "utils/Config.hpp" +#include "utils/IConfig.hpp" #include "utils/Service.hpp" #include "utils/WtLogger.hpp" @@ -42,28 +40,28 @@ std::vector generateWtConfig(std::string execPath) { std::vector args; - const std::filesystem::path wtConfigPath {ServiceProvider::get()->getPath("working-dir") / "wt_config.xml"}; - const std::filesystem::path wtLogFilePath {ServiceProvider::get()->getPath("log-file", "/var/log/lms.log")}; - const std::filesystem::path wtAccessLogFilePath {ServiceProvider::get()->getPath("access-log-file", "/var/log/lms.access.log")}; + const std::filesystem::path wtConfigPath {ServiceProvider::get()->getPath("working-dir") / "wt_config.xml"}; + const std::filesystem::path wtLogFilePath {ServiceProvider::get()->getPath("log-file", "/var/log/lms.log")}; + const std::filesystem::path wtAccessLogFilePath {ServiceProvider::get()->getPath("access-log-file", "/var/log/lms.access.log")}; args.push_back(execPath); args.push_back("--config=" + wtConfigPath.string()); - args.push_back("--docroot=" + ServiceProvider::get()->getString("docroot")); - args.push_back("--approot=" + ServiceProvider::get()->getString("approot")); - args.push_back("--resources-dir=" + ServiceProvider::get()->getString("wt-resources")); + args.push_back("--docroot=" + ServiceProvider::get()->getString("docroot")); + args.push_back("--approot=" + ServiceProvider::get()->getString("approot")); + args.push_back("--resources-dir=" + ServiceProvider::get()->getString("wt-resources")); - if (ServiceProvider::get()->getBool("tls-enable", false)) + if (ServiceProvider::get()->getBool("tls-enable", false)) { - args.push_back("--https-port=" + std::to_string( ServiceProvider::get()->getULong("listen-port", 5082))); - args.push_back("--https-address=" + ServiceProvider::get()->getString("listen-addr", "0.0.0.0")); - args.push_back("--ssl-certificate=" + ServiceProvider::get()->getString("tls-cert")); - args.push_back("--ssl-private-key=" + ServiceProvider::get()->getString("tls-key")); - args.push_back("--ssl-tmp-dh=" + ServiceProvider::get()->getString("tls-dh")); + args.push_back("--https-port=" + std::to_string( ServiceProvider::get()->getULong("listen-port", 5082))); + args.push_back("--https-address=" + ServiceProvider::get()->getString("listen-addr", "0.0.0.0")); + args.push_back("--ssl-certificate=" + ServiceProvider::get()->getString("tls-cert")); + args.push_back("--ssl-private-key=" + ServiceProvider::get()->getString("tls-key")); + args.push_back("--ssl-tmp-dh=" + ServiceProvider::get()->getString("tls-dh")); } else { - args.push_back("--http-port=" + std::to_string( ServiceProvider::get()->getULong("listen-port", 5082))); - args.push_back("--http-address=" + ServiceProvider::get()->getString("listen-addr", "0.0.0.0")); + args.push_back("--http-port=" + std::to_string( ServiceProvider::get()->getULong("listen-port", 5082))); + args.push_back("--http-address=" + ServiceProvider::get()->getString("listen-addr", "0.0.0.0")); } if (!wtAccessLogFilePath.empty()) @@ -74,8 +72,8 @@ std::vector generateWtConfig(std::string execPath) pt.put("server.application-settings..location", "*"); pt.put("server.application-settings.log-file", wtLogFilePath.string()); - pt.put("server.application-settings.log-config", ServiceProvider::get()->getString("log-config", "* -debug -info:WebRequest")); - pt.put("server.application-settings.behind-reverse-proxy", ServiceProvider::get()->getBool("behind-reverse-proxy", false)); + pt.put("server.application-settings.log-config", ServiceProvider::get()->getString("log-config", "* -debug -info:WebRequest")); + pt.put("server.application-settings.behind-reverse-proxy", ServiceProvider::get()->getBool("behind-reverse-proxy", false)); pt.put("server.application-settings.progressive-bootstrap", true); std::ofstream oss(wtConfigPath.string().c_str(), std::ios::out); @@ -108,17 +106,17 @@ int main(int argc, char* argv[]) // Make pstream work with ffmpeg close(STDIN_FILENO); - ServiceProvider::create(configFilePath); + ServiceProvider::assign(createConfig(configFilePath)); ServiceProvider::create(); // Make sure the working directory exists - std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir")); - std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir") / "cache"); + std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir")); + std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir") / "cache"); // Construct WT configuration and get the argc/argv back std::vector wtServerArgs = generateWtConfig(argv[0]); - const char* wtArgv[wtServerArgs.size()]; + std::vector wtArgv(wtServerArgs.size()); for (std::size_t i = 0; i < wtServerArgs.size(); ++i) { std::cout << "ARG = " << wtServerArgs[i] << std::endl; @@ -126,14 +124,13 @@ int main(int argc, char* argv[]) } Wt::WServer server(argv[0]); - server.setServerConfiguration (wtServerArgs.size(), const_cast(wtArgv)); + server.setServerConfiguration (wtServerArgs.size(), const_cast(&wtArgv[0])); // lib init - Image::init(argv[0]); Av::Transcoder::init(); // Initializing a connection pool to the database that will be shared along services - Database::Db database {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; + Database::Db database {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; { Database::Session session {database}; session.prepareTables(); @@ -142,23 +139,33 @@ int main(int argc, char* argv[]) UserInterface::LmsApplicationGroupContainer appGroups; // Service initialization order is important - ServiceProvider::create(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000)); - ServiceProvider::create(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000)); - Scanner::MediaScanner& mediaScanner {ServiceProvider::create(database)}; + ServiceProvider::assign(Auth::createAuthTokenService(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000))); + ServiceProvider::assign(Auth::createPasswordService(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000))); + Scanner::IMediaScanner& mediaScanner {ServiceProvider::assign(Scanner::createMediaScanner(database))}; - Similarity::FeaturesScannerAddon similarityFeaturesScannerAddon {database}; + Recommendation::IEngine& recommendationEngine {ServiceProvider::assign(Recommendation::createEngine(database))}; + mediaScanner.scanComplete().connect([&]() + { + auto status = mediaScanner.getStatus(); - mediaScanner.setAddon(similarityFeaturesScannerAddon); + if (status.lastCompleteScanStats->nbChanges() > 0 || status.lastCompleteScanStats->featuresFetched > 0) + { + LMS_LOG(MAIN, INFO) << "Scanner changed some files, reloading the recommendation engine..."; + recommendationEngine.requestReload(); + } + else + { + LMS_LOG(MAIN, INFO) << "Scanner did not change files, not reloading the recommendation engine..."; + } + }); - CoverArt::Grabber& coverArtGrabber {ServiceProvider::create()}; + CoverArt::IGrabber& coverArtGrabber {ServiceProvider::assign(CoverArt::createGrabber(argv[0]))}; coverArtGrabber.setDefaultCover(server.appRoot() + "/images/unknown-cover.jpg"); - ServiceProvider::create(similarityFeaturesScannerAddon); - API::Subsonic::SubsonicResource subsonicResource {database}; // bind API resources - if (ServiceProvider::get()->getBool("api-subsonic", true)) + if (ServiceProvider::get()->getBool("api-subsonic", true)) server.addResource(&subsonicResource, subsonicResource.getPath()); // bind UI entry point @@ -167,6 +174,9 @@ int main(int argc, char* argv[]) std::placeholders::_1, std::ref(database), std::ref(appGroups))); // Start + LMS_LOG(MAIN, INFO) << "Starting recommendation engine"; + recommendationEngine.start(); + LMS_LOG(MAIN, INFO) << "Starting media scanner..."; mediaScanner.start(); @@ -184,17 +194,12 @@ int main(int argc, char* argv[]) LMS_LOG(MAIN, INFO) << "Stopping media scanner..."; mediaScanner.stop(); + LMS_LOG(MAIN, INFO) << "Stopping recommendation engine..."; + recommendationEngine.stop(); + LMS_LOG(MAIN, INFO) << "Clean stop!"; res = EXIT_SUCCESS; } - catch( libconfig::FileIOException& e) - { - std::cerr << "Cannot open config file '" << configFilePath << "'" << std::endl; - } - catch( libconfig::ParseException& e) - { - std::cerr << "Caught libconfig::ParseException! error='" << e.getError() << "', file = '" << e.getFile() << "', line = " << e.getLine() << std::endl; - } catch(Wt::WServer::Exception& e) { std::cerr << "Caught a WServer::Exception: " << e.what() << std::endl; diff --git a/src/ui/Auth.cpp b/src/lms/ui/Auth.cpp similarity index 85% rename from src/ui/Auth.cpp rename to src/lms/ui/Auth.cpp index f1f37124..d450b0c7 100644 --- a/src/ui/Auth.cpp +++ b/src/lms/ui/Auth.cpp @@ -27,8 +27,8 @@ #include #include -#include "auth/AuthTokenService.hpp" -#include "auth/PasswordService.hpp" +#include "auth/IAuthTokenService.hpp" +#include "auth/IPasswordService.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -43,7 +43,7 @@ static void createAuthToken(Database::IdType userId, const Wt::WDateTime& expiry) { - const std::string secret {ServiceProvider<::Auth::AuthTokenService>::get()->createAuthToken(LmsApp->getDbSession(), userId, expiry)}; + const std::string secret {ServiceProvider<::Auth::IAuthTokenService>::get()->createAuthToken(LmsApp->getDbSession(), userId, expiry)}; LmsApp->setCookie(authCookieName, secret, @@ -61,15 +61,15 @@ processAuthToken(const Wt::WEnvironment& env) if (!authCookie) return std::nullopt; - const auto res {ServiceProvider<::Auth::AuthTokenService>::get()->processAuthToken(LmsApp->getDbSession(), boost::asio::ip::address::from_string(env.clientAddress()), *authCookie)}; + const auto res {ServiceProvider<::Auth::IAuthTokenService>::get()->processAuthToken(LmsApp->getDbSession(), boost::asio::ip::address::from_string(env.clientAddress()), *authCookie)}; switch (res.state) { - case ::Auth::AuthTokenService::AuthTokenProcessResult::State::NotFound: - case ::Auth::AuthTokenService::AuthTokenProcessResult::State::Throttled: + case ::Auth::IAuthTokenService::AuthTokenProcessResult::State::NotFound: + case ::Auth::IAuthTokenService::AuthTokenProcessResult::State::Throttled: LmsApp->setCookie(authCookieName, std::string {}, 0, "", "", env.urlScheme() == "https"); return std::nullopt; - case ::Auth::AuthTokenService::AuthTokenProcessResult::State::Found: + case ::Auth::IAuthTokenService::AuthTokenProcessResult::State::Found: createAuthToken(res.authTokenInfo->userId, res.authTokenInfo->expiry); break; } @@ -124,18 +124,18 @@ class AuthModel : public Wt::WFormModel if (field == PasswordField) { - switch (ServiceProvider<::Auth::PasswordService>::get()->checkUserPassword( + switch (ServiceProvider<::Auth::IPasswordService>::get()->checkUserPassword( LmsApp->getDbSession(), boost::asio::ip::address::from_string(LmsApp->environment().clientAddress()), valueText(LoginNameField).toUTF8(), valueText(PasswordField).toUTF8())) { - case ::Auth::PasswordService::PasswordCheckResult::Match: + case ::Auth::IPasswordService::PasswordCheckResult::Match: break; - case ::Auth::PasswordService::PasswordCheckResult::Mismatch: + case ::Auth::IPasswordService::PasswordCheckResult::Mismatch: error = Wt::WString::tr("Lms.password-bad-login-combination"); break; - case ::Auth::PasswordService::PasswordCheckResult::Throttled: + case ::Auth::IPasswordService::PasswordCheckResult::Throttled: error = Wt::WString::tr("Lms.password-client-throttled"); break; } diff --git a/src/ui/Auth.hpp b/src/lms/ui/Auth.hpp similarity index 100% rename from src/ui/Auth.hpp rename to src/lms/ui/Auth.hpp diff --git a/src/ui/LmsApplication.cpp b/src/lms/ui/LmsApplication.cpp similarity index 97% rename from src/ui/LmsApplication.cpp rename to src/lms/ui/LmsApplication.cpp index ebc16ae0..8a6dbcc2 100644 --- a/src/ui/LmsApplication.cpp +++ b/src/lms/ui/LmsApplication.cpp @@ -29,8 +29,7 @@ #include #include -#include "config/config.h" -#include "cover/CoverArtGrabber.hpp" +#include "cover/ICoverArtGrabber.hpp" #include "database/Artist.hpp" #include "database/Cluster.hpp" #include "database/Db.hpp" @@ -529,7 +528,7 @@ LmsApplication::createHome() // Events from MediaScanner { const std::string sessionId {LmsApp->sessionId()}; - ServiceProvider::get()->scanComplete().connect(this, [=] () + ServiceProvider::get()->scanComplete().connect(this, [=] () { Wt::WServer::instance()->post(sessionId, [=] { @@ -538,7 +537,7 @@ LmsApplication::createHome() }); }); - ServiceProvider::get()->scanInProgress().connect(this, [=] (Scanner::ScanProgressStats stats) + ServiceProvider::get()->scanInProgress().connect(this, [=] (Scanner::ScanProgressStats stats) { Wt::WServer::instance()->post(sessionId, [=] { @@ -547,7 +546,7 @@ LmsApplication::createHome() }); }); - ServiceProvider::get()->scheduled().connect(this, [=] (Wt::WDateTime dateTime) + ServiceProvider::get()->scheduled().connect(this, [=] (Wt::WDateTime dateTime) { Wt::WServer::instance()->post(sessionId, [=] { @@ -562,7 +561,7 @@ LmsApplication::createHome() { if (isUserAdmin()) { - const auto& stats {*ServiceProvider::get()->getStatus().lastCompleteScanStats}; + const auto& stats {*ServiceProvider::get()->getStatus().lastCompleteScanStats}; notifyMsg(MsgType::Info, Wt::WString::tr("Lms.Admin.Database.scan-complete") .arg(static_cast(stats.nbFiles())) diff --git a/src/ui/LmsApplication.hpp b/src/lms/ui/LmsApplication.hpp similarity index 99% rename from src/ui/LmsApplication.hpp rename to src/lms/ui/LmsApplication.hpp index 9e6eb57d..cd0f04ca 100644 --- a/src/ui/LmsApplication.hpp +++ b/src/lms/ui/LmsApplication.hpp @@ -25,7 +25,7 @@ #include "database/Db.hpp" #include "database/Session.hpp" -#include "scanner/MediaScanner.hpp" +#include "scanner/IMediaScanner.hpp" #include "LmsApplicationGroup.hpp" diff --git a/src/ui/LmsApplicationException.hpp b/src/lms/ui/LmsApplicationException.hpp similarity index 100% rename from src/ui/LmsApplicationException.hpp rename to src/lms/ui/LmsApplicationException.hpp diff --git a/src/ui/LmsApplicationGroup.cpp b/src/lms/ui/LmsApplicationGroup.cpp similarity index 97% rename from src/ui/LmsApplicationGroup.cpp rename to src/lms/ui/LmsApplicationGroup.cpp index 70fcbe64..d9c41199 100644 --- a/src/ui/LmsApplicationGroup.cpp +++ b/src/lms/ui/LmsApplicationGroup.cpp @@ -30,7 +30,7 @@ namespace UserInterface { LmsApplicationInfo LmsApplicationInfo::fromEnvironment(const Wt::WEnvironment& env) { - LmsApplicationInfo info = {.userAgent = env.agent()}; + LmsApplicationInfo info = {env.agent()}; return info; } diff --git a/src/ui/LmsApplicationGroup.hpp b/src/lms/ui/LmsApplicationGroup.hpp similarity index 100% rename from src/ui/LmsApplicationGroup.hpp rename to src/lms/ui/LmsApplicationGroup.hpp diff --git a/src/ui/MediaPlayer.cpp b/src/lms/ui/MediaPlayer.cpp similarity index 100% rename from src/ui/MediaPlayer.cpp rename to src/lms/ui/MediaPlayer.cpp diff --git a/src/ui/MediaPlayer.hpp b/src/lms/ui/MediaPlayer.hpp similarity index 100% rename from src/ui/MediaPlayer.hpp rename to src/lms/ui/MediaPlayer.hpp diff --git a/src/ui/PlayHistoryView.cpp b/src/lms/ui/PlayHistoryView.cpp similarity index 100% rename from src/ui/PlayHistoryView.cpp rename to src/lms/ui/PlayHistoryView.cpp diff --git a/src/ui/PlayHistoryView.hpp b/src/lms/ui/PlayHistoryView.hpp similarity index 100% rename from src/ui/PlayHistoryView.hpp rename to src/lms/ui/PlayHistoryView.hpp diff --git a/src/ui/PlayQueueView.cpp b/src/lms/ui/PlayQueueView.cpp similarity index 98% rename from src/ui/PlayQueueView.cpp rename to src/lms/ui/PlayQueueView.cpp index 91dc4c1b..e55476b6 100644 --- a/src/ui/PlayQueueView.cpp +++ b/src/lms/ui/PlayQueueView.cpp @@ -26,7 +26,7 @@ #include "database/Track.hpp" #include "database/TrackList.hpp" #include "database/User.hpp" -#include "similarity/SimilaritySearcher.hpp" +#include "recommendation/IEngine.hpp" #include "utils/Logger.hpp" #include "utils/Random.hpp" #include "utils/Service.hpp" @@ -417,7 +417,7 @@ PlayQueue::addSome() void PlayQueue::enqueueRadioTrack() { - const std::vector trackToAddIds {ServiceProvider::get()->getSimilarTracksFromTrackList(LmsApp->getDbSession(), _tracklistId, 1)}; + const std::vector trackToAddIds {ServiceProvider::get()->getSimilarTracksFromTrackList(LmsApp->getDbSession(), _tracklistId, 1)}; enqueueTracks(trackToAddIds); } diff --git a/src/ui/PlayQueueView.hpp b/src/lms/ui/PlayQueueView.hpp similarity index 100% rename from src/ui/PlayQueueView.hpp rename to src/lms/ui/PlayQueueView.hpp diff --git a/src/ui/SettingsView.cpp b/src/lms/ui/SettingsView.cpp similarity index 94% rename from src/ui/SettingsView.cpp rename to src/lms/ui/SettingsView.cpp index c1b430b6..d5aa9233 100644 --- a/src/ui/SettingsView.cpp +++ b/src/lms/ui/SettingsView.cpp @@ -30,7 +30,7 @@ #include "common/Validators.hpp" #include "common/ValueStringModel.hpp" -#include "auth/PasswordService.hpp" +#include "auth/IPasswordService.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -80,7 +80,7 @@ class SettingsModel : public Wt::WFormModel Database::User::PasswordHash passwordHash; if (!valueText(PasswordField).empty()) - passwordHash = ServiceProvider<::Auth::PasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8()); + passwordHash = ServiceProvider<::Auth::IPasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8()); auto transaction {LmsApp->getDbSession().createUniqueTransaction()}; @@ -133,18 +133,18 @@ class SettingsModel : public Wt::WFormModel { if (!valueText(PasswordOldField).empty()) { - switch (ServiceProvider<::Auth::PasswordService>::get()->checkUserPassword( + switch (ServiceProvider<::Auth::IPasswordService>::get()->checkUserPassword( LmsApp->getDbSession(), boost::asio::ip::address::from_string(LmsApp->environment().clientAddress()), LmsApp->getUserLoginName(), valueText(PasswordOldField).toUTF8())) { - case ::Auth::PasswordService::PasswordCheckResult::Match: + case ::Auth::IPasswordService::PasswordCheckResult::Match: break; - case ::Auth::PasswordService::PasswordCheckResult::Mismatch: + case ::Auth::IPasswordService::PasswordCheckResult::Mismatch: error = Wt::WString::tr("Lms.Settings.password-bad"); break; - case ::Auth::PasswordService::PasswordCheckResult::Throttled: + case ::Auth::IPasswordService::PasswordCheckResult::Throttled: error = Wt::WString::tr("Lms.password-client-throttled"); break; } @@ -161,7 +161,7 @@ class SettingsModel : public Wt::WFormModel { if (!valueText(PasswordField).empty()) { - if (!ServiceProvider<::Auth::PasswordService>::get()->evaluatePasswordStrength(LmsApp->getUserLoginName(), valueText(PasswordField).toUTF8())) + if (!ServiceProvider<::Auth::IPasswordService>::get()->evaluatePasswordStrength(LmsApp->getUserLoginName(), valueText(PasswordField).toUTF8())) error = Wt::WString::tr("Lms.password-too-weak"); } else diff --git a/src/ui/SettingsView.hpp b/src/lms/ui/SettingsView.hpp similarity index 100% rename from src/ui/SettingsView.hpp rename to src/lms/ui/SettingsView.hpp diff --git a/src/ui/TrackStringUtils.cpp b/src/lms/ui/TrackStringUtils.cpp similarity index 100% rename from src/ui/TrackStringUtils.cpp rename to src/lms/ui/TrackStringUtils.cpp diff --git a/src/ui/TrackStringUtils.hpp b/src/lms/ui/TrackStringUtils.hpp similarity index 100% rename from src/ui/TrackStringUtils.hpp rename to src/lms/ui/TrackStringUtils.hpp diff --git a/src/ui/admin/DatabaseSettingsView.cpp b/src/lms/ui/admin/DatabaseSettingsView.cpp similarity index 77% rename from src/ui/admin/DatabaseSettingsView.cpp rename to src/lms/ui/admin/DatabaseSettingsView.cpp index d4ac7200..323dfa7c 100644 --- a/src/ui/admin/DatabaseSettingsView.cpp +++ b/src/lms/ui/admin/DatabaseSettingsView.cpp @@ -27,6 +27,7 @@ #include #include "database/Cluster.hpp" +#include "database/ScanSettings.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" #include "utils/String.hpp" @@ -48,7 +49,7 @@ class DatabaseSettingsModel : public Wt::WFormModel static const Field MediaDirectoryField; static const Field UpdatePeriodField; static const Field UpdateStartTimeField; - static const Field SimilarityEngineTypeField; + static const Field RecommendationEngineTypeField; static const Field TagsField; DatabaseSettingsModel() @@ -59,7 +60,7 @@ class DatabaseSettingsModel : public Wt::WFormModel addField(MediaDirectoryField); addField(UpdatePeriodField); addField(UpdateStartTimeField); - addField(SimilarityEngineTypeField); + addField(RecommendationEngineTypeField); addField(TagsField); auto dirValidator {std::make_shared()}; @@ -68,7 +69,7 @@ class DatabaseSettingsModel : public Wt::WFormModel setValidator(UpdatePeriodField, createMandatoryValidator()); setValidator(UpdateStartTimeField, createMandatoryValidator()); - setValidator(SimilarityEngineTypeField, createMandatoryValidator()); + setValidator(RecommendationEngineTypeField, createMandatoryValidator()); setValidator(TagsField, createTagsValidator()); // populate the model with initial data @@ -77,7 +78,7 @@ class DatabaseSettingsModel : public Wt::WFormModel std::shared_ptr updatePeriodModel() { return _updatePeriodModel; } std::shared_ptr updateStartTimeModel() { return _updateStartTimeModel; } - std::shared_ptr similarityEngineTypeModel() { return _similarityEngineTypeModel; } + std::shared_ptr recommendationEngineTypeModel() { return _recommendationEngineTypeModel; } void loadData() { @@ -95,9 +96,9 @@ class DatabaseSettingsModel : public Wt::WFormModel if (startTimeRow) setValue(UpdateStartTimeField, _updateStartTimeModel->getString(*startTimeRow)); - auto similarityEngineTypeRow {_similarityEngineTypeModel->getRowFromValue(scanSettings->getSimilarityEngineType())}; - if (similarityEngineTypeRow) - setValue(SimilarityEngineTypeField, _similarityEngineTypeModel->getString(*similarityEngineTypeRow)); + auto recommendationEngineTypeRow {_recommendationEngineTypeModel->getRowFromValue(scanSettings->getRecommendationEngineType())}; + if (recommendationEngineTypeRow) + setValue(RecommendationEngineTypeField, _recommendationEngineTypeModel->getString(*recommendationEngineTypeRow)); auto clusterTypes {scanSettings->getClusterTypes()}; if (!clusterTypes.empty()) @@ -124,9 +125,9 @@ class DatabaseSettingsModel : public Wt::WFormModel if (startTimeRow) scanSettings.modify()->setUpdateStartTime(_updateStartTimeModel->getValue(*startTimeRow)); - auto similarityEngineTypeRow {_similarityEngineTypeModel->getRowFromString(valueText(SimilarityEngineTypeField))}; - if (similarityEngineTypeRow) - scanSettings.modify()->setSimilarityEngineType(_similarityEngineTypeModel->getValue(*similarityEngineTypeRow)); + auto recommendationEngineTypeRow {_recommendationEngineTypeModel->getRowFromString(valueText(RecommendationEngineTypeField))}; + if (recommendationEngineTypeRow) + scanSettings.modify()->setRecommendationEngineType(_recommendationEngineTypeModel->getValue(*recommendationEngineTypeRow)); auto clusterTypes {StringUtils::splitString(valueText(TagsField).toUTF8(), " ")}; scanSettings.modify()->setClusterTypes(LmsApp->getDbSession(), std::set(clusterTypes.begin(), clusterTypes.end())); @@ -155,22 +156,22 @@ class DatabaseSettingsModel : public Wt::WFormModel _updateStartTimeModel->add(time.toString(), time); } - _similarityEngineTypeModel = std::make_shared>(); - _similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.clusters"), ScanSettings::SimilarityEngineType::Clusters); - _similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.features"), ScanSettings::SimilarityEngineType::Features); + _recommendationEngineTypeModel = std::make_shared>(); + _recommendationEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.recommendation-engine-type.clusters"), ScanSettings::RecommendationEngineType::Clusters); + _recommendationEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.recommendation-engine-type.features"), ScanSettings::RecommendationEngineType::Features); } std::shared_ptr> _updatePeriodModel; std::shared_ptr> _updateStartTimeModel; - std::shared_ptr> _similarityEngineTypeModel; + std::shared_ptr> _recommendationEngineTypeModel; }; -const Wt::WFormModel::Field DatabaseSettingsModel::MediaDirectoryField = "media-directory"; -const Wt::WFormModel::Field DatabaseSettingsModel::UpdatePeriodField = "update-period"; -const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField = "update-start-time"; -const Wt::WFormModel::Field DatabaseSettingsModel::SimilarityEngineTypeField = "similarity-engine-type"; -const Wt::WFormModel::Field DatabaseSettingsModel::TagsField = "tags"; +const Wt::WFormModel::Field DatabaseSettingsModel::MediaDirectoryField = "media-directory"; +const Wt::WFormModel::Field DatabaseSettingsModel::UpdatePeriodField = "update-period"; +const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField = "update-start-time"; +const Wt::WFormModel::Field DatabaseSettingsModel::RecommendationEngineTypeField = "recommendation-engine-type"; +const Wt::WFormModel::Field DatabaseSettingsModel::TagsField = "tags"; DatabaseSettingsView::DatabaseSettingsView() { @@ -206,10 +207,10 @@ DatabaseSettingsView::refreshView() updateStartTime->setModel(model->updateStartTimeModel()); t->setFormWidget(DatabaseSettingsModel::UpdateStartTimeField, std::move(updateStartTime)); - // Similarity engine type - auto similarityEngineType {std::make_unique()}; - similarityEngineType->setModel(model->similarityEngineTypeModel()); - t->setFormWidget(DatabaseSettingsModel::SimilarityEngineTypeField, std::move(similarityEngineType)); + // recommendation engine type + auto recommendationEngineType {std::make_unique()}; + recommendationEngineType->setModel(model->recommendationEngineTypeModel()); + t->setFormWidget(DatabaseSettingsModel::RecommendationEngineTypeField, std::move(recommendationEngineType)); // Tags t->setFormWidget(DatabaseSettingsModel::TagsField, std::make_unique()); @@ -229,7 +230,7 @@ DatabaseSettingsView::refreshView() { model->saveData(); - ServiceProvider::get()->requestReschedule(); + ServiceProvider::get()->requestReschedule(); LmsApp->notifyMsg(MsgType::Success, Wt::WString::tr("Lms.Admin.Database.settings-saved")); } @@ -246,7 +247,7 @@ DatabaseSettingsView::refreshView() immScanBtn->clicked().connect([=] () { - ServiceProvider::get()->requestImmediateScan(); + ServiceProvider::get()->requestImmediateScan(); LmsApp->notifyMsg(MsgType::Info, Wt::WString::tr("Lms.Admin.Database.scan-launched")); }); diff --git a/src/ui/admin/DatabaseSettingsView.hpp b/src/lms/ui/admin/DatabaseSettingsView.hpp similarity index 100% rename from src/ui/admin/DatabaseSettingsView.hpp rename to src/lms/ui/admin/DatabaseSettingsView.hpp diff --git a/src/ui/admin/DatabaseStatus.cpp b/src/lms/ui/admin/DatabaseStatus.cpp similarity index 95% rename from src/ui/admin/DatabaseStatus.cpp rename to src/lms/ui/admin/DatabaseStatus.cpp index 23577bfa..34133e01 100644 --- a/src/ui/admin/DatabaseStatus.cpp +++ b/src/lms/ui/admin/DatabaseStatus.cpp @@ -27,7 +27,6 @@ #include #include "utils/Service.hpp" -#include "scanner/MediaScanner.hpp" #include "LmsApplication.hpp" namespace UserInterface { @@ -136,7 +135,7 @@ DatabaseStatus::refreshContents() Wt::WPushButton* reportBtn {bindNew("btn-report", Wt::WString::tr("Lms.Admin.Database.Status.get-report"))}; - const MediaScanner::Status status {ServiceProvider::get()->getStatus()}; + const IMediaScanner::Status status {ServiceProvider::get()->getStatus()}; if (status.lastCompleteScanStats) { bindString("last-scan", Wt::WString::tr("Lms.Admin.Database.Status.last-scan-status") @@ -160,14 +159,14 @@ DatabaseStatus::refreshContents() switch (status.currentState) { - case MediaScanner::State::NotScheduled: + case IMediaScanner::State::NotScheduled: bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-not-scheduled")); break; - case MediaScanner::State::Scheduled: + case IMediaScanner::State::Scheduled: bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-scheduled") .arg(status.nextScheduledScan.toString())); break; - case MediaScanner::State::InProgress: + case IMediaScanner::State::InProgress: { std::ostringstream oss; bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-in-progress") diff --git a/src/ui/admin/DatabaseStatus.hpp b/src/lms/ui/admin/DatabaseStatus.hpp similarity index 100% rename from src/ui/admin/DatabaseStatus.hpp rename to src/lms/ui/admin/DatabaseStatus.hpp diff --git a/src/ui/admin/InitWizardView.cpp b/src/lms/ui/admin/InitWizardView.cpp similarity index 93% rename from src/ui/admin/InitWizardView.cpp rename to src/lms/ui/admin/InitWizardView.cpp index 2eeba3e5..34d7c5fb 100644 --- a/src/ui/admin/InitWizardView.cpp +++ b/src/lms/ui/admin/InitWizardView.cpp @@ -23,7 +23,7 @@ #include #include -#include "auth/PasswordService.hpp" +#include "auth/IPasswordService.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -55,7 +55,7 @@ class InitWizardModel : public Wt::WFormModel void saveData() { - const Database::User::PasswordHash passwordHash {ServiceProvider<::Auth::PasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8())}; + const Database::User::PasswordHash passwordHash {ServiceProvider<::Auth::IPasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8())}; auto transaction(LmsApp->getDbSession().createUniqueTransaction()); @@ -77,7 +77,7 @@ class InitWizardModel : public Wt::WFormModel if (!valueText(PasswordField).empty()) { // Evaluate the strength of the password - if (!ServiceProvider<::Auth::PasswordService>::get()->evaluatePasswordStrength(valueText(AdminLoginField).toUTF8(), valueText(PasswordField).toUTF8())) + if (!ServiceProvider<::Auth::IPasswordService>::get()->evaluatePasswordStrength(valueText(AdminLoginField).toUTF8(), valueText(PasswordField).toUTF8())) error = Wt::WString::tr("Lms.password-too-weak"); } else diff --git a/src/ui/admin/InitWizardView.hpp b/src/lms/ui/admin/InitWizardView.hpp similarity index 100% rename from src/ui/admin/InitWizardView.hpp rename to src/lms/ui/admin/InitWizardView.hpp diff --git a/src/ui/admin/UserView.cpp b/src/lms/ui/admin/UserView.cpp similarity index 95% rename from src/ui/admin/UserView.cpp rename to src/lms/ui/admin/UserView.cpp index dc1dd83c..0b4d7556 100644 --- a/src/ui/admin/UserView.cpp +++ b/src/lms/ui/admin/UserView.cpp @@ -28,9 +28,9 @@ #include -#include "auth/PasswordService.hpp" +#include "auth/IPasswordService.hpp" #include "database/User.hpp" -#include "utils/Config.hpp" +#include "utils/IConfig.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -82,7 +82,7 @@ class UserModel : public Wt::WFormModel { std::optional passwordHash; if (!valueText(PasswordField).empty()) - passwordHash = ServiceProvider<::Auth::PasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8()); + passwordHash = ServiceProvider<::Auth::IPasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8()); auto transaction {LmsApp->getDbSession().createUniqueTransaction()}; @@ -174,7 +174,7 @@ class UserModel : public Wt::WFormModel else { // Evaluate the strength of the password for non demo accounts - if (!ServiceProvider<::Auth::PasswordService>::get()->evaluatePasswordStrength(getLoginName(), valueText(PasswordField).toUTF8())) + if (!ServiceProvider<::Auth::IPasswordService>::get()->evaluatePasswordStrength(getLoginName(), valueText(PasswordField).toUTF8())) error = Wt::WString::tr("Lms.password-too-weak"); } } @@ -270,7 +270,7 @@ UserView::refreshView() // Demo account t->setFormWidget(UserModel::DemoField, std::make_unique()); - if (!userId && ServiceProvider::get()->getBool("demo", false)) + if (!userId && ServiceProvider::get()->getBool("demo", false)) t->setCondition("if-demo", true); Wt::WPushButton* saveBtn = t->bindNew("save-btn", Wt::WString::tr(userId ? "Lms.save" : "Lms.create")); diff --git a/src/ui/admin/UserView.hpp b/src/lms/ui/admin/UserView.hpp similarity index 100% rename from src/ui/admin/UserView.hpp rename to src/lms/ui/admin/UserView.hpp diff --git a/src/ui/admin/UsersView.cpp b/src/lms/ui/admin/UsersView.cpp similarity index 100% rename from src/ui/admin/UsersView.cpp rename to src/lms/ui/admin/UsersView.cpp diff --git a/src/ui/admin/UsersView.hpp b/src/lms/ui/admin/UsersView.hpp similarity index 100% rename from src/ui/admin/UsersView.hpp rename to src/lms/ui/admin/UsersView.hpp diff --git a/src/ui/common/Validators.cpp b/src/lms/ui/common/Validators.cpp similarity index 100% rename from src/ui/common/Validators.cpp rename to src/lms/ui/common/Validators.cpp diff --git a/src/ui/common/Validators.hpp b/src/lms/ui/common/Validators.hpp similarity index 100% rename from src/ui/common/Validators.hpp rename to src/lms/ui/common/Validators.hpp diff --git a/src/ui/common/ValueStringModel.hpp b/src/lms/ui/common/ValueStringModel.hpp similarity index 100% rename from src/ui/common/ValueStringModel.hpp rename to src/lms/ui/common/ValueStringModel.hpp diff --git a/src/ui/explore/ArtistInfoView.cpp b/src/lms/ui/explore/ArtistInfoView.cpp similarity index 93% rename from src/ui/explore/ArtistInfoView.cpp rename to src/lms/ui/explore/ArtistInfoView.cpp index 8f806eba..e9d3a3d9 100644 --- a/src/ui/explore/ArtistInfoView.cpp +++ b/src/lms/ui/explore/ArtistInfoView.cpp @@ -20,7 +20,7 @@ #include "ArtistInfoView.hpp" #include "database/Artist.hpp" -#include "similarity/SimilaritySearcher.hpp" +#include "recommendation/IEngine.hpp" #include "utils/Service.hpp" #include "utils/String.hpp" @@ -63,7 +63,7 @@ ArtistInfo::refresh() if (!artistId) return; - const std::vector artistsIds {ServiceProvider::get()->getSimilarArtists(LmsApp->getDbSession(), *artistId, 5)}; + const std::vector artistsIds {ServiceProvider::get()->getSimilarArtists(LmsApp->getDbSession(), *artistId, 5)}; auto transaction {LmsApp->getDbSession().createSharedTransaction()}; diff --git a/src/ui/explore/ArtistInfoView.hpp b/src/lms/ui/explore/ArtistInfoView.hpp similarity index 100% rename from src/ui/explore/ArtistInfoView.hpp rename to src/lms/ui/explore/ArtistInfoView.hpp diff --git a/src/ui/explore/ArtistLink.cpp b/src/lms/ui/explore/ArtistLink.cpp similarity index 100% rename from src/ui/explore/ArtistLink.cpp rename to src/lms/ui/explore/ArtistLink.cpp diff --git a/src/ui/explore/ArtistLink.hpp b/src/lms/ui/explore/ArtistLink.hpp similarity index 100% rename from src/ui/explore/ArtistLink.hpp rename to src/lms/ui/explore/ArtistLink.hpp diff --git a/src/ui/explore/ArtistView.cpp b/src/lms/ui/explore/ArtistView.cpp similarity index 96% rename from src/ui/explore/ArtistView.cpp rename to src/lms/ui/explore/ArtistView.cpp index 7b40bd85..9bcaa9de 100644 --- a/src/ui/explore/ArtistView.cpp +++ b/src/lms/ui/explore/ArtistView.cpp @@ -26,6 +26,7 @@ #include "database/Artist.hpp" #include "database/Release.hpp" +#include "database/ScanSettings.hpp" #include "utils/Logger.hpp" #include "utils/String.hpp" @@ -139,11 +140,6 @@ Artist::refresh() entry->bindWidget("name", LmsApplication::createReleaseAnchor(release)); auto artists {release->getReleaseArtists()}; - LMS_LOG(UI, DEBUG) << "Found " << artists.size() << " release artists"; - - for (auto artist : artists) - LMS_LOG(UI, DEBUG) << "\tArtist = '" << artist->getName() << "'"; - if (artists.empty()) artists = release->getArtists(); diff --git a/src/ui/explore/ArtistView.hpp b/src/lms/ui/explore/ArtistView.hpp similarity index 100% rename from src/ui/explore/ArtistView.hpp rename to src/lms/ui/explore/ArtistView.hpp diff --git a/src/ui/explore/ArtistsInfoView.cpp b/src/lms/ui/explore/ArtistsInfoView.cpp similarity index 100% rename from src/ui/explore/ArtistsInfoView.cpp rename to src/lms/ui/explore/ArtistsInfoView.cpp diff --git a/src/ui/explore/ArtistsInfoView.hpp b/src/lms/ui/explore/ArtistsInfoView.hpp similarity index 100% rename from src/ui/explore/ArtistsInfoView.hpp rename to src/lms/ui/explore/ArtistsInfoView.hpp diff --git a/src/ui/explore/ArtistsView.cpp b/src/lms/ui/explore/ArtistsView.cpp similarity index 100% rename from src/ui/explore/ArtistsView.cpp rename to src/lms/ui/explore/ArtistsView.cpp diff --git a/src/ui/explore/ArtistsView.hpp b/src/lms/ui/explore/ArtistsView.hpp similarity index 100% rename from src/ui/explore/ArtistsView.hpp rename to src/lms/ui/explore/ArtistsView.hpp diff --git a/src/ui/explore/Explore.cpp b/src/lms/ui/explore/Explore.cpp similarity index 100% rename from src/ui/explore/Explore.cpp rename to src/lms/ui/explore/Explore.cpp diff --git a/src/ui/explore/Explore.hpp b/src/lms/ui/explore/Explore.hpp similarity index 100% rename from src/ui/explore/Explore.hpp rename to src/lms/ui/explore/Explore.hpp diff --git a/src/ui/explore/Filters.cpp b/src/lms/ui/explore/Filters.cpp similarity index 100% rename from src/ui/explore/Filters.cpp rename to src/lms/ui/explore/Filters.cpp diff --git a/src/ui/explore/Filters.hpp b/src/lms/ui/explore/Filters.hpp similarity index 100% rename from src/ui/explore/Filters.hpp rename to src/lms/ui/explore/Filters.hpp diff --git a/src/ui/explore/ReleaseInfoView.cpp b/src/lms/ui/explore/ReleaseInfoView.cpp similarity index 95% rename from src/ui/explore/ReleaseInfoView.cpp rename to src/lms/ui/explore/ReleaseInfoView.cpp index 7293a678..759b20de 100644 --- a/src/ui/explore/ReleaseInfoView.cpp +++ b/src/lms/ui/explore/ReleaseInfoView.cpp @@ -22,7 +22,7 @@ #include #include "database/Release.hpp" -#include "similarity/SimilaritySearcher.hpp" +#include "recommendation/IEngine.hpp" #include "utils/Service.hpp" #include "utils/String.hpp" @@ -68,7 +68,7 @@ ReleaseInfo::refresh() if (!releaseId) return; - const std::vector releasesIds {ServiceProvider::get()->getSimilarReleases(LmsApp->getDbSession(), *releaseId, 5)}; + const std::vector releasesIds {ServiceProvider::get()->getSimilarReleases(LmsApp->getDbSession(), *releaseId, 5)}; auto transaction {LmsApp->getDbSession().createSharedTransaction()}; diff --git a/src/ui/explore/ReleaseInfoView.hpp b/src/lms/ui/explore/ReleaseInfoView.hpp similarity index 100% rename from src/ui/explore/ReleaseInfoView.hpp rename to src/lms/ui/explore/ReleaseInfoView.hpp diff --git a/src/ui/explore/ReleaseLink.cpp b/src/lms/ui/explore/ReleaseLink.cpp similarity index 100% rename from src/ui/explore/ReleaseLink.cpp rename to src/lms/ui/explore/ReleaseLink.cpp diff --git a/src/ui/explore/ReleaseLink.hpp b/src/lms/ui/explore/ReleaseLink.hpp similarity index 100% rename from src/ui/explore/ReleaseLink.hpp rename to src/lms/ui/explore/ReleaseLink.hpp diff --git a/src/ui/explore/ReleaseView.cpp b/src/lms/ui/explore/ReleaseView.cpp similarity index 99% rename from src/ui/explore/ReleaseView.cpp rename to src/lms/ui/explore/ReleaseView.cpp index 8cb45f8d..11238e9a 100644 --- a/src/ui/explore/ReleaseView.cpp +++ b/src/lms/ui/explore/ReleaseView.cpp @@ -26,6 +26,7 @@ #include #include "database/Release.hpp" +#include "database/ScanSettings.hpp" #include "database/Track.hpp" #include "utils/Logger.hpp" diff --git a/src/ui/explore/ReleaseView.hpp b/src/lms/ui/explore/ReleaseView.hpp similarity index 100% rename from src/ui/explore/ReleaseView.hpp rename to src/lms/ui/explore/ReleaseView.hpp diff --git a/src/ui/explore/ReleasesInfoView.cpp b/src/lms/ui/explore/ReleasesInfoView.cpp similarity index 100% rename from src/ui/explore/ReleasesInfoView.cpp rename to src/lms/ui/explore/ReleasesInfoView.cpp diff --git a/src/ui/explore/ReleasesInfoView.hpp b/src/lms/ui/explore/ReleasesInfoView.hpp similarity index 100% rename from src/ui/explore/ReleasesInfoView.hpp rename to src/lms/ui/explore/ReleasesInfoView.hpp diff --git a/src/ui/explore/ReleasesView.cpp b/src/lms/ui/explore/ReleasesView.cpp similarity index 100% rename from src/ui/explore/ReleasesView.cpp rename to src/lms/ui/explore/ReleasesView.cpp diff --git a/src/ui/explore/ReleasesView.hpp b/src/lms/ui/explore/ReleasesView.hpp similarity index 100% rename from src/ui/explore/ReleasesView.hpp rename to src/lms/ui/explore/ReleasesView.hpp diff --git a/src/ui/explore/TracksInfoView.cpp b/src/lms/ui/explore/TracksInfoView.cpp similarity index 100% rename from src/ui/explore/TracksInfoView.cpp rename to src/lms/ui/explore/TracksInfoView.cpp diff --git a/src/ui/explore/TracksInfoView.hpp b/src/lms/ui/explore/TracksInfoView.hpp similarity index 100% rename from src/ui/explore/TracksInfoView.hpp rename to src/lms/ui/explore/TracksInfoView.hpp diff --git a/src/ui/explore/TracksView.cpp b/src/lms/ui/explore/TracksView.cpp similarity index 100% rename from src/ui/explore/TracksView.cpp rename to src/lms/ui/explore/TracksView.cpp diff --git a/src/ui/explore/TracksView.hpp b/src/lms/ui/explore/TracksView.hpp similarity index 100% rename from src/ui/explore/TracksView.hpp rename to src/lms/ui/explore/TracksView.hpp diff --git a/src/ui/resource/AudioResource.cpp b/src/lms/ui/resource/AudioResource.cpp similarity index 100% rename from src/ui/resource/AudioResource.cpp rename to src/lms/ui/resource/AudioResource.cpp diff --git a/src/ui/resource/AudioResource.hpp b/src/lms/ui/resource/AudioResource.hpp similarity index 100% rename from src/ui/resource/AudioResource.hpp rename to src/lms/ui/resource/AudioResource.hpp diff --git a/src/ui/resource/ImageResource.cpp b/src/lms/ui/resource/ImageResource.cpp similarity index 83% rename from src/ui/resource/ImageResource.cpp rename to src/lms/ui/resource/ImageResource.cpp index f0bade0c..6abd1d42 100644 --- a/src/ui/resource/ImageResource.cpp +++ b/src/lms/ui/resource/ImageResource.cpp @@ -22,7 +22,7 @@ #include #include -#include "cover/CoverArtGrabber.hpp" +#include "cover/ICoverArtGrabber.hpp" #include "database/Track.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" @@ -33,9 +33,6 @@ namespace UserInterface { -static const std::string unknownCoverPath = "/images/unknown-cover.jpg"; -static const std::string unknownArtistImagePath = "/images/unknown-artist.jpg"; - ImageResource::~ImageResource() { beingDeleted(); @@ -80,7 +77,7 @@ ImageResource::handleRequest(const Wt::Http::Request& request, Wt::Http::Respons // DbSession are not thread safe { Wt::WApplication::UpdateLock lock {LmsApp}; - cover = ServiceProvider::get()->getFromTrack(LmsApp->getDbSession(), *trackId, Image::Format::JPEG, *size); + cover = ServiceProvider::get()->getFromTrack(LmsApp->getDbSession(), *trackId, CoverArt::Format::JPEG, *size); } } else if (releaseIdStr) @@ -92,7 +89,7 @@ ImageResource::handleRequest(const Wt::Http::Request& request, Wt::Http::Respons // DbSession are not thread safe { Wt::WApplication::UpdateLock lock {LmsApp}; - cover = ServiceProvider::get()->getFromRelease(LmsApp->getDbSession(), *releaseId, Image::Format::JPEG, *size); + cover = ServiceProvider::get()->getFromRelease(LmsApp->getDbSession(), *releaseId, CoverArt::Format::JPEG, *size); } } else @@ -106,7 +103,7 @@ ImageResource::handleRequest(const Wt::Http::Request& request, Wt::Http::Respons std::string ImageResource::getMimeType() { - return Image::format_to_mimeType(Image::Format::JPEG); + return CoverArt::formatToMimeType(CoverArt::Format::JPEG); } } // namespace UserInterface diff --git a/src/ui/resource/ImageResource.hpp b/src/lms/ui/resource/ImageResource.hpp similarity index 97% rename from src/ui/resource/ImageResource.hpp rename to src/lms/ui/resource/ImageResource.hpp index 2f372b7d..29b5a595 100644 --- a/src/ui/resource/ImageResource.hpp +++ b/src/lms/ui/resource/ImageResource.hpp @@ -26,8 +26,6 @@ #include "database/Types.hpp" -#include "image/Image.hpp" - namespace UserInterface { diff --git a/src/similarity/SimilaritySearcher.cpp b/src/similarity/SimilaritySearcher.cpp deleted file mode 100644 index cf2513d0..00000000 --- a/src/similarity/SimilaritySearcher.cpp +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (C) 2019 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "SimilaritySearcher.hpp" - -#include "features/SimilarityFeaturesScannerAddon.hpp" -#include "cluster/SimilarityClusterSearcher.hpp" - -#include "database/ScanSettings.hpp" -#include "database/TrackList.hpp" - -namespace Similarity { - -Searcher::Searcher(FeaturesScannerAddon& somAddon) -: _somAddon(somAddon) -{} - -static -Database::ScanSettings::SimilarityEngineType -getEngineType(Database::Session& dbSession) -{ - auto transaction {dbSession.createSharedTransaction()}; - return Database::ScanSettings::get(dbSession)->getSimilarityEngineType(); -} - -std::vector -Searcher::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) -{ - auto engineType {getEngineType(session)}; - auto somSearcher {_somAddon.getSearcher()}; - - std::set trackIds; - { - auto transaction {session.createSharedTransaction()}; - Database::TrackList::pointer trackList {Database::TrackList::getById(session, trackListId)}; - if (trackList) - { - const std::vector orderedTrackIds {trackList->getTrackIds()}; - trackIds = std::set {std::cbegin(orderedTrackIds), std::cend(orderedTrackIds)}; - } - } - - if (trackIds.empty()) - return {}; - - if (engineType == Database::ScanSettings::SimilarityEngineType::Features - && somSearcher - && std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return somSearcher->isTrackClassified(trackId); } )) - { - return somSearcher->getSimilarTracks(trackIds, maxCount); - } - else - return ClusterSearcher::getSimilarTracksFromTrackList(session, trackListId, maxCount); -} - -std::vector -Searcher::getSimilarTracks(Database::Session& dbSession, const std::set& trackIds, std::size_t maxCount) -{ - auto engineType {getEngineType(dbSession)}; - auto somSearcher {_somAddon.getSearcher()}; - - if (engineType == Database::ScanSettings::SimilarityEngineType::Features - && somSearcher - && std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return somSearcher->isTrackClassified(trackId); } )) - { - return somSearcher->getSimilarTracks(trackIds, maxCount); - } - else - return ClusterSearcher::getSimilarTracks(dbSession, trackIds, maxCount); -} - -std::vector -Searcher::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) -{ - auto engineType {getEngineType(dbSession)}; - auto somSearcher {_somAddon.getSearcher()}; - - if (engineType == Database::ScanSettings::SimilarityEngineType::Features - && somSearcher - && somSearcher->isReleaseClassified(releaseId)) - { - return somSearcher->getSimilarReleases(releaseId, maxCount); - } - else - return ClusterSearcher::getSimilarReleases(dbSession, releaseId, maxCount); -} - -std::vector -Searcher::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) -{ - auto engineType {getEngineType(dbSession)}; - auto somSearcher {_somAddon.getSearcher()}; - - if (engineType == Database::ScanSettings::SimilarityEngineType::Features - && somSearcher - && somSearcher->isArtistClassified(artistId)) - { - return somSearcher->getSimilarArtists(artistId, maxCount); - } - else - return ClusterSearcher::getSimilarArtists(dbSession, artistId, maxCount); -} - -} // ns Similarity diff --git a/src/similarity/SimilaritySearcher.hpp b/src/similarity/SimilaritySearcher.hpp deleted file mode 100644 index 3a53cb9b..00000000 --- a/src/similarity/SimilaritySearcher.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2019 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include -#include - -#include "database/Types.hpp" - -namespace Database -{ - class Session; -} - -namespace Similarity { - -class FeaturesScannerAddon; - -class Searcher -{ - public: - Searcher(FeaturesScannerAddon& somAddon); - - // Closest results first - std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount); - std::vector getSimilarTracks(Database::Session& session, const std::set& tracksId, std::size_t maxCount); - std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount); - std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount); - - private: - - FeaturesScannerAddon& _somAddon; -}; - -} // ns Similarity diff --git a/src/similarity/features/SimilarityFeaturesScannerAddon.cpp b/src/similarity/features/SimilarityFeaturesScannerAddon.cpp deleted file mode 100644 index 20ecf2d3..00000000 --- a/src/similarity/features/SimilarityFeaturesScannerAddon.cpp +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright (C) 2019 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "SimilarityFeaturesScannerAddon.hpp" - -#include "AcousticBrainzUtils.hpp" -#include "database/ScanSettings.hpp" -#include "database/Track.hpp" -#include "database/TrackFeatures.hpp" -#include "similarity/features/SimilarityFeaturesCache.hpp" -#include "utils/Config.hpp" -#include "utils/Logger.hpp" - -namespace Similarity { - -static -bool -hasAtLeastOneTrackWithFeatures(Database::Session& session) -{ - auto transaction {session.createSharedTransaction()}; - return !Database::Track::getAllIdsWithFeatures(session, 1).empty(); -} - -struct TrackInfo -{ - Database::IdType id; - std::optional mbid; -}; - -static -std::vector -getTracksWithMBIDAndMissingFeatures(Database::Session& dbSession) -{ - std::vector res; - - auto transaction {dbSession.createSharedTransaction()}; - - auto tracks {Database::Track::getAllWithMBIDAndMissingFeatures(dbSession)}; - for (const Database::Track::pointer& track : tracks) - res.push_back({track.id(), track->getMBID()}); - - return res; -} - -FeaturesScannerAddon::FeaturesScannerAddon(Database::Db& db) -: _dbSession {db} -{ - std::optional cache {Similarity::FeaturesCache::read()}; - if (cache) - { - auto searcher {std::make_shared(_dbSession, *cache, [&]() { return _stopRequested; })}; - if (searcher->isValid()) - std::atomic_store(&_searcher, searcher); - } -} - -std::shared_ptr -FeaturesScannerAddon::getSearcher() -{ - return std::atomic_load(&_searcher); -} - -void -FeaturesScannerAddon::requestStop() -{ - _stopRequested = true; -} - -void -FeaturesScannerAddon::trackUpdated(Database::IdType trackId) -{ - auto uniqueTransaction {_dbSession.createUniqueTransaction()}; - - auto track {Database::Track::getById(_dbSession, trackId)}; - if (!track) - return; - - track.modify()->setFeatures({}); -} - -void -FeaturesScannerAddon::preScanComplete() -{ - { - auto transaction {_dbSession.createSharedTransaction()}; - - if (Database::ScanSettings::get(_dbSession)->getSimilarityEngineType() != Database::ScanSettings::SimilarityEngineType::Features) - { - LMS_LOG(DBUPDATER, INFO) << "Do not fetch features since the engine type does not make use of them"; - return; - } - } - - LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features..."; - const std::vector tracksInfo {getTracksWithMBIDAndMissingFeatures(_dbSession)}; - LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features DONE (found " << tracksInfo.size() << ")"; - - if (!tracksInfo.empty()) - Similarity::FeaturesCache::invalidate(); - - for (const TrackInfo& trackInfo : tracksInfo) - { - if (_stopRequested) - return; - - if (trackInfo.mbid) - fetchFeatures(trackInfo.id, *trackInfo.mbid); - } - - updateSearcher(); -} - -void -FeaturesScannerAddon::updateSearcher() -{ - LMS_LOG(SIMILARITY, INFO) << "Updating searcher..."; - - if (!hasAtLeastOneTrackWithFeatures(_dbSession)) - { - LMS_LOG(DBUPDATER, INFO) << "No track found with features!"; - std::atomic_store(&_searcher, std::shared_ptr{}); - return; - } - - Similarity::FeaturesSearcher::TrainSettings trainSettings; - trainSettings.featureSettingsMap = FeaturesSearcher::getDefaultTrainFeatureSettings(); - - auto searcher {std::make_shared(_dbSession, trainSettings, [&]() { return _stopRequested; })}; - if (searcher->isValid()) - { - std::atomic_store(&_searcher, searcher); - FeaturesCache cache{searcher->toCache()}; - cache.write(); - - LMS_LOG(DBUPDATER, INFO) << "New features similarity searcher instanciated"; - } - else - { - LMS_LOG(DBUPDATER, ERROR) << "Cannot set up a valid features similarity searcher!"; - std::atomic_store(&_searcher, std::shared_ptr{}); - } -} - -bool -FeaturesScannerAddon::fetchFeatures(Database::IdType trackId, const UUID& MBID) -{ - std::map features; - - LMS_LOG(DBUPDATER, DEBUG) << "Fetching low level features for track '" << MBID.getAsString() << "'"; - const std::string data {AcousticBrainz::extractLowLevelFeatures(MBID)}; - if (data.empty()) - { - LMS_LOG(DBUPDATER, ERROR) << "Track " << trackId << ", MBID = '" << MBID.getAsString() << "': cannot extract features using AcousticBrainz"; - return false; - } - - { - auto uniqueTransaction {_dbSession.createUniqueTransaction()}; - - Wt::Dbo::ptr track {Database::Track::getById(_dbSession, trackId)}; - if (!track) - return false; - - Database::TrackFeatures::create(_dbSession, track, data); - } - - return true; -} - -} // namespace Similarity - diff --git a/src/similarity/features/SimilarityFeaturesScannerAddon.hpp b/src/similarity/features/SimilarityFeaturesScannerAddon.hpp deleted file mode 100644 index 38651e37..00000000 --- a/src/similarity/features/SimilarityFeaturesScannerAddon.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include "database/Session.hpp" -#include "scanner/MediaScannerAddon.hpp" - -#include "utils/UUID.hpp" - -#include "SimilarityFeaturesSearcher.hpp" - - -namespace Database { - class Db; -} - -namespace Similarity { - -class FeaturesScannerAddon final : public Scanner::MediaScannerAddon -{ - public: - - FeaturesScannerAddon(Database::Db& db); - - std::shared_ptr getSearcher(); - - private: - - void refreshSettings() override {} - void requestStop() override; - void preScanComplete() override; - - void trackAdded(Database::IdType) override {} - void trackToRemove(Database::IdType) override {} - void trackUpdated(Database::IdType trackId) override; - - bool fetchFeatures(Database::IdType trackId, const UUID& MBID); - - void updateSearcher(); - - Database::Session _dbSession; - std::shared_ptr _searcher; - bool _stopRequested {}; -}; - -FeaturesScannerAddon* setFeaturesScannerAddon(FeaturesScannerAddon addon); -FeaturesScannerAddon* getFeaturesScannerAddon(); - -} // namespace Similarity - diff --git a/src/similarity/features/SimilarityFeaturesSearcher.cpp b/src/similarity/features/SimilarityFeaturesSearcher.cpp deleted file mode 100644 index a1ab5987..00000000 --- a/src/similarity/features/SimilarityFeaturesSearcher.cpp +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "SimilarityFeaturesSearcher.hpp" - -#include -#include - -#include "database/Artist.hpp" -#include "database/Release.hpp" -#include "database/Session.hpp" -#include "database/Track.hpp" -#include "database/TrackFeatures.hpp" -#include "som/DataNormalizer.hpp" -#include "utils/Logger.hpp" - - -namespace Similarity { - -const FeatureSettingsMap& -FeaturesSearcher::getDefaultTrainFeatureSettings() -{ - static FeatureSettingsMap defaultTrainFeatureSettings - { - { "lowlevel.spectral_energyband_high.mean", {1}}, - { "lowlevel.spectral_rolloff.median", {1}}, - { "lowlevel.spectral_contrast_valleys.var", {1}}, - { "lowlevel.erbbands.mean", {1}}, - { "lowlevel.gfcc.mean", {1}}, - }; - - return defaultTrainFeatureSettings; -} - -static -std::optional -getTrackFeatureValues(FeaturesSearcher::FeaturesFetchFunc func, Database::IdType trackId, const std::unordered_set& featureNames) -{ - return func(trackId, featureNames); -} - -static -std::optional -getTrackFeatureValuesFromDb(Database::Session& session, Database::IdType trackId, const std::unordered_set& featureNames) -{ - auto func = [&](Database::IdType trackId, const std::unordered_set& featureNames) - { - std::optional res; - - auto transaction {session.createSharedTransaction()}; - - Database::Track::pointer track {Database::Track::getById(session, trackId)}; - if (!track) - return res; - - res = track->getTrackFeatures()->getFeatureValuesMap(featureNames); - if (res->empty()) - res.reset(); - - return res; - }; - - return getTrackFeatureValues(func, trackId, featureNames); -} - -static -std::optional -convertFeatureValuesMapToInputVector(const FeatureValuesMap& featureValuesMap, std::size_t nbDimensions) -{ - std::size_t i {}; - std::optional res {SOM::InputVector {nbDimensions}}; - for (const auto& [featureName, values] : featureValuesMap) - { - if (values.size() != getFeatureDef(featureName).nbDimensions) - { - LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << values.size(); - res.reset(); - break; - } - - for (double val : values) - (*res)[i++] = val; - } - - return res; -} - -static -SOM::InputVector -getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t nbDimensions) -{ - SOM::InputVector weights {nbDimensions}; - std::size_t index {}; - for (const auto& [featureName, featureSettings] : featureSettingsMap) - { - const std::size_t featureNbDimensions {getFeatureDef(featureName).nbDimensions}; - - for (std::size_t i {}; i < featureNbDimensions; ++i) - weights[index++] = (1. / featureNbDimensions * featureSettings.weight); - } - - assert(index == nbDimensions); - - return weights; -} - -FeaturesSearcher::FeaturesSearcher(Database::Session& session, - const TrainSettings& trainSettings, - StopRequestedFunction stopRequested) -{ - LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher..."; - - std::unordered_set featureNames; - std::transform(std::cbegin(trainSettings.featureSettingsMap), std::cend(trainSettings.featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)), - [](const auto& itFeatureSetting) { return itFeatureSetting.first; }); - - const std::size_t nbDimensions {std::accumulate(std::cbegin(featureNames), std::cend(featureNames), std::size_t {0}, - [](std::size_t sum, const FeatureName& featureName) { return sum + getFeatureDef(featureName).nbDimensions; })}; - - LMS_LOG(SIMILARITY, DEBUG) << "Features dimension = " << nbDimensions; - - std::vector trackIds; - { - auto transaction {session.createSharedTransaction()}; - - LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features..."; - trackIds = Database::Track::getAllIdsWithFeatures(session); - LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features DONE (found " << trackIds.size() << " tracks)"; - } - - std::vector samples; - std::vector samplesTrackIds; - - samples.reserve(trackIds.size()); - samplesTrackIds.reserve(trackIds.size()); - - LMS_LOG(SIMILARITY, DEBUG) << "Extracting features..."; - for (Database::IdType trackId : trackIds) - { - if (stopRequested && stopRequested()) - return; - - std::optional featureValuesMap; - - if (_featuresFetchFunc) - featureValuesMap = getTrackFeatureValues(_featuresFetchFunc, trackId, featureNames); - else - featureValuesMap = getTrackFeatureValuesFromDb(session, trackId, featureNames); - - if (!featureValuesMap) - continue; - - std::optional inputVector {convertFeatureValuesMapToInputVector(*featureValuesMap, nbDimensions)}; - if (!inputVector) - continue; - - samples.emplace_back(std::move(*inputVector)); - samplesTrackIds.emplace_back(trackId); - } - LMS_LOG(SIMILARITY, DEBUG) << "Extracting features DONE"; - - if (samples.empty()) - { - LMS_LOG(SIMILARITY, INFO) << "Nothing to classify!"; - return; - } - - LMS_LOG(SIMILARITY, DEBUG) << "Normalizing data..."; - SOM::DataNormalizer dataNormalizer {nbDimensions}; - - dataNormalizer.computeNormalizationFactors(samples); - for (auto& sample : samples) - dataNormalizer.normalizeData(sample); - - SOM::Coordinate size {static_cast(std::sqrt(samples.size() / trainSettings.sampleCountPerNeuron))}; - LMS_LOG(SIMILARITY, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network"; - - SOM::Network network {size, size, nbDimensions}; - - SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)}; - network.setDataWeights(weights); - - auto progressIndicator{[](const auto& iter) - { - LMS_LOG(SIMILARITY, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount; - }}; - - LMS_LOG(SIMILARITY, DEBUG) << "Training network..."; - network.train(samples, trainSettings.iterationCount, progressIndicator, stopRequested); - LMS_LOG(SIMILARITY, DEBUG) << "Training network DONE"; - - if (stopRequested && stopRequested()) - return; - - LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks..."; - std::map> trackPositions; - for (std::size_t i {}; i < samples.size(); ++i) - { - if (stopRequested && stopRequested()) - return; - - const SOM::Position position {network.getClosestRefVectorPosition(samples[i])}; - - trackPositions[samplesTrackIds[i]].insert(position); - } - - LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE"; - - init(session, std::move(network), std::move(trackPositions), stopRequested); - - LMS_LOG(SIMILARITY, INFO) << "Successfully constructed features searcher"; -} - -FeaturesSearcher::FeaturesSearcher(Database::Session& session, FeaturesCache cache, StopRequestedFunction stopRequested) -{ - LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher from cache..."; - - init(session, std::move(cache._network), std::move(cache._trackPositions), stopRequested); - - LMS_LOG(SIMILARITY, INFO) << "Successfully constructed features searcher from cache"; -} - -bool -FeaturesSearcher::isValid() const -{ - return _network.get() != nullptr; -} - -bool -FeaturesSearcher::isTrackClassified(Database::IdType trackId) const -{ - return (_trackPositions.find(trackId) != _trackPositions.end()); -} - -bool -FeaturesSearcher::isReleaseClassified(Database::IdType releaseId) const -{ - return (_releasePositions.find(releaseId) != _releasePositions.end()); -} - -bool -FeaturesSearcher::isArtistClassified(Database::IdType artistId) const -{ - return (_artistPositions.find(artistId) != _artistPositions.end()); -} - -std::vector -FeaturesSearcher::getSimilarTracks(const std::set& tracksIds, std::size_t maxCount) const -{ - return getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount); -} - -std::vector -FeaturesSearcher::getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const -{ - return getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount); -} - -std::vector -FeaturesSearcher::getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const -{ - return getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount); -} - -void -FeaturesSearcher::dump(Database::Session& session, std::ostream& os) const -{ - if (!isValid()) - { - os << "Invalid searcher" << std::endl; - return; - } - - os << "Number of tracks classified: " << _trackPositions.size() << std::endl; - os << "Network size: " << _network->getWidth() << " * " << _network->getHeight() << std::endl; - os << "Ref vectors median distance = " << _networkRefVectorsDistanceMedian << std::endl; - - auto transaction {session.createSharedTransaction()}; - - for (SOM::Coordinate y {}; y < _network->getHeight(); ++y) - { - for (SOM::Coordinate x {}; x < _network->getWidth(); ++x) - { - const auto& trackIds {_tracksMap[{x, y}]}; - - os << "{" << x << ", " << y << "}"; - - if (y > 0) - os << " - {" << x << ", " << y - 1 << "}: " << _network->getRefVectorsDistance({x, y}, {x, y - 1}); - if (x > 0) - os << " - {" << x - 1 << ", " << y << "}: " << _network->getRefVectorsDistance({x, y}, {x - 1, y}); - if (y != _network->getHeight() - 1) - os << " - {" << x << ", " << y + 1 << "}: " << _network->getRefVectorsDistance({x, y}, {x, y + 1}); - if (x != _network->getWidth() - 1) - os << " - {" << x + 1 << ", " << y << "}: " << _network->getRefVectorsDistance({x, y}, {x + 1, y}); - os << std::endl; - - for (Database::IdType trackId : trackIds) - { - auto track {Database::Track::getById(session, trackId)}; - if (!track) - continue; - - os << "\t"; - for (auto artist : track->getArtists()) - os << artist->getName() << " - "; - if (track->getRelease()) - os << track->getRelease()->getName() << " - "; - os << track->getName() << std::endl; - } - - } - os << std::endl; - } -} - -FeaturesCache -FeaturesSearcher::toCache() const -{ - return FeaturesCache{*_network, _trackPositions}; -} - -void -FeaturesSearcher::init(Database::Session& session, - SOM::Network network, - std::map> tracksPosition, - std::function stopRequested) -{ - _networkRefVectorsDistanceMedian = network.computeRefVectorsDistanceMedian(); - LMS_LOG(SIMILARITY, DEBUG) << "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian; - - SOM::Coordinate width {network.getWidth()}; - SOM::Coordinate height {network.getHeight()}; - - _artistsMap = SOM::Matrix>{width, height}; - _releasesMap = SOM::Matrix>{width, height}; - _tracksMap = SOM::Matrix>{width, height}; - - LMS_LOG(SIMILARITY, DEBUG) << "Constructing maps..."; - - for (auto itTrackCoord : tracksPosition) - { - if (stopRequested && stopRequested()) - return; - - auto transaction {session.createSharedTransaction()}; - - Database::IdType trackId {itTrackCoord.first}; - const std::set& positionSet {itTrackCoord.second}; - - const Database::Track::pointer track {Database::Track::getById(session, trackId)}; - if (!track) - continue; - - for (const SOM::Position& position : positionSet) - { - _tracksMap[position].insert(trackId); - _trackPositions[trackId].insert(position); - - if (track->getRelease()) - { - _releasePositions[track->getRelease().id()].insert(position); - _releasesMap[position].insert(track->getRelease().id()); - } - for (const auto& artist : track->getArtists()) - { - _artistPositions[artist.id()].insert(position); - _artistsMap[position].insert(artist.id()); - } - } - } - - _network = std::make_unique(std::move(network)); - - LMS_LOG(SIMILARITY, DEBUG) << "Constructing maps... DONE"; - -} - -static -std::set -getMatchingRefVectorsPosition(const std::set& ids, const std::map>& objectPosition) -{ - std::set res; - - if (ids.empty()) - return res; - - for (auto id : ids) - { - auto it = objectPosition.find(id); - if (it == objectPosition.end()) - continue; - - for (const auto& position : it->second) - res.insert(position); - } - - return res; -} - -static -std::set -getObjectsIds(const std::set& positionSet, const SOM::Matrix>& objectsMap ) -{ - std::set res; - - for (const auto& position : positionSet) - { - for (auto id : objectsMap.get(position)) - res.insert(id); - } - - return res; -} - -std::vector -FeaturesSearcher::getSimilarObjects(const std::set& ids, - const SOM::Matrix>& objectsMap, - const std::map>& objectPosition, - std::size_t maxCount) const -{ - std::vector res; - - if (!isValid()) - return res; - - auto now {std::chrono::system_clock::now()}; - std::mt19937 randGenerator{static_cast(std::chrono::duration_cast(now.time_since_epoch()).count())}; - - std::set searchedRefVectorsPosition {getMatchingRefVectorsPosition(ids, objectPosition)}; - if (searchedRefVectorsPosition.empty()) - return res; - - while (1) - { - std::set closestObjectIds {getObjectsIds(searchedRefVectorsPosition, objectsMap)}; - - // Remove objects that are already in input or already reported - for (auto id : ids) - closestObjectIds.erase(id); - - for (auto id : res) - closestObjectIds.erase(id); - - { - std::vector objectIdsToAdd {closestObjectIds.begin(), closestObjectIds.end()}; - std::shuffle(objectIdsToAdd.begin(), objectIdsToAdd.end(), randGenerator); - std::copy(objectIdsToAdd.begin(), objectIdsToAdd.end(), std::back_inserter(res)); - } - - if (res.size() > maxCount) - res.resize(maxCount); - - if (res.size() == maxCount) - break; - - // If there is not enough objects, try again with closest neighbour until there is too much distance - std::optional closestRefVectorPosition {_network->getClosestRefVectorPosition(searchedRefVectorsPosition, _networkRefVectorsDistanceMedian * 0.75)}; - if (!closestRefVectorPosition) - break; - - searchedRefVectorsPosition.insert(*closestRefVectorPosition); - } - - return res; -} - - - -} // ns Similarity diff --git a/src/similarity/features/SimilarityFeaturesSearcher.hpp b/src/similarity/features/SimilarityFeaturesSearcher.hpp deleted file mode 100644 index 625877c1..00000000 --- a/src/similarity/features/SimilarityFeaturesSearcher.hpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include -#include -#include -#include - -#include "database/Types.hpp" -#include "som/DataNormalizer.hpp" -#include "som/Network.hpp" -#include "SimilarityFeaturesCache.hpp" -#include "SimilarityFeaturesDefs.hpp" - -namespace Database -{ - class Session; -} - -namespace Similarity { - -using FeatureWeight = double; - -class FeaturesSearcher -{ - public: - - using StopRequestedFunction = std::function; // return true if stop requested - - // Use cache - FeaturesSearcher(Database::Session& session, FeaturesCache cache, StopRequestedFunction stopRequested); - - // Use training (may be very slow) - struct TrainSettings - { - std::size_t iterationCount {10}; - float sampleCountPerNeuron {4}; - FeatureSettingsMap featureSettingsMap; - }; - FeaturesSearcher(Database::Session& session, const TrainSettings& trainSettings, StopRequestedFunction stopRequested = {}); - - static const FeatureSettingsMap& getDefaultTrainFeatureSettings(); - - bool isValid() const; - - bool isTrackClassified(Database::IdType trackId) const; - bool isReleaseClassified(Database::IdType releaseId) const; - bool isArtistClassified(Database::IdType artistId) const; - - std::vector getSimilarTracks(const std::set& tracksId, std::size_t maxCount) const; - std::vector getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const; - std::vector getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const; - - void dump(Database::Session& session, std::ostream& os) const; - - FeaturesCache toCache() const; - - using FeaturesFetchFunc = std::function>>(Database::IdType /*trackId*/, const std::unordered_set& /*features*/)>; - // Default is to retrieve the features from the database (may be slow). - // Use this only if you want to train different searchers with the same data - static void setFeaturesFetchFunc(FeaturesFetchFunc func) { _featuresFetchFunc = func; } - - private: - - using ObjectPositions = std::map>; - - void init(Database::Session& session, - SOM::Network network, - ObjectPositions tracksPosition, - StopRequestedFunction stopRequested); - - std::vector getSimilarObjects(const std::set& ids, - const SOM::Matrix>& objectsMap, - const ObjectPositions& objectPosition, - std::size_t maxCount) const; - - std::unique_ptr _network; - double _networkRefVectorsDistanceMedian {}; - - SOM::Matrix> _artistsMap; - ObjectPositions _artistPositions; - - SOM::Matrix> _releasesMap; - ObjectPositions _releasePositions; - - SOM::Matrix> _tracksMap; - ObjectPositions _trackPositions; - - static inline FeaturesFetchFunc _featuresFetchFunc; -}; - -} // ns Similarity diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt new file mode 100644 index 00000000..d2d34662 --- /dev/null +++ b/src/test/CMakeLists.txt @@ -0,0 +1,4 @@ + +add_subdirectory(database) +add_subdirectory(som) + diff --git a/src/test/database/CMakeLists.txt b/src/test/database/CMakeLists.txt new file mode 100644 index 00000000..ac42aa6b --- /dev/null +++ b/src/test/database/CMakeLists.txt @@ -0,0 +1,11 @@ + +add_executable(test-database + DatabaseTest.cpp + ) + +target_link_libraries(test-database PRIVATE + lmsdatabase + ) + +add_test(NAME database COMMAND test-database) + diff --git a/test/database/DatabaseTest.cpp b/src/test/database/DatabaseTest.cpp similarity index 97% rename from test/database/DatabaseTest.cpp rename to src/test/database/DatabaseTest.cpp index 005439d1..5ad5b60b 100644 --- a/test/database/DatabaseTest.cpp +++ b/src/test/database/DatabaseTest.cpp @@ -453,12 +453,24 @@ testSingleTrackSingleCluster(Session& session) CHECK(track->getClusterIds().empty()); } + { + auto transaction {session.createSharedTransaction()}; + CHECK(Track::getAllIdsWithClusters(session).empty()); + } + { auto transaction {session.createUniqueTransaction()}; cluster1.get().modify()->addTrack(track.get()); } + { + auto transaction {session.createSharedTransaction()}; + auto tracks {Track::getAllIdsWithClusters(session)}; + CHECK(tracks.size() == 1); + CHECK(tracks.front() == track.getId()); + } + { auto transaction {session.createSharedTransaction()}; auto clusters {Cluster::getAllOrphans(session)}; @@ -611,6 +623,11 @@ testSingleTrackSingleReleaseSingleCluster(Session& session) ScopedClusterType clusterType {session, "MyClusterType"}; ScopedCluster cluster {session, clusterType .lockAndGet(), "MyCluster"}; + { + auto transaction {session.createSharedTransaction()}; + CHECK(Release::getAllIdsWithClusters(session).empty()); + } + { auto transaction {session.createUniqueTransaction()}; @@ -618,6 +635,13 @@ testSingleTrackSingleReleaseSingleCluster(Session& session) cluster.get().modify()->addTrack(track.get()); } + { + auto transaction {session.createSharedTransaction()}; + auto releases {Release::getAllIdsWithClusters(session)}; + CHECK(releases.size() == 1); + CHECK(releases.front() == release.getId()); + } + { auto transaction {session.createSharedTransaction()}; @@ -858,6 +882,11 @@ testSingleTrackSingleReleaseSingleArtistSingleCluster(Session& session) ScopedClusterType clusterType {session, "MyType"}; ScopedCluster cluster {session, clusterType.lockAndGet(), "MyCluster"}; + { + auto transaction {session.createSharedTransaction()}; + CHECK(Artist::getAllIdsWithClusters(session).empty()); + } + { auto transaction {session.createUniqueTransaction()}; @@ -875,6 +904,13 @@ testSingleTrackSingleReleaseSingleArtistSingleCluster(Session& session) CHECK(Release::getAllOrphans(session).empty()); } + { + auto transaction {session.createSharedTransaction()}; + auto artists {Artist::getAllIdsWithClusters(session)}; + CHECK(artists.size() == 1); + CHECK(artists.front() == artist.getId()); + } + { auto transaction {session.createSharedTransaction()}; diff --git a/src/test/som/CMakeLists.txt b/src/test/som/CMakeLists.txt new file mode 100644 index 00000000..aaaf43d4 --- /dev/null +++ b/src/test/som/CMakeLists.txt @@ -0,0 +1,11 @@ + +add_executable(test-som + SomTest.cpp + ) + +target_link_libraries(test-som PRIVATE + lmssom + ) + +add_test(NAME som COMMAND test-som) + diff --git a/test/som/SomTest.cpp b/src/test/som/SomTest.cpp similarity index 96% rename from test/som/SomTest.cpp rename to src/test/som/SomTest.cpp index 70aad7a1..bfbf05a5 100644 --- a/test/som/SomTest.cpp +++ b/src/test/som/SomTest.cpp @@ -21,8 +21,8 @@ #include #include -#include "DataNormalizer.hpp" -#include "Network.hpp" +#include "som/DataNormalizer.hpp" +#include "som/Network.hpp" using namespace SOM; @@ -82,7 +82,7 @@ int main() assert((std::abs(distFunc({1, 0}, {1, 0.33}, weights) - distFunc({1, 0.66}, {1, 1.}, weights)) < EPSILON)); { - std::set positions; + std::unordered_set positions; for (const InputVector& data : trainData) positions.insert(network.getClosestRefVectorPosition(data)); assert(positions.size() == 4); diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt new file mode 100644 index 00000000..e5877a5b --- /dev/null +++ b/src/tools/CMakeLists.txt @@ -0,0 +1,5 @@ + +add_subdirectory(metadata) +add_subdirectory(recommendation) + + diff --git a/src/tools/metadata/CMakeLists.txt b/src/tools/metadata/CMakeLists.txt new file mode 100644 index 00000000..4c0706cf --- /dev/null +++ b/src/tools/metadata/CMakeLists.txt @@ -0,0 +1,11 @@ + +add_executable(lms-metadata + LmsMetadata.cpp + ) + +target_link_libraries(lms-metadata PRIVATE + lmsmetadata + ) + +install(TARGETS lms-metadata DESTINATION bin) + diff --git a/tools/metadata/LmsMetadata.cpp b/src/tools/metadata/LmsMetadata.cpp similarity index 96% rename from tools/metadata/LmsMetadata.cpp rename to src/tools/metadata/LmsMetadata.cpp index 9828c7f9..7a04e178 100644 --- a/tools/metadata/LmsMetadata.cpp +++ b/src/tools/metadata/LmsMetadata.cpp @@ -25,8 +25,7 @@ #include -#include "av/AvInfo.hpp" -#include "metadata/AvFormat.hpp" +#include "metadata/AvFormatParser.hpp" #include "metadata/TagLibParser.hpp" #include "utils/StreamLogger.hpp" @@ -52,7 +51,7 @@ std::ostream& operator<<(std::ostream& os, const MetaData::Album& album) -void parse(MetaData::Parser& parser, const std::filesystem::path& file) +void parse(MetaData::IParser& parser, const std::filesystem::path& file) { using namespace MetaData; @@ -155,7 +154,7 @@ int main(int argc, char *argv[]) { std::cout << "Using av:" << std::endl; - MetaData::AvFormat parser; + MetaData::AvFormatParser parser; parse(parser, file); } diff --git a/src/tools/recommendation/CMakeLists.txt b/src/tools/recommendation/CMakeLists.txt new file mode 100644 index 00000000..75b3e5d0 --- /dev/null +++ b/src/tools/recommendation/CMakeLists.txt @@ -0,0 +1,11 @@ + +add_executable(lms-recommendation + LmsRecommendation.cpp + ) + +target_link_libraries(lms-recommendation PRIVATE + lmsdatabase + lmsrecommendation + ) + +install(TARGETS lms-recommendation DESTINATION bin) diff --git a/src/tools/recommendation/LmsRecommendation.cpp b/src/tools/recommendation/LmsRecommendation.cpp new file mode 100644 index 00000000..954af8c0 --- /dev/null +++ b/src/tools/recommendation/LmsRecommendation.cpp @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include +#include +#include +#include + +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Db.hpp" +#include "database/Release.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "utils/IConfig.hpp" +#include "utils/Semaphore.hpp" +#include "utils/Service.hpp" +#include "utils/StreamLogger.hpp" +#include "recommendation/IEngine.hpp" + + +static +void +dumpRecommendation(Database::Session session, Recommendation::IEngine& engine) +{ + const std::vector trackIds {[&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Track::getAllIds(session); + }()}; + + std::cout << "*** Tracks (" << trackIds.size() << ") ***" << std::endl; + for (Database::IdType trackId : trackIds) + { + auto trackToString = [&](Database::IdType trackId) + { + std::string res; + auto transaction {session.createSharedTransaction()}; + Database::Track::pointer track {Database::Track::getById(session, trackId)}; + + res += track->getName(); + if (track->getRelease()) + res += " [" + track->getRelease()->getName() + "]"; + for (auto artist : track->getArtists()) + res += " - " + artist->getName(); + for (auto cluster : track->getClusters()) + res += " {" + cluster->getType()->getName() + "-"+ cluster->getName() + "}"; + + return res; + }; + + std::cout << "Processing track '" << trackToString(trackId) << std::endl; + for (Database::IdType similarTrackId : engine.getSimilarTracks(session, {trackId}, 3)) + std::cout << "\t- Similar track '" << trackToString(similarTrackId) << std::endl; + } + + const std::vector releaseIds = std::invoke([&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Release::getAllIds(session); + }); + + std::cout << "*** Releases ***" << std::endl; + for (Database::IdType releaseId : releaseIds) + { + auto releaseToString = [&](Database::IdType releaseId) + { + auto transaction {session.createSharedTransaction()}; + + Database::Release::pointer release {Database::Release::getById(session, releaseId)}; + return release->getName(); + }; + + std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl; + for (Database::IdType similarReleaseId : engine.getSimilarReleases(session, {releaseId}, 3)) + std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl; + } + + const std::vector artistIds = std::invoke([&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Artist::getAllIds(session); + }); + + std::cout << "*** Artists ***" << std::endl; + for (Database::IdType artistId : artistIds) + { + auto artistToString = [&](Database::IdType artistId) + { + auto transaction {session.createSharedTransaction()}; + + Database::Artist::pointer artist {Database::Artist::getById(session, artistId)}; + return artist->getName(); + }; + + std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl; + for (Database::IdType similarArtistId : engine.getSimilarArtists(session, {artistId}, 3)) + std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl; + } + + + + +} + + +int main(int argc, char *argv[]) +{ + try + { + // log to stdout + ServiceProvider::create(std::cout); + + std::filesystem::path configFilePath {"/etc/lms.conf"}; + if (argc >= 2) + configFilePath = std::string(argv[1], 0, 256); + + ServiceProvider::assign(createConfig(configFilePath)); + + Database::Db db {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; + Database::Session session {db}; + + std::cout << "Creating recommendation engine..." << std::endl; + const auto engine {Recommendation::createEngine(db)}; + std::cout << "Recommendation engine created!" << std::endl; + + Semaphore sem; + + engine->reloaded().connect([&]() + { + sem.notify(); + }); + + engine->start(); + + std::cout << "Waiting for the recommendation engine to be loaded..." << std::endl; + sem.wait(); + std::cout << "Recommendation engine loaded!" << std::endl; + + dumpRecommendation(db, *engine); + + engine->stop(); + } + catch( std::exception& e) + { + std::cerr << "Caught exception: " << e.what() << std::endl; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + diff --git a/tools/similarity-parameters/GeneticAlgorithm.hpp b/src/tools/similarity-parameters/GeneticAlgorithm.hpp similarity index 100% rename from tools/similarity-parameters/GeneticAlgorithm.hpp rename to src/tools/similarity-parameters/GeneticAlgorithm.hpp diff --git a/tools/similarity-parameters/LmsSimilarityParameters.cpp b/src/tools/similarity-parameters/LmsSimilarityParameters.cpp similarity index 100% rename from tools/similarity-parameters/LmsSimilarityParameters.cpp rename to src/tools/similarity-parameters/LmsSimilarityParameters.cpp diff --git a/tools/similarity-parameters/ParallelFor.hpp b/src/tools/similarity-parameters/ParallelFor.hpp similarity index 100% rename from tools/similarity-parameters/ParallelFor.hpp rename to src/tools/similarity-parameters/ParallelFor.hpp diff --git a/test/Makefile.am b/test/Makefile.am deleted file mode 100644 index 06d88e41..00000000 --- a/test/Makefile.am +++ /dev/null @@ -1,34 +0,0 @@ - -TESTS = test-som test-database - -check_PROGRAMS = test-som test-database - -test_som_SOURCES = \ - $(srcdir)/som/SomTest.cpp \ - $(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp \ - $(top_srcdir)/src/similarity/features/som/Network.cpp - -test_som_CXXFLAGS=-std=c++17 -I${top_srcdir}/src/ -I${top_srcdir}/src/similarity/features/som/ - - -test_database_SOURCES = \ - $(srcdir)/database/DatabaseTest.cpp \ - $(top_srcdir)/src/database/Artist.cpp \ - $(top_srcdir)/src/database/Cluster.cpp \ - $(top_srcdir)/src/database/Db.cpp \ - $(top_srcdir)/src/database/TrackArtistLink.cpp \ - $(top_srcdir)/src/database/TrackFeatures.cpp \ - $(top_srcdir)/src/database/TrackList.cpp \ - $(top_srcdir)/src/database/Release.cpp \ - $(top_srcdir)/src/database/ScanSettings.cpp \ - $(top_srcdir)/src/database/Session.cpp \ - $(top_srcdir)/src/database/SqlQuery.cpp \ - $(top_srcdir)/src/database/Track.cpp \ - $(top_srcdir)/src/database/TrackBookmark.cpp \ - $(top_srcdir)/src/database/User.cpp \ - $(top_srcdir)/src/utils/Logger.cpp \ - $(top_srcdir)/src/utils/StreamLogger.cpp \ - $(top_srcdir)/src/utils/String.cpp - -test_database_CXXFLAGS=-std=c++17 -I${top_srcdir}/src/ - diff --git a/tools/Makefile.am b/tools/Makefile.am deleted file mode 100644 index c382f8bc..00000000 --- a/tools/Makefile.am +++ /dev/null @@ -1,4 +0,0 @@ -if BUILD_TOOLS -SUBDIRS = similarity similarity-parameters metadata -endif - diff --git a/tools/metadata/Makefile.am b/tools/metadata/Makefile.am deleted file mode 100644 index e2a5fd10..00000000 --- a/tools/metadata/Makefile.am +++ /dev/null @@ -1,14 +0,0 @@ -bin_PROGRAMS = lms-metadata - -lms_metadata_SOURCES = \ - $(srcdir)/LmsMetadata.cpp \ - $(top_srcdir)/src/av/AvInfo.cpp \ - $(top_srcdir)/src/metadata/AvFormat.cpp \ - $(top_srcdir)/src/metadata/TagLibParser.cpp \ - $(top_srcdir)/src/utils/Logger.cpp \ - $(top_srcdir)/src/utils/StreamLogger.cpp \ - $(top_srcdir)/src/utils/String.cpp \ - $(top_srcdir)/src/utils/UUID.cpp - -lms_metadata_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT - diff --git a/tools/similarity-parameters/Makefile.am b/tools/similarity-parameters/Makefile.am deleted file mode 100644 index 0f7cb8f8..00000000 --- a/tools/similarity-parameters/Makefile.am +++ /dev/null @@ -1,29 +0,0 @@ -noinst_PROGRAMS = lms-similarity-parameters - -lms_similarity_parameters_SOURCES = \ - $(srcdir)/LmsSimilarityParameters.cpp \ - $(top_srcdir)/src/database/Artist.cpp \ - $(top_srcdir)/src/database/Cluster.cpp \ - $(top_srcdir)/src/database/Db.cpp \ - $(top_srcdir)/src/database/TrackFeatures.cpp \ - $(top_srcdir)/src/database/TrackList.cpp \ - $(top_srcdir)/src/database/Release.cpp \ - $(top_srcdir)/src/database/ScanSettings.cpp \ - $(top_srcdir)/src/database/Session.cpp \ - $(top_srcdir)/src/database/SessionPool.cpp \ - $(top_srcdir)/src/database/SqlQuery.cpp \ - $(top_srcdir)/src/database/Track.cpp \ - $(top_srcdir)/src/database/User.cpp \ - $(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp \ - $(top_srcdir)/src/similarity/features/som/Network.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesCache.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesSearcher.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesDefs.cpp \ - $(top_srcdir)/src/utils/Config.cpp \ - $(top_srcdir)/src/utils/Logger.cpp \ - $(top_srcdir)/src/utils/Random.cpp \ - $(top_srcdir)/src/utils/StreamLogger.cpp \ - $(top_srcdir)/src/utils/String.cpp - -lms_similarity_parameters_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT - diff --git a/tools/similarity/LmsSimilarity.cpp b/tools/similarity/LmsSimilarity.cpp deleted file mode 100644 index 522b6558..00000000 --- a/tools/similarity/LmsSimilarity.cpp +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (C) 2019 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include -#include -#include -#include - -#include "database/Artist.hpp" -#include "database/Cluster.hpp" -#include "database/Db.hpp" -#include "database/Release.hpp" -#include "database/Session.hpp" -#include "database/Track.hpp" -#include "utils/Config.hpp" -#include "utils/Service.hpp" -#include "utils/StreamLogger.hpp" -#include "similarity/features/SimilarityFeaturesSearcher.hpp" - -int main(int argc, char *argv[]) -{ - try - { - using namespace Similarity; - - // log to stdout - ServiceProvider::create(std::cout); - - std::filesystem::path configFilePath {"/etc/lms.conf"}; - if (argc >= 2) - configFilePath = std::string(argv[1], 0, 256); - - ServiceProvider::create(configFilePath); - - Database::Db db {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; - Database::Session session {db}; - - std::cout << "Classifying tracks..." << std::endl; - // may be long... - struct FeaturesSearcher::TrainSettings trainSettings; - trainSettings.featureSettingsMap = FeaturesSearcher::getDefaultTrainFeatureSettings(); - FeaturesSearcher searcher {session, trainSettings}; - std::cout << "Classifying tracks DONE" << std::endl; - - const std::vector trackIds = std::invoke([&]() - { - auto transaction {session.createSharedTransaction()}; - return Database::Track::getAllIdsWithFeatures(session); - }); - - std::cout << "*** Tracks (" << trackIds.size() << ") ***" << std::endl; - for (Database::IdType trackId : trackIds) - { - auto trackToString = [&](Database::IdType trackId) - { - std::string res; - auto transaction {session.createSharedTransaction()}; - Database::Track::pointer track {Database::Track::getById(session, trackId)}; - - res += track->getName(); - if (track->getRelease()) - res += " [" + track->getRelease()->getName() + "]"; - for (auto artist : track->getArtists()) - res += " - " + artist->getName(); - for (auto cluster : track->getClusters()) - res += " {" + cluster->getType()->getName() + "-"+ cluster->getName() + "}"; - - return res; - }; - - std::cout << "Processing track '" << trackToString(trackId) << std::endl; - for (Database::IdType similarTrackId : searcher.getSimilarTracks({trackId}, 3)) - std::cout << "\t- Similar track '" << trackToString(similarTrackId) << std::endl; - } - - const std::vector releaseIds = std::invoke([&]() - { - auto transaction {session.createSharedTransaction()}; - return Database::Release::getAllIds(session); - }); - - std::cout << "*** Releases ***" << std::endl; - for (Database::IdType releaseId : releaseIds) - { - auto releaseToString = [&](Database::IdType releaseId) - { - auto transaction {session.createSharedTransaction()}; - - Database::Release::pointer release {Database::Release::getById(session, releaseId)}; - return release->getName(); - }; - - std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl; - for (Database::IdType similarReleaseId : searcher.getSimilarReleases({releaseId}, 3)) - std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl; - } - - const std::vector artistIds = std::invoke([&]() - { - auto transaction {session.createSharedTransaction()}; - return Database::Artist::getAllIds(session); - }); - - std::cout << "*** Artists ***" << std::endl; - for (Database::IdType artistId : artistIds) - { - auto artistToString = [&](Database::IdType artistId) - { - auto transaction {session.createSharedTransaction()}; - - Database::Artist::pointer artist {Database::Artist::getById(session, artistId)}; - return artist->getName(); - }; - - std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl; - for (Database::IdType similarArtistId : searcher.getSimilarArtists({artistId}, 3)) - std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl; - } - - } - catch( std::exception& e) - { - std::cerr << "Caught exception: " << e.what() << std::endl; - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; -} - diff --git a/tools/similarity/Makefile.am b/tools/similarity/Makefile.am deleted file mode 100644 index 0473f464..00000000 --- a/tools/similarity/Makefile.am +++ /dev/null @@ -1,27 +0,0 @@ -noinst_PROGRAMS = lms-similarity - -lms_similarity_SOURCES = \ - $(srcdir)/LmsSimilarity.cpp \ - $(top_srcdir)/src/database/Artist.cpp \ - $(top_srcdir)/src/database/Cluster.cpp \ - $(top_srcdir)/src/database/Db.cpp \ - $(top_srcdir)/src/database/TrackFeatures.cpp \ - $(top_srcdir)/src/database/TrackList.cpp \ - $(top_srcdir)/src/database/Release.cpp \ - $(top_srcdir)/src/database/ScanSettings.cpp \ - $(top_srcdir)/src/database/Session.cpp \ - $(top_srcdir)/src/database/SqlQuery.cpp \ - $(top_srcdir)/src/database/Track.cpp \ - $(top_srcdir)/src/database/User.cpp \ - $(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp \ - $(top_srcdir)/src/similarity/features/som/Network.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesCache.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesSearcher.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesDefs.cpp \ - $(top_srcdir)/src/utils/Config.cpp \ - $(top_srcdir)/src/utils/Logger.cpp \ - $(top_srcdir)/src/utils/StreamLogger.cpp \ - $(top_srcdir)/src/utils/String.cpp - -lms_similarity_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT -