From 876fb12fe4399d6f5e1fdb848fdb6e5835a994ad Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 13 Feb 2020 13:00:22 +0100 Subject: [PATCH 01/23] WIP --- CMakeLists.txt | 18 ++ src/CMakeLists.txt | 4 + src/Makefile.am | 165 ------------------ src/config/.empty | 0 src/cover/CoverArtGrabber.hpp | 76 -------- src/liblms/CMakeLists.txt | 73 ++++++++ .../impl}/api/subsonic/SubsonicId.cpp | 0 .../impl}/api/subsonic/SubsonicId.hpp | 0 .../impl}/api/subsonic/SubsonicResource.cpp | 40 ++--- .../impl}/api/subsonic/SubsonicResponse.cpp | 0 .../impl}/api/subsonic/SubsonicResponse.hpp | 0 .../impl}/auth/AuthTokenService.cpp | 6 + .../impl}/auth/AuthTokenService.hpp | 35 +--- src/{ => liblms/impl}/auth/LoginThrottler.cpp | 0 src/{ => liblms/impl}/auth/LoginThrottler.hpp | 0 .../impl}/auth/PasswordService.cpp | 5 + .../impl}/auth/PasswordService.hpp | 47 +---- src/{ => liblms/impl}/av/AvInfo.cpp | 8 +- src/{ => liblms/impl}/av/AvTranscoder.cpp | 8 +- src/{ => liblms/impl}/av/AvTypes.cpp | 6 +- .../impl}/cover/CoverArtGrabber.cpp | 4 +- src/liblms/impl/cover/CoverArtGrabber.hpp | 78 +++++++++ src/{image => liblms/impl/cover}/Image.cpp | 2 +- src/{image => liblms/impl/cover}/Image.hpp | 0 src/{ => liblms/impl}/database/Artist.cpp | 14 +- src/{ => liblms/impl}/database/Cluster.cpp | 12 +- src/{ => liblms/impl}/database/Db.cpp | 4 +- src/{ => liblms/impl}/database/Release.cpp | 12 +- .../impl}/database/ScanSettings.cpp | 6 +- src/{ => liblms/impl}/database/Session.cpp | 24 +-- .../impl}/database/SessionPool.cpp | 6 +- src/{ => liblms/impl}/database/SqlQuery.cpp | 0 src/{ => liblms/impl}/database/SqlQuery.hpp | 0 src/{ => liblms/impl}/database/Track.cpp | 12 +- .../impl}/database/TrackArtistLink.cpp | 8 +- .../impl}/database/TrackBookmark.cpp | 8 +- .../impl}/database/TrackFeatures.cpp | 6 +- src/{ => liblms/impl}/database/TrackList.cpp | 15 +- src/{ => liblms/impl}/database/User.cpp | 12 +- src/{ => liblms/impl}/metadata/AvFormat.cpp | 4 +- .../impl}/metadata/TagLibParser.cpp | 4 +- .../impl/recommendation/Engine.cpp} | 60 ++++--- src/liblms/impl/recommendation/Engine.hpp | 52 ++++++ .../impl/recommendation/ProviderCreator.cpp | 38 ++++ .../cluster/SimilarityClusterSearcher.cpp | 3 - .../cluster/SimilarityClusterSearcher.hpp | 0 .../features/AcousticBrainzUtils.cpp | 4 +- .../features/AcousticBrainzUtils.hpp | 0 .../features/SimilarityFeaturesCache.cpp | 6 +- .../features/SimilarityFeaturesCache.hpp | 0 .../features/SimilarityFeaturesDefs.cpp | 0 .../features/SimilarityFeaturesDefs.hpp | 0 .../SimilarityFeaturesScannerAddon.cpp | 6 +- .../features/SimilarityFeaturesSearcher.cpp | 0 .../features/SimilarityFeaturesSearcher.hpp | 0 .../features/som/DataNormalizer.cpp | 0 .../features/som/DataNormalizer.hpp | 0 .../features/som/InputVector.hpp | 0 .../recommendation}/features/som/Matrix.hpp | 0 .../recommendation}/features/som/Network.cpp | 0 .../recommendation}/features/som/Network.hpp | 0 .../impl}/scanner/MediaScanner.cpp | 3 +- .../impl}/scanner/MediaScannerStats.cpp | 2 +- src/{ => liblms/impl}/utils/Config.cpp | 24 ++- src/{ => liblms/impl}/utils/Config.hpp | 15 +- src/{ => liblms/impl}/utils/Logger.cpp | 2 +- src/{ => liblms/impl}/utils/NetAddress.cpp | 2 +- src/{ => liblms/impl}/utils/Path.cpp | 2 +- src/{ => liblms/impl}/utils/Random.cpp | 2 +- src/{ => liblms/impl}/utils/StreamLogger.cpp | 2 +- src/{ => liblms/impl}/utils/String.cpp | 8 +- src/{ => liblms/impl}/utils/UUID.cpp | 2 +- src/{ => liblms/impl}/utils/WtLogger.cpp | 4 +- .../api/subsonic/SubsonicResource.hpp | 0 src/liblms/include/auth/IAuthTokenService.hpp | 71 ++++++++ src/liblms/include/auth/IPasswordService.hpp | 59 +++++++ src/{ => liblms/include}/av/AvInfo.hpp | 0 src/{ => liblms/include}/av/AvTranscoder.hpp | 0 src/{ => liblms/include}/av/AvTypes.hpp | 0 src/liblms/include/cover/CoverArt.hpp | 40 +++++ src/liblms/include/cover/ICoverArtGrabber.hpp | 47 +++++ src/{ => liblms/include}/database/Artist.hpp | 0 src/{ => liblms/include}/database/Cluster.hpp | 0 src/{ => liblms/include}/database/Db.hpp | 0 src/{ => liblms/include}/database/Release.hpp | 0 .../include}/database/ScanSettings.hpp | 0 src/{ => liblms/include}/database/Session.hpp | 4 +- .../include}/database/SessionPool.hpp | 0 src/{ => liblms/include}/database/Track.hpp | 0 .../include}/database/TrackArtistLink.hpp | 0 .../include}/database/TrackBookmark.hpp | 0 .../include}/database/TrackFeatures.hpp | 0 .../include}/database/TrackList.hpp | 0 src/{ => liblms/include}/database/Types.hpp | 0 src/{ => liblms/include}/database/User.hpp | 0 .../include}/metadata/AvFormat.hpp | 0 .../include}/metadata/MetaData.hpp | 0 .../include}/metadata/TagLibParser.hpp | 0 .../ClustersRecommendationProviderCreator.hpp | 35 ++++ .../FeaturesRecommendationProviderCreator.hpp | 40 +++++ src/liblms/include/recommendation/IEngine.hpp | 54 ++++++ .../include/recommendation/Provider.hpp | 50 ++++++ .../include}/scanner/MediaScanner.hpp | 0 .../include}/scanner/MediaScannerAddon.hpp | 0 .../include}/scanner/MediaScannerStats.hpp | 0 src/{ => liblms/include}/utils/Exception.hpp | 0 src/liblms/include/utils/IConfig.hpp | 40 +++++ src/{ => liblms/include}/utils/Logger.hpp | 0 src/{ => liblms/include}/utils/NetAddress.hpp | 0 src/{ => liblms/include}/utils/Path.hpp | 0 src/{ => liblms/include}/utils/Random.hpp | 0 src/{ => liblms/include}/utils/Service.hpp | 9 +- .../include}/utils/StreamLogger.hpp | 0 src/{ => liblms/include}/utils/String.hpp | 4 + src/{ => liblms/include}/utils/UUID.hpp | 0 src/{ => liblms/include}/utils/Utils.hpp | 0 src/{ => liblms/include}/utils/WtLogger.hpp | 0 src/lms/CMakeLists.txt | 47 +++++ src/{main => lms}/main.cpp | 77 ++++---- src/{ => lms}/ui/Auth.cpp | 22 +-- src/{ => lms}/ui/Auth.hpp | 0 src/{ => lms}/ui/LmsApplication.cpp | 1 - src/{ => lms}/ui/LmsApplication.hpp | 0 src/{ => lms}/ui/LmsApplicationException.hpp | 0 src/{ => lms}/ui/LmsApplicationGroup.cpp | 2 +- src/{ => lms}/ui/LmsApplicationGroup.hpp | 0 src/{ => lms}/ui/MediaPlayer.cpp | 0 src/{ => lms}/ui/MediaPlayer.hpp | 0 src/{ => lms}/ui/PlayHistoryView.cpp | 0 src/{ => lms}/ui/PlayHistoryView.hpp | 0 src/{ => lms}/ui/PlayQueueView.cpp | 4 +- src/{ => lms}/ui/PlayQueueView.hpp | 0 src/{ => lms}/ui/SettingsView.cpp | 14 +- src/{ => lms}/ui/SettingsView.hpp | 0 src/{ => lms}/ui/TrackStringUtils.cpp | 0 src/{ => lms}/ui/TrackStringUtils.hpp | 0 .../ui/admin/DatabaseSettingsView.cpp | 0 .../ui/admin/DatabaseSettingsView.hpp | 0 src/{ => lms}/ui/admin/DatabaseStatus.cpp | 0 src/{ => lms}/ui/admin/DatabaseStatus.hpp | 0 src/{ => lms}/ui/admin/InitWizardView.cpp | 6 +- src/{ => lms}/ui/admin/InitWizardView.hpp | 0 src/{ => lms}/ui/admin/UserView.cpp | 10 +- src/{ => lms}/ui/admin/UserView.hpp | 0 src/{ => lms}/ui/admin/UsersView.cpp | 0 src/{ => lms}/ui/admin/UsersView.hpp | 0 src/{ => lms}/ui/common/Validators.cpp | 0 src/{ => lms}/ui/common/Validators.hpp | 0 src/{ => lms}/ui/common/ValueStringModel.hpp | 0 src/{ => lms}/ui/explore/ArtistInfoView.cpp | 4 +- src/{ => lms}/ui/explore/ArtistInfoView.hpp | 0 src/{ => lms}/ui/explore/ArtistLink.cpp | 0 src/{ => lms}/ui/explore/ArtistLink.hpp | 0 src/{ => lms}/ui/explore/ArtistView.cpp | 0 src/{ => lms}/ui/explore/ArtistView.hpp | 0 src/{ => lms}/ui/explore/ArtistsInfoView.cpp | 0 src/{ => lms}/ui/explore/ArtistsInfoView.hpp | 0 src/{ => lms}/ui/explore/ArtistsView.cpp | 0 src/{ => lms}/ui/explore/ArtistsView.hpp | 0 src/{ => lms}/ui/explore/Explore.cpp | 0 src/{ => lms}/ui/explore/Explore.hpp | 0 src/{ => lms}/ui/explore/Filters.cpp | 0 src/{ => lms}/ui/explore/Filters.hpp | 0 src/{ => lms}/ui/explore/ReleaseInfoView.cpp | 4 +- src/{ => lms}/ui/explore/ReleaseInfoView.hpp | 0 src/{ => lms}/ui/explore/ReleaseLink.cpp | 0 src/{ => lms}/ui/explore/ReleaseLink.hpp | 0 src/{ => lms}/ui/explore/ReleaseView.cpp | 0 src/{ => lms}/ui/explore/ReleaseView.hpp | 0 src/{ => lms}/ui/explore/ReleasesInfoView.cpp | 0 src/{ => lms}/ui/explore/ReleasesInfoView.hpp | 0 src/{ => lms}/ui/explore/ReleasesView.cpp | 0 src/{ => lms}/ui/explore/ReleasesView.hpp | 0 src/{ => lms}/ui/explore/TracksInfoView.cpp | 0 src/{ => lms}/ui/explore/TracksInfoView.hpp | 0 src/{ => lms}/ui/explore/TracksView.cpp | 0 src/{ => lms}/ui/explore/TracksView.hpp | 0 src/{ => lms}/ui/resource/AudioResource.cpp | 0 src/{ => lms}/ui/resource/AudioResource.hpp | 0 src/{ => lms}/ui/resource/ImageResource.cpp | 0 src/{ => lms}/ui/resource/ImageResource.hpp | 0 src/similarity/SimilaritySearcher.hpp | 52 ------ .../SimilarityFeaturesScannerAddon.hpp | 67 ------- {tools => src/tools}/Makefile.am | 0 {tools => src/tools}/metadata/LmsMetadata.cpp | 0 {tools => src/tools}/metadata/Makefile.am | 0 .../GeneticAlgorithm.hpp | 0 .../LmsSimilarityParameters.cpp | 0 .../tools}/similarity-parameters/Makefile.am | 0 .../similarity-parameters/ParallelFor.hpp | 0 .../tools}/similarity/LmsSimilarity.cpp | 0 {tools => src/tools}/similarity/Makefile.am | 0 192 files changed, 1046 insertions(+), 676 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 src/CMakeLists.txt delete mode 100644 src/config/.empty delete mode 100644 src/cover/CoverArtGrabber.hpp create mode 100644 src/liblms/CMakeLists.txt rename src/{ => liblms/impl}/api/subsonic/SubsonicId.cpp (100%) rename src/{ => liblms/impl}/api/subsonic/SubsonicId.hpp (100%) rename src/{ => liblms/impl}/api/subsonic/SubsonicResource.cpp (97%) rename src/{ => liblms/impl}/api/subsonic/SubsonicResponse.cpp (100%) rename src/{ => liblms/impl}/api/subsonic/SubsonicResponse.hpp (100%) rename src/{ => liblms/impl}/auth/AuthTokenService.cpp (95%) rename src/{ => liblms/impl}/auth/AuthTokenService.hpp (72%) rename src/{ => liblms/impl}/auth/LoginThrottler.cpp (100%) rename src/{ => liblms/impl}/auth/LoginThrottler.hpp (100%) rename src/{ => liblms/impl}/auth/PasswordService.cpp (95%) rename src/{ => liblms/impl}/auth/PasswordService.hpp (60%) rename src/{ => liblms/impl}/av/AvInfo.cpp (96%) rename src/{ => liblms/impl}/av/AvTranscoder.cpp (97%) rename src/{ => liblms/impl}/av/AvTypes.cpp (93%) rename src/{ => liblms/impl}/cover/CoverArtGrabber.cpp (98%) create mode 100644 src/liblms/impl/cover/CoverArtGrabber.hpp rename src/{image => liblms/impl/cover}/Image.cpp (99%) rename src/{image => liblms/impl/cover}/Image.hpp (100%) rename src/{ => liblms/impl}/database/Artist.cpp (98%) rename src/{ => liblms/impl}/database/Cluster.cpp (96%) rename src/{ => liblms/impl}/database/Db.cpp (96%) rename src/{ => liblms/impl}/database/Release.cpp (98%) rename src/{ => liblms/impl}/database/ScanSettings.cpp (97%) rename src/{ => liblms/impl}/database/Session.cpp (96%) rename src/{ => liblms/impl}/database/SessionPool.cpp (96%) rename src/{ => liblms/impl}/database/SqlQuery.cpp (100%) rename src/{ => liblms/impl}/database/SqlQuery.hpp (100%) rename src/{ => liblms/impl}/database/Track.cpp (98%) rename src/{ => liblms/impl}/database/TrackArtistLink.cpp (90%) rename src/{ => liblms/impl}/database/TrackBookmark.cpp (94%) rename src/{ => liblms/impl}/database/TrackFeatures.cpp (95%) rename src/{ => liblms/impl}/database/TrackList.cpp (97%) rename src/{ => liblms/impl}/database/User.cpp (97%) rename src/{ => liblms/impl}/metadata/AvFormat.cpp (98%) rename src/{ => liblms/impl}/metadata/TagLibParser.cpp (98%) rename src/{similarity/SimilaritySearcher.cpp => liblms/impl/recommendation/Engine.cpp} (67%) create mode 100644 src/liblms/impl/recommendation/Engine.hpp create mode 100644 src/liblms/impl/recommendation/ProviderCreator.cpp rename src/{similarity => liblms/impl/recommendation}/cluster/SimilarityClusterSearcher.cpp (98%) rename src/{similarity => liblms/impl/recommendation}/cluster/SimilarityClusterSearcher.hpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/AcousticBrainzUtils.cpp (91%) rename src/{similarity => liblms/impl/recommendation}/features/AcousticBrainzUtils.hpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/SimilarityFeaturesCache.cpp (96%) rename src/{similarity => liblms/impl/recommendation}/features/SimilarityFeaturesCache.hpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/SimilarityFeaturesDefs.cpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/SimilarityFeaturesDefs.hpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/SimilarityFeaturesScannerAddon.cpp (97%) rename src/{similarity => liblms/impl/recommendation}/features/SimilarityFeaturesSearcher.cpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/SimilarityFeaturesSearcher.hpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/som/DataNormalizer.cpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/som/DataNormalizer.hpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/som/InputVector.hpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/som/Matrix.hpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/som/Network.cpp (100%) rename src/{similarity => liblms/impl/recommendation}/features/som/Network.hpp (100%) rename src/{ => liblms/impl}/scanner/MediaScanner.cpp (99%) rename src/{ => liblms/impl}/scanner/MediaScannerStats.cpp (97%) rename src/{ => liblms/impl}/utils/Config.cpp (73%) rename src/{ => liblms/impl}/utils/Config.hpp (77%) rename src/{ => liblms/impl}/utils/Logger.cpp (98%) rename src/{ => liblms/impl}/utils/NetAddress.cpp (97%) rename src/{ => liblms/impl}/utils/Path.cpp (98%) rename src/{ => liblms/impl}/utils/Random.cpp (96%) rename src/{ => liblms/impl}/utils/StreamLogger.cpp (96%) rename src/{ => liblms/impl}/utils/String.cpp (97%) rename src/{ => liblms/impl}/utils/UUID.cpp (97%) rename src/{ => liblms/impl}/utils/WtLogger.cpp (94%) rename src/{ => liblms/include}/api/subsonic/SubsonicResource.hpp (100%) create mode 100644 src/liblms/include/auth/IAuthTokenService.hpp create mode 100644 src/liblms/include/auth/IPasswordService.hpp rename src/{ => liblms/include}/av/AvInfo.hpp (100%) rename src/{ => liblms/include}/av/AvTranscoder.hpp (100%) rename src/{ => liblms/include}/av/AvTypes.hpp (100%) create mode 100644 src/liblms/include/cover/CoverArt.hpp create mode 100644 src/liblms/include/cover/ICoverArtGrabber.hpp rename src/{ => liblms/include}/database/Artist.hpp (100%) rename src/{ => liblms/include}/database/Cluster.hpp (100%) rename src/{ => liblms/include}/database/Db.hpp (100%) rename src/{ => liblms/include}/database/Release.hpp (100%) rename src/{ => liblms/include}/database/ScanSettings.hpp (100%) rename src/{ => liblms/include}/database/Session.hpp (94%) rename src/{ => liblms/include}/database/SessionPool.hpp (100%) rename src/{ => liblms/include}/database/Track.hpp (100%) rename src/{ => liblms/include}/database/TrackArtistLink.hpp (100%) rename src/{ => liblms/include}/database/TrackBookmark.hpp (100%) rename src/{ => liblms/include}/database/TrackFeatures.hpp (100%) rename src/{ => liblms/include}/database/TrackList.hpp (100%) rename src/{ => liblms/include}/database/Types.hpp (100%) rename src/{ => liblms/include}/database/User.hpp (100%) rename src/{ => liblms/include}/metadata/AvFormat.hpp (100%) rename src/{ => liblms/include}/metadata/MetaData.hpp (100%) rename src/{ => liblms/include}/metadata/TagLibParser.hpp (100%) create mode 100644 src/liblms/include/recommendation/ClustersRecommendationProviderCreator.hpp create mode 100644 src/liblms/include/recommendation/FeaturesRecommendationProviderCreator.hpp create mode 100644 src/liblms/include/recommendation/IEngine.hpp create mode 100644 src/liblms/include/recommendation/Provider.hpp rename src/{ => liblms/include}/scanner/MediaScanner.hpp (100%) rename src/{ => liblms/include}/scanner/MediaScannerAddon.hpp (100%) rename src/{ => liblms/include}/scanner/MediaScannerStats.hpp (100%) rename src/{ => liblms/include}/utils/Exception.hpp (100%) create mode 100644 src/liblms/include/utils/IConfig.hpp rename src/{ => liblms/include}/utils/Logger.hpp (100%) rename src/{ => liblms/include}/utils/NetAddress.hpp (100%) rename src/{ => liblms/include}/utils/Path.hpp (100%) rename src/{ => liblms/include}/utils/Random.hpp (100%) rename src/{ => liblms/include}/utils/Service.hpp (92%) rename src/{ => liblms/include}/utils/StreamLogger.hpp (100%) rename src/{ => liblms/include}/utils/String.hpp (97%) rename src/{ => liblms/include}/utils/UUID.hpp (100%) rename src/{ => liblms/include}/utils/Utils.hpp (100%) rename src/{ => liblms/include}/utils/WtLogger.hpp (100%) create mode 100644 src/lms/CMakeLists.txt rename src/{main => lms}/main.cpp (63%) rename src/{ => lms}/ui/Auth.cpp (85%) rename src/{ => lms}/ui/Auth.hpp (100%) rename src/{ => lms}/ui/LmsApplication.cpp (99%) rename src/{ => lms}/ui/LmsApplication.hpp (100%) rename src/{ => lms}/ui/LmsApplicationException.hpp (100%) rename src/{ => lms}/ui/LmsApplicationGroup.cpp (97%) rename src/{ => lms}/ui/LmsApplicationGroup.hpp (100%) rename src/{ => lms}/ui/MediaPlayer.cpp (100%) rename src/{ => lms}/ui/MediaPlayer.hpp (100%) rename src/{ => lms}/ui/PlayHistoryView.cpp (100%) rename src/{ => lms}/ui/PlayHistoryView.hpp (100%) rename src/{ => lms}/ui/PlayQueueView.cpp (98%) rename src/{ => lms}/ui/PlayQueueView.hpp (100%) rename src/{ => lms}/ui/SettingsView.cpp (94%) rename src/{ => lms}/ui/SettingsView.hpp (100%) rename src/{ => lms}/ui/TrackStringUtils.cpp (100%) rename src/{ => lms}/ui/TrackStringUtils.hpp (100%) rename src/{ => lms}/ui/admin/DatabaseSettingsView.cpp (100%) rename src/{ => lms}/ui/admin/DatabaseSettingsView.hpp (100%) rename src/{ => lms}/ui/admin/DatabaseStatus.cpp (100%) rename src/{ => lms}/ui/admin/DatabaseStatus.hpp (100%) rename src/{ => lms}/ui/admin/InitWizardView.cpp (93%) rename src/{ => lms}/ui/admin/InitWizardView.hpp (100%) rename src/{ => lms}/ui/admin/UserView.cpp (95%) rename src/{ => lms}/ui/admin/UserView.hpp (100%) rename src/{ => lms}/ui/admin/UsersView.cpp (100%) rename src/{ => lms}/ui/admin/UsersView.hpp (100%) rename src/{ => lms}/ui/common/Validators.cpp (100%) rename src/{ => lms}/ui/common/Validators.hpp (100%) rename src/{ => lms}/ui/common/ValueStringModel.hpp (100%) rename src/{ => lms}/ui/explore/ArtistInfoView.cpp (93%) rename src/{ => lms}/ui/explore/ArtistInfoView.hpp (100%) rename src/{ => lms}/ui/explore/ArtistLink.cpp (100%) rename src/{ => lms}/ui/explore/ArtistLink.hpp (100%) rename src/{ => lms}/ui/explore/ArtistView.cpp (100%) rename src/{ => lms}/ui/explore/ArtistView.hpp (100%) rename src/{ => lms}/ui/explore/ArtistsInfoView.cpp (100%) rename src/{ => lms}/ui/explore/ArtistsInfoView.hpp (100%) rename src/{ => lms}/ui/explore/ArtistsView.cpp (100%) rename src/{ => lms}/ui/explore/ArtistsView.hpp (100%) rename src/{ => lms}/ui/explore/Explore.cpp (100%) rename src/{ => lms}/ui/explore/Explore.hpp (100%) rename src/{ => lms}/ui/explore/Filters.cpp (100%) rename src/{ => lms}/ui/explore/Filters.hpp (100%) rename src/{ => lms}/ui/explore/ReleaseInfoView.cpp (95%) rename src/{ => lms}/ui/explore/ReleaseInfoView.hpp (100%) rename src/{ => lms}/ui/explore/ReleaseLink.cpp (100%) rename src/{ => lms}/ui/explore/ReleaseLink.hpp (100%) rename src/{ => lms}/ui/explore/ReleaseView.cpp (100%) rename src/{ => lms}/ui/explore/ReleaseView.hpp (100%) rename src/{ => lms}/ui/explore/ReleasesInfoView.cpp (100%) rename src/{ => lms}/ui/explore/ReleasesInfoView.hpp (100%) rename src/{ => lms}/ui/explore/ReleasesView.cpp (100%) rename src/{ => lms}/ui/explore/ReleasesView.hpp (100%) rename src/{ => lms}/ui/explore/TracksInfoView.cpp (100%) rename src/{ => lms}/ui/explore/TracksInfoView.hpp (100%) rename src/{ => lms}/ui/explore/TracksView.cpp (100%) rename src/{ => lms}/ui/explore/TracksView.hpp (100%) rename src/{ => lms}/ui/resource/AudioResource.cpp (100%) rename src/{ => lms}/ui/resource/AudioResource.hpp (100%) rename src/{ => lms}/ui/resource/ImageResource.cpp (100%) rename src/{ => lms}/ui/resource/ImageResource.hpp (100%) delete mode 100644 src/similarity/SimilaritySearcher.hpp delete mode 100644 src/similarity/features/SimilarityFeaturesScannerAddon.hpp rename {tools => src/tools}/Makefile.am (100%) rename {tools => src/tools}/metadata/LmsMetadata.cpp (100%) rename {tools => src/tools}/metadata/Makefile.am (100%) rename {tools => src/tools}/similarity-parameters/GeneticAlgorithm.hpp (100%) rename {tools => src/tools}/similarity-parameters/LmsSimilarityParameters.cpp (100%) rename {tools => src/tools}/similarity-parameters/Makefile.am (100%) rename {tools => src/tools}/similarity-parameters/ParallelFor.hpp (100%) rename {tools => src/tools}/similarity/LmsSimilarity.cpp (100%) rename {tools => src/tools}/similarity/Makefile.am (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..7eef4529 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.10) + +project(lms) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +find_package(PkgConfig) +pkg_check_modules(IMAGEMAGICKXX REQUIRED ImageMagick++) + + +SUBDIRS(src) + +install(DIRECTORY approot DESTINATION share/lms) +install(DIRECTORY docroot DESTINATION share/lms) +install(FILES systemd/default.service DESTINATION share/lms) +install(FILES conf/lms.conf DESTINATION share/lms) + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 00000000..8853a8c7 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,4 @@ +add_compile_options(-Wall -Wextra -pedantic -Werror) + +SUBDIRS(liblms lms) + diff --git a/src/Makefile.am b/src/Makefile.am index 4fd65f58..b5644219 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,171 +1,6 @@ bin_PROGRAMS = lms lms_SOURCES = \ - $(srcdir)/api/subsonic/SubsonicId.cpp \ - $(srcdir)/api/subsonic/SubsonicId.hpp \ - $(srcdir)/api/subsonic/SubsonicResource.cpp \ - $(srcdir)/api/subsonic/SubsonicResource.hpp \ - $(srcdir)/api/subsonic/SubsonicResponse.cpp \ - $(srcdir)/api/subsonic/SubsonicResponse.hpp \ - $(srcdir)/auth/AuthTokenService.cpp \ - $(srcdir)/auth/AuthTokenService.hpp \ - $(srcdir)/auth/PasswordService.cpp \ - $(srcdir)/auth/PasswordService.hpp \ - $(srcdir)/auth/LoginThrottler.cpp \ - $(srcdir)/auth/LoginThrottler.hpp \ - $(srcdir)/av/AvInfo.cpp \ - $(srcdir)/av/AvInfo.hpp \ - $(srcdir)/av/AvTranscoder.cpp \ - $(srcdir)/av/AvTranscoder.hpp \ - $(srcdir)/av/AvTypes.cpp \ - $(srcdir)/av/AvTypes.hpp \ - $(srcdir)/cover/CoverArtGrabber.cpp \ - $(srcdir)/cover/CoverArtGrabber.hpp \ - $(srcdir)/database/Artist.cpp \ - $(srcdir)/database/Artist.hpp \ - $(srcdir)/database/Cluster.cpp \ - $(srcdir)/database/Cluster.hpp \ - $(srcdir)/database/Db.cpp \ - $(srcdir)/database/Db.hpp \ - $(srcdir)/database/TrackArtistLink.cpp \ - $(srcdir)/database/TrackArtistLink.hpp \ - $(srcdir)/database/TrackFeatures.cpp \ - $(srcdir)/database/TrackFeatures.hpp \ - $(srcdir)/database/TrackList.cpp \ - $(srcdir)/database/TrackList.hpp \ - $(srcdir)/database/Types.hpp \ - $(srcdir)/database/Release.cpp \ - $(srcdir)/database/Release.hpp \ - $(srcdir)/database/ScanSettings.cpp \ - $(srcdir)/database/ScanSettings.hpp \ - $(srcdir)/database/Session.cpp \ - $(srcdir)/database/Session.hpp \ - $(srcdir)/database/SessionPool.cpp \ - $(srcdir)/database/SessionPool.hpp \ - $(srcdir)/database/SqlQuery.cpp \ - $(srcdir)/database/SqlQuery.hpp \ - $(srcdir)/database/Track.cpp \ - $(srcdir)/database/Track.hpp \ - $(srcdir)/database/TrackBookmark.cpp \ - $(srcdir)/database/TrackBookmark.hpp \ - $(srcdir)/database/User.cpp \ - $(srcdir)/database/User.hpp \ - $(srcdir)/image/Image.cpp \ - $(srcdir)/image/Image.hpp \ - $(srcdir)/main/main.cpp \ - $(srcdir)/metadata/AvFormat.cpp \ - $(srcdir)/metadata/AvFormat.hpp \ - $(srcdir)/metadata/MetaData.hpp \ - $(srcdir)/metadata/TagLibParser.cpp \ - $(srcdir)/metadata/TagLibParser.hpp \ - $(srcdir)/scanner/MediaScanner.cpp \ - $(srcdir)/scanner/MediaScanner.hpp \ - $(srcdir)/scanner/MediaScannerStats.cpp \ - $(srcdir)/scanner/MediaScannerStats.hpp \ - $(srcdir)/scanner/MediaScannerAddon.hpp \ - $(srcdir)/similarity/SimilaritySearcher.cpp \ - $(srcdir)/similarity/SimilaritySearcher.hpp \ - $(srcdir)/similarity/cluster/SimilarityClusterSearcher.cpp \ - $(srcdir)/similarity/cluster/SimilarityClusterSearcher.hpp \ - $(srcdir)/similarity/features/AcousticBrainzUtils.cpp \ - $(srcdir)/similarity/features/AcousticBrainzUtils.hpp \ - $(srcdir)/similarity/features/SimilarityFeaturesCache.cpp \ - $(srcdir)/similarity/features/SimilarityFeaturesCache.hpp \ - $(srcdir)/similarity/features/SimilarityFeaturesDefs.cpp \ - $(srcdir)/similarity/features/SimilarityFeaturesDefs.hpp \ - $(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.cpp \ - $(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.hpp \ - $(srcdir)/similarity/features/SimilarityFeaturesSearcher.cpp \ - $(srcdir)/similarity/features/SimilarityFeaturesSearcher.hpp \ - $(srcdir)/similarity/features/som/DataNormalizer.cpp \ - $(srcdir)/similarity/features/som/DataNormalizer.hpp \ - $(srcdir)/similarity/features/som/InputVector.hpp \ - $(srcdir)/similarity/features/som/Matrix.hpp \ - $(srcdir)/similarity/features/som/Network.cpp \ - $(srcdir)/similarity/features/som/Network.hpp \ - $(srcdir)/ui/Auth.cpp \ - $(srcdir)/ui/Auth.hpp \ - $(srcdir)/ui/LmsApplication.cpp \ - $(srcdir)/ui/LmsApplication.hpp \ - $(srcdir)/ui/LmsApplicationException.hpp \ - $(srcdir)/ui/LmsApplicationGroup.cpp \ - $(srcdir)/ui/LmsApplicationGroup.hpp \ - $(srcdir)/ui/MediaPlayer.cpp \ - $(srcdir)/ui/MediaPlayer.hpp \ - $(srcdir)/ui/PlayQueueView.cpp \ - $(srcdir)/ui/PlayQueueView.hpp \ - $(srcdir)/ui/PlayHistoryView.cpp \ - $(srcdir)/ui/PlayHistoryView.hpp \ - $(srcdir)/ui/SettingsView.cpp \ - $(srcdir)/ui/SettingsView.hpp \ - $(srcdir)/ui/TrackStringUtils.cpp \ - $(srcdir)/ui/TrackStringUtils.hpp \ - $(srcdir)/ui/admin/DatabaseSettingsView.cpp \ - $(srcdir)/ui/admin/DatabaseSettingsView.hpp \ - $(srcdir)/ui/admin/DatabaseStatus.cpp \ - $(srcdir)/ui/admin/DatabaseStatus.hpp \ - $(srcdir)/ui/admin/InitWizardView.cpp \ - $(srcdir)/ui/admin/InitWizardView.hpp \ - $(srcdir)/ui/admin/UserView.cpp \ - $(srcdir)/ui/admin/UserView.hpp \ - $(srcdir)/ui/admin/UsersView.cpp \ - $(srcdir)/ui/admin/UsersView.hpp \ - $(srcdir)/ui/common/Validators.cpp \ - $(srcdir)/ui/common/Validators.hpp \ - $(srcdir)/ui/common/ValueStringModel.hpp \ - $(srcdir)/ui/explore/ArtistInfoView.cpp \ - $(srcdir)/ui/explore/ArtistInfoView.hpp \ - $(srcdir)/ui/explore/ArtistLink.cpp \ - $(srcdir)/ui/explore/ArtistLink.hpp \ - $(srcdir)/ui/explore/ArtistsInfoView.cpp \ - $(srcdir)/ui/explore/ArtistsInfoView.hpp \ - $(srcdir)/ui/explore/ArtistView.cpp \ - $(srcdir)/ui/explore/ArtistView.hpp \ - $(srcdir)/ui/explore/ArtistsView.cpp \ - $(srcdir)/ui/explore/ArtistsView.hpp \ - $(srcdir)/ui/explore/Explore.cpp \ - $(srcdir)/ui/explore/Explore.hpp \ - $(srcdir)/ui/explore/Filters.cpp \ - $(srcdir)/ui/explore/Filters.hpp \ - $(srcdir)/ui/explore/ReleaseInfoView.cpp \ - $(srcdir)/ui/explore/ReleaseInfoView.hpp \ - $(srcdir)/ui/explore/ReleaseLink.cpp \ - $(srcdir)/ui/explore/ReleaseLink.hpp \ - $(srcdir)/ui/explore/ReleasesInfoView.cpp \ - $(srcdir)/ui/explore/ReleasesInfoView.hpp \ - $(srcdir)/ui/explore/ReleasesView.cpp \ - $(srcdir)/ui/explore/ReleasesView.hpp \ - $(srcdir)/ui/explore/ReleaseView.cpp \ - $(srcdir)/ui/explore/ReleaseView.hpp \ - $(srcdir)/ui/explore/TracksInfoView.cpp \ - $(srcdir)/ui/explore/TracksInfoView.hpp \ - $(srcdir)/ui/explore/TracksView.cpp \ - $(srcdir)/ui/explore/TracksView.hpp \ - $(srcdir)/ui/resource/ImageResource.cpp \ - $(srcdir)/ui/resource/ImageResource.hpp \ - $(srcdir)/ui/resource/AudioResource.cpp \ - $(srcdir)/ui/resource/AudioResource.hpp \ - $(srcdir)/utils/Config.cpp \ - $(srcdir)/utils/Config.hpp \ - $(srcdir)/utils/Exception.hpp \ - $(srcdir)/utils/Logger.cpp \ - $(srcdir)/utils/Logger.hpp \ - $(srcdir)/utils/NetAddress.cpp \ - $(srcdir)/utils/NetAddress.hpp \ - $(srcdir)/utils/Path.cpp \ - $(srcdir)/utils/Path.hpp \ - $(srcdir)/utils/Random.cpp \ - $(srcdir)/utils/Random.hpp \ - $(srcdir)/utils/Service.hpp \ - $(srcdir)/utils/StreamLogger.cpp \ - $(srcdir)/utils/StreamLogger.hpp \ - $(srcdir)/utils/String.cpp \ - $(srcdir)/utils/String.hpp \ - $(srcdir)/utils/Utils.hpp \ - $(srcdir)/utils/UUID.cpp \ - $(srcdir)/utils/UUID.hpp \ - $(srcdir)/utils/WtLogger.cpp \ - $(srcdir)/utils/WtLogger.hpp lms_CXXFLAGS=-std=c++17 -I$(srcdir)/ui $(MAGICKXX_CFLAGS) -D_REENTRANT lms_LDADD=$(MAGICKXX_LIBS) diff --git a/src/config/.empty b/src/config/.empty deleted file mode 100644 index e69de29b..00000000 diff --git a/src/cover/CoverArtGrabber.hpp b/src/cover/CoverArtGrabber.hpp deleted file mode 100644 index 07b4cc32..00000000 --- a/src/cover/CoverArtGrabber.hpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2015 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include -#include -#include -#include -#include - -#include "database/Types.hpp" - -#include "image/Image.hpp" - -namespace Database { - class Session; -} - -namespace CoverArt { - -class Grabber -{ - public: - Grabber(); - Grabber(const Grabber&) = delete; - Grabber& operator=(const Grabber&) = delete; - Grabber(Grabber&&) = delete; - Grabber& operator=(Grabber&&) = delete; - - void setDefaultCover(const std::filesystem::path& defaultCoverPath); - - std::vector getFromTrack(Database::Session& dbSession, Database::IdType trackId, Image::Format format, std::size_t size); - std::vector getFromRelease(Database::Session& dbSession, Database::IdType releaseId, Image::Format format, std::size_t size); - - private: - - Image::Image getFromTrack(Database::Session& dbSession, Database::IdType trackId, std::size_t size); - Image::Image getFromRelease(Database::Session& dbSession, Database::IdType releaseId, std::size_t size); - - std::optional getFromTrack(const std::filesystem::path& path) const; - std::vector getCoverPaths(const std::filesystem::path& directoryPath) const; - std::optional getFromDirectory(const std::filesystem::path& path) const; - - Image::Image getDefaultCover(std::size_t size); - - Image::Image _defaultCover; - - std::mutex _mutex; - std::map _defaultCovers; - - static inline const std::vector _fileExtensions {".jpg", ".jpeg", ".png", ".bmp"}; // TODO parametrize - - static inline const std::size_t _maxFileSize {10000000}; - - static inline const std::vector _preferredFileNames {"cover", "front"}; // TODO parametrize -}; - -} // namespace CoverArt - diff --git a/src/liblms/CMakeLists.txt b/src/liblms/CMakeLists.txt new file mode 100644 index 00000000..37ba7cfd --- /dev/null +++ b/src/liblms/CMakeLists.txt @@ -0,0 +1,73 @@ + +add_library(liblms SHARED + impl/api/subsonic/SubsonicId.cpp + impl/api/subsonic/SubsonicResource.cpp + impl/api/subsonic/SubsonicResponse.cpp + impl/auth/AuthTokenService.cpp + impl/auth/PasswordService.cpp + impl/auth/LoginThrottler.cpp + impl/av/AvInfo.cpp + impl/av/AvTranscoder.cpp + impl/av/AvTypes.cpp + impl/cover/CoverArtGrabber.cpp + impl/database/Artist.cpp + impl/database/Cluster.cpp + impl/database/Db.cpp + impl/database/TrackArtistLink.cpp + impl/database/TrackFeatures.cpp + impl/database/TrackList.cpp + impl/database/Release.cpp + impl/database/ScanSettings.cpp + impl/database/Session.cpp + impl/database/SessionPool.cpp + impl/database/SqlQuery.cpp + impl/database/Track.cpp + impl/database/TrackBookmark.cpp + impl/database/User.cpp + impl/metadata/AvFormat.cpp + impl/metadata/TagLibParser.cpp + impl/scanner/MediaScanner.cpp + impl/scanner/MediaScannerStats.cpp + impl/recommendation/Engine.cpp + impl/recommendation/ProviderCreator.cpp + impl/recommendation/features/som/DataNormalizer.cpp + impl/recommendation/features/som/Network.cpp + impl/utils/Config.cpp + impl/utils/Logger.cpp + impl/utils/NetAddress.cpp + impl/utils/Path.cpp + impl/utils/Random.cpp + impl/utils/StreamLogger.cpp + impl/utils/String.cpp + impl/utils/UUID.cpp + impl/utils/WtLogger.cpp + ) + +include_directories(liblms include/) + +target_include_directories(liblms INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ) + +# TODO make this private +target_include_directories(liblms PUBLIC ${IMAGEMAGICKXX_INCLUDE_DIRS}) +target_link_libraries(liblms PUBLIC ${IMAGEMAGICKXX_LIBRARIES}) +target_compile_options(liblms PUBLIC ${IMAGEMAGICKXX_CFLAGS_OTHER}) + +target_link_libraries(liblms PRIVATE + avformat + avutil + config++ + tag + stdc++fs + wtdbosqlite3 + ) + +target_link_libraries(liblms PUBLIC + pthread + boost_system + wtdbo + ) + +install(TARGETS liblms DESTINATION lib) + diff --git a/src/api/subsonic/SubsonicId.cpp b/src/liblms/impl/api/subsonic/SubsonicId.cpp similarity index 100% rename from src/api/subsonic/SubsonicId.cpp rename to src/liblms/impl/api/subsonic/SubsonicId.cpp diff --git a/src/api/subsonic/SubsonicId.hpp b/src/liblms/impl/api/subsonic/SubsonicId.hpp similarity index 100% rename from src/api/subsonic/SubsonicId.hpp rename to src/liblms/impl/api/subsonic/SubsonicId.hpp diff --git a/src/api/subsonic/SubsonicResource.cpp b/src/liblms/impl/api/subsonic/SubsonicResource.cpp similarity index 97% rename from src/api/subsonic/SubsonicResource.cpp rename to src/liblms/impl/api/subsonic/SubsonicResource.cpp index a52f5d22..c2e72223 100644 --- a/src/api/subsonic/SubsonicResource.cpp +++ b/src/liblms/impl/api/subsonic/SubsonicResource.cpp @@ -16,7 +16,7 @@ * You should have received a copy of the GNU General Public License * along with LMS. If not, see . */ -#include "SubsonicResource.hpp" +#include "api/subsonic/SubsonicResource.hpp" #include #include @@ -26,9 +26,9 @@ #include -#include "auth/PasswordService.hpp" +#include "auth/IPasswordService.hpp" #include "av/AvTranscoder.hpp" -#include "cover/CoverArtGrabber.hpp" +#include "cover/ICoverArtGrabber.hpp" #include "database/Artist.hpp" #include "database/Cluster.hpp" #include "database/Db.hpp" @@ -38,7 +38,7 @@ #include "database/TrackBookmark.hpp" #include "database/TrackList.hpp" #include "database/User.hpp" -#include "similarity/SimilaritySearcher.hpp" +#include "recommendation/IEngine.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" #include "utils/String.hpp" @@ -578,10 +578,10 @@ handleChangePassword(RequestContext& context) std::string username {getMandatoryParameterAs(context.parameters, "username")}; std::string password {decodePasswordIfNeeded(getMandatoryParameterAs(context.parameters, "password"))}; - if (!ServiceProvider::get()->evaluatePasswordStrength(username, password)) + if (!ServiceProvider::get()->evaluatePasswordStrength(username, password)) throw PasswordTooWeakGenericError {}; - const User::PasswordHash hash {ServiceProvider::get()->hashPassword(password)}; + const User::PasswordHash hash {ServiceProvider::get()->hashPassword(password)}; auto transaction {context.dbSession.createUniqueTransaction()}; @@ -660,10 +660,10 @@ handleCreateUserRequest(RequestContext& context) std::string password {decodePasswordIfNeeded(getMandatoryParameterAs(context.parameters, "password"))}; // Just ignore all the other fields as we don't handle them - if (!ServiceProvider::get()->evaluatePasswordStrength(username, password)) + if (!ServiceProvider::get()->evaluatePasswordStrength(username, password)) throw PasswordTooWeakGenericError {}; - const User::PasswordHash hash {ServiceProvider::get()->hashPassword(password)}; + const User::PasswordHash hash {ServiceProvider::get()->hashPassword(password)}; auto transaction {context.dbSession.createUniqueTransaction()}; @@ -944,7 +944,7 @@ handleGetArtistInfoRequestCommon(RequestContext& context, bool id3) artistInfoNode.createChild("musicBrainzId").setValue(artistMBID->getAsString()); } - auto similarArtistsId {ServiceProvider::get()->getSimilarArtists(context.dbSession, id.value, count)}; + auto similarArtistsId {ServiceProvider::get()->getSimilarArtists(context.dbSession, id.value, count)}; { auto transaction {context.dbSession.createSharedTransaction()}; @@ -1140,7 +1140,7 @@ handleGetSimilarSongsRequestCommon(RequestContext& context, bool id3) // Optional params std::size_t count {getParameterAs(context.parameters, "count").value_or(50)}; - auto similarArtistsId {ServiceProvider::get()->getSimilarArtists(context.dbSession, id.value, 5)}; + auto similarArtistsId {ServiceProvider::get()->getSimilarArtists(context.dbSession, id.value, 5)}; auto transaction {context.dbSession.createSharedTransaction()}; @@ -1588,10 +1588,10 @@ handleUpdateUserRequest(RequestContext& context) if (password) { *password = decodePasswordIfNeeded(*password); - if (!ServiceProvider::get()->evaluatePasswordStrength(username, *password)) + if (!ServiceProvider::get()->evaluatePasswordStrength(username, *password)) throw PasswordTooWeakGenericError {}; - hash = ServiceProvider::get()->hashPassword(*password); + hash = ServiceProvider::get()->hashPassword(*password); } auto transaction {context.dbSession.createUniqueTransaction()}; @@ -1895,16 +1895,16 @@ handleGetCoverArt(RequestContext& context, Wt::Http::ResponseContinuation*) switch (id.type) { case Id::Type::Track: - res.data = ServiceProvider::get()->getFromTrack(context.dbSession, id.value, Image::Format::JPEG, size); + res.data = ServiceProvider::get()->getFromTrack(context.dbSession, id.value, CoverArt::Format::JPEG, {size, size}); break; case Id::Type::Release: - res.data = ServiceProvider::get()->getFromRelease(context.dbSession, id.value, Image::Format::JPEG, size); + res.data = ServiceProvider::get()->getFromRelease(context.dbSession, id.value, CoverArt::Format::JPEG, {size, size}); break; default: throw BadParameterGenericError {"id"}; } - res.mimeType = Image::format_to_mimeType(Image::Format::JPEG); + res.mimeType = CoverArt::formatToMimeType(CoverArt::Format::JPEG); return res; } @@ -2056,19 +2056,19 @@ SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Resp SessionPool::ScopedSession dbSession {_sessionPool}; - switch (ServiceProvider::get()->checkUserPassword(dbSession.get(), + switch (ServiceProvider::get()->checkUserPassword(dbSession.get(), boost::asio::ip::address::from_string(request.clientAddress()), clientInfo.user, clientInfo.password)) { - case Auth::PasswordService::PasswordCheckResult::Match: + case Auth::IPasswordService::PasswordCheckResult::Match: break; - case Auth::PasswordService::PasswordCheckResult::Mismatch: + case Auth::IPasswordService::PasswordCheckResult::Mismatch: throw WrongUsernameOrPasswordError {}; - case Auth::PasswordService::PasswordCheckResult::Throttled: + case Auth::IPasswordService::PasswordCheckResult::Throttled: throw LoginThrottledGenericError {}; } - RequestContext requestContext {.parameters = parameters, .dbSession = dbSession.get(), .userName = clientInfo.user}; + RequestContext requestContext {parameters, dbSession.get(), clientInfo.user}; auto itEntryPoint {requestEntryPoints.find(requestPath)}; if (itEntryPoint != requestEntryPoints.end()) diff --git a/src/api/subsonic/SubsonicResponse.cpp b/src/liblms/impl/api/subsonic/SubsonicResponse.cpp similarity index 100% rename from src/api/subsonic/SubsonicResponse.cpp rename to src/liblms/impl/api/subsonic/SubsonicResponse.cpp diff --git a/src/api/subsonic/SubsonicResponse.hpp b/src/liblms/impl/api/subsonic/SubsonicResponse.hpp similarity index 100% rename from src/api/subsonic/SubsonicResponse.hpp rename to src/liblms/impl/api/subsonic/SubsonicResponse.hpp diff --git a/src/auth/AuthTokenService.cpp b/src/liblms/impl/auth/AuthTokenService.cpp similarity index 95% rename from src/auth/AuthTokenService.cpp rename to src/liblms/impl/auth/AuthTokenService.cpp index 14cb933c..ba37b35c 100644 --- a/src/auth/AuthTokenService.cpp +++ b/src/liblms/impl/auth/AuthTokenService.cpp @@ -26,11 +26,17 @@ #include #include "database/Session.hpp" +#include "database/User.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" namespace Auth { +std::unique_ptr createAuthTokenService(std::size_t maxThrottlerEntries) +{ + return std::make_unique(maxThrottlerEntries); +} + static const Wt::Auth::SHA1HashFunction sha1Function; AuthTokenService::AuthTokenService(std::size_t maxThrottlerEntries) diff --git a/src/auth/AuthTokenService.hpp b/src/liblms/impl/auth/AuthTokenService.hpp similarity index 72% rename from src/auth/AuthTokenService.hpp rename to src/liblms/impl/auth/AuthTokenService.hpp index b44ae3e7..36a9721a 100644 --- a/src/auth/AuthTokenService.hpp +++ b/src/liblms/impl/auth/AuthTokenService.hpp @@ -21,14 +21,9 @@ #pragma once -#include -#include - -#include +#include "auth/IAuthTokenService.hpp" #include "LoginThrottler.hpp" -#include "database/User.hpp" -#include "database/Types.hpp" namespace Database { @@ -38,7 +33,7 @@ namespace Database namespace Auth { - class AuthTokenService + class AuthTokenService : public IAuthTokenService { public: @@ -52,30 +47,8 @@ namespace Auth { AuthTokenService(AuthTokenService&&) = delete; AuthTokenService& operator=(AuthTokenService&&) = delete; - - // Auth Token services - struct AuthTokenProcessResult - { - enum class State - { - Found, - Throttled, - NotFound, - }; - - struct AuthTokenInfo - { - Database::IdType userId; - Wt::WDateTime expiry; - }; - - State state {State::NotFound}; - std::optional authTokenInfo {}; - }; - - // Removed if found - AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue); - std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry); + AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue) override; + std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry) override; private: diff --git a/src/auth/LoginThrottler.cpp b/src/liblms/impl/auth/LoginThrottler.cpp similarity index 100% rename from src/auth/LoginThrottler.cpp rename to src/liblms/impl/auth/LoginThrottler.cpp diff --git a/src/auth/LoginThrottler.hpp b/src/liblms/impl/auth/LoginThrottler.hpp similarity index 100% rename from src/auth/LoginThrottler.hpp rename to src/liblms/impl/auth/LoginThrottler.hpp diff --git a/src/auth/PasswordService.cpp b/src/liblms/impl/auth/PasswordService.cpp similarity index 95% rename from src/auth/PasswordService.cpp rename to src/liblms/impl/auth/PasswordService.cpp index 1c592f45..9c17de2b 100644 --- a/src/auth/PasswordService.cpp +++ b/src/liblms/impl/auth/PasswordService.cpp @@ -31,6 +31,11 @@ namespace Auth { +std::unique_ptr createPasswordService(std::size_t maxThrottlerEntries) +{ + return std::make_unique(maxThrottlerEntries); +} + PasswordService::PasswordService(std::size_t maxThrottlerEntries) : _loginThrottler{maxThrottlerEntries} { diff --git a/src/auth/PasswordService.hpp b/src/liblms/impl/auth/PasswordService.hpp similarity index 60% rename from src/auth/PasswordService.hpp rename to src/liblms/impl/auth/PasswordService.hpp index 303615be..99377e8b 100644 --- a/src/auth/PasswordService.hpp +++ b/src/liblms/impl/auth/PasswordService.hpp @@ -17,17 +17,12 @@ * along with LMS. If not, see . */ -/* This file contains some classes in order to get info from file using the libavconv */ - #pragma once -#include - -#include +#include +#include "auth/IPasswordService.hpp" #include "LoginThrottler.hpp" -#include "database/User.hpp" -#include "database/Types.hpp" namespace Database { @@ -37,7 +32,7 @@ namespace Database namespace Auth { - class PasswordService + class PasswordService : public IPasswordService { public: @@ -53,39 +48,9 @@ namespace Auth { // Password services - enum class PasswordCheckResult - { - Match, - Mismatch, - Throttled, - }; - PasswordCheckResult checkUserPassword(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& loginName, const std::string& password); - Database::User::PasswordHash hashPassword(const std::string& password) const; - bool evaluatePasswordStrength(const std::string& loginName, const std::string& password) const; - - // Auth Token services - struct AuthTokenProcessResult - { - enum class State - { - Found, - Throttled, - NotFound, - }; - - struct AuthTokenInfo - { - Database::IdType userId; - Wt::WDateTime expiry; - }; - - State state; - std::optional authTokenInfo; - }; - - // Removed if found - AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue); - std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry); + PasswordCheckResult checkUserPassword(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& loginName, const std::string& password) override; + Database::User::PasswordHash hashPassword(const std::string& password) const override; + bool evaluatePasswordStrength(const std::string& loginName, const std::string& password) const override; private: diff --git a/src/av/AvInfo.cpp b/src/liblms/impl/av/AvInfo.cpp similarity index 96% rename from src/av/AvInfo.cpp rename to src/liblms/impl/av/AvInfo.cpp index 2f7d9ee8..e9b1bd8c 100644 --- a/src/av/AvInfo.cpp +++ b/src/liblms/impl/av/AvInfo.cpp @@ -17,9 +17,7 @@ * along with LMS. If not, see . */ -#include "AvInfo.hpp" - -#include +#include "av/AvInfo.hpp" #include @@ -92,7 +90,7 @@ getMetaDataFromDictionnary(AVDictionary* dictionnary, std::map(tag->key)] = tag->value; + res[StringUtils::stringToUpper(tag->key)] = tag->value; } } @@ -141,7 +139,7 @@ MediaFile::getStreamInfo() const if (avstream->codecpar->codec_type != AVMEDIA_TYPE_AUDIO) continue; - res.push_back( {.id = i, .bitrate = static_cast(avstream->codecpar->bit_rate)} ); + res.push_back( {i, static_cast(avstream->codecpar->bit_rate)} ); } return res; diff --git a/src/av/AvTranscoder.cpp b/src/liblms/impl/av/AvTranscoder.cpp similarity index 97% rename from src/av/AvTranscoder.cpp rename to src/liblms/impl/av/AvTranscoder.cpp index 1dac0797..f98b3771 100644 --- a/src/av/AvTranscoder.cpp +++ b/src/liblms/impl/av/AvTranscoder.cpp @@ -17,13 +17,13 @@ * along with LMS. If not, see . */ -#include "AvTranscoder.hpp" +#include "av/AvTranscoder.hpp" #include #include -#include "AvInfo.hpp" -#include "utils/Config.hpp" +#include "av/AvInfo.hpp" +#include "utils/IConfig.hpp" #include "utils/Path.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -38,7 +38,7 @@ static std::filesystem::path ffmpegPath; void Transcoder::init() { - ffmpegPath = ServiceProvider::get()->getPath("ffmpeg-file", "/usr/bin/ffmpeg"); + ffmpegPath = ServiceProvider::get()->getPath("ffmpeg-file", "/usr/bin/ffmpeg"); if (!std::filesystem::exists(ffmpegPath)) throw LmsException {"File '" + ffmpegPath.string() + "' does not exist!"}; } diff --git a/src/av/AvTypes.cpp b/src/liblms/impl/av/AvTypes.cpp similarity index 93% rename from src/av/AvTypes.cpp rename to src/liblms/impl/av/AvTypes.cpp index 9ad5bd5d..7e556294 100644 --- a/src/av/AvTypes.cpp +++ b/src/liblms/impl/av/AvTypes.cpp @@ -17,9 +17,7 @@ * along with LMS. If not, see . */ -#include "AvTypes.hpp" - -#include +#include "av/AvTypes.hpp" namespace Av { @@ -34,7 +32,7 @@ const char* encodingToMimetype(Encoding encoding) case Encoding::WEBM_VORBIS: return "audio/webm"; } - throw AvException("Invalid encoding"); + throw AvException {"Invalid encoding"}; } } diff --git a/src/cover/CoverArtGrabber.cpp b/src/liblms/impl/cover/CoverArtGrabber.cpp similarity index 98% rename from src/cover/CoverArtGrabber.cpp rename to src/liblms/impl/cover/CoverArtGrabber.cpp index 8809a6c1..110e9ddd 100644 --- a/src/cover/CoverArtGrabber.cpp +++ b/src/liblms/impl/cover/CoverArtGrabber.cpp @@ -236,7 +236,7 @@ Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, } std::vector -Grabber::getFromTrack(Database::Session& session, Database::IdType trackId, Image::Format format, std::size_t size) +Grabber::getFromTrack(Database::Session& session, Database::IdType trackId, Format format, std::size_t size) { const Image::Image cover {getFromTrack(session, trackId, size)}; @@ -245,7 +245,7 @@ Grabber::getFromTrack(Database::Session& session, Database::IdType trackId, Imag } std::vector -Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, Image::Format format, std::size_t size) +Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, Format format, std::size_t size) { const Image::Image cover {getFromRelease(session, releaseId, size)}; diff --git a/src/liblms/impl/cover/CoverArtGrabber.hpp b/src/liblms/impl/cover/CoverArtGrabber.hpp new file mode 100644 index 00000000..3911ad43 --- /dev/null +++ b/src/liblms/impl/cover/CoverArtGrabber.hpp @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2015 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "cover/ICoverArtGrabber.hpp" +#include "database/Types.hpp" +#include "Image.hpp" + +namespace Database +{ + class Session; +} + +namespace CoverArt +{ + + class Grabber : public IGrabber + { + public: + Grabber(); + Grabber(const Grabber&) = delete; + Grabber& operator=(const Grabber&) = delete; + Grabber(Grabber&&) = delete; + Grabber& operator=(Grabber&&) = delete; + + void setDefaultCover(const std::filesystem::path& defaultCoverPath) override; + + std::vector getFromTrack(Database::Session& dbSession, Database::IdType trackId, Format format, std::size_t width) override; + std::vector getFromRelease(Database::Session& dbSession, Database::IdType releaseId, Format format, std::size_t width) override; + + private: + + Image::Image getFromTrack(Database::Session& dbSession, Database::IdType trackId, std::size_t size); + Image::Image getFromRelease(Database::Session& dbSession, Database::IdType releaseId, std::size_t size); + + std::optional getFromTrack(const std::filesystem::path& path) const; + std::vector getCoverPaths(const std::filesystem::path& directoryPath) const; + std::optional getFromDirectory(const std::filesystem::path& path) const; + + Image::Image getDefaultCover(std::size_t size); + + Image::Image _defaultCover; + + std::mutex _mutex; + std::map _defaultCovers; + + static inline const std::vector _fileExtensions {".jpg", ".jpeg", ".png", ".bmp"}; // TODO parametrize + + static inline const std::size_t _maxFileSize {10000000}; + + static inline const std::vector _preferredFileNames {"cover", "front"}; // TODO parametrize + }; + +} // namespace CoverArt + diff --git a/src/image/Image.cpp b/src/liblms/impl/cover/Image.cpp similarity index 99% rename from src/image/Image.cpp rename to src/liblms/impl/cover/Image.cpp index 2d50bebb..06532eca 100644 --- a/src/image/Image.cpp +++ b/src/liblms/impl/cover/Image.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "Image.hpp" +#include "image/Image.hpp" #include "utils/Logger.hpp" diff --git a/src/image/Image.hpp b/src/liblms/impl/cover/Image.hpp similarity index 100% rename from src/image/Image.hpp rename to src/liblms/impl/cover/Image.hpp diff --git a/src/database/Artist.cpp b/src/liblms/impl/database/Artist.cpp similarity index 98% rename from src/database/Artist.cpp rename to src/liblms/impl/database/Artist.cpp index 88ee6cb1..5d17df1a 100644 --- a/src/database/Artist.cpp +++ b/src/liblms/impl/database/Artist.cpp @@ -16,18 +16,18 @@ * You should have received a copy of the GNU General Public License * along with LMS. If not, see . */ -#include "Artist.hpp" +#include "database/Artist.hpp" #include +#include "database/Cluster.hpp" +#include "database/Release.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "database/User.hpp" #include "utils/Logger.hpp" - -#include "Cluster.hpp" -#include "Release.hpp" #include "SqlQuery.hpp" -#include "Session.hpp" -#include "Track.hpp" -#include "User.hpp" + namespace Database { diff --git a/src/database/Cluster.cpp b/src/liblms/impl/database/Cluster.cpp similarity index 96% rename from src/database/Cluster.cpp rename to src/liblms/impl/database/Cluster.cpp index ddfafb43..66e95c12 100644 --- a/src/database/Cluster.cpp +++ b/src/liblms/impl/database/Cluster.cpp @@ -17,14 +17,14 @@ * along with LMS. If not, see . */ -#include "Cluster.hpp" +#include "database/Cluster.hpp" -#include "Artist.hpp" -#include "Release.hpp" -#include "ScanSettings.hpp" -#include "Session.hpp" +#include "database/Artist.hpp" +#include "database/Release.hpp" +#include "database/ScanSettings.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" #include "SqlQuery.hpp" -#include "Track.hpp" namespace Database { diff --git a/src/database/Db.cpp b/src/liblms/impl/database/Db.cpp similarity index 96% rename from src/database/Db.cpp rename to src/liblms/impl/database/Db.cpp index 23edca35..87bdf5d7 100644 --- a/src/database/Db.cpp +++ b/src/liblms/impl/database/Db.cpp @@ -17,13 +17,13 @@ * along with LMS. If not, see . */ -#include "Db.hpp" +#include "database/Db.hpp" #include #include +#include "database/User.hpp" #include "utils/Logger.hpp" -#include "User.hpp" namespace Database { diff --git a/src/database/Release.cpp b/src/liblms/impl/database/Release.cpp similarity index 98% rename from src/database/Release.cpp rename to src/liblms/impl/database/Release.cpp index a4857f15..343a7ce1 100644 --- a/src/database/Release.cpp +++ b/src/liblms/impl/database/Release.cpp @@ -17,16 +17,16 @@ * along with LMS. If not, see . */ -#include "Release.hpp" +#include "database/Release.hpp" #include "utils/Logger.hpp" -#include "Artist.hpp" -#include "Cluster.hpp" -#include "Session.hpp" +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "database/User.hpp" #include "SqlQuery.hpp" -#include "Track.hpp" -#include "User.hpp" namespace Database { diff --git a/src/database/ScanSettings.cpp b/src/liblms/impl/database/ScanSettings.cpp similarity index 97% rename from src/database/ScanSettings.cpp rename to src/liblms/impl/database/ScanSettings.cpp index 1c024353..673e2a24 100644 --- a/src/database/ScanSettings.cpp +++ b/src/liblms/impl/database/ScanSettings.cpp @@ -17,15 +17,15 @@ * along with LMS. If not, see . */ -#include "ScanSettings.hpp" +#include "database/ScanSettings.hpp" #include #include "utils/Logger.hpp" #include "utils/String.hpp" -#include "Cluster.hpp" -#include "Session.hpp" +#include "database/Cluster.hpp" +#include "database/Session.hpp" namespace { diff --git a/src/database/Session.cpp b/src/liblms/impl/database/Session.cpp similarity index 96% rename from src/database/Session.cpp rename to src/liblms/impl/database/Session.cpp index 0c3aab24..36750ec7 100644 --- a/src/database/Session.cpp +++ b/src/liblms/impl/database/Session.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "Session.hpp" +#include "database/Session.hpp" #include #include @@ -26,17 +26,17 @@ #include "utils/Exception.hpp" #include "utils/Logger.hpp" -#include "Artist.hpp" -#include "Cluster.hpp" -#include "Db.hpp" -#include "Release.hpp" -#include "ScanSettings.hpp" -#include "Track.hpp" -#include "TrackBookmark.hpp" -#include "TrackArtistLink.hpp" -#include "TrackList.hpp" -#include "TrackFeatures.hpp" -#include "User.hpp" +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Db.hpp" +#include "database/Release.hpp" +#include "database/ScanSettings.hpp" +#include "database/Track.hpp" +#include "database/TrackBookmark.hpp" +#include "database/TrackArtistLink.hpp" +#include "database/TrackList.hpp" +#include "database/TrackFeatures.hpp" +#include "database/User.hpp" namespace Database { diff --git a/src/database/SessionPool.cpp b/src/liblms/impl/database/SessionPool.cpp similarity index 96% rename from src/database/SessionPool.cpp rename to src/liblms/impl/database/SessionPool.cpp index f6526aca..ff6b1111 100644 --- a/src/database/SessionPool.cpp +++ b/src/liblms/impl/database/SessionPool.cpp @@ -17,13 +17,13 @@ * along with LMS. If not, see . */ -#include "SessionPool.hpp" +#include "database/SessionPool.hpp" + +#include "database/Session.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" -#include "Session.hpp" - namespace Database { SessionPool::SessionPool(Db& database, std::size_t maxSessionCount) diff --git a/src/database/SqlQuery.cpp b/src/liblms/impl/database/SqlQuery.cpp similarity index 100% rename from src/database/SqlQuery.cpp rename to src/liblms/impl/database/SqlQuery.cpp diff --git a/src/database/SqlQuery.hpp b/src/liblms/impl/database/SqlQuery.hpp similarity index 100% rename from src/database/SqlQuery.hpp rename to src/liblms/impl/database/SqlQuery.hpp diff --git a/src/database/Track.cpp b/src/liblms/impl/database/Track.cpp similarity index 98% rename from src/database/Track.cpp rename to src/liblms/impl/database/Track.cpp index 65503bbf..a84af8e1 100644 --- a/src/database/Track.cpp +++ b/src/liblms/impl/database/Track.cpp @@ -17,17 +17,17 @@ * along with LMS. If not, see . */ -#include "Track.hpp" +#include "database/Track.hpp" #include +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Release.hpp" +#include "database/TrackFeatures.hpp" +#include "database/Session.hpp" #include "utils/Logger.hpp" -#include "Artist.hpp" -#include "Cluster.hpp" -#include "Release.hpp" -#include "TrackFeatures.hpp" -#include "Session.hpp" #include "SqlQuery.hpp" namespace Database { diff --git a/src/database/TrackArtistLink.cpp b/src/liblms/impl/database/TrackArtistLink.cpp similarity index 90% rename from src/database/TrackArtistLink.cpp rename to src/liblms/impl/database/TrackArtistLink.cpp index 64ce9281..57526eab 100644 --- a/src/database/TrackArtistLink.cpp +++ b/src/liblms/impl/database/TrackArtistLink.cpp @@ -17,11 +17,11 @@ * along with LMS. If not, see . */ -#include "TrackArtistLink.hpp" +#include "database/TrackArtistLink.hpp" -#include "Artist.hpp" -#include "Session.hpp" -#include "Track.hpp" +#include "database/Artist.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" namespace Database { diff --git a/src/database/TrackBookmark.cpp b/src/liblms/impl/database/TrackBookmark.cpp similarity index 94% rename from src/database/TrackBookmark.cpp rename to src/liblms/impl/database/TrackBookmark.cpp index 1307f0a7..2a0c4bef 100644 --- a/src/database/TrackBookmark.cpp +++ b/src/liblms/impl/database/TrackBookmark.cpp @@ -17,11 +17,11 @@ * along with LMS. If not, see . */ -#include "TrackBookmark.hpp" +#include "database/TrackBookmark.hpp" -#include "Session.hpp" -#include "Track.hpp" -#include "User.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "database/User.hpp" namespace Database { diff --git a/src/database/TrackFeatures.cpp b/src/liblms/impl/database/TrackFeatures.cpp similarity index 95% rename from src/database/TrackFeatures.cpp rename to src/liblms/impl/database/TrackFeatures.cpp index 6a812bc9..6fb3c963 100644 --- a/src/database/TrackFeatures.cpp +++ b/src/liblms/impl/database/TrackFeatures.cpp @@ -17,14 +17,14 @@ * along with LMS. If not, see . */ -#include "TrackFeatures.hpp" +#include "database/TrackFeatures.hpp" #include #include +#include "database/Session.hpp" +#include "database/Track.hpp" #include "utils/Logger.hpp" -#include "Session.hpp" -#include "Track.hpp" namespace Database { diff --git a/src/database/TrackList.cpp b/src/liblms/impl/database/TrackList.cpp similarity index 97% rename from src/database/TrackList.cpp rename to src/liblms/impl/database/TrackList.cpp index d63312ca..a7fc145c 100644 --- a/src/database/TrackList.cpp +++ b/src/liblms/impl/database/TrackList.cpp @@ -16,19 +16,18 @@ * You should have received a copy of the GNU General Public License * along with LMS. If not, see . */ -#include "TrackList.hpp" +#include "database/TrackList.hpp" #include -#include #include "utils/Logger.hpp" -#include "Artist.hpp" -#include "Cluster.hpp" -#include "Release.hpp" -#include "Session.hpp" -#include "User.hpp" -#include "Track.hpp" +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Release.hpp" +#include "database/Session.hpp" +#include "database/User.hpp" +#include "database/Track.hpp" namespace Database { diff --git a/src/database/User.cpp b/src/liblms/impl/database/User.cpp similarity index 97% rename from src/database/User.cpp rename to src/liblms/impl/database/User.cpp index 2f0e52ef..6b980130 100644 --- a/src/database/User.cpp +++ b/src/liblms/impl/database/User.cpp @@ -17,14 +17,14 @@ * along with LMS. If not, see . */ -#include "User.hpp" +#include "database/User.hpp" +#include "database/Artist.hpp" +#include "database/Release.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "database/TrackList.hpp" #include "utils/Logger.hpp" -#include "Artist.hpp" -#include "Release.hpp" -#include "Session.hpp" -#include "Track.hpp" -#include "TrackList.hpp" namespace Database { diff --git a/src/metadata/AvFormat.cpp b/src/liblms/impl/metadata/AvFormat.cpp similarity index 98% rename from src/metadata/AvFormat.cpp rename to src/liblms/impl/metadata/AvFormat.cpp index 38790589..85311c7a 100644 --- a/src/metadata/AvFormat.cpp +++ b/src/liblms/impl/metadata/AvFormat.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "AvFormat.hpp" +#include "metadata/AvFormat.hpp" #include #include @@ -142,7 +142,7 @@ AvFormat::parse(const std::filesystem::path& p, bool debug) for (auto stream : mediaFile.getStreamInfo()) { - MetaData::AudioStream audioStream {.bitRate = static_cast(stream.bitrate)}; + MetaData::AudioStream audioStream {static_cast(stream.bitrate)}; track.audioStreams.emplace_back(audioStream); } } diff --git a/src/metadata/TagLibParser.cpp b/src/liblms/impl/metadata/TagLibParser.cpp similarity index 98% rename from src/metadata/TagLibParser.cpp rename to src/liblms/impl/metadata/TagLibParser.cpp index 5f430e14..78949ce8 100644 --- a/src/metadata/TagLibParser.cpp +++ b/src/liblms/impl/metadata/TagLibParser.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "TagLibParser.hpp" +#include "metadata/TagLibParser.hpp" #include #include @@ -180,7 +180,7 @@ TagLibParser::parse(const std::filesystem::path& p, bool debug) track.duration = std::chrono::milliseconds {properties->length() * 1000}; - MetaData::AudioStream audioStream {.bitRate = static_cast(properties->bitrate() * 1000)}; + MetaData::AudioStream audioStream {static_cast(properties->bitrate() * 1000)}; track.audioStreams = {std::move(audioStream)}; } diff --git a/src/similarity/SimilaritySearcher.cpp b/src/liblms/impl/recommendation/Engine.cpp similarity index 67% rename from src/similarity/SimilaritySearcher.cpp rename to src/liblms/impl/recommendation/Engine.cpp index cf2513d0..a2c61a61 100644 --- a/src/similarity/SimilaritySearcher.cpp +++ b/src/liblms/impl/recommendation/Engine.cpp @@ -17,31 +17,38 @@ * along with LMS. If not, see . */ -#include "SimilaritySearcher.hpp" +#include "Engine.hpp" -#include "features/SimilarityFeaturesScannerAddon.hpp" -#include "cluster/SimilarityClusterSearcher.hpp" +//#include "features/SimilarityFeaturesScannerAddon.hpp" +//#include "cluster/SimilarityClusterSearcher.hpp" #include "database/ScanSettings.hpp" #include "database/TrackList.hpp" -namespace Similarity { +namespace Recommendation { -Searcher::Searcher(FeaturesScannerAddon& somAddon) -: _somAddon(somAddon) -{} - -static -Database::ScanSettings::SimilarityEngineType -getEngineType(Database::Session& dbSession) +std::unique_ptr +createEngine() { - auto transaction {dbSession.createSharedTransaction()}; - return Database::ScanSettings::get(dbSession)->getSimilarityEngineType(); + return std::make_unique(); +} + +void +Engine::clearProviders() +{ + _providers.clear(); +} + +void +Engine::addProvider(std::unique_ptr provider, unsigned priority) +{ + _providers.emplace(priority, std::move(provider)); } std::vector -Searcher::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) +Engine::getSimilarTracksFromTrackList(Database::Session& /*session*/, Database::IdType /*trackListId*/, std::size_t /*maxCount*/) { +#if 0 auto engineType {getEngineType(session)}; auto somSearcher {_somAddon.getSearcher()}; @@ -66,12 +73,15 @@ Searcher::getSimilarTracksFromTrackList(Database::Session& session, Database::Id return somSearcher->getSimilarTracks(trackIds, maxCount); } else - return ClusterSearcher::getSimilarTracksFromTrackList(session, trackListId, maxCount); + return ClusterEngine::getSimilarTracksFromTrackList(session, trackListId, maxCount); +#endif + return {}; } std::vector -Searcher::getSimilarTracks(Database::Session& dbSession, const std::set& trackIds, std::size_t maxCount) +Engine::getSimilarTracks(Database::Session& /*dbSession*/, const std::unordered_set& /*trackIds*/, std::size_t /*maxCount*/) { +#if 0 auto engineType {getEngineType(dbSession)}; auto somSearcher {_somAddon.getSearcher()}; @@ -82,12 +92,15 @@ Searcher::getSimilarTracks(Database::Session& dbSession, const std::setgetSimilarTracks(trackIds, maxCount); } else - return ClusterSearcher::getSimilarTracks(dbSession, trackIds, maxCount); + return ClusterEngine::getSimilarTracks(dbSession, trackIds, maxCount); +#endif + return {}; } std::vector -Searcher::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) +Engine::getSimilarReleases(Database::Session& /*dbSession*/, Database::IdType /*releaseId*/, std::size_t /*maxCount*/) { +#if 0 auto engineType {getEngineType(dbSession)}; auto somSearcher {_somAddon.getSearcher()}; @@ -98,12 +111,15 @@ Searcher::getSimilarReleases(Database::Session& dbSession, Database::IdType rele return somSearcher->getSimilarReleases(releaseId, maxCount); } else - return ClusterSearcher::getSimilarReleases(dbSession, releaseId, maxCount); + return ClusterEngine::getSimilarReleases(dbSession, releaseId, maxCount); +#endif + return {}; } std::vector -Searcher::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) +Engine::getSimilarArtists(Database::Session& /*dbSession*/, Database::IdType /*artistId*/, std::size_t /*maxCount*/) { +#if 0 auto engineType {getEngineType(dbSession)}; auto somSearcher {_somAddon.getSearcher()}; @@ -114,7 +130,9 @@ Searcher::getSimilarArtists(Database::Session& dbSession, Database::IdType artis return somSearcher->getSimilarArtists(artistId, maxCount); } else - return ClusterSearcher::getSimilarArtists(dbSession, artistId, maxCount); + return ClusterEngine::getSimilarArtists(dbSession, artistId, maxCount); +#endif + return {}; } } // ns Similarity diff --git a/src/liblms/impl/recommendation/Engine.hpp b/src/liblms/impl/recommendation/Engine.hpp new file mode 100644 index 00000000..92dacc08 --- /dev/null +++ b/src/liblms/impl/recommendation/Engine.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +#include "recommendation/IEngine.hpp" +#include "recommendation/Provider.hpp" + +namespace Database +{ + class Session; +} + +namespace Recommendation +{ + class Engine : public IEngine + { + public: + void clearProviders() override; + void addProvider(std::unique_ptr provider, unsigned priority) override; + + // Closest results first + std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override; + std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) override; + std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override; + std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override; + + private: + + std::map> _providers; + }; + +} // ns Recommendation + diff --git a/src/liblms/impl/recommendation/ProviderCreator.cpp b/src/liblms/impl/recommendation/ProviderCreator.cpp new file mode 100644 index 00000000..b451e480 --- /dev/null +++ b/src/liblms/impl/recommendation/ProviderCreator.cpp @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2020 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "recommendation/ClustersRecommendationProviderCreator.hpp" +#include "recommendation/FeaturesRecommendationProviderCreator.hpp" +#include "recommendation/Provider.hpp" + +namespace Recommendation +{ + + std::unique_ptr createClustersRecommendationProvider() + { + return {}; + } + + std::unique_ptr createFeaturesRecommendationProvider(Scanner::MediaScanner&) + { + return {}; + } + +} + diff --git a/src/similarity/cluster/SimilarityClusterSearcher.cpp b/src/liblms/impl/recommendation/cluster/SimilarityClusterSearcher.cpp similarity index 98% rename from src/similarity/cluster/SimilarityClusterSearcher.cpp rename to src/liblms/impl/recommendation/cluster/SimilarityClusterSearcher.cpp index 037dd3c4..7b24b5e4 100644 --- a/src/similarity/cluster/SimilarityClusterSearcher.cpp +++ b/src/liblms/impl/recommendation/cluster/SimilarityClusterSearcher.cpp @@ -19,9 +19,6 @@ #include "SimilarityClusterSearcher.hpp" -#include -#include - #include "database/Artist.hpp" #include "database/Cluster.hpp" #include "database/Release.hpp" diff --git a/src/similarity/cluster/SimilarityClusterSearcher.hpp b/src/liblms/impl/recommendation/cluster/SimilarityClusterSearcher.hpp similarity index 100% rename from src/similarity/cluster/SimilarityClusterSearcher.hpp rename to src/liblms/impl/recommendation/cluster/SimilarityClusterSearcher.hpp diff --git a/src/similarity/features/AcousticBrainzUtils.cpp b/src/liblms/impl/recommendation/features/AcousticBrainzUtils.cpp similarity index 91% rename from src/similarity/features/AcousticBrainzUtils.cpp rename to src/liblms/impl/recommendation/features/AcousticBrainzUtils.cpp index 7e460782..b33f5c4f 100644 --- a/src/similarity/features/AcousticBrainzUtils.cpp +++ b/src/liblms/impl/recommendation/features/AcousticBrainzUtils.cpp @@ -25,7 +25,7 @@ #include #include -#include "utils/Config.hpp" +#include "utils/IConfig.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -39,7 +39,7 @@ getJsonData(const UUID& mbid) { static const std::string defaultAPIURL = "https://acousticbrainz.org/api/v1/"; - const std::string url {ServiceProvider::get()->getString("acousticbrainz-api-url", defaultAPIURL) + std::string {mbid.getAsString()} + "/low-level"}; + const std::string url {ServiceProvider::get()->getString("acousticbrainz-api-url", defaultAPIURL) + std::string {mbid.getAsString()} + "/low-level"}; boost::asio::io_service ioService; diff --git a/src/similarity/features/AcousticBrainzUtils.hpp b/src/liblms/impl/recommendation/features/AcousticBrainzUtils.hpp similarity index 100% rename from src/similarity/features/AcousticBrainzUtils.hpp rename to src/liblms/impl/recommendation/features/AcousticBrainzUtils.hpp diff --git a/src/similarity/features/SimilarityFeaturesCache.cpp b/src/liblms/impl/recommendation/features/SimilarityFeaturesCache.cpp similarity index 96% rename from src/similarity/features/SimilarityFeaturesCache.cpp rename to src/liblms/impl/recommendation/features/SimilarityFeaturesCache.cpp index 1f81c010..235ed331 100644 --- a/src/similarity/features/SimilarityFeaturesCache.cpp +++ b/src/liblms/impl/recommendation/features/SimilarityFeaturesCache.cpp @@ -22,7 +22,7 @@ #include #include -#include "utils/Config.hpp" +#include "utils/IConfig.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -32,7 +32,7 @@ namespace Similarity { static std::filesystem::path getCacheDirectory() { - return ServiceProvider::get()->getPath("working-dir") / "cache" / "features"; + return ServiceProvider::get()->getPath("working-dir") / "cache" / "features"; } static std::filesystem::path getCacheNetworkFilePath() @@ -242,7 +242,7 @@ FeaturesCache::read() void FeaturesCache::write() { - std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir") / "cache" / "features"); + std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir") / "cache" / "features"); if (!networkToCacheFile(_network, getCacheNetworkFilePath()) || !objectPositionToCacheFile(_trackPositions, getCacheTrackPositionsFilePath())) diff --git a/src/similarity/features/SimilarityFeaturesCache.hpp b/src/liblms/impl/recommendation/features/SimilarityFeaturesCache.hpp similarity index 100% rename from src/similarity/features/SimilarityFeaturesCache.hpp rename to src/liblms/impl/recommendation/features/SimilarityFeaturesCache.hpp diff --git a/src/similarity/features/SimilarityFeaturesDefs.cpp b/src/liblms/impl/recommendation/features/SimilarityFeaturesDefs.cpp similarity index 100% rename from src/similarity/features/SimilarityFeaturesDefs.cpp rename to src/liblms/impl/recommendation/features/SimilarityFeaturesDefs.cpp diff --git a/src/similarity/features/SimilarityFeaturesDefs.hpp b/src/liblms/impl/recommendation/features/SimilarityFeaturesDefs.hpp similarity index 100% rename from src/similarity/features/SimilarityFeaturesDefs.hpp rename to src/liblms/impl/recommendation/features/SimilarityFeaturesDefs.hpp diff --git a/src/similarity/features/SimilarityFeaturesScannerAddon.cpp b/src/liblms/impl/recommendation/features/SimilarityFeaturesScannerAddon.cpp similarity index 97% rename from src/similarity/features/SimilarityFeaturesScannerAddon.cpp rename to src/liblms/impl/recommendation/features/SimilarityFeaturesScannerAddon.cpp index 20ecf2d3..620cedc8 100644 --- a/src/similarity/features/SimilarityFeaturesScannerAddon.cpp +++ b/src/liblms/impl/recommendation/features/SimilarityFeaturesScannerAddon.cpp @@ -17,16 +17,16 @@ * along with LMS. If not, see . */ -#include "SimilarityFeaturesScannerAddon.hpp" +#include "similarity/features/SimilarityFeaturesScannerAddon.hpp" #include "AcousticBrainzUtils.hpp" #include "database/ScanSettings.hpp" #include "database/Track.hpp" #include "database/TrackFeatures.hpp" -#include "similarity/features/SimilarityFeaturesCache.hpp" -#include "utils/Config.hpp" #include "utils/Logger.hpp" +#include "SimilarityFeaturesCache.hpp" + namespace Similarity { static diff --git a/src/similarity/features/SimilarityFeaturesSearcher.cpp b/src/liblms/impl/recommendation/features/SimilarityFeaturesSearcher.cpp similarity index 100% rename from src/similarity/features/SimilarityFeaturesSearcher.cpp rename to src/liblms/impl/recommendation/features/SimilarityFeaturesSearcher.cpp diff --git a/src/similarity/features/SimilarityFeaturesSearcher.hpp b/src/liblms/impl/recommendation/features/SimilarityFeaturesSearcher.hpp similarity index 100% rename from src/similarity/features/SimilarityFeaturesSearcher.hpp rename to src/liblms/impl/recommendation/features/SimilarityFeaturesSearcher.hpp diff --git a/src/similarity/features/som/DataNormalizer.cpp b/src/liblms/impl/recommendation/features/som/DataNormalizer.cpp similarity index 100% rename from src/similarity/features/som/DataNormalizer.cpp rename to src/liblms/impl/recommendation/features/som/DataNormalizer.cpp diff --git a/src/similarity/features/som/DataNormalizer.hpp b/src/liblms/impl/recommendation/features/som/DataNormalizer.hpp similarity index 100% rename from src/similarity/features/som/DataNormalizer.hpp rename to src/liblms/impl/recommendation/features/som/DataNormalizer.hpp diff --git a/src/similarity/features/som/InputVector.hpp b/src/liblms/impl/recommendation/features/som/InputVector.hpp similarity index 100% rename from src/similarity/features/som/InputVector.hpp rename to src/liblms/impl/recommendation/features/som/InputVector.hpp diff --git a/src/similarity/features/som/Matrix.hpp b/src/liblms/impl/recommendation/features/som/Matrix.hpp similarity index 100% rename from src/similarity/features/som/Matrix.hpp rename to src/liblms/impl/recommendation/features/som/Matrix.hpp diff --git a/src/similarity/features/som/Network.cpp b/src/liblms/impl/recommendation/features/som/Network.cpp similarity index 100% rename from src/similarity/features/som/Network.cpp rename to src/liblms/impl/recommendation/features/som/Network.cpp diff --git a/src/similarity/features/som/Network.hpp b/src/liblms/impl/recommendation/features/som/Network.hpp similarity index 100% rename from src/similarity/features/som/Network.hpp rename to src/liblms/impl/recommendation/features/som/Network.hpp diff --git a/src/scanner/MediaScanner.cpp b/src/liblms/impl/scanner/MediaScanner.cpp similarity index 99% rename from src/scanner/MediaScanner.cpp rename to src/liblms/impl/scanner/MediaScanner.cpp index fb835855..a88893e3 100644 --- a/src/scanner/MediaScanner.cpp +++ b/src/liblms/impl/scanner/MediaScanner.cpp @@ -17,13 +17,12 @@ * along with LMS. If not, see . */ -#include "MediaScanner.hpp" +#include "scanner/MediaScanner.hpp" #include #include -#include "cover/CoverArtGrabber.hpp" #include "database/Artist.hpp" #include "database/Cluster.hpp" #include "database/Release.hpp" diff --git a/src/scanner/MediaScannerStats.cpp b/src/liblms/impl/scanner/MediaScannerStats.cpp similarity index 97% rename from src/scanner/MediaScannerStats.cpp rename to src/liblms/impl/scanner/MediaScannerStats.cpp index f2f991f5..a2a984da 100644 --- a/src/scanner/MediaScannerStats.cpp +++ b/src/liblms/impl/scanner/MediaScannerStats.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "MediaScannerStats.hpp" +#include "scanner/MediaScannerStats.hpp" namespace Scanner { diff --git a/src/utils/Config.cpp b/src/liblms/impl/utils/Config.cpp similarity index 73% rename from src/utils/Config.cpp rename to src/liblms/impl/utils/Config.cpp index 40855b64..ff8c4846 100644 --- a/src/utils/Config.cpp +++ b/src/liblms/impl/utils/Config.cpp @@ -19,16 +19,36 @@ #include "Config.hpp" +#include "utils/Exception.hpp" #include "utils/Logger.hpp" +std::unique_ptr createConfig(const std::filesystem::path& p) +{ + return std::make_unique(p); +} Config::Config(const std::filesystem::path& p) { - _config.readFile(p.string().c_str()); + try + { + _config.readFile(p.string().c_str()); + } + catch( libconfig::FileIOException& e) + { + throw LmsException {"Cannot open config file '" + p.string() + "'"}; + } + catch( libconfig::ParseException& e) + { + throw LmsException {"Cannot parse config file '" + p.string() + "', line = " + std::to_string(e.getLine()) + ", error = '" + e.getError() + "'"}; + } + catch (libconfig::ConfigException& e) + { + throw LmsException {"Cannot open config file '" + p.string() + "': " + e.what()}; + } } std::string -Config::getString(const std::string& setting, const std::string& def, const std::set& allowedValues) +Config::getString(const std::string& setting, const std::string& def, const std::unordered_set& allowedValues) { try { std::string res {(const char*)_config.lookup(setting)}; diff --git a/src/utils/Config.hpp b/src/liblms/impl/utils/Config.hpp similarity index 77% rename from src/utils/Config.hpp rename to src/liblms/impl/utils/Config.hpp index 0b248e20..e1e9600e 100644 --- a/src/utils/Config.hpp +++ b/src/liblms/impl/utils/Config.hpp @@ -18,13 +18,12 @@ */ #pragma once -#include -#include +#include "utils/IConfig.hpp" #include // Used to get config values from configuration files -class Config final +class Config final : public IConfig { public: Config(const std::filesystem::path& p); @@ -36,11 +35,11 @@ class Config final Config& operator=(Config&&) = delete; // Default values are returned in case of setting not found - std::string getString(const std::string& setting, const std::string& def = "", const std::set& allowedValues = {}); - std::filesystem::path getPath(const std::string& setting, const std::filesystem::path& def = std::filesystem::path()); - unsigned long getULong(const std::string& setting, unsigned long def = 0); - long getLong(const std::string& setting, long def = 0); - bool getBool(const std::string& setting, bool def = false); + std::string getString(const std::string& setting, const std::string& def = "", const std::unordered_set& allowedValues = {}) override; + std::filesystem::path getPath(const std::string& setting, const std::filesystem::path& def = std::filesystem::path()) override; + unsigned long getULong(const std::string& setting, unsigned long def = 0) override; + long getLong(const std::string& setting, long def = 0) override; + bool getBool(const std::string& setting, bool def = false) override; private: diff --git a/src/utils/Logger.cpp b/src/liblms/impl/utils/Logger.cpp similarity index 98% rename from src/utils/Logger.cpp rename to src/liblms/impl/utils/Logger.cpp index 79bfafe9..b0d99437 100644 --- a/src/utils/Logger.cpp +++ b/src/liblms/impl/utils/Logger.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "Logger.hpp" +#include "utils/Logger.hpp" const char* getModuleName(Module mod) { diff --git a/src/utils/NetAddress.cpp b/src/liblms/impl/utils/NetAddress.cpp similarity index 97% rename from src/utils/NetAddress.cpp rename to src/liblms/impl/utils/NetAddress.cpp index 723de502..cede3029 100644 --- a/src/utils/NetAddress.cpp +++ b/src/liblms/impl/utils/NetAddress.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "NetAddress.hpp" +#include "utils/NetAddress.hpp" namespace std { diff --git a/src/utils/Path.cpp b/src/liblms/impl/utils/Path.cpp similarity index 98% rename from src/utils/Path.cpp rename to src/liblms/impl/utils/Path.cpp index dd2c4b21..a5c0d882 100644 --- a/src/utils/Path.cpp +++ b/src/liblms/impl/utils/Path.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "Path.hpp" +#include "utils/Path.hpp" #include #include diff --git a/src/utils/Random.cpp b/src/liblms/impl/utils/Random.cpp similarity index 96% rename from src/utils/Random.cpp rename to src/liblms/impl/utils/Random.cpp index c7114ddf..eb081ae0 100644 --- a/src/utils/Random.cpp +++ b/src/liblms/impl/utils/Random.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "Random.hpp" +#include "utils/Random.hpp" namespace Random { diff --git a/src/utils/StreamLogger.cpp b/src/liblms/impl/utils/StreamLogger.cpp similarity index 96% rename from src/utils/StreamLogger.cpp rename to src/liblms/impl/utils/StreamLogger.cpp index f4fc9cf5..94434332 100644 --- a/src/utils/StreamLogger.cpp +++ b/src/liblms/impl/utils/StreamLogger.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "StreamLogger.hpp" +#include "utils/StreamLogger.hpp" StreamLogger::StreamLogger(std::ostream& os) : _os {os} diff --git a/src/utils/String.cpp b/src/liblms/impl/utils/String.cpp similarity index 97% rename from src/utils/String.cpp rename to src/liblms/impl/utils/String.cpp index 299c177e..be6a79a1 100644 --- a/src/utils/String.cpp +++ b/src/liblms/impl/utils/String.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "String.hpp" +#include "utils/String.hpp" #include @@ -103,6 +103,12 @@ stringToLower(const std::string& str) return boost::algorithm::to_lower_copy(str); } +std::string +stringToUpper(const std::string& str) +{ + return boost::to_upper_copy(str); +} + std::string bufferToString(const std::vector& data) { diff --git a/src/utils/UUID.cpp b/src/liblms/impl/utils/UUID.cpp similarity index 97% rename from src/utils/UUID.cpp rename to src/liblms/impl/utils/UUID.cpp index 9edbdad4..f95209e2 100644 --- a/src/utils/UUID.cpp +++ b/src/liblms/impl/utils/UUID.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "UUID.hpp" +#include "utils/UUID.hpp" #include diff --git a/src/utils/WtLogger.cpp b/src/liblms/impl/utils/WtLogger.cpp similarity index 94% rename from src/utils/WtLogger.cpp rename to src/liblms/impl/utils/WtLogger.cpp index bb1359b0..126af923 100644 --- a/src/utils/WtLogger.cpp +++ b/src/liblms/impl/utils/WtLogger.cpp @@ -17,12 +17,12 @@ * along with LMS. If not, see . */ -#include "WtLogger.hpp" +#include "utils/WtLogger.hpp" #include #include -#include "Logger.hpp" +#include "utils/Logger.hpp" void WtLogger::processLog(const Log& log) diff --git a/src/api/subsonic/SubsonicResource.hpp b/src/liblms/include/api/subsonic/SubsonicResource.hpp similarity index 100% rename from src/api/subsonic/SubsonicResource.hpp rename to src/liblms/include/api/subsonic/SubsonicResource.hpp diff --git a/src/liblms/include/auth/IAuthTokenService.hpp b/src/liblms/include/auth/IAuthTokenService.hpp new file mode 100644 index 00000000..f85148dd --- /dev/null +++ b/src/liblms/include/auth/IAuthTokenService.hpp @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +/* This file contains some classes in order to get info from file using the libavconv */ + +#pragma once + +#include +#include +#include +#include + +#include "database/Types.hpp" + +namespace Database +{ + class Session; +} + + +namespace Auth { + + class IAuthTokenService + { + public: + + // Auth Token services + struct AuthTokenProcessResult + { + enum class State + { + Found, + Throttled, + NotFound, + }; + + struct AuthTokenInfo + { + Database::IdType userId; + Wt::WDateTime expiry; + }; + + State state {State::NotFound}; + std::optional authTokenInfo {}; + }; + + // Removed if found + virtual AuthTokenProcessResult processAuthToken(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& tokenValue) = 0; + virtual std::string createAuthToken(Database::Session& session, Database::IdType userid, const Wt::WDateTime& expiry) = 0; + }; + + std::unique_ptr createAuthTokenService(std::size_t maxThrottlerEntryCount); + +} + diff --git a/src/liblms/include/auth/IPasswordService.hpp b/src/liblms/include/auth/IPasswordService.hpp new file mode 100644 index 00000000..fb5f9551 --- /dev/null +++ b/src/liblms/include/auth/IPasswordService.hpp @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +#include + +#include "database/User.hpp" +#include "database/Types.hpp" + +namespace Database +{ + class Session; +} + + +namespace Auth { + + class IPasswordService + { + public: + + virtual ~IPasswordService() = default; + + // Password services + enum class PasswordCheckResult + { + Match, + Mismatch, + Throttled, + }; + virtual PasswordCheckResult checkUserPassword(Database::Session& session, const boost::asio::ip::address& clientAddress, const std::string& loginName, const std::string& password) = 0; + virtual Database::User::PasswordHash hashPassword(const std::string& password) const = 0; + virtual bool evaluatePasswordStrength(const std::string& loginName, const std::string& password) const = 0; + }; + + std::unique_ptr createPasswordService(std::size_t maxThrottlerEntryCount); + +} + diff --git a/src/av/AvInfo.hpp b/src/liblms/include/av/AvInfo.hpp similarity index 100% rename from src/av/AvInfo.hpp rename to src/liblms/include/av/AvInfo.hpp diff --git a/src/av/AvTranscoder.hpp b/src/liblms/include/av/AvTranscoder.hpp similarity index 100% rename from src/av/AvTranscoder.hpp rename to src/liblms/include/av/AvTranscoder.hpp diff --git a/src/av/AvTypes.hpp b/src/liblms/include/av/AvTypes.hpp similarity index 100% rename from src/av/AvTypes.hpp rename to src/liblms/include/av/AvTypes.hpp diff --git a/src/liblms/include/cover/CoverArt.hpp b/src/liblms/include/cover/CoverArt.hpp new file mode 100644 index 00000000..db392e6f --- /dev/null +++ b/src/liblms/include/cover/CoverArt.hpp @@ -0,0 +1,40 @@ + +/* + * Copyright (C) 2015 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +namespace CoverArt +{ + + enum class Format + { + JPEG, + }; + std::string formatToMimeType(Format format); + + struct Geometry + { + std::size_t width; + std::size_t height; + }; + +} diff --git a/src/liblms/include/cover/ICoverArtGrabber.hpp b/src/liblms/include/cover/ICoverArtGrabber.hpp new file mode 100644 index 00000000..e018521a --- /dev/null +++ b/src/liblms/include/cover/ICoverArtGrabber.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2015 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +#include "database/Types.hpp" +#include "cover/CoverArt.hpp" + +namespace Database { + class Session; +} + +namespace CoverArt { + +class IGrabber +{ + public: + virtual ~IGrabber() = default; + + virtual void setDefaultCover(const std::filesystem::path& defaultCoverPath) = 0; + + virtual std::vector getFromTrack(Database::Session& dbSession, Database::IdType trackId, Format format, std::size_t width) = 0; + virtual std::vector getFromRelease(Database::Session& dbSession, Database::IdType releaseId, Format format, std::size_t width) = 0; + +}; + +} // namespace CoverArt + diff --git a/src/database/Artist.hpp b/src/liblms/include/database/Artist.hpp similarity index 100% rename from src/database/Artist.hpp rename to src/liblms/include/database/Artist.hpp diff --git a/src/database/Cluster.hpp b/src/liblms/include/database/Cluster.hpp similarity index 100% rename from src/database/Cluster.hpp rename to src/liblms/include/database/Cluster.hpp diff --git a/src/database/Db.hpp b/src/liblms/include/database/Db.hpp similarity index 100% rename from src/database/Db.hpp rename to src/liblms/include/database/Db.hpp diff --git a/src/database/Release.hpp b/src/liblms/include/database/Release.hpp similarity index 100% rename from src/database/Release.hpp rename to src/liblms/include/database/Release.hpp diff --git a/src/database/ScanSettings.hpp b/src/liblms/include/database/ScanSettings.hpp similarity index 100% rename from src/database/ScanSettings.hpp rename to src/liblms/include/database/ScanSettings.hpp diff --git a/src/database/Session.hpp b/src/liblms/include/database/Session.hpp similarity index 94% rename from src/database/Session.hpp rename to src/liblms/include/database/Session.hpp index 427fc6fd..303501bd 100644 --- a/src/database/Session.hpp +++ b/src/liblms/include/database/Session.hpp @@ -67,8 +67,8 @@ class Session Session& operator=(const Session&) = delete; Session& operator=(Session&&) = delete; - UniqueTransaction createUniqueTransaction(); - SharedTransaction createSharedTransaction(); + [[nodiscard]] UniqueTransaction createUniqueTransaction(); + [[nodiscard]] SharedTransaction createSharedTransaction(); void checkUniqueLocked(); void checkSharedLocked(); diff --git a/src/database/SessionPool.hpp b/src/liblms/include/database/SessionPool.hpp similarity index 100% rename from src/database/SessionPool.hpp rename to src/liblms/include/database/SessionPool.hpp diff --git a/src/database/Track.hpp b/src/liblms/include/database/Track.hpp similarity index 100% rename from src/database/Track.hpp rename to src/liblms/include/database/Track.hpp diff --git a/src/database/TrackArtistLink.hpp b/src/liblms/include/database/TrackArtistLink.hpp similarity index 100% rename from src/database/TrackArtistLink.hpp rename to src/liblms/include/database/TrackArtistLink.hpp diff --git a/src/database/TrackBookmark.hpp b/src/liblms/include/database/TrackBookmark.hpp similarity index 100% rename from src/database/TrackBookmark.hpp rename to src/liblms/include/database/TrackBookmark.hpp diff --git a/src/database/TrackFeatures.hpp b/src/liblms/include/database/TrackFeatures.hpp similarity index 100% rename from src/database/TrackFeatures.hpp rename to src/liblms/include/database/TrackFeatures.hpp diff --git a/src/database/TrackList.hpp b/src/liblms/include/database/TrackList.hpp similarity index 100% rename from src/database/TrackList.hpp rename to src/liblms/include/database/TrackList.hpp diff --git a/src/database/Types.hpp b/src/liblms/include/database/Types.hpp similarity index 100% rename from src/database/Types.hpp rename to src/liblms/include/database/Types.hpp diff --git a/src/database/User.hpp b/src/liblms/include/database/User.hpp similarity index 100% rename from src/database/User.hpp rename to src/liblms/include/database/User.hpp diff --git a/src/metadata/AvFormat.hpp b/src/liblms/include/metadata/AvFormat.hpp similarity index 100% rename from src/metadata/AvFormat.hpp rename to src/liblms/include/metadata/AvFormat.hpp diff --git a/src/metadata/MetaData.hpp b/src/liblms/include/metadata/MetaData.hpp similarity index 100% rename from src/metadata/MetaData.hpp rename to src/liblms/include/metadata/MetaData.hpp diff --git a/src/metadata/TagLibParser.hpp b/src/liblms/include/metadata/TagLibParser.hpp similarity index 100% rename from src/metadata/TagLibParser.hpp rename to src/liblms/include/metadata/TagLibParser.hpp diff --git a/src/liblms/include/recommendation/ClustersRecommendationProviderCreator.hpp b/src/liblms/include/recommendation/ClustersRecommendationProviderCreator.hpp new file mode 100644 index 00000000..99b11666 --- /dev/null +++ b/src/liblms/include/recommendation/ClustersRecommendationProviderCreator.hpp @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2020 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +namespace Database +{ + class Session; +} + +namespace Recommendation +{ + class Provider; + + std::unique_ptr createClustersRecommendationProvider(); +} + diff --git a/src/liblms/include/recommendation/FeaturesRecommendationProviderCreator.hpp b/src/liblms/include/recommendation/FeaturesRecommendationProviderCreator.hpp new file mode 100644 index 00000000..01c07787 --- /dev/null +++ b/src/liblms/include/recommendation/FeaturesRecommendationProviderCreator.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2020 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +namespace Database +{ + class Session; +} + +namespace Scanner +{ + class MediaScanner; +} + +namespace Recommendation +{ + class Provider; + + std::unique_ptr createFeaturesRecommendationProvider(Scanner::MediaScanner& scanner); +} + diff --git a/src/liblms/include/recommendation/IEngine.hpp b/src/liblms/include/recommendation/IEngine.hpp new file mode 100644 index 00000000..bc420aff --- /dev/null +++ b/src/liblms/include/recommendation/IEngine.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +#include "database/Types.hpp" +#include "Provider.hpp" + +namespace Database +{ + class Session; +} + +namespace Recommendation +{ + class Provider; + class IEngine + { + public: + virtual ~IEngine() = default; + + virtual void clearProviders() = 0; + virtual void addProvider(std::unique_ptr provider, unsigned priority) = 0; + + // Closest results first + virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0; + virtual std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) = 0; + virtual std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) = 0; + virtual std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0; + }; + + std::unique_ptr createEngine(); + +} // ns Recommendation + diff --git a/src/liblms/include/recommendation/Provider.hpp b/src/liblms/include/recommendation/Provider.hpp new file mode 100644 index 00000000..d40f686e --- /dev/null +++ b/src/liblms/include/recommendation/Provider.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2020 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +#include "database/Types.hpp" + +namespace Database +{ + class Session; +} + +namespace Recommendation +{ + + class Provider + { + public: + virtual ~Provider() = default; + + virtual bool isTrackClassified(Database::IdType trackId) const = 0; + virtual bool isReleaseClassified(Database::IdType releaseId) const = 0; + virtual bool isArtistClassified(Database::IdType artistId) const = 0; + + virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0; + virtual std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const = 0; + virtual std::vector getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const = 0; + virtual std::vector getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const = 0; + }; + +} // ns Recommendation diff --git a/src/scanner/MediaScanner.hpp b/src/liblms/include/scanner/MediaScanner.hpp similarity index 100% rename from src/scanner/MediaScanner.hpp rename to src/liblms/include/scanner/MediaScanner.hpp diff --git a/src/scanner/MediaScannerAddon.hpp b/src/liblms/include/scanner/MediaScannerAddon.hpp similarity index 100% rename from src/scanner/MediaScannerAddon.hpp rename to src/liblms/include/scanner/MediaScannerAddon.hpp diff --git a/src/scanner/MediaScannerStats.hpp b/src/liblms/include/scanner/MediaScannerStats.hpp similarity index 100% rename from src/scanner/MediaScannerStats.hpp rename to src/liblms/include/scanner/MediaScannerStats.hpp diff --git a/src/utils/Exception.hpp b/src/liblms/include/utils/Exception.hpp similarity index 100% rename from src/utils/Exception.hpp rename to src/liblms/include/utils/Exception.hpp diff --git a/src/liblms/include/utils/IConfig.hpp b/src/liblms/include/utils/IConfig.hpp new file mode 100644 index 00000000..6b62169b --- /dev/null +++ b/src/liblms/include/utils/IConfig.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2016 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ +#pragma once + +#include +#include +#include + +// Used to get config values from configuration files +class IConfig +{ + public: + + // Default values are returned in case of setting not found + virtual std::string getString(const std::string& setting, const std::string& def = "", const std::unordered_set& allowedValues = {}) = 0; + virtual std::filesystem::path getPath(const std::string& setting, const std::filesystem::path& def = std::filesystem::path()) = 0; + virtual unsigned long getULong(const std::string& setting, unsigned long def = 0) = 0; + virtual long getLong(const std::string& setting, long def = 0) = 0; + virtual bool getBool(const std::string& setting, bool def = false) = 0; +}; + + +std::unique_ptr createConfig(const std::filesystem::path& p); + diff --git a/src/utils/Logger.hpp b/src/liblms/include/utils/Logger.hpp similarity index 100% rename from src/utils/Logger.hpp rename to src/liblms/include/utils/Logger.hpp diff --git a/src/utils/NetAddress.hpp b/src/liblms/include/utils/NetAddress.hpp similarity index 100% rename from src/utils/NetAddress.hpp rename to src/liblms/include/utils/NetAddress.hpp diff --git a/src/utils/Path.hpp b/src/liblms/include/utils/Path.hpp similarity index 100% rename from src/utils/Path.hpp rename to src/liblms/include/utils/Path.hpp diff --git a/src/utils/Random.hpp b/src/liblms/include/utils/Random.hpp similarity index 100% rename from src/utils/Random.hpp rename to src/liblms/include/utils/Random.hpp diff --git a/src/utils/Service.hpp b/src/liblms/include/utils/Service.hpp similarity index 92% rename from src/utils/Service.hpp rename to src/liblms/include/utils/Service.hpp index a18363ce..c53a8c81 100644 --- a/src/utils/Service.hpp +++ b/src/liblms/include/utils/Service.hpp @@ -46,7 +46,14 @@ class ServiceProvider return *get(); } - static void assign(std::unique_ptr service) { _service = std::move(service); } + static + Class& + assign(std::unique_ptr service) + { + _service = std::move(service); + return *get(); + } + static void clear() { _service.reset(); } static Class* get() { return _service.get(); } diff --git a/src/utils/StreamLogger.hpp b/src/liblms/include/utils/StreamLogger.hpp similarity index 100% rename from src/utils/StreamLogger.hpp rename to src/liblms/include/utils/StreamLogger.hpp diff --git a/src/utils/String.hpp b/src/liblms/include/utils/String.hpp similarity index 97% rename from src/utils/String.hpp rename to src/liblms/include/utils/String.hpp index 44f67faa..1ee1a964 100644 --- a/src/utils/String.hpp +++ b/src/liblms/include/utils/String.hpp @@ -26,6 +26,7 @@ namespace StringUtils { + std::vector splitString(const std::string& string, const std::string& separators); @@ -41,6 +42,9 @@ stringTrimEnd(const std::string& str, const std::string& whitespaces = " \t"); std::string stringToLower(const std::string& str); +std::string +stringToUpper(const std::string& str); + std::string bufferToString(const std::vector& data); diff --git a/src/utils/UUID.hpp b/src/liblms/include/utils/UUID.hpp similarity index 100% rename from src/utils/UUID.hpp rename to src/liblms/include/utils/UUID.hpp diff --git a/src/utils/Utils.hpp b/src/liblms/include/utils/Utils.hpp similarity index 100% rename from src/utils/Utils.hpp rename to src/liblms/include/utils/Utils.hpp diff --git a/src/utils/WtLogger.hpp b/src/liblms/include/utils/WtLogger.hpp similarity index 100% rename from src/utils/WtLogger.hpp rename to src/liblms/include/utils/WtLogger.hpp diff --git a/src/lms/CMakeLists.txt b/src/lms/CMakeLists.txt new file mode 100644 index 00000000..94526da9 --- /dev/null +++ b/src/lms/CMakeLists.txt @@ -0,0 +1,47 @@ + +add_executable(lms + main.cpp + ui/Auth.cpp + ui/LmsApplication.cpp + ui/LmsApplicationGroup.cpp + ui/MediaPlayer.cpp + ui/PlayQueueView.cpp + ui/PlayHistoryView.cpp + ui/SettingsView.cpp + ui/TrackStringUtils.cpp + ui/admin/DatabaseSettingsView.cpp + ui/admin/DatabaseStatus.cpp + ui/admin/InitWizardView.cpp + ui/admin/UserView.cpp + ui/admin/UsersView.cpp + ui/common/Validators.cpp + ui/explore/ArtistInfoView.cpp + ui/explore/ArtistLink.cpp + ui/explore/ArtistsInfoView.cpp + ui/explore/ArtistView.cpp + ui/explore/ArtistsView.cpp + ui/explore/Explore.cpp + ui/explore/Filters.cpp + ui/explore/ReleaseInfoView.cpp + ui/explore/ReleaseLink.cpp + ui/explore/ReleasesInfoView.cpp + ui/explore/ReleasesView.cpp + ui/explore/ReleaseView.cpp + ui/explore/TracksInfoView.cpp + ui/explore/TracksView.cpp + ui/resource/ImageResource.cpp + ui/resource/AudioResource.cpp + ) + +target_include_directories(lms PRIVATE + ui/ + ) + +target_link_libraries(lms PRIVATE + liblms + wt + wthttp + ) + +install(TARGETS lms DESTINATION bin) + diff --git a/src/main/main.cpp b/src/lms/main.cpp similarity index 63% rename from src/main/main.cpp rename to src/lms/main.cpp index 805de6c0..b51512d0 100644 --- a/src/main/main.cpp +++ b/src/lms/main.cpp @@ -25,16 +25,17 @@ #include "api/subsonic/SubsonicResource.hpp" #include "av/AvInfo.hpp" #include "av/AvTranscoder.hpp" -#include "auth/AuthTokenService.hpp" -#include "auth/PasswordService.hpp" +#include "auth/IAuthTokenService.hpp" +#include "auth/IPasswordService.hpp" #include "cover/CoverArtGrabber.hpp" #include "database/Db.hpp" #include "image/Image.hpp" #include "scanner/MediaScanner.hpp" -#include "similarity/features/SimilarityFeaturesScannerAddon.hpp" -#include "similarity/SimilaritySearcher.hpp" +#include "recommendation/FeaturesRecommendationProviderCreator.hpp" +#include "recommendation/IEngine.hpp" +#include "recommendation/ClustersRecommendationProviderCreator.hpp" #include "ui/LmsApplication.hpp" -#include "utils/Config.hpp" +#include "utils/IConfig.hpp" #include "utils/Service.hpp" #include "utils/WtLogger.hpp" @@ -42,28 +43,28 @@ std::vector generateWtConfig(std::string execPath) { std::vector args; - const std::filesystem::path wtConfigPath {ServiceProvider::get()->getPath("working-dir") / "wt_config.xml"}; - const std::filesystem::path wtLogFilePath {ServiceProvider::get()->getPath("log-file", "/var/log/lms.log")}; - const std::filesystem::path wtAccessLogFilePath {ServiceProvider::get()->getPath("access-log-file", "/var/log/lms.access.log")}; + const std::filesystem::path wtConfigPath {ServiceProvider::get()->getPath("working-dir") / "wt_config.xml"}; + const std::filesystem::path wtLogFilePath {ServiceProvider::get()->getPath("log-file", "/var/log/lms.log")}; + const std::filesystem::path wtAccessLogFilePath {ServiceProvider::get()->getPath("access-log-file", "/var/log/lms.access.log")}; args.push_back(execPath); args.push_back("--config=" + wtConfigPath.string()); - args.push_back("--docroot=" + ServiceProvider::get()->getString("docroot")); - args.push_back("--approot=" + ServiceProvider::get()->getString("approot")); - args.push_back("--resources-dir=" + ServiceProvider::get()->getString("wt-resources")); + args.push_back("--docroot=" + ServiceProvider::get()->getString("docroot")); + args.push_back("--approot=" + ServiceProvider::get()->getString("approot")); + args.push_back("--resources-dir=" + ServiceProvider::get()->getString("wt-resources")); - if (ServiceProvider::get()->getBool("tls-enable", false)) + if (ServiceProvider::get()->getBool("tls-enable", false)) { - args.push_back("--https-port=" + std::to_string( ServiceProvider::get()->getULong("listen-port", 5082))); - args.push_back("--https-address=" + ServiceProvider::get()->getString("listen-addr", "0.0.0.0")); - args.push_back("--ssl-certificate=" + ServiceProvider::get()->getString("tls-cert")); - args.push_back("--ssl-private-key=" + ServiceProvider::get()->getString("tls-key")); - args.push_back("--ssl-tmp-dh=" + ServiceProvider::get()->getString("tls-dh")); + args.push_back("--https-port=" + std::to_string( ServiceProvider::get()->getULong("listen-port", 5082))); + args.push_back("--https-address=" + ServiceProvider::get()->getString("listen-addr", "0.0.0.0")); + args.push_back("--ssl-certificate=" + ServiceProvider::get()->getString("tls-cert")); + args.push_back("--ssl-private-key=" + ServiceProvider::get()->getString("tls-key")); + args.push_back("--ssl-tmp-dh=" + ServiceProvider::get()->getString("tls-dh")); } else { - args.push_back("--http-port=" + std::to_string( ServiceProvider::get()->getULong("listen-port", 5082))); - args.push_back("--http-address=" + ServiceProvider::get()->getString("listen-addr", "0.0.0.0")); + args.push_back("--http-port=" + std::to_string( ServiceProvider::get()->getULong("listen-port", 5082))); + args.push_back("--http-address=" + ServiceProvider::get()->getString("listen-addr", "0.0.0.0")); } if (!wtAccessLogFilePath.empty()) @@ -74,8 +75,8 @@ std::vector generateWtConfig(std::string execPath) pt.put("server.application-settings..location", "*"); pt.put("server.application-settings.log-file", wtLogFilePath.string()); - pt.put("server.application-settings.log-config", ServiceProvider::get()->getString("log-config", "* -debug -info:WebRequest")); - pt.put("server.application-settings.behind-reverse-proxy", ServiceProvider::get()->getBool("behind-reverse-proxy", false)); + pt.put("server.application-settings.log-config", ServiceProvider::get()->getString("log-config", "* -debug -info:WebRequest")); + pt.put("server.application-settings.behind-reverse-proxy", ServiceProvider::get()->getBool("behind-reverse-proxy", false)); pt.put("server.application-settings.progressive-bootstrap", true); std::ofstream oss(wtConfigPath.string().c_str(), std::ios::out); @@ -108,17 +109,17 @@ int main(int argc, char* argv[]) // Make pstream work with ffmpeg close(STDIN_FILENO); - ServiceProvider::create(configFilePath); + ServiceProvider::assign(createConfig(configFilePath)); ServiceProvider::create(); // Make sure the working directory exists - std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir")); - std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir") / "cache"); + std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir")); + std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir") / "cache"); // Construct WT configuration and get the argc/argv back std::vector wtServerArgs = generateWtConfig(argv[0]); - const char* wtArgv[wtServerArgs.size()]; + std::vector wtArgv(wtServerArgs.size()); for (std::size_t i = 0; i < wtServerArgs.size(); ++i) { std::cout << "ARG = " << wtServerArgs[i] << std::endl; @@ -126,14 +127,14 @@ int main(int argc, char* argv[]) } Wt::WServer server(argv[0]); - server.setServerConfiguration (wtServerArgs.size(), const_cast(wtArgv)); + server.setServerConfiguration (wtServerArgs.size(), const_cast(&wtArgv[0])); // lib init Image::init(argv[0]); Av::Transcoder::init(); // Initializing a connection pool to the database that will be shared along services - Database::Db database {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; + Database::Db database {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; { Database::Session session {database}; session.prepareTables(); @@ -142,23 +143,21 @@ int main(int argc, char* argv[]) UserInterface::LmsApplicationGroupContainer appGroups; // Service initialization order is important - ServiceProvider::create(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000)); - ServiceProvider::create(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000)); + ServiceProvider::assign(Auth::createAuthTokenService(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000))); + ServiceProvider::assign(Auth::createPasswordService(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000))); Scanner::MediaScanner& mediaScanner {ServiceProvider::create(database)}; - Similarity::FeaturesScannerAddon similarityFeaturesScannerAddon {database}; - - mediaScanner.setAddon(similarityFeaturesScannerAddon); + Recommendation::IEngine& recommendationEngine {ServiceProvider::assign(Recommendation::createEngine())}; + recommendationEngine.addProvider(Recommendation::createFeaturesRecommendationProvider(mediaScanner), 0); + recommendationEngine.addProvider(Recommendation::createClustersRecommendationProvider(), 1); CoverArt::Grabber& coverArtGrabber {ServiceProvider::create()}; coverArtGrabber.setDefaultCover(server.appRoot() + "/images/unknown-cover.jpg"); - ServiceProvider::create(similarityFeaturesScannerAddon); - API::Subsonic::SubsonicResource subsonicResource {database}; // bind API resources - if (ServiceProvider::get()->getBool("api-subsonic", true)) + if (ServiceProvider::get()->getBool("api-subsonic", true)) server.addResource(&subsonicResource, subsonicResource.getPath()); // bind UI entry point @@ -187,14 +186,6 @@ int main(int argc, char* argv[]) LMS_LOG(MAIN, INFO) << "Clean stop!"; res = EXIT_SUCCESS; } - catch( libconfig::FileIOException& e) - { - std::cerr << "Cannot open config file '" << configFilePath << "'" << std::endl; - } - catch( libconfig::ParseException& e) - { - std::cerr << "Caught libconfig::ParseException! error='" << e.getError() << "', file = '" << e.getFile() << "', line = " << e.getLine() << std::endl; - } catch(Wt::WServer::Exception& e) { std::cerr << "Caught a WServer::Exception: " << e.what() << std::endl; diff --git a/src/ui/Auth.cpp b/src/lms/ui/Auth.cpp similarity index 85% rename from src/ui/Auth.cpp rename to src/lms/ui/Auth.cpp index f1f37124..d450b0c7 100644 --- a/src/ui/Auth.cpp +++ b/src/lms/ui/Auth.cpp @@ -27,8 +27,8 @@ #include #include -#include "auth/AuthTokenService.hpp" -#include "auth/PasswordService.hpp" +#include "auth/IAuthTokenService.hpp" +#include "auth/IPasswordService.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -43,7 +43,7 @@ static void createAuthToken(Database::IdType userId, const Wt::WDateTime& expiry) { - const std::string secret {ServiceProvider<::Auth::AuthTokenService>::get()->createAuthToken(LmsApp->getDbSession(), userId, expiry)}; + const std::string secret {ServiceProvider<::Auth::IAuthTokenService>::get()->createAuthToken(LmsApp->getDbSession(), userId, expiry)}; LmsApp->setCookie(authCookieName, secret, @@ -61,15 +61,15 @@ processAuthToken(const Wt::WEnvironment& env) if (!authCookie) return std::nullopt; - const auto res {ServiceProvider<::Auth::AuthTokenService>::get()->processAuthToken(LmsApp->getDbSession(), boost::asio::ip::address::from_string(env.clientAddress()), *authCookie)}; + const auto res {ServiceProvider<::Auth::IAuthTokenService>::get()->processAuthToken(LmsApp->getDbSession(), boost::asio::ip::address::from_string(env.clientAddress()), *authCookie)}; switch (res.state) { - case ::Auth::AuthTokenService::AuthTokenProcessResult::State::NotFound: - case ::Auth::AuthTokenService::AuthTokenProcessResult::State::Throttled: + case ::Auth::IAuthTokenService::AuthTokenProcessResult::State::NotFound: + case ::Auth::IAuthTokenService::AuthTokenProcessResult::State::Throttled: LmsApp->setCookie(authCookieName, std::string {}, 0, "", "", env.urlScheme() == "https"); return std::nullopt; - case ::Auth::AuthTokenService::AuthTokenProcessResult::State::Found: + case ::Auth::IAuthTokenService::AuthTokenProcessResult::State::Found: createAuthToken(res.authTokenInfo->userId, res.authTokenInfo->expiry); break; } @@ -124,18 +124,18 @@ class AuthModel : public Wt::WFormModel if (field == PasswordField) { - switch (ServiceProvider<::Auth::PasswordService>::get()->checkUserPassword( + switch (ServiceProvider<::Auth::IPasswordService>::get()->checkUserPassword( LmsApp->getDbSession(), boost::asio::ip::address::from_string(LmsApp->environment().clientAddress()), valueText(LoginNameField).toUTF8(), valueText(PasswordField).toUTF8())) { - case ::Auth::PasswordService::PasswordCheckResult::Match: + case ::Auth::IPasswordService::PasswordCheckResult::Match: break; - case ::Auth::PasswordService::PasswordCheckResult::Mismatch: + case ::Auth::IPasswordService::PasswordCheckResult::Mismatch: error = Wt::WString::tr("Lms.password-bad-login-combination"); break; - case ::Auth::PasswordService::PasswordCheckResult::Throttled: + case ::Auth::IPasswordService::PasswordCheckResult::Throttled: error = Wt::WString::tr("Lms.password-client-throttled"); break; } diff --git a/src/ui/Auth.hpp b/src/lms/ui/Auth.hpp similarity index 100% rename from src/ui/Auth.hpp rename to src/lms/ui/Auth.hpp diff --git a/src/ui/LmsApplication.cpp b/src/lms/ui/LmsApplication.cpp similarity index 99% rename from src/ui/LmsApplication.cpp rename to src/lms/ui/LmsApplication.cpp index ebc16ae0..669417e2 100644 --- a/src/ui/LmsApplication.cpp +++ b/src/lms/ui/LmsApplication.cpp @@ -29,7 +29,6 @@ #include #include -#include "config/config.h" #include "cover/CoverArtGrabber.hpp" #include "database/Artist.hpp" #include "database/Cluster.hpp" diff --git a/src/ui/LmsApplication.hpp b/src/lms/ui/LmsApplication.hpp similarity index 100% rename from src/ui/LmsApplication.hpp rename to src/lms/ui/LmsApplication.hpp diff --git a/src/ui/LmsApplicationException.hpp b/src/lms/ui/LmsApplicationException.hpp similarity index 100% rename from src/ui/LmsApplicationException.hpp rename to src/lms/ui/LmsApplicationException.hpp diff --git a/src/ui/LmsApplicationGroup.cpp b/src/lms/ui/LmsApplicationGroup.cpp similarity index 97% rename from src/ui/LmsApplicationGroup.cpp rename to src/lms/ui/LmsApplicationGroup.cpp index 70fcbe64..d9c41199 100644 --- a/src/ui/LmsApplicationGroup.cpp +++ b/src/lms/ui/LmsApplicationGroup.cpp @@ -30,7 +30,7 @@ namespace UserInterface { LmsApplicationInfo LmsApplicationInfo::fromEnvironment(const Wt::WEnvironment& env) { - LmsApplicationInfo info = {.userAgent = env.agent()}; + LmsApplicationInfo info = {env.agent()}; return info; } diff --git a/src/ui/LmsApplicationGroup.hpp b/src/lms/ui/LmsApplicationGroup.hpp similarity index 100% rename from src/ui/LmsApplicationGroup.hpp rename to src/lms/ui/LmsApplicationGroup.hpp diff --git a/src/ui/MediaPlayer.cpp b/src/lms/ui/MediaPlayer.cpp similarity index 100% rename from src/ui/MediaPlayer.cpp rename to src/lms/ui/MediaPlayer.cpp diff --git a/src/ui/MediaPlayer.hpp b/src/lms/ui/MediaPlayer.hpp similarity index 100% rename from src/ui/MediaPlayer.hpp rename to src/lms/ui/MediaPlayer.hpp diff --git a/src/ui/PlayHistoryView.cpp b/src/lms/ui/PlayHistoryView.cpp similarity index 100% rename from src/ui/PlayHistoryView.cpp rename to src/lms/ui/PlayHistoryView.cpp diff --git a/src/ui/PlayHistoryView.hpp b/src/lms/ui/PlayHistoryView.hpp similarity index 100% rename from src/ui/PlayHistoryView.hpp rename to src/lms/ui/PlayHistoryView.hpp diff --git a/src/ui/PlayQueueView.cpp b/src/lms/ui/PlayQueueView.cpp similarity index 98% rename from src/ui/PlayQueueView.cpp rename to src/lms/ui/PlayQueueView.cpp index 91dc4c1b..e55476b6 100644 --- a/src/ui/PlayQueueView.cpp +++ b/src/lms/ui/PlayQueueView.cpp @@ -26,7 +26,7 @@ #include "database/Track.hpp" #include "database/TrackList.hpp" #include "database/User.hpp" -#include "similarity/SimilaritySearcher.hpp" +#include "recommendation/IEngine.hpp" #include "utils/Logger.hpp" #include "utils/Random.hpp" #include "utils/Service.hpp" @@ -417,7 +417,7 @@ PlayQueue::addSome() void PlayQueue::enqueueRadioTrack() { - const std::vector trackToAddIds {ServiceProvider::get()->getSimilarTracksFromTrackList(LmsApp->getDbSession(), _tracklistId, 1)}; + const std::vector trackToAddIds {ServiceProvider::get()->getSimilarTracksFromTrackList(LmsApp->getDbSession(), _tracklistId, 1)}; enqueueTracks(trackToAddIds); } diff --git a/src/ui/PlayQueueView.hpp b/src/lms/ui/PlayQueueView.hpp similarity index 100% rename from src/ui/PlayQueueView.hpp rename to src/lms/ui/PlayQueueView.hpp diff --git a/src/ui/SettingsView.cpp b/src/lms/ui/SettingsView.cpp similarity index 94% rename from src/ui/SettingsView.cpp rename to src/lms/ui/SettingsView.cpp index c1b430b6..d5aa9233 100644 --- a/src/ui/SettingsView.cpp +++ b/src/lms/ui/SettingsView.cpp @@ -30,7 +30,7 @@ #include "common/Validators.hpp" #include "common/ValueStringModel.hpp" -#include "auth/PasswordService.hpp" +#include "auth/IPasswordService.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -80,7 +80,7 @@ class SettingsModel : public Wt::WFormModel Database::User::PasswordHash passwordHash; if (!valueText(PasswordField).empty()) - passwordHash = ServiceProvider<::Auth::PasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8()); + passwordHash = ServiceProvider<::Auth::IPasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8()); auto transaction {LmsApp->getDbSession().createUniqueTransaction()}; @@ -133,18 +133,18 @@ class SettingsModel : public Wt::WFormModel { if (!valueText(PasswordOldField).empty()) { - switch (ServiceProvider<::Auth::PasswordService>::get()->checkUserPassword( + switch (ServiceProvider<::Auth::IPasswordService>::get()->checkUserPassword( LmsApp->getDbSession(), boost::asio::ip::address::from_string(LmsApp->environment().clientAddress()), LmsApp->getUserLoginName(), valueText(PasswordOldField).toUTF8())) { - case ::Auth::PasswordService::PasswordCheckResult::Match: + case ::Auth::IPasswordService::PasswordCheckResult::Match: break; - case ::Auth::PasswordService::PasswordCheckResult::Mismatch: + case ::Auth::IPasswordService::PasswordCheckResult::Mismatch: error = Wt::WString::tr("Lms.Settings.password-bad"); break; - case ::Auth::PasswordService::PasswordCheckResult::Throttled: + case ::Auth::IPasswordService::PasswordCheckResult::Throttled: error = Wt::WString::tr("Lms.password-client-throttled"); break; } @@ -161,7 +161,7 @@ class SettingsModel : public Wt::WFormModel { if (!valueText(PasswordField).empty()) { - if (!ServiceProvider<::Auth::PasswordService>::get()->evaluatePasswordStrength(LmsApp->getUserLoginName(), valueText(PasswordField).toUTF8())) + if (!ServiceProvider<::Auth::IPasswordService>::get()->evaluatePasswordStrength(LmsApp->getUserLoginName(), valueText(PasswordField).toUTF8())) error = Wt::WString::tr("Lms.password-too-weak"); } else diff --git a/src/ui/SettingsView.hpp b/src/lms/ui/SettingsView.hpp similarity index 100% rename from src/ui/SettingsView.hpp rename to src/lms/ui/SettingsView.hpp diff --git a/src/ui/TrackStringUtils.cpp b/src/lms/ui/TrackStringUtils.cpp similarity index 100% rename from src/ui/TrackStringUtils.cpp rename to src/lms/ui/TrackStringUtils.cpp diff --git a/src/ui/TrackStringUtils.hpp b/src/lms/ui/TrackStringUtils.hpp similarity index 100% rename from src/ui/TrackStringUtils.hpp rename to src/lms/ui/TrackStringUtils.hpp diff --git a/src/ui/admin/DatabaseSettingsView.cpp b/src/lms/ui/admin/DatabaseSettingsView.cpp similarity index 100% rename from src/ui/admin/DatabaseSettingsView.cpp rename to src/lms/ui/admin/DatabaseSettingsView.cpp diff --git a/src/ui/admin/DatabaseSettingsView.hpp b/src/lms/ui/admin/DatabaseSettingsView.hpp similarity index 100% rename from src/ui/admin/DatabaseSettingsView.hpp rename to src/lms/ui/admin/DatabaseSettingsView.hpp diff --git a/src/ui/admin/DatabaseStatus.cpp b/src/lms/ui/admin/DatabaseStatus.cpp similarity index 100% rename from src/ui/admin/DatabaseStatus.cpp rename to src/lms/ui/admin/DatabaseStatus.cpp diff --git a/src/ui/admin/DatabaseStatus.hpp b/src/lms/ui/admin/DatabaseStatus.hpp similarity index 100% rename from src/ui/admin/DatabaseStatus.hpp rename to src/lms/ui/admin/DatabaseStatus.hpp diff --git a/src/ui/admin/InitWizardView.cpp b/src/lms/ui/admin/InitWizardView.cpp similarity index 93% rename from src/ui/admin/InitWizardView.cpp rename to src/lms/ui/admin/InitWizardView.cpp index 2eeba3e5..34d7c5fb 100644 --- a/src/ui/admin/InitWizardView.cpp +++ b/src/lms/ui/admin/InitWizardView.cpp @@ -23,7 +23,7 @@ #include #include -#include "auth/PasswordService.hpp" +#include "auth/IPasswordService.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -55,7 +55,7 @@ class InitWizardModel : public Wt::WFormModel void saveData() { - const Database::User::PasswordHash passwordHash {ServiceProvider<::Auth::PasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8())}; + const Database::User::PasswordHash passwordHash {ServiceProvider<::Auth::IPasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8())}; auto transaction(LmsApp->getDbSession().createUniqueTransaction()); @@ -77,7 +77,7 @@ class InitWizardModel : public Wt::WFormModel if (!valueText(PasswordField).empty()) { // Evaluate the strength of the password - if (!ServiceProvider<::Auth::PasswordService>::get()->evaluatePasswordStrength(valueText(AdminLoginField).toUTF8(), valueText(PasswordField).toUTF8())) + if (!ServiceProvider<::Auth::IPasswordService>::get()->evaluatePasswordStrength(valueText(AdminLoginField).toUTF8(), valueText(PasswordField).toUTF8())) error = Wt::WString::tr("Lms.password-too-weak"); } else diff --git a/src/ui/admin/InitWizardView.hpp b/src/lms/ui/admin/InitWizardView.hpp similarity index 100% rename from src/ui/admin/InitWizardView.hpp rename to src/lms/ui/admin/InitWizardView.hpp diff --git a/src/ui/admin/UserView.cpp b/src/lms/ui/admin/UserView.cpp similarity index 95% rename from src/ui/admin/UserView.cpp rename to src/lms/ui/admin/UserView.cpp index dc1dd83c..0b4d7556 100644 --- a/src/ui/admin/UserView.cpp +++ b/src/lms/ui/admin/UserView.cpp @@ -28,9 +28,9 @@ #include -#include "auth/PasswordService.hpp" +#include "auth/IPasswordService.hpp" #include "database/User.hpp" -#include "utils/Config.hpp" +#include "utils/IConfig.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -82,7 +82,7 @@ class UserModel : public Wt::WFormModel { std::optional passwordHash; if (!valueText(PasswordField).empty()) - passwordHash = ServiceProvider<::Auth::PasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8()); + passwordHash = ServiceProvider<::Auth::IPasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8()); auto transaction {LmsApp->getDbSession().createUniqueTransaction()}; @@ -174,7 +174,7 @@ class UserModel : public Wt::WFormModel else { // Evaluate the strength of the password for non demo accounts - if (!ServiceProvider<::Auth::PasswordService>::get()->evaluatePasswordStrength(getLoginName(), valueText(PasswordField).toUTF8())) + if (!ServiceProvider<::Auth::IPasswordService>::get()->evaluatePasswordStrength(getLoginName(), valueText(PasswordField).toUTF8())) error = Wt::WString::tr("Lms.password-too-weak"); } } @@ -270,7 +270,7 @@ UserView::refreshView() // Demo account t->setFormWidget(UserModel::DemoField, std::make_unique()); - if (!userId && ServiceProvider::get()->getBool("demo", false)) + if (!userId && ServiceProvider::get()->getBool("demo", false)) t->setCondition("if-demo", true); Wt::WPushButton* saveBtn = t->bindNew("save-btn", Wt::WString::tr(userId ? "Lms.save" : "Lms.create")); diff --git a/src/ui/admin/UserView.hpp b/src/lms/ui/admin/UserView.hpp similarity index 100% rename from src/ui/admin/UserView.hpp rename to src/lms/ui/admin/UserView.hpp diff --git a/src/ui/admin/UsersView.cpp b/src/lms/ui/admin/UsersView.cpp similarity index 100% rename from src/ui/admin/UsersView.cpp rename to src/lms/ui/admin/UsersView.cpp diff --git a/src/ui/admin/UsersView.hpp b/src/lms/ui/admin/UsersView.hpp similarity index 100% rename from src/ui/admin/UsersView.hpp rename to src/lms/ui/admin/UsersView.hpp diff --git a/src/ui/common/Validators.cpp b/src/lms/ui/common/Validators.cpp similarity index 100% rename from src/ui/common/Validators.cpp rename to src/lms/ui/common/Validators.cpp diff --git a/src/ui/common/Validators.hpp b/src/lms/ui/common/Validators.hpp similarity index 100% rename from src/ui/common/Validators.hpp rename to src/lms/ui/common/Validators.hpp diff --git a/src/ui/common/ValueStringModel.hpp b/src/lms/ui/common/ValueStringModel.hpp similarity index 100% rename from src/ui/common/ValueStringModel.hpp rename to src/lms/ui/common/ValueStringModel.hpp diff --git a/src/ui/explore/ArtistInfoView.cpp b/src/lms/ui/explore/ArtistInfoView.cpp similarity index 93% rename from src/ui/explore/ArtistInfoView.cpp rename to src/lms/ui/explore/ArtistInfoView.cpp index 8f806eba..e9d3a3d9 100644 --- a/src/ui/explore/ArtistInfoView.cpp +++ b/src/lms/ui/explore/ArtistInfoView.cpp @@ -20,7 +20,7 @@ #include "ArtistInfoView.hpp" #include "database/Artist.hpp" -#include "similarity/SimilaritySearcher.hpp" +#include "recommendation/IEngine.hpp" #include "utils/Service.hpp" #include "utils/String.hpp" @@ -63,7 +63,7 @@ ArtistInfo::refresh() if (!artistId) return; - const std::vector artistsIds {ServiceProvider::get()->getSimilarArtists(LmsApp->getDbSession(), *artistId, 5)}; + const std::vector artistsIds {ServiceProvider::get()->getSimilarArtists(LmsApp->getDbSession(), *artistId, 5)}; auto transaction {LmsApp->getDbSession().createSharedTransaction()}; diff --git a/src/ui/explore/ArtistInfoView.hpp b/src/lms/ui/explore/ArtistInfoView.hpp similarity index 100% rename from src/ui/explore/ArtistInfoView.hpp rename to src/lms/ui/explore/ArtistInfoView.hpp diff --git a/src/ui/explore/ArtistLink.cpp b/src/lms/ui/explore/ArtistLink.cpp similarity index 100% rename from src/ui/explore/ArtistLink.cpp rename to src/lms/ui/explore/ArtistLink.cpp diff --git a/src/ui/explore/ArtistLink.hpp b/src/lms/ui/explore/ArtistLink.hpp similarity index 100% rename from src/ui/explore/ArtistLink.hpp rename to src/lms/ui/explore/ArtistLink.hpp diff --git a/src/ui/explore/ArtistView.cpp b/src/lms/ui/explore/ArtistView.cpp similarity index 100% rename from src/ui/explore/ArtistView.cpp rename to src/lms/ui/explore/ArtistView.cpp diff --git a/src/ui/explore/ArtistView.hpp b/src/lms/ui/explore/ArtistView.hpp similarity index 100% rename from src/ui/explore/ArtistView.hpp rename to src/lms/ui/explore/ArtistView.hpp diff --git a/src/ui/explore/ArtistsInfoView.cpp b/src/lms/ui/explore/ArtistsInfoView.cpp similarity index 100% rename from src/ui/explore/ArtistsInfoView.cpp rename to src/lms/ui/explore/ArtistsInfoView.cpp diff --git a/src/ui/explore/ArtistsInfoView.hpp b/src/lms/ui/explore/ArtistsInfoView.hpp similarity index 100% rename from src/ui/explore/ArtistsInfoView.hpp rename to src/lms/ui/explore/ArtistsInfoView.hpp diff --git a/src/ui/explore/ArtistsView.cpp b/src/lms/ui/explore/ArtistsView.cpp similarity index 100% rename from src/ui/explore/ArtistsView.cpp rename to src/lms/ui/explore/ArtistsView.cpp diff --git a/src/ui/explore/ArtistsView.hpp b/src/lms/ui/explore/ArtistsView.hpp similarity index 100% rename from src/ui/explore/ArtistsView.hpp rename to src/lms/ui/explore/ArtistsView.hpp diff --git a/src/ui/explore/Explore.cpp b/src/lms/ui/explore/Explore.cpp similarity index 100% rename from src/ui/explore/Explore.cpp rename to src/lms/ui/explore/Explore.cpp diff --git a/src/ui/explore/Explore.hpp b/src/lms/ui/explore/Explore.hpp similarity index 100% rename from src/ui/explore/Explore.hpp rename to src/lms/ui/explore/Explore.hpp diff --git a/src/ui/explore/Filters.cpp b/src/lms/ui/explore/Filters.cpp similarity index 100% rename from src/ui/explore/Filters.cpp rename to src/lms/ui/explore/Filters.cpp diff --git a/src/ui/explore/Filters.hpp b/src/lms/ui/explore/Filters.hpp similarity index 100% rename from src/ui/explore/Filters.hpp rename to src/lms/ui/explore/Filters.hpp diff --git a/src/ui/explore/ReleaseInfoView.cpp b/src/lms/ui/explore/ReleaseInfoView.cpp similarity index 95% rename from src/ui/explore/ReleaseInfoView.cpp rename to src/lms/ui/explore/ReleaseInfoView.cpp index 7293a678..759b20de 100644 --- a/src/ui/explore/ReleaseInfoView.cpp +++ b/src/lms/ui/explore/ReleaseInfoView.cpp @@ -22,7 +22,7 @@ #include #include "database/Release.hpp" -#include "similarity/SimilaritySearcher.hpp" +#include "recommendation/IEngine.hpp" #include "utils/Service.hpp" #include "utils/String.hpp" @@ -68,7 +68,7 @@ ReleaseInfo::refresh() if (!releaseId) return; - const std::vector releasesIds {ServiceProvider::get()->getSimilarReleases(LmsApp->getDbSession(), *releaseId, 5)}; + const std::vector releasesIds {ServiceProvider::get()->getSimilarReleases(LmsApp->getDbSession(), *releaseId, 5)}; auto transaction {LmsApp->getDbSession().createSharedTransaction()}; diff --git a/src/ui/explore/ReleaseInfoView.hpp b/src/lms/ui/explore/ReleaseInfoView.hpp similarity index 100% rename from src/ui/explore/ReleaseInfoView.hpp rename to src/lms/ui/explore/ReleaseInfoView.hpp diff --git a/src/ui/explore/ReleaseLink.cpp b/src/lms/ui/explore/ReleaseLink.cpp similarity index 100% rename from src/ui/explore/ReleaseLink.cpp rename to src/lms/ui/explore/ReleaseLink.cpp diff --git a/src/ui/explore/ReleaseLink.hpp b/src/lms/ui/explore/ReleaseLink.hpp similarity index 100% rename from src/ui/explore/ReleaseLink.hpp rename to src/lms/ui/explore/ReleaseLink.hpp diff --git a/src/ui/explore/ReleaseView.cpp b/src/lms/ui/explore/ReleaseView.cpp similarity index 100% rename from src/ui/explore/ReleaseView.cpp rename to src/lms/ui/explore/ReleaseView.cpp diff --git a/src/ui/explore/ReleaseView.hpp b/src/lms/ui/explore/ReleaseView.hpp similarity index 100% rename from src/ui/explore/ReleaseView.hpp rename to src/lms/ui/explore/ReleaseView.hpp diff --git a/src/ui/explore/ReleasesInfoView.cpp b/src/lms/ui/explore/ReleasesInfoView.cpp similarity index 100% rename from src/ui/explore/ReleasesInfoView.cpp rename to src/lms/ui/explore/ReleasesInfoView.cpp diff --git a/src/ui/explore/ReleasesInfoView.hpp b/src/lms/ui/explore/ReleasesInfoView.hpp similarity index 100% rename from src/ui/explore/ReleasesInfoView.hpp rename to src/lms/ui/explore/ReleasesInfoView.hpp diff --git a/src/ui/explore/ReleasesView.cpp b/src/lms/ui/explore/ReleasesView.cpp similarity index 100% rename from src/ui/explore/ReleasesView.cpp rename to src/lms/ui/explore/ReleasesView.cpp diff --git a/src/ui/explore/ReleasesView.hpp b/src/lms/ui/explore/ReleasesView.hpp similarity index 100% rename from src/ui/explore/ReleasesView.hpp rename to src/lms/ui/explore/ReleasesView.hpp diff --git a/src/ui/explore/TracksInfoView.cpp b/src/lms/ui/explore/TracksInfoView.cpp similarity index 100% rename from src/ui/explore/TracksInfoView.cpp rename to src/lms/ui/explore/TracksInfoView.cpp diff --git a/src/ui/explore/TracksInfoView.hpp b/src/lms/ui/explore/TracksInfoView.hpp similarity index 100% rename from src/ui/explore/TracksInfoView.hpp rename to src/lms/ui/explore/TracksInfoView.hpp diff --git a/src/ui/explore/TracksView.cpp b/src/lms/ui/explore/TracksView.cpp similarity index 100% rename from src/ui/explore/TracksView.cpp rename to src/lms/ui/explore/TracksView.cpp diff --git a/src/ui/explore/TracksView.hpp b/src/lms/ui/explore/TracksView.hpp similarity index 100% rename from src/ui/explore/TracksView.hpp rename to src/lms/ui/explore/TracksView.hpp diff --git a/src/ui/resource/AudioResource.cpp b/src/lms/ui/resource/AudioResource.cpp similarity index 100% rename from src/ui/resource/AudioResource.cpp rename to src/lms/ui/resource/AudioResource.cpp diff --git a/src/ui/resource/AudioResource.hpp b/src/lms/ui/resource/AudioResource.hpp similarity index 100% rename from src/ui/resource/AudioResource.hpp rename to src/lms/ui/resource/AudioResource.hpp diff --git a/src/ui/resource/ImageResource.cpp b/src/lms/ui/resource/ImageResource.cpp similarity index 100% rename from src/ui/resource/ImageResource.cpp rename to src/lms/ui/resource/ImageResource.cpp diff --git a/src/ui/resource/ImageResource.hpp b/src/lms/ui/resource/ImageResource.hpp similarity index 100% rename from src/ui/resource/ImageResource.hpp rename to src/lms/ui/resource/ImageResource.hpp diff --git a/src/similarity/SimilaritySearcher.hpp b/src/similarity/SimilaritySearcher.hpp deleted file mode 100644 index 3a53cb9b..00000000 --- a/src/similarity/SimilaritySearcher.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2019 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include -#include - -#include "database/Types.hpp" - -namespace Database -{ - class Session; -} - -namespace Similarity { - -class FeaturesScannerAddon; - -class Searcher -{ - public: - Searcher(FeaturesScannerAddon& somAddon); - - // Closest results first - std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount); - std::vector getSimilarTracks(Database::Session& session, const std::set& tracksId, std::size_t maxCount); - std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount); - std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount); - - private: - - FeaturesScannerAddon& _somAddon; -}; - -} // ns Similarity diff --git a/src/similarity/features/SimilarityFeaturesScannerAddon.hpp b/src/similarity/features/SimilarityFeaturesScannerAddon.hpp deleted file mode 100644 index 38651e37..00000000 --- a/src/similarity/features/SimilarityFeaturesScannerAddon.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include "database/Session.hpp" -#include "scanner/MediaScannerAddon.hpp" - -#include "utils/UUID.hpp" - -#include "SimilarityFeaturesSearcher.hpp" - - -namespace Database { - class Db; -} - -namespace Similarity { - -class FeaturesScannerAddon final : public Scanner::MediaScannerAddon -{ - public: - - FeaturesScannerAddon(Database::Db& db); - - std::shared_ptr getSearcher(); - - private: - - void refreshSettings() override {} - void requestStop() override; - void preScanComplete() override; - - void trackAdded(Database::IdType) override {} - void trackToRemove(Database::IdType) override {} - void trackUpdated(Database::IdType trackId) override; - - bool fetchFeatures(Database::IdType trackId, const UUID& MBID); - - void updateSearcher(); - - Database::Session _dbSession; - std::shared_ptr _searcher; - bool _stopRequested {}; -}; - -FeaturesScannerAddon* setFeaturesScannerAddon(FeaturesScannerAddon addon); -FeaturesScannerAddon* getFeaturesScannerAddon(); - -} // namespace Similarity - diff --git a/tools/Makefile.am b/src/tools/Makefile.am similarity index 100% rename from tools/Makefile.am rename to src/tools/Makefile.am diff --git a/tools/metadata/LmsMetadata.cpp b/src/tools/metadata/LmsMetadata.cpp similarity index 100% rename from tools/metadata/LmsMetadata.cpp rename to src/tools/metadata/LmsMetadata.cpp diff --git a/tools/metadata/Makefile.am b/src/tools/metadata/Makefile.am similarity index 100% rename from tools/metadata/Makefile.am rename to src/tools/metadata/Makefile.am diff --git a/tools/similarity-parameters/GeneticAlgorithm.hpp b/src/tools/similarity-parameters/GeneticAlgorithm.hpp similarity index 100% rename from tools/similarity-parameters/GeneticAlgorithm.hpp rename to src/tools/similarity-parameters/GeneticAlgorithm.hpp diff --git a/tools/similarity-parameters/LmsSimilarityParameters.cpp b/src/tools/similarity-parameters/LmsSimilarityParameters.cpp similarity index 100% rename from tools/similarity-parameters/LmsSimilarityParameters.cpp rename to src/tools/similarity-parameters/LmsSimilarityParameters.cpp diff --git a/tools/similarity-parameters/Makefile.am b/src/tools/similarity-parameters/Makefile.am similarity index 100% rename from tools/similarity-parameters/Makefile.am rename to src/tools/similarity-parameters/Makefile.am diff --git a/tools/similarity-parameters/ParallelFor.hpp b/src/tools/similarity-parameters/ParallelFor.hpp similarity index 100% rename from tools/similarity-parameters/ParallelFor.hpp rename to src/tools/similarity-parameters/ParallelFor.hpp diff --git a/tools/similarity/LmsSimilarity.cpp b/src/tools/similarity/LmsSimilarity.cpp similarity index 100% rename from tools/similarity/LmsSimilarity.cpp rename to src/tools/similarity/LmsSimilarity.cpp diff --git a/tools/similarity/Makefile.am b/src/tools/similarity/Makefile.am similarity index 100% rename from tools/similarity/Makefile.am rename to src/tools/similarity/Makefile.am From 1e2c1caeed95e0aeed78f820413970f5a672daf2 Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 13 Feb 2020 16:10:43 +0100 Subject: [PATCH 02/23] Isolated ImageMagick++ in liblms --- src/liblms/CMakeLists.txt | 16 ++++-- .../impl/api/subsonic/SubsonicResource.cpp | 4 +- src/liblms/impl/cover/CoverArtGrabber.cpp | 55 +++++++++++-------- src/liblms/impl/cover/CoverArtGrabber.hpp | 21 ++++--- src/liblms/impl/cover/Image.cpp | 36 +++++++----- src/liblms/impl/cover/Image.hpp | 50 +++++++---------- src/liblms/include/cover/ICoverArtGrabber.hpp | 3 +- src/lms/main.cpp | 6 +- src/lms/ui/LmsApplication.cpp | 2 +- src/lms/ui/resource/ImageResource.cpp | 11 ++-- src/lms/ui/resource/ImageResource.hpp | 2 - 11 files changed, 107 insertions(+), 99 deletions(-) diff --git a/src/liblms/CMakeLists.txt b/src/liblms/CMakeLists.txt index 37ba7cfd..2c1644ff 100644 --- a/src/liblms/CMakeLists.txt +++ b/src/liblms/CMakeLists.txt @@ -10,6 +10,7 @@ add_library(liblms SHARED impl/av/AvTranscoder.cpp impl/av/AvTypes.cpp impl/cover/CoverArtGrabber.cpp + impl/cover/Image.cpp impl/database/Artist.cpp impl/database/Cluster.cpp impl/database/Db.cpp @@ -43,18 +44,21 @@ add_library(liblms SHARED impl/utils/WtLogger.cpp ) -include_directories(liblms include/) - target_include_directories(liblms INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include ) -# TODO make this private -target_include_directories(liblms PUBLIC ${IMAGEMAGICKXX_INCLUDE_DIRS}) -target_link_libraries(liblms PUBLIC ${IMAGEMAGICKXX_LIBRARIES}) -target_compile_options(liblms PUBLIC ${IMAGEMAGICKXX_CFLAGS_OTHER}) +target_include_directories(liblms PRIVATE + include/ + ${IMAGEMAGICKXX_INCLUDE_DIRS} + ) + +target_compile_options(liblms PRIVATE + ${IMAGEMAGICKXX_CFLAGS_OTHER} + ) target_link_libraries(liblms PRIVATE + ${IMAGEMAGICKXX_LIBRARIES} avformat avutil config++ diff --git a/src/liblms/impl/api/subsonic/SubsonicResource.cpp b/src/liblms/impl/api/subsonic/SubsonicResource.cpp index c2e72223..3a4db863 100644 --- a/src/liblms/impl/api/subsonic/SubsonicResource.cpp +++ b/src/liblms/impl/api/subsonic/SubsonicResource.cpp @@ -1895,10 +1895,10 @@ handleGetCoverArt(RequestContext& context, Wt::Http::ResponseContinuation*) switch (id.type) { case Id::Type::Track: - res.data = ServiceProvider::get()->getFromTrack(context.dbSession, id.value, CoverArt::Format::JPEG, {size, size}); + res.data = ServiceProvider::get()->getFromTrack(context.dbSession, id.value, CoverArt::Format::JPEG, size); break; case Id::Type::Release: - res.data = ServiceProvider::get()->getFromRelease(context.dbSession, id.value, CoverArt::Format::JPEG, {size, size}); + res.data = ServiceProvider::get()->getFromRelease(context.dbSession, id.value, CoverArt::Format::JPEG, size); break; default: throw BadParameterGenericError {"id"}; diff --git a/src/liblms/impl/cover/CoverArtGrabber.cpp b/src/liblms/impl/cover/CoverArtGrabber.cpp index 110e9ddd..110b80d1 100644 --- a/src/liblms/impl/cover/CoverArtGrabber.cpp +++ b/src/liblms/impl/cover/CoverArtGrabber.cpp @@ -39,8 +39,19 @@ isFileSupported(const std::filesystem::path& file, const std::vector createGrabber(const std::filesystem::path& execPath) { + return std::make_unique(execPath); +} + +Grabber::Grabber(const std::filesystem::path& execPath) +{ + init(execPath); +} + +Grabber::~Grabber() +{ + deinit(); } void @@ -50,7 +61,7 @@ Grabber::setDefaultCover(const std::filesystem::path& p) throw LmsException("Cannot read default cover file '" + p.string() + "'"); } -Image::Image +Image Grabber::getDefaultCover(std::size_t size) { LMS_LOG(COVER, DEBUG) << "Getting a default cover using size = " << size; @@ -59,12 +70,12 @@ Grabber::getDefaultCover(std::size_t size) auto it = _defaultCovers.find(size); if (it == _defaultCovers.end()) { - Image::Image cover = _defaultCover; + Image cover = _defaultCover; LMS_LOG(COVER, DEBUG) << "default cover size = " << cover.getSize().width << " x " << cover.getSize().height; LMS_LOG(COVER, DEBUG) << "Scaling cover to size = " << size; - cover.scale(Image::Geometry{size, size}); + cover.scale(Geometry{size, size}); LMS_LOG(COVER, DEBUG) << "Scaling DONE"; auto res = _defaultCovers.insert(std::make_pair(size, cover)); assert(res.second); @@ -74,14 +85,14 @@ Grabber::getDefaultCover(std::size_t size) return it->second; } -static std::optional +static std::optional getFromAvMediaFile(const Av::MediaFile& input) { - std::vector res; + std::vector res; for (auto& picture : input.getAttachedPictures(2)) { - Image::Image image; + Image image; if (image.load(picture.data)) return image; @@ -93,12 +104,12 @@ getFromAvMediaFile(const Av::MediaFile& input) return std::nullopt; } -std::optional +std::optional Grabber::getFromDirectory(const std::filesystem::path& p) const { for (auto coverPath : getCoverPaths(p)) { - Image::Image image; + Image image; if (image.load(coverPath)) return image; @@ -143,7 +154,7 @@ Grabber::getCoverPaths(const std::filesystem::path& directoryPath) const return res; } -std::optional +std::optional Grabber::getFromTrack(const std::filesystem::path& p) const { try @@ -159,12 +170,12 @@ Grabber::getFromTrack(const std::filesystem::path& p) const } } -Image::Image +Image Grabber::getFromTrack(Database::Session& dbSession, Database::IdType trackId, std::size_t size) { using namespace Database; - std::optional cover; + std::optional cover; bool hasCover {}; bool isMultiDisc {}; @@ -200,16 +211,16 @@ Grabber::getFromTrack(Database::Session& dbSession, Database::IdType trackId, st if (!cover) cover = getDefaultCover(size); else - cover->scale(Image::Geometry {size, size}); + cover->scale(Geometry {size, size}); return *cover; } -Image::Image +Image Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, std::size_t size) { - std::optional cover; + std::optional cover; std::optional trackId; { @@ -230,26 +241,26 @@ Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, if (!cover) cover = getDefaultCover(size); else - cover->scale(Image::Geometry {size, size}); + cover->scale(Geometry {size, size}); return *cover; } std::vector -Grabber::getFromTrack(Database::Session& session, Database::IdType trackId, Format format, std::size_t size) +Grabber::getFromTrack(Database::Session& session, Database::IdType trackId, Format format, std::size_t width) { - const Image::Image cover {getFromTrack(session, trackId, size)}; + const Image cover {getFromTrack(session, trackId, width)}; - assert(format == Image::Format::JPEG); + assert(format == Format::JPEG); return cover.save(format); } std::vector -Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, Format format, std::size_t size) +Grabber::getFromRelease(Database::Session& session, Database::IdType releaseId, Format format, std::size_t width) { - const Image::Image cover {getFromRelease(session, releaseId, size)}; + const Image cover {getFromRelease(session, releaseId, width)}; - assert(format == Image::Format::JPEG); + assert(format == Format::JPEG); return cover.save(format); } diff --git a/src/liblms/impl/cover/CoverArtGrabber.hpp b/src/liblms/impl/cover/CoverArtGrabber.hpp index 3911ad43..bc921355 100644 --- a/src/liblms/impl/cover/CoverArtGrabber.hpp +++ b/src/liblms/impl/cover/CoverArtGrabber.hpp @@ -40,7 +40,9 @@ namespace CoverArt class Grabber : public IGrabber { public: - Grabber(); + Grabber(const std::filesystem::path& execPath); + ~Grabber(); + Grabber(const Grabber&) = delete; Grabber& operator=(const Grabber&) = delete; Grabber(Grabber&&) = delete; @@ -53,24 +55,21 @@ namespace CoverArt private: - Image::Image getFromTrack(Database::Session& dbSession, Database::IdType trackId, std::size_t size); - Image::Image getFromRelease(Database::Session& dbSession, Database::IdType releaseId, std::size_t size); + Image getFromTrack(Database::Session& dbSession, Database::IdType trackId, std::size_t size); + Image getFromRelease(Database::Session& dbSession, Database::IdType releaseId, std::size_t size); - std::optional getFromTrack(const std::filesystem::path& path) const; + std::optional getFromTrack(const std::filesystem::path& path) const; std::vector getCoverPaths(const std::filesystem::path& directoryPath) const; - std::optional getFromDirectory(const std::filesystem::path& path) const; + std::optional getFromDirectory(const std::filesystem::path& path) const; + Image getDefaultCover(std::size_t size); - Image::Image getDefaultCover(std::size_t size); - - Image::Image _defaultCover; + Image _defaultCover; std::mutex _mutex; - std::map _defaultCovers; + std::map _defaultCovers; static inline const std::vector _fileExtensions {".jpg", ".jpeg", ".png", ".bmp"}; // TODO parametrize - static inline const std::size_t _maxFileSize {10000000}; - static inline const std::vector _preferredFileNames {"cover", "front"}; // TODO parametrize }; diff --git a/src/liblms/impl/cover/Image.cpp b/src/liblms/impl/cover/Image.cpp index 06532eca..a39c0061 100644 --- a/src/liblms/impl/cover/Image.cpp +++ b/src/liblms/impl/cover/Image.cpp @@ -17,14 +17,27 @@ * along with LMS. If not, see . */ -#include "image/Image.hpp" +#include "Image.hpp" #include "utils/Logger.hpp" -namespace Image { +namespace CoverArt { + +void +init(const std::filesystem::path& path) +{ + Magick::InitializeMagick(path.string().c_str()); +} + +void +deinit() +{ + MagickCore::MagickCoreTerminus(); +} static -std::string format_to_magick(Format format) +std::string +formatToMagick(Format format) { switch (format) { @@ -34,7 +47,8 @@ std::string format_to_magick(Format format) return "JPEG"; } -std::string format_to_mimeType(Format format) +std::string +formatToMimeType(Format format) { switch (format) { @@ -44,18 +58,13 @@ std::string format_to_mimeType(Format format) return "application/octet-stream"; } -void -init(const char *path) -{ - Magick::InitializeMagick(path); -} bool Image::load(const std::vector& rawData) { try { - Magick::Blob blob(&rawData[0], rawData.size()); + Magick::Blob blob {&rawData[0], rawData.size()}; _image.read(blob); return true; @@ -116,9 +125,9 @@ Image::save(Format format) const try { - Magick::Image outputImage(_image); + Magick::Image outputImage {_image}; - outputImage.magick( format_to_magick(format)); + outputImage.magick(formatToMagick(format)); Magick::Blob blob; outputImage.write(&blob); @@ -135,4 +144,5 @@ Image::save(Format format) const } } -} // namespace Image +} // namespace CoverArt + diff --git a/src/liblms/impl/cover/Image.hpp b/src/liblms/impl/cover/Image.hpp index 40bc2d6b..6932825f 100644 --- a/src/liblms/impl/cover/Image.hpp +++ b/src/liblms/impl/cover/Image.hpp @@ -24,44 +24,34 @@ #include -namespace Image +#include "cover/CoverArt.hpp" + +namespace CoverArt { -enum class Format -{ - JPEG, -}; + void init(const std::filesystem::path& path); + void deinit(); -std::string format_to_mimeType(Format format); + class Image + { + public: -void init(const char *path); + // input + bool load(const std::vector& rawData); + bool load(const std::filesystem::path& p); -struct Geometry -{ - std::size_t width; - std::size_t height; -}; + Geometry getSize() const; -class Image -{ - public: + // Operations + bool scale(Geometry geometry); - // input - bool load(const std::vector& rawData); - bool load(const std::filesystem::path& p); + // output + std::vector save(Format format) const; - Geometry getSize() const; - - // Operations - bool scale(Geometry geometry); - - // output - std::vector save(Format format) const; - - private: - Magick::Image _image; -}; + private: + Magick::Image _image; + }; -} // namespace Image +} // namespace CoverArt diff --git a/src/liblms/include/cover/ICoverArtGrabber.hpp b/src/liblms/include/cover/ICoverArtGrabber.hpp index e018521a..c62e3955 100644 --- a/src/liblms/include/cover/ICoverArtGrabber.hpp +++ b/src/liblms/include/cover/ICoverArtGrabber.hpp @@ -40,8 +40,9 @@ class IGrabber virtual std::vector getFromTrack(Database::Session& dbSession, Database::IdType trackId, Format format, std::size_t width) = 0; virtual std::vector getFromRelease(Database::Session& dbSession, Database::IdType releaseId, Format format, std::size_t width) = 0; - }; +std::unique_ptr createGrabber(const std::filesystem::path& execPath); + } // namespace CoverArt diff --git a/src/lms/main.cpp b/src/lms/main.cpp index b51512d0..5f5bf313 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -27,9 +27,8 @@ #include "av/AvTranscoder.hpp" #include "auth/IAuthTokenService.hpp" #include "auth/IPasswordService.hpp" -#include "cover/CoverArtGrabber.hpp" +#include "cover/ICoverArtGrabber.hpp" #include "database/Db.hpp" -#include "image/Image.hpp" #include "scanner/MediaScanner.hpp" #include "recommendation/FeaturesRecommendationProviderCreator.hpp" #include "recommendation/IEngine.hpp" @@ -130,7 +129,6 @@ int main(int argc, char* argv[]) server.setServerConfiguration (wtServerArgs.size(), const_cast(&wtArgv[0])); // lib init - Image::init(argv[0]); Av::Transcoder::init(); // Initializing a connection pool to the database that will be shared along services @@ -151,7 +149,7 @@ int main(int argc, char* argv[]) recommendationEngine.addProvider(Recommendation::createFeaturesRecommendationProvider(mediaScanner), 0); recommendationEngine.addProvider(Recommendation::createClustersRecommendationProvider(), 1); - CoverArt::Grabber& coverArtGrabber {ServiceProvider::create()}; + CoverArt::IGrabber& coverArtGrabber {ServiceProvider::assign(CoverArt::createGrabber(argv[0]))}; coverArtGrabber.setDefaultCover(server.appRoot() + "/images/unknown-cover.jpg"); API::Subsonic::SubsonicResource subsonicResource {database}; diff --git a/src/lms/ui/LmsApplication.cpp b/src/lms/ui/LmsApplication.cpp index 669417e2..56480d94 100644 --- a/src/lms/ui/LmsApplication.cpp +++ b/src/lms/ui/LmsApplication.cpp @@ -29,7 +29,7 @@ #include #include -#include "cover/CoverArtGrabber.hpp" +#include "cover/ICoverArtGrabber.hpp" #include "database/Artist.hpp" #include "database/Cluster.hpp" #include "database/Db.hpp" diff --git a/src/lms/ui/resource/ImageResource.cpp b/src/lms/ui/resource/ImageResource.cpp index f0bade0c..6abd1d42 100644 --- a/src/lms/ui/resource/ImageResource.cpp +++ b/src/lms/ui/resource/ImageResource.cpp @@ -22,7 +22,7 @@ #include #include -#include "cover/CoverArtGrabber.hpp" +#include "cover/ICoverArtGrabber.hpp" #include "database/Track.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" @@ -33,9 +33,6 @@ namespace UserInterface { -static const std::string unknownCoverPath = "/images/unknown-cover.jpg"; -static const std::string unknownArtistImagePath = "/images/unknown-artist.jpg"; - ImageResource::~ImageResource() { beingDeleted(); @@ -80,7 +77,7 @@ ImageResource::handleRequest(const Wt::Http::Request& request, Wt::Http::Respons // DbSession are not thread safe { Wt::WApplication::UpdateLock lock {LmsApp}; - cover = ServiceProvider::get()->getFromTrack(LmsApp->getDbSession(), *trackId, Image::Format::JPEG, *size); + cover = ServiceProvider::get()->getFromTrack(LmsApp->getDbSession(), *trackId, CoverArt::Format::JPEG, *size); } } else if (releaseIdStr) @@ -92,7 +89,7 @@ ImageResource::handleRequest(const Wt::Http::Request& request, Wt::Http::Respons // DbSession are not thread safe { Wt::WApplication::UpdateLock lock {LmsApp}; - cover = ServiceProvider::get()->getFromRelease(LmsApp->getDbSession(), *releaseId, Image::Format::JPEG, *size); + cover = ServiceProvider::get()->getFromRelease(LmsApp->getDbSession(), *releaseId, CoverArt::Format::JPEG, *size); } } else @@ -106,7 +103,7 @@ ImageResource::handleRequest(const Wt::Http::Request& request, Wt::Http::Respons std::string ImageResource::getMimeType() { - return Image::format_to_mimeType(Image::Format::JPEG); + return CoverArt::formatToMimeType(CoverArt::Format::JPEG); } } // namespace UserInterface diff --git a/src/lms/ui/resource/ImageResource.hpp b/src/lms/ui/resource/ImageResource.hpp index 2f372b7d..29b5a595 100644 --- a/src/lms/ui/resource/ImageResource.hpp +++ b/src/lms/ui/resource/ImageResource.hpp @@ -26,8 +26,6 @@ #include "database/Types.hpp" -#include "image/Image.hpp" - namespace UserInterface { From 15e53caa2dfc31d279c1e4f434641f249bdcc961 Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 13 Feb 2020 18:04:35 +0100 Subject: [PATCH 03/23] Split the lib in smaller libs to ease unit tests --- src/CMakeLists.txt | 4 +- src/liblms/CMakeLists.txt | 77 ----------------- src/libs/CMakeLists.txt | 11 +++ src/libs/auth/CMakeLists.txt | 28 ++++++ .../auth/impl}/AuthTokenService.cpp | 0 .../auth/impl}/AuthTokenService.hpp | 0 .../auth/impl}/LoginThrottler.cpp | 0 .../auth/impl}/LoginThrottler.hpp | 0 .../auth/impl}/PasswordService.cpp | 0 .../auth/impl}/PasswordService.hpp | 0 .../auth}/include/auth/IAuthTokenService.hpp | 0 .../auth}/include/auth/IPasswordService.hpp | 0 src/libs/av/CMakeLists.txt | 24 ++++++ .../impl/av => libs/av/impl}/AvInfo.cpp | 0 .../impl/av => libs/av/impl}/AvTranscoder.cpp | 0 .../impl/av => libs/av/impl}/AvTypes.cpp | 0 src/{liblms => libs/av}/include/av/AvInfo.hpp | 0 .../av}/include/av/AvTranscoder.hpp | 0 .../av}/include/av/AvTypes.hpp | 0 src/libs/cover/CMakeLists.txt | 27 ++++++ .../cover/impl}/CoverArtGrabber.cpp | 0 .../cover/impl}/CoverArtGrabber.hpp | 0 .../impl/cover => libs/cover/impl}/Image.cpp | 0 .../impl/cover => libs/cover/impl}/Image.hpp | 0 .../cover}/include/cover/CoverArt.hpp | 0 .../cover}/include/cover/ICoverArtGrabber.hpp | 0 src/libs/database/CMakeLists.txt | 37 ++++++++ .../database/impl}/Artist.cpp | 0 .../database/impl}/Cluster.cpp | 0 .../database => libs/database/impl}/Db.cpp | 0 .../database/impl}/Release.cpp | 0 .../database/impl}/ScanSettings.cpp | 0 .../database/impl}/Session.cpp | 0 .../database/impl}/SessionPool.cpp | 0 .../database/impl}/SqlQuery.cpp | 0 .../database/impl}/SqlQuery.hpp | 0 .../database => libs/database/impl}/Track.cpp | 0 .../database/impl}/TrackArtistLink.cpp | 0 .../database/impl}/TrackBookmark.cpp | 0 .../database/impl}/TrackFeatures.cpp | 0 .../database/impl}/TrackList.cpp | 0 .../database => libs/database/impl}/User.cpp | 0 .../database}/include/database/Artist.hpp | 0 .../database}/include/database/Cluster.hpp | 0 .../database}/include/database/Db.hpp | 0 .../database}/include/database/Release.hpp | 0 .../include/database/ScanSettings.hpp | 0 .../database}/include/database/Session.hpp | 0 .../include/database/SessionPool.hpp | 0 .../database}/include/database/Track.hpp | 0 .../include/database/TrackArtistLink.hpp | 0 .../include/database/TrackBookmark.hpp | 0 .../include/database/TrackFeatures.hpp | 0 .../database}/include/database/TrackList.hpp | 0 .../database}/include/database/Types.hpp | 0 .../database}/include/database/User.hpp | 0 src/libs/recommendation/CMakeLists.txt | 25 ++++++ .../recommendation/impl}/Engine.cpp | 0 .../recommendation/impl}/Engine.hpp | 0 .../recommendation/impl}/ProviderCreator.cpp | 2 +- .../cluster/SimilarityClusterSearcher.cpp | 0 .../cluster/SimilarityClusterSearcher.hpp | 0 .../impl}/features/AcousticBrainzUtils.cpp | 0 .../impl}/features/AcousticBrainzUtils.hpp | 0 .../features/SimilarityFeaturesCache.cpp | 0 .../features/SimilarityFeaturesCache.hpp | 0 .../impl}/features/SimilarityFeaturesDefs.cpp | 0 .../impl}/features/SimilarityFeaturesDefs.hpp | 0 .../SimilarityFeaturesScannerAddon.cpp | 0 .../features/SimilarityFeaturesSearcher.cpp | 0 .../features/SimilarityFeaturesSearcher.hpp | 0 .../impl}/features/som/DataNormalizer.cpp | 0 .../impl}/features/som/DataNormalizer.hpp | 0 .../impl}/features/som/InputVector.hpp | 0 .../impl}/features/som/Matrix.hpp | 0 .../impl}/features/som/Network.cpp | 0 .../impl}/features/som/Network.hpp | 0 .../ClustersRecommendationProviderCreator.hpp | 0 .../FeaturesRecommendationProviderCreator.hpp | 4 +- .../include/recommendation/IEngine.hpp | 0 .../include/recommendation/Provider.hpp | 0 src/libs/scanner/CMakeLists.txt | 29 +++++++ .../scanner/impl}/MediaScanner.cpp | 8 +- .../scanner/impl}/MediaScanner.hpp | 47 +++------- .../scanner/impl}/MediaScannerStats.cpp | 0 .../scanner}/impl/metadata/AvFormat.cpp | 2 +- .../scanner/impl}/metadata/AvFormat.hpp | 0 .../scanner/impl}/metadata/MetaData.hpp | 0 .../scanner}/impl/metadata/TagLibParser.cpp | 2 +- .../scanner/impl}/metadata/TagLibParser.hpp | 0 .../scanner/include/scanner/IMediaScanner.hpp | 85 +++++++++++++++++++ .../include/scanner/MediaScannerAddon.hpp | 0 .../include/scanner/MediaScannerStats.hpp | 0 src/libs/subsonic/CMakeLists.txt | 30 +++++++ .../subsonic/impl}/SubsonicId.cpp | 0 .../subsonic/impl}/SubsonicId.hpp | 0 .../subsonic/impl}/SubsonicResource.cpp | 2 +- .../subsonic/impl}/SubsonicResponse.cpp | 0 .../subsonic/impl}/SubsonicResponse.hpp | 0 .../include}/subsonic/SubsonicResource.hpp | 0 src/libs/utils/CMakeLists.txt | 31 +++++++ .../impl/utils => libs/utils/impl}/Config.cpp | 0 .../impl/utils => libs/utils/impl}/Config.hpp | 0 .../impl/utils => libs/utils/impl}/Logger.cpp | 0 .../utils => libs/utils/impl}/NetAddress.cpp | 0 .../impl/utils => libs/utils/impl}/Path.cpp | 0 .../impl/utils => libs/utils/impl}/Random.cpp | 0 .../utils/impl}/StreamLogger.cpp | 0 .../impl/utils => libs/utils/impl}/String.cpp | 0 .../impl/utils => libs/utils/impl}/UUID.cpp | 0 .../utils => libs/utils/impl}/WtLogger.cpp | 0 .../utils}/include/utils/Exception.hpp | 0 .../utils}/include/utils/IConfig.hpp | 0 .../utils}/include/utils/Logger.hpp | 0 .../utils}/include/utils/NetAddress.hpp | 0 .../utils}/include/utils/Path.hpp | 0 .../utils}/include/utils/Random.hpp | 0 .../utils}/include/utils/Service.hpp | 0 .../utils}/include/utils/StreamLogger.hpp | 0 .../utils}/include/utils/String.hpp | 0 .../utils}/include/utils/UUID.hpp | 0 .../utils}/include/utils/Utils.hpp | 0 .../utils}/include/utils/WtLogger.hpp | 0 src/lms/CMakeLists.txt | 9 +- src/lms/main.cpp | 10 +-- src/lms/ui/LmsApplication.cpp | 8 +- src/lms/ui/LmsApplication.hpp | 2 +- src/lms/ui/admin/DatabaseSettingsView.cpp | 5 +- src/lms/ui/admin/DatabaseStatus.cpp | 9 +- src/lms/ui/explore/ArtistView.cpp | 1 + src/lms/ui/explore/ReleaseView.cpp | 1 + 131 files changed, 382 insertions(+), 138 deletions(-) delete mode 100644 src/liblms/CMakeLists.txt create mode 100644 src/libs/CMakeLists.txt create mode 100644 src/libs/auth/CMakeLists.txt rename src/{liblms/impl/auth => libs/auth/impl}/AuthTokenService.cpp (100%) rename src/{liblms/impl/auth => libs/auth/impl}/AuthTokenService.hpp (100%) rename src/{liblms/impl/auth => libs/auth/impl}/LoginThrottler.cpp (100%) rename src/{liblms/impl/auth => libs/auth/impl}/LoginThrottler.hpp (100%) rename src/{liblms/impl/auth => libs/auth/impl}/PasswordService.cpp (100%) rename src/{liblms/impl/auth => libs/auth/impl}/PasswordService.hpp (100%) rename src/{liblms => libs/auth}/include/auth/IAuthTokenService.hpp (100%) rename src/{liblms => libs/auth}/include/auth/IPasswordService.hpp (100%) create mode 100644 src/libs/av/CMakeLists.txt rename src/{liblms/impl/av => libs/av/impl}/AvInfo.cpp (100%) rename src/{liblms/impl/av => libs/av/impl}/AvTranscoder.cpp (100%) rename src/{liblms/impl/av => libs/av/impl}/AvTypes.cpp (100%) rename src/{liblms => libs/av}/include/av/AvInfo.hpp (100%) rename src/{liblms => libs/av}/include/av/AvTranscoder.hpp (100%) rename src/{liblms => libs/av}/include/av/AvTypes.hpp (100%) create mode 100644 src/libs/cover/CMakeLists.txt rename src/{liblms/impl/cover => libs/cover/impl}/CoverArtGrabber.cpp (100%) rename src/{liblms/impl/cover => libs/cover/impl}/CoverArtGrabber.hpp (100%) rename src/{liblms/impl/cover => libs/cover/impl}/Image.cpp (100%) rename src/{liblms/impl/cover => libs/cover/impl}/Image.hpp (100%) rename src/{liblms => libs/cover}/include/cover/CoverArt.hpp (100%) rename src/{liblms => libs/cover}/include/cover/ICoverArtGrabber.hpp (100%) create mode 100644 src/libs/database/CMakeLists.txt rename src/{liblms/impl/database => libs/database/impl}/Artist.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/Cluster.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/Db.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/Release.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/ScanSettings.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/Session.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/SessionPool.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/SqlQuery.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/SqlQuery.hpp (100%) rename src/{liblms/impl/database => libs/database/impl}/Track.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/TrackArtistLink.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/TrackBookmark.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/TrackFeatures.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/TrackList.cpp (100%) rename src/{liblms/impl/database => libs/database/impl}/User.cpp (100%) rename src/{liblms => libs/database}/include/database/Artist.hpp (100%) rename src/{liblms => libs/database}/include/database/Cluster.hpp (100%) rename src/{liblms => libs/database}/include/database/Db.hpp (100%) rename src/{liblms => libs/database}/include/database/Release.hpp (100%) rename src/{liblms => libs/database}/include/database/ScanSettings.hpp (100%) rename src/{liblms => libs/database}/include/database/Session.hpp (100%) rename src/{liblms => libs/database}/include/database/SessionPool.hpp (100%) rename src/{liblms => libs/database}/include/database/Track.hpp (100%) rename src/{liblms => libs/database}/include/database/TrackArtistLink.hpp (100%) rename src/{liblms => libs/database}/include/database/TrackBookmark.hpp (100%) rename src/{liblms => libs/database}/include/database/TrackFeatures.hpp (100%) rename src/{liblms => libs/database}/include/database/TrackList.hpp (100%) rename src/{liblms => libs/database}/include/database/Types.hpp (100%) rename src/{liblms => libs/database}/include/database/User.hpp (100%) create mode 100644 src/libs/recommendation/CMakeLists.txt rename src/{liblms/impl/recommendation => libs/recommendation/impl}/Engine.cpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/Engine.hpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/ProviderCreator.cpp (97%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/cluster/SimilarityClusterSearcher.cpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/cluster/SimilarityClusterSearcher.hpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/AcousticBrainzUtils.cpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/AcousticBrainzUtils.hpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/SimilarityFeaturesCache.cpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/SimilarityFeaturesCache.hpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/SimilarityFeaturesDefs.cpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/SimilarityFeaturesDefs.hpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/SimilarityFeaturesScannerAddon.cpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/SimilarityFeaturesSearcher.cpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/SimilarityFeaturesSearcher.hpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/som/DataNormalizer.cpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/som/DataNormalizer.hpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/som/InputVector.hpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/som/Matrix.hpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/som/Network.cpp (100%) rename src/{liblms/impl/recommendation => libs/recommendation/impl}/features/som/Network.hpp (100%) rename src/{liblms => libs/recommendation}/include/recommendation/ClustersRecommendationProviderCreator.hpp (100%) rename src/{liblms => libs/recommendation}/include/recommendation/FeaturesRecommendationProviderCreator.hpp (94%) rename src/{liblms => libs/recommendation}/include/recommendation/IEngine.hpp (100%) rename src/{liblms => libs/recommendation}/include/recommendation/Provider.hpp (100%) create mode 100644 src/libs/scanner/CMakeLists.txt rename src/{liblms/impl/scanner => libs/scanner/impl}/MediaScanner.cpp (99%) rename src/{liblms/include/scanner => libs/scanner/impl}/MediaScanner.hpp (76%) rename src/{liblms/impl/scanner => libs/scanner/impl}/MediaScannerStats.cpp (100%) rename src/{liblms => libs/scanner}/impl/metadata/AvFormat.cpp (99%) rename src/{liblms/include => libs/scanner/impl}/metadata/AvFormat.hpp (100%) rename src/{liblms/include => libs/scanner/impl}/metadata/MetaData.hpp (100%) rename src/{liblms => libs/scanner}/impl/metadata/TagLibParser.cpp (99%) rename src/{liblms/include => libs/scanner/impl}/metadata/TagLibParser.hpp (100%) create mode 100644 src/libs/scanner/include/scanner/IMediaScanner.hpp rename src/{liblms => libs/scanner}/include/scanner/MediaScannerAddon.hpp (100%) rename src/{liblms => libs/scanner}/include/scanner/MediaScannerStats.hpp (100%) create mode 100644 src/libs/subsonic/CMakeLists.txt rename src/{liblms/impl/api/subsonic => libs/subsonic/impl}/SubsonicId.cpp (100%) rename src/{liblms/impl/api/subsonic => libs/subsonic/impl}/SubsonicId.hpp (100%) rename src/{liblms/impl/api/subsonic => libs/subsonic/impl}/SubsonicResource.cpp (99%) rename src/{liblms/impl/api/subsonic => libs/subsonic/impl}/SubsonicResponse.cpp (100%) rename src/{liblms/impl/api/subsonic => libs/subsonic/impl}/SubsonicResponse.hpp (100%) rename src/{liblms/include/api => libs/subsonic/include}/subsonic/SubsonicResource.hpp (100%) create mode 100644 src/libs/utils/CMakeLists.txt rename src/{liblms/impl/utils => libs/utils/impl}/Config.cpp (100%) rename src/{liblms/impl/utils => libs/utils/impl}/Config.hpp (100%) rename src/{liblms/impl/utils => libs/utils/impl}/Logger.cpp (100%) rename src/{liblms/impl/utils => libs/utils/impl}/NetAddress.cpp (100%) rename src/{liblms/impl/utils => libs/utils/impl}/Path.cpp (100%) rename src/{liblms/impl/utils => libs/utils/impl}/Random.cpp (100%) rename src/{liblms/impl/utils => libs/utils/impl}/StreamLogger.cpp (100%) rename src/{liblms/impl/utils => libs/utils/impl}/String.cpp (100%) rename src/{liblms/impl/utils => libs/utils/impl}/UUID.cpp (100%) rename src/{liblms/impl/utils => libs/utils/impl}/WtLogger.cpp (100%) rename src/{liblms => libs/utils}/include/utils/Exception.hpp (100%) rename src/{liblms => libs/utils}/include/utils/IConfig.hpp (100%) rename src/{liblms => libs/utils}/include/utils/Logger.hpp (100%) rename src/{liblms => libs/utils}/include/utils/NetAddress.hpp (100%) rename src/{liblms => libs/utils}/include/utils/Path.hpp (100%) rename src/{liblms => libs/utils}/include/utils/Random.hpp (100%) rename src/{liblms => libs/utils}/include/utils/Service.hpp (100%) rename src/{liblms => libs/utils}/include/utils/StreamLogger.hpp (100%) rename src/{liblms => libs/utils}/include/utils/String.hpp (100%) rename src/{liblms => libs/utils}/include/utils/UUID.hpp (100%) rename src/{liblms => libs/utils}/include/utils/Utils.hpp (100%) rename src/{liblms => libs/utils}/include/utils/WtLogger.hpp (100%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8853a8c7..49f688ab 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,6 @@ add_compile_options(-Wall -Wextra -pedantic -Werror) -SUBDIRS(liblms lms) +add_subdirectory(libs) +add_subdirectory(lms) + diff --git a/src/liblms/CMakeLists.txt b/src/liblms/CMakeLists.txt deleted file mode 100644 index 2c1644ff..00000000 --- a/src/liblms/CMakeLists.txt +++ /dev/null @@ -1,77 +0,0 @@ - -add_library(liblms SHARED - impl/api/subsonic/SubsonicId.cpp - impl/api/subsonic/SubsonicResource.cpp - impl/api/subsonic/SubsonicResponse.cpp - impl/auth/AuthTokenService.cpp - impl/auth/PasswordService.cpp - impl/auth/LoginThrottler.cpp - impl/av/AvInfo.cpp - impl/av/AvTranscoder.cpp - impl/av/AvTypes.cpp - impl/cover/CoverArtGrabber.cpp - impl/cover/Image.cpp - impl/database/Artist.cpp - impl/database/Cluster.cpp - impl/database/Db.cpp - impl/database/TrackArtistLink.cpp - impl/database/TrackFeatures.cpp - impl/database/TrackList.cpp - impl/database/Release.cpp - impl/database/ScanSettings.cpp - impl/database/Session.cpp - impl/database/SessionPool.cpp - impl/database/SqlQuery.cpp - impl/database/Track.cpp - impl/database/TrackBookmark.cpp - impl/database/User.cpp - impl/metadata/AvFormat.cpp - impl/metadata/TagLibParser.cpp - impl/scanner/MediaScanner.cpp - impl/scanner/MediaScannerStats.cpp - impl/recommendation/Engine.cpp - impl/recommendation/ProviderCreator.cpp - impl/recommendation/features/som/DataNormalizer.cpp - impl/recommendation/features/som/Network.cpp - impl/utils/Config.cpp - impl/utils/Logger.cpp - impl/utils/NetAddress.cpp - impl/utils/Path.cpp - impl/utils/Random.cpp - impl/utils/StreamLogger.cpp - impl/utils/String.cpp - impl/utils/UUID.cpp - impl/utils/WtLogger.cpp - ) - -target_include_directories(liblms INTERFACE - ${CMAKE_CURRENT_SOURCE_DIR}/include - ) - -target_include_directories(liblms PRIVATE - include/ - ${IMAGEMAGICKXX_INCLUDE_DIRS} - ) - -target_compile_options(liblms PRIVATE - ${IMAGEMAGICKXX_CFLAGS_OTHER} - ) - -target_link_libraries(liblms PRIVATE - ${IMAGEMAGICKXX_LIBRARIES} - avformat - avutil - config++ - tag - stdc++fs - wtdbosqlite3 - ) - -target_link_libraries(liblms PUBLIC - pthread - boost_system - wtdbo - ) - -install(TARGETS liblms DESTINATION lib) - diff --git a/src/libs/CMakeLists.txt b/src/libs/CMakeLists.txt new file mode 100644 index 00000000..97b099a9 --- /dev/null +++ b/src/libs/CMakeLists.txt @@ -0,0 +1,11 @@ + +add_subdirectory(auth) +add_subdirectory(av) +add_subdirectory(cover) +add_subdirectory(database) +add_subdirectory(recommendation) +add_subdirectory(scanner) +add_subdirectory(subsonic) +add_subdirectory(utils) + + diff --git a/src/libs/auth/CMakeLists.txt b/src/libs/auth/CMakeLists.txt new file mode 100644 index 00000000..9be20d0c --- /dev/null +++ b/src/libs/auth/CMakeLists.txt @@ -0,0 +1,28 @@ + +add_library(lmsauth SHARED + impl/AuthTokenService.cpp + impl/PasswordService.cpp + impl/LoginThrottler.cpp + ) + +target_include_directories(lmsauth INTERFACE + include + ) + +target_include_directories(lmsauth PRIVATE + include/ + ) + +target_link_libraries(lmsauth PRIVATE + lmsutils + lmsdatabase + ) + +target_link_libraries(lmsauth PUBLIC + pthread + boost_system + wt + ) + +install(TARGETS lmsauth DESTINATION lib) + diff --git a/src/liblms/impl/auth/AuthTokenService.cpp b/src/libs/auth/impl/AuthTokenService.cpp similarity index 100% rename from src/liblms/impl/auth/AuthTokenService.cpp rename to src/libs/auth/impl/AuthTokenService.cpp diff --git a/src/liblms/impl/auth/AuthTokenService.hpp b/src/libs/auth/impl/AuthTokenService.hpp similarity index 100% rename from src/liblms/impl/auth/AuthTokenService.hpp rename to src/libs/auth/impl/AuthTokenService.hpp diff --git a/src/liblms/impl/auth/LoginThrottler.cpp b/src/libs/auth/impl/LoginThrottler.cpp similarity index 100% rename from src/liblms/impl/auth/LoginThrottler.cpp rename to src/libs/auth/impl/LoginThrottler.cpp diff --git a/src/liblms/impl/auth/LoginThrottler.hpp b/src/libs/auth/impl/LoginThrottler.hpp similarity index 100% rename from src/liblms/impl/auth/LoginThrottler.hpp rename to src/libs/auth/impl/LoginThrottler.hpp diff --git a/src/liblms/impl/auth/PasswordService.cpp b/src/libs/auth/impl/PasswordService.cpp similarity index 100% rename from src/liblms/impl/auth/PasswordService.cpp rename to src/libs/auth/impl/PasswordService.cpp diff --git a/src/liblms/impl/auth/PasswordService.hpp b/src/libs/auth/impl/PasswordService.hpp similarity index 100% rename from src/liblms/impl/auth/PasswordService.hpp rename to src/libs/auth/impl/PasswordService.hpp diff --git a/src/liblms/include/auth/IAuthTokenService.hpp b/src/libs/auth/include/auth/IAuthTokenService.hpp similarity index 100% rename from src/liblms/include/auth/IAuthTokenService.hpp rename to src/libs/auth/include/auth/IAuthTokenService.hpp diff --git a/src/liblms/include/auth/IPasswordService.hpp b/src/libs/auth/include/auth/IPasswordService.hpp similarity index 100% rename from src/liblms/include/auth/IPasswordService.hpp rename to src/libs/auth/include/auth/IPasswordService.hpp diff --git a/src/libs/av/CMakeLists.txt b/src/libs/av/CMakeLists.txt new file mode 100644 index 00000000..c2d03eb6 --- /dev/null +++ b/src/libs/av/CMakeLists.txt @@ -0,0 +1,24 @@ + +add_library(lmsav SHARED + impl/AvInfo.cpp + impl/AvTranscoder.cpp + impl/AvTypes.cpp + ) + +target_include_directories(lmsav INTERFACE + include + ) + +target_include_directories(lmsav PRIVATE + include/ + ) + +# TODO make these private +target_link_libraries(lmsav PUBLIC + lmsutils + avformat + avutil + ) + +install(TARGETS lmsav DESTINATION lib) + diff --git a/src/liblms/impl/av/AvInfo.cpp b/src/libs/av/impl/AvInfo.cpp similarity index 100% rename from src/liblms/impl/av/AvInfo.cpp rename to src/libs/av/impl/AvInfo.cpp diff --git a/src/liblms/impl/av/AvTranscoder.cpp b/src/libs/av/impl/AvTranscoder.cpp similarity index 100% rename from src/liblms/impl/av/AvTranscoder.cpp rename to src/libs/av/impl/AvTranscoder.cpp diff --git a/src/liblms/impl/av/AvTypes.cpp b/src/libs/av/impl/AvTypes.cpp similarity index 100% rename from src/liblms/impl/av/AvTypes.cpp rename to src/libs/av/impl/AvTypes.cpp diff --git a/src/liblms/include/av/AvInfo.hpp b/src/libs/av/include/av/AvInfo.hpp similarity index 100% rename from src/liblms/include/av/AvInfo.hpp rename to src/libs/av/include/av/AvInfo.hpp diff --git a/src/liblms/include/av/AvTranscoder.hpp b/src/libs/av/include/av/AvTranscoder.hpp similarity index 100% rename from src/liblms/include/av/AvTranscoder.hpp rename to src/libs/av/include/av/AvTranscoder.hpp diff --git a/src/liblms/include/av/AvTypes.hpp b/src/libs/av/include/av/AvTypes.hpp similarity index 100% rename from src/liblms/include/av/AvTypes.hpp rename to src/libs/av/include/av/AvTypes.hpp diff --git a/src/libs/cover/CMakeLists.txt b/src/libs/cover/CMakeLists.txt new file mode 100644 index 00000000..62eb5d43 --- /dev/null +++ b/src/libs/cover/CMakeLists.txt @@ -0,0 +1,27 @@ + +add_library(lmscover SHARED + impl/CoverArtGrabber.cpp + impl/Image.cpp + ) + +target_include_directories(lmscover INTERFACE + include + ) + +target_include_directories(lmscover PRIVATE + include + ${IMAGEMAGICKXX_INCLUDE_DIRS} + ) + +target_compile_options(lmscover PRIVATE + ${IMAGEMAGICKXX_CFLAGS_OTHER} + ) + +target_link_libraries(lmscover PRIVATE + lmsav + lmsdatabase + ${IMAGEMAGICKXX_LIBRARIES} + ) + +install(TARGETS lmscover DESTINATION lib) + diff --git a/src/liblms/impl/cover/CoverArtGrabber.cpp b/src/libs/cover/impl/CoverArtGrabber.cpp similarity index 100% rename from src/liblms/impl/cover/CoverArtGrabber.cpp rename to src/libs/cover/impl/CoverArtGrabber.cpp diff --git a/src/liblms/impl/cover/CoverArtGrabber.hpp b/src/libs/cover/impl/CoverArtGrabber.hpp similarity index 100% rename from src/liblms/impl/cover/CoverArtGrabber.hpp rename to src/libs/cover/impl/CoverArtGrabber.hpp diff --git a/src/liblms/impl/cover/Image.cpp b/src/libs/cover/impl/Image.cpp similarity index 100% rename from src/liblms/impl/cover/Image.cpp rename to src/libs/cover/impl/Image.cpp diff --git a/src/liblms/impl/cover/Image.hpp b/src/libs/cover/impl/Image.hpp similarity index 100% rename from src/liblms/impl/cover/Image.hpp rename to src/libs/cover/impl/Image.hpp diff --git a/src/liblms/include/cover/CoverArt.hpp b/src/libs/cover/include/cover/CoverArt.hpp similarity index 100% rename from src/liblms/include/cover/CoverArt.hpp rename to src/libs/cover/include/cover/CoverArt.hpp diff --git a/src/liblms/include/cover/ICoverArtGrabber.hpp b/src/libs/cover/include/cover/ICoverArtGrabber.hpp similarity index 100% rename from src/liblms/include/cover/ICoverArtGrabber.hpp rename to src/libs/cover/include/cover/ICoverArtGrabber.hpp diff --git a/src/libs/database/CMakeLists.txt b/src/libs/database/CMakeLists.txt new file mode 100644 index 00000000..638fb2e2 --- /dev/null +++ b/src/libs/database/CMakeLists.txt @@ -0,0 +1,37 @@ + +add_library(lmsdatabase SHARED + impl/Artist.cpp + impl/Cluster.cpp + impl/Db.cpp + impl/TrackArtistLink.cpp + impl/TrackFeatures.cpp + impl/TrackList.cpp + impl/Release.cpp + impl/ScanSettings.cpp + impl/Session.cpp + impl/SessionPool.cpp + impl/SqlQuery.cpp + impl/Track.cpp + impl/TrackBookmark.cpp + impl/User.cpp + ) + +target_include_directories(lmsdatabase INTERFACE + include + ) + +target_include_directories(lmsdatabase PRIVATE + include + ) + +target_link_libraries(lmsdatabase PRIVATE + wtdbosqlite3 + ) + +target_link_libraries(lmsdatabase PUBLIC + lmsutils + wtdbo + ) + +install(TARGETS lmsdatabase DESTINATION lib) + diff --git a/src/liblms/impl/database/Artist.cpp b/src/libs/database/impl/Artist.cpp similarity index 100% rename from src/liblms/impl/database/Artist.cpp rename to src/libs/database/impl/Artist.cpp diff --git a/src/liblms/impl/database/Cluster.cpp b/src/libs/database/impl/Cluster.cpp similarity index 100% rename from src/liblms/impl/database/Cluster.cpp rename to src/libs/database/impl/Cluster.cpp diff --git a/src/liblms/impl/database/Db.cpp b/src/libs/database/impl/Db.cpp similarity index 100% rename from src/liblms/impl/database/Db.cpp rename to src/libs/database/impl/Db.cpp diff --git a/src/liblms/impl/database/Release.cpp b/src/libs/database/impl/Release.cpp similarity index 100% rename from src/liblms/impl/database/Release.cpp rename to src/libs/database/impl/Release.cpp diff --git a/src/liblms/impl/database/ScanSettings.cpp b/src/libs/database/impl/ScanSettings.cpp similarity index 100% rename from src/liblms/impl/database/ScanSettings.cpp rename to src/libs/database/impl/ScanSettings.cpp diff --git a/src/liblms/impl/database/Session.cpp b/src/libs/database/impl/Session.cpp similarity index 100% rename from src/liblms/impl/database/Session.cpp rename to src/libs/database/impl/Session.cpp diff --git a/src/liblms/impl/database/SessionPool.cpp b/src/libs/database/impl/SessionPool.cpp similarity index 100% rename from src/liblms/impl/database/SessionPool.cpp rename to src/libs/database/impl/SessionPool.cpp diff --git a/src/liblms/impl/database/SqlQuery.cpp b/src/libs/database/impl/SqlQuery.cpp similarity index 100% rename from src/liblms/impl/database/SqlQuery.cpp rename to src/libs/database/impl/SqlQuery.cpp diff --git a/src/liblms/impl/database/SqlQuery.hpp b/src/libs/database/impl/SqlQuery.hpp similarity index 100% rename from src/liblms/impl/database/SqlQuery.hpp rename to src/libs/database/impl/SqlQuery.hpp diff --git a/src/liblms/impl/database/Track.cpp b/src/libs/database/impl/Track.cpp similarity index 100% rename from src/liblms/impl/database/Track.cpp rename to src/libs/database/impl/Track.cpp diff --git a/src/liblms/impl/database/TrackArtistLink.cpp b/src/libs/database/impl/TrackArtistLink.cpp similarity index 100% rename from src/liblms/impl/database/TrackArtistLink.cpp rename to src/libs/database/impl/TrackArtistLink.cpp diff --git a/src/liblms/impl/database/TrackBookmark.cpp b/src/libs/database/impl/TrackBookmark.cpp similarity index 100% rename from src/liblms/impl/database/TrackBookmark.cpp rename to src/libs/database/impl/TrackBookmark.cpp diff --git a/src/liblms/impl/database/TrackFeatures.cpp b/src/libs/database/impl/TrackFeatures.cpp similarity index 100% rename from src/liblms/impl/database/TrackFeatures.cpp rename to src/libs/database/impl/TrackFeatures.cpp diff --git a/src/liblms/impl/database/TrackList.cpp b/src/libs/database/impl/TrackList.cpp similarity index 100% rename from src/liblms/impl/database/TrackList.cpp rename to src/libs/database/impl/TrackList.cpp diff --git a/src/liblms/impl/database/User.cpp b/src/libs/database/impl/User.cpp similarity index 100% rename from src/liblms/impl/database/User.cpp rename to src/libs/database/impl/User.cpp diff --git a/src/liblms/include/database/Artist.hpp b/src/libs/database/include/database/Artist.hpp similarity index 100% rename from src/liblms/include/database/Artist.hpp rename to src/libs/database/include/database/Artist.hpp diff --git a/src/liblms/include/database/Cluster.hpp b/src/libs/database/include/database/Cluster.hpp similarity index 100% rename from src/liblms/include/database/Cluster.hpp rename to src/libs/database/include/database/Cluster.hpp diff --git a/src/liblms/include/database/Db.hpp b/src/libs/database/include/database/Db.hpp similarity index 100% rename from src/liblms/include/database/Db.hpp rename to src/libs/database/include/database/Db.hpp diff --git a/src/liblms/include/database/Release.hpp b/src/libs/database/include/database/Release.hpp similarity index 100% rename from src/liblms/include/database/Release.hpp rename to src/libs/database/include/database/Release.hpp diff --git a/src/liblms/include/database/ScanSettings.hpp b/src/libs/database/include/database/ScanSettings.hpp similarity index 100% rename from src/liblms/include/database/ScanSettings.hpp rename to src/libs/database/include/database/ScanSettings.hpp diff --git a/src/liblms/include/database/Session.hpp b/src/libs/database/include/database/Session.hpp similarity index 100% rename from src/liblms/include/database/Session.hpp rename to src/libs/database/include/database/Session.hpp diff --git a/src/liblms/include/database/SessionPool.hpp b/src/libs/database/include/database/SessionPool.hpp similarity index 100% rename from src/liblms/include/database/SessionPool.hpp rename to src/libs/database/include/database/SessionPool.hpp diff --git a/src/liblms/include/database/Track.hpp b/src/libs/database/include/database/Track.hpp similarity index 100% rename from src/liblms/include/database/Track.hpp rename to src/libs/database/include/database/Track.hpp diff --git a/src/liblms/include/database/TrackArtistLink.hpp b/src/libs/database/include/database/TrackArtistLink.hpp similarity index 100% rename from src/liblms/include/database/TrackArtistLink.hpp rename to src/libs/database/include/database/TrackArtistLink.hpp diff --git a/src/liblms/include/database/TrackBookmark.hpp b/src/libs/database/include/database/TrackBookmark.hpp similarity index 100% rename from src/liblms/include/database/TrackBookmark.hpp rename to src/libs/database/include/database/TrackBookmark.hpp diff --git a/src/liblms/include/database/TrackFeatures.hpp b/src/libs/database/include/database/TrackFeatures.hpp similarity index 100% rename from src/liblms/include/database/TrackFeatures.hpp rename to src/libs/database/include/database/TrackFeatures.hpp diff --git a/src/liblms/include/database/TrackList.hpp b/src/libs/database/include/database/TrackList.hpp similarity index 100% rename from src/liblms/include/database/TrackList.hpp rename to src/libs/database/include/database/TrackList.hpp diff --git a/src/liblms/include/database/Types.hpp b/src/libs/database/include/database/Types.hpp similarity index 100% rename from src/liblms/include/database/Types.hpp rename to src/libs/database/include/database/Types.hpp diff --git a/src/liblms/include/database/User.hpp b/src/libs/database/include/database/User.hpp similarity index 100% rename from src/liblms/include/database/User.hpp rename to src/libs/database/include/database/User.hpp diff --git a/src/libs/recommendation/CMakeLists.txt b/src/libs/recommendation/CMakeLists.txt new file mode 100644 index 00000000..c83ee6aa --- /dev/null +++ b/src/libs/recommendation/CMakeLists.txt @@ -0,0 +1,25 @@ + +add_library(lmsrecommendation SHARED + impl/Engine.cpp + impl/ProviderCreator.cpp + impl/features/som/DataNormalizer.cpp + impl/features/som/Network.cpp + ) + +target_include_directories(lmsrecommendation INTERFACE + include + ) + +target_include_directories(lmsrecommendation PRIVATE + include + ) + +target_link_libraries(lmsrecommendation PRIVATE + lmsdatabase + ) + +target_link_libraries(lmsrecommendation PUBLIC + ) + +install(TARGETS lmsrecommendation DESTINATION lib) + diff --git a/src/liblms/impl/recommendation/Engine.cpp b/src/libs/recommendation/impl/Engine.cpp similarity index 100% rename from src/liblms/impl/recommendation/Engine.cpp rename to src/libs/recommendation/impl/Engine.cpp diff --git a/src/liblms/impl/recommendation/Engine.hpp b/src/libs/recommendation/impl/Engine.hpp similarity index 100% rename from src/liblms/impl/recommendation/Engine.hpp rename to src/libs/recommendation/impl/Engine.hpp diff --git a/src/liblms/impl/recommendation/ProviderCreator.cpp b/src/libs/recommendation/impl/ProviderCreator.cpp similarity index 97% rename from src/liblms/impl/recommendation/ProviderCreator.cpp rename to src/libs/recommendation/impl/ProviderCreator.cpp index b451e480..00d0b0e9 100644 --- a/src/liblms/impl/recommendation/ProviderCreator.cpp +++ b/src/libs/recommendation/impl/ProviderCreator.cpp @@ -29,7 +29,7 @@ namespace Recommendation return {}; } - std::unique_ptr createFeaturesRecommendationProvider(Scanner::MediaScanner&) + std::unique_ptr createFeaturesRecommendationProvider(Scanner::IMediaScanner&) { return {}; } diff --git a/src/liblms/impl/recommendation/cluster/SimilarityClusterSearcher.cpp b/src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.cpp similarity index 100% rename from src/liblms/impl/recommendation/cluster/SimilarityClusterSearcher.cpp rename to src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.cpp diff --git a/src/liblms/impl/recommendation/cluster/SimilarityClusterSearcher.hpp b/src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.hpp similarity index 100% rename from src/liblms/impl/recommendation/cluster/SimilarityClusterSearcher.hpp rename to src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.hpp diff --git a/src/liblms/impl/recommendation/features/AcousticBrainzUtils.cpp b/src/libs/recommendation/impl/features/AcousticBrainzUtils.cpp similarity index 100% rename from src/liblms/impl/recommendation/features/AcousticBrainzUtils.cpp rename to src/libs/recommendation/impl/features/AcousticBrainzUtils.cpp diff --git a/src/liblms/impl/recommendation/features/AcousticBrainzUtils.hpp b/src/libs/recommendation/impl/features/AcousticBrainzUtils.hpp similarity index 100% rename from src/liblms/impl/recommendation/features/AcousticBrainzUtils.hpp rename to src/libs/recommendation/impl/features/AcousticBrainzUtils.hpp diff --git a/src/liblms/impl/recommendation/features/SimilarityFeaturesCache.cpp b/src/libs/recommendation/impl/features/SimilarityFeaturesCache.cpp similarity index 100% rename from src/liblms/impl/recommendation/features/SimilarityFeaturesCache.cpp rename to src/libs/recommendation/impl/features/SimilarityFeaturesCache.cpp diff --git a/src/liblms/impl/recommendation/features/SimilarityFeaturesCache.hpp b/src/libs/recommendation/impl/features/SimilarityFeaturesCache.hpp similarity index 100% rename from src/liblms/impl/recommendation/features/SimilarityFeaturesCache.hpp rename to src/libs/recommendation/impl/features/SimilarityFeaturesCache.hpp diff --git a/src/liblms/impl/recommendation/features/SimilarityFeaturesDefs.cpp b/src/libs/recommendation/impl/features/SimilarityFeaturesDefs.cpp similarity index 100% rename from src/liblms/impl/recommendation/features/SimilarityFeaturesDefs.cpp rename to src/libs/recommendation/impl/features/SimilarityFeaturesDefs.cpp diff --git a/src/liblms/impl/recommendation/features/SimilarityFeaturesDefs.hpp b/src/libs/recommendation/impl/features/SimilarityFeaturesDefs.hpp similarity index 100% rename from src/liblms/impl/recommendation/features/SimilarityFeaturesDefs.hpp rename to src/libs/recommendation/impl/features/SimilarityFeaturesDefs.hpp diff --git a/src/liblms/impl/recommendation/features/SimilarityFeaturesScannerAddon.cpp b/src/libs/recommendation/impl/features/SimilarityFeaturesScannerAddon.cpp similarity index 100% rename from src/liblms/impl/recommendation/features/SimilarityFeaturesScannerAddon.cpp rename to src/libs/recommendation/impl/features/SimilarityFeaturesScannerAddon.cpp diff --git a/src/liblms/impl/recommendation/features/SimilarityFeaturesSearcher.cpp b/src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.cpp similarity index 100% rename from src/liblms/impl/recommendation/features/SimilarityFeaturesSearcher.cpp rename to src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.cpp diff --git a/src/liblms/impl/recommendation/features/SimilarityFeaturesSearcher.hpp b/src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.hpp similarity index 100% rename from src/liblms/impl/recommendation/features/SimilarityFeaturesSearcher.hpp rename to src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.hpp diff --git a/src/liblms/impl/recommendation/features/som/DataNormalizer.cpp b/src/libs/recommendation/impl/features/som/DataNormalizer.cpp similarity index 100% rename from src/liblms/impl/recommendation/features/som/DataNormalizer.cpp rename to src/libs/recommendation/impl/features/som/DataNormalizer.cpp diff --git a/src/liblms/impl/recommendation/features/som/DataNormalizer.hpp b/src/libs/recommendation/impl/features/som/DataNormalizer.hpp similarity index 100% rename from src/liblms/impl/recommendation/features/som/DataNormalizer.hpp rename to src/libs/recommendation/impl/features/som/DataNormalizer.hpp diff --git a/src/liblms/impl/recommendation/features/som/InputVector.hpp b/src/libs/recommendation/impl/features/som/InputVector.hpp similarity index 100% rename from src/liblms/impl/recommendation/features/som/InputVector.hpp rename to src/libs/recommendation/impl/features/som/InputVector.hpp diff --git a/src/liblms/impl/recommendation/features/som/Matrix.hpp b/src/libs/recommendation/impl/features/som/Matrix.hpp similarity index 100% rename from src/liblms/impl/recommendation/features/som/Matrix.hpp rename to src/libs/recommendation/impl/features/som/Matrix.hpp diff --git a/src/liblms/impl/recommendation/features/som/Network.cpp b/src/libs/recommendation/impl/features/som/Network.cpp similarity index 100% rename from src/liblms/impl/recommendation/features/som/Network.cpp rename to src/libs/recommendation/impl/features/som/Network.cpp diff --git a/src/liblms/impl/recommendation/features/som/Network.hpp b/src/libs/recommendation/impl/features/som/Network.hpp similarity index 100% rename from src/liblms/impl/recommendation/features/som/Network.hpp rename to src/libs/recommendation/impl/features/som/Network.hpp diff --git a/src/liblms/include/recommendation/ClustersRecommendationProviderCreator.hpp b/src/libs/recommendation/include/recommendation/ClustersRecommendationProviderCreator.hpp similarity index 100% rename from src/liblms/include/recommendation/ClustersRecommendationProviderCreator.hpp rename to src/libs/recommendation/include/recommendation/ClustersRecommendationProviderCreator.hpp diff --git a/src/liblms/include/recommendation/FeaturesRecommendationProviderCreator.hpp b/src/libs/recommendation/include/recommendation/FeaturesRecommendationProviderCreator.hpp similarity index 94% rename from src/liblms/include/recommendation/FeaturesRecommendationProviderCreator.hpp rename to src/libs/recommendation/include/recommendation/FeaturesRecommendationProviderCreator.hpp index 01c07787..756bf605 100644 --- a/src/liblms/include/recommendation/FeaturesRecommendationProviderCreator.hpp +++ b/src/libs/recommendation/include/recommendation/FeaturesRecommendationProviderCreator.hpp @@ -28,13 +28,13 @@ namespace Database namespace Scanner { - class MediaScanner; + class IMediaScanner; } namespace Recommendation { class Provider; - std::unique_ptr createFeaturesRecommendationProvider(Scanner::MediaScanner& scanner); + std::unique_ptr createFeaturesRecommendationProvider(Scanner::IMediaScanner& scanner); } diff --git a/src/liblms/include/recommendation/IEngine.hpp b/src/libs/recommendation/include/recommendation/IEngine.hpp similarity index 100% rename from src/liblms/include/recommendation/IEngine.hpp rename to src/libs/recommendation/include/recommendation/IEngine.hpp diff --git a/src/liblms/include/recommendation/Provider.hpp b/src/libs/recommendation/include/recommendation/Provider.hpp similarity index 100% rename from src/liblms/include/recommendation/Provider.hpp rename to src/libs/recommendation/include/recommendation/Provider.hpp diff --git a/src/libs/scanner/CMakeLists.txt b/src/libs/scanner/CMakeLists.txt new file mode 100644 index 00000000..ff0df591 --- /dev/null +++ b/src/libs/scanner/CMakeLists.txt @@ -0,0 +1,29 @@ + +add_library(lmsscanner SHARED + impl/metadata/AvFormat.cpp + impl/metadata/TagLibParser.cpp + impl/MediaScanner.cpp + impl/MediaScannerStats.cpp + ) + +target_include_directories(lmsscanner INTERFACE + include + ) + +target_include_directories(lmsscanner PRIVATE + include + ) + +target_link_libraries(lmsscanner PRIVATE + lmsav + lmsdatabase + lmsutils + tag + ) + +target_link_libraries(lmsscanner PUBLIC + wt + ) + +install(TARGETS lmsscanner DESTINATION lib) + diff --git a/src/liblms/impl/scanner/MediaScanner.cpp b/src/libs/scanner/impl/MediaScanner.cpp similarity index 99% rename from src/liblms/impl/scanner/MediaScanner.cpp rename to src/libs/scanner/impl/MediaScanner.cpp index a88893e3..a1a88753 100644 --- a/src/liblms/impl/scanner/MediaScanner.cpp +++ b/src/libs/scanner/impl/MediaScanner.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "scanner/MediaScanner.hpp" +#include "MediaScanner.hpp" #include @@ -191,6 +191,12 @@ getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames) namespace Scanner { +std::unique_ptr +createMediaScanner(Database::Db& db) +{ + return std::make_unique(db); +} + MediaScanner::MediaScanner(Database::Db& db) : _dbSession {db} { diff --git a/src/liblms/include/scanner/MediaScanner.hpp b/src/libs/scanner/impl/MediaScanner.hpp similarity index 76% rename from src/liblms/include/scanner/MediaScanner.hpp rename to src/libs/scanner/impl/MediaScanner.hpp index 0ad1586a..f7b13d3a 100644 --- a/src/liblms/include/scanner/MediaScanner.hpp +++ b/src/libs/scanner/impl/MediaScanner.hpp @@ -29,56 +29,33 @@ #include +#include "scanner/IMediaScanner.hpp" #include "database/ScanSettings.hpp" #include "database/Session.hpp" #include "metadata/TagLibParser.hpp" -#include "MediaScannerAddon.hpp" -#include "MediaScannerStats.hpp" namespace Scanner { -class MediaScanner +class MediaScanner : public IMediaScanner { public: MediaScanner(Database::Db& db); - void setAddon(MediaScannerAddon& addon); + void setAddon(MediaScannerAddon& addon) override; - void start(); - void stop(); - void restart(); + void start() override; + void stop() override; + void restart() override; - // Async requests - void requestImmediateScan(); - void requestReschedule(); + void requestImmediateScan() override; + void requestReschedule() override ; + Status getStatus() override; - enum class State - { - NotScheduled, - Scheduled, - InProgress, - }; - - struct Status - { - State currentState {State::NotScheduled}; - Wt::WDateTime nextScheduledScan; - std::optional lastCompleteScanStats; - std::optional inProgressScanStats; - }; - - Status getStatus(); - - // Called just after scan complete - Wt::Signal<>& scanComplete() { return _sigScanComplete; } - - // Called during scan in progress - Wt::Signal& scanInProgress() { return _sigScanInProgress; } - - // Called after a schedule - Wt::Signal& scheduled() { return _sigScheduled; } + Wt::Signal<>& scanComplete() override { return _sigScanComplete; } + Wt::Signal& scanInProgress() override { return _sigScanInProgress; } + Wt::Signal& scheduled() override { return _sigScheduled; } private: diff --git a/src/liblms/impl/scanner/MediaScannerStats.cpp b/src/libs/scanner/impl/MediaScannerStats.cpp similarity index 100% rename from src/liblms/impl/scanner/MediaScannerStats.cpp rename to src/libs/scanner/impl/MediaScannerStats.cpp diff --git a/src/liblms/impl/metadata/AvFormat.cpp b/src/libs/scanner/impl/metadata/AvFormat.cpp similarity index 99% rename from src/liblms/impl/metadata/AvFormat.cpp rename to src/libs/scanner/impl/metadata/AvFormat.cpp index 85311c7a..37af1554 100644 --- a/src/liblms/impl/metadata/AvFormat.cpp +++ b/src/libs/scanner/impl/metadata/AvFormat.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "metadata/AvFormat.hpp" +#include "AvFormat.hpp" #include #include diff --git a/src/liblms/include/metadata/AvFormat.hpp b/src/libs/scanner/impl/metadata/AvFormat.hpp similarity index 100% rename from src/liblms/include/metadata/AvFormat.hpp rename to src/libs/scanner/impl/metadata/AvFormat.hpp diff --git a/src/liblms/include/metadata/MetaData.hpp b/src/libs/scanner/impl/metadata/MetaData.hpp similarity index 100% rename from src/liblms/include/metadata/MetaData.hpp rename to src/libs/scanner/impl/metadata/MetaData.hpp diff --git a/src/liblms/impl/metadata/TagLibParser.cpp b/src/libs/scanner/impl/metadata/TagLibParser.cpp similarity index 99% rename from src/liblms/impl/metadata/TagLibParser.cpp rename to src/libs/scanner/impl/metadata/TagLibParser.cpp index 78949ce8..75104d53 100644 --- a/src/liblms/impl/metadata/TagLibParser.cpp +++ b/src/libs/scanner/impl/metadata/TagLibParser.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "metadata/TagLibParser.hpp" +#include "TagLibParser.hpp" #include #include diff --git a/src/liblms/include/metadata/TagLibParser.hpp b/src/libs/scanner/impl/metadata/TagLibParser.hpp similarity index 100% rename from src/liblms/include/metadata/TagLibParser.hpp rename to src/libs/scanner/impl/metadata/TagLibParser.hpp diff --git a/src/libs/scanner/include/scanner/IMediaScanner.hpp b/src/libs/scanner/include/scanner/IMediaScanner.hpp new file mode 100644 index 00000000..49b68048 --- /dev/null +++ b/src/libs/scanner/include/scanner/IMediaScanner.hpp @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2013 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +#include +#include + +#include "MediaScannerAddon.hpp" +#include "MediaScannerStats.hpp" + +namespace Database +{ + class Db; +} + +namespace Scanner { + +class IMediaScanner +{ + public: + virtual ~IMediaScanner() = default; + + virtual void setAddon(MediaScannerAddon& addon) = 0; + + virtual void start() = 0; + virtual void stop() = 0; + virtual void restart() = 0; + + // Async requests + virtual void requestImmediateScan() = 0; + virtual void requestReschedule() = 0; + + + enum class State + { + NotScheduled, + Scheduled, + InProgress, + }; + + struct Status + { + State currentState {State::NotScheduled}; + Wt::WDateTime nextScheduledScan; + std::optional lastCompleteScanStats; + std::optional inProgressScanStats; + }; + + virtual Status getStatus() = 0; + + // Called just after scan complete + virtual Wt::Signal<>& scanComplete() = 0; + + // Called during scan in progress + virtual Wt::Signal& scanInProgress() = 0; + + // Called after a schedule + virtual Wt::Signal& scheduled() = 0; + +}; + +std::unique_ptr createMediaScanner(Database::Db& db); + + +} // Scanner + diff --git a/src/liblms/include/scanner/MediaScannerAddon.hpp b/src/libs/scanner/include/scanner/MediaScannerAddon.hpp similarity index 100% rename from src/liblms/include/scanner/MediaScannerAddon.hpp rename to src/libs/scanner/include/scanner/MediaScannerAddon.hpp diff --git a/src/liblms/include/scanner/MediaScannerStats.hpp b/src/libs/scanner/include/scanner/MediaScannerStats.hpp similarity index 100% rename from src/liblms/include/scanner/MediaScannerStats.hpp rename to src/libs/scanner/include/scanner/MediaScannerStats.hpp diff --git a/src/libs/subsonic/CMakeLists.txt b/src/libs/subsonic/CMakeLists.txt new file mode 100644 index 00000000..0531b7bf --- /dev/null +++ b/src/libs/subsonic/CMakeLists.txt @@ -0,0 +1,30 @@ + +add_library(lmssubsonic SHARED + impl/SubsonicId.cpp + impl/SubsonicResource.cpp + impl/SubsonicResponse.cpp + ) + +target_include_directories(lmssubsonic INTERFACE + include + ) + +target_include_directories(lmssubsonic PRIVATE + include + ) + +target_link_libraries(lmssubsonic PRIVATE + lmsauth + lmsav + lmscover + lmsrecommendation + lmsutils + ) + +target_link_libraries(lmssubsonic PUBLIC + lmsdatabase + wt + ) + +install(TARGETS lmssubsonic DESTINATION lib) + diff --git a/src/liblms/impl/api/subsonic/SubsonicId.cpp b/src/libs/subsonic/impl/SubsonicId.cpp similarity index 100% rename from src/liblms/impl/api/subsonic/SubsonicId.cpp rename to src/libs/subsonic/impl/SubsonicId.cpp diff --git a/src/liblms/impl/api/subsonic/SubsonicId.hpp b/src/libs/subsonic/impl/SubsonicId.hpp similarity index 100% rename from src/liblms/impl/api/subsonic/SubsonicId.hpp rename to src/libs/subsonic/impl/SubsonicId.hpp diff --git a/src/liblms/impl/api/subsonic/SubsonicResource.cpp b/src/libs/subsonic/impl/SubsonicResource.cpp similarity index 99% rename from src/liblms/impl/api/subsonic/SubsonicResource.cpp rename to src/libs/subsonic/impl/SubsonicResource.cpp index 3a4db863..056005c5 100644 --- a/src/liblms/impl/api/subsonic/SubsonicResource.cpp +++ b/src/libs/subsonic/impl/SubsonicResource.cpp @@ -16,7 +16,7 @@ * You should have received a copy of the GNU General Public License * along with LMS. If not, see . */ -#include "api/subsonic/SubsonicResource.hpp" +#include "subsonic/SubsonicResource.hpp" #include #include diff --git a/src/liblms/impl/api/subsonic/SubsonicResponse.cpp b/src/libs/subsonic/impl/SubsonicResponse.cpp similarity index 100% rename from src/liblms/impl/api/subsonic/SubsonicResponse.cpp rename to src/libs/subsonic/impl/SubsonicResponse.cpp diff --git a/src/liblms/impl/api/subsonic/SubsonicResponse.hpp b/src/libs/subsonic/impl/SubsonicResponse.hpp similarity index 100% rename from src/liblms/impl/api/subsonic/SubsonicResponse.hpp rename to src/libs/subsonic/impl/SubsonicResponse.hpp diff --git a/src/liblms/include/api/subsonic/SubsonicResource.hpp b/src/libs/subsonic/include/subsonic/SubsonicResource.hpp similarity index 100% rename from src/liblms/include/api/subsonic/SubsonicResource.hpp rename to src/libs/subsonic/include/subsonic/SubsonicResource.hpp diff --git a/src/libs/utils/CMakeLists.txt b/src/libs/utils/CMakeLists.txt new file mode 100644 index 00000000..ba5c5f01 --- /dev/null +++ b/src/libs/utils/CMakeLists.txt @@ -0,0 +1,31 @@ + +add_library(lmsutils SHARED + impl/Config.cpp + impl/Logger.cpp + impl/NetAddress.cpp + impl/Path.cpp + impl/Random.cpp + impl/StreamLogger.cpp + impl/String.cpp + impl/UUID.cpp + impl/WtLogger.cpp + ) + +target_include_directories(lmsutils INTERFACE + include + ) + +target_include_directories(lmsutils PRIVATE + include + ) + +target_link_libraries(lmsutils PRIVATE + config++ + ) + +target_link_libraries(lmsutils PUBLIC + stdc++fs + ) + +install(TARGETS lmsutils DESTINATION lib) + diff --git a/src/liblms/impl/utils/Config.cpp b/src/libs/utils/impl/Config.cpp similarity index 100% rename from src/liblms/impl/utils/Config.cpp rename to src/libs/utils/impl/Config.cpp diff --git a/src/liblms/impl/utils/Config.hpp b/src/libs/utils/impl/Config.hpp similarity index 100% rename from src/liblms/impl/utils/Config.hpp rename to src/libs/utils/impl/Config.hpp diff --git a/src/liblms/impl/utils/Logger.cpp b/src/libs/utils/impl/Logger.cpp similarity index 100% rename from src/liblms/impl/utils/Logger.cpp rename to src/libs/utils/impl/Logger.cpp diff --git a/src/liblms/impl/utils/NetAddress.cpp b/src/libs/utils/impl/NetAddress.cpp similarity index 100% rename from src/liblms/impl/utils/NetAddress.cpp rename to src/libs/utils/impl/NetAddress.cpp diff --git a/src/liblms/impl/utils/Path.cpp b/src/libs/utils/impl/Path.cpp similarity index 100% rename from src/liblms/impl/utils/Path.cpp rename to src/libs/utils/impl/Path.cpp diff --git a/src/liblms/impl/utils/Random.cpp b/src/libs/utils/impl/Random.cpp similarity index 100% rename from src/liblms/impl/utils/Random.cpp rename to src/libs/utils/impl/Random.cpp diff --git a/src/liblms/impl/utils/StreamLogger.cpp b/src/libs/utils/impl/StreamLogger.cpp similarity index 100% rename from src/liblms/impl/utils/StreamLogger.cpp rename to src/libs/utils/impl/StreamLogger.cpp diff --git a/src/liblms/impl/utils/String.cpp b/src/libs/utils/impl/String.cpp similarity index 100% rename from src/liblms/impl/utils/String.cpp rename to src/libs/utils/impl/String.cpp diff --git a/src/liblms/impl/utils/UUID.cpp b/src/libs/utils/impl/UUID.cpp similarity index 100% rename from src/liblms/impl/utils/UUID.cpp rename to src/libs/utils/impl/UUID.cpp diff --git a/src/liblms/impl/utils/WtLogger.cpp b/src/libs/utils/impl/WtLogger.cpp similarity index 100% rename from src/liblms/impl/utils/WtLogger.cpp rename to src/libs/utils/impl/WtLogger.cpp diff --git a/src/liblms/include/utils/Exception.hpp b/src/libs/utils/include/utils/Exception.hpp similarity index 100% rename from src/liblms/include/utils/Exception.hpp rename to src/libs/utils/include/utils/Exception.hpp diff --git a/src/liblms/include/utils/IConfig.hpp b/src/libs/utils/include/utils/IConfig.hpp similarity index 100% rename from src/liblms/include/utils/IConfig.hpp rename to src/libs/utils/include/utils/IConfig.hpp diff --git a/src/liblms/include/utils/Logger.hpp b/src/libs/utils/include/utils/Logger.hpp similarity index 100% rename from src/liblms/include/utils/Logger.hpp rename to src/libs/utils/include/utils/Logger.hpp diff --git a/src/liblms/include/utils/NetAddress.hpp b/src/libs/utils/include/utils/NetAddress.hpp similarity index 100% rename from src/liblms/include/utils/NetAddress.hpp rename to src/libs/utils/include/utils/NetAddress.hpp diff --git a/src/liblms/include/utils/Path.hpp b/src/libs/utils/include/utils/Path.hpp similarity index 100% rename from src/liblms/include/utils/Path.hpp rename to src/libs/utils/include/utils/Path.hpp diff --git a/src/liblms/include/utils/Random.hpp b/src/libs/utils/include/utils/Random.hpp similarity index 100% rename from src/liblms/include/utils/Random.hpp rename to src/libs/utils/include/utils/Random.hpp diff --git a/src/liblms/include/utils/Service.hpp b/src/libs/utils/include/utils/Service.hpp similarity index 100% rename from src/liblms/include/utils/Service.hpp rename to src/libs/utils/include/utils/Service.hpp diff --git a/src/liblms/include/utils/StreamLogger.hpp b/src/libs/utils/include/utils/StreamLogger.hpp similarity index 100% rename from src/liblms/include/utils/StreamLogger.hpp rename to src/libs/utils/include/utils/StreamLogger.hpp diff --git a/src/liblms/include/utils/String.hpp b/src/libs/utils/include/utils/String.hpp similarity index 100% rename from src/liblms/include/utils/String.hpp rename to src/libs/utils/include/utils/String.hpp diff --git a/src/liblms/include/utils/UUID.hpp b/src/libs/utils/include/utils/UUID.hpp similarity index 100% rename from src/liblms/include/utils/UUID.hpp rename to src/libs/utils/include/utils/UUID.hpp diff --git a/src/liblms/include/utils/Utils.hpp b/src/libs/utils/include/utils/Utils.hpp similarity index 100% rename from src/liblms/include/utils/Utils.hpp rename to src/libs/utils/include/utils/Utils.hpp diff --git a/src/liblms/include/utils/WtLogger.hpp b/src/libs/utils/include/utils/WtLogger.hpp similarity index 100% rename from src/liblms/include/utils/WtLogger.hpp rename to src/libs/utils/include/utils/WtLogger.hpp diff --git a/src/lms/CMakeLists.txt b/src/lms/CMakeLists.txt index 94526da9..d422a93b 100644 --- a/src/lms/CMakeLists.txt +++ b/src/lms/CMakeLists.txt @@ -38,7 +38,14 @@ target_include_directories(lms PRIVATE ) target_link_libraries(lms PRIVATE - liblms + lmsav + lmsauth + lmscover + lmsdatabase + lmsrecommendation + lmsscanner + lmssubsonic + lmsutils wt wthttp ) diff --git a/src/lms/main.cpp b/src/lms/main.cpp index 5f5bf313..126c7402 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -22,17 +22,17 @@ #include #include -#include "api/subsonic/SubsonicResource.hpp" -#include "av/AvInfo.hpp" -#include "av/AvTranscoder.hpp" #include "auth/IAuthTokenService.hpp" #include "auth/IPasswordService.hpp" +#include "av/AvInfo.hpp" +#include "av/AvTranscoder.hpp" #include "cover/ICoverArtGrabber.hpp" #include "database/Db.hpp" -#include "scanner/MediaScanner.hpp" +#include "scanner/IMediaScanner.hpp" #include "recommendation/FeaturesRecommendationProviderCreator.hpp" #include "recommendation/IEngine.hpp" #include "recommendation/ClustersRecommendationProviderCreator.hpp" +#include "subsonic/SubsonicResource.hpp" #include "ui/LmsApplication.hpp" #include "utils/IConfig.hpp" #include "utils/Service.hpp" @@ -143,7 +143,7 @@ int main(int argc, char* argv[]) // Service initialization order is important ServiceProvider::assign(Auth::createAuthTokenService(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000))); ServiceProvider::assign(Auth::createPasswordService(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000))); - Scanner::MediaScanner& mediaScanner {ServiceProvider::create(database)}; + Scanner::IMediaScanner& mediaScanner {ServiceProvider::assign(Scanner::createMediaScanner(database))}; Recommendation::IEngine& recommendationEngine {ServiceProvider::assign(Recommendation::createEngine())}; recommendationEngine.addProvider(Recommendation::createFeaturesRecommendationProvider(mediaScanner), 0); diff --git a/src/lms/ui/LmsApplication.cpp b/src/lms/ui/LmsApplication.cpp index 56480d94..8a6dbcc2 100644 --- a/src/lms/ui/LmsApplication.cpp +++ b/src/lms/ui/LmsApplication.cpp @@ -528,7 +528,7 @@ LmsApplication::createHome() // Events from MediaScanner { const std::string sessionId {LmsApp->sessionId()}; - ServiceProvider::get()->scanComplete().connect(this, [=] () + ServiceProvider::get()->scanComplete().connect(this, [=] () { Wt::WServer::instance()->post(sessionId, [=] { @@ -537,7 +537,7 @@ LmsApplication::createHome() }); }); - ServiceProvider::get()->scanInProgress().connect(this, [=] (Scanner::ScanProgressStats stats) + ServiceProvider::get()->scanInProgress().connect(this, [=] (Scanner::ScanProgressStats stats) { Wt::WServer::instance()->post(sessionId, [=] { @@ -546,7 +546,7 @@ LmsApplication::createHome() }); }); - ServiceProvider::get()->scheduled().connect(this, [=] (Wt::WDateTime dateTime) + ServiceProvider::get()->scheduled().connect(this, [=] (Wt::WDateTime dateTime) { Wt::WServer::instance()->post(sessionId, [=] { @@ -561,7 +561,7 @@ LmsApplication::createHome() { if (isUserAdmin()) { - const auto& stats {*ServiceProvider::get()->getStatus().lastCompleteScanStats}; + const auto& stats {*ServiceProvider::get()->getStatus().lastCompleteScanStats}; notifyMsg(MsgType::Info, Wt::WString::tr("Lms.Admin.Database.scan-complete") .arg(static_cast(stats.nbFiles())) diff --git a/src/lms/ui/LmsApplication.hpp b/src/lms/ui/LmsApplication.hpp index 9e6eb57d..cd0f04ca 100644 --- a/src/lms/ui/LmsApplication.hpp +++ b/src/lms/ui/LmsApplication.hpp @@ -25,7 +25,7 @@ #include "database/Db.hpp" #include "database/Session.hpp" -#include "scanner/MediaScanner.hpp" +#include "scanner/IMediaScanner.hpp" #include "LmsApplicationGroup.hpp" diff --git a/src/lms/ui/admin/DatabaseSettingsView.cpp b/src/lms/ui/admin/DatabaseSettingsView.cpp index d4ac7200..3c4423e1 100644 --- a/src/lms/ui/admin/DatabaseSettingsView.cpp +++ b/src/lms/ui/admin/DatabaseSettingsView.cpp @@ -27,6 +27,7 @@ #include #include "database/Cluster.hpp" +#include "database/ScanSettings.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" #include "utils/String.hpp" @@ -229,7 +230,7 @@ DatabaseSettingsView::refreshView() { model->saveData(); - ServiceProvider::get()->requestReschedule(); + ServiceProvider::get()->requestReschedule(); LmsApp->notifyMsg(MsgType::Success, Wt::WString::tr("Lms.Admin.Database.settings-saved")); } @@ -246,7 +247,7 @@ DatabaseSettingsView::refreshView() immScanBtn->clicked().connect([=] () { - ServiceProvider::get()->requestImmediateScan(); + ServiceProvider::get()->requestImmediateScan(); LmsApp->notifyMsg(MsgType::Info, Wt::WString::tr("Lms.Admin.Database.scan-launched")); }); diff --git a/src/lms/ui/admin/DatabaseStatus.cpp b/src/lms/ui/admin/DatabaseStatus.cpp index 23577bfa..34133e01 100644 --- a/src/lms/ui/admin/DatabaseStatus.cpp +++ b/src/lms/ui/admin/DatabaseStatus.cpp @@ -27,7 +27,6 @@ #include #include "utils/Service.hpp" -#include "scanner/MediaScanner.hpp" #include "LmsApplication.hpp" namespace UserInterface { @@ -136,7 +135,7 @@ DatabaseStatus::refreshContents() Wt::WPushButton* reportBtn {bindNew("btn-report", Wt::WString::tr("Lms.Admin.Database.Status.get-report"))}; - const MediaScanner::Status status {ServiceProvider::get()->getStatus()}; + const IMediaScanner::Status status {ServiceProvider::get()->getStatus()}; if (status.lastCompleteScanStats) { bindString("last-scan", Wt::WString::tr("Lms.Admin.Database.Status.last-scan-status") @@ -160,14 +159,14 @@ DatabaseStatus::refreshContents() switch (status.currentState) { - case MediaScanner::State::NotScheduled: + case IMediaScanner::State::NotScheduled: bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-not-scheduled")); break; - case MediaScanner::State::Scheduled: + case IMediaScanner::State::Scheduled: bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-scheduled") .arg(status.nextScheduledScan.toString())); break; - case MediaScanner::State::InProgress: + case IMediaScanner::State::InProgress: { std::ostringstream oss; bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-in-progress") diff --git a/src/lms/ui/explore/ArtistView.cpp b/src/lms/ui/explore/ArtistView.cpp index 7b40bd85..089f8714 100644 --- a/src/lms/ui/explore/ArtistView.cpp +++ b/src/lms/ui/explore/ArtistView.cpp @@ -26,6 +26,7 @@ #include "database/Artist.hpp" #include "database/Release.hpp" +#include "database/ScanSettings.hpp" #include "utils/Logger.hpp" #include "utils/String.hpp" diff --git a/src/lms/ui/explore/ReleaseView.cpp b/src/lms/ui/explore/ReleaseView.cpp index 8cb45f8d..11238e9a 100644 --- a/src/lms/ui/explore/ReleaseView.cpp +++ b/src/lms/ui/explore/ReleaseView.cpp @@ -26,6 +26,7 @@ #include #include "database/Release.hpp" +#include "database/ScanSettings.hpp" #include "database/Track.hpp" #include "utils/Logger.hpp" From f44addc2a257630b07b38e4bc37468512c0528a5 Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 13 Feb 2020 20:04:20 +0100 Subject: [PATCH 04/23] Restored unit tests --- CMakeLists.txt | 3 ++- src/libs/CMakeLists.txt | 1 + src/libs/database/CMakeLists.txt | 1 + src/libs/database/test/CMakeLists.txt | 11 +++++++++++ .../libs/database/test}/DatabaseTest.cpp | 0 src/libs/recommendation/CMakeLists.txt | 6 +----- src/libs/som/CMakeLists.txt | 19 +++++++++++++++++++ .../som => som/impl}/DataNormalizer.cpp | 2 +- .../features/som => som/impl}/Network.cpp | 2 +- .../include}/som/DataNormalizer.hpp | 0 .../include}/som/InputVector.hpp | 2 ++ .../features => som/include}/som/Matrix.hpp | 0 .../features => som/include}/som/Network.hpp | 1 - src/libs/som/test/CMakeLists.txt | 11 +++++++++++ {test/som => src/libs/som/test}/SomTest.cpp | 4 ++-- src/libs/utils/CMakeLists.txt | 2 ++ 16 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 src/libs/database/test/CMakeLists.txt rename {test/database => src/libs/database/test}/DatabaseTest.cpp (100%) create mode 100644 src/libs/som/CMakeLists.txt rename src/libs/{recommendation/impl/features/som => som/impl}/DataNormalizer.cpp (98%) rename src/libs/{recommendation/impl/features/som => som/impl}/Network.cpp (99%) rename src/libs/{recommendation/impl/features => som/include}/som/DataNormalizer.hpp (100%) rename src/libs/{recommendation/impl/features => som/include}/som/InputVector.hpp (99%) rename src/libs/{recommendation/impl/features => som/include}/som/Matrix.hpp (100%) rename src/libs/{recommendation/impl/features => som/include}/som/Network.hpp (99%) create mode 100644 src/libs/som/test/CMakeLists.txt rename {test/som => src/libs/som/test}/SomTest.cpp (98%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7eef4529..c41adcdb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,8 +8,9 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) find_package(PkgConfig) pkg_check_modules(IMAGEMAGICKXX REQUIRED ImageMagick++) +enable_testing() -SUBDIRS(src) +add_subdirectory(src) install(DIRECTORY approot DESTINATION share/lms) install(DIRECTORY docroot DESTINATION share/lms) diff --git a/src/libs/CMakeLists.txt b/src/libs/CMakeLists.txt index 97b099a9..2619ec6e 100644 --- a/src/libs/CMakeLists.txt +++ b/src/libs/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(cover) add_subdirectory(database) add_subdirectory(recommendation) add_subdirectory(scanner) +add_subdirectory(som) add_subdirectory(subsonic) add_subdirectory(utils) diff --git a/src/libs/database/CMakeLists.txt b/src/libs/database/CMakeLists.txt index 638fb2e2..aae05b4f 100644 --- a/src/libs/database/CMakeLists.txt +++ b/src/libs/database/CMakeLists.txt @@ -35,3 +35,4 @@ target_link_libraries(lmsdatabase PUBLIC install(TARGETS lmsdatabase DESTINATION lib) +add_subdirectory(test) diff --git a/src/libs/database/test/CMakeLists.txt b/src/libs/database/test/CMakeLists.txt new file mode 100644 index 00000000..ac42aa6b --- /dev/null +++ b/src/libs/database/test/CMakeLists.txt @@ -0,0 +1,11 @@ + +add_executable(test-database + DatabaseTest.cpp + ) + +target_link_libraries(test-database PRIVATE + lmsdatabase + ) + +add_test(NAME database COMMAND test-database) + diff --git a/test/database/DatabaseTest.cpp b/src/libs/database/test/DatabaseTest.cpp similarity index 100% rename from test/database/DatabaseTest.cpp rename to src/libs/database/test/DatabaseTest.cpp diff --git a/src/libs/recommendation/CMakeLists.txt b/src/libs/recommendation/CMakeLists.txt index c83ee6aa..cdcf20bd 100644 --- a/src/libs/recommendation/CMakeLists.txt +++ b/src/libs/recommendation/CMakeLists.txt @@ -2,8 +2,6 @@ add_library(lmsrecommendation SHARED impl/Engine.cpp impl/ProviderCreator.cpp - impl/features/som/DataNormalizer.cpp - impl/features/som/Network.cpp ) target_include_directories(lmsrecommendation INTERFACE @@ -16,9 +14,7 @@ target_include_directories(lmsrecommendation PRIVATE target_link_libraries(lmsrecommendation PRIVATE lmsdatabase - ) - -target_link_libraries(lmsrecommendation PUBLIC + lmssom ) install(TARGETS lmsrecommendation DESTINATION lib) diff --git a/src/libs/som/CMakeLists.txt b/src/libs/som/CMakeLists.txt new file mode 100644 index 00000000..2d7ca9ef --- /dev/null +++ b/src/libs/som/CMakeLists.txt @@ -0,0 +1,19 @@ + +add_library(lmssom STATIC + impl/DataNormalizer.cpp + impl/Network.cpp + ) + +target_include_directories(lmssom INTERFACE + include + ) + +target_include_directories(lmssom PRIVATE + include + ) + +target_link_libraries(lmssom PUBLIC + lmsutils + ) + +add_subdirectory(test) diff --git a/src/libs/recommendation/impl/features/som/DataNormalizer.cpp b/src/libs/som/impl/DataNormalizer.cpp similarity index 98% rename from src/libs/recommendation/impl/features/som/DataNormalizer.cpp rename to src/libs/som/impl/DataNormalizer.cpp index d6498fcf..a4c39351 100644 --- a/src/libs/recommendation/impl/features/som/DataNormalizer.cpp +++ b/src/libs/som/impl/DataNormalizer.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "DataNormalizer.hpp" +#include "som/DataNormalizer.hpp" #include #include diff --git a/src/libs/recommendation/impl/features/som/Network.cpp b/src/libs/som/impl/Network.cpp similarity index 99% rename from src/libs/recommendation/impl/features/som/Network.cpp rename to src/libs/som/impl/Network.cpp index 0431c966..deea6f7b 100644 --- a/src/libs/recommendation/impl/features/som/Network.cpp +++ b/src/libs/som/impl/Network.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "Network.hpp" +#include "som/Network.hpp" #include #include diff --git a/src/libs/recommendation/impl/features/som/DataNormalizer.hpp b/src/libs/som/include/som/DataNormalizer.hpp similarity index 100% rename from src/libs/recommendation/impl/features/som/DataNormalizer.hpp rename to src/libs/som/include/som/DataNormalizer.hpp diff --git a/src/libs/recommendation/impl/features/som/InputVector.hpp b/src/libs/som/include/som/InputVector.hpp similarity index 99% rename from src/libs/recommendation/impl/features/som/InputVector.hpp rename to src/libs/som/include/som/InputVector.hpp index 68d5834e..058e0fee 100644 --- a/src/libs/recommendation/impl/features/som/InputVector.hpp +++ b/src/libs/som/include/som/InputVector.hpp @@ -23,6 +23,8 @@ #include #include +#include "utils/Exception.hpp" + namespace SOM { diff --git a/src/libs/recommendation/impl/features/som/Matrix.hpp b/src/libs/som/include/som/Matrix.hpp similarity index 100% rename from src/libs/recommendation/impl/features/som/Matrix.hpp rename to src/libs/som/include/som/Matrix.hpp diff --git a/src/libs/recommendation/impl/features/som/Network.hpp b/src/libs/som/include/som/Network.hpp similarity index 99% rename from src/libs/recommendation/impl/features/som/Network.hpp rename to src/libs/som/include/som/Network.hpp index cf64a83c..cf04cf72 100644 --- a/src/libs/recommendation/impl/features/som/Network.hpp +++ b/src/libs/som/include/som/Network.hpp @@ -25,7 +25,6 @@ #include #include -#include "utils/Exception.hpp" #include "InputVector.hpp" #include "Matrix.hpp" diff --git a/src/libs/som/test/CMakeLists.txt b/src/libs/som/test/CMakeLists.txt new file mode 100644 index 00000000..aaaf43d4 --- /dev/null +++ b/src/libs/som/test/CMakeLists.txt @@ -0,0 +1,11 @@ + +add_executable(test-som + SomTest.cpp + ) + +target_link_libraries(test-som PRIVATE + lmssom + ) + +add_test(NAME som COMMAND test-som) + diff --git a/test/som/SomTest.cpp b/src/libs/som/test/SomTest.cpp similarity index 98% rename from test/som/SomTest.cpp rename to src/libs/som/test/SomTest.cpp index 70aad7a1..6f4edb37 100644 --- a/test/som/SomTest.cpp +++ b/src/libs/som/test/SomTest.cpp @@ -21,8 +21,8 @@ #include #include -#include "DataNormalizer.hpp" -#include "Network.hpp" +#include "som/DataNormalizer.hpp" +#include "som/Network.hpp" using namespace SOM; diff --git a/src/libs/utils/CMakeLists.txt b/src/libs/utils/CMakeLists.txt index ba5c5f01..c468be2f 100644 --- a/src/libs/utils/CMakeLists.txt +++ b/src/libs/utils/CMakeLists.txt @@ -24,7 +24,9 @@ target_link_libraries(lmsutils PRIVATE ) target_link_libraries(lmsutils PUBLIC + boost_system stdc++fs + wt ) install(TARGETS lmsutils DESTINATION lib) From 8f86850da047a30f5dae327f41d717315b6d3cc4 Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 13 Feb 2020 20:06:10 +0100 Subject: [PATCH 05/23] Removed autotools --- Makefile.am | 56 ----------------------------- configure.ac | 94 ------------------------------------------------ src/Makefile.am | 7 ---- test/Makefile.am | 34 ------------------ 4 files changed, 191 deletions(-) delete mode 100644 Makefile.am delete mode 100644 configure.ac delete mode 100644 src/Makefile.am delete mode 100644 test/Makefile.am diff --git a/Makefile.am b/Makefile.am deleted file mode 100644 index a82bd6d3..00000000 --- a/Makefile.am +++ /dev/null @@ -1,56 +0,0 @@ - -AUTOMAKE_OPTIONS = dist-bzip2 no-dist-gzip - -SUBDIRS = test tools src - -lms_docrootdir=$(pkgdatadir)/docroot -lms_approotdir=$(pkgdatadir)/approot - -lms_cssdir=$(lms_docrootdir)/css -lms_jsdir=$(lms_docrootdir)/js -lms_imagesdir=$(lms_approotdir)/images - -dist_pkgdata_DATA = \ - systemd/default.service \ - conf/lms.conf - -dist_lms_css_DATA = \ - docroot/css/lms.css - -dist_lms_js_DATA = \ - docroot/js/bootstrap-notify.js \ - docroot/js/jquery-1.10.2.min.js \ - docroot/js/mediaplayer.js - -dist_lms_images_DATA = \ - approot/images/unknown-cover.jpg \ - approot/images/unknown-artist.jpg - -dist_lms_approot_DATA = \ - approot/admin-database.xml \ - approot/admin-user.xml \ - approot/admin-users.xml \ - approot/admin-initwizard.xml \ - approot/artist.xml \ - approot/artistinfo.xml \ - approot/artistlink.xml \ - approot/artists.xml \ - approot/artistsinfo.xml \ - approot/error.xml \ - approot/explore.xml \ - approot/login.xml \ - approot/mediaplayer.xml \ - approot/messages.xml \ - approot/messages_fr.xml \ - approot/playhistory.xml \ - approot/playqueue.xml \ - approot/release.xml \ - approot/releaseinfo.xml \ - approot/releaselink.xml \ - approot/releases.xml \ - approot/releasesinfo.xml \ - approot/settings.xml \ - approot/templates.xml \ - approot/tracks.xml \ - approot/tracksinfo.xml - diff --git a/configure.ac b/configure.ac deleted file mode 100644 index 638b5bed..00000000 --- a/configure.ac +++ /dev/null @@ -1,94 +0,0 @@ -AC_PREREQ(2.59) -AC_INIT(lms, 3.6.2, test@test) -AM_INIT_AUTOMAKE -AC_CONFIG_HEADER(src/config/config.h) - -AC_LANG_CPLUSPLUS - -AC_PROG_CXX - -PKG_CHECK_MODULES(IMAGEMAGICKXX, "ImageMagick++", [ HAVE_IMAGEMAGICKXX=yes ], [ ]) -if test -n "$HAVE_IMAGEMAGICKXX"; then - MAGICKXX_CFLAGS="$IMAGEMAGICKXX_CFLAGS" - MAGICKXX_LIBS="$IMAGEMAGICKXX_LIBS" -fi -AC_SUBST(MAGICKXX_CFLAGS) -AC_SUBST(MAGICKXX_LIBS) - -AC_CHECK_HEADERS([Wt/WApplication.h pstreams/pstream.h boost/asio.hpp], - [], - [AC_MSG_ERROR([Header not found or unusable !])]) - -AC_CHECK_LIB([stdc++fs], - [main], - , - [AC_MSG_ERROR([lib filesystem not found!])]) - -AC_CHECK_LIB([pthread], - [pthread_rwlock_unlock], - , - [AC_MSG_ERROR([libpthread not found!])]) - -AC_CHECK_LIB([boost_system], - [main], - , - [AC_MSG_ERROR([libboost_system not found!])]) - -AC_CHECK_LIB([wt], - [main], - , - [AC_MSG_ERROR([libwt not found!])]) - -AC_CHECK_LIB([wtdbo], - [main], - , - [AC_MSG_ERROR([libwtdbo not found!])]) - -AC_CHECK_LIB([wtdbosqlite3], - [main], - , - [AC_MSG_ERROR([libwtdbosqlite3 not found!])]) - -AC_CHECK_LIB([wthttp], - [main], - , - [AC_MSG_ERROR([libwthttp not found!])]) - - -AC_CHECK_LIB([avutil], - [av_free], - , - [AC_MSG_ERROR([libavutil not found!])]) - -AC_CHECK_LIB([avformat], - [av_read_frame], - , - [AC_MSG_ERROR([libavformat not found!])]) - -AC_CHECK_LIB( [tag], - [main], - , - [AC_MSG_ERROR([libtag not found!])]) - -AC_CHECK_LIB( [config++], - [main], - , - [AC_MSG_ERROR([libconfig++ not found!])]) - -AC_CONFIG_FILES([Makefile - src/Makefile - test/Makefile - tools/Makefile - tools/similarity/Makefile - tools/similarity-parameters/Makefile - tools/metadata/Makefile]) - -AC_ARG_ENABLE([tools], - [AC_HELP_STRING([--enable-tools], [Build the tools])], - [:], - [enable_tools=no]) - -AM_CONDITIONAL([BUILD_TOOLS], [test "$enable_tools" = "yes"]) - -AC_OUTPUT - diff --git a/src/Makefile.am b/src/Makefile.am deleted file mode 100644 index b5644219..00000000 --- a/src/Makefile.am +++ /dev/null @@ -1,7 +0,0 @@ -bin_PROGRAMS = lms - -lms_SOURCES = \ - -lms_CXXFLAGS=-std=c++17 -I$(srcdir)/ui $(MAGICKXX_CFLAGS) -D_REENTRANT -lms_LDADD=$(MAGICKXX_LIBS) - diff --git a/test/Makefile.am b/test/Makefile.am deleted file mode 100644 index 06d88e41..00000000 --- a/test/Makefile.am +++ /dev/null @@ -1,34 +0,0 @@ - -TESTS = test-som test-database - -check_PROGRAMS = test-som test-database - -test_som_SOURCES = \ - $(srcdir)/som/SomTest.cpp \ - $(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp \ - $(top_srcdir)/src/similarity/features/som/Network.cpp - -test_som_CXXFLAGS=-std=c++17 -I${top_srcdir}/src/ -I${top_srcdir}/src/similarity/features/som/ - - -test_database_SOURCES = \ - $(srcdir)/database/DatabaseTest.cpp \ - $(top_srcdir)/src/database/Artist.cpp \ - $(top_srcdir)/src/database/Cluster.cpp \ - $(top_srcdir)/src/database/Db.cpp \ - $(top_srcdir)/src/database/TrackArtistLink.cpp \ - $(top_srcdir)/src/database/TrackFeatures.cpp \ - $(top_srcdir)/src/database/TrackList.cpp \ - $(top_srcdir)/src/database/Release.cpp \ - $(top_srcdir)/src/database/ScanSettings.cpp \ - $(top_srcdir)/src/database/Session.cpp \ - $(top_srcdir)/src/database/SqlQuery.cpp \ - $(top_srcdir)/src/database/Track.cpp \ - $(top_srcdir)/src/database/TrackBookmark.cpp \ - $(top_srcdir)/src/database/User.cpp \ - $(top_srcdir)/src/utils/Logger.cpp \ - $(top_srcdir)/src/utils/StreamLogger.cpp \ - $(top_srcdir)/src/utils/String.cpp - -test_database_CXXFLAGS=-std=c++17 -I${top_srcdir}/src/ - From 942d0b4631a642308433516e608fd2df335866f3 Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 13 Feb 2020 20:10:41 +0100 Subject: [PATCH 06/23] Adapted travis ci script --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f53ac887..1193accd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,15 +16,13 @@ matrix: - gcc before_install: - eval "${MATRIX_EVAL}" -- sudo apt-get install build-essential autoconf automake cmake libboost-all-dev libconfig++-dev libavcodec-dev libavutil-dev libavformat-dev ffmpeg libmagick++-dev libpstreams-dev libconfig++-dev libpstreams-dev libtag1-dev +- sudo apt-get install build-essential cmake libboost-all-dev libconfig++-dev libavcodec-dev libavutil-dev libavformat-dev ffmpeg libmagick++-dev libpstreams-dev libconfig++-dev libpstreams-dev libtag1-dev - git clone https://github.com/emweb/wt.git wt - pushd wt; - git checkout 4.1.0 - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr && sudo make install - popd -script: autoreconf -vfi && CXXFLAGS="-Wall -Wextra -Werror" ./configure --enable-tools && make distcheck +script: cmake . && make && make test env: global: - MAKEFLAGS="-j 2" -after_failure: - - cat config.log From 4c535da6dc923ee9eaa7a5f605d7620cf9a09819 Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 13 Feb 2020 20:31:49 +0100 Subject: [PATCH 07/23] Removed Werror that was used for testing purpose --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 49f688ab..d9c97633 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -add_compile_options(-Wall -Wextra -pedantic -Werror) +add_compile_options(-Wall -Wextra -pedantict) add_subdirectory(libs) add_subdirectory(lms) From ed3d728c028e6cc23ff35ac35e1ce6d0e83c69ea Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 13 Feb 2020 21:03:58 +0100 Subject: [PATCH 08/23] Fixed typo --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d9c97633..48f68d61 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -add_compile_options(-Wall -Wextra -pedantict) +add_compile_options(-Wall -Wextra -pedantic) add_subdirectory(libs) add_subdirectory(lms) From d06dfde187b5456bd19db2932b54592ffb9efd41 Mon Sep 17 00:00:00 2001 From: emeric Date: Fri, 14 Feb 2020 13:53:46 +0100 Subject: [PATCH 09/23] Added tools --- .travis.yml | 2 +- src/CMakeLists.txt | 1 + src/libs/CMakeLists.txt | 1 + src/libs/metadata/CMakeLists.txt | 25 +++++++++++++++++++ .../impl/AvFormatParser.cpp} | 5 ++-- .../impl}/TagLibParser.cpp | 2 +- .../include/metadata/AvFormatParser.hpp} | 4 +-- .../include/metadata/IParser.hpp} | 3 +-- .../include}/metadata/TagLibParser.hpp | 4 +-- src/libs/scanner/CMakeLists.txt | 5 +--- src/libs/scanner/impl/MediaScanner.hpp | 2 +- src/tools/CMakeLists.txt | 4 +++ src/tools/Makefile.am | 4 --- src/tools/metadata/CMakeLists.txt | 11 ++++++++ src/tools/metadata/LmsMetadata.cpp | 7 +++--- src/tools/metadata/Makefile.am | 14 ----------- 16 files changed, 56 insertions(+), 38 deletions(-) create mode 100644 src/libs/metadata/CMakeLists.txt rename src/libs/{scanner/impl/metadata/AvFormat.cpp => metadata/impl/AvFormatParser.cpp} (98%) rename src/libs/{scanner/impl/metadata => metadata/impl}/TagLibParser.cpp (99%) rename src/libs/{scanner/impl/metadata/AvFormat.hpp => metadata/include/metadata/AvFormatParser.hpp} (92%) rename src/libs/{scanner/impl/metadata/MetaData.hpp => metadata/include/metadata/IParser.hpp} (98%) rename src/libs/{scanner/impl => metadata/include}/metadata/TagLibParser.hpp (92%) create mode 100644 src/tools/CMakeLists.txt delete mode 100644 src/tools/Makefile.am create mode 100644 src/tools/metadata/CMakeLists.txt delete mode 100644 src/tools/metadata/Makefile.am diff --git a/.travis.yml b/.travis.yml index 1193accd..f2f798b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ before_install: - git checkout 4.1.0 - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr && sudo make install - popd -script: cmake . && make && make test +script: cmake -DCMAKE_BUILD_TYPE=Release . && make && make test env: global: - MAKEFLAGS="-j 2" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 48f68d61..d88e62ec 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,5 +2,6 @@ add_compile_options(-Wall -Wextra -pedantic) add_subdirectory(libs) add_subdirectory(lms) +add_subdirectory(tools) diff --git a/src/libs/CMakeLists.txt b/src/libs/CMakeLists.txt index 2619ec6e..7302b579 100644 --- a/src/libs/CMakeLists.txt +++ b/src/libs/CMakeLists.txt @@ -3,6 +3,7 @@ add_subdirectory(auth) add_subdirectory(av) add_subdirectory(cover) add_subdirectory(database) +add_subdirectory(metadata) add_subdirectory(recommendation) add_subdirectory(scanner) add_subdirectory(som) diff --git a/src/libs/metadata/CMakeLists.txt b/src/libs/metadata/CMakeLists.txt new file mode 100644 index 00000000..cc6a1dbb --- /dev/null +++ b/src/libs/metadata/CMakeLists.txt @@ -0,0 +1,25 @@ + +add_library(lmsmetadata SHARED + impl/AvFormatParser.cpp + impl/TagLibParser.cpp + ) + +target_include_directories(lmsmetadata INTERFACE + include + ) + +target_include_directories(lmsmetadata PRIVATE + include + ) + +target_link_libraries(lmsmetadata PRIVATE + lmsav + tag + ) + +target_link_libraries(lmsmetadata PUBLIC + lmsutils + ) + +install(TARGETS lmsmetadata DESTINATION lib) + diff --git a/src/libs/scanner/impl/metadata/AvFormat.cpp b/src/libs/metadata/impl/AvFormatParser.cpp similarity index 98% rename from src/libs/scanner/impl/metadata/AvFormat.cpp rename to src/libs/metadata/impl/AvFormatParser.cpp index 37af1554..7dfd8af4 100644 --- a/src/libs/scanner/impl/metadata/AvFormat.cpp +++ b/src/libs/metadata/impl/AvFormatParser.cpp @@ -17,13 +17,12 @@ * along with LMS. If not, see . */ -#include "AvFormat.hpp" +#include "metadata/AvFormatParser.hpp" #include #include #include "av/AvInfo.hpp" - #include "utils/Logger.hpp" #include "utils/String.hpp" @@ -128,7 +127,7 @@ getArtists(const MetadataMap& metadataMap) } std::optional -AvFormat::parse(const std::filesystem::path& p, bool debug) +AvFormatParser::parse(const std::filesystem::path& p, bool debug) { Track track; diff --git a/src/libs/scanner/impl/metadata/TagLibParser.cpp b/src/libs/metadata/impl/TagLibParser.cpp similarity index 99% rename from src/libs/scanner/impl/metadata/TagLibParser.cpp rename to src/libs/metadata/impl/TagLibParser.cpp index 75104d53..78949ce8 100644 --- a/src/libs/scanner/impl/metadata/TagLibParser.cpp +++ b/src/libs/metadata/impl/TagLibParser.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "TagLibParser.hpp" +#include "metadata/TagLibParser.hpp" #include #include diff --git a/src/libs/scanner/impl/metadata/AvFormat.hpp b/src/libs/metadata/include/metadata/AvFormatParser.hpp similarity index 92% rename from src/libs/scanner/impl/metadata/AvFormat.hpp rename to src/libs/metadata/include/metadata/AvFormatParser.hpp index 28c25a8e..009d4c2e 100644 --- a/src/libs/scanner/impl/metadata/AvFormat.hpp +++ b/src/libs/metadata/include/metadata/AvFormatParser.hpp @@ -19,13 +19,13 @@ #pragma once -#include "MetaData.hpp" +#include "metadata/IParser.hpp" namespace MetaData { // Parse that makes use of AvFormat -class AvFormat : public Parser +class AvFormatParser : public IParser { public: std::optional parse(const std::filesystem::path& p, bool debug = false) override; diff --git a/src/libs/scanner/impl/metadata/MetaData.hpp b/src/libs/metadata/include/metadata/IParser.hpp similarity index 98% rename from src/libs/scanner/impl/metadata/MetaData.hpp rename to src/libs/metadata/include/metadata/IParser.hpp index b4f652ad..4a8b5362 100644 --- a/src/libs/scanner/impl/metadata/MetaData.hpp +++ b/src/libs/metadata/include/metadata/IParser.hpp @@ -26,7 +26,6 @@ #include #include -//#include "utils/Utils.hpp" #include "utils/UUID.hpp" namespace MetaData @@ -73,7 +72,7 @@ namespace MetaData std::string copyrightURL; }; - class Parser + class IParser { public: virtual std::optional parse(const std::filesystem::path& p, bool debug = false) = 0; diff --git a/src/libs/scanner/impl/metadata/TagLibParser.hpp b/src/libs/metadata/include/metadata/TagLibParser.hpp similarity index 92% rename from src/libs/scanner/impl/metadata/TagLibParser.hpp rename to src/libs/metadata/include/metadata/TagLibParser.hpp index 5dad7a3f..1c055b76 100644 --- a/src/libs/scanner/impl/metadata/TagLibParser.hpp +++ b/src/libs/metadata/include/metadata/TagLibParser.hpp @@ -19,13 +19,13 @@ #pragma once -#include "MetaData.hpp" +#include "metadata/IParser.hpp" namespace MetaData { // Parse that makes use of AvFormat -class TagLibParser : public Parser +class TagLibParser : public IParser { public: std::optional parse(const std::filesystem::path& p, bool debug = false) override; diff --git a/src/libs/scanner/CMakeLists.txt b/src/libs/scanner/CMakeLists.txt index ff0df591..3d7a77f3 100644 --- a/src/libs/scanner/CMakeLists.txt +++ b/src/libs/scanner/CMakeLists.txt @@ -1,7 +1,5 @@ add_library(lmsscanner SHARED - impl/metadata/AvFormat.cpp - impl/metadata/TagLibParser.cpp impl/MediaScanner.cpp impl/MediaScannerStats.cpp ) @@ -15,10 +13,9 @@ target_include_directories(lmsscanner PRIVATE ) target_link_libraries(lmsscanner PRIVATE - lmsav lmsdatabase + lmsmetadata lmsutils - tag ) target_link_libraries(lmsscanner PUBLIC diff --git a/src/libs/scanner/impl/MediaScanner.hpp b/src/libs/scanner/impl/MediaScanner.hpp index f7b13d3a..c6452156 100644 --- a/src/libs/scanner/impl/MediaScanner.hpp +++ b/src/libs/scanner/impl/MediaScanner.hpp @@ -29,10 +29,10 @@ #include -#include "scanner/IMediaScanner.hpp" #include "database/ScanSettings.hpp" #include "database/Session.hpp" #include "metadata/TagLibParser.hpp" +#include "scanner/IMediaScanner.hpp" namespace Scanner { diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt new file mode 100644 index 00000000..f66ea24a --- /dev/null +++ b/src/tools/CMakeLists.txt @@ -0,0 +1,4 @@ + +add_subdirectory(metadata) + + diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am deleted file mode 100644 index c382f8bc..00000000 --- a/src/tools/Makefile.am +++ /dev/null @@ -1,4 +0,0 @@ -if BUILD_TOOLS -SUBDIRS = similarity similarity-parameters metadata -endif - diff --git a/src/tools/metadata/CMakeLists.txt b/src/tools/metadata/CMakeLists.txt new file mode 100644 index 00000000..4c0706cf --- /dev/null +++ b/src/tools/metadata/CMakeLists.txt @@ -0,0 +1,11 @@ + +add_executable(lms-metadata + LmsMetadata.cpp + ) + +target_link_libraries(lms-metadata PRIVATE + lmsmetadata + ) + +install(TARGETS lms-metadata DESTINATION bin) + diff --git a/src/tools/metadata/LmsMetadata.cpp b/src/tools/metadata/LmsMetadata.cpp index 9828c7f9..7a04e178 100644 --- a/src/tools/metadata/LmsMetadata.cpp +++ b/src/tools/metadata/LmsMetadata.cpp @@ -25,8 +25,7 @@ #include -#include "av/AvInfo.hpp" -#include "metadata/AvFormat.hpp" +#include "metadata/AvFormatParser.hpp" #include "metadata/TagLibParser.hpp" #include "utils/StreamLogger.hpp" @@ -52,7 +51,7 @@ std::ostream& operator<<(std::ostream& os, const MetaData::Album& album) -void parse(MetaData::Parser& parser, const std::filesystem::path& file) +void parse(MetaData::IParser& parser, const std::filesystem::path& file) { using namespace MetaData; @@ -155,7 +154,7 @@ int main(int argc, char *argv[]) { std::cout << "Using av:" << std::endl; - MetaData::AvFormat parser; + MetaData::AvFormatParser parser; parse(parser, file); } diff --git a/src/tools/metadata/Makefile.am b/src/tools/metadata/Makefile.am deleted file mode 100644 index e2a5fd10..00000000 --- a/src/tools/metadata/Makefile.am +++ /dev/null @@ -1,14 +0,0 @@ -bin_PROGRAMS = lms-metadata - -lms_metadata_SOURCES = \ - $(srcdir)/LmsMetadata.cpp \ - $(top_srcdir)/src/av/AvInfo.cpp \ - $(top_srcdir)/src/metadata/AvFormat.cpp \ - $(top_srcdir)/src/metadata/TagLibParser.cpp \ - $(top_srcdir)/src/utils/Logger.cpp \ - $(top_srcdir)/src/utils/StreamLogger.cpp \ - $(top_srcdir)/src/utils/String.cpp \ - $(top_srcdir)/src/utils/UUID.cpp - -lms_metadata_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT - From 4c6af6f9d326a7734fcb25bb7caa97b993e1af0f Mon Sep 17 00:00:00 2001 From: emeric Date: Fri, 14 Feb 2020 14:01:27 +0100 Subject: [PATCH 10/23] Made tests optional --- CMakeLists.txt | 3 +-- src/CMakeLists.txt | 4 ++++ src/libs/database/CMakeLists.txt | 1 - src/libs/som/CMakeLists.txt | 1 - src/test/CMakeLists.txt | 4 ++++ src/{libs/database/test => test/database}/CMakeLists.txt | 0 src/{libs/database/test => test/database}/DatabaseTest.cpp | 0 src/{libs/som/test => test/som}/CMakeLists.txt | 0 src/{libs/som/test => test/som}/SomTest.cpp | 0 9 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 src/test/CMakeLists.txt rename src/{libs/database/test => test/database}/CMakeLists.txt (100%) rename src/{libs/database/test => test/database}/DatabaseTest.cpp (100%) rename src/{libs/som/test => test/som}/CMakeLists.txt (100%) rename src/{libs/som/test => test/som}/SomTest.cpp (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index c41adcdb..516595ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,11 +5,10 @@ project(lms) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) +include(CTest) find_package(PkgConfig) pkg_check_modules(IMAGEMAGICKXX REQUIRED ImageMagick++) -enable_testing() - add_subdirectory(src) install(DIRECTORY approot DESTINATION share/lms) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d88e62ec..c796f0e9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,4 +4,8 @@ add_subdirectory(libs) add_subdirectory(lms) add_subdirectory(tools) +if(BUILD_TESTING) + add_subdirectory(test) +endif() + diff --git a/src/libs/database/CMakeLists.txt b/src/libs/database/CMakeLists.txt index aae05b4f..638fb2e2 100644 --- a/src/libs/database/CMakeLists.txt +++ b/src/libs/database/CMakeLists.txt @@ -35,4 +35,3 @@ target_link_libraries(lmsdatabase PUBLIC install(TARGETS lmsdatabase DESTINATION lib) -add_subdirectory(test) diff --git a/src/libs/som/CMakeLists.txt b/src/libs/som/CMakeLists.txt index 2d7ca9ef..7cb1b45e 100644 --- a/src/libs/som/CMakeLists.txt +++ b/src/libs/som/CMakeLists.txt @@ -16,4 +16,3 @@ target_link_libraries(lmssom PUBLIC lmsutils ) -add_subdirectory(test) diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt new file mode 100644 index 00000000..d2d34662 --- /dev/null +++ b/src/test/CMakeLists.txt @@ -0,0 +1,4 @@ + +add_subdirectory(database) +add_subdirectory(som) + diff --git a/src/libs/database/test/CMakeLists.txt b/src/test/database/CMakeLists.txt similarity index 100% rename from src/libs/database/test/CMakeLists.txt rename to src/test/database/CMakeLists.txt diff --git a/src/libs/database/test/DatabaseTest.cpp b/src/test/database/DatabaseTest.cpp similarity index 100% rename from src/libs/database/test/DatabaseTest.cpp rename to src/test/database/DatabaseTest.cpp diff --git a/src/libs/som/test/CMakeLists.txt b/src/test/som/CMakeLists.txt similarity index 100% rename from src/libs/som/test/CMakeLists.txt rename to src/test/som/CMakeLists.txt diff --git a/src/libs/som/test/SomTest.cpp b/src/test/som/SomTest.cpp similarity index 100% rename from src/libs/som/test/SomTest.cpp rename to src/test/som/SomTest.cpp From ef5cf9b5aa9432a51b6ca288462503fae9198e2e Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 15 Feb 2020 13:44:58 +0100 Subject: [PATCH 11/23] Added some more tools --- src/libs/recommendation/CMakeLists.txt | 2 +- ...viderCreator.cpp => ClassifierCreator.cpp} | 10 +++---- src/libs/recommendation/impl/Engine.cpp | 8 ++--- src/libs/recommendation/impl/Engine.hpp | 9 +++--- .../{Provider.hpp => Classifier.hpp} | 6 ++-- ...ator.hpp => ClustersClassifierCreator.hpp} | 4 +-- ...ator.hpp => FeaturesClassifierCreator.hpp} | 9 ++---- .../include/recommendation/IEngine.hpp | 7 ++--- src/lms/main.cpp | 6 +--- src/tools/CMakeLists.txt | 1 + .../recommendation-features/CMakeLists.txt | 10 +++++++ .../LmsRecommendationFeatures.cpp} | 23 +++++++-------- src/tools/similarity-parameters/Makefile.am | 29 ------------------- src/tools/similarity/Makefile.am | 27 ----------------- 14 files changed, 47 insertions(+), 104 deletions(-) rename src/libs/recommendation/impl/{ProviderCreator.cpp => ClassifierCreator.cpp} (70%) rename src/libs/recommendation/include/recommendation/{Provider.hpp => Classifier.hpp} (95%) rename src/libs/recommendation/include/recommendation/{ClustersRecommendationProviderCreator.hpp => ClustersClassifierCreator.hpp} (90%) rename src/libs/recommendation/include/recommendation/{FeaturesRecommendationProviderCreator.hpp => FeaturesClassifierCreator.hpp} (83%) create mode 100644 src/tools/recommendation-features/CMakeLists.txt rename src/tools/{similarity/LmsSimilarity.cpp => recommendation-features/LmsRecommendationFeatures.cpp} (85%) delete mode 100644 src/tools/similarity-parameters/Makefile.am delete mode 100644 src/tools/similarity/Makefile.am diff --git a/src/libs/recommendation/CMakeLists.txt b/src/libs/recommendation/CMakeLists.txt index cdcf20bd..b2161dc2 100644 --- a/src/libs/recommendation/CMakeLists.txt +++ b/src/libs/recommendation/CMakeLists.txt @@ -1,7 +1,7 @@ add_library(lmsrecommendation SHARED impl/Engine.cpp - impl/ProviderCreator.cpp + impl/ClassifierCreator.cpp ) target_include_directories(lmsrecommendation INTERFACE diff --git a/src/libs/recommendation/impl/ProviderCreator.cpp b/src/libs/recommendation/impl/ClassifierCreator.cpp similarity index 70% rename from src/libs/recommendation/impl/ProviderCreator.cpp rename to src/libs/recommendation/impl/ClassifierCreator.cpp index 00d0b0e9..865dd1c9 100644 --- a/src/libs/recommendation/impl/ProviderCreator.cpp +++ b/src/libs/recommendation/impl/ClassifierCreator.cpp @@ -17,19 +17,19 @@ * along with LMS. If not, see . */ -#include "recommendation/ClustersRecommendationProviderCreator.hpp" -#include "recommendation/FeaturesRecommendationProviderCreator.hpp" -#include "recommendation/Provider.hpp" +#include "recommendation/ClustersClassifierCreator.hpp" +#include "recommendation/FeaturesClassifierCreator.hpp" +#include "recommendation/Classifier.hpp" namespace Recommendation { - std::unique_ptr createClustersRecommendationProvider() + std::unique_ptr createClustersClassifier() { return {}; } - std::unique_ptr createFeaturesRecommendationProvider(Scanner::IMediaScanner&) + std::unique_ptr createFeaturesClassifier() { return {}; } diff --git a/src/libs/recommendation/impl/Engine.cpp b/src/libs/recommendation/impl/Engine.cpp index a2c61a61..384f6862 100644 --- a/src/libs/recommendation/impl/Engine.cpp +++ b/src/libs/recommendation/impl/Engine.cpp @@ -34,15 +34,15 @@ createEngine() } void -Engine::clearProviders() +Engine::clearClassifiers() { - _providers.clear(); + _classifiers.clear(); } void -Engine::addProvider(std::unique_ptr provider, unsigned priority) +Engine::addClassifier(std::unique_ptr classifier, unsigned priority) { - _providers.emplace(priority, std::move(provider)); + _classifiers.emplace(priority, std::move(classifier)); } std::vector diff --git a/src/libs/recommendation/impl/Engine.hpp b/src/libs/recommendation/impl/Engine.hpp index 92dacc08..cbf66a2b 100644 --- a/src/libs/recommendation/impl/Engine.hpp +++ b/src/libs/recommendation/impl/Engine.hpp @@ -22,7 +22,7 @@ #include #include "recommendation/IEngine.hpp" -#include "recommendation/Provider.hpp" +#include "recommendation/Classifier.hpp" namespace Database { @@ -34,9 +34,6 @@ namespace Recommendation class Engine : public IEngine { public: - void clearProviders() override; - void addProvider(std::unique_ptr provider, unsigned priority) override; - // Closest results first std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override; std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) override; @@ -44,8 +41,10 @@ namespace Recommendation std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override; private: + void clearClassifiers(); + void addClassifier(std::unique_ptr classifier, unsigned priority); - std::map> _providers; + std::map> _classifiers; }; } // ns Recommendation diff --git a/src/libs/recommendation/include/recommendation/Provider.hpp b/src/libs/recommendation/include/recommendation/Classifier.hpp similarity index 95% rename from src/libs/recommendation/include/recommendation/Provider.hpp rename to src/libs/recommendation/include/recommendation/Classifier.hpp index d40f686e..3c7c7e63 100644 --- a/src/libs/recommendation/include/recommendation/Provider.hpp +++ b/src/libs/recommendation/include/recommendation/Classifier.hpp @@ -32,10 +32,12 @@ namespace Database namespace Recommendation { - class Provider + class Classifier { public: - virtual ~Provider() = default; + virtual ~Classifier() = default; + + virtual void classify() = 0; virtual bool isTrackClassified(Database::IdType trackId) const = 0; virtual bool isReleaseClassified(Database::IdType releaseId) const = 0; diff --git a/src/libs/recommendation/include/recommendation/ClustersRecommendationProviderCreator.hpp b/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp similarity index 90% rename from src/libs/recommendation/include/recommendation/ClustersRecommendationProviderCreator.hpp rename to src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp index 99b11666..71e92ddc 100644 --- a/src/libs/recommendation/include/recommendation/ClustersRecommendationProviderCreator.hpp +++ b/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp @@ -28,8 +28,8 @@ namespace Database namespace Recommendation { - class Provider; + class Classifier; - std::unique_ptr createClustersRecommendationProvider(); + std::unique_ptr createClustersClassifier(); } diff --git a/src/libs/recommendation/include/recommendation/FeaturesRecommendationProviderCreator.hpp b/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp similarity index 83% rename from src/libs/recommendation/include/recommendation/FeaturesRecommendationProviderCreator.hpp rename to src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp index 756bf605..ce4e3a74 100644 --- a/src/libs/recommendation/include/recommendation/FeaturesRecommendationProviderCreator.hpp +++ b/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp @@ -26,15 +26,10 @@ namespace Database class Session; } -namespace Scanner -{ - class IMediaScanner; -} - namespace Recommendation { - class Provider; + class Classifier; - std::unique_ptr createFeaturesRecommendationProvider(Scanner::IMediaScanner& scanner); + std::unique_ptr createFeaturesClassifier(); } diff --git a/src/libs/recommendation/include/recommendation/IEngine.hpp b/src/libs/recommendation/include/recommendation/IEngine.hpp index bc420aff..a28adfaf 100644 --- a/src/libs/recommendation/include/recommendation/IEngine.hpp +++ b/src/libs/recommendation/include/recommendation/IEngine.hpp @@ -23,7 +23,7 @@ #include #include "database/Types.hpp" -#include "Provider.hpp" +#include "Classifier.hpp" namespace Database { @@ -32,15 +32,12 @@ namespace Database namespace Recommendation { - class Provider; + class Classifier; class IEngine { public: virtual ~IEngine() = default; - virtual void clearProviders() = 0; - virtual void addProvider(std::unique_ptr provider, unsigned priority) = 0; - // Closest results first virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0; virtual std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) = 0; diff --git a/src/lms/main.cpp b/src/lms/main.cpp index 126c7402..7cf6af96 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -29,9 +29,7 @@ #include "cover/ICoverArtGrabber.hpp" #include "database/Db.hpp" #include "scanner/IMediaScanner.hpp" -#include "recommendation/FeaturesRecommendationProviderCreator.hpp" #include "recommendation/IEngine.hpp" -#include "recommendation/ClustersRecommendationProviderCreator.hpp" #include "subsonic/SubsonicResource.hpp" #include "ui/LmsApplication.hpp" #include "utils/IConfig.hpp" @@ -145,9 +143,7 @@ int main(int argc, char* argv[]) ServiceProvider::assign(Auth::createPasswordService(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000))); Scanner::IMediaScanner& mediaScanner {ServiceProvider::assign(Scanner::createMediaScanner(database))}; - Recommendation::IEngine& recommendationEngine {ServiceProvider::assign(Recommendation::createEngine())}; - recommendationEngine.addProvider(Recommendation::createFeaturesRecommendationProvider(mediaScanner), 0); - recommendationEngine.addProvider(Recommendation::createClustersRecommendationProvider(), 1); + ServiceProvider::assign(Recommendation::createEngine()); CoverArt::IGrabber& coverArtGrabber {ServiceProvider::assign(CoverArt::createGrabber(argv[0]))}; coverArtGrabber.setDefaultCover(server.appRoot() + "/images/unknown-cover.jpg"); diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index f66ea24a..4fcdba31 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(metadata) +add_subdirectory(recommendation-features) diff --git a/src/tools/recommendation-features/CMakeLists.txt b/src/tools/recommendation-features/CMakeLists.txt new file mode 100644 index 00000000..25508174 --- /dev/null +++ b/src/tools/recommendation-features/CMakeLists.txt @@ -0,0 +1,10 @@ + +add_executable(lms-recommendation-features + LmsRecommendationFeatures.cpp + ) + +target_link_libraries(lms-recommendation-features PRIVATE + lmsdatabase + lmsrecommendation + ) + diff --git a/src/tools/similarity/LmsSimilarity.cpp b/src/tools/recommendation-features/LmsRecommendationFeatures.cpp similarity index 85% rename from src/tools/similarity/LmsSimilarity.cpp rename to src/tools/recommendation-features/LmsRecommendationFeatures.cpp index 522b6558..6426dff5 100644 --- a/src/tools/similarity/LmsSimilarity.cpp +++ b/src/tools/recommendation-features/LmsRecommendationFeatures.cpp @@ -28,17 +28,16 @@ #include "database/Release.hpp" #include "database/Session.hpp" #include "database/Track.hpp" -#include "utils/Config.hpp" +#include "utils/IConfig.hpp" #include "utils/Service.hpp" #include "utils/StreamLogger.hpp" -#include "similarity/features/SimilarityFeaturesSearcher.hpp" +#include "recommendation/IEngine.hpp" +#include "recommendation/FeaturesClassifierCreator.hpp" int main(int argc, char *argv[]) { try { - using namespace Similarity; - // log to stdout ServiceProvider::create(std::cout); @@ -46,16 +45,16 @@ int main(int argc, char *argv[]) if (argc >= 2) configFilePath = std::string(argv[1], 0, 256); - ServiceProvider::create(configFilePath); + ServiceProvider::assign(createConfig(configFilePath)); - Database::Db db {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; + Database::Db db {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; Database::Session session {db}; + auto classifier {Recommendation::createFeaturesClassifier()}; + std::cout << "Classifying tracks..." << std::endl; // may be long... - struct FeaturesSearcher::TrainSettings trainSettings; - trainSettings.featureSettingsMap = FeaturesSearcher::getDefaultTrainFeatureSettings(); - FeaturesSearcher searcher {session, trainSettings}; + classifier->classify(); std::cout << "Classifying tracks DONE" << std::endl; const std::vector trackIds = std::invoke([&]() @@ -85,7 +84,7 @@ int main(int argc, char *argv[]) }; std::cout << "Processing track '" << trackToString(trackId) << std::endl; - for (Database::IdType similarTrackId : searcher.getSimilarTracks({trackId}, 3)) + for (Database::IdType similarTrackId : classifier->getSimilarTracks(session, {trackId}, 3)) std::cout << "\t- Similar track '" << trackToString(similarTrackId) << std::endl; } @@ -107,7 +106,7 @@ int main(int argc, char *argv[]) }; std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl; - for (Database::IdType similarReleaseId : searcher.getSimilarReleases({releaseId}, 3)) + for (Database::IdType similarReleaseId : classifier->getSimilarReleases({releaseId}, 3)) std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl; } @@ -129,7 +128,7 @@ int main(int argc, char *argv[]) }; std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl; - for (Database::IdType similarArtistId : searcher.getSimilarArtists({artistId}, 3)) + for (Database::IdType similarArtistId : classifier->getSimilarArtists({artistId}, 3)) std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl; } diff --git a/src/tools/similarity-parameters/Makefile.am b/src/tools/similarity-parameters/Makefile.am deleted file mode 100644 index 0f7cb8f8..00000000 --- a/src/tools/similarity-parameters/Makefile.am +++ /dev/null @@ -1,29 +0,0 @@ -noinst_PROGRAMS = lms-similarity-parameters - -lms_similarity_parameters_SOURCES = \ - $(srcdir)/LmsSimilarityParameters.cpp \ - $(top_srcdir)/src/database/Artist.cpp \ - $(top_srcdir)/src/database/Cluster.cpp \ - $(top_srcdir)/src/database/Db.cpp \ - $(top_srcdir)/src/database/TrackFeatures.cpp \ - $(top_srcdir)/src/database/TrackList.cpp \ - $(top_srcdir)/src/database/Release.cpp \ - $(top_srcdir)/src/database/ScanSettings.cpp \ - $(top_srcdir)/src/database/Session.cpp \ - $(top_srcdir)/src/database/SessionPool.cpp \ - $(top_srcdir)/src/database/SqlQuery.cpp \ - $(top_srcdir)/src/database/Track.cpp \ - $(top_srcdir)/src/database/User.cpp \ - $(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp \ - $(top_srcdir)/src/similarity/features/som/Network.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesCache.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesSearcher.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesDefs.cpp \ - $(top_srcdir)/src/utils/Config.cpp \ - $(top_srcdir)/src/utils/Logger.cpp \ - $(top_srcdir)/src/utils/Random.cpp \ - $(top_srcdir)/src/utils/StreamLogger.cpp \ - $(top_srcdir)/src/utils/String.cpp - -lms_similarity_parameters_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT - diff --git a/src/tools/similarity/Makefile.am b/src/tools/similarity/Makefile.am deleted file mode 100644 index 0473f464..00000000 --- a/src/tools/similarity/Makefile.am +++ /dev/null @@ -1,27 +0,0 @@ -noinst_PROGRAMS = lms-similarity - -lms_similarity_SOURCES = \ - $(srcdir)/LmsSimilarity.cpp \ - $(top_srcdir)/src/database/Artist.cpp \ - $(top_srcdir)/src/database/Cluster.cpp \ - $(top_srcdir)/src/database/Db.cpp \ - $(top_srcdir)/src/database/TrackFeatures.cpp \ - $(top_srcdir)/src/database/TrackList.cpp \ - $(top_srcdir)/src/database/Release.cpp \ - $(top_srcdir)/src/database/ScanSettings.cpp \ - $(top_srcdir)/src/database/Session.cpp \ - $(top_srcdir)/src/database/SqlQuery.cpp \ - $(top_srcdir)/src/database/Track.cpp \ - $(top_srcdir)/src/database/User.cpp \ - $(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp \ - $(top_srcdir)/src/similarity/features/som/Network.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesCache.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesSearcher.cpp \ - $(top_srcdir)/src/similarity/features/SimilarityFeaturesDefs.cpp \ - $(top_srcdir)/src/utils/Config.cpp \ - $(top_srcdir)/src/utils/Logger.cpp \ - $(top_srcdir)/src/utils/StreamLogger.cpp \ - $(top_srcdir)/src/utils/String.cpp - -lms_similarity_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT - From 7f05a1d4e67cecdb3241bc85378bf9e30ffae37a Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 15 Feb 2020 14:04:04 +0100 Subject: [PATCH 12/23] Updated README --- README.md | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6e07b808..a083a22c 100644 --- a/README.md +++ b/README.md @@ -21,24 +21,29 @@ A [demo](http://lms.demo.poupon.io) instance is available, with the following li * Multi-value tags: artists, genres, ... * Custom tags (ex: _mood_, _genre_, _albummood_, _albumgrouping_, ...) * MusicBrainzID support to handle duplicated artist and release names -* Playlists, (only using Subsonic API for now) -* Starred Album/Artist/Tracks (only using Subsonic API for now) * _Systemd_ integration +* Subsonic-only features: + * Playlists + * Starred Album/Artist/Tracks + * Bookmarks -## Recommendation engine +## Music discovery _LMS_ provides several ways to help you find the music you like: * Tag-based filters (ex: _Rock_, _Metal_ and _Aggressive_, _Electronic_ and _Relaxed_, ...) * Recommendations for similar artists and albums -* Radio mode +* Radio mode, based on what is in the current playqueue * Searches in album, artist and track names * Most played/Recently added music -The recommendation engine makes use of [Self-Organizing Maps](https://en.wikipedia.org/wiki/Self-organizing_map).
-__Notes__: -* constructing the map requires significant computation time on large collections (ex: half an hour for 40k tracks) -* audio data is pulled from [AcousticBrainz](https://acousticbrainz.org/). Therefore your music files must contain the [MusicBrainz Identifier](https://musicbrainz.org/doc/MusicBrainz_Identifier) for the recommendation engine to work properly (otherwise, only tag-based recommendations are provided) -* to use the _self-organizing map_ based engine, you have to enable it first in the settings panel. +The recommendation engine uses two different sources: +1. Tags that are present in the audio files +2. Acoustic similarities of the audio files, using a trained [Self-Organizing Map](https://en.wikipedia.org/wiki/Self-organizing_map) + +__Notes on the self-organizing map__: +* training the map requires significant computation time on large collections (ex: half an hour for 40k tracks) +* audio acoustic data is pulled from [AcousticBrainz](https://acousticbrainz.org/). Therefore your audio files _must_ contain the [MusicBrainz Identifier](https://musicbrainz.org/doc/MusicBrainz_Identifier). +* to enable the audio similarity source, you have to enable it first in the settings panel. ## Subsonic API The API version implemented is 1.12.0 and has been tested on _Android_ using the official application, _Ultrasonic_ and _DSub_. @@ -80,7 +85,7 @@ __Notes__: * a C++17 compiler is needed * ffmpeg version 4 minimum is required ```sh -apt-get install g++ autoconf automake libboost-filesystem-dev libboost-system-dev libavutil-dev libavformat-dev libmagick++-dev libpstreams-dev libconfig++-dev libpstreams-dev ffmpeg libtag1-dev +apt-get install g++ cmake libboost-system-dev libavutil-dev libavformat-dev libmagick++-dev libconfig++-dev libpstreams-dev ffmpeg libtag1-dev ``` You also need _Wt4_, which is not packaged yet on _Debian_. See [installation instructions](https://www.webtoolkit.eu/wt/doc/reference/html/InstallationUnix.html).
@@ -92,16 +97,10 @@ Get the latest stable release and build it: ```sh git clone https://github.com/epoupon/lms.git lms cd lms -autoreconf -vfi mkdir build -cd build -../configure --prefix=/usr +cmake .. -DCMAKE_BUILD_TYPE=Release ``` -configure will report any missing library. - -__Note__: in order to customize the installation directories, you can use the following options of the `configure` script: -* _--prefix_ (defaults to `/usr/local`). -* _--bindir_ (defaults to `$PREFIX/bin`). +__Note__: in order to customize the installation directory, you can use the _-DCMAKE_INSTALL_PREFIX_ option (defaults to `/usr/local`). ```sh make From d1bdaf62033ebd126af1f9b5d7e1630af4f8e155 Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 15 Feb 2020 15:52:39 +0100 Subject: [PATCH 13/23] Started to restore clusters based recommendation engine --- README.md | 2 +- approot/messages.xml | 6 +- approot/messages_fr.xml | 6 +- src/libs/database/impl/Artist.cpp | 15 +++ src/libs/database/impl/Release.cpp | 15 +++ src/libs/database/impl/Session.cpp | 2 +- src/libs/database/impl/Track.cpp | 15 ++- src/libs/database/include/database/Artist.hpp | 1 + .../database/include/database/Release.hpp | 1 + .../include/database/ScanSettings.hpp | 10 +- src/libs/database/include/database/Track.hpp | 6 +- src/libs/recommendation/CMakeLists.txt | 1 + .../recommendation/impl/ClassifierCreator.cpp | 9 +- src/libs/recommendation/impl/Engine.cpp | 105 ++++++++++-------- src/libs/recommendation/impl/Engine.hpp | 13 ++- .../cluster/SimilarityClusterSearcher.hpp | 40 ------- .../ClustersClassifier.cpp} | 66 +++++++++-- .../impl/clusters/ClustersClassifier.hpp | 56 ++++++++++ .../ClustersClassifierCreator.hpp | 9 +- .../FeaturesClassifierCreator.hpp | 9 +- .../{Classifier.hpp => IClassifier.hpp} | 12 +- .../include/recommendation/IEngine.hpp | 6 +- src/lms/main.cpp | 5 +- src/lms/ui/admin/DatabaseSettingsView.cpp | 46 ++++---- src/test/database/DatabaseTest.cpp | 36 ++++++ .../LmsRecommendationFeatures.cpp | 11 +- 26 files changed, 326 insertions(+), 177 deletions(-) delete mode 100644 src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.hpp rename src/libs/recommendation/impl/{cluster/SimilarityClusterSearcher.cpp => clusters/ClustersClassifier.cpp} (57%) create mode 100644 src/libs/recommendation/impl/clusters/ClustersClassifier.hpp rename src/libs/recommendation/include/recommendation/{Classifier.hpp => IClassifier.hpp} (85%) diff --git a/README.md b/README.md index a083a22c..25b903a4 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ A [demo](http://lms.demo.poupon.io) instance is available, with the following li * Audio transcode for maximum interoperability and low bandwith requirements * Persistent play queue across sessions * Subsonic API -* Album artist +* Compilation support * Multi-value tags: artists, genres, ... * Custom tags (ex: _mood_, _genre_, _albummood_, _albumgrouping_, ...) * MusicBrainzID support to handle duplicated artist and release names diff --git a/approot/messages.xml b/approot/messages.xml index 17d5ded8..20677bc8 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -39,9 +39,9 @@ Monthly Never Media root directory -Recommendation engine -Tags based -Audio analysis based +Recommendation engine +Tags based +Audio analysis based Scan complete: {1} total files, {2} additions, {3} updates, {4} deletions, {5} duplicates, {6} errors Scan launched! Scan options diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index 2d61dc1d..934cd400 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -39,9 +39,9 @@ Tous les mois Jamais Dossier racine des fichiers de musique -Moteur de recommandation -Basé sur les tags -Basé sur l'analyse audio +Moteur de recommandation +Basé sur les tags +Basé sur l'analyse audio Scan terminé : {1} fichiers, {2} ajouts, {3} mises à jour, {4} suppressions, {5} duplicatas, {6} erreurs Scan lancé ! Options diff --git a/src/libs/database/impl/Artist.cpp b/src/libs/database/impl/Artist.cpp index 5d17df1a..b5c6d221 100644 --- a/src/libs/database/impl/Artist.cpp +++ b/src/libs/database/impl/Artist.cpp @@ -104,6 +104,21 @@ Artist::getAllOrphans(Session& session) return std::vector(res.begin(), res.end()); } +std::vector +Artist::getAllIdsWithClusters(Session& session, std::optional limit) +{ + session.checkSharedLocked(); + + Wt::Dbo::collection res = session.getDboSession().query + ("SELECT DISTINCT a.id FROM artist a" + " INNER JOIN track t ON t.id = t_a_l.track_id INNER JOIN track_artist_link t_a_l ON t_a_l.artist_id = a.id" + " INNER JOIN track_cluster t_c ON t_c.track_id = t.id") + .limit(limit ? static_cast(*limit) : -1); + + return std::vector(res.begin(), res.end()); +} + + static Wt::Dbo::Query getQuery(Session& session, diff --git a/src/libs/database/impl/Release.cpp b/src/libs/database/impl/Release.cpp index 343a7ce1..7fdd639a 100644 --- a/src/libs/database/impl/Release.cpp +++ b/src/libs/database/impl/Release.cpp @@ -252,6 +252,21 @@ Release::getByFilter(Session& session, return res; } +std::vector +Release::getAllIdsWithClusters(Session& session, std::optional limit) +{ + session.checkSharedLocked(); + + Wt::Dbo::collection res = session.getDboSession().query + ("SELECT DISTINCT r.id FROM release r" + " INNER JOIN track t ON t.release_id = r.id" + " INNER JOIN track_cluster t_c ON t_c.track_id = t.id") + .limit(limit ? static_cast(*limit) : -1); + + return std::vector(res.begin(), res.end()); +} + + std::optional Release::getTotalTrackNumber(void) const { diff --git a/src/libs/database/impl/Session.cpp b/src/libs/database/impl/Session.cpp index 36750ec7..081fe8a8 100644 --- a/src/libs/database/impl/Session.cpp +++ b/src/libs/database/impl/Session.cpp @@ -118,7 +118,7 @@ Session::doDatabaseMigrationIfNeeded() { _session.execute("DROP TABLE similarity_settings"); _session.execute("DROP TABLE similarity_settings_feature"); - _session.execute("ALTER TABLE scan_settings ADD similarity_engine_type INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(ScanSettings::SimilarityEngineType::Clusters)) + ")"); + _session.execute("ALTER TABLE scan_settings ADD similarity_engine_type INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(ScanSettings::RecommendationEngineType::Clusters)) + ")"); } else if (version == 8) { diff --git a/src/libs/database/impl/Track.cpp b/src/libs/database/impl/Track.cpp index a84af8e1..2acff171 100644 --- a/src/libs/database/impl/Track.cpp +++ b/src/libs/database/impl/Track.cpp @@ -163,6 +163,19 @@ Track::getAllIdsWithFeatures(Session& session, std::optional limit) return std::vector(res.begin(), res.end()); } +std::vector +Track::getAllIdsWithClusters(Session& session, std::optional limit) +{ + session.checkSharedLocked(); + + Wt::Dbo::collection res = session.getDboSession().query + ("SELECT DISTINCT t.id FROM track t" + " INNER JOIN track_cluster t_c ON t_c.track_id = t.id") + .limit(limit ? static_cast(*limit) : -1); + + return std::vector(res.begin(), res.end()); +} + std::vector Track::getClusters(void) const { @@ -263,7 +276,7 @@ Track::getByFilter(Session& session, std::vector Track::getSimilarTracks(Session& session, - const std::set& tracks, + const std::unordered_set& tracks, std::optional offset, std::optional size) { diff --git a/src/libs/database/include/database/Artist.hpp b/src/libs/database/include/database/Artist.hpp index 01d261b3..f53082c1 100644 --- a/src/libs/database/include/database/Artist.hpp +++ b/src/libs/database/include/database/Artist.hpp @@ -68,6 +68,7 @@ class Artist : public Wt::Dbo::Dbo static std::vector getAllIds(Session& session); static std::vector getAllOrphans(Session& session); // No track related static std::vector getLastAdded(Session& session, Wt::WDateTime after, std::optional size = {}); + static std::vector getAllIdsWithClusters(Session& session, std::optional limit = {}); // Accessors const std::string& getName(void) const { return _name; } diff --git a/src/libs/database/include/database/Release.hpp b/src/libs/database/include/database/Release.hpp index c0a638f3..695bcdc5 100644 --- a/src/libs/database/include/database/Release.hpp +++ b/src/libs/database/include/database/Release.hpp @@ -66,6 +66,7 @@ class Release : public Wt::Dbo::Dbo std::optional offset, std::optional size, bool& moreExpected); + static std::vector getAllIdsWithClusters(Session& session, std::optional limit = {}); std::vector> getTracks(const std::set& clusters = std::set()) const; std::size_t getTracksCount() const; diff --git a/src/libs/database/include/database/ScanSettings.hpp b/src/libs/database/include/database/ScanSettings.hpp index 0ce37a49..7151df23 100644 --- a/src/libs/database/include/database/ScanSettings.hpp +++ b/src/libs/database/include/database/ScanSettings.hpp @@ -43,7 +43,7 @@ class ScanSettings : public Wt::Dbo::Dbo }; // Do not modify values (just add) - enum class SimilarityEngineType + enum class RecommendationEngineType { Clusters = 0, Features, @@ -60,7 +60,7 @@ class ScanSettings : public Wt::Dbo::Dbo UpdatePeriod getUpdatePeriod() const { return _updatePeriod; } std::vector> getClusterTypes() const; std::set getAudioFileExtensions() const; - SimilarityEngineType getSimilarityEngineType() const { return _similarityEngineType; } + RecommendationEngineType getRecommendationEngineType() const { return _recommendationEngineType; } // Setters void addAudioFileExtension(const std::filesystem::path& ext); @@ -68,7 +68,7 @@ class ScanSettings : public Wt::Dbo::Dbo void setUpdateStartTime(Wt::WTime t) { _startTime = t; } void setUpdatePeriod(UpdatePeriod p) { _updatePeriod = p; } void setClusterTypes(Session& session, const std::set& clusterTypeNames); - void setSimilarityEngineType(SimilarityEngineType type) { _similarityEngineType = type; } + void setRecommendationEngineType(RecommendationEngineType type) { _recommendationEngineType = type; } void incScanVersion(); template @@ -79,7 +79,7 @@ class ScanSettings : public Wt::Dbo::Dbo Wt::Dbo::field(a, _startTime, "start_time"); Wt::Dbo::field(a, _updatePeriod, "update_period"); Wt::Dbo::field(a, _audioFileExtensions, "audio_file_extensions"); - Wt::Dbo::field(a, _similarityEngineType,"similarity_engine_type"); + Wt::Dbo::field(a, _recommendationEngineType,"similarity_engine_type"); Wt::Dbo::hasMany(a, _clusterTypes, Wt::Dbo::ManyToOne, "scan_settings"); } @@ -89,7 +89,7 @@ class ScanSettings : public Wt::Dbo::Dbo std::string _mediaDirectory; Wt::WTime _startTime = Wt::WTime {0,0,0}; UpdatePeriod _updatePeriod {UpdatePeriod::Never}; - SimilarityEngineType _similarityEngineType {SimilarityEngineType::Clusters}; + RecommendationEngineType _recommendationEngineType {RecommendationEngineType::Clusters}; std::string _audioFileExtensions {".alac .mp3 .ogg .oga .aac .m4a .m4b .flac .wav .wma .aif .aiff .ape .mpc .shn .opus"}; Wt::Dbo::collection> _clusterTypes; }; diff --git a/src/libs/database/include/database/Track.hpp b/src/libs/database/include/database/Track.hpp index d3bcd824..24113f21 100644 --- a/src/libs/database/include/database/Track.hpp +++ b/src/libs/database/include/database/Track.hpp @@ -22,8 +22,9 @@ #include #include #include -#include #include +#include +#include #include #include @@ -58,7 +59,7 @@ class Track : public Wt::Dbo::Dbo static pointer getById(Session& session, IdType id); static pointer getByMBID(Session& session, const UUID& MBID); static std::vector getSimilarTracks(Session& session, - const std::set& trackIds, + const std::unordered_set& trackIds, std::optional offset = {}, std::optional size = {}); static std::vector getByClusters(Session& session, @@ -78,6 +79,7 @@ class Track : public Wt::Dbo::Dbo static std::vector getLastAdded(Session& session, const Wt::WDateTime& after, std::optional size = 1); static std::vector getAllWithMBIDAndMissingFeatures(Session& session); static std::vector getAllIdsWithFeatures(Session& session, std::optional limit = {}); + static std::vector getAllIdsWithClusters(Session& session, std::optional limit = {}); // Create utility static pointer create(Session& session, const std::filesystem::path& p); diff --git a/src/libs/recommendation/CMakeLists.txt b/src/libs/recommendation/CMakeLists.txt index b2161dc2..260f137b 100644 --- a/src/libs/recommendation/CMakeLists.txt +++ b/src/libs/recommendation/CMakeLists.txt @@ -1,5 +1,6 @@ add_library(lmsrecommendation SHARED + impl/clusters/ClustersClassifier.cpp impl/Engine.cpp impl/ClassifierCreator.cpp ) diff --git a/src/libs/recommendation/impl/ClassifierCreator.cpp b/src/libs/recommendation/impl/ClassifierCreator.cpp index 865dd1c9..90dd651b 100644 --- a/src/libs/recommendation/impl/ClassifierCreator.cpp +++ b/src/libs/recommendation/impl/ClassifierCreator.cpp @@ -17,22 +17,17 @@ * along with LMS. If not, see . */ -#include "recommendation/ClustersClassifierCreator.hpp" #include "recommendation/FeaturesClassifierCreator.hpp" -#include "recommendation/Classifier.hpp" +#include "recommendation/IClassifier.hpp" namespace Recommendation { - std::unique_ptr createClustersClassifier() + std::unique_ptr createFeaturesClassifier() { return {}; } - std::unique_ptr createFeaturesClassifier() - { - return {}; - } } diff --git a/src/libs/recommendation/impl/Engine.cpp b/src/libs/recommendation/impl/Engine.cpp index 384f6862..a5cf2a63 100644 --- a/src/libs/recommendation/impl/Engine.cpp +++ b/src/libs/recommendation/impl/Engine.cpp @@ -19,30 +19,49 @@ #include "Engine.hpp" -//#include "features/SimilarityFeaturesScannerAddon.hpp" -//#include "cluster/SimilarityClusterSearcher.hpp" +#include "recommendation/ClustersClassifierCreator.hpp" +#include "recommendation/FeaturesClassifierCreator.hpp" #include "database/ScanSettings.hpp" +#include "database/Session.hpp" #include "database/TrackList.hpp" namespace Recommendation { std::unique_ptr -createEngine() +createEngine(Database::Session& session) { - return std::make_unique(); + return std::make_unique(session); +} + +Engine::Engine(Database::Session& session) +{ + reloadSettings(session); } void -Engine::clearClassifiers() +Engine::reloadSettings(Database::Session& session) { - _classifiers.clear(); -} + using namespace Database; -void -Engine::addClassifier(std::unique_ptr classifier, unsigned priority) -{ - _classifiers.emplace(priority, std::move(classifier)); + const ScanSettings::RecommendationEngineType engineType {[&]() + { + auto transaction {session.createSharedTransaction()}; + return ScanSettings::get(session)->getRecommendationEngineType(); + }()}; + + clearClassifiers(); + + switch (engineType) + { + case ScanSettings::RecommendationEngineType::Features: +// _classifiers.emplace(0, createFeaturesClassifier()); // higher priority +// [[fallthrough]]; + + case ScanSettings::RecommendationEngineType::Clusters: + _classifiers.emplace(1, createClustersClassifier(session)); // lower priority + break; + } } std::vector @@ -79,60 +98,52 @@ Engine::getSimilarTracksFromTrackList(Database::Session& /*session*/, Database:: } std::vector -Engine::getSimilarTracks(Database::Session& /*dbSession*/, const std::unordered_set& /*trackIds*/, std::size_t /*maxCount*/) +Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set& trackIds, std::size_t maxCount) { -#if 0 - auto engineType {getEngineType(dbSession)}; - auto somSearcher {_somAddon.getSearcher()}; - - if (engineType == Database::ScanSettings::SimilarityEngineType::Features - && somSearcher - && std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return somSearcher->isTrackClassified(trackId); } )) + for (const auto& [priority, classifier] : _classifiers) { - return somSearcher->getSimilarTracks(trackIds, maxCount); + if (std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return classifier->isTrackClassified(trackId); } )) + return classifier->getSimilarTracks(dbSession, trackIds, maxCount); } - else - return ClusterEngine::getSimilarTracks(dbSession, trackIds, maxCount); -#endif + return {}; } std::vector -Engine::getSimilarReleases(Database::Session& /*dbSession*/, Database::IdType /*releaseId*/, std::size_t /*maxCount*/) +Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) { -#if 0 - auto engineType {getEngineType(dbSession)}; - auto somSearcher {_somAddon.getSearcher()}; - - if (engineType == Database::ScanSettings::SimilarityEngineType::Features - && somSearcher - && somSearcher->isReleaseClassified(releaseId)) + for (const auto& [priority, classifier] : _classifiers) { - return somSearcher->getSimilarReleases(releaseId, maxCount); + if (classifier->isReleaseClassified(releaseId)) + return classifier->getSimilarReleases(dbSession, releaseId, maxCount); } - else - return ClusterEngine::getSimilarReleases(dbSession, releaseId, maxCount); -#endif + return {}; } std::vector -Engine::getSimilarArtists(Database::Session& /*dbSession*/, Database::IdType /*artistId*/, std::size_t /*maxCount*/) +Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) { -#if 0 - auto engineType {getEngineType(dbSession)}; - auto somSearcher {_somAddon.getSearcher()}; - - if (engineType == Database::ScanSettings::SimilarityEngineType::Features - && somSearcher - && somSearcher->isArtistClassified(artistId)) + for (const auto& [priority, classifier] : _classifiers) { - return somSearcher->getSimilarArtists(artistId, maxCount); + if (classifier->isArtistClassified(artistId)) + return classifier->getSimilarArtists(dbSession, artistId, maxCount); } - else - return ClusterEngine::getSimilarArtists(dbSession, artistId, maxCount); -#endif + return {}; } +void +Engine::clearClassifiers() +{ + _classifiers.clear(); +} + +void +Engine::addClassifier(std::unique_ptr classifier, unsigned priority) +{ + _classifiers.emplace(priority, std::move(classifier)); +} + + } // ns Similarity diff --git a/src/libs/recommendation/impl/Engine.hpp b/src/libs/recommendation/impl/Engine.hpp index cbf66a2b..2fdace41 100644 --- a/src/libs/recommendation/impl/Engine.hpp +++ b/src/libs/recommendation/impl/Engine.hpp @@ -22,7 +22,7 @@ #include #include "recommendation/IEngine.hpp" -#include "recommendation/Classifier.hpp" +#include "recommendation/IClassifier.hpp" namespace Database { @@ -34,17 +34,22 @@ namespace Recommendation class Engine : public IEngine { public: + Engine(Database::Session& session); + + private: + + void reloadSettings(Database::Session& session) override; + // Closest results first std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override; std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) override; std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override; std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override; - private: void clearClassifiers(); - void addClassifier(std::unique_ptr classifier, unsigned priority); + void addClassifier(std::unique_ptr classifier, unsigned priority); - std::map> _classifiers; + std::map> _classifiers; }; } // ns Recommendation diff --git a/src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.hpp b/src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.hpp deleted file mode 100644 index 90ef0db8..00000000 --- a/src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.hpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include - -#include "database/Types.hpp" - -namespace Database { - class Session; -} - -namespace Similarity { - -namespace ClusterSearcher -{ - std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount); - std::vector getSimilarTracks(Database::Session& session, const std::set& tracksId, std::size_t maxCount); - std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount); - std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount); -}; - -} // namespace Similarity diff --git a/src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.cpp b/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp similarity index 57% rename from src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.cpp rename to src/libs/recommendation/impl/clusters/ClustersClassifier.cpp index 7b24b5e4..f0aee487 100644 --- a/src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.cpp +++ b/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "SimilarityClusterSearcher.hpp" +#include "ClustersClassifier.hpp" #include "database/Artist.hpp" #include "database/Cluster.hpp" @@ -26,11 +26,60 @@ #include "database/Track.hpp" #include "database/TrackList.hpp" -namespace Similarity { -namespace ClusterSearcher { +namespace Recommendation { + +std::unique_ptr createClustersClassifier(Database::Session& session) +{ + return std::make_unique(session); +} + +ClusterClassifier::ClusterClassifier(Database::Session& session) +{ + classify(session); +} + +void +ClusterClassifier::classify(Database::Session& session) +{ + auto transaction {session.createSharedTransaction()}; + + { + std::vector trackIds {Database::Track::getAllIdsWithClusters(session)}; + _classifiedTracks = std::unordered_set(std::cbegin(trackIds), std::cend(trackIds)); + } + + { + std::vector releaseIds {Database::Release::getAllIdsWithClusters(session)}; + _classifiedReleases = std::unordered_set(std::cbegin(releaseIds), std::cend(releaseIds)); + } + + { + std::vector artistIds {Database::Artist::getAllIdsWithClusters(session)}; + _classifiedArtists = std::unordered_set(std::cbegin(artistIds), std::cend(artistIds)); + } +} + +bool +ClusterClassifier::isTrackClassified(Database::IdType trackId) const +{ + return _classifiedTracks.find(trackId) != std::cend(_classifiedTracks); +} + +bool +ClusterClassifier::isReleaseClassified(Database::IdType releaseId) const +{ + return _classifiedReleases.find(releaseId) != std::cend(_classifiedReleases); +} + +bool +ClusterClassifier::isArtistClassified(Database::IdType artistId) const +{ + return _classifiedArtists.find(artistId) != std::cend(_classifiedArtists); +} + std::vector -getSimilarTracks(Database::Session& dbSession, const std::set& trackIds, std::size_t maxCount) +ClusterClassifier::getSimilarTracks(Database::Session& dbSession, const std::unordered_set& trackIds, std::size_t maxCount) const { auto transaction {dbSession.createSharedTransaction()}; @@ -43,7 +92,7 @@ getSimilarTracks(Database::Session& dbSession, const std::set& } std::vector -getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) +ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const { std::vector res; @@ -62,7 +111,7 @@ getSimilarTracksFromTrackList(Database::Session& session, Database::IdType track } std::vector -getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) +ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) const { std::vector res; @@ -80,7 +129,7 @@ getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std } std::vector -getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) +ClusterClassifier::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) const { std::vector res; @@ -97,5 +146,4 @@ getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std:: return res; } -} // namespace ClusterSearcher -} // namespace Similarity +} // namespace Recommendation diff --git a/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp b/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp new file mode 100644 index 00000000..2f44205e --- /dev/null +++ b/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "recommendation/IClassifier.hpp" + + +namespace Recommendation +{ + + class ClusterClassifier : public IClassifier + { + public: + ClusterClassifier(Database::Session& session); + ClusterClassifier(const ClusterClassifier&) = delete; + ClusterClassifier(ClusterClassifier&&) = delete; + ClusterClassifier& operator=(const ClusterClassifier&) = delete; + ClusterClassifier& operator=(ClusterClassifier&&) = delete; + + private: + + bool isTrackClassified(Database::IdType trackId) const override; + bool isReleaseClassified(Database::IdType releaseId) const override; + bool isArtistClassified(Database::IdType artistId) const override; + + std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override; + std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const override; + std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override; + std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override; + + void classify(Database::Session& session); + + std::unordered_set _classifiedArtists; + std::unordered_set _classifiedReleases; + std::unordered_set _classifiedTracks; +}; + +} // namespace Recommendation + diff --git a/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp b/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp index 71e92ddc..3f4dfbe8 100644 --- a/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp +++ b/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp @@ -21,15 +21,10 @@ #include -namespace Database -{ - class Session; -} - namespace Recommendation { - class Classifier; + class IClassifier; - std::unique_ptr createClustersClassifier(); + std::unique_ptr createClustersClassifier(Database::Session& session); } diff --git a/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp b/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp index ce4e3a74..9a763020 100644 --- a/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp +++ b/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp @@ -21,15 +21,10 @@ #include -namespace Database -{ - class Session; -} - namespace Recommendation { - class Classifier; + class IClassifier; - std::unique_ptr createFeaturesClassifier(); + std::unique_ptr createFeaturesClassifier(); } diff --git a/src/libs/recommendation/include/recommendation/Classifier.hpp b/src/libs/recommendation/include/recommendation/IClassifier.hpp similarity index 85% rename from src/libs/recommendation/include/recommendation/Classifier.hpp rename to src/libs/recommendation/include/recommendation/IClassifier.hpp index 3c7c7e63..6619ccd2 100644 --- a/src/libs/recommendation/include/recommendation/Classifier.hpp +++ b/src/libs/recommendation/include/recommendation/IClassifier.hpp @@ -32,21 +32,19 @@ namespace Database namespace Recommendation { - class Classifier + class IClassifier { public: - virtual ~Classifier() = default; - - virtual void classify() = 0; + virtual ~IClassifier() = default; virtual bool isTrackClassified(Database::IdType trackId) const = 0; virtual bool isReleaseClassified(Database::IdType releaseId) const = 0; virtual bool isArtistClassified(Database::IdType artistId) const = 0; - virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0; + virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0; virtual std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const = 0; - virtual std::vector getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const = 0; - virtual std::vector getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const = 0; + virtual std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const = 0; + virtual std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const = 0; }; } // ns Recommendation diff --git a/src/libs/recommendation/include/recommendation/IEngine.hpp b/src/libs/recommendation/include/recommendation/IEngine.hpp index a28adfaf..91f10c50 100644 --- a/src/libs/recommendation/include/recommendation/IEngine.hpp +++ b/src/libs/recommendation/include/recommendation/IEngine.hpp @@ -23,7 +23,6 @@ #include #include "database/Types.hpp" -#include "Classifier.hpp" namespace Database { @@ -32,12 +31,13 @@ namespace Database namespace Recommendation { - class Classifier; class IEngine { public: virtual ~IEngine() = default; + virtual void reloadSettings(Database::Session& session) = 0; + // Closest results first virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0; virtual std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) = 0; @@ -45,7 +45,7 @@ namespace Recommendation virtual std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0; }; - std::unique_ptr createEngine(); + std::unique_ptr createEngine(Database::Session& session); } // ns Recommendation diff --git a/src/lms/main.cpp b/src/lms/main.cpp index 7cf6af96..f2a71fa0 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -143,7 +143,10 @@ int main(int argc, char* argv[]) ServiceProvider::assign(Auth::createPasswordService(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000))); Scanner::IMediaScanner& mediaScanner {ServiceProvider::assign(Scanner::createMediaScanner(database))}; - ServiceProvider::assign(Recommendation::createEngine()); + { + Database::Session session {database}; + ServiceProvider::assign(Recommendation::createEngine(session)); + } CoverArt::IGrabber& coverArtGrabber {ServiceProvider::assign(CoverArt::createGrabber(argv[0]))}; coverArtGrabber.setDefaultCover(server.appRoot() + "/images/unknown-cover.jpg"); diff --git a/src/lms/ui/admin/DatabaseSettingsView.cpp b/src/lms/ui/admin/DatabaseSettingsView.cpp index 3c4423e1..323dfa7c 100644 --- a/src/lms/ui/admin/DatabaseSettingsView.cpp +++ b/src/lms/ui/admin/DatabaseSettingsView.cpp @@ -49,7 +49,7 @@ class DatabaseSettingsModel : public Wt::WFormModel static const Field MediaDirectoryField; static const Field UpdatePeriodField; static const Field UpdateStartTimeField; - static const Field SimilarityEngineTypeField; + static const Field RecommendationEngineTypeField; static const Field TagsField; DatabaseSettingsModel() @@ -60,7 +60,7 @@ class DatabaseSettingsModel : public Wt::WFormModel addField(MediaDirectoryField); addField(UpdatePeriodField); addField(UpdateStartTimeField); - addField(SimilarityEngineTypeField); + addField(RecommendationEngineTypeField); addField(TagsField); auto dirValidator {std::make_shared()}; @@ -69,7 +69,7 @@ class DatabaseSettingsModel : public Wt::WFormModel setValidator(UpdatePeriodField, createMandatoryValidator()); setValidator(UpdateStartTimeField, createMandatoryValidator()); - setValidator(SimilarityEngineTypeField, createMandatoryValidator()); + setValidator(RecommendationEngineTypeField, createMandatoryValidator()); setValidator(TagsField, createTagsValidator()); // populate the model with initial data @@ -78,7 +78,7 @@ class DatabaseSettingsModel : public Wt::WFormModel std::shared_ptr updatePeriodModel() { return _updatePeriodModel; } std::shared_ptr updateStartTimeModel() { return _updateStartTimeModel; } - std::shared_ptr similarityEngineTypeModel() { return _similarityEngineTypeModel; } + std::shared_ptr recommendationEngineTypeModel() { return _recommendationEngineTypeModel; } void loadData() { @@ -96,9 +96,9 @@ class DatabaseSettingsModel : public Wt::WFormModel if (startTimeRow) setValue(UpdateStartTimeField, _updateStartTimeModel->getString(*startTimeRow)); - auto similarityEngineTypeRow {_similarityEngineTypeModel->getRowFromValue(scanSettings->getSimilarityEngineType())}; - if (similarityEngineTypeRow) - setValue(SimilarityEngineTypeField, _similarityEngineTypeModel->getString(*similarityEngineTypeRow)); + auto recommendationEngineTypeRow {_recommendationEngineTypeModel->getRowFromValue(scanSettings->getRecommendationEngineType())}; + if (recommendationEngineTypeRow) + setValue(RecommendationEngineTypeField, _recommendationEngineTypeModel->getString(*recommendationEngineTypeRow)); auto clusterTypes {scanSettings->getClusterTypes()}; if (!clusterTypes.empty()) @@ -125,9 +125,9 @@ class DatabaseSettingsModel : public Wt::WFormModel if (startTimeRow) scanSettings.modify()->setUpdateStartTime(_updateStartTimeModel->getValue(*startTimeRow)); - auto similarityEngineTypeRow {_similarityEngineTypeModel->getRowFromString(valueText(SimilarityEngineTypeField))}; - if (similarityEngineTypeRow) - scanSettings.modify()->setSimilarityEngineType(_similarityEngineTypeModel->getValue(*similarityEngineTypeRow)); + auto recommendationEngineTypeRow {_recommendationEngineTypeModel->getRowFromString(valueText(RecommendationEngineTypeField))}; + if (recommendationEngineTypeRow) + scanSettings.modify()->setRecommendationEngineType(_recommendationEngineTypeModel->getValue(*recommendationEngineTypeRow)); auto clusterTypes {StringUtils::splitString(valueText(TagsField).toUTF8(), " ")}; scanSettings.modify()->setClusterTypes(LmsApp->getDbSession(), std::set(clusterTypes.begin(), clusterTypes.end())); @@ -156,22 +156,22 @@ class DatabaseSettingsModel : public Wt::WFormModel _updateStartTimeModel->add(time.toString(), time); } - _similarityEngineTypeModel = std::make_shared>(); - _similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.clusters"), ScanSettings::SimilarityEngineType::Clusters); - _similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.features"), ScanSettings::SimilarityEngineType::Features); + _recommendationEngineTypeModel = std::make_shared>(); + _recommendationEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.recommendation-engine-type.clusters"), ScanSettings::RecommendationEngineType::Clusters); + _recommendationEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.recommendation-engine-type.features"), ScanSettings::RecommendationEngineType::Features); } std::shared_ptr> _updatePeriodModel; std::shared_ptr> _updateStartTimeModel; - std::shared_ptr> _similarityEngineTypeModel; + std::shared_ptr> _recommendationEngineTypeModel; }; -const Wt::WFormModel::Field DatabaseSettingsModel::MediaDirectoryField = "media-directory"; -const Wt::WFormModel::Field DatabaseSettingsModel::UpdatePeriodField = "update-period"; -const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField = "update-start-time"; -const Wt::WFormModel::Field DatabaseSettingsModel::SimilarityEngineTypeField = "similarity-engine-type"; -const Wt::WFormModel::Field DatabaseSettingsModel::TagsField = "tags"; +const Wt::WFormModel::Field DatabaseSettingsModel::MediaDirectoryField = "media-directory"; +const Wt::WFormModel::Field DatabaseSettingsModel::UpdatePeriodField = "update-period"; +const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField = "update-start-time"; +const Wt::WFormModel::Field DatabaseSettingsModel::RecommendationEngineTypeField = "recommendation-engine-type"; +const Wt::WFormModel::Field DatabaseSettingsModel::TagsField = "tags"; DatabaseSettingsView::DatabaseSettingsView() { @@ -207,10 +207,10 @@ DatabaseSettingsView::refreshView() updateStartTime->setModel(model->updateStartTimeModel()); t->setFormWidget(DatabaseSettingsModel::UpdateStartTimeField, std::move(updateStartTime)); - // Similarity engine type - auto similarityEngineType {std::make_unique()}; - similarityEngineType->setModel(model->similarityEngineTypeModel()); - t->setFormWidget(DatabaseSettingsModel::SimilarityEngineTypeField, std::move(similarityEngineType)); + // recommendation engine type + auto recommendationEngineType {std::make_unique()}; + recommendationEngineType->setModel(model->recommendationEngineTypeModel()); + t->setFormWidget(DatabaseSettingsModel::RecommendationEngineTypeField, std::move(recommendationEngineType)); // Tags t->setFormWidget(DatabaseSettingsModel::TagsField, std::make_unique()); diff --git a/src/test/database/DatabaseTest.cpp b/src/test/database/DatabaseTest.cpp index 005439d1..5ad5b60b 100644 --- a/src/test/database/DatabaseTest.cpp +++ b/src/test/database/DatabaseTest.cpp @@ -453,12 +453,24 @@ testSingleTrackSingleCluster(Session& session) CHECK(track->getClusterIds().empty()); } + { + auto transaction {session.createSharedTransaction()}; + CHECK(Track::getAllIdsWithClusters(session).empty()); + } + { auto transaction {session.createUniqueTransaction()}; cluster1.get().modify()->addTrack(track.get()); } + { + auto transaction {session.createSharedTransaction()}; + auto tracks {Track::getAllIdsWithClusters(session)}; + CHECK(tracks.size() == 1); + CHECK(tracks.front() == track.getId()); + } + { auto transaction {session.createSharedTransaction()}; auto clusters {Cluster::getAllOrphans(session)}; @@ -611,6 +623,11 @@ testSingleTrackSingleReleaseSingleCluster(Session& session) ScopedClusterType clusterType {session, "MyClusterType"}; ScopedCluster cluster {session, clusterType .lockAndGet(), "MyCluster"}; + { + auto transaction {session.createSharedTransaction()}; + CHECK(Release::getAllIdsWithClusters(session).empty()); + } + { auto transaction {session.createUniqueTransaction()}; @@ -618,6 +635,13 @@ testSingleTrackSingleReleaseSingleCluster(Session& session) cluster.get().modify()->addTrack(track.get()); } + { + auto transaction {session.createSharedTransaction()}; + auto releases {Release::getAllIdsWithClusters(session)}; + CHECK(releases.size() == 1); + CHECK(releases.front() == release.getId()); + } + { auto transaction {session.createSharedTransaction()}; @@ -858,6 +882,11 @@ testSingleTrackSingleReleaseSingleArtistSingleCluster(Session& session) ScopedClusterType clusterType {session, "MyType"}; ScopedCluster cluster {session, clusterType.lockAndGet(), "MyCluster"}; + { + auto transaction {session.createSharedTransaction()}; + CHECK(Artist::getAllIdsWithClusters(session).empty()); + } + { auto transaction {session.createUniqueTransaction()}; @@ -875,6 +904,13 @@ testSingleTrackSingleReleaseSingleArtistSingleCluster(Session& session) CHECK(Release::getAllOrphans(session).empty()); } + { + auto transaction {session.createSharedTransaction()}; + auto artists {Artist::getAllIdsWithClusters(session)}; + CHECK(artists.size() == 1); + CHECK(artists.front() == artist.getId()); + } + { auto transaction {session.createSharedTransaction()}; diff --git a/src/tools/recommendation-features/LmsRecommendationFeatures.cpp b/src/tools/recommendation-features/LmsRecommendationFeatures.cpp index 6426dff5..a8c591be 100644 --- a/src/tools/recommendation-features/LmsRecommendationFeatures.cpp +++ b/src/tools/recommendation-features/LmsRecommendationFeatures.cpp @@ -32,6 +32,7 @@ #include "utils/Service.hpp" #include "utils/StreamLogger.hpp" #include "recommendation/IEngine.hpp" +#include "recommendation/IClassifier.hpp" #include "recommendation/FeaturesClassifierCreator.hpp" int main(int argc, char *argv[]) @@ -50,11 +51,9 @@ int main(int argc, char *argv[]) Database::Db db {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; Database::Session session {db}; - auto classifier {Recommendation::createFeaturesClassifier()}; - - std::cout << "Classifying tracks..." << std::endl; // may be long... - classifier->classify(); + std::cout << "Classifying tracks..." << std::endl; + auto classifier {Recommendation::createFeaturesClassifier()}; std::cout << "Classifying tracks DONE" << std::endl; const std::vector trackIds = std::invoke([&]() @@ -106,7 +105,7 @@ int main(int argc, char *argv[]) }; std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl; - for (Database::IdType similarReleaseId : classifier->getSimilarReleases({releaseId}, 3)) + for (Database::IdType similarReleaseId : classifier->getSimilarReleases(session, {releaseId}, 3)) std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl; } @@ -128,7 +127,7 @@ int main(int argc, char *argv[]) }; std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl; - for (Database::IdType similarArtistId : classifier->getSimilarArtists({artistId}, 3)) + for (Database::IdType similarArtistId : classifier->getSimilarArtists(session, {artistId}, 3)) std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl; } From cfb8bbe2ceb58079da4603ff173cf4f2f4bfbeff Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 15 Feb 2020 16:22:28 +0100 Subject: [PATCH 14/23] Reloading the recommendation engine once scan is complete --- approot/admin-database.xml | 8 ++-- src/libs/recommendation/impl/Engine.cpp | 45 +++++++++++-------- src/libs/recommendation/impl/Engine.hpp | 4 +- .../include/recommendation/IEngine.hpp | 2 +- src/libs/scanner/impl/MediaScanner.cpp | 8 ++-- src/lms/main.cpp | 18 +++++++- 6 files changed, 55 insertions(+), 30 deletions(-) diff --git a/approot/admin-database.xml b/approot/admin-database.xml index e5c8ec6f..6e2b08ec 100644 --- a/approot/admin-database.xml +++ b/approot/admin-database.xml @@ -47,14 +47,14 @@
-
diff --git a/src/libs/recommendation/impl/Engine.cpp b/src/libs/recommendation/impl/Engine.cpp index a5cf2a63..d53d8645 100644 --- a/src/libs/recommendation/impl/Engine.cpp +++ b/src/libs/recommendation/impl/Engine.cpp @@ -36,11 +36,11 @@ createEngine(Database::Session& session) Engine::Engine(Database::Session& session) { - reloadSettings(session); + reload(session); } void -Engine::reloadSettings(Database::Session& session) +Engine::reload(Database::Session& session) { using namespace Database; @@ -50,56 +50,57 @@ Engine::reloadSettings(Database::Session& session) return ScanSettings::get(session)->getRecommendationEngineType(); }()}; + // TODO: just replace the classifier once it is ready clearClassifiers(); switch (engineType) { case ScanSettings::RecommendationEngineType::Features: -// _classifiers.emplace(0, createFeaturesClassifier()); // higher priority +// addClassifier(createFeaturesClassifier(), 0); // higher priority // [[fallthrough]]; case ScanSettings::RecommendationEngineType::Clusters: - _classifiers.emplace(1, createClustersClassifier(session)); // lower priority + addClassifier(createClustersClassifier(session), 1); // lower priority break; } } std::vector -Engine::getSimilarTracksFromTrackList(Database::Session& /*session*/, Database::IdType /*trackListId*/, std::size_t /*maxCount*/) +Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) { -#if 0 - auto engineType {getEngineType(session)}; - auto somSearcher {_somAddon.getSearcher()}; - - std::set trackIds; + const std::unordered_set trackIds {[&]() -> std::unordered_set { auto transaction {session.createSharedTransaction()}; + Database::TrackList::pointer trackList {Database::TrackList::getById(session, trackListId)}; if (trackList) { const std::vector orderedTrackIds {trackList->getTrackIds()}; - trackIds = std::set {std::cbegin(orderedTrackIds), std::cend(orderedTrackIds)}; + return std::unordered_set {std::cbegin(orderedTrackIds), std::cend(orderedTrackIds)}; } - } + + return {}; + }()}; if (trackIds.empty()) return {}; - if (engineType == Database::ScanSettings::SimilarityEngineType::Features - && somSearcher - && std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return somSearcher->isTrackClassified(trackId); } )) + std::shared_lock lock {_mutex}; + + for (const auto& [priority, classifier] : _classifiers) { - return somSearcher->getSimilarTracks(trackIds, maxCount); + if (std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return classifier->isTrackClassified(trackId); } )) + return classifier->getSimilarTracksFromTrackList(session, trackListId, maxCount); } - else - return ClusterEngine::getSimilarTracksFromTrackList(session, trackListId, maxCount); -#endif + return {}; } std::vector Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set& trackIds, std::size_t maxCount) { + std::shared_lock lock {_mutex}; + for (const auto& [priority, classifier] : _classifiers) { if (std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return classifier->isTrackClassified(trackId); } )) @@ -112,6 +113,8 @@ Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set< std::vector Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) { + std::shared_lock lock {_mutex}; + for (const auto& [priority, classifier] : _classifiers) { if (classifier->isReleaseClassified(releaseId)) @@ -124,6 +127,8 @@ Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releas std::vector Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) { + std::shared_lock lock {_mutex}; + for (const auto& [priority, classifier] : _classifiers) { if (classifier->isArtistClassified(artistId)) @@ -136,12 +141,14 @@ Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistI void Engine::clearClassifiers() { + std::unique_lock lock {_mutex}; _classifiers.clear(); } void Engine::addClassifier(std::unique_ptr classifier, unsigned priority) { + std::unique_lock lock {_mutex}; _classifiers.emplace(priority, std::move(classifier)); } diff --git a/src/libs/recommendation/impl/Engine.hpp b/src/libs/recommendation/impl/Engine.hpp index 2fdace41..17b9787c 100644 --- a/src/libs/recommendation/impl/Engine.hpp +++ b/src/libs/recommendation/impl/Engine.hpp @@ -20,6 +20,7 @@ #pragma once #include +#include #include "recommendation/IEngine.hpp" #include "recommendation/IClassifier.hpp" @@ -38,7 +39,7 @@ namespace Recommendation private: - void reloadSettings(Database::Session& session) override; + void reload(Database::Session& session) override; // Closest results first std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override; @@ -49,6 +50,7 @@ namespace Recommendation void clearClassifiers(); void addClassifier(std::unique_ptr classifier, unsigned priority); + std::shared_mutex _mutex; std::map> _classifiers; }; diff --git a/src/libs/recommendation/include/recommendation/IEngine.hpp b/src/libs/recommendation/include/recommendation/IEngine.hpp index 91f10c50..4561f19d 100644 --- a/src/libs/recommendation/include/recommendation/IEngine.hpp +++ b/src/libs/recommendation/include/recommendation/IEngine.hpp @@ -36,7 +36,7 @@ namespace Recommendation public: virtual ~IEngine() = default; - virtual void reloadSettings(Database::Session& session) = 0; + virtual void reload(Database::Session& session) = 0; // Closest results first virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0; diff --git a/src/libs/scanner/impl/MediaScanner.cpp b/src/libs/scanner/impl/MediaScanner.cpp index a1a88753..6a606773 100644 --- a/src/libs/scanner/impl/MediaScanner.cpp +++ b/src/libs/scanner/impl/MediaScanner.cpp @@ -427,6 +427,10 @@ MediaScanner::scan(boost::system::error_code err) addon->preScanComplete(); } + LMS_LOG(DBUPDATER, INFO) << "Optimizing db..."; + _dbSession.optimize(); + LMS_LOG(DBUPDATER, INFO) << "Optimize db done!"; + if (_running) { stats.stopTime = Wt::WLocalDateTime::currentDateTime().toUTC(); @@ -448,10 +452,6 @@ MediaScanner::scan(boost::system::error_code err) _curState = State::NotScheduled; _inProgressScanStats.reset(); } - - LMS_LOG(DBUPDATER, INFO) << "Optimizing db..."; - _dbSession.optimize(); - LMS_LOG(DBUPDATER, INFO) << "Optimize db done!"; } void diff --git a/src/lms/main.cpp b/src/lms/main.cpp index f2a71fa0..b82a7917 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -145,7 +145,23 @@ int main(int argc, char* argv[]) { Database::Session session {database}; - ServiceProvider::assign(Recommendation::createEngine(session)); + Recommendation::IEngine& recommendationEngine {ServiceProvider::assign(Recommendation::createEngine(session))}; + + mediaScanner.scanComplete().connect([&]() + { + auto status = mediaScanner.getStatus(); + + if (status.lastCompleteScanStats->nbChanges() > 0) + { + LMS_LOG(MAIN, INFO) << "Scanner changed some files, reloading the recommendation engine..."; + Database::Session session {database}; + recommendationEngine.reload(session); + } + else + { + LMS_LOG(MAIN, INFO) << "Scanner did not change files, not reloading the recommendation engine..."; + } + }); } CoverArt::IGrabber& coverArtGrabber {ServiceProvider::assign(CoverArt::createGrabber(argv[0]))}; From 6239064396d7bca88c88734bf5d2a5286b4edccd Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 15 Feb 2020 16:33:05 +0100 Subject: [PATCH 15/23] Now have a recommendation tool for the whole database --- src/tools/CMakeLists.txt | 2 +- .../recommendation-features/CMakeLists.txt | 10 -------- src/tools/recommendation/CMakeLists.txt | 11 ++++++++ .../LmsRecommendation.cpp} | 25 ++++++++----------- 4 files changed, 23 insertions(+), 25 deletions(-) delete mode 100644 src/tools/recommendation-features/CMakeLists.txt create mode 100644 src/tools/recommendation/CMakeLists.txt rename src/tools/{recommendation-features/LmsRecommendationFeatures.cpp => recommendation/LmsRecommendation.cpp} (83%) diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 4fcdba31..e5877a5b 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -1,5 +1,5 @@ add_subdirectory(metadata) -add_subdirectory(recommendation-features) +add_subdirectory(recommendation) diff --git a/src/tools/recommendation-features/CMakeLists.txt b/src/tools/recommendation-features/CMakeLists.txt deleted file mode 100644 index 25508174..00000000 --- a/src/tools/recommendation-features/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ - -add_executable(lms-recommendation-features - LmsRecommendationFeatures.cpp - ) - -target_link_libraries(lms-recommendation-features PRIVATE - lmsdatabase - lmsrecommendation - ) - diff --git a/src/tools/recommendation/CMakeLists.txt b/src/tools/recommendation/CMakeLists.txt new file mode 100644 index 00000000..75b3e5d0 --- /dev/null +++ b/src/tools/recommendation/CMakeLists.txt @@ -0,0 +1,11 @@ + +add_executable(lms-recommendation + LmsRecommendation.cpp + ) + +target_link_libraries(lms-recommendation PRIVATE + lmsdatabase + lmsrecommendation + ) + +install(TARGETS lms-recommendation DESTINATION bin) diff --git a/src/tools/recommendation-features/LmsRecommendationFeatures.cpp b/src/tools/recommendation/LmsRecommendation.cpp similarity index 83% rename from src/tools/recommendation-features/LmsRecommendationFeatures.cpp rename to src/tools/recommendation/LmsRecommendation.cpp index a8c591be..2ff53459 100644 --- a/src/tools/recommendation-features/LmsRecommendationFeatures.cpp +++ b/src/tools/recommendation/LmsRecommendation.cpp @@ -32,8 +32,6 @@ #include "utils/Service.hpp" #include "utils/StreamLogger.hpp" #include "recommendation/IEngine.hpp" -#include "recommendation/IClassifier.hpp" -#include "recommendation/FeaturesClassifierCreator.hpp" int main(int argc, char *argv[]) { @@ -51,16 +49,15 @@ int main(int argc, char *argv[]) Database::Db db {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; Database::Session session {db}; - // may be long... - std::cout << "Classifying tracks..." << std::endl; - auto classifier {Recommendation::createFeaturesClassifier()}; - std::cout << "Classifying tracks DONE" << std::endl; + std::cout << "Creating recommendation engine..." << std::endl; + const auto engine {Recommendation::createEngine(session)}; + std::cout << "DONE!" << std::endl; - const std::vector trackIds = std::invoke([&]() - { - auto transaction {session.createSharedTransaction()}; - return Database::Track::getAllIdsWithFeatures(session); - }); + const std::vector trackIds {[&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Track::getAllIds(session); + }()}; std::cout << "*** Tracks (" << trackIds.size() << ") ***" << std::endl; for (Database::IdType trackId : trackIds) @@ -83,7 +80,7 @@ int main(int argc, char *argv[]) }; std::cout << "Processing track '" << trackToString(trackId) << std::endl; - for (Database::IdType similarTrackId : classifier->getSimilarTracks(session, {trackId}, 3)) + for (Database::IdType similarTrackId : engine->getSimilarTracks(session, {trackId}, 3)) std::cout << "\t- Similar track '" << trackToString(similarTrackId) << std::endl; } @@ -105,7 +102,7 @@ int main(int argc, char *argv[]) }; std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl; - for (Database::IdType similarReleaseId : classifier->getSimilarReleases(session, {releaseId}, 3)) + for (Database::IdType similarReleaseId : engine->getSimilarReleases(session, {releaseId}, 3)) std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl; } @@ -127,7 +124,7 @@ int main(int argc, char *argv[]) }; std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl; - for (Database::IdType similarArtistId : classifier->getSimilarArtists(session, {artistId}, 3)) + for (Database::IdType similarArtistId : engine->getSimilarArtists(session, {artistId}, 3)) std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl; } From a414a80bbb875d68151f1479f31ca1c6dda54c06 Mon Sep 17 00:00:00 2001 From: emeric Date: Sun, 16 Feb 2020 12:39:27 +0100 Subject: [PATCH 16/23] Further decoupled recommendation engine: it now has its own thread --- src/libs/recommendation/CMakeLists.txt | 1 + src/libs/recommendation/impl/Engine.cpp | 103 +++++++---- src/libs/recommendation/impl/Engine.hpp | 26 ++- .../include/recommendation/IEngine.hpp | 12 +- src/libs/scanner/impl/MediaScanner.cpp | 55 +++--- src/libs/scanner/impl/MediaScanner.hpp | 14 +- .../scanner/include/scanner/IMediaScanner.hpp | 3 - src/libs/utils/impl/Logger.cpp | 2 +- src/libs/utils/include/utils/Logger.hpp | 2 +- .../include/utils/Semaphore.hpp} | 41 +++-- src/lms/main.cpp | 35 ++-- .../recommendation/LmsRecommendation.cpp | 170 ++++++++++-------- 12 files changed, 280 insertions(+), 184 deletions(-) rename src/libs/{scanner/include/scanner/MediaScannerAddon.hpp => utils/include/utils/Semaphore.hpp} (53%) diff --git a/src/libs/recommendation/CMakeLists.txt b/src/libs/recommendation/CMakeLists.txt index 260f137b..26e29757 100644 --- a/src/libs/recommendation/CMakeLists.txt +++ b/src/libs/recommendation/CMakeLists.txt @@ -16,6 +16,7 @@ target_include_directories(lmsrecommendation PRIVATE target_link_libraries(lmsrecommendation PRIVATE lmsdatabase lmssom + wt ) install(TARGETS lmsrecommendation DESTINATION lib) diff --git a/src/libs/recommendation/impl/Engine.cpp b/src/libs/recommendation/impl/Engine.cpp index d53d8645..5b3d818f 100644 --- a/src/libs/recommendation/impl/Engine.cpp +++ b/src/libs/recommendation/impl/Engine.cpp @@ -23,46 +23,51 @@ #include "recommendation/FeaturesClassifierCreator.hpp" #include "database/ScanSettings.hpp" -#include "database/Session.hpp" #include "database/TrackList.hpp" +#include "utils/Logger.hpp" namespace Recommendation { std::unique_ptr -createEngine(Database::Session& session) +createEngine(Database::Db& db) { - return std::make_unique(session); + return std::make_unique(db); } -Engine::Engine(Database::Session& session) +Engine::Engine(Database::Db& db) +: _dbSession {db} { - reload(session); } void -Engine::reload(Database::Session& session) +Engine::start() { - using namespace Database; + assert(!_running); + _running = true; - const ScanSettings::RecommendationEngineType engineType {[&]() + requestReload(); + + _ioService.start(); +} + +void +Engine::stop() +{ + assert(_running); + _running = false; + + _ioService.stop(); +} + +void +Engine::requestReload() +{ + LMS_LOG(RECOMMENDATION, DEBUG) << "Reload requested..."; + + _ioService.post([&]() { - auto transaction {session.createSharedTransaction()}; - return ScanSettings::get(session)->getRecommendationEngineType(); - }()}; - - // TODO: just replace the classifier once it is ready - clearClassifiers(); - - switch (engineType) - { - case ScanSettings::RecommendationEngineType::Features: -// addClassifier(createFeaturesClassifier(), 0); // higher priority -// [[fallthrough]]; - - case ScanSettings::RecommendationEngineType::Clusters: - addClassifier(createClustersClassifier(session), 1); // lower priority - break; - } + reload(); + }); } std::vector @@ -85,7 +90,7 @@ Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdTy if (trackIds.empty()) return {}; - std::shared_lock lock {_mutex}; + std::shared_lock lock {_classifiersMutex}; for (const auto& [priority, classifier] : _classifiers) { @@ -99,7 +104,7 @@ Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdTy std::vector Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set& trackIds, std::size_t maxCount) { - std::shared_lock lock {_mutex}; + std::shared_lock lock {_classifiersMutex}; for (const auto& [priority, classifier] : _classifiers) { @@ -113,7 +118,7 @@ Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set< std::vector Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) { - std::shared_lock lock {_mutex}; + std::shared_lock lock {_classifiersMutex}; for (const auto& [priority, classifier] : _classifiers) { @@ -127,7 +132,7 @@ Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releas std::vector Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) { - std::shared_lock lock {_mutex}; + std::shared_lock lock {_classifiersMutex}; for (const auto& [priority, classifier] : _classifiers) { @@ -138,17 +143,53 @@ Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistI return {}; } +void +Engine::reload() +{ + using namespace Database; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Reloading recommendation engines..."; + + const ScanSettings::RecommendationEngineType engineType {[&]() + { + auto transaction {_dbSession.createSharedTransaction()}; + + return ScanSettings::get(_dbSession)->getRecommendationEngineType(); + }()}; + + // TODO: just replace the classifier once it is ready + clearClassifiers(); + + switch (engineType) + { + case ScanSettings::RecommendationEngineType::Features: +// addClassifier(createFeaturesClassifier(), 0); // higher priority +// [[fallthrough]]; + + case ScanSettings::RecommendationEngineType::Clusters: + addClassifier(createClustersClassifier(_dbSession), 1); // lower priority + break; + } + + LMS_LOG(RECOMMENDATION, DEBUG) << "Recommendation engines reloaded!"; + + _sigReloaded.emit(); +} + + void Engine::clearClassifiers() { - std::unique_lock lock {_mutex}; + std::unique_lock lock {_classifiersMutex}; + _classifiers.clear(); } void Engine::addClassifier(std::unique_ptr classifier, unsigned priority) { - std::unique_lock lock {_mutex}; + std::unique_lock lock {_classifiersMutex}; + _classifiers.emplace(priority, std::move(classifier)); } diff --git a/src/libs/recommendation/impl/Engine.hpp b/src/libs/recommendation/impl/Engine.hpp index 17b9787c..bc6eea96 100644 --- a/src/libs/recommendation/impl/Engine.hpp +++ b/src/libs/recommendation/impl/Engine.hpp @@ -22,35 +22,43 @@ #include #include +#include + +#include "database/Session.hpp" #include "recommendation/IEngine.hpp" #include "recommendation/IClassifier.hpp" -namespace Database -{ - class Session; -} - namespace Recommendation { class Engine : public IEngine { public: - Engine(Database::Session& session); + Engine(Database::Db& db); private: - void reload(Database::Session& session) override; + void start() override; + void stop() override; + + void requestReload() override; + Wt::Signal<>& reloaded() override { return _sigReloaded; } - // Closest results first std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override; std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) override; std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override; std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override; + + void reload(); void clearClassifiers(); void addClassifier(std::unique_ptr classifier, unsigned priority); - std::shared_mutex _mutex; + bool _running {}; + Wt::WIOService _ioService; + Database::Session _dbSession; + Wt::Signal<> _sigReloaded; + + std::shared_mutex _classifiersMutex; std::map> _classifiers; }; diff --git a/src/libs/recommendation/include/recommendation/IEngine.hpp b/src/libs/recommendation/include/recommendation/IEngine.hpp index 4561f19d..37f7cb8c 100644 --- a/src/libs/recommendation/include/recommendation/IEngine.hpp +++ b/src/libs/recommendation/include/recommendation/IEngine.hpp @@ -19,13 +19,17 @@ #pragma once +#include #include #include +#include + #include "database/Types.hpp" namespace Database { + class Db; class Session; } @@ -36,7 +40,11 @@ namespace Recommendation public: virtual ~IEngine() = default; - virtual void reload(Database::Session& session) = 0; + virtual void start() = 0; + virtual void stop() = 0; + + virtual void requestReload() = 0; + virtual Wt::Signal<>& reloaded() = 0; // Closest results first virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0; @@ -45,7 +53,7 @@ namespace Recommendation virtual std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0; }; - std::unique_ptr createEngine(Database::Session& session); + std::unique_ptr createEngine(Database::Db& db); } // ns Recommendation diff --git a/src/libs/scanner/impl/MediaScanner.cpp b/src/libs/scanner/impl/MediaScanner.cpp index 6a606773..84e78d92 100644 --- a/src/libs/scanner/impl/MediaScanner.cpp +++ b/src/libs/scanner/impl/MediaScanner.cpp @@ -28,6 +28,7 @@ #include "database/Release.hpp" #include "database/ScanSettings.hpp" #include "database/Track.hpp" +#include "metadata/TagLibParser.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" #include "utils/Path.hpp" @@ -200,15 +201,18 @@ createMediaScanner(Database::Db& db) MediaScanner::MediaScanner(Database::Db& db) : _dbSession {db} { + // For now, always use TagLib + _metadataParser = std::make_unique(); + _ioService.setThreadCount(1); refreshScanSettings(); } -void -MediaScanner::setAddon(MediaScannerAddon& addon) +MediaScanner::~MediaScanner() { - _addons.push_back(&addon); + if (_running) + stop(); } void @@ -233,9 +237,6 @@ MediaScanner::stop(void) { _running = false; - for (auto& addon : _addons) - addon->requestStop(); - _scheduleTimer.cancel(); _ioService.stop(); @@ -281,7 +282,7 @@ MediaScanner::scheduleNextScan() refreshScanSettings(); - Wt::WDateTime now {Wt::WLocalDateTime::currentServerDateTime().toUTC()}; + const Wt::WDateTime now {Wt::WLocalDateTime::currentServerDateTime().toUTC()}; Wt::WDate nextScanDate; switch (_updatePeriod) @@ -421,12 +422,6 @@ MediaScanner::scan(boost::system::error_code err) LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), duplicates = " << stats.duplicates.size(); - if (_running) - { - for (auto& addon : _addons) - addon->preScanComplete(); - } - LMS_LOG(DBUPDATER, INFO) << "Optimizing db..."; _dbSession.optimize(); LMS_LOG(DBUPDATER, INFO) << "Optimize db done!"; @@ -457,32 +452,28 @@ MediaScanner::scan(boost::system::error_code err) void MediaScanner::refreshScanSettings() { - { - auto transaction {_dbSession.createSharedTransaction()}; + auto transaction {_dbSession.createSharedTransaction()}; - ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)}; + ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)}; - LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion(); + LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion(); - _scanVersion = scanSettings->getScanVersion(); - _startTime = scanSettings->getUpdateStartTime(); - _updatePeriod = scanSettings->getUpdatePeriod(); + _scanVersion = scanSettings->getScanVersion(); + _startTime = scanSettings->getUpdateStartTime(); + _updatePeriod = scanSettings->getUpdatePeriod(); - _fileExtensions = scanSettings->getAudioFileExtensions(); - _mediaDirectory = scanSettings->getMediaDirectory(); + _fileExtensions = scanSettings->getAudioFileExtensions(); + _mediaDirectory = scanSettings->getMediaDirectory(); - auto clusterTypes = scanSettings->getClusterTypes(); - std::set clusterTypeNames; + auto clusterTypes = scanSettings->getClusterTypes(); + std::set clusterTypeNames; - std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes), - std::inserter(clusterTypeNames, clusterTypeNames.begin()), - [](ClusterType::pointer clusterType) { return clusterType->getName(); }); + std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes), + std::inserter(clusterTypeNames, clusterTypeNames.begin()), + [](ClusterType::pointer clusterType) { return clusterType->getName(); }); - _metadataParser.setClusterTypeNames(clusterTypeNames); - } + _metadataParser->setClusterTypeNames(clusterTypeNames); - for (auto& addon : _addons) - addon->refreshSettings(); } void @@ -539,7 +530,7 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S } } - std::optional trackInfo {_metadataParser.parse(file)}; + std::optional trackInfo {_metadataParser->parse(file)}; if (!trackInfo) { stats.errors.emplace_back(file, ScanErrorType::CannotParseFile); diff --git a/src/libs/scanner/impl/MediaScanner.hpp b/src/libs/scanner/impl/MediaScanner.hpp index c6452156..54bb70ce 100644 --- a/src/libs/scanner/impl/MediaScanner.hpp +++ b/src/libs/scanner/impl/MediaScanner.hpp @@ -29,9 +29,10 @@ #include +#include "database/Types.hpp" #include "database/ScanSettings.hpp" #include "database/Session.hpp" -#include "metadata/TagLibParser.hpp" +#include "metadata/IParser.hpp" #include "scanner/IMediaScanner.hpp" @@ -41,8 +42,12 @@ class MediaScanner : public IMediaScanner { public: MediaScanner(Database::Db& db); + ~MediaScanner(); - void setAddon(MediaScannerAddon& addon) override; + MediaScanner(const MediaScanner&) = delete; + MediaScanner(MediaScanner&&) = delete; + MediaScanner& operator=(const MediaScanner&) = delete; + MediaScanner& operator=(MediaScanner&&) = delete; void start() override; void stop() override; @@ -80,7 +85,7 @@ class MediaScanner : public IMediaScanner void notifyInProgressIfNeeded(const ScanStats& stats); void notifyInProgress(const ScanStats& stats); - bool _running {false}; + bool _running {}; Wt::WIOService _ioService; boost::asio::system_timer _scheduleTimer {_ioService}; Wt::Signal<> _sigScanComplete; @@ -88,8 +93,7 @@ class MediaScanner : public IMediaScanner std::chrono::system_clock::time_point _lastScanInProgressEmit {}; Wt::Signal _sigScheduled; Database::Session _dbSession; - MetaData::TagLibParser _metadataParser; - std::vector _addons; + std::unique_ptr _metadataParser; std::mutex _statusMutex; State _curState {State::NotScheduled}; diff --git a/src/libs/scanner/include/scanner/IMediaScanner.hpp b/src/libs/scanner/include/scanner/IMediaScanner.hpp index 49b68048..e50be005 100644 --- a/src/libs/scanner/include/scanner/IMediaScanner.hpp +++ b/src/libs/scanner/include/scanner/IMediaScanner.hpp @@ -24,7 +24,6 @@ #include #include -#include "MediaScannerAddon.hpp" #include "MediaScannerStats.hpp" namespace Database @@ -39,8 +38,6 @@ class IMediaScanner public: virtual ~IMediaScanner() = default; - virtual void setAddon(MediaScannerAddon& addon) = 0; - virtual void start() = 0; virtual void stop() = 0; virtual void restart() = 0; diff --git a/src/libs/utils/impl/Logger.cpp b/src/libs/utils/impl/Logger.cpp index b0d99437..602fdbc3 100644 --- a/src/libs/utils/impl/Logger.cpp +++ b/src/libs/utils/impl/Logger.cpp @@ -34,7 +34,7 @@ const char* getModuleName(Module mod) case Module::METADATA: return "METADATA"; case Module::REMOTE: return "REMOTE"; case Module::SERVICE: return "SERVICE"; - case Module::SIMILARITY: return "SIMILARITY"; + case Module::RECOMMENDATION: return "RECOMMENDATION"; case Module::TRANSCODE: return "TRANSCODE"; case Module::UI: return "UI"; } diff --git a/src/libs/utils/include/utils/Logger.hpp b/src/libs/utils/include/utils/Logger.hpp index 9597aa5c..9f5b2ec5 100644 --- a/src/libs/utils/include/utils/Logger.hpp +++ b/src/libs/utils/include/utils/Logger.hpp @@ -46,7 +46,7 @@ enum class Module METADATA, REMOTE, SERVICE, - SIMILARITY, + RECOMMENDATION, TRANSCODE, UI, }; diff --git a/src/libs/scanner/include/scanner/MediaScannerAddon.hpp b/src/libs/utils/include/utils/Semaphore.hpp similarity index 53% rename from src/libs/scanner/include/scanner/MediaScannerAddon.hpp rename to src/libs/utils/include/utils/Semaphore.hpp index fdfd9577..7b82879d 100644 --- a/src/libs/scanner/include/scanner/MediaScannerAddon.hpp +++ b/src/libs/utils/include/utils/Semaphore.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018 Emeric Poupon + * Copyright (C) 2020 Emeric Poupon * * This file is part of LMS. * @@ -19,23 +19,40 @@ #pragma once -#include "database/Types.hpp" +#include +#include -namespace Scanner { - -class MediaScannerAddon +class Semaphore { public: + Semaphore() = default; + Semaphore(const Semaphore&) = delete; + Semaphore(Semaphore&&) = delete; + Semaphore& operator=(const Semaphore&) = delete; + Semaphore& operator=(Semaphore&&) = delete; - virtual void refreshSettings() = 0; - virtual void requestStop() = 0; - virtual void preScanComplete() = 0; + void notify() + { + std::unique_lock lock {_mutex}; - virtual void trackAdded(Database::IdType trackId) = 0; - virtual void trackToRemove(Database::IdType trackId) = 0; - virtual void trackUpdated(Database::IdType trackId) = 0; + _count++; + _cv.notify_one(); + } + void wait() + { + std::unique_lock lock(_mutex); + + while (_count == 0) + _cv.wait(lock); + + _count--; + } + + private: + std::mutex _mutex; + std::condition_variable _cv; + unsigned _count {}; }; -} // ns Scanner diff --git a/src/lms/main.cpp b/src/lms/main.cpp index b82a7917..1b6e504e 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -143,26 +143,21 @@ int main(int argc, char* argv[]) ServiceProvider::assign(Auth::createPasswordService(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000))); Scanner::IMediaScanner& mediaScanner {ServiceProvider::assign(Scanner::createMediaScanner(database))}; + Recommendation::IEngine& recommendationEngine {ServiceProvider::assign(Recommendation::createEngine(database))}; + mediaScanner.scanComplete().connect([&]() { - Database::Session session {database}; - Recommendation::IEngine& recommendationEngine {ServiceProvider::assign(Recommendation::createEngine(session))}; + auto status = mediaScanner.getStatus(); - mediaScanner.scanComplete().connect([&]() + if (status.lastCompleteScanStats->nbChanges() > 0) { - auto status = mediaScanner.getStatus(); - - if (status.lastCompleteScanStats->nbChanges() > 0) - { - LMS_LOG(MAIN, INFO) << "Scanner changed some files, reloading the recommendation engine..."; - Database::Session session {database}; - recommendationEngine.reload(session); - } - else - { - LMS_LOG(MAIN, INFO) << "Scanner did not change files, not reloading the recommendation engine..."; - } - }); - } + LMS_LOG(MAIN, INFO) << "Scanner changed some files, reloading the recommendation engine..."; + recommendationEngine.requestReload(); + } + else + { + LMS_LOG(MAIN, INFO) << "Scanner did not change files, not reloading the recommendation engine..."; + } + }); CoverArt::IGrabber& coverArtGrabber {ServiceProvider::assign(CoverArt::createGrabber(argv[0]))}; coverArtGrabber.setDefaultCover(server.appRoot() + "/images/unknown-cover.jpg"); @@ -179,6 +174,9 @@ int main(int argc, char* argv[]) std::placeholders::_1, std::ref(database), std::ref(appGroups))); // Start + LMS_LOG(MAIN, INFO) << "Starting recommendation engine"; + recommendationEngine.start(); + LMS_LOG(MAIN, INFO) << "Starting media scanner..."; mediaScanner.start(); @@ -196,6 +194,9 @@ int main(int argc, char* argv[]) LMS_LOG(MAIN, INFO) << "Stopping media scanner..."; mediaScanner.stop(); + LMS_LOG(MAIN, INFO) << "Stopping recommendation engine..."; + recommendationEngine.stop(); + LMS_LOG(MAIN, INFO) << "Clean stop!"; res = EXIT_SUCCESS; } diff --git a/src/tools/recommendation/LmsRecommendation.cpp b/src/tools/recommendation/LmsRecommendation.cpp index 2ff53459..b5ea5aa4 100644 --- a/src/tools/recommendation/LmsRecommendation.cpp +++ b/src/tools/recommendation/LmsRecommendation.cpp @@ -29,10 +29,97 @@ #include "database/Session.hpp" #include "database/Track.hpp" #include "utils/IConfig.hpp" +#include "utils/Semaphore.hpp" #include "utils/Service.hpp" #include "utils/StreamLogger.hpp" #include "recommendation/IEngine.hpp" + +static +void +dumpRecommendation(Database::Session session, Recommendation::IEngine& engine) +{ + const std::vector trackIds {[&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Track::getAllIds(session); + }()}; + + std::cout << "*** Tracks (" << trackIds.size() << ") ***" << std::endl; + for (Database::IdType trackId : trackIds) + { + auto trackToString = [&](Database::IdType trackId) + { + std::string res; + auto transaction {session.createSharedTransaction()}; + Database::Track::pointer track {Database::Track::getById(session, trackId)}; + + res += track->getName(); + if (track->getRelease()) + res += " [" + track->getRelease()->getName() + "]"; + for (auto artist : track->getArtists()) + res += " - " + artist->getName(); + for (auto cluster : track->getClusters()) + res += " {" + cluster->getType()->getName() + "-"+ cluster->getName() + "}"; + + return res; + }; + + std::cout << "Processing track '" << trackToString(trackId) << std::endl; + for (Database::IdType similarTrackId : engine.getSimilarTracks(session, {trackId}, 3)) + std::cout << "\t- Similar track '" << trackToString(similarTrackId) << std::endl; + } + + const std::vector releaseIds = std::invoke([&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Release::getAllIds(session); + }); + + std::cout << "*** Releases ***" << std::endl; + for (Database::IdType releaseId : releaseIds) + { + auto releaseToString = [&](Database::IdType releaseId) + { + auto transaction {session.createSharedTransaction()}; + + Database::Release::pointer release {Database::Release::getById(session, releaseId)}; + return release->getName(); + }; + + std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl; + for (Database::IdType similarReleaseId : engine.getSimilarReleases(session, {releaseId}, 3)) + std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl; + } + + const std::vector artistIds = std::invoke([&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Artist::getAllIds(session); + }); + + std::cout << "*** Artists ***" << std::endl; + for (Database::IdType artistId : artistIds) + { + auto artistToString = [&](Database::IdType artistId) + { + auto transaction {session.createSharedTransaction()}; + + Database::Artist::pointer artist {Database::Artist::getById(session, artistId)}; + return artist->getName(); + }; + + std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl; + for (Database::IdType similarArtistId : engine.getSimilarArtists(session, {artistId}, 3)) + std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl; + } + + + + +} + + int main(int argc, char *argv[]) { try @@ -50,84 +137,25 @@ int main(int argc, char *argv[]) Database::Session session {db}; std::cout << "Creating recommendation engine..." << std::endl; - const auto engine {Recommendation::createEngine(session)}; - std::cout << "DONE!" << std::endl; + const auto engine {Recommendation::createEngine(db)}; + std::cout << "Recommendation engine created!" << std::endl; - const std::vector trackIds {[&]() - { - auto transaction {session.createSharedTransaction()}; - return Database::Track::getAllIds(session); - }()}; + Semaphore sem; - std::cout << "*** Tracks (" << trackIds.size() << ") ***" << std::endl; - for (Database::IdType trackId : trackIds) + engine->reloaded().connect([&]() { - auto trackToString = [&](Database::IdType trackId) - { - std::string res; - auto transaction {session.createSharedTransaction()}; - Database::Track::pointer track {Database::Track::getById(session, trackId)}; + sem.notify(); + }); - res += track->getName(); - if (track->getRelease()) - res += " [" + track->getRelease()->getName() + "]"; - for (auto artist : track->getArtists()) - res += " - " + artist->getName(); - for (auto cluster : track->getClusters()) - res += " {" + cluster->getType()->getName() + "-"+ cluster->getName() + "}"; + engine->start(); - return res; - }; + std::cout << "Wating for the recommendation engine to be loaded..." << std::endl; + sem.wait(); + std::cout << "Recommendation engine loaded!" << std::endl; - std::cout << "Processing track '" << trackToString(trackId) << std::endl; - for (Database::IdType similarTrackId : engine->getSimilarTracks(session, {trackId}, 3)) - std::cout << "\t- Similar track '" << trackToString(similarTrackId) << std::endl; - } - - const std::vector releaseIds = std::invoke([&]() - { - auto transaction {session.createSharedTransaction()}; - return Database::Release::getAllIds(session); - }); - - std::cout << "*** Releases ***" << std::endl; - for (Database::IdType releaseId : releaseIds) - { - auto releaseToString = [&](Database::IdType releaseId) - { - auto transaction {session.createSharedTransaction()}; - - Database::Release::pointer release {Database::Release::getById(session, releaseId)}; - return release->getName(); - }; - - std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl; - for (Database::IdType similarReleaseId : engine->getSimilarReleases(session, {releaseId}, 3)) - std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl; - } - - const std::vector artistIds = std::invoke([&]() - { - auto transaction {session.createSharedTransaction()}; - return Database::Artist::getAllIds(session); - }); - - std::cout << "*** Artists ***" << std::endl; - for (Database::IdType artistId : artistIds) - { - auto artistToString = [&](Database::IdType artistId) - { - auto transaction {session.createSharedTransaction()}; - - Database::Artist::pointer artist {Database::Artist::getById(session, artistId)}; - return artist->getName(); - }; - - std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl; - for (Database::IdType similarArtistId : engine->getSimilarArtists(session, {artistId}, 3)) - std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl; - } + dumpRecommendation(db, *engine); + engine->stop(); } catch( std::exception& e) { From 53b3429c74fb15880bf831f50c743e716e3d008f Mon Sep 17 00:00:00 2001 From: emeric Date: Sun, 16 Feb 2020 16:33:38 +0100 Subject: [PATCH 17/23] Atomatically reload recommendation engines --- src/libs/recommendation/impl/Engine.cpp | 19 ++++++++++--------- src/libs/recommendation/impl/Engine.hpp | 4 +++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/libs/recommendation/impl/Engine.cpp b/src/libs/recommendation/impl/Engine.cpp index 5b3d818f..2a4aa772 100644 --- a/src/libs/recommendation/impl/Engine.cpp +++ b/src/libs/recommendation/impl/Engine.cpp @@ -148,7 +148,7 @@ Engine::reload() { using namespace Database; - LMS_LOG(RECOMMENDATION, DEBUG) << "Reloading recommendation engines..."; + LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines..."; const ScanSettings::RecommendationEngineType engineType {[&]() { @@ -157,21 +157,25 @@ Engine::reload() return ScanSettings::get(_dbSession)->getRecommendationEngineType(); }()}; - // TODO: just replace the classifier once it is ready - clearClassifiers(); + std::map> newClassifiers; switch (engineType) { case ScanSettings::RecommendationEngineType::Features: -// addClassifier(createFeaturesClassifier(), 0); // higher priority +// newClassifiers.emplace_back(0, createFeaturesClassifier()); // higher priority // [[fallthrough]]; case ScanSettings::RecommendationEngineType::Clusters: - addClassifier(createClustersClassifier(_dbSession), 1); // lower priority + newClassifiers.emplace(1, createClustersClassifier(_dbSession)); // lower priority break; } - LMS_LOG(RECOMMENDATION, DEBUG) << "Recommendation engines reloaded!"; + { + std::unique_lock lock {_classifiersMutex}; + _classifiers.swap(newClassifiers); + } + + LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines reloaded!"; _sigReloaded.emit(); } @@ -180,9 +184,6 @@ Engine::reload() void Engine::clearClassifiers() { - std::unique_lock lock {_classifiersMutex}; - - _classifiers.clear(); } void diff --git a/src/libs/recommendation/impl/Engine.hpp b/src/libs/recommendation/impl/Engine.hpp index bc6eea96..9fde430e 100644 --- a/src/libs/recommendation/impl/Engine.hpp +++ b/src/libs/recommendation/impl/Engine.hpp @@ -37,6 +37,8 @@ namespace Recommendation private: + using ClassifierPriority = unsigned; + void start() override; void stop() override; @@ -59,7 +61,7 @@ namespace Recommendation Wt::Signal<> _sigReloaded; std::shared_mutex _classifiersMutex; - std::map> _classifiers; + std::map> _classifiers; }; } // ns Recommendation From efbec56d26b08ea16120f8703e4f74e816587448 Mon Sep 17 00:00:00 2001 From: emeric Date: Tue, 18 Feb 2020 13:11:00 +0100 Subject: [PATCH 18/23] Restored audio simimarity based classifier --- src/libs/recommendation/CMakeLists.txt | 4 +- .../recommendation/impl/ClassifierCreator.cpp | 33 -- src/libs/recommendation/impl/Engine.cpp | 103 ++-- src/libs/recommendation/impl/Engine.hpp | 6 +- .../impl/clusters/ClustersClassifier.cpp | 49 +- .../impl/clusters/ClustersClassifier.hpp | 14 +- .../impl/features/FeaturesClassifier.cpp | 445 ++++++++++++++++ .../impl/features/FeaturesClassifier.hpp | 112 ++++ ...sCache.cpp => FeaturesClassifierCache.cpp} | 58 +-- ...sCache.hpp => FeaturesClassifierCache.hpp} | 27 +- ...arityFeaturesDefs.cpp => FeaturesDefs.cpp} | 6 +- ...arityFeaturesDefs.hpp => FeaturesDefs.hpp} | 4 +- .../SimilarityFeaturesScannerAddon.cpp | 187 ------- .../features/SimilarityFeaturesSearcher.cpp | 487 ------------------ .../features/SimilarityFeaturesSearcher.hpp | 110 ---- .../ClustersClassifierCreator.hpp | 2 +- .../FeaturesClassifierCreator.hpp | 3 +- .../include/recommendation/IClassifier.hpp | 8 +- src/libs/scanner/CMakeLists.txt | 1 + .../impl}/AcousticBrainzUtils.cpp | 7 +- .../impl}/AcousticBrainzUtils.hpp | 2 +- src/libs/scanner/impl/MediaScanner.cpp | 80 ++- src/libs/scanner/impl/MediaScanner.hpp | 8 +- .../include/scanner/MediaScannerStats.hpp | 5 +- src/libs/som/CMakeLists.txt | 2 + src/libs/som/impl/Network.cpp | 4 +- src/libs/som/include/som/InputVector.hpp | 2 +- src/libs/som/include/som/Matrix.hpp | 19 +- src/libs/som/include/som/Network.hpp | 4 +- src/lms/main.cpp | 2 +- src/test/som/SomTest.cpp | 2 +- .../recommendation/LmsRecommendation.cpp | 2 +- 32 files changed, 813 insertions(+), 985 deletions(-) delete mode 100644 src/libs/recommendation/impl/ClassifierCreator.cpp create mode 100644 src/libs/recommendation/impl/features/FeaturesClassifier.cpp create mode 100644 src/libs/recommendation/impl/features/FeaturesClassifier.hpp rename src/libs/recommendation/impl/features/{SimilarityFeaturesCache.cpp => FeaturesClassifierCache.cpp} (76%) rename src/libs/recommendation/impl/features/{SimilarityFeaturesCache.hpp => FeaturesClassifierCache.hpp} (53%) rename src/libs/recommendation/impl/features/{SimilarityFeaturesDefs.cpp => FeaturesDefs.cpp} (99%) rename src/libs/recommendation/impl/features/{SimilarityFeaturesDefs.hpp => FeaturesDefs.hpp} (95%) delete mode 100644 src/libs/recommendation/impl/features/SimilarityFeaturesScannerAddon.cpp delete mode 100644 src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.cpp delete mode 100644 src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.hpp rename src/libs/{recommendation/impl/features => scanner/impl}/AcousticBrainzUtils.cpp (84%) rename src/libs/{recommendation/impl/features => scanner/impl}/AcousticBrainzUtils.hpp (96%) diff --git a/src/libs/recommendation/CMakeLists.txt b/src/libs/recommendation/CMakeLists.txt index 26e29757..b7a52f12 100644 --- a/src/libs/recommendation/CMakeLists.txt +++ b/src/libs/recommendation/CMakeLists.txt @@ -1,8 +1,10 @@ add_library(lmsrecommendation SHARED impl/clusters/ClustersClassifier.cpp + impl/features/FeaturesClassifierCache.cpp + impl/features/FeaturesClassifier.cpp + impl/features/FeaturesDefs.cpp impl/Engine.cpp - impl/ClassifierCreator.cpp ) target_include_directories(lmsrecommendation INTERFACE diff --git a/src/libs/recommendation/impl/ClassifierCreator.cpp b/src/libs/recommendation/impl/ClassifierCreator.cpp deleted file mode 100644 index 90dd651b..00000000 --- a/src/libs/recommendation/impl/ClassifierCreator.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "recommendation/FeaturesClassifierCreator.hpp" -#include "recommendation/IClassifier.hpp" - -namespace Recommendation -{ - - std::unique_ptr createFeaturesClassifier() - { - return {}; - } - - -} - diff --git a/src/libs/recommendation/impl/Engine.cpp b/src/libs/recommendation/impl/Engine.cpp index 2a4aa772..4b4a6b67 100644 --- a/src/libs/recommendation/impl/Engine.cpp +++ b/src/libs/recommendation/impl/Engine.cpp @@ -24,6 +24,7 @@ #include "database/ScanSettings.hpp" #include "database/TrackList.hpp" +#include "utils/Exception.hpp" #include "utils/Logger.hpp" namespace Recommendation { @@ -56,6 +57,8 @@ Engine::stop() assert(_running); _running = false; + cancelPendingClassifiers(); + _ioService.stop(); } @@ -73,32 +76,18 @@ Engine::requestReload() std::vector Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) { - const std::unordered_set trackIds {[&]() -> std::unordered_set - { - auto transaction {session.createSharedTransaction()}; - - Database::TrackList::pointer trackList {Database::TrackList::getById(session, trackListId)}; - if (trackList) - { - const std::vector orderedTrackIds {trackList->getTrackIds()}; - return std::unordered_set {std::cbegin(orderedTrackIds), std::cend(orderedTrackIds)}; - } - - return {}; - }()}; - - if (trackIds.empty()) - return {}; - std::shared_lock lock {_classifiersMutex}; + std::vector res; + for (const auto& [priority, classifier] : _classifiers) { - if (std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return classifier->isTrackClassified(trackId); } )) - return classifier->getSimilarTracksFromTrackList(session, trackListId, maxCount); + res = classifier->getSimilarTracksFromTrackList(session, trackListId, maxCount); + if (!res.empty()) + break; } - return {}; + return res; } std::vector @@ -106,13 +95,16 @@ Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set< { std::shared_lock lock {_classifiersMutex}; + std::vector res; + for (const auto& [priority, classifier] : _classifiers) { - if (std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return classifier->isTrackClassified(trackId); } )) - return classifier->getSimilarTracks(dbSession, trackIds, maxCount); + res = classifier->getSimilarTracks(dbSession, trackIds, maxCount); + if (!res.empty()) + break; } - return {}; + return res; } std::vector @@ -120,13 +112,16 @@ Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releas { std::shared_lock lock {_classifiersMutex}; + std::vector res; + for (const auto& [priority, classifier] : _classifiers) { - if (classifier->isReleaseClassified(releaseId)) - return classifier->getSimilarReleases(dbSession, releaseId, maxCount); + res = classifier->getSimilarReleases(dbSession, releaseId, maxCount); + if (!res.empty()) + break; } - return {}; + return res; } std::vector @@ -134,13 +129,16 @@ Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistI { std::shared_lock lock {_classifiersMutex}; + std::vector res; + for (const auto& [priority, classifier] : _classifiers) { - if (classifier->isArtistClassified(artistId)) - return classifier->getSimilarArtists(dbSession, artistId, maxCount); + res = classifier->getSimilarArtists(dbSession, artistId, maxCount); + if (!res.empty()) + return res; } - return {}; + return res; } void @@ -159,14 +157,35 @@ Engine::reload() std::map> newClassifiers; + // TODO RAII this + auto addClassifier = [&](ClassifierPriority prio, std::unique_ptr classifier) + { + try + { + addPendingClassifier(*classifier.get()); + bool res {classifier->init(_dbSession)}; + removePendingClassifier(*classifier.get()); + + if (res) + newClassifiers.emplace(prio, std::move(classifier)); + + return res; + } + catch (LmsException& e) + { + removePendingClassifier(*classifier.get()); + throw; + } + }; + switch (engineType) { case ScanSettings::RecommendationEngineType::Features: -// newClassifiers.emplace_back(0, createFeaturesClassifier()); // higher priority -// [[fallthrough]]; + addClassifier(0, createFeaturesClassifier()); // higher priority + [[fallthrough]]; case ScanSettings::RecommendationEngineType::Clusters: - newClassifiers.emplace(1, createClustersClassifier(_dbSession)); // lower priority + addClassifier(1, createClustersClassifier()); // lower priority break; } @@ -180,19 +199,29 @@ Engine::reload() _sigReloaded.emit(); } - void -Engine::clearClassifiers() +Engine::cancelPendingClassifiers() { + std::unique_lock lock {_classifiersMutex}; + + for (IClassifier* classifier : _pendingClassifiers) + classifier->requestCancelInit(); } void -Engine::addClassifier(std::unique_ptr classifier, unsigned priority) +Engine::addPendingClassifier(IClassifier& classifier) { - std::unique_lock lock {_classifiersMutex}; + std::unique_lock lock {_classifiersMutex}; - _classifiers.emplace(priority, std::move(classifier)); + _pendingClassifiers.insert(&classifier); } +void +Engine::removePendingClassifier(IClassifier& classifier) +{ + std::unique_lock lock {_classifiersMutex}; + + _pendingClassifiers.erase(&classifier); +} } // ns Similarity diff --git a/src/libs/recommendation/impl/Engine.hpp b/src/libs/recommendation/impl/Engine.hpp index 9fde430e..17208b77 100644 --- a/src/libs/recommendation/impl/Engine.hpp +++ b/src/libs/recommendation/impl/Engine.hpp @@ -52,8 +52,9 @@ namespace Recommendation void reload(); - void clearClassifiers(); - void addClassifier(std::unique_ptr classifier, unsigned priority); + void cancelPendingClassifiers(); + void addPendingClassifier(IClassifier& classifier); + void removePendingClassifier(IClassifier& classifier); bool _running {}; Wt::WIOService _ioService; @@ -62,6 +63,7 @@ namespace Recommendation std::shared_mutex _classifiersMutex; std::map> _classifiers; + std::unordered_set _pendingClassifiers; }; } // ns Recommendation diff --git a/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp b/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp index f0aee487..b433d833 100644 --- a/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp +++ b/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp @@ -28,56 +28,11 @@ namespace Recommendation { -std::unique_ptr createClustersClassifier(Database::Session& session) +std::unique_ptr createClustersClassifier() { - return std::make_unique(session); + return std::make_unique(); } -ClusterClassifier::ClusterClassifier(Database::Session& session) -{ - classify(session); -} - -void -ClusterClassifier::classify(Database::Session& session) -{ - auto transaction {session.createSharedTransaction()}; - - { - std::vector trackIds {Database::Track::getAllIdsWithClusters(session)}; - _classifiedTracks = std::unordered_set(std::cbegin(trackIds), std::cend(trackIds)); - } - - { - std::vector releaseIds {Database::Release::getAllIdsWithClusters(session)}; - _classifiedReleases = std::unordered_set(std::cbegin(releaseIds), std::cend(releaseIds)); - } - - { - std::vector artistIds {Database::Artist::getAllIdsWithClusters(session)}; - _classifiedArtists = std::unordered_set(std::cbegin(artistIds), std::cend(artistIds)); - } -} - -bool -ClusterClassifier::isTrackClassified(Database::IdType trackId) const -{ - return _classifiedTracks.find(trackId) != std::cend(_classifiedTracks); -} - -bool -ClusterClassifier::isReleaseClassified(Database::IdType releaseId) const -{ - return _classifiedReleases.find(releaseId) != std::cend(_classifiedReleases); -} - -bool -ClusterClassifier::isArtistClassified(Database::IdType artistId) const -{ - return _classifiedArtists.find(artistId) != std::cend(_classifiedArtists); -} - - std::vector ClusterClassifier::getSimilarTracks(Database::Session& dbSession, const std::unordered_set& trackIds, std::size_t maxCount) const { diff --git a/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp b/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp index 2f44205e..469a6ee8 100644 --- a/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp +++ b/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp @@ -28,7 +28,7 @@ namespace Recommendation class ClusterClassifier : public IClassifier { public: - ClusterClassifier(Database::Session& session); + ClusterClassifier() = default; ClusterClassifier(const ClusterClassifier&) = delete; ClusterClassifier(ClusterClassifier&&) = delete; ClusterClassifier& operator=(const ClusterClassifier&) = delete; @@ -36,20 +36,16 @@ namespace Recommendation private: - bool isTrackClassified(Database::IdType trackId) const override; - bool isReleaseClassified(Database::IdType releaseId) const override; - bool isArtistClassified(Database::IdType artistId) const override; + std::string_view getName() const { return "Clusters"; } + + bool init(Database::Session&) override {return true;} + void requestCancelInit() override {} std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override; std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const override; std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override; std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override; - void classify(Database::Session& session); - - std::unordered_set _classifiedArtists; - std::unordered_set _classifiedReleases; - std::unordered_set _classifiedTracks; }; } // namespace Recommendation diff --git a/src/libs/recommendation/impl/features/FeaturesClassifier.cpp b/src/libs/recommendation/impl/features/FeaturesClassifier.cpp new file mode 100644 index 00000000..f4f4ad90 --- /dev/null +++ b/src/libs/recommendation/impl/features/FeaturesClassifier.cpp @@ -0,0 +1,445 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "FeaturesClassifier.hpp" + +#include + +#include "database/Artist.hpp" +#include "database/Release.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "database/TrackFeatures.hpp" +#include "database/TrackList.hpp" +#include "som/DataNormalizer.hpp" +#include "utils/Logger.hpp" +#include "utils/Random.hpp" + + +namespace Recommendation { + +std::unique_ptr createFeaturesClassifier() +{ + return std::make_unique(); +} + +const FeatureSettingsMap& +FeaturesClassifier::getDefaultTrainFeatureSettings() +{ + static const FeatureSettingsMap defaultTrainFeatureSettings + { + { "lowlevel.spectral_energyband_high.mean", {1}}, + { "lowlevel.spectral_rolloff.median", {1}}, + { "lowlevel.spectral_contrast_valleys.var", {1}}, + { "lowlevel.erbbands.mean", {1}}, + { "lowlevel.gfcc.mean", {1}}, + }; + + return defaultTrainFeatureSettings; +} + +static +std::optional +getTrackFeatureValues(FeaturesClassifier::FeaturesFetchFunc func, Database::IdType trackId, const std::unordered_set& featureNames) +{ + return func(trackId, featureNames); +} + +static +std::optional +getTrackFeatureValuesFromDb(Database::Session& session, Database::IdType trackId, const std::unordered_set& featureNames) +{ + auto func = [&](Database::IdType trackId, const std::unordered_set& featureNames) + { + std::optional res; + + auto transaction {session.createSharedTransaction()}; + + Database::Track::pointer track {Database::Track::getById(session, trackId)}; + if (!track) + return res; + + res = track->getTrackFeatures()->getFeatureValuesMap(featureNames); + if (res->empty()) + res.reset(); + + return res; + }; + + return getTrackFeatureValues(func, trackId, featureNames); +} + +static +std::optional +convertFeatureValuesMapToInputVector(const FeatureValuesMap& featureValuesMap, std::size_t nbDimensions) +{ + std::size_t i {}; + std::optional res {SOM::InputVector {nbDimensions}}; + for (const auto& [featureName, values] : featureValuesMap) + { + if (values.size() != getFeatureDef(featureName).nbDimensions) + { + LMS_LOG(RECOMMENDATION, WARNING) << "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << values.size(); + res.reset(); + break; + } + + for (double val : values) + (*res)[i++] = val; + } + + return res; +} + +static +SOM::InputVector +getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t nbDimensions) +{ + SOM::InputVector weights {nbDimensions}; + std::size_t index {}; + for (const auto& [featureName, featureSettings] : featureSettingsMap) + { + const std::size_t featureNbDimensions {getFeatureDef(featureName).nbDimensions}; + + for (std::size_t i {}; i < featureNbDimensions; ++i) + weights[index++] = (1. / featureNbDimensions * featureSettings.weight); + } + + assert(index == nbDimensions); + + return weights; +} + +bool +FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSettings& trainSettings) +{ + LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier..."; + + std::unordered_set featureNames; + std::transform(std::cbegin(trainSettings.featureSettingsMap), std::cend(trainSettings.featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)), + [](const auto& itFeatureSetting) { return itFeatureSetting.first; }); + + const std::size_t nbDimensions {std::accumulate(std::cbegin(featureNames), std::cend(featureNames), std::size_t {0}, + [](std::size_t sum, const FeatureName& featureName) { return sum + getFeatureDef(featureName).nbDimensions; })}; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Features dimension = " << nbDimensions; + + std::vector trackIds; + { + auto transaction {session.createSharedTransaction()}; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Getting Tracks with features..."; + trackIds = Database::Track::getAllIdsWithFeatures(session); + LMS_LOG(RECOMMENDATION, DEBUG) << "Getting Tracks with features DONE (found " << trackIds.size() << " tracks)"; + } + + std::vector samples; + std::vector samplesTrackIds; + + samples.reserve(trackIds.size()); + samplesTrackIds.reserve(trackIds.size()); + + LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features..."; + for (Database::IdType trackId : trackIds) + { + if (_initCancelled) + return false; + + std::optional featureValuesMap; + + if (_featuresFetchFunc) + featureValuesMap = getTrackFeatureValues(_featuresFetchFunc, trackId, featureNames); + else + featureValuesMap = getTrackFeatureValuesFromDb(session, trackId, featureNames); + + if (!featureValuesMap) + continue; + + std::optional inputVector {convertFeatureValuesMapToInputVector(*featureValuesMap, nbDimensions)}; + if (!inputVector) + continue; + + samples.emplace_back(std::move(*inputVector)); + samplesTrackIds.emplace_back(trackId); + } + LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features DONE"; + + if (samples.empty()) + { + LMS_LOG(RECOMMENDATION, INFO) << "Nothing to classify!"; + return false; + } + + LMS_LOG(RECOMMENDATION, DEBUG) << "Normalizing data..."; + SOM::DataNormalizer dataNormalizer {nbDimensions}; + + dataNormalizer.computeNormalizationFactors(samples); + for (auto& sample : samples) + dataNormalizer.normalizeData(sample); + + const SOM::Coordinate size {static_cast(std::sqrt(samples.size() / trainSettings.sampleCountPerNeuron))}; + LMS_LOG(RECOMMENDATION, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network"; + + SOM::Network network {size, size, nbDimensions}; + + SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)}; + network.setDataWeights(weights); + + auto progressIndicator{[](const auto& iter) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount; + }}; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Training network..."; + network.train(samples, trainSettings.iterationCount, progressIndicator); + LMS_LOG(RECOMMENDATION, DEBUG) << "Training network DONE"; + + if (_initCancelled) + return false; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks..."; + ObjectPositions trackPositions; + for (std::size_t i {}; i < samples.size(); ++i) + { + if (_initCancelled) + return false; + + const SOM::Position position {network.getClosestRefVectorPosition(samples[i])}; + + trackPositions[samplesTrackIds[i]].insert(position); + } + + LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks DONE"; + + return init(session, std::move(network), std::move(trackPositions)); +} + +bool +FeaturesClassifier::initFromCache(Database::Session& session, const FeaturesClassifierCache& cache) +{ + LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier from cache..."; + + return init(session, std::move(cache._network), cache._trackPositions); +} + +std::vector +FeaturesClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) const +{ + const std::unordered_set trackIds {[&]() -> std::unordered_set + { + auto transaction {session.createSharedTransaction()}; + + const Database::TrackList::pointer trackList {Database::TrackList::getById(session, trackListId)}; + if (trackList) + { + const std::vector orderedTrackIds {trackList->getTrackIds()}; + return std::unordered_set {std::cbegin(orderedTrackIds), std::cend(orderedTrackIds)}; + } + + return {}; + }()}; + + return getSimilarTracks(session, trackIds, maxCount); +} + +std::vector +FeaturesClassifier::getSimilarTracks(Database::Session&, const std::unordered_set& tracksIds, std::size_t maxCount) const +{ + return getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount); +} + +std::vector +FeaturesClassifier::getSimilarReleases(Database::Session&, Database::IdType releaseId, std::size_t maxCount) const +{ + return getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount); +} + +std::vector +FeaturesClassifier::getSimilarArtists(Database::Session&, Database::IdType artistId, std::size_t maxCount) const +{ + return getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount); +} + +FeaturesClassifierCache +FeaturesClassifier::toCache() const +{ + return FeaturesClassifierCache {*_network, _trackPositions}; +} + +bool +FeaturesClassifier::init(Database::Session& session) +{ + std::optional cache {FeaturesClassifierCache::read()}; + if (cache) + return initFromCache(session, *cache); + + TrainSettings trainSettings; + trainSettings.featureSettingsMap = getDefaultTrainFeatureSettings(); + + bool res {initFromTraining(session, trainSettings)}; + if (res) + toCache().write(); + + return res; +} + +void +FeaturesClassifier::requestCancelInit() +{ + LMS_LOG(RECOMMENDATION, DEBUG) << "Requesting init cancellation"; + _initCancelled = true; +} + +bool +FeaturesClassifier::init(Database::Session& session, + SOM::Network network, + const ObjectPositions& tracksPosition) +{ + _networkRefVectorsDistanceMedian = network.computeRefVectorsDistanceMedian(); + LMS_LOG(RECOMMENDATION, DEBUG) << "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian; + + const SOM::Coordinate width {network.getWidth()}; + const SOM::Coordinate height {network.getHeight()}; + + _artistsMap = MatrixOfObjects {width, height}; + _releasesMap = MatrixOfObjects {width, height}; + _tracksMap = MatrixOfObjects {width, height}; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Constructing maps..."; + + for (auto itTrackCoord : tracksPosition) + { + if (_initCancelled) + return false; + + auto transaction {session.createSharedTransaction()}; + + Database::IdType trackId {itTrackCoord.first}; + const std::unordered_set& positionSet {itTrackCoord.second}; + + const Database::Track::pointer track {Database::Track::getById(session, trackId)}; + if (!track) + continue; + + for (const SOM::Position& position : positionSet) + { + _tracksMap[position].insert(trackId); + _trackPositions[trackId].insert(position); + + if (track->getRelease()) + { + _releasePositions[track->getRelease().id()].insert(position); + _releasesMap[position].insert(track->getRelease().id()); + } + for (const auto& artist : track->getArtists()) + { + _artistPositions[artist.id()].insert(position); + _artistsMap[position].insert(artist.id()); + } + } + } + + _network = std::make_unique(std::move(network)); + + LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully initialized!"; + + return true; +} + +std::unordered_set +FeaturesClassifier::getMatchingRefVectorsPosition(const std::unordered_set& ids, const ObjectPositions& objectPositions) +{ + std::unordered_set res; + + if (ids.empty()) + return res; + + for (auto id : ids) + { + auto it = objectPositions.find(id); + if (it == objectPositions.end()) + continue; + + for (const auto& position : it->second) + res.insert(position); + } + + return res; +} + +std::unordered_set +FeaturesClassifier::getObjectsIds(const std::unordered_set& positionSet, const MatrixOfObjects& objectsMap) +{ + std::unordered_set res; + + for (const auto& position : positionSet) + { + for (auto id : objectsMap.get(position)) + res.insert(id); + } + + return res; +} + +std::vector +FeaturesClassifier::getSimilarObjects(const std::unordered_set& ids, + const MatrixOfObjects& objectsMap, + const ObjectPositions& objectPosition, + std::size_t maxCount) const +{ + std::vector res; + + std::unordered_set searchedRefVectorsPosition {getMatchingRefVectorsPosition(ids, objectPosition)}; + if (searchedRefVectorsPosition.empty()) + return res; + + while (1) + { + std::unordered_set closestObjectIds {getObjectsIds(searchedRefVectorsPosition, objectsMap)}; + + // Remove objects that are already in input or already reported + for (auto id : ids) + closestObjectIds.erase(id); + + { + std::vector objectIdsToAdd {std::cbegin(closestObjectIds), std::cend(closestObjectIds)}; + Random::shuffleContainer(objectIdsToAdd ); + std::copy(std::cbegin(objectIdsToAdd), std::cend(objectIdsToAdd), std::back_inserter(res)); + } + + if (res.size() > maxCount) + res.resize(maxCount); + + if (res.size() == maxCount) + break; + + // If there is not enough objects, try again with closest neighbour until there is too much distance + const std::optional closestRefVectorPosition {_network->getClosestRefVectorPosition(searchedRefVectorsPosition, _networkRefVectorsDistanceMedian * 0.75)}; + if (!closestRefVectorPosition) + break; + + searchedRefVectorsPosition.insert(closestRefVectorPosition.value()); + } + + return res; +} + + + +} // ns Recommendation diff --git a/src/libs/recommendation/impl/features/FeaturesClassifier.hpp b/src/libs/recommendation/impl/features/FeaturesClassifier.hpp new file mode 100644 index 00000000..d4856a96 --- /dev/null +++ b/src/libs/recommendation/impl/features/FeaturesClassifier.hpp @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include "recommendation/IClassifier.hpp" +#include "som/DataNormalizer.hpp" +#include "som/Network.hpp" +#include "FeaturesClassifierCache.hpp" +#include "FeaturesDefs.hpp" + +namespace Database +{ + class Session; +} + +namespace Recommendation { + +using FeatureWeight = double; + +class FeaturesClassifier : public IClassifier +{ + public: + FeaturesClassifier() = default; + FeaturesClassifier(const FeaturesClassifier&) = delete; + FeaturesClassifier(FeaturesClassifier&&) = delete; + FeaturesClassifier& operator=(const FeaturesClassifier&) = delete; + FeaturesClassifier& operator=(FeaturesClassifier&&) = delete; + + using FeaturesFetchFunc = std::function>>(Database::IdType /*trackId*/, const std::unordered_set& /*features*/)>; + // Default is to retrieve the features from the database (may be slow). + // Use this only if you want to train different searchers with some cached data + static void setFeaturesFetchFunc(FeaturesFetchFunc func) { _featuresFetchFunc = func; } + + static const FeatureSettingsMap& getDefaultTrainFeatureSettings(); + private: + + std::string_view getName() const { return "Features"; } + + bool init(Database::Session& session) override; + void requestCancelInit() override; + + std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override; + std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const override; + std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const; + std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const; + + bool initFromCache(Database::Session& session, const FeaturesClassifierCache& cache); + + // Use training (may be very slow) + struct TrainSettings + { + std::size_t iterationCount {10}; + float sampleCountPerNeuron {4}; + FeatureSettingsMap featureSettingsMap; + }; + bool initFromTraining(Database::Session& session, const TrainSettings& trainSettings); + + using ObjectPositions = std::unordered_map>; + using MatrixOfObjects = SOM::Matrix>; + + bool init(Database::Session& session, + SOM::Network network, + const ObjectPositions& tracksPosition); + + FeaturesClassifierCache toCache() const; + + static std::unordered_set getMatchingRefVectorsPosition(const std::unordered_set& ids, const ObjectPositions& objectPositions); + static std::unordered_set getObjectsIds(const std::unordered_set& positionSet, const MatrixOfObjects& objectsMap); + + std::vector getSimilarObjects(const std::unordered_set& ids, + const SOM::Matrix>& objectsMap, + const ObjectPositions& objectPosition, + std::size_t maxCount) const; + + bool _initCancelled {}; + std::unique_ptr _network; + double _networkRefVectorsDistanceMedian {}; + + MatrixOfObjects _artistsMap; + ObjectPositions _artistPositions; + + MatrixOfObjects _releasesMap; + ObjectPositions _releasePositions; + + MatrixOfObjects _tracksMap; + ObjectPositions _trackPositions; + + static inline FeaturesFetchFunc _featuresFetchFunc; +}; + +} // ns Recommendation diff --git a/src/libs/recommendation/impl/features/SimilarityFeaturesCache.cpp b/src/libs/recommendation/impl/features/FeaturesClassifierCache.cpp similarity index 76% rename from src/libs/recommendation/impl/features/SimilarityFeaturesCache.cpp rename to src/libs/recommendation/impl/features/FeaturesClassifierCache.cpp index 235ed331..fc80098d 100644 --- a/src/libs/recommendation/impl/features/SimilarityFeaturesCache.cpp +++ b/src/libs/recommendation/impl/features/FeaturesClassifierCache.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "SimilarityFeaturesCache.hpp" +#include "FeaturesClassifierCache.hpp" #include #include @@ -26,7 +26,7 @@ #include "utils/Logger.hpp" #include "utils/Service.hpp" -namespace Similarity { +namespace Recommendation { static @@ -38,7 +38,7 @@ std::filesystem::path getCacheDirectory() static std::filesystem::path getCacheNetworkFilePath() { return getCacheDirectory() / "network"; -}; +} static std::filesystem::path getCacheTrackPositionsFilePath() { @@ -79,26 +79,25 @@ networkToCacheFile(const SOM::Network& network, std::filesystem::path path) boost::property_tree::write_xml(path.string(), root); - LMS_LOG(SIMILARITY, DEBUG) << "Created network cache"; + LMS_LOG(RECOMMENDATION, DEBUG) << "Created network cache"; return true; } catch (boost::property_tree::ptree_error& error) { - LMS_LOG(SIMILARITY, ERROR) << "Cannot create network cache: " << error.what(); + LMS_LOG(RECOMMENDATION, ERROR) << "Cannot create network cache: " << error.what(); return false; } } -static std::optional -createNetworkFromCacheFile(const std::filesystem::path& path) +FeaturesClassifierCache::createNetworkFromCacheFile(const std::filesystem::path& path) { if (!std::filesystem::exists(path)) return std::nullopt; try { - LMS_LOG(SIMILARITY, INFO) << "Reading network from cache..."; + LMS_LOG(RECOMMENDATION, INFO) << "Reading network from cache..."; boost::property_tree::ptree root; @@ -132,20 +131,19 @@ createNetworkFromCacheFile(const std::filesystem::path& path) res.setRefVector({x, y}, refVector); } - LMS_LOG(SIMILARITY, INFO) << "Successfully read network from cache"; + LMS_LOG(RECOMMENDATION, INFO) << "Successfully read network from cache"; return res; } catch (boost::property_tree::ptree_error& error) { - LMS_LOG(SIMILARITY, ERROR) << "Cannot read network cache: " << error.what(); + LMS_LOG(RECOMMENDATION, ERROR) << "Cannot read network cache: " << error.what(); return std::nullopt; } } -static bool -objectPositionToCacheFile(const std::map>& objectsPosition, std::filesystem::path path) +FeaturesClassifierCache::objectPositionToCacheFile(const ObjectPositions& objectsPosition, const std::filesystem::path& path) { try { @@ -174,24 +172,23 @@ objectPositionToCacheFile(const std::map>> -createObjectPositionsFromCacheFile(std::filesystem::path path) +std::optional +FeaturesClassifierCache::createObjectPositionsFromCacheFile(const std::filesystem::path& path) { try { - LMS_LOG(SIMILARITY, INFO) << "Reading object position from cache..."; + LMS_LOG(RECOMMENDATION, INFO) << "Reading object position from cache..."; boost::property_tree::ptree root; boost::property_tree::read_xml(path.string(), root); - std::map> res; + ObjectPositions res; for (const auto& object : root.get_child("objects")) { @@ -205,42 +202,40 @@ createObjectPositionsFromCacheFile(std::filesystem::path path) } } - LMS_LOG(SIMILARITY, INFO) << "Successfully read object position from cache"; + LMS_LOG(RECOMMENDATION, INFO) << "Successfully read object position from cache"; return res; } catch (boost::property_tree::ptree_error& error) { - LMS_LOG(SIMILARITY, ERROR) << "Cannot create object position from cache file: " << error.what(); + LMS_LOG(RECOMMENDATION, ERROR) << "Cannot create object position from cache file: " << error.what(); return std::nullopt; } } void -FeaturesCache::invalidate() +FeaturesClassifierCache::invalidate() { std::filesystem::remove(getCacheNetworkFilePath()); std::filesystem::remove(getCacheTrackPositionsFilePath()); } -std::optional -FeaturesCache::read() +std::optional +FeaturesClassifierCache::read() { - std::optional res; - auto network{createNetworkFromCacheFile(getCacheNetworkFilePath())}; if (!network) - return res; + return std::nullopt; auto trackPositions{createObjectPositionsFromCacheFile(getCacheTrackPositionsFilePath())}; if (!trackPositions) - return res; + return std::nullopt; - return FeaturesCache{std::move(*network), std::move(*trackPositions)}; + return FeaturesClassifierCache {std::move(*network), std::move(*trackPositions)}; } void -FeaturesCache::write() +FeaturesClassifierCache::write() const { std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir") / "cache" / "features"); @@ -251,11 +246,10 @@ FeaturesCache::write() } } -FeaturesCache::FeaturesCache(SOM::Network network, ObjectPositions trackPositions) +FeaturesClassifierCache::FeaturesClassifierCache(SOM::Network network, ObjectPositions trackPositions) : _network {std::move(network)}, _trackPositions {std::move(trackPositions)} { } - -} // namespace Similarity +} // namespace Recommendation diff --git a/src/libs/recommendation/impl/features/SimilarityFeaturesCache.hpp b/src/libs/recommendation/impl/features/FeaturesClassifierCache.hpp similarity index 53% rename from src/libs/recommendation/impl/features/SimilarityFeaturesCache.hpp rename to src/libs/recommendation/impl/features/FeaturesClassifierCache.hpp index a12334f3..25ddaeff 100644 --- a/src/libs/recommendation/impl/features/SimilarityFeaturesCache.hpp +++ b/src/libs/recommendation/impl/features/FeaturesClassifierCache.hpp @@ -19,33 +19,36 @@ #pragma once -#include -#include -#include +#include +#include +#include #include "database/Types.hpp" #include "som/Network.hpp" -namespace Similarity { +namespace Recommendation { -class FeaturesCache +class FeaturesClassifierCache { public: - static void invalidate(); - static std::optional read(); - void write(); + static std::optional read(); + void write() const; private: - using ObjectPositions = std::map>; + using ObjectPositions = std::unordered_map>; - FeaturesCache(SOM::Network network, ObjectPositions trackPositions); + FeaturesClassifierCache(SOM::Network network, ObjectPositions trackPositions); - friend class FeaturesSearcher; + static std::optional createNetworkFromCacheFile(const std::filesystem::path& path); + static std::optional createObjectPositionsFromCacheFile(const std::filesystem::path& path); + static bool objectPositionToCacheFile(const ObjectPositions& objectsPosition, const std::filesystem::path& path); + + friend class FeaturesClassifier; SOM::Network _network; ObjectPositions _trackPositions; }; -} // namespace Similarity +} // namespace Recommendation diff --git a/src/libs/recommendation/impl/features/SimilarityFeaturesDefs.cpp b/src/libs/recommendation/impl/features/FeaturesDefs.cpp similarity index 99% rename from src/libs/recommendation/impl/features/SimilarityFeaturesDefs.cpp rename to src/libs/recommendation/impl/features/FeaturesDefs.cpp index a0aecc4d..b3de7dcb 100644 --- a/src/libs/recommendation/impl/features/SimilarityFeaturesDefs.cpp +++ b/src/libs/recommendation/impl/features/FeaturesDefs.cpp @@ -17,14 +17,14 @@ * along with LMS. If not, see . */ -#include "SimilarityFeaturesDefs.hpp" +#include "FeaturesDefs.hpp" #include #include #include "utils/Exception.hpp" -namespace Similarity { +namespace Recommendation { static const std::unordered_map featureDefinitions { @@ -398,5 +398,5 @@ getFeatureNames() return res; } -} // namespace Similarity +} // namespace Recommendation diff --git a/src/libs/recommendation/impl/features/SimilarityFeaturesDefs.hpp b/src/libs/recommendation/impl/features/FeaturesDefs.hpp similarity index 95% rename from src/libs/recommendation/impl/features/SimilarityFeaturesDefs.hpp rename to src/libs/recommendation/impl/features/FeaturesDefs.hpp index 58eed273..d6e1d759 100644 --- a/src/libs/recommendation/impl/features/SimilarityFeaturesDefs.hpp +++ b/src/libs/recommendation/impl/features/FeaturesDefs.hpp @@ -24,7 +24,7 @@ #include #include -namespace Similarity { +namespace Recommendation { using FeatureName = std::string; using FeatureNames = std::unordered_set; @@ -46,4 +46,4 @@ struct FeatureSettings }; using FeatureSettingsMap = std::unordered_map; -} // namespace Similarity +} // namespace Recommendation diff --git a/src/libs/recommendation/impl/features/SimilarityFeaturesScannerAddon.cpp b/src/libs/recommendation/impl/features/SimilarityFeaturesScannerAddon.cpp deleted file mode 100644 index 620cedc8..00000000 --- a/src/libs/recommendation/impl/features/SimilarityFeaturesScannerAddon.cpp +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright (C) 2019 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "similarity/features/SimilarityFeaturesScannerAddon.hpp" - -#include "AcousticBrainzUtils.hpp" -#include "database/ScanSettings.hpp" -#include "database/Track.hpp" -#include "database/TrackFeatures.hpp" -#include "utils/Logger.hpp" - -#include "SimilarityFeaturesCache.hpp" - -namespace Similarity { - -static -bool -hasAtLeastOneTrackWithFeatures(Database::Session& session) -{ - auto transaction {session.createSharedTransaction()}; - return !Database::Track::getAllIdsWithFeatures(session, 1).empty(); -} - -struct TrackInfo -{ - Database::IdType id; - std::optional mbid; -}; - -static -std::vector -getTracksWithMBIDAndMissingFeatures(Database::Session& dbSession) -{ - std::vector res; - - auto transaction {dbSession.createSharedTransaction()}; - - auto tracks {Database::Track::getAllWithMBIDAndMissingFeatures(dbSession)}; - for (const Database::Track::pointer& track : tracks) - res.push_back({track.id(), track->getMBID()}); - - return res; -} - -FeaturesScannerAddon::FeaturesScannerAddon(Database::Db& db) -: _dbSession {db} -{ - std::optional cache {Similarity::FeaturesCache::read()}; - if (cache) - { - auto searcher {std::make_shared(_dbSession, *cache, [&]() { return _stopRequested; })}; - if (searcher->isValid()) - std::atomic_store(&_searcher, searcher); - } -} - -std::shared_ptr -FeaturesScannerAddon::getSearcher() -{ - return std::atomic_load(&_searcher); -} - -void -FeaturesScannerAddon::requestStop() -{ - _stopRequested = true; -} - -void -FeaturesScannerAddon::trackUpdated(Database::IdType trackId) -{ - auto uniqueTransaction {_dbSession.createUniqueTransaction()}; - - auto track {Database::Track::getById(_dbSession, trackId)}; - if (!track) - return; - - track.modify()->setFeatures({}); -} - -void -FeaturesScannerAddon::preScanComplete() -{ - { - auto transaction {_dbSession.createSharedTransaction()}; - - if (Database::ScanSettings::get(_dbSession)->getSimilarityEngineType() != Database::ScanSettings::SimilarityEngineType::Features) - { - LMS_LOG(DBUPDATER, INFO) << "Do not fetch features since the engine type does not make use of them"; - return; - } - } - - LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features..."; - const std::vector tracksInfo {getTracksWithMBIDAndMissingFeatures(_dbSession)}; - LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features DONE (found " << tracksInfo.size() << ")"; - - if (!tracksInfo.empty()) - Similarity::FeaturesCache::invalidate(); - - for (const TrackInfo& trackInfo : tracksInfo) - { - if (_stopRequested) - return; - - if (trackInfo.mbid) - fetchFeatures(trackInfo.id, *trackInfo.mbid); - } - - updateSearcher(); -} - -void -FeaturesScannerAddon::updateSearcher() -{ - LMS_LOG(SIMILARITY, INFO) << "Updating searcher..."; - - if (!hasAtLeastOneTrackWithFeatures(_dbSession)) - { - LMS_LOG(DBUPDATER, INFO) << "No track found with features!"; - std::atomic_store(&_searcher, std::shared_ptr{}); - return; - } - - Similarity::FeaturesSearcher::TrainSettings trainSettings; - trainSettings.featureSettingsMap = FeaturesSearcher::getDefaultTrainFeatureSettings(); - - auto searcher {std::make_shared(_dbSession, trainSettings, [&]() { return _stopRequested; })}; - if (searcher->isValid()) - { - std::atomic_store(&_searcher, searcher); - FeaturesCache cache{searcher->toCache()}; - cache.write(); - - LMS_LOG(DBUPDATER, INFO) << "New features similarity searcher instanciated"; - } - else - { - LMS_LOG(DBUPDATER, ERROR) << "Cannot set up a valid features similarity searcher!"; - std::atomic_store(&_searcher, std::shared_ptr{}); - } -} - -bool -FeaturesScannerAddon::fetchFeatures(Database::IdType trackId, const UUID& MBID) -{ - std::map features; - - LMS_LOG(DBUPDATER, DEBUG) << "Fetching low level features for track '" << MBID.getAsString() << "'"; - const std::string data {AcousticBrainz::extractLowLevelFeatures(MBID)}; - if (data.empty()) - { - LMS_LOG(DBUPDATER, ERROR) << "Track " << trackId << ", MBID = '" << MBID.getAsString() << "': cannot extract features using AcousticBrainz"; - return false; - } - - { - auto uniqueTransaction {_dbSession.createUniqueTransaction()}; - - Wt::Dbo::ptr track {Database::Track::getById(_dbSession, trackId)}; - if (!track) - return false; - - Database::TrackFeatures::create(_dbSession, track, data); - } - - return true; -} - -} // namespace Similarity - diff --git a/src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.cpp b/src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.cpp deleted file mode 100644 index a1ab5987..00000000 --- a/src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.cpp +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "SimilarityFeaturesSearcher.hpp" - -#include -#include - -#include "database/Artist.hpp" -#include "database/Release.hpp" -#include "database/Session.hpp" -#include "database/Track.hpp" -#include "database/TrackFeatures.hpp" -#include "som/DataNormalizer.hpp" -#include "utils/Logger.hpp" - - -namespace Similarity { - -const FeatureSettingsMap& -FeaturesSearcher::getDefaultTrainFeatureSettings() -{ - static FeatureSettingsMap defaultTrainFeatureSettings - { - { "lowlevel.spectral_energyband_high.mean", {1}}, - { "lowlevel.spectral_rolloff.median", {1}}, - { "lowlevel.spectral_contrast_valleys.var", {1}}, - { "lowlevel.erbbands.mean", {1}}, - { "lowlevel.gfcc.mean", {1}}, - }; - - return defaultTrainFeatureSettings; -} - -static -std::optional -getTrackFeatureValues(FeaturesSearcher::FeaturesFetchFunc func, Database::IdType trackId, const std::unordered_set& featureNames) -{ - return func(trackId, featureNames); -} - -static -std::optional -getTrackFeatureValuesFromDb(Database::Session& session, Database::IdType trackId, const std::unordered_set& featureNames) -{ - auto func = [&](Database::IdType trackId, const std::unordered_set& featureNames) - { - std::optional res; - - auto transaction {session.createSharedTransaction()}; - - Database::Track::pointer track {Database::Track::getById(session, trackId)}; - if (!track) - return res; - - res = track->getTrackFeatures()->getFeatureValuesMap(featureNames); - if (res->empty()) - res.reset(); - - return res; - }; - - return getTrackFeatureValues(func, trackId, featureNames); -} - -static -std::optional -convertFeatureValuesMapToInputVector(const FeatureValuesMap& featureValuesMap, std::size_t nbDimensions) -{ - std::size_t i {}; - std::optional res {SOM::InputVector {nbDimensions}}; - for (const auto& [featureName, values] : featureValuesMap) - { - if (values.size() != getFeatureDef(featureName).nbDimensions) - { - LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << values.size(); - res.reset(); - break; - } - - for (double val : values) - (*res)[i++] = val; - } - - return res; -} - -static -SOM::InputVector -getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t nbDimensions) -{ - SOM::InputVector weights {nbDimensions}; - std::size_t index {}; - for (const auto& [featureName, featureSettings] : featureSettingsMap) - { - const std::size_t featureNbDimensions {getFeatureDef(featureName).nbDimensions}; - - for (std::size_t i {}; i < featureNbDimensions; ++i) - weights[index++] = (1. / featureNbDimensions * featureSettings.weight); - } - - assert(index == nbDimensions); - - return weights; -} - -FeaturesSearcher::FeaturesSearcher(Database::Session& session, - const TrainSettings& trainSettings, - StopRequestedFunction stopRequested) -{ - LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher..."; - - std::unordered_set featureNames; - std::transform(std::cbegin(trainSettings.featureSettingsMap), std::cend(trainSettings.featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)), - [](const auto& itFeatureSetting) { return itFeatureSetting.first; }); - - const std::size_t nbDimensions {std::accumulate(std::cbegin(featureNames), std::cend(featureNames), std::size_t {0}, - [](std::size_t sum, const FeatureName& featureName) { return sum + getFeatureDef(featureName).nbDimensions; })}; - - LMS_LOG(SIMILARITY, DEBUG) << "Features dimension = " << nbDimensions; - - std::vector trackIds; - { - auto transaction {session.createSharedTransaction()}; - - LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features..."; - trackIds = Database::Track::getAllIdsWithFeatures(session); - LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features DONE (found " << trackIds.size() << " tracks)"; - } - - std::vector samples; - std::vector samplesTrackIds; - - samples.reserve(trackIds.size()); - samplesTrackIds.reserve(trackIds.size()); - - LMS_LOG(SIMILARITY, DEBUG) << "Extracting features..."; - for (Database::IdType trackId : trackIds) - { - if (stopRequested && stopRequested()) - return; - - std::optional featureValuesMap; - - if (_featuresFetchFunc) - featureValuesMap = getTrackFeatureValues(_featuresFetchFunc, trackId, featureNames); - else - featureValuesMap = getTrackFeatureValuesFromDb(session, trackId, featureNames); - - if (!featureValuesMap) - continue; - - std::optional inputVector {convertFeatureValuesMapToInputVector(*featureValuesMap, nbDimensions)}; - if (!inputVector) - continue; - - samples.emplace_back(std::move(*inputVector)); - samplesTrackIds.emplace_back(trackId); - } - LMS_LOG(SIMILARITY, DEBUG) << "Extracting features DONE"; - - if (samples.empty()) - { - LMS_LOG(SIMILARITY, INFO) << "Nothing to classify!"; - return; - } - - LMS_LOG(SIMILARITY, DEBUG) << "Normalizing data..."; - SOM::DataNormalizer dataNormalizer {nbDimensions}; - - dataNormalizer.computeNormalizationFactors(samples); - for (auto& sample : samples) - dataNormalizer.normalizeData(sample); - - SOM::Coordinate size {static_cast(std::sqrt(samples.size() / trainSettings.sampleCountPerNeuron))}; - LMS_LOG(SIMILARITY, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network"; - - SOM::Network network {size, size, nbDimensions}; - - SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)}; - network.setDataWeights(weights); - - auto progressIndicator{[](const auto& iter) - { - LMS_LOG(SIMILARITY, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount; - }}; - - LMS_LOG(SIMILARITY, DEBUG) << "Training network..."; - network.train(samples, trainSettings.iterationCount, progressIndicator, stopRequested); - LMS_LOG(SIMILARITY, DEBUG) << "Training network DONE"; - - if (stopRequested && stopRequested()) - return; - - LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks..."; - std::map> trackPositions; - for (std::size_t i {}; i < samples.size(); ++i) - { - if (stopRequested && stopRequested()) - return; - - const SOM::Position position {network.getClosestRefVectorPosition(samples[i])}; - - trackPositions[samplesTrackIds[i]].insert(position); - } - - LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE"; - - init(session, std::move(network), std::move(trackPositions), stopRequested); - - LMS_LOG(SIMILARITY, INFO) << "Successfully constructed features searcher"; -} - -FeaturesSearcher::FeaturesSearcher(Database::Session& session, FeaturesCache cache, StopRequestedFunction stopRequested) -{ - LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher from cache..."; - - init(session, std::move(cache._network), std::move(cache._trackPositions), stopRequested); - - LMS_LOG(SIMILARITY, INFO) << "Successfully constructed features searcher from cache"; -} - -bool -FeaturesSearcher::isValid() const -{ - return _network.get() != nullptr; -} - -bool -FeaturesSearcher::isTrackClassified(Database::IdType trackId) const -{ - return (_trackPositions.find(trackId) != _trackPositions.end()); -} - -bool -FeaturesSearcher::isReleaseClassified(Database::IdType releaseId) const -{ - return (_releasePositions.find(releaseId) != _releasePositions.end()); -} - -bool -FeaturesSearcher::isArtistClassified(Database::IdType artistId) const -{ - return (_artistPositions.find(artistId) != _artistPositions.end()); -} - -std::vector -FeaturesSearcher::getSimilarTracks(const std::set& tracksIds, std::size_t maxCount) const -{ - return getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount); -} - -std::vector -FeaturesSearcher::getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const -{ - return getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount); -} - -std::vector -FeaturesSearcher::getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const -{ - return getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount); -} - -void -FeaturesSearcher::dump(Database::Session& session, std::ostream& os) const -{ - if (!isValid()) - { - os << "Invalid searcher" << std::endl; - return; - } - - os << "Number of tracks classified: " << _trackPositions.size() << std::endl; - os << "Network size: " << _network->getWidth() << " * " << _network->getHeight() << std::endl; - os << "Ref vectors median distance = " << _networkRefVectorsDistanceMedian << std::endl; - - auto transaction {session.createSharedTransaction()}; - - for (SOM::Coordinate y {}; y < _network->getHeight(); ++y) - { - for (SOM::Coordinate x {}; x < _network->getWidth(); ++x) - { - const auto& trackIds {_tracksMap[{x, y}]}; - - os << "{" << x << ", " << y << "}"; - - if (y > 0) - os << " - {" << x << ", " << y - 1 << "}: " << _network->getRefVectorsDistance({x, y}, {x, y - 1}); - if (x > 0) - os << " - {" << x - 1 << ", " << y << "}: " << _network->getRefVectorsDistance({x, y}, {x - 1, y}); - if (y != _network->getHeight() - 1) - os << " - {" << x << ", " << y + 1 << "}: " << _network->getRefVectorsDistance({x, y}, {x, y + 1}); - if (x != _network->getWidth() - 1) - os << " - {" << x + 1 << ", " << y << "}: " << _network->getRefVectorsDistance({x, y}, {x + 1, y}); - os << std::endl; - - for (Database::IdType trackId : trackIds) - { - auto track {Database::Track::getById(session, trackId)}; - if (!track) - continue; - - os << "\t"; - for (auto artist : track->getArtists()) - os << artist->getName() << " - "; - if (track->getRelease()) - os << track->getRelease()->getName() << " - "; - os << track->getName() << std::endl; - } - - } - os << std::endl; - } -} - -FeaturesCache -FeaturesSearcher::toCache() const -{ - return FeaturesCache{*_network, _trackPositions}; -} - -void -FeaturesSearcher::init(Database::Session& session, - SOM::Network network, - std::map> tracksPosition, - std::function stopRequested) -{ - _networkRefVectorsDistanceMedian = network.computeRefVectorsDistanceMedian(); - LMS_LOG(SIMILARITY, DEBUG) << "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian; - - SOM::Coordinate width {network.getWidth()}; - SOM::Coordinate height {network.getHeight()}; - - _artistsMap = SOM::Matrix>{width, height}; - _releasesMap = SOM::Matrix>{width, height}; - _tracksMap = SOM::Matrix>{width, height}; - - LMS_LOG(SIMILARITY, DEBUG) << "Constructing maps..."; - - for (auto itTrackCoord : tracksPosition) - { - if (stopRequested && stopRequested()) - return; - - auto transaction {session.createSharedTransaction()}; - - Database::IdType trackId {itTrackCoord.first}; - const std::set& positionSet {itTrackCoord.second}; - - const Database::Track::pointer track {Database::Track::getById(session, trackId)}; - if (!track) - continue; - - for (const SOM::Position& position : positionSet) - { - _tracksMap[position].insert(trackId); - _trackPositions[trackId].insert(position); - - if (track->getRelease()) - { - _releasePositions[track->getRelease().id()].insert(position); - _releasesMap[position].insert(track->getRelease().id()); - } - for (const auto& artist : track->getArtists()) - { - _artistPositions[artist.id()].insert(position); - _artistsMap[position].insert(artist.id()); - } - } - } - - _network = std::make_unique(std::move(network)); - - LMS_LOG(SIMILARITY, DEBUG) << "Constructing maps... DONE"; - -} - -static -std::set -getMatchingRefVectorsPosition(const std::set& ids, const std::map>& objectPosition) -{ - std::set res; - - if (ids.empty()) - return res; - - for (auto id : ids) - { - auto it = objectPosition.find(id); - if (it == objectPosition.end()) - continue; - - for (const auto& position : it->second) - res.insert(position); - } - - return res; -} - -static -std::set -getObjectsIds(const std::set& positionSet, const SOM::Matrix>& objectsMap ) -{ - std::set res; - - for (const auto& position : positionSet) - { - for (auto id : objectsMap.get(position)) - res.insert(id); - } - - return res; -} - -std::vector -FeaturesSearcher::getSimilarObjects(const std::set& ids, - const SOM::Matrix>& objectsMap, - const std::map>& objectPosition, - std::size_t maxCount) const -{ - std::vector res; - - if (!isValid()) - return res; - - auto now {std::chrono::system_clock::now()}; - std::mt19937 randGenerator{static_cast(std::chrono::duration_cast(now.time_since_epoch()).count())}; - - std::set searchedRefVectorsPosition {getMatchingRefVectorsPosition(ids, objectPosition)}; - if (searchedRefVectorsPosition.empty()) - return res; - - while (1) - { - std::set closestObjectIds {getObjectsIds(searchedRefVectorsPosition, objectsMap)}; - - // Remove objects that are already in input or already reported - for (auto id : ids) - closestObjectIds.erase(id); - - for (auto id : res) - closestObjectIds.erase(id); - - { - std::vector objectIdsToAdd {closestObjectIds.begin(), closestObjectIds.end()}; - std::shuffle(objectIdsToAdd.begin(), objectIdsToAdd.end(), randGenerator); - std::copy(objectIdsToAdd.begin(), objectIdsToAdd.end(), std::back_inserter(res)); - } - - if (res.size() > maxCount) - res.resize(maxCount); - - if (res.size() == maxCount) - break; - - // If there is not enough objects, try again with closest neighbour until there is too much distance - std::optional closestRefVectorPosition {_network->getClosestRefVectorPosition(searchedRefVectorsPosition, _networkRefVectorsDistanceMedian * 0.75)}; - if (!closestRefVectorPosition) - break; - - searchedRefVectorsPosition.insert(*closestRefVectorPosition); - } - - return res; -} - - - -} // ns Similarity diff --git a/src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.hpp b/src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.hpp deleted file mode 100644 index 625877c1..00000000 --- a/src/libs/recommendation/impl/features/SimilarityFeaturesSearcher.hpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include -#include -#include -#include - -#include "database/Types.hpp" -#include "som/DataNormalizer.hpp" -#include "som/Network.hpp" -#include "SimilarityFeaturesCache.hpp" -#include "SimilarityFeaturesDefs.hpp" - -namespace Database -{ - class Session; -} - -namespace Similarity { - -using FeatureWeight = double; - -class FeaturesSearcher -{ - public: - - using StopRequestedFunction = std::function; // return true if stop requested - - // Use cache - FeaturesSearcher(Database::Session& session, FeaturesCache cache, StopRequestedFunction stopRequested); - - // Use training (may be very slow) - struct TrainSettings - { - std::size_t iterationCount {10}; - float sampleCountPerNeuron {4}; - FeatureSettingsMap featureSettingsMap; - }; - FeaturesSearcher(Database::Session& session, const TrainSettings& trainSettings, StopRequestedFunction stopRequested = {}); - - static const FeatureSettingsMap& getDefaultTrainFeatureSettings(); - - bool isValid() const; - - bool isTrackClassified(Database::IdType trackId) const; - bool isReleaseClassified(Database::IdType releaseId) const; - bool isArtistClassified(Database::IdType artistId) const; - - std::vector getSimilarTracks(const std::set& tracksId, std::size_t maxCount) const; - std::vector getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const; - std::vector getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const; - - void dump(Database::Session& session, std::ostream& os) const; - - FeaturesCache toCache() const; - - using FeaturesFetchFunc = std::function>>(Database::IdType /*trackId*/, const std::unordered_set& /*features*/)>; - // Default is to retrieve the features from the database (may be slow). - // Use this only if you want to train different searchers with the same data - static void setFeaturesFetchFunc(FeaturesFetchFunc func) { _featuresFetchFunc = func; } - - private: - - using ObjectPositions = std::map>; - - void init(Database::Session& session, - SOM::Network network, - ObjectPositions tracksPosition, - StopRequestedFunction stopRequested); - - std::vector getSimilarObjects(const std::set& ids, - const SOM::Matrix>& objectsMap, - const ObjectPositions& objectPosition, - std::size_t maxCount) const; - - std::unique_ptr _network; - double _networkRefVectorsDistanceMedian {}; - - SOM::Matrix> _artistsMap; - ObjectPositions _artistPositions; - - SOM::Matrix> _releasesMap; - ObjectPositions _releasePositions; - - SOM::Matrix> _tracksMap; - ObjectPositions _trackPositions; - - static inline FeaturesFetchFunc _featuresFetchFunc; -}; - -} // ns Similarity diff --git a/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp b/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp index 3f4dfbe8..c9527ef8 100644 --- a/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp +++ b/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp @@ -25,6 +25,6 @@ namespace Recommendation { class IClassifier; - std::unique_ptr createClustersClassifier(Database::Session& session); + std::unique_ptr createClustersClassifier(); } diff --git a/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp b/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp index 9a763020..28a079d7 100644 --- a/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp +++ b/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp @@ -20,11 +20,10 @@ #pragma once #include +#include "recommendation/IClassifier.hpp" namespace Recommendation { - class IClassifier; - std::unique_ptr createFeaturesClassifier(); } diff --git a/src/libs/recommendation/include/recommendation/IClassifier.hpp b/src/libs/recommendation/include/recommendation/IClassifier.hpp index 6619ccd2..e1c01f2f 100644 --- a/src/libs/recommendation/include/recommendation/IClassifier.hpp +++ b/src/libs/recommendation/include/recommendation/IClassifier.hpp @@ -19,6 +19,7 @@ #pragma once +#include #include #include @@ -37,9 +38,10 @@ namespace Recommendation public: virtual ~IClassifier() = default; - virtual bool isTrackClassified(Database::IdType trackId) const = 0; - virtual bool isReleaseClassified(Database::IdType releaseId) const = 0; - virtual bool isArtistClassified(Database::IdType artistId) const = 0; + virtual std::string_view getName() const = 0; + + virtual bool init(Database::Session& session) = 0; + virtual void requestCancelInit() = 0; virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0; virtual std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const = 0; diff --git a/src/libs/scanner/CMakeLists.txt b/src/libs/scanner/CMakeLists.txt index 3d7a77f3..57fae63a 100644 --- a/src/libs/scanner/CMakeLists.txt +++ b/src/libs/scanner/CMakeLists.txt @@ -1,5 +1,6 @@ add_library(lmsscanner SHARED + impl/AcousticBrainzUtils.cpp impl/MediaScanner.cpp impl/MediaScannerStats.cpp ) diff --git a/src/libs/recommendation/impl/features/AcousticBrainzUtils.cpp b/src/libs/scanner/impl/AcousticBrainzUtils.cpp similarity index 84% rename from src/libs/recommendation/impl/features/AcousticBrainzUtils.cpp rename to src/libs/scanner/impl/AcousticBrainzUtils.cpp index b33f5c4f..970e04da 100644 --- a/src/libs/recommendation/impl/features/AcousticBrainzUtils.cpp +++ b/src/libs/scanner/impl/AcousticBrainzUtils.cpp @@ -28,6 +28,7 @@ #include "utils/IConfig.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" +#include "utils/UUID.hpp" namespace AcousticBrainz @@ -50,7 +51,7 @@ getJsonData(const UUID& mbid) if (!client.get(url)) { - LMS_LOG(SIMILARITY, ERROR) << "Cannot perform a GET request to url '" << url << "'"; + LMS_LOG(DBUPDATER, ERROR) << "Cannot perform a GET request to url '" << url << "'"; return {}; } @@ -59,13 +60,13 @@ getJsonData(const UUID& mbid) { if (ec) { - LMS_LOG(SIMILARITY, ERROR) << "GET request to url '" << url << "' failed: " << ec.message(); + LMS_LOG(DBUPDATER, ERROR) << "GET request to url '" << url << "' failed: " << ec.message(); return; } if (msg.status() != 200) { - LMS_LOG(SIMILARITY, ERROR) << "GET request to url '" << url << "' failed: status = " << msg.status() << ", body = " << msg.body(); + LMS_LOG(DBUPDATER, ERROR) << "GET request to url '" << url << "' failed: status = " << msg.status() << ", body = " << msg.body(); return; } diff --git a/src/libs/recommendation/impl/features/AcousticBrainzUtils.hpp b/src/libs/scanner/impl/AcousticBrainzUtils.hpp similarity index 96% rename from src/libs/recommendation/impl/features/AcousticBrainzUtils.hpp rename to src/libs/scanner/impl/AcousticBrainzUtils.hpp index 70d8fdf6..cdd73d77 100644 --- a/src/libs/recommendation/impl/features/AcousticBrainzUtils.hpp +++ b/src/libs/scanner/impl/AcousticBrainzUtils.hpp @@ -21,7 +21,7 @@ #include -#include "utils/UUID.hpp" +class UUID; namespace AcousticBrainz { diff --git a/src/libs/scanner/impl/MediaScanner.cpp b/src/libs/scanner/impl/MediaScanner.cpp index 84e78d92..236333d4 100644 --- a/src/libs/scanner/impl/MediaScanner.cpp +++ b/src/libs/scanner/impl/MediaScanner.cpp @@ -28,10 +28,13 @@ #include "database/Release.hpp" #include "database/ScanSettings.hpp" #include "database/Track.hpp" +#include "database/TrackFeatures.hpp" #include "metadata/TagLibParser.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" #include "utils/Path.hpp" +#include "utils/UUID.hpp" +#include "AcousticBrainzUtils.hpp" using namespace Database; @@ -420,7 +423,10 @@ MediaScanner::scan(boost::system::error_code err) if (_running) checkDuplicatedAudioFiles(stats); - LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), duplicates = " << stats.duplicates.size(); + // Now update all the track features if needed + fetchTrackFeatures(stats); + + LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << "/" << stats.featuresToFetch <<", duplicates = " << stats.duplicates.size(); LMS_LOG(DBUPDATER, INFO) << "Optimizing db..."; _dbSession.optimize(); @@ -449,6 +455,76 @@ MediaScanner::scan(boost::system::error_code err) } } +bool +MediaScanner::fetchTrackFeatures(Database::IdType trackId, const UUID& MBID) +{ + std::map features; + + LMS_LOG(DBUPDATER, INFO) << "Fetching low level features for track '" << MBID.getAsString() << "'"; + const std::string data {AcousticBrainz::extractLowLevelFeatures(MBID)}; + if (data.empty()) + { + LMS_LOG(DBUPDATER, ERROR) << "Track " << trackId << ", MBID = '" << MBID.getAsString() << "': cannot extract features using AcousticBrainz"; + return false; + } + + { + auto uniqueTransaction {_dbSession.createUniqueTransaction()}; + + Wt::Dbo::ptr track {Database::Track::getById(_dbSession, trackId)}; + if (!track) + return false; + + Database::TrackFeatures::create(_dbSession, track, data); + } + + return true; +} + +void +MediaScanner::fetchTrackFeatures(ScanStats& stats) +{ + if (_recommendationEngineType != ScanSettings::RecommendationEngineType::Features) + return; + + LMS_LOG(DBUPDATER, INFO) << "Fetching missing track features..."; + + struct TrackInfo + { + Database::IdType id; + UUID mbid; + }; + + const auto tracksToFetch {[&]() + { + std::vector res; + + auto transaction {_dbSession.createSharedTransaction()}; + + auto tracks {Database::Track::getAllWithMBIDAndMissingFeatures(_dbSession)}; + for (const auto& track : tracks) + res.emplace_back(TrackInfo {track.id(), *track->getMBID()}); + + return res; + }()}; + + stats.featuresToFetch = tracksToFetch.size(); + + LMS_LOG(DBUPDATER, INFO) << "Found " << tracksToFetch.size() << " track(s) to fetch!"; + + for (const TrackInfo& trackToFetch : tracksToFetch) + { + if (!_running) + return; + + if (fetchTrackFeatures(trackToFetch.id, trackToFetch.mbid)) + stats.featuresFetched++; + + } + + LMS_LOG(DBUPDATER, INFO) << "Track features fetched!"; +} + void MediaScanner::refreshScanSettings() { @@ -464,6 +540,7 @@ MediaScanner::refreshScanSettings() _fileExtensions = scanSettings->getAudioFileExtensions(); _mediaDirectory = scanSettings->getMediaDirectory(); + _recommendationEngineType = scanSettings->getRecommendationEngineType(); auto clusterTypes = scanSettings->getClusterTypes(); std::set clusterTypeNames; @@ -648,6 +725,7 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S track.modify()->setYear(*trackInfo->originalYear); track.modify()->setMBID(trackInfo->musicBrainzRecordID); + track.modify()->setFeatures({}); // TODO: only if MBID changed? track.modify()->setHasCover(trackInfo->hasCover); track.modify()->setCopyright(trackInfo->copyright); track.modify()->setCopyrightURL(trackInfo->copyrightURL); diff --git a/src/libs/scanner/impl/MediaScanner.hpp b/src/libs/scanner/impl/MediaScanner.hpp index 54bb70ce..645538a0 100644 --- a/src/libs/scanner/impl/MediaScanner.hpp +++ b/src/libs/scanner/impl/MediaScanner.hpp @@ -35,6 +35,7 @@ #include "metadata/IParser.hpp" #include "scanner/IMediaScanner.hpp" +class UUID; namespace Scanner { @@ -72,6 +73,8 @@ class MediaScanner : public IMediaScanner void scan(boost::system::error_code ec); void scanMediaDirectory( const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats); + bool fetchTrackFeatures(Database::IdType trackId, const UUID& MBID); + void fetchTrackFeatures(ScanStats& stats); // Helpers void refreshScanSettings(); @@ -102,11 +105,12 @@ class MediaScanner : public IMediaScanner Wt::WDateTime _nextScheduledScan; // Current scan settings - std::size_t _scanVersion {}; - Wt::WTime _startTime; + std::size_t _scanVersion {}; + Wt::WTime _startTime; Database::ScanSettings::UpdatePeriod _updatePeriod {Database::ScanSettings::UpdatePeriod::Never}; std::set _fileExtensions; std::filesystem::path _mediaDirectory; + Database::ScanSettings::RecommendationEngineType _recommendationEngineType; }; // class MediaScanner diff --git a/src/libs/scanner/include/scanner/MediaScannerStats.hpp b/src/libs/scanner/include/scanner/MediaScannerStats.hpp index 6f4e514d..99c898b6 100644 --- a/src/libs/scanner/include/scanner/MediaScannerStats.hpp +++ b/src/libs/scanner/include/scanner/MediaScannerStats.hpp @@ -77,10 +77,13 @@ namespace Scanner { std::size_t skips {}; // no change since last scan std::size_t scans {}; // actually scanned filed - std::size_t additions {}; // Added in DB + std::size_t additions {}; // added in DB std::size_t deletions {}; // removed from DB std::size_t updates {}; // updated file in DB + std::size_t featuresFetched {}; // features fetched in DB + std::size_t featuresToFetch {}; // features to be fetched in DB + std::vector errors; std::vector duplicates; diff --git a/src/libs/som/CMakeLists.txt b/src/libs/som/CMakeLists.txt index 7cb1b45e..2e19b9d2 100644 --- a/src/libs/som/CMakeLists.txt +++ b/src/libs/som/CMakeLists.txt @@ -16,3 +16,5 @@ target_link_libraries(lmssom PUBLIC lmsutils ) +set_property(TARGET lmssom PROPERTY POSITION_INDEPENDENT_CODE ON) + diff --git a/src/libs/som/impl/Network.cpp b/src/libs/som/impl/Network.cpp index deea6f7b..75e60fce 100644 --- a/src/libs/som/impl/Network.cpp +++ b/src/libs/som/impl/Network.cpp @@ -201,9 +201,9 @@ Network::getClosestRefVectorPosition(const InputVector& data, InputVector::Dista } std::optional -Network::getClosestRefVectorPosition(const std::set& refVectorsPosition, InputVector::Distance maxDistance) const +Network::getClosestRefVectorPosition(const std::unordered_set& refVectorsPosition, InputVector::Distance maxDistance) const { - std::set neighboursPosition; + std::unordered_set neighboursPosition; for (const Position& refVectorPosition : refVectorsPosition) { if (refVectorPosition.y > 0) diff --git a/src/libs/som/include/som/InputVector.hpp b/src/libs/som/include/som/InputVector.hpp index 058e0fee..50737fec 100644 --- a/src/libs/som/include/som/InputVector.hpp +++ b/src/libs/som/include/som/InputVector.hpp @@ -31,7 +31,7 @@ namespace SOM class Exception : public LmsException { public: - Exception(const std::string& msg) : LmsException(msg) {} + using LmsException::LmsException; }; class InputVector diff --git a/src/libs/som/include/som/Matrix.hpp b/src/libs/som/include/som/Matrix.hpp index 32df1bff..2688d729 100644 --- a/src/libs/som/include/som/Matrix.hpp +++ b/src/libs/som/include/som/Matrix.hpp @@ -21,7 +21,7 @@ #include #include -#include +#include #include namespace SOM @@ -116,3 +116,20 @@ class Matrix }; } // ns SOM + +namespace std { + +template<> +class hash +{ + public: + size_t operator()(const SOM::Position& s) const + { + size_t h1 = std::hash()(s.x); + size_t h2 = std::hash()(s.y); + return h1 ^ (h2 << 1); + } +}; + +} // ns std + diff --git a/src/libs/som/include/som/Network.hpp b/src/libs/som/include/som/Network.hpp index cf04cf72..f3ba2df1 100644 --- a/src/libs/som/include/som/Network.hpp +++ b/src/libs/som/include/som/Network.hpp @@ -20,7 +20,7 @@ #pragma once #include -#include +#include #include #include #include @@ -70,7 +70,7 @@ class Network Position getClosestRefVectorPosition(const InputVector& data) const; std::optional getClosestRefVectorPosition(const InputVector& data, InputVector::Distance maxDistance) const; - std::optional getClosestRefVectorPosition(const std::set& refVectorsPosition, InputVector::Distance maxDistance) const; + std::optional getClosestRefVectorPosition(const std::unordered_set& refVectorsPosition, InputVector::Distance maxDistance) const; InputVector::Distance getRefVectorsDistance(const Position& position1, const Position& position2) const; diff --git a/src/lms/main.cpp b/src/lms/main.cpp index 1b6e504e..54fe076c 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -148,7 +148,7 @@ int main(int argc, char* argv[]) { auto status = mediaScanner.getStatus(); - if (status.lastCompleteScanStats->nbChanges() > 0) + if (status.lastCompleteScanStats->nbChanges() > 0 || status.lastCompleteScanStats->featuresFetched > 0) { LMS_LOG(MAIN, INFO) << "Scanner changed some files, reloading the recommendation engine..."; recommendationEngine.requestReload(); diff --git a/src/test/som/SomTest.cpp b/src/test/som/SomTest.cpp index 6f4edb37..bfbf05a5 100644 --- a/src/test/som/SomTest.cpp +++ b/src/test/som/SomTest.cpp @@ -82,7 +82,7 @@ int main() assert((std::abs(distFunc({1, 0}, {1, 0.33}, weights) - distFunc({1, 0.66}, {1, 1.}, weights)) < EPSILON)); { - std::set positions; + std::unordered_set positions; for (const InputVector& data : trainData) positions.insert(network.getClosestRefVectorPosition(data)); assert(positions.size() == 4); diff --git a/src/tools/recommendation/LmsRecommendation.cpp b/src/tools/recommendation/LmsRecommendation.cpp index b5ea5aa4..954af8c0 100644 --- a/src/tools/recommendation/LmsRecommendation.cpp +++ b/src/tools/recommendation/LmsRecommendation.cpp @@ -149,7 +149,7 @@ int main(int argc, char *argv[]) engine->start(); - std::cout << "Wating for the recommendation engine to be loaded..." << std::endl; + std::cout << "Waiting for the recommendation engine to be loaded..." << std::endl; sem.wait(); std::cout << "Recommendation engine loaded!" << std::endl; From d8712de5858df4f58520c0efc693ea4989b85862 Mon Sep 17 00:00:00 2001 From: emeric Date: Tue, 18 Feb 2020 13:51:30 +0100 Subject: [PATCH 19/23] Adapted dockerfile to build using cmake --- docker/Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2a94c55d..4ac83cd4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,7 +7,7 @@ ARG FFMPEG_VERSION=4.1.4 ARG WT_VERSION=4.2.0 ARG IMAGEMAGICK6_VERSION=6.9.10-71 ARG PSTREAMS_VERSION=1.0.1 -ARG LMS_VERSION=3.5.0 +ARG LMS_VERSION=v3.6.3 ARG PREFIX="/tmp/install" @@ -153,23 +153,21 @@ RUN \ # LMS RUN \ DIR=/tmp/lms && mkdir -p ${DIR} && cd ${DIR} && \ - curl -sLO https://github.com/epoupon/lms/archive/v${LMS_VERSION}.tar.gz && \ - tar -x --strip-components=1 -f v${LMS_VERSION}.tar.gz + curl -sL https://github.com/epoupon/lms/archive/${LMS_VERSION}.tar.gz -o ${LMS_VERSION}.tar.gz && \ + tar -x --strip-components=1 -f ${LMS_VERSION}.tar.gz RUN \ DIR=/tmp/lms && mkdir -p ${DIR} && cd ${DIR} && \ - autoreconf -vfi && \ - PKG_CONFIG_PATH=/tmp/install/lib/pkgconfig CXXFLAGS="-O2 -I${PREFIX}/include" LDFLAGS="-L${PREFIX}/lib -Wl,--rpath-link=${PREFIX}/lib" ./configure --prefix=${PREFIX} && \ + PKG_CONFIG_PATH=/tmp/install/lib/pkgconfig CXXFLAGS="-I${PREFIX}/include -L${PREFIX}/lib -Wl,--rpath-link=${PREFIX}/lib" cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PREFIX} && \ make && \ make install && \ - make distclean && \ mkdir -p ${PREFIX}/etc/ && \ cp conf/lms.conf ${PREFIX}/etc # Now copy all the stuff installed in a new folder (/tmp/fakeroot/) RUN \ mkdir -p /tmp/fakeroot/bin && \ - for bin in ${PREFIX}/bin/ffmpeg ${PREFIX}/bin/lms; \ + for bin in ${PREFIX}/bin/ffmpeg ${PREFIX}/bin/lms*; \ do \ strip --strip-all $bin && \ cp $bin /tmp/fakeroot/bin/; \ From 428ab70d27c4a80c4689fa692b861b3af7b2bcdd Mon Sep 17 00:00:00 2001 From: emeric Date: Wed, 19 Feb 2020 13:35:31 +0100 Subject: [PATCH 20/23] Attempt to check why some files are missing --- approot/messages.xml | 2 ++ approot/messages_fr.xml | 2 ++ src/libs/database/impl/ScanSettings.cpp | 5 +++-- .../include/database/ScanSettings.hpp | 6 ++++-- src/libs/scanner/impl/MediaScanner.cpp | 20 +++++++++++++++++-- src/libs/scanner/impl/MediaScanner.hpp | 2 +- .../include/scanner/MediaScannerStats.hpp | 2 ++ src/libs/utils/include/utils/Path.hpp | 7 +++++++ src/lms/ui/admin/DatabaseStatus.cpp | 2 ++ 9 files changed, 41 insertions(+), 7 deletions(-) diff --git a/approot/messages.xml b/approot/messages.xml index 20677bc8..925ac67c 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -61,6 +61,8 @@ Not available Scanned {1} files in {2} on {3} ({4} errors, {5} duplicates) No audio track +Not a regular file +File type not supported Duplicated file hash Duplicated MBID Status diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index 934cd400..5f5951b2 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -61,6 +61,8 @@ Non disponible {1} fichiers scannés en {2} le {3} ({4} erreurs, {5} duplicatas) Pas de piste audio +Fichier non régulier +Type de fichier non supporté Hash dupliqué MBID dupliqué Statut diff --git a/src/libs/database/impl/ScanSettings.cpp b/src/libs/database/impl/ScanSettings.cpp index 673e2a24..3d9a1a86 100644 --- a/src/libs/database/impl/ScanSettings.cpp +++ b/src/libs/database/impl/ScanSettings.cpp @@ -21,6 +21,7 @@ #include +#include "utils/Path.hpp" #include "utils/Logger.hpp" #include "utils/String.hpp" @@ -62,11 +63,11 @@ ScanSettings::get(Session& session) return session.getDboSession().find(); } -std::set +std::unordered_set ScanSettings::getAudioFileExtensions() const { auto extensions = StringUtils::splitString(_audioFileExtensions, " "); - return std::set(std::cbegin(extensions), std::cend(extensions)); + return std::unordered_set(std::cbegin(extensions), std::cend(extensions)); } void diff --git a/src/libs/database/include/database/ScanSettings.hpp b/src/libs/database/include/database/ScanSettings.hpp index 7151df23..0a2ee450 100644 --- a/src/libs/database/include/database/ScanSettings.hpp +++ b/src/libs/database/include/database/ScanSettings.hpp @@ -19,11 +19,13 @@ #pragma once -#include +#include #include #include +#include "utils/Path.hpp" + namespace Database { class ClusterType; @@ -59,7 +61,7 @@ class ScanSettings : public Wt::Dbo::Dbo Wt::WTime getUpdateStartTime() const { return _startTime; } UpdatePeriod getUpdatePeriod() const { return _updatePeriod; } std::vector> getClusterTypes() const; - std::set getAudioFileExtensions() const; + std::unordered_set getAudioFileExtensions() const; RecommendationEngineType getRecommendationEngineType() const { return _recommendationEngineType; } // Setters diff --git a/src/libs/scanner/impl/MediaScanner.cpp b/src/libs/scanner/impl/MediaScanner.cpp index 586f2995..b213f9f8 100644 --- a/src/libs/scanner/impl/MediaScanner.cpp +++ b/src/libs/scanner/impl/MediaScanner.cpp @@ -63,7 +63,7 @@ getNextFirstOfMonth(Wt::WDate current) } bool -isFileSupported(const std::filesystem::path& file, const std::set& extensions) +isFileSupported(const std::filesystem::path& file, const std::unordered_set& extensions) { return (extensions.find(file.extension()) != extensions.end()); } @@ -768,10 +768,26 @@ MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bo LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message(); stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()}); } + else if (std::filesystem::is_directory(path)) + { + ; + } else if (std::filesystem::is_regular_file(path)) { if (isFileSupported(path, _fileExtensions)) + { scanAudioFile(path, forceScan, stats ); + } + else + { + LMS_LOG(DBUPDATER, ERROR) << "Skipped '" << path.string() << "': file not supported"; + stats.errors.emplace_back(ScanError {path, ScanErrorType::NotSupported}); + } + } + else + { + LMS_LOG(DBUPDATER, ERROR) << "Skipped '" << path.string() << "': not a regular file"; + stats.errors.emplace_back(ScanError {path, ScanErrorType::NotRegular}); } itPath.increment(ec); @@ -782,7 +798,7 @@ MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bo // Check if a file exists and is still in a media directory static bool -checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory, const std::set& extensions) +checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory, const std::unordered_set& extensions) { try { diff --git a/src/libs/scanner/impl/MediaScanner.hpp b/src/libs/scanner/impl/MediaScanner.hpp index 645538a0..a04ceb39 100644 --- a/src/libs/scanner/impl/MediaScanner.hpp +++ b/src/libs/scanner/impl/MediaScanner.hpp @@ -108,7 +108,7 @@ class MediaScanner : public IMediaScanner std::size_t _scanVersion {}; Wt::WTime _startTime; Database::ScanSettings::UpdatePeriod _updatePeriod {Database::ScanSettings::UpdatePeriod::Never}; - std::set _fileExtensions; + std::unordered_set _fileExtensions; std::filesystem::path _mediaDirectory; Database::ScanSettings::RecommendationEngineType _recommendationEngineType; diff --git a/src/libs/scanner/include/scanner/MediaScannerStats.hpp b/src/libs/scanner/include/scanner/MediaScannerStats.hpp index 99c898b6..7356306e 100644 --- a/src/libs/scanner/include/scanner/MediaScannerStats.hpp +++ b/src/libs/scanner/include/scanner/MediaScannerStats.hpp @@ -32,6 +32,8 @@ namespace Scanner { CannotParseFile, // cannot parse file NoAudioTrack, // no audio track found BadDuration, // bad duration + NotSupported, // not supported + NotRegular, }; enum class DuplicateReason diff --git a/src/libs/utils/include/utils/Path.hpp b/src/libs/utils/include/utils/Path.hpp index f7e82893..caf3a148 100644 --- a/src/libs/utils/include/utils/Path.hpp +++ b/src/libs/utils/include/utils/Path.hpp @@ -34,3 +34,10 @@ bool ensureDirectory(const std::filesystem::path& dir); // Get the last write time since Epoch Wt::WDateTime getLastWriteTime(const std::filesystem::path& dir); +namespace std { + template <> + struct hash { + inline std::size_t operator()(const std::filesystem::path &path) const { return hash_value(path); } + }; +} + diff --git a/src/lms/ui/admin/DatabaseStatus.cpp b/src/lms/ui/admin/DatabaseStatus.cpp index 34133e01..017ef1a8 100644 --- a/src/lms/ui/admin/DatabaseStatus.cpp +++ b/src/lms/ui/admin/DatabaseStatus.cpp @@ -94,6 +94,8 @@ class ReportResource : public Wt::WResource case Scanner::ScanErrorType::CannotParseFile: return Wt::WString::tr("Lms.Admin.Database.Status.cannot-parse-file"); case Scanner::ScanErrorType::NoAudioTrack: return Wt::WString::tr("Lms.Admin.Database.Status.no-audio-track"); case Scanner::ScanErrorType::BadDuration: return Wt::WString::tr("Lms.Admin.Database.Status.bad-duration"); + case Scanner::ScanErrorType::NotSupported: return Wt::WString::tr("Lms.Admin.Database.Status.not-supported"); + case Scanner::ScanErrorType::NotRegular: return Wt::WString::tr("Lms.Admin.Database.Status.not-regular"); } return "?"; } From 284231ff92a64d691ee8c7e9a6138b0df5c2e4b3 Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 20 Feb 2020 13:32:48 +0100 Subject: [PATCH 21/23] Attempt to properly scan all the files during scan --- approot/messages.xml | 2 - approot/messages_fr.xml | 2 - src/libs/scanner/impl/MediaScanner.cpp | 73 ++++--------------- .../include/scanner/MediaScannerStats.hpp | 2 - src/libs/utils/impl/Path.cpp | 36 +++++++++ src/libs/utils/include/utils/Path.hpp | 10 ++- src/lms/ui/admin/DatabaseStatus.cpp | 2 - 7 files changed, 59 insertions(+), 68 deletions(-) diff --git a/approot/messages.xml b/approot/messages.xml index 925ac67c..20677bc8 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -61,8 +61,6 @@ Not available Scanned {1} files in {2} on {3} ({4} errors, {5} duplicates) No audio track -Not a regular file -File type not supported Duplicated file hash Duplicated MBID Status diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index 5f5951b2..934cd400 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -61,8 +61,6 @@ Non disponible {1} fichiers scannés en {2} le {3} ({4} erreurs, {5} duplicatas) Pas de piste audio -Fichier non régulier -Type de fichier non supporté Hash dupliqué MBID dupliqué Statut diff --git a/src/libs/scanner/impl/MediaScanner.cpp b/src/libs/scanner/impl/MediaScanner.cpp index b213f9f8..a6d27213 100644 --- a/src/libs/scanner/impl/MediaScanner.cpp +++ b/src/libs/scanner/impl/MediaScanner.cpp @@ -350,33 +350,21 @@ MediaScanner::scheduleNextScan() void MediaScanner::countAllFiles(ScanStats& stats) { - std::error_code ec; - stats.filesToScan = 0; - std::filesystem::recursive_directory_iterator itPath {_mediaDirectory, std::filesystem::directory_options::follow_directory_symlink, ec}; - if (ec) + exploreFilesRecursive(_mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path) { - LMS_LOG(DBUPDATER, ERROR) << "Cannot iterate over '" << _mediaDirectory.string() << "': " << ec.message(); - return; - } + if (ec) + return; - std::filesystem::recursive_directory_iterator itEnd; - while (_running && itPath != itEnd) - { - const std::filesystem::path& path {*itPath}; - - if (!ec) + if (isFileSupported(path, _fileExtensions)) { - if (std::filesystem::is_regular_file(path) && isFileSupported(path, _fileExtensions)) - stats.filesToScan ++; + stats.filesToScan++; if (stats.filesToScan % 250 == 0) notifyInProgressIfNeeded(stats); } - - itPath.increment(ec); - } + }); } void @@ -592,8 +580,6 @@ MediaScanner::notifyInProgressIfNeeded(const ScanStats& stats) void MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats) { - notifyInProgressIfNeeded(stats); - Wt::WDateTime lastWriteTime; try { @@ -748,50 +734,23 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S void MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats) { - std::error_code ec; - - std::filesystem::recursive_directory_iterator itPath {_mediaDirectory, std::filesystem::directory_options::follow_directory_symlink, ec}; - if (ec) + exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path) { - LMS_LOG(DBUPDATER, ERROR) << "Cannot iterate over '" << mediaDirectory.string() << "': " << ec.message(); - stats.errors.emplace_back(ScanError {mediaDirectory, ScanErrorType::CannotReadFile, ec.message()}); - return; - } - - std::filesystem::recursive_directory_iterator itEnd; - while (_running && itPath != itEnd) - { - const std::filesystem::path& path {*itPath}; - if (ec) { LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message(); stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()}); - } - else if (std::filesystem::is_directory(path)) - { - ; - } - else if (std::filesystem::is_regular_file(path)) - { - if (isFileSupported(path, _fileExtensions)) - { - scanAudioFile(path, forceScan, stats ); - } - else - { - LMS_LOG(DBUPDATER, ERROR) << "Skipped '" << path.string() << "': file not supported"; - stats.errors.emplace_back(ScanError {path, ScanErrorType::NotSupported}); - } - } - else - { - LMS_LOG(DBUPDATER, ERROR) << "Skipped '" << path.string() << "': not a regular file"; - stats.errors.emplace_back(ScanError {path, ScanErrorType::NotRegular}); + + return; } - itPath.increment(ec); - } + if (isFileSupported(path, _fileExtensions)) + { + scanAudioFile(path, forceScan, stats ); + + notifyInProgressIfNeeded(stats); + } + }); notifyInProgress(stats); } diff --git a/src/libs/scanner/include/scanner/MediaScannerStats.hpp b/src/libs/scanner/include/scanner/MediaScannerStats.hpp index 7356306e..99c898b6 100644 --- a/src/libs/scanner/include/scanner/MediaScannerStats.hpp +++ b/src/libs/scanner/include/scanner/MediaScannerStats.hpp @@ -32,8 +32,6 @@ namespace Scanner { CannotParseFile, // cannot parse file NoAudioTrack, // no audio track found BadDuration, // bad duration - NotSupported, // not supported - NotRegular, }; enum class DuplicateReason diff --git a/src/libs/utils/impl/Path.cpp b/src/libs/utils/impl/Path.cpp index a5c0d882..246215f1 100644 --- a/src/libs/utils/impl/Path.cpp +++ b/src/libs/utils/impl/Path.cpp @@ -87,3 +87,39 @@ getLastWriteTime(const std::filesystem::path& file) return Wt::WDateTime::fromTime_t(sb.st_mtime); } +void +exploreFilesRecursive(const std::filesystem::path& directory, std::function cb) +{ + std::error_code ec; + std::filesystem::directory_iterator itPath {directory, std::filesystem::directory_options::follow_directory_symlink, ec}; + + if (ec) + { + cb(ec, directory); + return; + } + + std::filesystem::directory_iterator itEnd; + while (itPath != itEnd) + { + if (ec) + { + cb(ec, *itPath); + } + else + { + if (std::filesystem::is_regular_file(*itPath, ec)) + cb(ec, *itPath); + else if (std::filesystem::is_directory(*itPath, ec)) + { + if (!ec) + exploreFilesRecursive(*itPath, cb); + else + cb(ec, *itPath); + } + } + + itPath.increment(ec); + } +} + diff --git a/src/libs/utils/include/utils/Path.hpp b/src/libs/utils/include/utils/Path.hpp index caf3a148..6367ce03 100644 --- a/src/libs/utils/include/utils/Path.hpp +++ b/src/libs/utils/include/utils/Path.hpp @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -34,10 +35,13 @@ bool ensureDirectory(const std::filesystem::path& dir); // Get the last write time since Epoch Wt::WDateTime getLastWriteTime(const std::filesystem::path& dir); -namespace std { +void exploreFilesRecursive(const std::filesystem::path& directory, std::function cb); + +namespace std +{ template <> - struct hash { + struct hash + { inline std::size_t operator()(const std::filesystem::path &path) const { return hash_value(path); } }; } - diff --git a/src/lms/ui/admin/DatabaseStatus.cpp b/src/lms/ui/admin/DatabaseStatus.cpp index 017ef1a8..34133e01 100644 --- a/src/lms/ui/admin/DatabaseStatus.cpp +++ b/src/lms/ui/admin/DatabaseStatus.cpp @@ -94,8 +94,6 @@ class ReportResource : public Wt::WResource case Scanner::ScanErrorType::CannotParseFile: return Wt::WString::tr("Lms.Admin.Database.Status.cannot-parse-file"); case Scanner::ScanErrorType::NoAudioTrack: return Wt::WString::tr("Lms.Admin.Database.Status.no-audio-track"); case Scanner::ScanErrorType::BadDuration: return Wt::WString::tr("Lms.Admin.Database.Status.bad-duration"); - case Scanner::ScanErrorType::NotSupported: return Wt::WString::tr("Lms.Admin.Database.Status.not-supported"); - case Scanner::ScanErrorType::NotRegular: return Wt::WString::tr("Lms.Admin.Database.Status.not-regular"); } return "?"; } From 0ad4973a5a78cb11e36f2158944317287242e8e6 Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 20 Feb 2020 14:09:52 +0100 Subject: [PATCH 22/23] Restored cancellable scan --- src/libs/scanner/impl/MediaScanner.cpp | 18 +++++++++++------- src/libs/utils/impl/Path.cpp | 15 +++++++++++---- src/libs/utils/include/utils/Path.hpp | 2 +- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/libs/scanner/impl/MediaScanner.cpp b/src/libs/scanner/impl/MediaScanner.cpp index a6d27213..9de04c5b 100644 --- a/src/libs/scanner/impl/MediaScanner.cpp +++ b/src/libs/scanner/impl/MediaScanner.cpp @@ -354,16 +354,18 @@ MediaScanner::countAllFiles(ScanStats& stats) exploreFilesRecursive(_mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path) { - if (ec) - return; + if (!_running) + return false; - if (isFileSupported(path, _fileExtensions)) + if (!ec && isFileSupported(path, _fileExtensions)) { stats.filesToScan++; if (stats.filesToScan % 250 == 0) notifyInProgressIfNeeded(stats); } + + return true; }); } @@ -736,20 +738,22 @@ MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bo { exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path) { + if (!_running) + return false; + if (ec) { LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message(); stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()}); - - return; } - - if (isFileSupported(path, _fileExtensions)) + else if (isFileSupported(path, _fileExtensions)) { scanAudioFile(path, forceScan, stats ); notifyInProgressIfNeeded(stats); } + + return true; }); notifyInProgress(stats); diff --git a/src/libs/utils/impl/Path.cpp b/src/libs/utils/impl/Path.cpp index 246215f1..0bfc807b 100644 --- a/src/libs/utils/impl/Path.cpp +++ b/src/libs/utils/impl/Path.cpp @@ -88,7 +88,7 @@ getLastWriteTime(const std::filesystem::path& file) } void -exploreFilesRecursive(const std::filesystem::path& directory, std::function cb) +exploreFilesRecursive(const std::filesystem::path& directory, std::function cb) { std::error_code ec; std::filesystem::directory_iterator itPath {directory, std::filesystem::directory_options::follow_directory_symlink, ec}; @@ -102,23 +102,30 @@ exploreFilesRecursive(const std::filesystem::path& directory, std::function cb); +void exploreFilesRecursive(const std::filesystem::path& directory, std::function cb); namespace std { From d02a7ca86bd1ced9765031fa7a0c7ba8efefd618 Mon Sep 17 00:00:00 2001 From: emeric Date: Fri, 21 Feb 2020 15:03:42 +0100 Subject: [PATCH 23/23] Made the recommendation engine more usable during initial scan/rescans --- src/libs/recommendation/impl/Engine.cpp | 135 ++++++++++++------ src/libs/recommendation/impl/Engine.hpp | 32 ++++- .../impl/clusters/ClustersClassifier.hpp | 2 +- .../impl/features/FeaturesClassifier.cpp | 56 +++++++- .../impl/features/FeaturesClassifier.hpp | 2 +- .../include/recommendation/IClassifier.hpp | 4 +- src/libs/scanner/impl/MediaScanner.cpp | 6 +- src/lms/ui/explore/ArtistView.cpp | 5 - 8 files changed, 177 insertions(+), 65 deletions(-) diff --git a/src/libs/recommendation/impl/Engine.cpp b/src/libs/recommendation/impl/Engine.cpp index 4b4a6b67..072c3d48 100644 --- a/src/libs/recommendation/impl/Engine.cpp +++ b/src/libs/recommendation/impl/Engine.cpp @@ -46,7 +46,7 @@ Engine::start() assert(!_running); _running = true; - requestReload(); + requestReloadInternal(false); _ioService.start(); } @@ -64,12 +64,18 @@ Engine::stop() void Engine::requestReload() +{ + requestReloadInternal(true); +} + +void +Engine::requestReloadInternal(bool databaseChanged) { LMS_LOG(RECOMMENDATION, DEBUG) << "Reload requested..."; - _ioService.post([&]() + _ioService.post([=]() { - reload(); + reload(databaseChanged); }); } @@ -80,9 +86,13 @@ Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdTy std::vector res; - for (const auto& [priority, classifier] : _classifiers) + for (const auto& classifierName : _classifierPriorities) { - res = classifier->getSimilarTracksFromTrackList(session, trackListId, maxCount); + auto itClassifier {_classifiers.find(classifierName)}; + if (itClassifier == std::cend(_classifiers)) + continue; + + res = itClassifier->second->getSimilarTracksFromTrackList(session, trackListId, maxCount); if (!res.empty()) break; } @@ -97,11 +107,18 @@ Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set< std::vector res; - for (const auto& [priority, classifier] : _classifiers) + for (const auto& classifierName : _classifierPriorities) { - res = classifier->getSimilarTracks(dbSession, trackIds, maxCount); + auto itClassifier {_classifiers.find(classifierName)}; + if (itClassifier == std::cend(_classifiers)) + continue; + + res = itClassifier->second->getSimilarTracks(dbSession, trackIds, maxCount); if (!res.empty()) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using classifier '" << classifierName << "'"; break; + } } return res; @@ -114,11 +131,18 @@ Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releas std::vector res; - for (const auto& [priority, classifier] : _classifiers) + for (const auto& classifierName : _classifierPriorities) { - res = classifier->getSimilarReleases(dbSession, releaseId, maxCount); + auto itClassifier {_classifiers.find(classifierName)}; + if (itClassifier == std::cend(_classifiers)) + continue; + + res = itClassifier->second->getSimilarReleases(dbSession, releaseId, maxCount); if (!res.empty()) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using classifier '" << classifierName << "'"; break; + } } return res; @@ -131,18 +155,25 @@ Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistI std::vector res; - for (const auto& [priority, classifier] : _classifiers) + for (const auto& classifierName : _classifierPriorities) { - res = classifier->getSimilarArtists(dbSession, artistId, maxCount); + auto itClassifier {_classifiers.find(classifierName)}; + if (itClassifier == std::cend(_classifiers)) + continue; + + res = itClassifier->second->getSimilarArtists(dbSession, artistId, maxCount); if (!res.empty()) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using classifier '" << classifierName << "'"; return res; + } } return res; } void -Engine::reload() +Engine::reload(bool databaseChanged) { using namespace Database; @@ -155,43 +186,29 @@ Engine::reload() return ScanSettings::get(_dbSession)->getRecommendationEngineType(); }()}; - std::map> newClassifiers; - - // TODO RAII this - auto addClassifier = [&](ClassifierPriority prio, std::unique_ptr classifier) - { - try - { - addPendingClassifier(*classifier.get()); - bool res {classifier->init(_dbSession)}; - removePendingClassifier(*classifier.get()); - - if (res) - newClassifiers.emplace(prio, std::move(classifier)); - - return res; - } - catch (LmsException& e) - { - removePendingClassifier(*classifier.get()); - throw; - } - }; + clearClassifiers(); switch (engineType) { case ScanSettings::RecommendationEngineType::Features: - addClassifier(0, createFeaturesClassifier()); // higher priority - [[fallthrough]]; + { + auto clustersClassifier {createClustersClassifier()}; + auto featuresClassifier {createFeaturesClassifier()}; + + setClassifierPriorities({featuresClassifier->getName(), clustersClassifier->getName()}); + + initAndAddClassifier(std::move(clustersClassifier), databaseChanged); // init first since faster + initAndAddClassifier(std::move(featuresClassifier), databaseChanged); + break; + } case ScanSettings::RecommendationEngineType::Clusters: - addClassifier(1, createClustersClassifier()); // lower priority - break; - } + auto clustersClassifier {createClustersClassifier()}; - { - std::unique_lock lock {_classifiersMutex}; - _classifiers.swap(newClassifiers); + setClassifierPriorities({clustersClassifier->getName()}); + + initAndAddClassifier(std::move(clustersClassifier), databaseChanged); + break; } LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines reloaded!"; @@ -199,6 +216,40 @@ Engine::reload() _sigReloaded.emit(); } +void +Engine::setClassifierPriorities(std::initializer_list classifierPriorities) +{ + std::unique_lock lock {_classifiersMutex}; + + _classifierPriorities.clear(); + std::transform(std::cbegin(classifierPriorities), std::cend(classifierPriorities), std::back_inserter(_classifierPriorities), [](std::string_view name) { return std::string {name}; }); +} + +void +Engine::clearClassifiers() +{ + std::unique_lock lock {_classifiersMutex}; + + _classifiers.clear(); +} + +void +Engine::initAndAddClassifier(std::unique_ptr classifier, bool databaseChanged) +{ + PendingClassifierHandler pendingClassifier {*this, *classifier.get()}; + + LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "'..."; + bool res {classifier->init(_dbSession, databaseChanged)}; + LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "': " << (res ? "SUCCESS" : "FAILURE"); + + if (res) + { + std::unique_lock lock {_classifiersMutex}; + + _classifiers.emplace(classifier->getName(), std::move(classifier)); + } +} + void Engine::cancelPendingClassifiers() { diff --git a/src/libs/recommendation/impl/Engine.hpp b/src/libs/recommendation/impl/Engine.hpp index 17208b77..4fab63f1 100644 --- a/src/libs/recommendation/impl/Engine.hpp +++ b/src/libs/recommendation/impl/Engine.hpp @@ -21,6 +21,7 @@ #include #include +#include #include @@ -37,8 +38,6 @@ namespace Recommendation private: - using ClassifierPriority = unsigned; - void start() override; void stop() override; @@ -51,7 +50,31 @@ namespace Recommendation std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override; - void reload(); + void requestReloadInternal(bool databaseChanged); + void reload(bool databaseChanged); + + void setClassifierPriorities(std::initializer_list classifierNames); + void clearClassifiers(); + void initAndAddClassifier(std::unique_ptr classifier, bool databaseChanged); + + class PendingClassifierHandler + { + public: + PendingClassifierHandler(Engine& engine, IClassifier& classifier) : _engine {engine}, _classifier {classifier} + { + _engine.addPendingClassifier(_classifier); + } + + ~PendingClassifierHandler() + { + _engine.removePendingClassifier(_classifier); + } + + private: + Engine& _engine; + IClassifier& _classifier; + }; + void cancelPendingClassifiers(); void addPendingClassifier(IClassifier& classifier); void removePendingClassifier(IClassifier& classifier); @@ -62,7 +85,8 @@ namespace Recommendation Wt::Signal<> _sigReloaded; std::shared_mutex _classifiersMutex; - std::map> _classifiers; + std::map> _classifiers; + std::vector _classifierPriorities; // ordered by priority std::unordered_set _pendingClassifiers; }; diff --git a/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp b/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp index 469a6ee8..4619d5dc 100644 --- a/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp +++ b/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp @@ -38,7 +38,7 @@ namespace Recommendation std::string_view getName() const { return "Clusters"; } - bool init(Database::Session&) override {return true;} + bool init(Database::Session&, bool) override {return true;} void requestCancelInit() override {} std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override; diff --git a/src/libs/recommendation/impl/features/FeaturesClassifier.cpp b/src/libs/recommendation/impl/features/FeaturesClassifier.cpp index f4f4ad90..d3b13d5d 100644 --- a/src/libs/recommendation/impl/features/FeaturesClassifier.cpp +++ b/src/libs/recommendation/impl/features/FeaturesClassifier.cpp @@ -259,21 +259,57 @@ FeaturesClassifier::getSimilarTracksFromTrackList(Database::Session& session, Da } std::vector -FeaturesClassifier::getSimilarTracks(Database::Session&, const std::unordered_set& tracksIds, std::size_t maxCount) const +FeaturesClassifier::getSimilarTracks(Database::Session& session, const std::unordered_set& tracksIds, std::size_t maxCount) const { - return getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount); + std::vector similarTrackIds {getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount)}; + + if (!similarTrackIds.empty()) + { + // Report only existing ids + auto transaction {session.createSharedTransaction()}; + + similarTrackIds.erase(std::remove_if(std::begin(similarTrackIds), std::end(similarTrackIds), + [&](Database::IdType trackId) { return Database::Track::getById(session, trackId) == Database::Track::pointer {}; }), + std::cend(similarTrackIds)); + } + + return similarTrackIds; } std::vector -FeaturesClassifier::getSimilarReleases(Database::Session&, Database::IdType releaseId, std::size_t maxCount) const +FeaturesClassifier::getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const { - return getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount); + std::vector similarReleaseIds {getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount)}; + + if (!similarReleaseIds.empty()) + { + // Report only existing ids + auto transaction {session.createSharedTransaction()}; + + similarReleaseIds.erase(std::remove_if(std::begin(similarReleaseIds), std::end(similarReleaseIds), + [&](Database::IdType releaseId) { return Database::Release::getById(session, releaseId) == Database::Release::pointer {}; }), + std::cend(similarReleaseIds)); + } + + return similarReleaseIds; } std::vector -FeaturesClassifier::getSimilarArtists(Database::Session&, Database::IdType artistId, std::size_t maxCount) const +FeaturesClassifier::getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const { - return getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount); + std::vector similarArtistIds {getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount)}; + + if (!similarArtistIds.empty()) + { + // Report only existing ids + auto transaction {session.createSharedTransaction()}; + + similarArtistIds.erase(std::remove_if(std::begin(similarArtistIds), std::end(similarArtistIds), + [&](Database::IdType artistId) { return Database::Artist::getById(session, artistId) == Database::Artist::pointer {}; }), + std::cend(similarArtistIds)); + } + + return similarArtistIds; } FeaturesClassifierCache @@ -283,8 +319,14 @@ FeaturesClassifier::toCache() const } bool -FeaturesClassifier::init(Database::Session& session) +FeaturesClassifier::init(Database::Session& session, bool databaseChanged) { + if (databaseChanged) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Database changed: invidating cache"; + FeaturesClassifierCache::invalidate(); + } + std::optional cache {FeaturesClassifierCache::read()}; if (cache) return initFromCache(session, *cache); diff --git a/src/libs/recommendation/impl/features/FeaturesClassifier.hpp b/src/libs/recommendation/impl/features/FeaturesClassifier.hpp index d4856a96..ecf3b947 100644 --- a/src/libs/recommendation/impl/features/FeaturesClassifier.hpp +++ b/src/libs/recommendation/impl/features/FeaturesClassifier.hpp @@ -57,7 +57,7 @@ class FeaturesClassifier : public IClassifier std::string_view getName() const { return "Features"; } - bool init(Database::Session& session) override; + bool init(Database::Session& session, bool databaseChanged) override; void requestCancelInit() override; std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override; diff --git a/src/libs/recommendation/include/recommendation/IClassifier.hpp b/src/libs/recommendation/include/recommendation/IClassifier.hpp index e1c01f2f..44fa47d5 100644 --- a/src/libs/recommendation/include/recommendation/IClassifier.hpp +++ b/src/libs/recommendation/include/recommendation/IClassifier.hpp @@ -19,7 +19,7 @@ #pragma once -#include +#include #include #include @@ -40,7 +40,7 @@ namespace Recommendation virtual std::string_view getName() const = 0; - virtual bool init(Database::Session& session) = 0; + virtual bool init(Database::Session& session, bool databaseChanged) = 0; virtual void requestCancelInit() = 0; virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0; diff --git a/src/libs/scanner/impl/MediaScanner.cpp b/src/libs/scanner/impl/MediaScanner.cpp index 9de04c5b..e70b1c43 100644 --- a/src/libs/scanner/impl/MediaScanner.cpp +++ b/src/libs/scanner/impl/MediaScanner.cpp @@ -360,9 +360,7 @@ MediaScanner::countAllFiles(ScanStats& stats) if (!ec && isFileSupported(path, _fileExtensions)) { stats.filesToScan++; - - if (stats.filesToScan % 250 == 0) - notifyInProgressIfNeeded(stats); + notifyInProgressIfNeeded(stats); } return true; @@ -822,6 +820,8 @@ MediaScanner::removeMissingTracks(ScanStats& stats) stats.deletions++; } } + + notifyInProgressIfNeeded(stats); } } diff --git a/src/lms/ui/explore/ArtistView.cpp b/src/lms/ui/explore/ArtistView.cpp index 089f8714..9bcaa9de 100644 --- a/src/lms/ui/explore/ArtistView.cpp +++ b/src/lms/ui/explore/ArtistView.cpp @@ -140,11 +140,6 @@ Artist::refresh() entry->bindWidget("name", LmsApplication::createReleaseAnchor(release)); auto artists {release->getReleaseArtists()}; - LMS_LOG(UI, DEBUG) << "Found " << artists.size() << " release artists"; - - for (auto artist : artists) - LMS_LOG(UI, DEBUG) << "\tArtist = '" << artist->getName() << "'"; - if (artists.empty()) artists = release->getArtists();