From 636b70450bcfa1c8175e1e180baa271a8b5cd7e7 Mon Sep 17 00:00:00 2001 From: emeric Date: Fri, 10 Nov 2023 20:59:25 +0100 Subject: [PATCH] Simplified the similarity engine loading (since there is no more the features-based engine) --- .../impl/RecommendationService.cpp | 283 ++----- .../impl/RecommendationService.hpp | 72 +- .../recommendation/IRecommendationService.hpp | 4 +- .../services/scanner/impl/ScannerService.cpp | 737 +++++++++--------- .../services/scanner/impl/ScannerService.hpp | 96 ++- .../services/scanner/impl/ScannerSettings.hpp | 2 - .../services/scanner/IScannerService.hpp | 7 +- src/lms/main.cpp | 2 +- src/lms/ui/admin/DatabaseSettingsView.cpp | 2 + .../recommendation/LmsRecommendation.cpp | 10 +- 10 files changed, 504 insertions(+), 711 deletions(-) diff --git a/src/libs/services/recommendation/impl/RecommendationService.cpp b/src/libs/services/recommendation/impl/RecommendationService.cpp index 4ec0ac75..02e77870 100644 --- a/src/libs/services/recommendation/impl/RecommendationService.cpp +++ b/src/libs/services/recommendation/impl/RecommendationService.cpp @@ -33,240 +33,91 @@ namespace Recommendation { + namespace + { + Database::ScanSettings::SimilarityEngineType getSimilarityEngineType(Database::Session& session) + { + auto transaction{ session.createSharedTransaction() }; - static - std::string_view - engineTypeToString(EngineType engineType) - { - switch (engineType) - { - case EngineType::Clusters: return "clusters"; - case EngineType::Features: return "features"; - } + return Database::ScanSettings::get(session)->getSimilarityEngineType(); + } + } - throw LmsException {"Internal error"}; - } + std::unique_ptr createRecommendationService(Database::Db& db) + { + return std::make_unique(db); + } - std::unique_ptr - createRecommendationService(Database::Db& db) - { - return std::make_unique(db); - } + RecommendationService::RecommendationService(Database::Db& db) + : _db{ db } + { + load(); + } - RecommendationService::RecommendationService(Database::Db& db) - : _db {db} - { - } + TrackContainer RecommendationService::findSimilarTracks(Database::TrackListId trackListId, std::size_t maxCount) const + { + TrackContainer res; - TrackContainer - RecommendationService::findSimilarTracks(Database::TrackListId trackListId, std::size_t maxCount) const - { - TrackContainer res; + if (!_engine) + return res; - std::shared_lock lock {_enginesMutex}; - for (const auto& engineType : _enginePriorities) - { - auto itEngine {_engines.find(engineType)}; - if (itEngine == std::cend(_engines)) - continue; + return _engine->findSimilarTracksFromTrackList(trackListId, maxCount); + } - res = itEngine->second->findSimilarTracksFromTrackList(trackListId, maxCount); - if (!res.empty()) - break; - } + TrackContainer RecommendationService::findSimilarTracks(const std::vector& trackIds, std::size_t maxCount) const + { + TrackContainer res; - return res; - } + if (!_engine) + return res; - TrackContainer - RecommendationService::findSimilarTracks(const std::vector& trackIds, std::size_t maxCount) const - { - TrackContainer res; + return _engine->findSimilarTracks(trackIds, maxCount); + } - std::shared_lock lock {_enginesMutex}; - for (EngineType engineType : _enginePriorities) - { - auto itEngine {_engines.find(engineType)}; - if (itEngine == std::cend(_engines)) - continue; + ReleaseContainer RecommendationService::getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const + { + ReleaseContainer res; - LMS_LOG(RECOMMENDATION, DEBUG) << "Trying engine '" << engineTypeToString(engineType) << "' to get similar tracks"; + if (!_engine) + return res; - const IEngine& engine {*itEngine->second}; - res = engine.findSimilarTracks(trackIds, maxCount); - if (!res.empty()) - { - LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using engine '" << engineTypeToString(engineType) << "'"; - break; - } - } + return _engine->getSimilarReleases(releaseId, maxCount);; + } - return res; - } + ArtistContainer RecommendationService::getSimilarArtists(Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) const + { + ArtistContainer res; - ReleaseContainer - RecommendationService::getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const - { - ReleaseContainer res; + if (!_engine) + return res; - std::shared_lock lock {_enginesMutex}; - for (EngineType engineType : _enginePriorities) - { - auto itEngine {_engines.find(engineType)}; - if (itEngine == std::cend(_engines)) - continue; + return _engine->getSimilarArtists(artistId, linkTypes, maxCount); - LMS_LOG(RECOMMENDATION, DEBUG) << "Trying engine '" << engineTypeToString(engineType) << "' to get similar releases"; + return res; + } - const IEngine& engine {*itEngine->second}; - res = engine.getSimilarReleases(releaseId, maxCount); - if (!res.empty()) - { - LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using engine '" << engineTypeToString(engineType) << "'"; - break; - } + void RecommendationService::load() + { + using namespace Database; - LMS_LOG(RECOMMENDATION, DEBUG) << "No result using engine '" << engineTypeToString(engineType) << "'"; - } + switch (getSimilarityEngineType(_db.getTLSSession())) + { + case ScanSettings::SimilarityEngineType::Clusters: + if (_engineType != EngineType::Clusters) + { + _engineType = EngineType::Clusters; + _engine = createClustersEngine(_db); + } + break; - return res; - } - - ArtistContainer - RecommendationService::getSimilarArtists(Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) const - { - ArtistContainer res; - - std::shared_lock lock {_enginesMutex}; - for (EngineType engineType : _enginePriorities) - { - auto itEngine {_engines.find(engineType)}; - if (itEngine == std::cend(_engines)) - continue; - - LMS_LOG(RECOMMENDATION, DEBUG) << "Trying engine '" << engineTypeToString(engineType) << "' to get similar artists"; - - const IEngine& engine {*itEngine->second}; - res = engine.getSimilarArtists(artistId, linkTypes, maxCount); - if (!res.empty()) - { - LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using engine '" << engineTypeToString(engineType) << "'"; - return res; - } - } - - return res; - } - - static - Database::ScanSettings::SimilarityEngineType - getSimilarityEngineType(Database::Session& session) - { - auto transaction {session.createSharedTransaction()}; - - return Database::ScanSettings::get(session)->getSimilarityEngineType(); - } - - void - RecommendationService::load(bool forceReload, const ProgressCallback& progressCallback) - { - using namespace Database; - - LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines..."; - - EngineContainer enginesToLoad; - - { - std::unique_lock controlLock {_controlMutex}; - - { - std::unique_lock lock {_enginesMutex}; - _engines.clear(); - } - - switch (getSimilarityEngineType(_db.getTLSSession())) - { - case ScanSettings::SimilarityEngineType::Clusters: - _enginePriorities = {EngineType::Clusters}; - enginesToLoad.try_emplace(EngineType::Clusters, createClustersEngine(_db)); - break; - - case ScanSettings::SimilarityEngineType::Features: - _enginePriorities = {EngineType::Features, EngineType::Clusters}; - - // not same order since clusters is faster to load - enginesToLoad.try_emplace(EngineType::Clusters, createClustersEngine(_db)); - enginesToLoad.try_emplace(EngineType::Features, createFeaturesEngine(_db)); - break; - - case ScanSettings::SimilarityEngineType::None: - _enginePriorities.clear(); - break; - } - - assert(_pendingEngines.empty()); - for (auto& [engineType, engine] : enginesToLoad) - _pendingEngines.push_back(engine.get()); - } - - for (auto& [engineType, engine] : enginesToLoad) - loadPendingEngine(engineType, std::move(engine), forceReload, progressCallback); - - _pendingEnginesCondvar.notify_all(); - - LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines loaded!"; - } - - void - RecommendationService::loadPendingEngine(EngineType engineType, std::unique_ptr engine, bool forceReload, const ProgressCallback& progressCallback) - { - if (!_loadCancelled) - { - LMS_LOG(RECOMMENDATION, INFO) << "Initializing engine '" << engineTypeToString(engineType) << "'..."; - - auto progress {[&](const Progress& progress) - { - progressCallback(progress); - }}; - - engine->load(forceReload, progressCallback ? progress : ProgressCallback {}); - - LMS_LOG(RECOMMENDATION, INFO) << "Initializing engine '" << engineTypeToString(engineType) << "': " << (_loadCancelled ? "aborted" : "complete"); - } - - { - std::scoped_lock lock {_controlMutex}; - _pendingEngines.erase(std::find(std::begin(_pendingEngines), std::end(_pendingEngines), engine.get())); - } - - if (!_loadCancelled) - { - std::unique_lock lock {_enginesMutex}; - _engines.emplace(engineType, std::move(engine)); - } - } - - void - RecommendationService::cancelLoad() - { - LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading..."; - - std::unique_lock controlLock {_controlMutex}; - - assert(!_loadCancelled); - _loadCancelled = true; - - LMS_LOG(RECOMMENDATION, DEBUG) << "Still " << _pendingEngines.size() << " pending engines!"; - - for (IEngine* engine : _pendingEngines) - { - engine->requestCancelLoad(); - } - - _pendingEnginesCondvar.wait(controlLock, [this] {return _pendingEngines.empty();}); - _loadCancelled = false; - - LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading DONE"; - } + case ScanSettings::SimilarityEngineType::Features: + case ScanSettings::SimilarityEngineType::None: + _engineType.reset(); + _engine.reset(); + break; + } + if (_engine) + _engine->load(false); + } } // ns Similarity diff --git a/src/libs/services/recommendation/impl/RecommendationService.hpp b/src/libs/services/recommendation/impl/RecommendationService.hpp index 11b10cfc..a27956e2 100644 --- a/src/libs/services/recommendation/impl/RecommendationService.hpp +++ b/src/libs/services/recommendation/impl/RecommendationService.hpp @@ -19,67 +19,49 @@ #pragma once -#include -#include -#include -#include -#include +#include #include "services/recommendation/IRecommendationService.hpp" #include "IEngine.hpp" namespace Database { - class Db; + class Db; } namespace Recommendation { - enum class EngineType - { - Clusters, - Features, - }; + enum class EngineType + { + Clusters, + Features, + }; - class RecommendationService : public IRecommendationService - { - public: - RecommendationService(Database::Db& db); - ~RecommendationService() = default; + class RecommendationService : public IRecommendationService + { + public: + RecommendationService(Database::Db& db); + ~RecommendationService() = default; - RecommendationService(const RecommendationService&) = delete; - RecommendationService(RecommendationService&&) = delete; - RecommendationService& operator=(const RecommendationService&) = delete; - RecommendationService& operator=(RecommendationService&&) = delete; + RecommendationService(const RecommendationService&) = delete; + RecommendationService& operator=(const RecommendationService&) = delete; - private: - void load(bool forceReload, const ProgressCallback& progressCallback) override; - void cancelLoad() override; + private: + void load() override; - TrackContainer findSimilarTracks(Database::TrackListId tracklistId, std::size_t maxCount) const override; - TrackContainer findSimilarTracks(const std::vector& tracksId, std::size_t maxCount) const override; - ReleaseContainer getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const override; - ArtistContainer getSimilarArtists(Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) const override; + TrackContainer findSimilarTracks(Database::TrackListId tracklistId, std::size_t maxCount) const override; + TrackContainer findSimilarTracks(const std::vector& tracksId, std::size_t maxCount) const override; + ReleaseContainer getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const override; + ArtistContainer getSimilarArtists(Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) const override; - void setEnginePriorities(const std::vector& engineTypes); - void clearEngines(); - void loadPendingEngine(EngineType engineType, std::unique_ptr engine, bool forceReload, const ProgressCallback& progressCallback); + void setEnginePriorities(const std::vector& engineTypes); + void clearEngines(); + void loadPendingEngine(EngineType engineType, std::unique_ptr engine, bool forceReload, const ProgressCallback& progressCallback); - Database::Db& _db; - - std::mutex _controlMutex; - bool _loadCancelled {}; - - using EngineContainer = std::unordered_map>; - EngineContainer _engines; - mutable std::shared_mutex _enginesMutex; - - std::vector _pendingEngines; - std::shared_mutex _pendingEnginesMutex; - std::condition_variable _pendingEnginesCondvar; - - std::vector _enginePriorities; // ordered by priority - }; + Database::Db& _db; + std::optional _engineType; + std::unique_ptr _engine; + }; } // ns Recommendation diff --git a/src/libs/services/recommendation/include/services/recommendation/IRecommendationService.hpp b/src/libs/services/recommendation/include/services/recommendation/IRecommendationService.hpp index f38f7bd0..7643d775 100644 --- a/src/libs/services/recommendation/include/services/recommendation/IRecommendationService.hpp +++ b/src/libs/services/recommendation/include/services/recommendation/IRecommendationService.hpp @@ -20,6 +20,7 @@ #pragma once #include +#include #include "utils/EnumSet.hpp" #include "services/database/TrackListId.hpp" #include "services/database/Types.hpp" @@ -37,8 +38,7 @@ namespace Recommendation public: virtual ~IRecommendationService() = default; - virtual void load(bool forceReload, const ProgressCallback& progressCallback = {}) = 0; - virtual void cancelLoad() = 0; // wait for cancel done + virtual void load() = 0; virtual TrackContainer findSimilarTracks(Database::TrackListId tracklistId, std::size_t maxCount) const = 0; virtual TrackContainer findSimilarTracks(const std::vector& tracksId, std::size_t maxCount) const = 0; diff --git a/src/libs/services/scanner/impl/ScannerService.cpp b/src/libs/services/scanner/impl/ScannerService.cpp index 81907181..17eba786 100644 --- a/src/libs/services/scanner/impl/ScannerService.cpp +++ b/src/libs/services/scanner/impl/ScannerService.cpp @@ -25,7 +25,6 @@ #include "services/database/Cluster.hpp" #include "services/database/TrackFeatures.hpp" #include "services/database/ScanSettings.hpp" -#include "services/recommendation/IRecommendationService.hpp" #include "utils/Exception.hpp" #include "utils/IConfig.hpp" #include "utils/Logger.hpp" @@ -38,387 +37,361 @@ #include "ScanStepScanFiles.hpp" #include "ScanStepComputeClusterStats.hpp" -using namespace Database; - -namespace { - -Wt::WDate -getNextMonday(Wt::WDate current) +namespace Scanner { - do - { - current = current.addDays(1); - } while (current.dayOfWeek() != 1); - - return current; -} - -Wt::WDate -getNextFirstOfMonth(Wt::WDate current) -{ - do - { - current = current.addDays(1); - } while (current.day() != 1); - - return current; -} - -} // namespace - -namespace Scanner { - -std::unique_ptr -createScannerService(Db& db, Recommendation::IRecommendationService& recommendationService) -{ - return std::make_unique(db, recommendationService); -} - -ScannerService::ScannerService(Db& db, Recommendation::IRecommendationService& recommendationService) -: _recommendationService {recommendationService} -, _db {db} -, _dbSession {db} -{ - _ioService.setThreadCount(1); - - refreshScanSettings(); - - start(); -} - -ScannerService::~ScannerService() -{ - LMS_LOG(DBUPDATER, INFO) << "Stopping service..."; - stop(); - LMS_LOG(DBUPDATER, INFO) << "Service stopped!"; -} - -void -ScannerService::start() -{ - std::scoped_lock lock {_controlMutex}; - - _ioService.post([this] - { - if (_abortScan) - return; - - _recommendationService.load(false, - [](const Recommendation::Progress& progress) - { - LMS_LOG(DBUPDATER, DEBUG) << "Reloading recommendation : " << progress.processedElems << "/" << progress.totalElems; - }); - scheduleNextScan(); - }); - - _ioService.start(); -} - -void -ScannerService::stop() -{ - std::scoped_lock lock {_controlMutex}; - - _abortScan = true; - _scheduleTimer.cancel(); - _recommendationService.cancelLoad(); - _ioService.stop(); -} - -void -ScannerService::abortScan() -{ - LMS_LOG(DBUPDATER, DEBUG) << "Aborting scan..."; - std::scoped_lock lock {_controlMutex}; - - LMS_LOG(DBUPDATER, DEBUG) << "Waiting for the scan to abort..."; - - _abortScan = true; - _scheduleTimer.cancel(); - _recommendationService.cancelLoad(); - _ioService.stop(); - LMS_LOG(DBUPDATER, DEBUG) << "Scan abort done!"; - - _abortScan = false; - _ioService.start(); -} - -void -ScannerService::requestImmediateScan(bool force) -{ - abortScan(); - _ioService.post([=]() - { - if (_abortScan) - return; - - scheduleScan(force); - }); -} - -void -ScannerService::requestReload() -{ - abortScan(); - _ioService.post([=]() - { - if (_abortScan) - return; - - scheduleNextScan(); - }); -} - -ScannerService::Status -ScannerService::getStatus() const -{ - Status res; - - std::shared_lock lock {_statusMutex}; - - res.currentState = _curState; - res.nextScheduledScan = _nextScheduledScan; - res.lastCompleteScanStats = _lastCompleteScanStats; - res.currentScanStepStats = _currentScanStepStats; - - return res; -} - -void -ScannerService::scheduleNextScan() -{ - LMS_LOG(DBUPDATER, DEBUG) << "Scheduling next scan"; - - refreshScanSettings(); - - const Wt::WDateTime now {Wt::WDateTime::currentDateTime()}; - - Wt::WDateTime nextScanDateTime; - switch (_settings.updatePeriod) - { - case ScanSettings::UpdatePeriod::Daily: - if (now.time() < _settings.startTime) - nextScanDateTime = {now.date(), _settings.startTime}; - else - nextScanDateTime = {now.date().addDays(1), _settings.startTime}; - break; - - case ScanSettings::UpdatePeriod::Weekly: - if (now.time() < _settings.startTime && now.date().dayOfWeek() == 1) - nextScanDateTime = {now.date(), _settings.startTime}; - else - nextScanDateTime = {getNextMonday(now.date()), _settings.startTime}; - break; - - case ScanSettings::UpdatePeriod::Monthly: - if (now.time() < _settings.startTime && now.date().day() == 1) - nextScanDateTime = {now.date(), _settings.startTime}; - else - nextScanDateTime = {getNextFirstOfMonth(now.date()), _settings.startTime}; - break; - - case ScanSettings::UpdatePeriod::Hourly: - nextScanDateTime = {now.date(), now.time().addSecs(3600)}; - break; - - case ScanSettings::UpdatePeriod::Never: - LMS_LOG(DBUPDATER, INFO) << "Auto scan disabled!"; - break; - } - - if (nextScanDateTime.isValid()) - scheduleScan(false, nextScanDateTime); - - { - std::unique_lock lock {_statusMutex}; - _curState = nextScanDateTime.isValid() ? State::Scheduled : State::NotScheduled; - _nextScheduledScan = nextScanDateTime; - } - - _events.scanScheduled.emit(_nextScheduledScan); -} - -void -ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime) -{ - auto cb {[=](boost::system::error_code ec) - { - if (ec) - return; - - scan(force); - }}; - - if (dateTime.isNull()) - { - LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan right now"; - _scheduleTimer.expires_from_now(std::chrono::seconds {0}); - _scheduleTimer.async_wait(cb); - } - else - { - std::chrono::system_clock::time_point timePoint {dateTime.toTimePoint()}; - std::time_t t {std::chrono::system_clock::to_time_t(timePoint)}; - char ctimeStr[26]; - - LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan at " << std::string(::ctime_r(&t, ctimeStr)); - _scheduleTimer.expires_at(timePoint); - _scheduleTimer.async_wait(cb); - } -} - -void -ScannerService::scan(bool forceScan) -{ - _events.scanStarted.emit(); - - { - std::unique_lock lock {_statusMutex}; - _curState = State::InProgress; - _nextScheduledScan = {}; - } - - - LMS_LOG(UI, INFO) << "New scan started!"; - - refreshScanSettings(); - - IScanStep::ScanContext scanContext {_settings.mediaDirectory, forceScan, ScanStats {}, ScanStepStats {}}; - ScanStats& stats {scanContext.stats}; - stats.startTime = Wt::WDateTime::currentDateTime(); - - for (auto& scanStep : _scanSteps) - { - LMS_LOG(DBUPDATER, DEBUG) << "Starting scan step '" << scanStep->getStepName() << "'"; - scanContext.currentStepStats = ScanStepStats {Wt::WDateTime::currentDateTime(), scanStep->getStep()}; - - notifyInProgress(scanContext.currentStepStats); - scanStep->process(scanContext); - notifyInProgress(scanContext.currentStepStats); - LMS_LOG(DBUPDATER, DEBUG) << "Completed scan step '" << scanStep->getStepName() << "'"; - } - - LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size(); - - _dbSession.analyze(); - - if (!_abortScan) - { - stats.stopTime = Wt::WDateTime::currentDateTime(); - { - std::unique_lock lock {_statusMutex}; - - _lastCompleteScanStats = stats; - _currentScanStepStats.reset(); - } - - LMS_LOG(DBUPDATER, DEBUG) << "Scan not aborted, scheduling next scan!"; - scheduleNextScan(); - - _events.scanComplete.emit(stats); - } - else - { - LMS_LOG(DBUPDATER, DEBUG) << "Scan aborted, not scheduling next scan!"; - - std::unique_lock lock {_statusMutex}; - - _curState = State::NotScheduled; - _currentScanStepStats.reset(); - } -} - -void -ScannerService::refreshScanSettings() -{ - ScannerSettings newSettings {readSettings()}; - if (_settings == newSettings) - return; - - LMS_LOG(DBUPDATER, DEBUG) << "Scanner settings updated"; - LMS_LOG(DBUPDATER, DEBUG) << "skipDuplicateMBID = " << newSettings.skipDuplicateMBID; - LMS_LOG(DBUPDATER, DEBUG) << "Using scan settings version " << newSettings.scanVersion; - - _settings = std::move(newSettings); - - auto cbFunc {[this](const ScanStepStats& stats) - { - notifyInProgressIfNeeded(stats); - }}; - - ScanStepBase::InitParams params - { - _settings, - cbFunc, - _abortScan, - _db - }; - - _scanSteps.clear(); - _scanSteps.push_back(std::make_unique(params)); - _scanSteps.push_back(std::make_unique(params)); - _scanSteps.push_back(std::make_unique(params)); - _scanSteps.push_back(std::make_unique(params)); - _scanSteps.push_back(std::make_unique(params)); -} - -ScannerSettings -ScannerService::readSettings() -{ - ScannerSettings newSettings; - - newSettings.skipDuplicateMBID = Service::get()->getBool("scanner-skip-duplicate-mbid", false); - { - auto transaction {_dbSession.createSharedTransaction()}; - - const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)}; - - newSettings.scanVersion = scanSettings->getScanVersion(); - newSettings.startTime = scanSettings->getUpdateStartTime(); - newSettings.updatePeriod = scanSettings->getUpdatePeriod(); - - { - const auto fileExtensions {scanSettings->getAudioFileExtensions()}; - newSettings.supportedExtensions.reserve(fileExtensions.size()); - std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(newSettings.supportedExtensions), - [](const std::filesystem::path& extension) { return std::filesystem::path{ StringUtils::stringToLower(extension.string()) }; }); - } - newSettings.similarityServiceType = scanSettings->getSimilarityEngineType(); - newSettings.mediaDirectory = scanSettings->getMediaDirectory(); - - const auto clusterTypes = scanSettings->getClusterTypes(); - std::set clusterTypeNames; - - std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes), - std::inserter(clusterTypeNames, clusterTypeNames.begin()), - [](ClusterType::pointer clusterType) { return clusterType->getName(); }); - - newSettings.clusterTypeNames = std::move(clusterTypeNames); - } - - return newSettings; -} - -void -ScannerService::notifyInProgress(const ScanStepStats& stepStats) -{ - { - std::unique_lock lock {_statusMutex}; - _currentScanStepStats = stepStats; - } - - const std::chrono::system_clock::time_point now {std::chrono::system_clock::now()}; - _events.scanInProgress(stepStats); - _lastScanInProgressEmit = now; -} - -void -ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats) -{ - std::chrono::system_clock::time_point now {std::chrono::system_clock::now()}; - - if (std::chrono::duration_cast(now - _lastScanInProgressEmit).count() > 1) - notifyInProgress(stepStats); -} + using namespace Database; + + namespace + { + Wt::WDate getNextMonday(Wt::WDate current) + { + do + { + current = current.addDays(1); + } while (current.dayOfWeek() != 1); + + return current; + } + + Wt::WDate getNextFirstOfMonth(Wt::WDate current) + { + do + { + current = current.addDays(1); + } while (current.day() != 1); + + return current; + } + } // namespace + + std::unique_ptr createScannerService(Db& db) + { + return std::make_unique(db); + } + + ScannerService::ScannerService(Db& db) + : _db{ db } + , _dbSession{ db } + { + _ioService.setThreadCount(1); + + refreshScanSettings(); + + start(); + } + + ScannerService::~ScannerService() + { + LMS_LOG(DBUPDATER, INFO) << "Stopping service..."; + stop(); + LMS_LOG(DBUPDATER, INFO) << "Service stopped!"; + } + + void ScannerService::start() + { + std::scoped_lock lock{ _controlMutex }; + + _ioService.post([this] + { + if (_abortScan) + return; + + scheduleNextScan(); + }); + + _ioService.start(); + } + + void ScannerService::stop() + { + std::scoped_lock lock{ _controlMutex }; + + _abortScan = true; + _scheduleTimer.cancel(); + _ioService.stop(); + } + + void ScannerService::abortScan() + { + LMS_LOG(DBUPDATER, DEBUG) << "Aborting scan..."; + std::scoped_lock lock{ _controlMutex }; + + LMS_LOG(DBUPDATER, DEBUG) << "Waiting for the scan to abort..."; + + _abortScan = true; + _scheduleTimer.cancel(); + _ioService.stop(); + LMS_LOG(DBUPDATER, DEBUG) << "Scan abort done!"; + + _abortScan = false; + _ioService.start(); + } + + void ScannerService::requestImmediateScan(bool force) + { + abortScan(); + _ioService.post([=]() + { + if (_abortScan) + return; + + scheduleScan(force); + }); + } + + void ScannerService::requestReload() + { + abortScan(); + _ioService.post([=]() + { + if (_abortScan) + return; + + scheduleNextScan(); + }); + } + + ScannerService::Status ScannerService::getStatus() const + { + Status res; + + std::shared_lock lock{ _statusMutex }; + + res.currentState = _curState; + res.nextScheduledScan = _nextScheduledScan; + res.lastCompleteScanStats = _lastCompleteScanStats; + res.currentScanStepStats = _currentScanStepStats; + + return res; + } + + void ScannerService::scheduleNextScan() + { + LMS_LOG(DBUPDATER, DEBUG) << "Scheduling next scan"; + + refreshScanSettings(); + + const Wt::WDateTime now{ Wt::WDateTime::currentDateTime() }; + + Wt::WDateTime nextScanDateTime; + switch (_settings.updatePeriod) + { + case ScanSettings::UpdatePeriod::Daily: + if (now.time() < _settings.startTime) + nextScanDateTime = { now.date(), _settings.startTime }; + else + nextScanDateTime = { now.date().addDays(1), _settings.startTime }; + break; + + case ScanSettings::UpdatePeriod::Weekly: + if (now.time() < _settings.startTime && now.date().dayOfWeek() == 1) + nextScanDateTime = { now.date(), _settings.startTime }; + else + nextScanDateTime = { getNextMonday(now.date()), _settings.startTime }; + break; + + case ScanSettings::UpdatePeriod::Monthly: + if (now.time() < _settings.startTime && now.date().day() == 1) + nextScanDateTime = { now.date(), _settings.startTime }; + else + nextScanDateTime = { getNextFirstOfMonth(now.date()), _settings.startTime }; + break; + + case ScanSettings::UpdatePeriod::Hourly: + nextScanDateTime = { now.date(), now.time().addSecs(3600) }; + break; + + case ScanSettings::UpdatePeriod::Never: + LMS_LOG(DBUPDATER, INFO) << "Auto scan disabled!"; + break; + } + + if (nextScanDateTime.isValid()) + scheduleScan(false, nextScanDateTime); + + { + std::unique_lock lock{ _statusMutex }; + _curState = nextScanDateTime.isValid() ? State::Scheduled : State::NotScheduled; + _nextScheduledScan = nextScanDateTime; + } + + _events.scanScheduled.emit(_nextScheduledScan); + } + + void ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime) + { + auto cb{ [=](boost::system::error_code ec) + { + if (ec) + return; + + scan(force); + } }; + + if (dateTime.isNull()) + { + LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan right now"; + _scheduleTimer.expires_from_now(std::chrono::seconds{ 0 }); + _scheduleTimer.async_wait(cb); + } + else + { + std::chrono::system_clock::time_point timePoint{ dateTime.toTimePoint() }; + std::time_t t{ std::chrono::system_clock::to_time_t(timePoint) }; + char ctimeStr[26]; + + LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan at " << std::string(::ctime_r(&t, ctimeStr)); + _scheduleTimer.expires_at(timePoint); + _scheduleTimer.async_wait(cb); + } + } + + void ScannerService::scan(bool forceScan) + { + _events.scanStarted.emit(); + + { + std::unique_lock lock{ _statusMutex }; + _curState = State::InProgress; + _nextScheduledScan = {}; + } + + + LMS_LOG(UI, INFO) << "New scan started!"; + + refreshScanSettings(); + + IScanStep::ScanContext scanContext{ _settings.mediaDirectory, forceScan, ScanStats {}, ScanStepStats {} }; + ScanStats& stats{ scanContext.stats }; + stats.startTime = Wt::WDateTime::currentDateTime(); + + for (auto& scanStep : _scanSteps) + { + LMS_LOG(DBUPDATER, DEBUG) << "Starting scan step '" << scanStep->getStepName() << "'"; + scanContext.currentStepStats = ScanStepStats{ Wt::WDateTime::currentDateTime(), scanStep->getStep() }; + + notifyInProgress(scanContext.currentStepStats); + scanStep->process(scanContext); + notifyInProgress(scanContext.currentStepStats); + LMS_LOG(DBUPDATER, DEBUG) << "Completed scan step '" << scanStep->getStepName() << "'"; + } + + LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size(); + + _dbSession.analyze(); + + if (!_abortScan) + { + stats.stopTime = Wt::WDateTime::currentDateTime(); + { + std::unique_lock lock{ _statusMutex }; + + _lastCompleteScanStats = stats; + _currentScanStepStats.reset(); + } + + LMS_LOG(DBUPDATER, DEBUG) << "Scan not aborted, scheduling next scan!"; + scheduleNextScan(); + + _events.scanComplete.emit(stats); + } + else + { + LMS_LOG(DBUPDATER, DEBUG) << "Scan aborted, not scheduling next scan!"; + + std::unique_lock lock{ _statusMutex }; + + _curState = State::NotScheduled; + _currentScanStepStats.reset(); + } + } + + void ScannerService::refreshScanSettings() + { + ScannerSettings newSettings{ readSettings() }; + if (_settings == newSettings) + return; + + LMS_LOG(DBUPDATER, DEBUG) << "Scanner settings updated"; + LMS_LOG(DBUPDATER, DEBUG) << "skipDuplicateMBID = " << newSettings.skipDuplicateMBID; + LMS_LOG(DBUPDATER, DEBUG) << "Using scan settings version " << newSettings.scanVersion; + + _settings = std::move(newSettings); + + auto cbFunc{ [this](const ScanStepStats& stats) + { + notifyInProgressIfNeeded(stats); + } }; + + ScanStepBase::InitParams params + { + _settings, + cbFunc, + _abortScan, + _db + }; + + _scanSteps.clear(); + _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); + } + + ScannerSettings ScannerService::readSettings() + { + ScannerSettings newSettings; + + newSettings.skipDuplicateMBID = Service::get()->getBool("scanner-skip-duplicate-mbid", false); + { + auto transaction{ _dbSession.createSharedTransaction() }; + + const ScanSettings::pointer scanSettings{ ScanSettings::get(_dbSession) }; + + newSettings.scanVersion = scanSettings->getScanVersion(); + newSettings.startTime = scanSettings->getUpdateStartTime(); + newSettings.updatePeriod = scanSettings->getUpdatePeriod(); + + { + const auto fileExtensions{ scanSettings->getAudioFileExtensions() }; + newSettings.supportedExtensions.reserve(fileExtensions.size()); + std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(newSettings.supportedExtensions), + [](const std::filesystem::path& extension) { return std::filesystem::path{ StringUtils::stringToLower(extension.string()) }; }); + } + newSettings.mediaDirectory = scanSettings->getMediaDirectory(); + + const auto clusterTypes = scanSettings->getClusterTypes(); + std::set clusterTypeNames; + + std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes), + std::inserter(clusterTypeNames, clusterTypeNames.begin()), + [](ClusterType::pointer clusterType) { return clusterType->getName(); }); + + newSettings.clusterTypeNames = std::move(clusterTypeNames); + } + + return newSettings; + } + + void ScannerService::notifyInProgress(const ScanStepStats& stepStats) + { + { + std::unique_lock lock{ _statusMutex }; + _currentScanStepStats = stepStats; + } + + const std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() }; + _events.scanInProgress(stepStats); + _lastScanInProgressEmit = now; + } + + void ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats) + { + std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() }; + + if (std::chrono::duration_cast(now - _lastScanInProgressEmit).count() > 1) + notifyInProgress(stepStats); + } } // namespace Scanner diff --git a/src/libs/services/scanner/impl/ScannerService.hpp b/src/libs/services/scanner/impl/ScannerService.hpp index 133dfef2..02493a5c 100644 --- a/src/libs/services/scanner/impl/ScannerService.hpp +++ b/src/libs/services/scanner/impl/ScannerService.hpp @@ -38,73 +38,65 @@ #include "IScanStep.hpp" #include "ScannerSettings.hpp" -namespace Recommendation -{ - class IRecommendationService; -} - namespace Scanner { - class ScannerService : public IScannerService - { - public: - ScannerService(Database::Db& db, Recommendation::IRecommendationService& recommendationService); - ~ScannerService(); + class ScannerService : public IScannerService + { + public: + ScannerService(Database::Db& db); + ~ScannerService(); - ScannerService(const ScannerService&) = delete; - ScannerService(ScannerService&&) = delete; - ScannerService& operator=(const ScannerService&) = delete; - ScannerService& operator=(ScannerService&&) = delete; + ScannerService(const ScannerService&) = delete; + ScannerService& operator=(const ScannerService&) = delete; - void requestReload() override; - void requestImmediateScan(bool force) override; + void requestReload() override; + void requestImmediateScan(bool force) override; - Status getStatus() const override; - Events& getEvents() override { return _events; } + Status getStatus() const override; + Events& getEvents() override { return _events; } - private: - void start(); - void stop(); + private: + void start(); + void stop(); - // Job handling - void scheduleNextScan(); - void scheduleScan(bool force, const Wt::WDateTime& dateTime = {}); + // Job handling + void scheduleNextScan(); + void scheduleScan(bool force, const Wt::WDateTime& dateTime = {}); - void abortScan(); + void abortScan(); - // Update database (scheduled callback) - void scan(bool force); + // Update database (scheduled callback) + void scan(bool force); - void scanMediaDirectory( const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats); + void scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats); - // Helpers - void refreshScanSettings(); - ScannerSettings readSettings(); + // Helpers + void refreshScanSettings(); + ScannerSettings readSettings(); + void reloadRecommendationService(); - void notifyInProgressIfNeeded(const ScanStepStats& stats); - void notifyInProgress(const ScanStepStats& stats); - void reloadSimilarityEngine(ScanStats& stats); + void notifyInProgressIfNeeded(const ScanStepStats& stats); + void notifyInProgress(const ScanStepStats& stats); + void reloadSimilarityEngine(ScanStats& stats); - Recommendation::IRecommendationService& _recommendationService; + std::vector> _scanSteps; - std::vector> _scanSteps; + std::mutex _controlMutex; + bool _abortScan{}; + Wt::WIOService _ioService; + boost::asio::system_timer _scheduleTimer{ _ioService }; + Events _events; + std::chrono::system_clock::time_point _lastScanInProgressEmit{}; + Database::Db& _db; + Database::Session _dbSession; - std::mutex _controlMutex; - bool _abortScan {}; - Wt::WIOService _ioService; - boost::asio::system_timer _scheduleTimer {_ioService}; - Events _events; - std::chrono::system_clock::time_point _lastScanInProgressEmit {}; - Database::Db& _db; - Database::Session _dbSession; + mutable std::shared_mutex _statusMutex; + State _curState{ State::NotScheduled }; + std::optional _lastCompleteScanStats; + std::optional _currentScanStepStats; + Wt::WDateTime _nextScheduledScan; - mutable std::shared_mutex _statusMutex; - State _curState {State::NotScheduled}; - std::optional _lastCompleteScanStats; - std::optional _currentScanStepStats; - Wt::WDateTime _nextScheduledScan; - - ScannerSettings _settings; - }; + ScannerSettings _settings; + }; } // Scanner diff --git a/src/libs/services/scanner/impl/ScannerSettings.hpp b/src/libs/services/scanner/impl/ScannerSettings.hpp index a102dea3..9a469c78 100644 --- a/src/libs/services/scanner/impl/ScannerSettings.hpp +++ b/src/libs/services/scanner/impl/ScannerSettings.hpp @@ -34,7 +34,6 @@ namespace Scanner Wt::WTime startTime; Database::ScanSettings::UpdatePeriod updatePeriod {Database::ScanSettings::UpdatePeriod::Never}; std::vector supportedExtensions; - Database::ScanSettings::SimilarityEngineType similarityServiceType; std::filesystem::path mediaDirectory; bool skipDuplicateMBID {}; std::set clusterTypeNames; @@ -45,7 +44,6 @@ namespace Scanner && startTime == rhs.startTime && updatePeriod == rhs.updatePeriod && supportedExtensions == rhs.supportedExtensions - && similarityServiceType == rhs.similarityServiceType && mediaDirectory == rhs.mediaDirectory && skipDuplicateMBID == rhs.skipDuplicateMBID && clusterTypeNames == rhs.clusterTypeNames; diff --git a/src/libs/services/scanner/include/services/scanner/IScannerService.hpp b/src/libs/services/scanner/include/services/scanner/IScannerService.hpp index 6085f6b8..77c694f1 100644 --- a/src/libs/services/scanner/include/services/scanner/IScannerService.hpp +++ b/src/libs/services/scanner/include/services/scanner/IScannerService.hpp @@ -29,11 +29,6 @@ namespace Database class Db; } -namespace Recommendation -{ - class IRecommendationService; -} - namespace Scanner { @@ -66,7 +61,7 @@ namespace Scanner virtual Events& getEvents() = 0; }; - std::unique_ptr createScannerService(Database::Db& db, Recommendation::IRecommendationService& recommendationEngine); + std::unique_ptr createScannerService(Database::Db& db); } // Scanner diff --git a/src/lms/main.cpp b/src/lms/main.cpp index 543c2475..ebef9dd4 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -270,7 +270,7 @@ int main(int argc, char* argv[]) Service coverService{ Cover::createCoverService(database, argv[0], server.appRoot() + "/images/unknown-cover.jpg") }; Service recommendationService{ Recommendation::createRecommendationService(database) }; Service playlistGeneratorService{ Recommendation::createPlaylistGeneratorService(database, *recommendationService.get()) }; - Service scannerService{ Scanner::createScannerService(database, *recommendationService) }; + Service scannerService{ Scanner::createScannerService(database) }; scannerService->getEvents().scanComplete.connect([&] { diff --git a/src/lms/ui/admin/DatabaseSettingsView.cpp b/src/lms/ui/admin/DatabaseSettingsView.cpp index 38d9877e..97db0680 100644 --- a/src/lms/ui/admin/DatabaseSettingsView.cpp +++ b/src/lms/ui/admin/DatabaseSettingsView.cpp @@ -29,6 +29,7 @@ #include "services/database/Cluster.hpp" #include "services/database/ScanSettings.hpp" #include "services/database/Session.hpp" +#include "services/recommendation/IRecommendationService.hpp" #include "services/scanner/IScannerService.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -232,6 +233,7 @@ DatabaseSettingsView::refreshView() { model->saveData(); + Service::get()->load(); Service::get()->requestImmediateScan(false); LmsApp->notifyMsg(Notification::Type::Info, Wt::WString::tr("Lms.Admin.Database.database"), Wt::WString::tr("Lms.Admin.Database.settings-saved")); } diff --git a/src/tools/recommendation/LmsRecommendation.cpp b/src/tools/recommendation/LmsRecommendation.cpp index a4a71f17..43659f0d 100644 --- a/src/tools/recommendation/LmsRecommendation.cpp +++ b/src/tools/recommendation/LmsRecommendation.cpp @@ -157,16 +157,16 @@ int main(int argc, char* argv[]) Db db{ config->getPath("working-dir") / "lms.db" }; Session session{ db }; - std::cout << "Creating recommendation recommendationService..." << std::endl; + std::cout << "Creating recommendation service..." << std::endl; const auto recommendationService{ Recommendation::createRecommendationService(db) }; - std::cout << "Recommendation recommendationService created!" << std::endl; + std::cout << "Recommendation service created!" << std::endl; - std::cout << "Loading recommendation recommendationService..." << std::endl; - recommendationService->load(false); + std::cout << "Loading recommendation service..." << std::endl; + recommendationService->load(); unsigned maxSimilarityCount{ vm["max"].as() }; - std::cout << "Recommendation recommendationService loaded!" << std::endl; + std::cout << "Recommendation service loaded!" << std::endl; if (vm.count("tracks")) dumpTracksRecommendation(db, *recommendationService, maxSimilarityCount);