owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Simplified the similarity engine loading (since there is no more the features-based engine)

Browse Source
This commit is contained in:
emeric
2023-11-10 20:59:25 +01:00
parent 8ea532fa26
commit 636b70450b
10 changed files with 504 additions and 711 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/recommendation/impl/RecommendationService.cpp
+36 -185
View File
@@ -33,240 +33,91 @@
namespace Recommendation namespace Recommendation
{ {
namespace
{
Database::ScanSettings::SimilarityEngineType getSimilarityEngineType(Database::Session& session)
{
auto transaction{ session.createSharedTransaction() };
static return Database::ScanSettings::get(session)->getSimilarityEngineType();
std::string_view }
engineTypeToString(EngineType engineType)
{
switch (engineType)
{
case EngineType::Clusters: return "clusters";
case EngineType::Features: return "features";
} }
throw LmsException {"Internal error"}; std::unique_ptr<IRecommendationService> createRecommendationService(Database::Db& db)
}
std::unique_ptr<IRecommendationService>
createRecommendationService(Database::Db& db)
{ {
return std::make_unique<RecommendationService>(db); return std::make_unique<RecommendationService>(db);
} }
RecommendationService::RecommendationService(Database::Db& db) RecommendationService::RecommendationService(Database::Db& db)
: _db {db} : _db{ db }
{ {
load();
} }
TrackContainer TrackContainer RecommendationService::findSimilarTracks(Database::TrackListId trackListId, std::size_t maxCount) const
RecommendationService::findSimilarTracks(Database::TrackListId trackListId, std::size_t maxCount) const
{ {
TrackContainer res; TrackContainer res;
std::shared_lock lock {_enginesMutex}; if (!_engine)
for (const auto& engineType : _enginePriorities)
{
auto itEngine {_engines.find(engineType)};
if (itEngine == std::cend(_engines))
continue;
res = itEngine->second->findSimilarTracksFromTrackList(trackListId, maxCount);
if (!res.empty())
break;
}
return res; return res;
return _engine->findSimilarTracksFromTrackList(trackListId, maxCount);
} }
TrackContainer TrackContainer RecommendationService::findSimilarTracks(const std::vector<Database::TrackId>& trackIds, std::size_t maxCount) const
RecommendationService::findSimilarTracks(const std::vector<Database::TrackId>& trackIds, std::size_t maxCount) const
{ {
TrackContainer res; TrackContainer res;
std::shared_lock lock {_enginesMutex}; if (!_engine)
for (EngineType engineType : _enginePriorities)
{
auto itEngine {_engines.find(engineType)};
if (itEngine == std::cend(_engines))
continue;
LMS_LOG(RECOMMENDATION, DEBUG) << "Trying engine '" << engineTypeToString(engineType) << "' to get similar tracks";
const IEngine& engine {*itEngine->second};
res = engine.findSimilarTracks(trackIds, maxCount);
if (!res.empty())
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using engine '" << engineTypeToString(engineType) << "'";
break;
}
}
return res; return res;
return _engine->findSimilarTracks(trackIds, maxCount);
} }
ReleaseContainer ReleaseContainer RecommendationService::getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const
RecommendationService::getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const
{ {
ReleaseContainer res; ReleaseContainer res;
std::shared_lock lock {_enginesMutex}; if (!_engine)
for (EngineType engineType : _enginePriorities)
{
auto itEngine {_engines.find(engineType)};
if (itEngine == std::cend(_engines))
continue;
LMS_LOG(RECOMMENDATION, DEBUG) << "Trying engine '" << engineTypeToString(engineType) << "' to get similar releases";
const IEngine& engine {*itEngine->second};
res = engine.getSimilarReleases(releaseId, maxCount);
if (!res.empty())
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using engine '" << engineTypeToString(engineType) << "'";
break;
}
LMS_LOG(RECOMMENDATION, DEBUG) << "No result using engine '" << engineTypeToString(engineType) << "'";
}
return res; return res;
return _engine->getSimilarReleases(releaseId, maxCount);;
} }
ArtistContainer ArtistContainer RecommendationService::getSimilarArtists(Database::ArtistId artistId, EnumSet<Database::TrackArtistLinkType> linkTypes, std::size_t maxCount) const
RecommendationService::getSimilarArtists(Database::ArtistId artistId, EnumSet<Database::TrackArtistLinkType> linkTypes, std::size_t maxCount) const
{ {
ArtistContainer res; ArtistContainer res;
std::shared_lock lock {_enginesMutex}; if (!_engine)
for (EngineType engineType : _enginePriorities)
{
auto itEngine {_engines.find(engineType)};
if (itEngine == std::cend(_engines))
continue;
LMS_LOG(RECOMMENDATION, DEBUG) << "Trying engine '" << engineTypeToString(engineType) << "' to get similar artists";
const IEngine& engine {*itEngine->second};
res = engine.getSimilarArtists(artistId, linkTypes, maxCount);
if (!res.empty())
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using engine '" << engineTypeToString(engineType) << "'";
return res; return res;
}
} return _engine->getSimilarArtists(artistId, linkTypes, maxCount);
return res; return res;
} }
static void RecommendationService::load()
Database::ScanSettings::SimilarityEngineType
getSimilarityEngineType(Database::Session& session)
{
auto transaction {session.createSharedTransaction()};
return Database::ScanSettings::get(session)->getSimilarityEngineType();
}
void
RecommendationService::load(bool forceReload, const ProgressCallback& progressCallback)
{ {
using namespace Database; using namespace Database;
LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines...";
EngineContainer enginesToLoad;
{
std::unique_lock controlLock {_controlMutex};
{
std::unique_lock lock {_enginesMutex};
_engines.clear();
}
switch (getSimilarityEngineType(_db.getTLSSession())) switch (getSimilarityEngineType(_db.getTLSSession()))
{ {
case ScanSettings::SimilarityEngineType::Clusters: case ScanSettings::SimilarityEngineType::Clusters:
_enginePriorities = {EngineType::Clusters}; if (_engineType != EngineType::Clusters)
enginesToLoad.try_emplace(EngineType::Clusters, createClustersEngine(_db)); {
_engineType = EngineType::Clusters;
_engine = createClustersEngine(_db);
}
break; break;
case ScanSettings::SimilarityEngineType::Features: case ScanSettings::SimilarityEngineType::Features:
_enginePriorities = {EngineType::Features, EngineType::Clusters};
// not same order since clusters is faster to load
enginesToLoad.try_emplace(EngineType::Clusters, createClustersEngine(_db));
enginesToLoad.try_emplace(EngineType::Features, createFeaturesEngine(_db));
break;
case ScanSettings::SimilarityEngineType::None: case ScanSettings::SimilarityEngineType::None:
_enginePriorities.clear(); _engineType.reset();
_engine.reset();
break; break;
} }
assert(_pendingEngines.empty()); if (_engine)
for (auto& [engineType, engine] : enginesToLoad) _engine->load(false);
_pendingEngines.push_back(engine.get());
} }
for (auto& [engineType, engine] : enginesToLoad)
loadPendingEngine(engineType, std::move(engine), forceReload, progressCallback);
_pendingEnginesCondvar.notify_all();
LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines loaded!";
}
void
RecommendationService::loadPendingEngine(EngineType engineType, std::unique_ptr<IEngine> engine, bool forceReload, const ProgressCallback& progressCallback)
{
if (!_loadCancelled)
{
LMS_LOG(RECOMMENDATION, INFO) << "Initializing engine '" << engineTypeToString(engineType) << "'...";
auto progress {[&](const Progress& progress)
{
progressCallback(progress);
}};
engine->load(forceReload, progressCallback ? progress : ProgressCallback {});
LMS_LOG(RECOMMENDATION, INFO) << "Initializing engine '" << engineTypeToString(engineType) << "': " << (_loadCancelled ? "aborted" : "complete");
}
{
std::scoped_lock lock {_controlMutex};
_pendingEngines.erase(std::find(std::begin(_pendingEngines), std::end(_pendingEngines), engine.get()));
}
if (!_loadCancelled)
{
std::unique_lock lock {_enginesMutex};
_engines.emplace(engineType, std::move(engine));
}
}
void
RecommendationService::cancelLoad()
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading...";
std::unique_lock controlLock {_controlMutex};
assert(!_loadCancelled);
_loadCancelled = true;
LMS_LOG(RECOMMENDATION, DEBUG) << "Still " << _pendingEngines.size() << " pending engines!";
for (IEngine* engine : _pendingEngines)
{
engine->requestCancelLoad();
}
_pendingEnginesCondvar.wait(controlLock, [this] {return _pendingEngines.empty();});
_loadCancelled = false;
LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading DONE";
}
} // ns Similarity } // ns Similarity
src/libs/services/recommendation/impl/RecommendationService.hpp
+4 -22
View File
@@ -19,11 +19,7 @@
#pragma once #pragma once
#include <condition_variable> #include <optional>
#include <mutex>
#include <shared_mutex>
#include <unordered_map>
#include <vector>
#include "services/recommendation/IRecommendationService.hpp" #include "services/recommendation/IRecommendationService.hpp"
#include "IEngine.hpp" #include "IEngine.hpp"
@@ -48,13 +44,10 @@ namespace Recommendation
~RecommendationService() = default; ~RecommendationService() = default;
RecommendationService(const RecommendationService&) = delete; RecommendationService(const RecommendationService&) = delete;
RecommendationService(RecommendationService&&) = delete;
RecommendationService& operator=(const RecommendationService&) = delete; RecommendationService& operator=(const RecommendationService&) = delete;
RecommendationService& operator=(RecommendationService&&) = delete;
private: private:
void load(bool forceReload, const ProgressCallback& progressCallback) override; void load() override;
void cancelLoad() override;
TrackContainer findSimilarTracks(Database::TrackListId tracklistId, std::size_t maxCount) const override; TrackContainer findSimilarTracks(Database::TrackListId tracklistId, std::size_t maxCount) const override;
TrackContainer findSimilarTracks(const std::vector<Database::TrackId>& tracksId, std::size_t maxCount) const override; TrackContainer findSimilarTracks(const std::vector<Database::TrackId>& tracksId, std::size_t maxCount) const override;
@@ -66,19 +59,8 @@ namespace Recommendation
void loadPendingEngine(EngineType engineType, std::unique_ptr<IEngine> engine, bool forceReload, const ProgressCallback& progressCallback); void loadPendingEngine(EngineType engineType, std::unique_ptr<IEngine> engine, bool forceReload, const ProgressCallback& progressCallback);
Database::Db& _db; Database::Db& _db;
std::optional<EngineType> _engineType;
std::mutex _controlMutex; std::unique_ptr<IEngine> _engine;
bool _loadCancelled {};
using EngineContainer = std::unordered_map<EngineType, std::unique_ptr<IEngine>>;
EngineContainer _engines;
mutable std::shared_mutex _enginesMutex;
std::vector<IEngine*> _pendingEngines;
std::shared_mutex _pendingEnginesMutex;
std::condition_variable _pendingEnginesCondvar;
std::vector<EngineType> _enginePriorities; // ordered by priority
}; };
} // ns Recommendation } // ns Recommendation
src/libs/services/recommendation/include/services/recommendation/IRecommendationService.hpp
+2 -2
View File
@@ -20,6 +20,7 @@
#pragma once #pragma once
#include <memory> #include <memory>
#include <vector>
#include "utils/EnumSet.hpp" #include "utils/EnumSet.hpp"
#include "services/database/TrackListId.hpp" #include "services/database/TrackListId.hpp"
#include "services/database/Types.hpp" #include "services/database/Types.hpp"
@@ -37,8 +38,7 @@ namespace Recommendation
public: public:
virtual ~IRecommendationService() = default; virtual ~IRecommendationService() = default;
virtual void load(bool forceReload, const ProgressCallback& progressCallback = {}) = 0; virtual void load() = 0;
virtual void cancelLoad() = 0; // wait for cancel done
virtual TrackContainer findSimilarTracks(Database::TrackListId tracklistId, std::size_t maxCount) const = 0; virtual TrackContainer findSimilarTracks(Database::TrackListId tracklistId, std::size_t maxCount) const = 0;
virtual TrackContainer findSimilarTracks(const std::vector<Database::TrackId>& tracksId, std::size_t maxCount) const = 0; virtual TrackContainer findSimilarTracks(const std::vector<Database::TrackId>& tracksId, std::size_t maxCount) const = 0;
src/libs/services/scanner/impl/ScannerService.cpp
+96 -123
View File
@@ -25,7 +25,6 @@
#include "services/database/Cluster.hpp" #include "services/database/Cluster.hpp"
#include "services/database/TrackFeatures.hpp" #include "services/database/TrackFeatures.hpp"
#include "services/database/ScanSettings.hpp" #include "services/database/ScanSettings.hpp"
#include "services/recommendation/IRecommendationService.hpp"
#include "utils/Exception.hpp" #include "utils/Exception.hpp"
#include "utils/IConfig.hpp" #include "utils/IConfig.hpp"
#include "utils/Logger.hpp" #include "utils/Logger.hpp"
@@ -38,114 +37,98 @@
#include "ScanStepScanFiles.hpp" #include "ScanStepScanFiles.hpp"
#include "ScanStepComputeClusterStats.hpp" #include "ScanStepComputeClusterStats.hpp"
using namespace Database; namespace Scanner
namespace {
Wt::WDate
getNextMonday(Wt::WDate current)
{ {
using namespace Database;
namespace
{
Wt::WDate getNextMonday(Wt::WDate current)
{
do do
{ {
current = current.addDays(1); current = current.addDays(1);
} while (current.dayOfWeek() != 1); } while (current.dayOfWeek() != 1);
return current; return current;
} }
Wt::WDate Wt::WDate getNextFirstOfMonth(Wt::WDate current)
getNextFirstOfMonth(Wt::WDate current) {
{
do do
{ {
current = current.addDays(1); current = current.addDays(1);
} while (current.day() != 1); } while (current.day() != 1);
return current; return current;
} }
} // namespace
} // namespace std::unique_ptr<IScannerService> createScannerService(Db& db)
{
return std::make_unique<ScannerService>(db);
}
namespace Scanner { ScannerService::ScannerService(Db& db)
: _db{ db }
std::unique_ptr<IScannerService> , _dbSession{ db }
createScannerService(Db& db, Recommendation::IRecommendationService& recommendationService) {
{
return std::make_unique<ScannerService>(db, recommendationService);
}
ScannerService::ScannerService(Db& db, Recommendation::IRecommendationService& recommendationService)
: _recommendationService {recommendationService}
, _db {db}
, _dbSession {db}
{
_ioService.setThreadCount(1); _ioService.setThreadCount(1);
refreshScanSettings(); refreshScanSettings();
start(); start();
} }
ScannerService::~ScannerService() ScannerService::~ScannerService()
{ {
LMS_LOG(DBUPDATER, INFO) << "Stopping service..."; LMS_LOG(DBUPDATER, INFO) << "Stopping service...";
stop(); stop();
LMS_LOG(DBUPDATER, INFO) << "Service stopped!"; LMS_LOG(DBUPDATER, INFO) << "Service stopped!";
} }
void void ScannerService::start()
ScannerService::start() {
{ std::scoped_lock lock{ _controlMutex };
std::scoped_lock lock {_controlMutex};
_ioService.post([this] _ioService.post([this]
{ {
if (_abortScan) if (_abortScan)
return; return;
_recommendationService.load(false,
[](const Recommendation::Progress& progress)
{
LMS_LOG(DBUPDATER, DEBUG) << "Reloading recommendation : " << progress.processedElems << "/" << progress.totalElems;
});
scheduleNextScan(); scheduleNextScan();
}); });
_ioService.start(); _ioService.start();
} }
void void ScannerService::stop()
ScannerService::stop() {
{ std::scoped_lock lock{ _controlMutex };
std::scoped_lock lock {_controlMutex};
_abortScan = true; _abortScan = true;
_scheduleTimer.cancel(); _scheduleTimer.cancel();
_recommendationService.cancelLoad();
_ioService.stop(); _ioService.stop();
} }
void void ScannerService::abortScan()
ScannerService::abortScan() {
{
LMS_LOG(DBUPDATER, DEBUG) << "Aborting scan..."; LMS_LOG(DBUPDATER, DEBUG) << "Aborting scan...";
std::scoped_lock lock {_controlMutex}; std::scoped_lock lock{ _controlMutex };
LMS_LOG(DBUPDATER, DEBUG) << "Waiting for the scan to abort..."; LMS_LOG(DBUPDATER, DEBUG) << "Waiting for the scan to abort...";
_abortScan = true; _abortScan = true;
_scheduleTimer.cancel(); _scheduleTimer.cancel();
_recommendationService.cancelLoad();
_ioService.stop(); _ioService.stop();
LMS_LOG(DBUPDATER, DEBUG) << "Scan abort done!"; LMS_LOG(DBUPDATER, DEBUG) << "Scan abort done!";
_abortScan = false; _abortScan = false;
_ioService.start(); _ioService.start();
} }
void void ScannerService::requestImmediateScan(bool force)
ScannerService::requestImmediateScan(bool force) {
{
abortScan(); abortScan();
_ioService.post([=]() _ioService.post([=]()
{ {
@@ -154,11 +137,10 @@ ScannerService::requestImmediateScan(bool force)
scheduleScan(force); scheduleScan(force);
}); });
} }
void void ScannerService::requestReload()
ScannerService::requestReload() {
{
abortScan(); abortScan();
_ioService.post([=]() _ioService.post([=]()
{ {
@@ -167,14 +149,13 @@ ScannerService::requestReload()
scheduleNextScan(); scheduleNextScan();
}); });
} }
ScannerService::Status ScannerService::Status ScannerService::getStatus() const
ScannerService::getStatus() const {
{
Status res; Status res;
std::shared_lock lock {_statusMutex}; std::shared_lock lock{ _statusMutex };
res.currentState = _curState; res.currentState = _curState;
res.nextScheduledScan = _nextScheduledScan; res.nextScheduledScan = _nextScheduledScan;
@@ -182,43 +163,42 @@ ScannerService::getStatus() const
res.currentScanStepStats = _currentScanStepStats; res.currentScanStepStats = _currentScanStepStats;
return res; return res;
} }
void void ScannerService::scheduleNextScan()
ScannerService::scheduleNextScan() {
{
LMS_LOG(DBUPDATER, DEBUG) << "Scheduling next scan"; LMS_LOG(DBUPDATER, DEBUG) << "Scheduling next scan";
refreshScanSettings(); refreshScanSettings();
const Wt::WDateTime now {Wt::WDateTime::currentDateTime()}; const Wt::WDateTime now{ Wt::WDateTime::currentDateTime() };
Wt::WDateTime nextScanDateTime; Wt::WDateTime nextScanDateTime;
switch (_settings.updatePeriod) switch (_settings.updatePeriod)
{ {
case ScanSettings::UpdatePeriod::Daily: case ScanSettings::UpdatePeriod::Daily:
if (now.time() < _settings.startTime) if (now.time() < _settings.startTime)
nextScanDateTime = {now.date(), _settings.startTime}; nextScanDateTime = { now.date(), _settings.startTime };
else else
nextScanDateTime = {now.date().addDays(1), _settings.startTime}; nextScanDateTime = { now.date().addDays(1), _settings.startTime };
break; break;
case ScanSettings::UpdatePeriod::Weekly: case ScanSettings::UpdatePeriod::Weekly:
if (now.time() < _settings.startTime && now.date().dayOfWeek() == 1) if (now.time() < _settings.startTime && now.date().dayOfWeek() == 1)
nextScanDateTime = {now.date(), _settings.startTime}; nextScanDateTime = { now.date(), _settings.startTime };
else else
nextScanDateTime = {getNextMonday(now.date()), _settings.startTime}; nextScanDateTime = { getNextMonday(now.date()), _settings.startTime };
break; break;
case ScanSettings::UpdatePeriod::Monthly: case ScanSettings::UpdatePeriod::Monthly:
if (now.time() < _settings.startTime && now.date().day() == 1) if (now.time() < _settings.startTime && now.date().day() == 1)
nextScanDateTime = {now.date(), _settings.startTime}; nextScanDateTime = { now.date(), _settings.startTime };
else else
nextScanDateTime = {getNextFirstOfMonth(now.date()), _settings.startTime}; nextScanDateTime = { getNextFirstOfMonth(now.date()), _settings.startTime };
break; break;
case ScanSettings::UpdatePeriod::Hourly: case ScanSettings::UpdatePeriod::Hourly:
nextScanDateTime = {now.date(), now.time().addSecs(3600)}; nextScanDateTime = { now.date(), now.time().addSecs(3600) };
break; break;
case ScanSettings::UpdatePeriod::Never: case ScanSettings::UpdatePeriod::Never:
@@ -230,50 +210,48 @@ ScannerService::scheduleNextScan()
scheduleScan(false, nextScanDateTime); scheduleScan(false, nextScanDateTime);
{ {
std::unique_lock lock {_statusMutex}; std::unique_lock lock{ _statusMutex };
_curState = nextScanDateTime.isValid() ? State::Scheduled : State::NotScheduled; _curState = nextScanDateTime.isValid() ? State::Scheduled : State::NotScheduled;
_nextScheduledScan = nextScanDateTime; _nextScheduledScan = nextScanDateTime;
} }
_events.scanScheduled.emit(_nextScheduledScan); _events.scanScheduled.emit(_nextScheduledScan);
} }
void void ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime)
ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime) {
{ auto cb{ [=](boost::system::error_code ec)
auto cb {[=](boost::system::error_code ec)
{ {
if (ec) if (ec)
return; return;
scan(force); scan(force);
}}; } };
if (dateTime.isNull()) if (dateTime.isNull())
{ {
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan right now"; LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan right now";
_scheduleTimer.expires_from_now(std::chrono::seconds {0}); _scheduleTimer.expires_from_now(std::chrono::seconds{ 0 });
_scheduleTimer.async_wait(cb); _scheduleTimer.async_wait(cb);
} }
else else
{ {
std::chrono::system_clock::time_point timePoint {dateTime.toTimePoint()}; std::chrono::system_clock::time_point timePoint{ dateTime.toTimePoint() };
std::time_t t {std::chrono::system_clock::to_time_t(timePoint)}; std::time_t t{ std::chrono::system_clock::to_time_t(timePoint) };
char ctimeStr[26]; char ctimeStr[26];
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan at " << std::string(::ctime_r(&t, ctimeStr)); LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan at " << std::string(::ctime_r(&t, ctimeStr));
_scheduleTimer.expires_at(timePoint); _scheduleTimer.expires_at(timePoint);
_scheduleTimer.async_wait(cb); _scheduleTimer.async_wait(cb);
} }
} }
void void ScannerService::scan(bool forceScan)
ScannerService::scan(bool forceScan) {
{
_events.scanStarted.emit(); _events.scanStarted.emit();
{ {
std::unique_lock lock {_statusMutex}; std::unique_lock lock{ _statusMutex };
_curState = State::InProgress; _curState = State::InProgress;
_nextScheduledScan = {}; _nextScheduledScan = {};
} }
@@ -283,14 +261,14 @@ ScannerService::scan(bool forceScan)
refreshScanSettings(); refreshScanSettings();
IScanStep::ScanContext scanContext {_settings.mediaDirectory, forceScan, ScanStats {}, ScanStepStats {}}; IScanStep::ScanContext scanContext{ _settings.mediaDirectory, forceScan, ScanStats {}, ScanStepStats {} };
ScanStats& stats {scanContext.stats}; ScanStats& stats{ scanContext.stats };
stats.startTime = Wt::WDateTime::currentDateTime(); stats.startTime = Wt::WDateTime::currentDateTime();
for (auto& scanStep : _scanSteps) for (auto& scanStep : _scanSteps)
{ {
LMS_LOG(DBUPDATER, DEBUG) << "Starting scan step '" << scanStep->getStepName() << "'"; LMS_LOG(DBUPDATER, DEBUG) << "Starting scan step '" << scanStep->getStepName() << "'";
scanContext.currentStepStats = ScanStepStats {Wt::WDateTime::currentDateTime(), scanStep->getStep()}; scanContext.currentStepStats = ScanStepStats{ Wt::WDateTime::currentDateTime(), scanStep->getStep() };
notifyInProgress(scanContext.currentStepStats); notifyInProgress(scanContext.currentStepStats);
scanStep->process(scanContext); scanStep->process(scanContext);
@@ -306,7 +284,7 @@ ScannerService::scan(bool forceScan)
{ {
stats.stopTime = Wt::WDateTime::currentDateTime(); stats.stopTime = Wt::WDateTime::currentDateTime();
{ {
std::unique_lock lock {_statusMutex}; std::unique_lock lock{ _statusMutex };
_lastCompleteScanStats = stats; _lastCompleteScanStats = stats;
_currentScanStepStats.reset(); _currentScanStepStats.reset();
@@ -321,17 +299,16 @@ ScannerService::scan(bool forceScan)
{ {
LMS_LOG(DBUPDATER, DEBUG) << "Scan aborted, not scheduling next scan!"; LMS_LOG(DBUPDATER, DEBUG) << "Scan aborted, not scheduling next scan!";
std::unique_lock lock {_statusMutex}; std::unique_lock lock{ _statusMutex };
_curState = State::NotScheduled; _curState = State::NotScheduled;
_currentScanStepStats.reset(); _currentScanStepStats.reset();
} }
} }
void void ScannerService::refreshScanSettings()
ScannerService::refreshScanSettings() {
{ ScannerSettings newSettings{ readSettings() };
ScannerSettings newSettings {readSettings()};
if (_settings == newSettings) if (_settings == newSettings)
return; return;
@@ -341,10 +318,10 @@ ScannerService::refreshScanSettings()
_settings = std::move(newSettings); _settings = std::move(newSettings);
auto cbFunc {[this](const ScanStepStats& stats) auto cbFunc{ [this](const ScanStepStats& stats)
{ {
notifyInProgressIfNeeded(stats); notifyInProgressIfNeeded(stats);
}}; } };
ScanStepBase::InitParams params ScanStepBase::InitParams params
{ {
@@ -360,30 +337,28 @@ ScannerService::refreshScanSettings()
_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params)); _scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params)); _scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params));
_scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params)); _scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
} }
ScannerSettings ScannerSettings ScannerService::readSettings()
ScannerService::readSettings() {
{
ScannerSettings newSettings; ScannerSettings newSettings;
newSettings.skipDuplicateMBID = Service<IConfig>::get()->getBool("scanner-skip-duplicate-mbid", false); newSettings.skipDuplicateMBID = Service<IConfig>::get()->getBool("scanner-skip-duplicate-mbid", false);
{ {
auto transaction {_dbSession.createSharedTransaction()}; auto transaction{ _dbSession.createSharedTransaction() };
const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)}; const ScanSettings::pointer scanSettings{ ScanSettings::get(_dbSession) };
newSettings.scanVersion = scanSettings->getScanVersion(); newSettings.scanVersion = scanSettings->getScanVersion();
newSettings.startTime = scanSettings->getUpdateStartTime(); newSettings.startTime = scanSettings->getUpdateStartTime();
newSettings.updatePeriod = scanSettings->getUpdatePeriod(); newSettings.updatePeriod = scanSettings->getUpdatePeriod();
{ {
const auto fileExtensions {scanSettings->getAudioFileExtensions()}; const auto fileExtensions{ scanSettings->getAudioFileExtensions() };
newSettings.supportedExtensions.reserve(fileExtensions.size()); newSettings.supportedExtensions.reserve(fileExtensions.size());
std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(newSettings.supportedExtensions), std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(newSettings.supportedExtensions),
[](const std::filesystem::path& extension) { return std::filesystem::path{ StringUtils::stringToLower(extension.string()) }; }); [](const std::filesystem::path& extension) { return std::filesystem::path{ StringUtils::stringToLower(extension.string()) }; });
} }
newSettings.similarityServiceType = scanSettings->getSimilarityEngineType();
newSettings.mediaDirectory = scanSettings->getMediaDirectory(); newSettings.mediaDirectory = scanSettings->getMediaDirectory();
const auto clusterTypes = scanSettings->getClusterTypes(); const auto clusterTypes = scanSettings->getClusterTypes();
@@ -397,28 +372,26 @@ ScannerService::readSettings()
} }
return newSettings; return newSettings;
} }
void void ScannerService::notifyInProgress(const ScanStepStats& stepStats)
ScannerService::notifyInProgress(const ScanStepStats& stepStats)
{
{ {
std::unique_lock lock {_statusMutex}; {
std::unique_lock lock{ _statusMutex };
_currentScanStepStats = stepStats; _currentScanStepStats = stepStats;
} }
const std::chrono::system_clock::time_point now {std::chrono::system_clock::now()}; const std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() };
_events.scanInProgress(stepStats); _events.scanInProgress(stepStats);
_lastScanInProgressEmit = now; _lastScanInProgressEmit = now;
} }
void void ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats) {
{ std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() };
std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
if (std::chrono::duration_cast<std::chrono::seconds>(now - _lastScanInProgressEmit).count() > 1) if (std::chrono::duration_cast<std::chrono::seconds>(now - _lastScanInProgressEmit).count() > 1)
notifyInProgress(stepStats); notifyInProgress(stepStats);
} }
} // namespace Scanner } // namespace Scanner
src/libs/services/scanner/impl/ScannerService.hpp
+7 -15
View File
@@ -38,23 +38,16 @@
#include "IScanStep.hpp" #include "IScanStep.hpp"
#include "ScannerSettings.hpp" #include "ScannerSettings.hpp"
namespace Recommendation
{
class IRecommendationService;
}
namespace Scanner namespace Scanner
{ {
class ScannerService : public IScannerService class ScannerService : public IScannerService
{ {
public: public:
ScannerService(Database::Db& db, Recommendation::IRecommendationService& recommendationService); ScannerService(Database::Db& db);
~ScannerService(); ~ScannerService();
ScannerService(const ScannerService&) = delete; ScannerService(const ScannerService&) = delete;
ScannerService(ScannerService&&) = delete;
ScannerService& operator=(const ScannerService&) = delete; ScannerService& operator=(const ScannerService&) = delete;
ScannerService& operator=(ScannerService&&) = delete;
void requestReload() override; void requestReload() override;
void requestImmediateScan(bool force) override; void requestImmediateScan(bool force) override;
@@ -75,31 +68,30 @@ namespace Scanner
// Update database (scheduled callback) // Update database (scheduled callback)
void scan(bool force); void scan(bool force);
void scanMediaDirectory( const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats); void scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats);
// Helpers // Helpers
void refreshScanSettings(); void refreshScanSettings();
ScannerSettings readSettings(); ScannerSettings readSettings();
void reloadRecommendationService();
void notifyInProgressIfNeeded(const ScanStepStats& stats); void notifyInProgressIfNeeded(const ScanStepStats& stats);
void notifyInProgress(const ScanStepStats& stats); void notifyInProgress(const ScanStepStats& stats);
void reloadSimilarityEngine(ScanStats& stats); void reloadSimilarityEngine(ScanStats& stats);
Recommendation::IRecommendationService& _recommendationService;
std::vector<std::unique_ptr<IScanStep>> _scanSteps; std::vector<std::unique_ptr<IScanStep>> _scanSteps;
std::mutex _controlMutex; std::mutex _controlMutex;
bool _abortScan {}; bool _abortScan{};
Wt::WIOService _ioService; Wt::WIOService _ioService;
boost::asio::system_timer _scheduleTimer {_ioService}; boost::asio::system_timer _scheduleTimer{ _ioService };
Events _events; Events _events;
std::chrono::system_clock::time_point _lastScanInProgressEmit {}; std::chrono::system_clock::time_point _lastScanInProgressEmit{};
Database::Db& _db; Database::Db& _db;
Database::Session _dbSession; Database::Session _dbSession;
mutable std::shared_mutex _statusMutex; mutable std::shared_mutex _statusMutex;
State _curState {State::NotScheduled}; State _curState{ State::NotScheduled };
std::optional<ScanStats> _lastCompleteScanStats; std::optional<ScanStats> _lastCompleteScanStats;
std::optional<ScanStepStats> _currentScanStepStats; std::optional<ScanStepStats> _currentScanStepStats;
Wt::WDateTime _nextScheduledScan; Wt::WDateTime _nextScheduledScan;
src/libs/services/scanner/impl/ScannerSettings.hpp
-2
View File
@@ -34,7 +34,6 @@ namespace Scanner
Wt::WTime startTime; Wt::WTime startTime;
Database::ScanSettings::UpdatePeriod updatePeriod {Database::ScanSettings::UpdatePeriod::Never}; Database::ScanSettings::UpdatePeriod updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
std::vector<std::filesystem::path> supportedExtensions; std::vector<std::filesystem::path> supportedExtensions;
Database::ScanSettings::SimilarityEngineType similarityServiceType;
std::filesystem::path mediaDirectory; std::filesystem::path mediaDirectory;
bool skipDuplicateMBID {}; bool skipDuplicateMBID {};
std::set<std::string> clusterTypeNames; std::set<std::string> clusterTypeNames;
@@ -45,7 +44,6 @@ namespace Scanner
&& startTime == rhs.startTime && startTime == rhs.startTime
&& updatePeriod == rhs.updatePeriod && updatePeriod == rhs.updatePeriod
&& supportedExtensions == rhs.supportedExtensions && supportedExtensions == rhs.supportedExtensions
&& similarityServiceType == rhs.similarityServiceType
&& mediaDirectory == rhs.mediaDirectory && mediaDirectory == rhs.mediaDirectory
&& skipDuplicateMBID == rhs.skipDuplicateMBID && skipDuplicateMBID == rhs.skipDuplicateMBID
&& clusterTypeNames == rhs.clusterTypeNames; && clusterTypeNames == rhs.clusterTypeNames;
src/libs/services/scanner/include/services/scanner/IScannerService.hpp
+1 -6
View File
@@ -29,11 +29,6 @@ namespace Database
class Db; class Db;
} }
namespace Recommendation
{
class IRecommendationService;
}
namespace Scanner namespace Scanner
{ {
@@ -66,7 +61,7 @@ namespace Scanner
virtual Events& getEvents() = 0; virtual Events& getEvents() = 0;
}; };
std::unique_ptr<IScannerService> createScannerService(Database::Db& db, Recommendation::IRecommendationService& recommendationEngine); std::unique_ptr<IScannerService> createScannerService(Database::Db& db);
} // Scanner } // Scanner
src/lms/main.cpp
+1 -1
View File
@@ -270,7 +270,7 @@ int main(int argc, char* argv[])
Service<Cover::ICoverService> coverService{ Cover::createCoverService(database, argv[0], server.appRoot() + "/images/unknown-cover.jpg") }; Service<Cover::ICoverService> coverService{ Cover::createCoverService(database, argv[0], server.appRoot() + "/images/unknown-cover.jpg") };
Service<Recommendation::IRecommendationService> recommendationService{ Recommendation::createRecommendationService(database) }; Service<Recommendation::IRecommendationService> recommendationService{ Recommendation::createRecommendationService(database) };
Service<Recommendation::IPlaylistGeneratorService> playlistGeneratorService{ Recommendation::createPlaylistGeneratorService(database, *recommendationService.get()) }; Service<Recommendation::IPlaylistGeneratorService> playlistGeneratorService{ Recommendation::createPlaylistGeneratorService(database, *recommendationService.get()) };
Service<Scanner::IScannerService> scannerService{ Scanner::createScannerService(database, *recommendationService) }; Service<Scanner::IScannerService> scannerService{ Scanner::createScannerService(database) };
scannerService->getEvents().scanComplete.connect([&] scannerService->getEvents().scanComplete.connect([&]
{ {
src/lms/ui/admin/DatabaseSettingsView.cpp
+2
View File
@@ -29,6 +29,7 @@
#include "services/database/Cluster.hpp" #include "services/database/Cluster.hpp"
#include "services/database/ScanSettings.hpp" #include "services/database/ScanSettings.hpp"
#include "services/database/Session.hpp" #include "services/database/Session.hpp"
#include "services/recommendation/IRecommendationService.hpp"
#include "services/scanner/IScannerService.hpp" #include "services/scanner/IScannerService.hpp"
#include "utils/Logger.hpp" #include "utils/Logger.hpp"
#include "utils/Service.hpp" #include "utils/Service.hpp"
@@ -232,6 +233,7 @@ DatabaseSettingsView::refreshView()
{ {
model->saveData(); model->saveData();
Service<Recommendation::IRecommendationService>::get()->load();
Service<Scanner::IScannerService>::get()->requestImmediateScan(false); Service<Scanner::IScannerService>::get()->requestImmediateScan(false);
LmsApp->notifyMsg(Notification::Type::Info, Wt::WString::tr("Lms.Admin.Database.database"), Wt::WString::tr("Lms.Admin.Database.settings-saved")); LmsApp->notifyMsg(Notification::Type::Info, Wt::WString::tr("Lms.Admin.Database.database"), Wt::WString::tr("Lms.Admin.Database.settings-saved"));
} }
src/tools/recommendation/LmsRecommendation.cpp
+5 -5
View File
@@ -157,16 +157,16 @@ int main(int argc, char* argv[])
Db db{ config->getPath("working-dir") / "lms.db" }; Db db{ config->getPath("working-dir") / "lms.db" };
Session session{ db }; Session session{ db };
std::cout << "Creating recommendation recommendationService..." << std::endl; std::cout << "Creating recommendation service..." << std::endl;
const auto recommendationService{ Recommendation::createRecommendationService(db) }; const auto recommendationService{ Recommendation::createRecommendationService(db) };
std::cout << "Recommendation recommendationService created!" << std::endl; std::cout << "Recommendation service created!" << std::endl;
std::cout << "Loading recommendation recommendationService..." << std::endl; std::cout << "Loading recommendation service..." << std::endl;
recommendationService->load(false); recommendationService->load();
unsigned maxSimilarityCount{ vm["max"].as<unsigned>() }; unsigned maxSimilarityCount{ vm["max"].as<unsigned>() };
std::cout << "Recommendation recommendationService loaded!" << std::endl; std::cout << "Recommendation service loaded!" << std::endl;
if (vm.count("tracks")) if (vm.count("tracks"))
dumpTracksRecommendation(db, *recommendationService, maxSimilarityCount); dumpTracksRecommendation(db, *recommendationService, maxSimilarityCount);