owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrated scanner stuff

Browse Source
This commit is contained in:
emeric
2021-10-20 20:00:14 +02:00
parent 7386c89ae7
commit 398925588f
13 changed files with 173 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/scanner/CMakeLists.txt
+1 -1
View File
@@ -1,7 +1,7 @@
add_library(lmsscanner SHARED
impl/AcousticBrainzUtils.cpp
impl/Scanner.cpp
impl/ScannerService.cpp
impl/ScannerStats.cpp
)
src/libs/services/scanner/impl/Scanner.hpp
-122
View File
@@ -1,122 +0,0 @@
/*
* Copyright (C) 2013 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <chrono>
#include <shared_mutex>
#include <optional>
#include <unordered_set>
#include <Wt/WDateTime.h>
#include <Wt/WIOService.h>
#include <Wt/WSignal.h>
#include <boost/asio/system_timer.hpp>
#include "database/Types.hpp"
#include "database/ScanSettings.hpp"
#include "database/Session.hpp"
#include "metadata/IParser.hpp"
#include "scanner/IScanner.hpp"
#include "utils/Path.hpp"
class UUID;
namespace Recommendation
{
class IRecommendationService;
}
namespace Scanner {
class Scanner : public IScanner
{
public:
Scanner(Database::Db& db, Recommendation::IRecommendationService& recommendationService);
~Scanner();
Scanner(const Scanner&) = delete;
Scanner(Scanner&&) = delete;
Scanner& operator=(const Scanner&) = delete;
Scanner& operator=(Scanner&&) = delete;
void requestReload() override;
void requestImmediateScan(bool force) override;
Status getStatus() const override;
Events& getEvents() override { return _events; }
private:
void start();
void stop();
// Job handling
void scheduleNextScan();
void scheduleScan(bool force, const Wt::WDateTime& dateTime = {});
void abortScan();
// Update database (scheduled callback)
void scan(bool force);
void scanMediaDirectory( const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats);
bool fetchTrackFeatures(Database::TrackId trackId, const UUID& MBID);
void fetchTrackFeatures(ScanStats& stats);
// Helpers
void refreshScanSettings();
void countAllFiles(ScanStats& stats);
void removeMissingTracks(ScanStats& stats);
void removeOrphanEntries();
void checkDuplicatedAudioFiles(ScanStats& stats);
void scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats);
void notifyInProgressIfNeeded(const ScanStepStats& stats);
void notifyInProgress(const ScanStepStats& stats);
void reloadSimilarityEngine(ScanStats& stats);
Recommendation::IRecommendationService& _recommendationService;
std::mutex _controlMutex;
std::atomic<bool> _abortScan {};
Wt::WIOService _ioService;
boost::asio::system_timer _scheduleTimer {_ioService};
Events _events;
std::chrono::system_clock::time_point _lastScanInProgressEmit {};
Database::Session _dbSession;
std::unique_ptr<MetaData::IParser> _metadataParser;
mutable std::shared_mutex _statusMutex;
State _curState {State::NotScheduled};
std::optional<ScanStats> _lastCompleteScanStats;
std::optional<ScanStepStats> _currentScanStepStats;
Wt::WDateTime _nextScheduledScan;
// Current scan settings
std::size_t _scanVersion {};
Wt::WTime _startTime;
Database::ScanSettings::UpdatePeriod _updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
std::unordered_set<std::filesystem::path> _fileExtensions;
std::filesystem::path _mediaDirectory;
Database::ScanSettings::RecommendationEngineType _recommendationServiceType;
};
} // Scanner
src/libs/services/scanner/impl/Scanner.cpp → src/libs/services/scanner/impl/ScannerService.cpp
+28 -28
View File
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Scanner.hpp"
#include "ScannerService.hpp"
#include <ctime>
#include <boost/asio/placeholders.hpp>
@@ -248,13 +248,13 @@ getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames)
namespace Scanner {
std::unique_ptr<IScanner>
createScanner(Database::Db& db, Recommendation::IRecommendationService& recommendationService)
std::unique_ptr<IScannerService>
createScannerService(Database::Db& db, Recommendation::IRecommendationService& recommendationService)
{
return std::make_unique<Scanner>(db, recommendationService);
return std::make_unique<ScannerService>(db, recommendationService);
}
Scanner::Scanner(Database::Db& db, Recommendation::IRecommendationService& recommendationService)
ScannerService::ScannerService(Database::Db& db, Recommendation::IRecommendationService& recommendationService)
: _recommendationService {recommendationService}
, _dbSession {db}
{
@@ -268,14 +268,14 @@ Scanner::Scanner(Database::Db& db, Recommendation::IRecommendationService& recom
start();
}
Scanner::~Scanner()
ScannerService::~ScannerService()
{
LMS_LOG(DBUPDATER, INFO) << "Shutting down Scanner...";
stop();
}
void
Scanner::start()
ScannerService::start()
{
std::scoped_lock lock {_controlMutex};
@@ -296,7 +296,7 @@ Scanner::start()
}
void
Scanner::stop()
ScannerService::stop()
{
std::scoped_lock lock {_controlMutex};
@@ -307,7 +307,7 @@ Scanner::stop()
}
void
Scanner::abortScan()
ScannerService::abortScan()
{
LMS_LOG(DBUPDATER, DEBUG) << "Aborting scan...";
std::scoped_lock lock {_controlMutex};
@@ -325,7 +325,7 @@ Scanner::abortScan()
}
void
Scanner::requestImmediateScan(bool force)
ScannerService::requestImmediateScan(bool force)
{
abortScan();
_ioService.post([=]()
@@ -338,7 +338,7 @@ Scanner::requestImmediateScan(bool force)
}
void
Scanner::requestReload()
ScannerService::requestReload()
{
abortScan();
_ioService.post([=]()
@@ -350,8 +350,8 @@ Scanner::requestReload()
});
}
Scanner::Status
Scanner::getStatus() const
ScannerService::Status
ScannerService::getStatus() const
{
Status res;
@@ -366,7 +366,7 @@ Scanner::getStatus() const
}
void
Scanner::scheduleNextScan()
ScannerService::scheduleNextScan()
{
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan";
@@ -420,7 +420,7 @@ Scanner::scheduleNextScan()
}
void
Scanner::countAllFiles(ScanStats& stats)
ScannerService::countAllFiles(ScanStats& stats)
{
ScanStepStats stepStats{stats.startTime, ScanProgressStep::DiscoveringFiles};
@@ -445,7 +445,7 @@ Scanner::countAllFiles(ScanStats& stats)
}
void
Scanner::scheduleScan(bool force, const Wt::WDateTime& dateTime)
ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime)
{
auto cb {[=](boost::system::error_code ec)
{
@@ -474,7 +474,7 @@ Scanner::scheduleScan(bool force, const Wt::WDateTime& dateTime)
}
void
Scanner::scan(bool forceScan)
ScannerService::scan(bool forceScan)
{
_events.scanStarted.emit();
@@ -543,7 +543,7 @@ Scanner::scan(bool forceScan)
}
bool
Scanner::fetchTrackFeatures(Database::TrackId trackId, const UUID& recordingMBID)
ScannerService::fetchTrackFeatures(Database::TrackId trackId, const UUID& recordingMBID)
{
std::map<std::string, double> features;
@@ -569,7 +569,7 @@ Scanner::fetchTrackFeatures(Database::TrackId trackId, const UUID& recordingMBID
}
void
Scanner::fetchTrackFeatures(ScanStats& stats)
ScannerService::fetchTrackFeatures(ScanStats& stats)
{
if (_recommendationServiceType != ScanSettings::RecommendationEngineType::Features)
return;
@@ -619,7 +619,7 @@ Scanner::fetchTrackFeatures(ScanStats& stats)
}
void
Scanner::refreshScanSettings()
ScannerService::refreshScanSettings()
{
auto transaction {_dbSession.createSharedTransaction()};
@@ -651,7 +651,7 @@ Scanner::refreshScanSettings()
}
void
Scanner::notifyInProgress(const ScanStepStats& stepStats)
ScannerService::notifyInProgress(const ScanStepStats& stepStats)
{
{
std::unique_lock lock {_statusMutex};
@@ -664,7 +664,7 @@ Scanner::notifyInProgress(const ScanStepStats& stepStats)
}
void
Scanner::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
{
std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
@@ -673,7 +673,7 @@ Scanner::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
}
void
Scanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats)
ScannerService::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats)
{
Wt::WDateTime lastWriteTime;
try
@@ -836,7 +836,7 @@ Scanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanSt
}
void
Scanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats)
ScannerService::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats)
{
ScanStepStats stepStats{stats.startTime, ScanProgressStep::ScanningFiles};
stepStats.totalElems = stats.filesScanned;
@@ -903,7 +903,7 @@ checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDire
}
void
Scanner::removeMissingTracks(ScanStats& stats)
ScannerService::removeMissingTracks(ScanStats& stats)
{
static constexpr std::size_t batchSize {50};
@@ -970,7 +970,7 @@ Scanner::removeMissingTracks(ScanStats& stats)
}
void
Scanner::removeOrphanEntries()
ScannerService::removeOrphanEntries()
{
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters...";
{
@@ -1013,7 +1013,7 @@ Scanner::removeOrphanEntries()
}
void
Scanner::checkDuplicatedAudioFiles(ScanStats& stats)
ScannerService::checkDuplicatedAudioFiles(ScanStats& stats)
{
LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files";
@@ -1033,7 +1033,7 @@ Scanner::checkDuplicatedAudioFiles(ScanStats& stats)
}
void
Scanner::reloadSimilarityEngine(ScanStats& stats)
ScannerService::reloadSimilarityEngine(ScanStats& stats)
{
ScanStepStats stepStats {stats.startTime, ScanProgressStep::ReloadingSimilarityEngine};
src/libs/services/scanner/impl/ScannerService.hpp
+121
View File
@@ -0,0 +1,121 @@
/*
* Copyright (C) 2013 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <chrono>
#include <shared_mutex>
#include <optional>
#include <unordered_set>
#include <Wt/WDateTime.h>
#include <Wt/WIOService.h>
#include <Wt/WSignal.h>
#include <boost/asio/system_timer.hpp>
#include "database/Types.hpp"
#include "database/ScanSettings.hpp"
#include "database/Session.hpp"
#include "metadata/IParser.hpp"
#include "services/scanner/IScannerService.hpp"
#include "utils/Path.hpp"
class UUID;
namespace Recommendation
{
class IRecommendationService;
}
namespace Scanner
{
class ScannerService : public IScannerService
{
public:
ScannerService(Database::Db& db, Recommendation::IRecommendationService& recommendationService);
~ScannerService();
ScannerService(const ScannerService&) = delete;
ScannerService(ScannerService&&) = delete;
ScannerService& operator=(const ScannerService&) = delete;
ScannerService& operator=(ScannerService&&) = delete;
void requestReload() override;
void requestImmediateScan(bool force) override;
Status getStatus() const override;
Events& getEvents() override { return _events; }
private:
void start();
void stop();
// Job handling
void scheduleNextScan();
void scheduleScan(bool force, const Wt::WDateTime& dateTime = {});
void abortScan();
// Update database (scheduled callback)
void scan(bool force);
void scanMediaDirectory( const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats);
bool fetchTrackFeatures(Database::TrackId trackId, const UUID& MBID);
void fetchTrackFeatures(ScanStats& stats);
// Helpers
void refreshScanSettings();
void countAllFiles(ScanStats& stats);
void removeMissingTracks(ScanStats& stats);
void removeOrphanEntries();
void checkDuplicatedAudioFiles(ScanStats& stats);
void scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats);
void notifyInProgressIfNeeded(const ScanStepStats& stats);
void notifyInProgress(const ScanStepStats& stats);
void reloadSimilarityEngine(ScanStats& stats);
Recommendation::IRecommendationService& _recommendationService;
std::mutex _controlMutex;
std::atomic<bool> _abortScan {};
Wt::WIOService _ioService;
boost::asio::system_timer _scheduleTimer {_ioService};
Events _events;
std::chrono::system_clock::time_point _lastScanInProgressEmit {};
Database::Session _dbSession;
std::unique_ptr<MetaData::IParser> _metadataParser;
mutable std::shared_mutex _statusMutex;
State _curState {State::NotScheduled};
std::optional<ScanStats> _lastCompleteScanStats;
std::optional<ScanStepStats> _currentScanStepStats;
Wt::WDateTime _nextScheduledScan;
// Current scan settings
std::size_t _scanVersion {};
Wt::WTime _startTime;
Database::ScanSettings::UpdatePeriod _updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
std::unordered_set<std::filesystem::path> _fileExtensions;
std::filesystem::path _mediaDirectory;
Database::ScanSettings::RecommendationEngineType _recommendationServiceType;
};
} // Scanner
src/libs/services/scanner/impl/ScannerStats.cpp
+1 -1
View File
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "scanner/ScannerStats.hpp"
#include "services/scanner/ScannerStats.hpp"
namespace Scanner {
src/libs/services/scanner/include/scanner/IScanner.hpp → src/libs/services/scanner/include/services/scanner/IScannerService.hpp
+3 -3
View File
@@ -37,10 +37,10 @@ namespace Recommendation
namespace Scanner
{
class IScanner
class IScannerService
{
public:
virtual ~IScanner() = default;
virtual ~IScannerService() = default;
// Async requests
virtual void requestReload() = 0;
@@ -66,7 +66,7 @@ namespace Scanner
virtual Events& getEvents() = 0;
};
std::unique_ptr<IScanner> createScanner(Database::Db& db, Recommendation::IRecommendationService& recommendationEngine);
std::unique_ptr<IScannerService> createScannerService(Database::Db& db, Recommendation::IRecommendationService& recommendationEngine);
} // Scanner
src/libs/services/scanner/include/scanner/ScannerEvents.hpp → src/libs/services/scanner/include/services/scanner/ScannerEvents.hpp
View File
src/libs/services/scanner/include/scanner/ScannerStats.hpp → src/libs/services/scanner/include/services/scanner/ScannerStats.hpp
View File
src/libs/subsonic/impl/Scan.cpp
+5 -5
View File
@@ -19,7 +19,7 @@
#include "Scan.hpp"
#include "scanner/IScanner.hpp"
#include "services/scanner/IScannerService.hpp"
#include "utils/Service.hpp"
namespace API::Subsonic::Scan
@@ -32,10 +32,10 @@ namespace API::Subsonic::Scan
{
Response::Node statusResponse;
const IScanner::Status scanStatus {Service<IScanner>::get()->getStatus()};
const IScannerService::Status scanStatus {Service<IScannerService>::get()->getStatus()};
statusResponse.setAttribute("scanning", scanStatus.currentState == IScanner::State::InProgress);
if (scanStatus.currentState == IScanner::State::InProgress)
statusResponse.setAttribute("scanning", scanStatus.currentState == IScannerService::State::InProgress);
if (scanStatus.currentState == IScannerService::State::InProgress)
{
std::size_t count{};
@@ -61,7 +61,7 @@ namespace API::Subsonic::Scan
Response
handleStartScan(RequestContext& context)
{
Service<IScanner>::get()->requestImmediateScan(false);
Service<IScannerService>::get()->requestImmediateScan(false);
Response response {Response::createOkResponse(context.serverProtocolVersion)};
response.addNode("scanStatus", createStatusResponseNode());
src/lms/main.cpp
+3 -3
View File
@@ -31,7 +31,7 @@
#include "services/cover/ICoverService.hpp"
#include "database/Db.hpp"
#include "database/Session.hpp"
#include "scanner/IScanner.hpp"
#include "services/scanner/IScannerService.hpp"
#include "subsonic/SubsonicResource.hpp"
#include "services/recommendation/IRecommendationService.hpp"
#include "services/scrobbling/IScrobblingService.hpp"
@@ -132,7 +132,7 @@ generateWtConfig(std::string execPath)
static
void
proxyScannerEventsToApplication(Scanner::IScanner& scanner, Wt::WServer& server)
proxyScannerEventsToApplication(Scanner::IScannerService& scanner, Wt::WServer& server)
{
auto postAll {[](Wt::WServer& server, std::function<void()> cb)
{
@@ -259,7 +259,7 @@ int main(int argc, char* argv[])
Service<Http::IClient> httpClient {Http::createClient(ioContext)};
Service<Cover::ICoverService> coverService {Cover::createCoverService(database, argv[0], server.appRoot() + "/images/unknown-cover.jpg")};
Service<Recommendation::IRecommendationService> recommendationService {Recommendation::createRecommendationService(database)};
Service<Scanner::IScanner> scannerService {Scanner::createScanner(database, *recommendationService)};
Service<Scanner::IScannerService> scannerService {Scanner::createScannerService(database, *recommendationService)};
scannerService->getEvents().scanComplete.connect([&]
{
src/lms/ui/LmsApplication.hpp
+1 -1
View File
@@ -23,7 +23,7 @@
#include <Wt/WApplication.h>
#include "scanner/ScannerEvents.hpp"
#include "services/scanner/ScannerEvents.hpp"
namespace Database
{
src/lms/ui/admin/DatabaseSettingsView.cpp
+3 -3
View File
@@ -29,7 +29,7 @@
#include "database/Cluster.hpp"
#include "database/ScanSettings.hpp"
#include "database/Session.hpp"
#include "scanner/IScanner.hpp"
#include "services/scanner/IScannerService.hpp"
#include "utils/Logger.hpp"
#include "utils/Service.hpp"
#include "utils/String.hpp"
@@ -240,7 +240,7 @@ DatabaseSettingsView::refreshView()
{
model->saveData();
Service<Scanner::IScanner>::get()->requestImmediateScan(false);
Service<Scanner::IScannerService>::get()->requestImmediateScan(false);
LmsApp->notifyMsg(LmsApplication::MsgType::Success, Wt::WString::tr("Lms.Admin.Database.settings-saved"));
}
@@ -257,7 +257,7 @@ DatabaseSettingsView::refreshView()
immScanBtn->clicked().connect([=]
{
Service<Scanner::IScanner>::get()->requestImmediateScan(false);
Service<Scanner::IScannerService>::get()->requestImmediateScan(false);
});
t->updateView(model.get());
src/lms/ui/admin/ScannerController.cpp
+7 -7
View File
@@ -30,7 +30,7 @@
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "scanner/IScanner.hpp"
#include "services/scanner/IScannerService.hpp"
#include "utils/Service.hpp"
#include "LmsApplication.hpp"
@@ -163,20 +163,20 @@ ScannerController::refreshContents()
actionBtn->actionButton()->setText(Wt::WString::tr("Lms.Admin.ScannerController.scan-now"));
actionBtn->actionButton()->clicked().connect([]
{
Service<Scanner::IScanner>::get()->requestImmediateScan(false);
Service<Scanner::IScannerService>::get()->requestImmediateScan(false);
});
auto popup = std::make_unique<Wt::WPopupMenu>();
popup->addItem(Wt::WString::tr("Lms.Admin.ScannerController.force-scan-now"));
popup->itemSelected().connect([]
{
Service<Scanner::IScanner>::get()->requestImmediateScan(true);
Service<Scanner::IScannerService>::get()->requestImmediateScan(true);
});
actionBtn->dropDownButton()->setMenu(std::move(popup));
actionBtn->dropDownButton()->addStyleClass("btn-primary");
const IScanner::Status status {Service<IScanner>::get()->getStatus()};
const IScannerService::Status status {Service<IScannerService>::get()->getStatus()};
if (status.lastCompleteScanStats)
{
bindString("last-scan", Wt::WString::tr("Lms.Admin.ScannerController.last-scan-status")
@@ -200,16 +200,16 @@ ScannerController::refreshContents()
switch (status.currentState)
{
case IScanner::State::NotScheduled:
case IScannerService::State::NotScheduled:
bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-not-scheduled"));
bindEmpty("step-status");
break;
case IScanner::State::Scheduled:
case IScannerService::State::Scheduled:
bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-scheduled")
.arg(status.nextScheduledScan.toString()));
bindEmpty("step-status");
break;
case IScanner::State::InProgress:
case IScannerService::State::InProgress:
bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress")
.arg(static_cast<int>(status.currentScanStepStats->currentStep) + 1)
.arg(Scanner::ScanProgressStepCount));