owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added some feedback about the last/current/next media directory scans

Browse Source
This commit is contained in:
emeric
2019-06-04 00:04:01 +02:00
parent 6d3ddcddee
commit 51a8de8af4
8 changed files with 336 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
approot/admin-database.xml
+23
View File
@@ -76,6 +76,29 @@
</div>
</div>
</div>
${status}
</message>
<message id="Lms.Admin.Database.Status.template">
<legend>${tr:Lms.Admin.Database.Status.status}</legend>
<div class="form-horizontal">
<div class="form-group">
<label class="control-label col-sm-2" for="${id:last-scan}">
${tr:Lms.Admin.Database.Status.last-scan}
</label>
<div class="col-sm-5 well well-sm">
${last-scan}
</div>
</div>
<div class="form-group">
<label class="control-label col-sm-2" for="${id:status}">
${tr:Lms.Admin.Database.Status.status}
</label>
<div class="col-sm-5 well well-sm">
${status}
</div>
</div>
</div>
</message>
</messages>
approot/messages.xml
+8
View File
@@ -38,6 +38,14 @@
<message id="Lms.Admin.Database.update-start-time">Update start time</message>
<message id="Lms.Admin.Database.weekly">Weekly</message>
<message id="Lms.Admin.Database.Status.status">Status</message>
<message id="Lms.Admin.Database.Status.last-scan">Last scan</message>
<message id="Lms.Admin.Database.Status.last-scan-not-available">Not available</message>
<message id="Lms.Admin.Database.Status.last-scan-status">Scanned {1} files in {2} on {3} ({4} errors)</message>
<message id="Lms.Admin.Database.Status.status-not-scheduled">Not scheduled</message>
<message id="Lms.Admin.Database.Status.status-scheduled">Scheduled on {1}</message>
<message id="Lms.Admin.Database.Status.status-in-progress">Scanning {1} of {2} files ({3} %)</message>
<!--Users-->
<message id="Lms.Admin.Users.add">New user</message>
<message id="Lms.Admin.Users.admin">Admin</message>
approot/messages_fr.xml
+8
View File
@@ -38,6 +38,14 @@
<message id="Lms.Admin.Database.update-start-time">Heure de départ de la mise à jour</message>
<message id="Lms.Admin.Database.weekly">Toutes les semaines</message>
<message id="Lms.Admin.Database.Status.status">Statut</message>
<message id="Lms.Admin.Database.Status.last-scan">Dernier scan</message>
<message id="Lms.Admin.Database.Status.last-scan-not-available">Non disponible</message>
<message id="Lms.Admin.Database.Status.last-scan-status">{1} fichiers scannés en {2} le {3} ({4} erreurs)</message>
<message id="Lms.Admin.Database.Status.status-not-scheduled">Non planifié</message>
<message id="Lms.Admin.Database.Status.status-scheduled">Planifié le {1}</message>
<message id="Lms.Admin.Database.Status.status-in-progress">Scan de {1} fichiers sur {2} ({3} %)</message>
<!--Users-->
<message id="Lms.Admin.Users.add">Ajouter</message>
<message id="Lms.Admin.Users.admin">Admin</message>
src/scanner/MediaScanner.cpp
+127 -31
View File
@@ -194,9 +194,7 @@ getOrCreateClusters(Wt::Dbo::Session& session, const MetaData::Clusters& cluster
namespace Scanner {
MediaScanner::MediaScanner(Wt::Dbo::SqlConnectionPool& connectionPool)
: _running(false),
_scheduleTimer(_ioService),
_db(connectionPool)
: _db {connectionPool}
{
_ioService.setThreadCount(1);
@@ -221,8 +219,7 @@ MediaScanner::start(void)
{
_running = true;
// post some jobs in the io_service
scheduleScan();
scheduleNextScan();
_ioService.start();
}
@@ -241,27 +238,41 @@ MediaScanner::stop(void)
}
void
MediaScanner::scheduleImmediateScan()
MediaScanner::requestImmediateScan()
{
_ioService.post([=]()
{
LMS_LOG(DBUPDATER, INFO) << "Schedule immediate scan";
scheduleScan(std::chrono::seconds(0));
});
}
void
MediaScanner::reschedule()
{
_ioService.post([=]()
{
LMS_LOG(DBUPDATER, INFO) << "Rescheduling scan";
scheduleScan();
});
}
void
MediaScanner::scheduleScan()
MediaScanner::requestReschedule()
{
_ioService.post([=]()
{
LMS_LOG(DBUPDATER, INFO) << "Rescheduling scan";
scheduleNextScan();
});
}
MediaScanner::Status
MediaScanner::getStatus()
{
Status res;
std::unique_lock<std::mutex> lock {_statusMutex};
res.currentState = _curState;
res.nextScheduledScan = _nextScheduledScan;
res.lastScanStats = _lastScanStats;
res.inProgressStats = _inProgressStats;
return res;
}
void
MediaScanner::scheduleNextScan()
{
refreshScanSettings();
@@ -296,25 +307,71 @@ MediaScanner::scheduleScan()
break;
}
Wt::WDateTime nextScanDateTime;
if (nextScanDate.isValid())
scheduleScan( Wt::WDateTime(nextScanDate, _startTime).toTimePoint() );
{
nextScanDateTime = Wt::WDateTime {nextScanDate, _startTime};
scheduleScan(nextScanDateTime);
}
{
std::unique_lock<std::mutex> lock {_statusMutex};
_curState = nextScanDateTime.isValid() ? State::Scheduled : State::NotScheduled;
_nextScheduledScan = nextScanDateTime;
}
_sigScheduled.emit(_nextScheduledScan);
}
void
MediaScanner::scheduleScan(std::chrono::seconds duration)
MediaScanner::countAllFiles(Stats& stats)
{
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan in " << duration.count() << " seconds";
_scheduleTimer.expires_from_now(duration);
_scheduleTimer.async_wait( std::bind( &MediaScanner::scan, this, std::placeholders::_1) );
boost::system::error_code ec;
stats.totalFiles = 0;
boost::filesystem::recursive_directory_iterator itPath(_mediaDirectory, ec);
if (ec)
{
LMS_LOG(DBUPDATER, ERROR) << "Cannot iterate over '" << _mediaDirectory.string() << "': " << ec.message();
return;
}
boost::filesystem::recursive_directory_iterator itEnd;
while (itPath != itEnd && _running)
{
if (stats.totalFiles % 250 == 0)
notifyInProgressIfNeeded(stats);
const boost::filesystem::path& path {*itPath++};
if (!_running)
break;
if (boost::filesystem::is_regular(path) && isFileSupported(path, _fileExtensions))
stats.totalFiles++;
}
}
void
MediaScanner::scheduleScan(std::chrono::system_clock::time_point timePoint)
MediaScanner::scheduleScan(const Wt::WDateTime& dateTime)
{
std::time_t t = std::chrono::system_clock::to_time_t(timePoint);
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan at " << std::string(std::ctime(&t));
_scheduleTimer.expires_at(timePoint);
_scheduleTimer.async_wait(std::bind(&MediaScanner::scan, this, std::placeholders::_1));
if (dateTime.isNull())
{
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan right now";
_scheduleTimer.expires_from_now(std::chrono::seconds(0));
_scheduleTimer.async_wait(std::bind(&MediaScanner::scan, this, std::placeholders::_1));
}
else
{
std::chrono::system_clock::time_point timePoint {dateTime.toTimePoint()};
std::time_t t {std::chrono::system_clock::to_time_t(timePoint)};
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan at " << std::string(std::ctime(&t));
_scheduleTimer.expires_at(timePoint);
_scheduleTimer.async_wait(std::bind(&MediaScanner::scan, this, std::placeholders::_1));
}
}
void
@@ -323,12 +380,25 @@ MediaScanner::scan(boost::system::error_code err)
if (err)
return;
{
std::unique_lock<std::mutex> lock {_statusMutex};
_curState = State::InProgress;
_nextScheduledScan = {};
_inProgressStats = Stats{};
_inProgressStats->startTime = Wt::WLocalDateTime::currentDateTime().toUTC();
}
Stats& stats {*_inProgressStats};
LMS_LOG(UI, INFO) << "New scan started!";
refreshScanSettings();
bool forceScan = false;
Stats stats;
bool forceScan {false};
LMS_LOG(DBUPDATER, DEBUG) << "Counting files in media directory '" << _mediaDirectory.string() << "'...";
countAllFiles(stats);
LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.totalFiles;
removeMissingTracks(stats);
@@ -351,10 +421,23 @@ MediaScanner::scan(boost::system::error_code err)
for (auto& addon : _addons)
addon->preScanComplete();
scheduleScan();
stats.stopTime = Wt::WLocalDateTime::currentDateTime().toUTC();
{
std::unique_lock<std::mutex> lock {_statusMutex};
_lastScanStats = std::move(_inProgressStats);
_inProgressStats.reset();
}
scheduleNextScan();
scanComplete().emit(stats);
}
else
{
std::unique_lock<std::mutex> lock {_statusMutex};
_curState = State::NotScheduled;
_inProgressStats.reset();
}
}
void
@@ -388,9 +471,22 @@ MediaScanner::refreshScanSettings()
addon->refreshSettings();
}
void MediaScanner::notifyInProgressIfNeeded(Stats& stats)
{
std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
if (std::chrono::duration_cast<std::chrono::seconds>(now - _lastScanInProgressEmit).count() > 2)
{
_sigScanInProgress(stats);
_lastScanInProgressEmit = now;
}
}
void
MediaScanner::scanAudioFile(const boost::filesystem::path& file, bool forceScan, Stats& stats)
{
notifyInProgressIfNeeded(stats);
auto lastWriteTime = Wt::WDateTime::fromTime_t(boost::filesystem::last_write_time(file));
if (!forceScan)
@@ -561,7 +657,7 @@ MediaScanner::scanMediaDirectory(boost::filesystem::path mediaDirectory, bool fo
boost::filesystem::recursive_directory_iterator itEnd;
while (itPath != itEnd)
{
boost::filesystem::path path = *itPath;
const boost::filesystem::path& path {*itPath};
if (!_running)
return;
src/scanner/MediaScanner.hpp
+52 -16
View File
@@ -20,7 +20,9 @@
#pragma once
#include <chrono>
#include <mutex>
#include <Wt/WDateTime.h>
#include <Wt/WIOService.h>
#include <Wt/WSignal.h>
@@ -37,7 +39,6 @@ namespace Scanner {
class MediaScanner
{
public:
MediaScanner(Wt::Dbo::SqlConnectionPool& connectionPool);
void setAddon(MediaScannerAddon& addon);
@@ -46,11 +47,14 @@ class MediaScanner
void stop();
void restart();
void scheduleImmediateScan();
void reschedule();
// Async requests
void requestImmediateScan();
void requestReschedule();
struct Stats
{
Wt::WDateTime startTime;
Wt::WDateTime stopTime;
std::size_t skips = 0; // no change since last scan
std::size_t scans = 0; // actually scanned filed
std::size_t scanErrors = 0; // cannot scan file
@@ -62,21 +66,46 @@ class MediaScanner
std::size_t duplicateHashes = 0; // Same file hashes
std::size_t duplicateMBID = 0; // Same MBID
std::size_t totalFiles = 0; // Total number of files to be scanned
std::size_t nbFiles() const { return skips + additions + updates; }
std::size_t nbChanges() const { return additions + deletions + updates; }
std::size_t nbErrors() const { return scanErrors + incompleteScans; }
std::size_t nbDuplicates() const { return duplicateHashes + duplicateMBID; }
float progress() const { return (nbFiles() - totalFiles) / static_cast<float>(totalFiles); }
};
enum class State
{
NotScheduled,
Scheduled,
InProgress,
};
struct Status
{
State currentState {State::NotScheduled};
Wt::WDateTime nextScheduledScan;
boost::optional<Stats> lastScanStats;
boost::optional<Stats> inProgressStats;
};
Status getStatus();
// Called just after scan complete
Wt::Signal<Stats>& scanComplete() { return _sigScanComplete; }
// Called during scan in progress
Wt::Signal<Stats>& scanInProgress() { return _sigScanInProgress; }
// Called after a schedule
Wt::Signal<Wt::WDateTime>& scheduled() { return _sigScheduled; }
private:
// Job handling
void scheduleScan();
void scheduleScan(std::chrono::seconds duration);
void scheduleScan(std::chrono::system_clock::time_point time);
void scheduleNextScan();
void scheduleScan(const Wt::WDateTime& dateTime = {});
// Update database (scheduled callback)
void scan(boost::system::error_code ec);
@@ -86,30 +115,37 @@ class MediaScanner
// Helpers
void refreshScanSettings();
void countAllFiles(Stats& stats);
void removeMissingTracks(Stats& stats);
void removeOrphanEntries();
void checkDuplicatedAudioFiles(Stats& stats);
void scanAudioFile(const boost::filesystem::path& file, bool forceScan, Stats& stats);
void notifyInProgressIfNeeded(Stats& stats);
bool _running;
bool _running {false};
Wt::WIOService _ioService;
boost::asio::system_timer _scheduleTimer {_ioService};
Wt::Signal<Stats> _sigScanComplete;
boost::asio::system_timer _scheduleTimer;
Wt::Signal<Stats> _sigScanInProgress;
std::chrono::system_clock::time_point _lastScanInProgressEmit {};
Wt::Signal<Wt::WDateTime> _sigScheduled;
Database::Handler _db;
MetaData::TagLibParser _metadataParser;
std::vector<MediaScannerAddon*> _addons;
std::mutex _statusMutex;
State _curState {State::NotScheduled};
boost::optional<Stats> _inProgressStats;
boost::optional<Stats> _lastScanStats;
Wt::WDateTime _nextScheduledScan;
// Current scan settings
std::size_t _scanVersion;
std::size_t _scanVersion {};
Wt::WTime _startTime;
Database::ScanSettings::UpdatePeriod _updatePeriod;
Database::ScanSettings::UpdatePeriod _updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
std::set<boost::filesystem::path> _fileExtensions;
boost::filesystem::path _mediaDirectory;
MetaData::TagLibParser _metadataParser;
std::vector<MediaScannerAddon*> _addons;
}; // class MediaScanner
src/ui/LmsApplication.cpp
+27 -7
View File
@@ -458,16 +458,36 @@ LmsApplication::createHome()
});
// Events from MediaScanner
std::string sessionId = LmsApp->sessionId();
getService<Scanner::MediaScanner>()->scanComplete().connect([=] (Scanner::MediaScanner::Stats stats)
{
// Runs from media scanner context
Wt::WServer::instance()->post(sessionId, [=]
std::string sessionId = LmsApp->sessionId();
getService<Scanner::MediaScanner>()->scanComplete().connect(this, [=] (Scanner::MediaScanner::Stats stats)
{
_events.dbScanned.emit(stats);
triggerUpdate();
Wt::WServer::instance()->post(sessionId, [=]
{
_events.dbScanned.emit(stats);
triggerUpdate();
});
});
});
getService<Scanner::MediaScanner>()->scanInProgress().connect(this, [=] (Scanner::MediaScanner::Stats stats)
{
Wt::WServer::instance()->post(sessionId, [=]
{
_events.dbScanInProgress.emit(stats);
triggerUpdate();
});
});
getService<Scanner::MediaScanner>()->scheduled().connect(this, [=] (Wt::WDateTime dateTime)
{
Wt::WServer::instance()->post(sessionId, [=]
{
_events.dbScanScheduled.emit(dateTime);
triggerUpdate();
});
});
}
_events.dbScanned.connect([=] (Scanner::MediaScanner::Stats stats)
{
src/ui/LmsApplication.hpp
+3 -1
View File
@@ -55,8 +55,10 @@ struct Events
// Unload current track
Wt::Signal<> trackUnloaded;
// A database scan is complete
// Database events
Wt::Signal<Scanner::MediaScanner::Stats> dbScanned;
Wt::Signal<Scanner::MediaScanner::Stats> dbScanInProgress;
Wt::Signal<Wt::WDateTime> dbScanScheduled;
};
// Used to classify the message sent to the user
src/ui/admin/DatabaseSettingsView.cpp
+88 -3
View File
@@ -19,10 +19,14 @@
#include "DatabaseSettingsView.hpp"
#include <ostream>
#include <iomanip>
#include <Wt/WComboBox.h>
#include <Wt/WFormModel.h>
#include <Wt/WLineEdit.h>
#include <Wt/WPushButton.h>
#include <Wt/WServer.h>
#include <Wt/WString.h>
#include <Wt/WTemplateFormView.h>
@@ -62,7 +66,7 @@ class DatabaseSettingsModel : public Wt::WFormModel
addField(SimilarityEngineTypeField);
addField(TagsField);
auto dirValidator = std::make_shared<DirectoryValidator>();
auto dirValidator {std::make_shared<DirectoryValidator>()};
dirValidator->setMandatory(true);
setValidator(MediaDirectoryField, dirValidator);
@@ -174,6 +178,85 @@ const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField = "upda
const Wt::WFormModel::Field DatabaseSettingsModel::SimilarityEngineTypeField = "similarity-engine-type";
const Wt::WFormModel::Field DatabaseSettingsModel::TagsField = "tags";
static
std::string durationToString(const Wt::WDateTime& begin, const Wt::WDateTime& end)
{
auto secs {std::chrono::duration_cast<std::chrono::seconds>(end.toTimePoint() - begin.toTimePoint()).count()};
std::ostringstream oss;
if (secs >= 3600)
oss << secs/3600 << "h";
if (secs >= 60)
oss << std::setw(2) << std::setfill('0') << (secs % 3600) / 60 << "m";
oss << std::setw(2) << std::setfill('0') << (secs % 60) << "s";
return oss.str();
}
class DatabaseStatus : public Wt::WTemplate
{
public:
DatabaseStatus(): WTemplate {Wt::WString::tr("Lms.Admin.Database.Status.template")}
{
addFunction("tr", &Wt::WTemplate::Functions::tr);
using namespace Scanner;
auto onDbEvent = [&]() { refreshContents(); };
LmsApp->getEvents().dbScanned.connect(this, onDbEvent);
LmsApp->getEvents().dbScanInProgress.connect(this, onDbEvent);
LmsApp->getEvents().dbScanScheduled.connect(this, onDbEvent);
refreshContents();
}
private:
void refreshContents()
{
using namespace Scanner;
MediaScanner::Status status {getService<MediaScanner>()->getStatus()};
if (status.lastScanStats)
{
bindString("last-scan", Wt::WString::tr("Lms.Admin.Database.Status.last-scan-status")
.arg(status.lastScanStats->totalFiles)
.arg(durationToString(status.lastScanStats->startTime, status.lastScanStats->stopTime))
.arg(status.lastScanStats->stopTime.toString())
.arg(status.lastScanStats->nbErrors())
);
}
else
{
bindString("last-scan", Wt::WString::tr("Lms.Admin.Database.Status.last-scan-not-available"));
}
switch (status.currentState)
{
case MediaScanner::State::NotScheduled:
bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-not-scheduled"));
break;
case MediaScanner::State::Scheduled:
bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-scheduled")
.arg(status.nextScheduledScan.toString()));
break;
case MediaScanner::State::InProgress:
{
std::ostringstream oss;
bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-in-progress")
.arg(status.inProgressStats->nbFiles())
.arg(status.inProgressStats->totalFiles)
.arg(static_cast<int>(status.inProgressStats->nbFiles() / static_cast<float>(status.inProgressStats->totalFiles ? status.inProgressStats->totalFiles : 1) * 100)));
}
break;
}
}
private:
};
DatabaseSettingsView::DatabaseSettingsView()
{
wApp->internalPathChanged().connect(std::bind([=]
@@ -221,6 +304,8 @@ DatabaseSettingsView::refreshView()
Wt::WPushButton *discardBtn = t->bindWidget("discard-btn", std::make_unique<Wt::WPushButton>(Wt::WString::tr("Lms.discard")));
Wt::WPushButton *immScanBtn = t->bindWidget("immediate-scan-btn", std::make_unique<Wt::WPushButton>(Wt::WString::tr("Lms.Admin.Database.immediate-scan")));
t->bindNew<DatabaseStatus>("status");
saveBtn->clicked().connect([=] ()
{
t->updateModel(model.get());
@@ -229,7 +314,7 @@ DatabaseSettingsView::refreshView()
{
model->saveData();
getService<Scanner::MediaScanner>()->reschedule();
getService<Scanner::MediaScanner>()->requestReschedule();
LmsApp->notifyMsg(MsgType::Success, Wt::WString::tr("Lms.Admin.Database.settings-saved"));
}
@@ -246,7 +331,7 @@ DatabaseSettingsView::refreshView()
immScanBtn->clicked().connect([=] ()
{
getService<Scanner::MediaScanner>()->scheduleImmediateScan();
getService<Scanner::MediaScanner>()->requestImmediateScan();
LmsApp->notifyMsg(MsgType::Info, Wt::WString::tr("Lms.Admin.Database.scan-launched"));
});