owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added a scan report button (Fixes #4), also fixed the regression about files being always rescanned even if no change has been made

Browse Source
This commit is contained in:
emeric
2019-10-10 13:05:05 +02:00
parent 736e1bd9ec
commit 322f90dc93
14 changed files with 517 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1,4 +1,5 @@
*.sw? *.sw?
*.*~
*.in *.in
build/ build/
Makefile.in Makefile.in
approot/admin-database.xml
+36 -22
View File
@@ -76,28 +76,12 @@
</div> </div>
</div> </div>
</div> </div>
${status} <br/>
</message> <div class="row">
<div class="col-sm-7">
<message id="Lms.Admin.Database.Status.template"> <div class="panel panel-info">
<legend>${tr:Lms.Admin.Database.Status.status}</legend> <div class="panel-heading">${tr:Lms.Admin.Database.Status.status}</div>
<div class="form-horizontal"> <div class="panel-body">
<div class="form-group">
<label class="control-label col-sm-2" for="${id:last-scan}">
${tr:Lms.Admin.Database.Status.last-scan}
</label>
<div class="col-sm-5">
<div class="well well-sm">
${last-scan}
</div>
</div>
</div>
<div class="form-group">
<label class="control-label col-sm-2" for="${id:status}">
${tr:Lms.Admin.Database.Status.status}
</label>
<div class="col-sm-5">
<div class="well well-sm">
${status} ${status}
</div> </div>
</div> </div>
@@ -105,4 +89,34 @@
</div> </div>
</message> </message>
<message id="Lms.Admin.Database.Status.template">
<div class="form-horizontal">
<div class="form-group">
<label class="control-label col-sm-2">
${tr:Lms.Admin.Database.Status.status}
</label>
<div class="col-sm-10">
<div class="well well-sm">
${status}
</div>
</div>
</div>
<div class="form-group">
<label class="control-label col-sm-2">
${tr:Lms.Admin.Database.Status.last-scan}
</label>
<div class="col-sm-10">
<div class="well well-sm">
${last-scan}
</div>
</div>
</div>
<div class="form-group">
<div class="col-sm-offset-2 col-sm-10">
${btn-report class="btn"}
</div>
</div>
</div>
</message>
</messages> </messages>
approot/messages.xml
+11 -2
View File
@@ -51,10 +51,19 @@
<message id="Lms.Admin.Database.update-start-time">Update start time</message> <message id="Lms.Admin.Database.update-start-time">Update start time</message>
<message id="Lms.Admin.Database.weekly">Weekly</message> <message id="Lms.Admin.Database.weekly">Weekly</message>
<message id="Lms.Admin.Database.Status.status">Status</message> <message id="Lms.Admin.Database.Status.bad-duration">Cannot get track duration</message>
<message id="Lms.Admin.Database.Status.cannot-parse-file">Cannot parse file</message>
<message id="Lms.Admin.Database.Status.cannot-read-file">Cannot read file</message>
<message id="Lms.Admin.Database.Status.duplicates-header">{1} duplicate files:</message>
<message id="Lms.Admin.Database.Status.errors-header">{1} errors:</message>
<message id="Lms.Admin.Database.Status.get-report">Get report</message>
<message id="Lms.Admin.Database.Status.last-scan">Last scan</message> <message id="Lms.Admin.Database.Status.last-scan">Last scan</message>
<message id="Lms.Admin.Database.Status.last-scan-not-available">Not available</message> <message id="Lms.Admin.Database.Status.last-scan-not-available">Not available</message>
<message id="Lms.Admin.Database.Status.last-scan-status">Scanned {1} files in {2} on {3} ({4} errors)</message> <message id="Lms.Admin.Database.Status.last-scan-status">Scanned {1} files in {2} on {3} ({4} errors, {5} duplicates)</message>
<message id="Lms.Admin.Database.Status.no-audio-track">No audio track</message>
<message id="Lms.Admin.Database.Status.same-hash">Duplicated file hash</message>
<message id="Lms.Admin.Database.Status.same-mbid">Duplicated MBID</message>
<message id="Lms.Admin.Database.Status.status">Status</message>
<message id="Lms.Admin.Database.Status.status-not-scheduled">Not scheduled</message> <message id="Lms.Admin.Database.Status.status-not-scheduled">Not scheduled</message>
<message id="Lms.Admin.Database.Status.status-scheduled">Scheduled on {1}</message> <message id="Lms.Admin.Database.Status.status-scheduled">Scheduled on {1}</message>
<message id="Lms.Admin.Database.Status.status-in-progress">Scanning {1} of {2} files ({3} %)</message> <message id="Lms.Admin.Database.Status.status-in-progress">Scanning {1} of {2} files ({3} %)</message>
approot/messages_fr.xml
+11 -2
View File
@@ -51,10 +51,19 @@
<message id="Lms.Admin.Database.update-start-time">Heure de départ de la mise à jour</message> <message id="Lms.Admin.Database.update-start-time">Heure de départ de la mise à jour</message>
<message id="Lms.Admin.Database.weekly">Toutes les semaines</message> <message id="Lms.Admin.Database.weekly">Toutes les semaines</message>
<message id="Lms.Admin.Database.Status.status">Statut</message> <message id="Lms.Admin.Database.Status.bad-duration">Impossible de récupérer la durée de la piste</message>
<message id="Lms.Admin.Database.Status.cannot-parse-file">Impossible d'analyser le fichier</message>
<message id="Lms.Admin.Database.Status.cannot-read-file">Impossible de lire le fichier</message>
<message id="Lms.Admin.Database.Status.duplicates-header">{1} fichiers dupliqués :</message>
<message id="Lms.Admin.Database.Status.errors-header">{1} erreurs :</message>
<message id="Lms.Admin.Database.Status.get-report">Rapport</message>
<message id="Lms.Admin.Database.Status.last-scan">Dernier scan</message> <message id="Lms.Admin.Database.Status.last-scan">Dernier scan</message>
<message id="Lms.Admin.Database.Status.last-scan-not-available">Non disponible</message> <message id="Lms.Admin.Database.Status.last-scan-not-available">Non disponible</message>
<message id="Lms.Admin.Database.Status.last-scan-status">{1} fichiers scannés en {2} le {3} ({4} erreurs)</message> <message id="Lms.Admin.Database.Status.last-scan-status">{1} fichiers scannés en {2} le {3} ({4} erreurs, {5} duplicatas)</message>
<message id="Lms.Admin.Database.Status.no-audio-track">Pas de piste audio</message>
<message id="Lms.Admin.Database.Status.same-hash">Hash dupliqué</message>
<message id="Lms.Admin.Database.Status.same-mbid">MBID dupliqué</message>
<message id="Lms.Admin.Database.Status.status">Statut</message>
<message id="Lms.Admin.Database.Status.status-not-scheduled">Non planifié</message> <message id="Lms.Admin.Database.Status.status-not-scheduled">Non planifié</message>
<message id="Lms.Admin.Database.Status.status-scheduled">Planifié le {1}</message> <message id="Lms.Admin.Database.Status.status-scheduled">Planifié le {1}</message>
<message id="Lms.Admin.Database.Status.status-in-progress">Scan de {1} fichiers sur {2} ({3} %)</message> <message id="Lms.Admin.Database.Status.status-in-progress">Scan de {1} fichiers sur {2} ({3} %)</message>
src/Makefile.am
+4
View File
@@ -59,6 +59,8 @@ lms_SOURCES = \
$(srcdir)/metadata/TagLibParser.hpp \ $(srcdir)/metadata/TagLibParser.hpp \
$(srcdir)/scanner/MediaScanner.cpp \ $(srcdir)/scanner/MediaScanner.cpp \
$(srcdir)/scanner/MediaScanner.hpp \ $(srcdir)/scanner/MediaScanner.hpp \
$(srcdir)/scanner/MediaScannerStats.cpp \
$(srcdir)/scanner/MediaScannerStats.hpp \
$(srcdir)/scanner/MediaScannerAddon.hpp \ $(srcdir)/scanner/MediaScannerAddon.hpp \
$(srcdir)/similarity/SimilaritySearcher.cpp \ $(srcdir)/similarity/SimilaritySearcher.cpp \
$(srcdir)/similarity/SimilaritySearcher.hpp \ $(srcdir)/similarity/SimilaritySearcher.hpp \
@@ -95,6 +97,8 @@ lms_SOURCES = \
$(srcdir)/ui/SettingsView.hpp \ $(srcdir)/ui/SettingsView.hpp \
$(srcdir)/ui/admin/DatabaseSettingsView.cpp \ $(srcdir)/ui/admin/DatabaseSettingsView.cpp \
$(srcdir)/ui/admin/DatabaseSettingsView.hpp \ $(srcdir)/ui/admin/DatabaseSettingsView.hpp \
$(srcdir)/ui/admin/DatabaseStatus.cpp \
$(srcdir)/ui/admin/DatabaseStatus.hpp \
$(srcdir)/ui/admin/InitWizardView.cpp \ $(srcdir)/ui/admin/InitWizardView.cpp \
$(srcdir)/ui/admin/InitWizardView.hpp \ $(srcdir)/ui/admin/InitWizardView.hpp \
$(srcdir)/ui/admin/UserView.cpp \ $(srcdir)/ui/admin/UserView.cpp \
src/scanner/MediaScanner.cpp
+44 -39
View File
@@ -19,8 +19,6 @@
#include "MediaScanner.hpp" #include "MediaScanner.hpp"
#include <stdexcept>
#include <boost/asio/placeholders.hpp> #include <boost/asio/placeholders.hpp>
#include <Wt/WLocalDateTime.h> #include <Wt/WLocalDateTime.h>
@@ -72,7 +70,7 @@ isPathInParentPath(const std::filesystem::path& path, const std::filesystem::pat
{ {
std::filesystem::path curPath = path; std::filesystem::path curPath = path;
while (curPath.has_parent_path()) while (curPath.parent_path() != curPath)
{ {
curPath = curPath.parent_path(); curPath = curPath.parent_path();
@@ -260,10 +258,11 @@ MediaScanner::getStatus()
Status res; Status res;
std::unique_lock<std::mutex> lock {_statusMutex}; std::unique_lock<std::mutex> lock {_statusMutex};
res.currentState = _curState; res.currentState = _curState;
res.nextScheduledScan = _nextScheduledScan; res.nextScheduledScan = _nextScheduledScan;
res.lastScanStats = _lastScanStats; res.lastCompleteScanStats = _lastCompleteScanStats;
res.inProgressStats = _inProgressStats; res.inProgressScanStats = _inProgressScanStats;
return res; return res;
} }
@@ -275,7 +274,7 @@ MediaScanner::scheduleNextScan()
refreshScanSettings(); refreshScanSettings();
Wt::WDateTime now = Wt::WLocalDateTime::currentServerDateTime().toUTC(); Wt::WDateTime now {Wt::WLocalDateTime::currentServerDateTime().toUTC()};
Wt::WDate nextScanDate; Wt::WDate nextScanDate;
switch (_updatePeriod) switch (_updatePeriod)
@@ -324,11 +323,11 @@ MediaScanner::scheduleNextScan()
} }
void void
MediaScanner::countAllFiles(Stats& stats) MediaScanner::countAllFiles(ScanStats& stats)
{ {
std::error_code ec; std::error_code ec;
stats.totalFiles = 0; stats.filesToScan = 0;
std::filesystem::recursive_directory_iterator itPath {_mediaDirectory, ec}; std::filesystem::recursive_directory_iterator itPath {_mediaDirectory, ec};
if (ec) if (ec)
@@ -345,9 +344,9 @@ MediaScanner::countAllFiles(Stats& stats)
if (!ec) if (!ec)
{ {
if (std::filesystem::is_regular_file(path) && isFileSupported(path, _fileExtensions)) if (std::filesystem::is_regular_file(path) && isFileSupported(path, _fileExtensions))
stats.totalFiles++; stats.filesToScan ++;
if (stats.totalFiles % 250 == 0) if (stats.filesToScan % 250 == 0)
notifyInProgressIfNeeded(stats); notifyInProgressIfNeeded(stats);
} }
@@ -385,11 +384,10 @@ MediaScanner::scan(boost::system::error_code err)
std::unique_lock<std::mutex> lock {_statusMutex}; std::unique_lock<std::mutex> lock {_statusMutex};
_curState = State::InProgress; _curState = State::InProgress;
_nextScheduledScan = {}; _nextScheduledScan = {};
_inProgressStats = Stats{};
_inProgressStats->startTime = Wt::WLocalDateTime::currentDateTime().toUTC();
} }
Stats& stats {*_inProgressStats}; ScanStats stats;
stats.startTime = Wt::WLocalDateTime::currentDateTime().toUTC();
LMS_LOG(UI, INFO) << "New scan started!"; LMS_LOG(UI, INFO) << "New scan started!";
@@ -399,7 +397,7 @@ MediaScanner::scan(boost::system::error_code err)
LMS_LOG(DBUPDATER, DEBUG) << "Counting files in media directory '" << _mediaDirectory.string() << "'..."; LMS_LOG(DBUPDATER, DEBUG) << "Counting files in media directory '" << _mediaDirectory.string() << "'...";
countAllFiles(stats); countAllFiles(stats);
LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.totalFiles; LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesToScan;
removeMissingTracks(stats); removeMissingTracks(stats);
@@ -414,7 +412,7 @@ MediaScanner::scan(boost::system::error_code err)
if (_running) if (_running)
checkDuplicatedAudioFiles(stats); checkDuplicatedAudioFiles(stats);
LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.scanErrors << ", not imported = " << stats.incompleteScans << "), duplicates = " << stats.nbDuplicates() << " (hash = " << stats.duplicateHashes << ", mbid = " << stats.duplicateMBID << ")"; LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), duplicates = " << stats.duplicates.size();
if (_running) if (_running)
{ {
@@ -427,19 +425,21 @@ MediaScanner::scan(boost::system::error_code err)
stats.stopTime = Wt::WLocalDateTime::currentDateTime().toUTC(); stats.stopTime = Wt::WLocalDateTime::currentDateTime().toUTC();
{ {
std::unique_lock<std::mutex> lock {_statusMutex}; std::unique_lock<std::mutex> lock {_statusMutex};
_lastScanStats = std::move(_inProgressStats);
_inProgressStats.reset(); _lastCompleteScanStats = std::move(stats);
_inProgressScanStats.reset();
} }
scheduleNextScan(); scheduleNextScan();
scanComplete().emit(stats); scanComplete().emit();
} }
else else
{ {
std::unique_lock<std::mutex> lock {_statusMutex}; std::unique_lock<std::mutex> lock {_statusMutex};
_curState = State::NotScheduled; _curState = State::NotScheduled;
_inProgressStats.reset(); _inProgressScanStats.reset();
} }
LMS_LOG(DBUPDATER, INFO) << "Optimizing db..."; LMS_LOG(DBUPDATER, INFO) << "Optimizing db...";
@@ -478,39 +478,44 @@ MediaScanner::refreshScanSettings()
addon->refreshSettings(); addon->refreshSettings();
} }
void MediaScanner::notifyInProgress(Stats& stats) void
MediaScanner::notifyInProgress(const ScanStats& stats)
{ {
{
std::unique_lock<std::mutex> lock {_statusMutex};
_inProgressScanStats = stats.toProgressStats();
}
std::chrono::system_clock::time_point now {std::chrono::system_clock::now()}; std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
_sigScanInProgress(stats); _sigScanInProgress(*_inProgressScanStats);
_lastScanInProgressEmit = now; _lastScanInProgressEmit = now;
} }
void MediaScanner::notifyInProgressIfNeeded(Stats& stats) void
MediaScanner::notifyInProgressIfNeeded(const ScanStats& stats)
{ {
std::chrono::system_clock::time_point now {std::chrono::system_clock::now()}; std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
if (std::chrono::duration_cast<std::chrono::seconds>(now - _lastScanInProgressEmit).count() > 2) if (std::chrono::duration_cast<std::chrono::seconds>(now - _lastScanInProgressEmit).count() > 2)
{ notifyInProgress(stats);
_sigScanInProgress(stats);
_lastScanInProgressEmit = now;
}
} }
void void
MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, Stats& stats) MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats)
{ {
notifyInProgressIfNeeded(stats); notifyInProgressIfNeeded(stats);
const auto lastWriteTime {Wt::WDateTime::fromTimePoint(std::filesystem::last_write_time(file))}; const Wt::WDateTime lastWriteTime {std::filesystem::last_write_time(file)};
if (!forceScan) if (!forceScan)
{ {
// Skip file if last write is the same // Skip file if last write is the same
auto transaction {_dbSession->createSharedTransaction()}; auto transaction {_dbSession->createSharedTransaction()};
Track::pointer track {Track::getByPath(*_dbSession, file)}; const Track::pointer track {Track::getByPath(*_dbSession, file)};
if (track && track->getLastWriteTime() == lastWriteTime && track->getScanVersion() == _scanVersion) if (track && track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t()
&& track->getScanVersion() == _scanVersion)
{ {
stats.skips++; stats.skips++;
return; return;
@@ -520,7 +525,7 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S
std::optional<MetaData::Track> trackInfo {_metadataParser.parse(file)}; std::optional<MetaData::Track> trackInfo {_metadataParser.parse(file)};
if (!trackInfo) if (!trackInfo)
{ {
stats.scanErrors++; stats.errors.emplace_back(file, ScanErrorType::CannotParseFile);
return; return;
} }
@@ -543,7 +548,7 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S
track.remove(); track.remove();
stats.deletions++; stats.deletions++;
} }
stats.incompleteScans++; stats.errors.emplace_back(ScanError {file, ScanErrorType::NoAudioTrack});
return; return;
} }
if (trackInfo->duration == std::chrono::milliseconds::zero()) if (trackInfo->duration == std::chrono::milliseconds::zero())
@@ -556,7 +561,7 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S
track.remove(); track.remove();
stats.deletions++; stats.deletions++;
} }
stats.incompleteScans++; stats.errors.emplace_back(ScanError {file, ScanErrorType::BadDuration});
return; return;
} }
@@ -641,7 +646,7 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S
} }
void void
MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, Stats& stats) MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats)
{ {
std::error_code ec; std::error_code ec;
@@ -649,7 +654,7 @@ MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bo
if (ec) if (ec)
{ {
LMS_LOG(DBUPDATER, ERROR) << "Cannot iterate over '" << mediaDirectory.string() << "': " << ec.message(); LMS_LOG(DBUPDATER, ERROR) << "Cannot iterate over '" << mediaDirectory.string() << "': " << ec.message();
stats.scanErrors++; stats.errors.emplace_back(ScanError {mediaDirectory, ScanErrorType::CannotReadFile, ec.message()});
return; return;
} }
@@ -661,7 +666,7 @@ MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bo
if (ec) if (ec)
{ {
LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message(); LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message();
stats.scanErrors++; stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()});
} }
else if (std::filesystem::is_regular_file(path)) else if (std::filesystem::is_regular_file(path))
{ {
@@ -713,7 +718,7 @@ checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDire
} }
void void
MediaScanner::removeMissingTracks(Stats& stats) MediaScanner::removeMissingTracks(ScanStats& stats)
{ {
std::vector<std::filesystem::path> trackPaths; std::vector<std::filesystem::path> trackPaths;
{ {
@@ -785,7 +790,7 @@ MediaScanner::removeOrphanEntries()
} }
void void
MediaScanner::checkDuplicatedAudioFiles(Stats& stats) MediaScanner::checkDuplicatedAudioFiles(ScanStats& stats)
{ {
LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files"; LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files";
@@ -795,7 +800,7 @@ MediaScanner::checkDuplicatedAudioFiles(Stats& stats)
for (const Track::pointer& track : tracks) for (const Track::pointer& track : tracks)
{ {
LMS_LOG(DBUPDATER, INFO) << "Found duplicated MBID [" << track->getMBID() << "], file: " << track->getPath().string() << " - " << track->getName(); LMS_LOG(DBUPDATER, INFO) << "Found duplicated MBID [" << track->getMBID() << "], file: " << track->getPath().string() << " - " << track->getName();
stats.duplicateMBID++; stats.duplicates.emplace_back(ScanDuplicate {track->getPath(), DuplicateReason::SameMBID});
} }
LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!"; LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!";
src/scanner/MediaScanner.hpp
+29 -51
View File
@@ -34,6 +34,7 @@
#include "metadata/TagLibParser.hpp" #include "metadata/TagLibParser.hpp"
#include "MediaScannerAddon.hpp" #include "MediaScannerAddon.hpp"
#include "MediaScannerStats.hpp"
namespace Scanner { namespace Scanner {
@@ -52,29 +53,6 @@ class MediaScanner
void requestImmediateScan(); void requestImmediateScan();
void requestReschedule(); void requestReschedule();
struct Stats
{
Wt::WDateTime startTime;
Wt::WDateTime stopTime;
std::size_t skips = 0; // no change since last scan
std::size_t scans = 0; // actually scanned filed
std::size_t scanErrors = 0; // cannot scan file
std::size_t incompleteScans = 0; // Scanned, but not imported (criteria not filled)
std::size_t additions = 0; // Added in DB
std::size_t deletions = 0; // removed from DB
std::size_t updates = 0; // updated file in DB
std::size_t duplicateHashes = 0; // Same file hashes
std::size_t duplicateMBID = 0; // Same MBID
std::size_t totalFiles = 0; // Total number of files to be scanned
std::size_t nbFiles() const { return skips + additions + updates; }
std::size_t nbChanges() const { return additions + deletions + updates; }
std::size_t nbErrors() const { return scanErrors + incompleteScans; }
std::size_t nbDuplicates() const { return duplicateHashes + duplicateMBID; }
float progress() const { return (nbFiles() - totalFiles) / static_cast<float>(totalFiles); }
};
enum class State enum class State
{ {
@@ -85,19 +63,19 @@ class MediaScanner
struct Status struct Status
{ {
State currentState {State::NotScheduled}; State currentState {State::NotScheduled};
Wt::WDateTime nextScheduledScan; Wt::WDateTime nextScheduledScan;
std::optional<Stats> lastScanStats; std::optional<ScanStats> lastCompleteScanStats;
std::optional<Stats> inProgressStats; std::optional<ScanProgressStats> inProgressScanStats;
}; };
Status getStatus(); Status getStatus();
// Called just after scan complete // Called just after scan complete
Wt::Signal<Stats>& scanComplete() { return _sigScanComplete; } Wt::Signal<>& scanComplete() { return _sigScanComplete; }
// Called during scan in progress // Called during scan in progress
Wt::Signal<Stats>& scanInProgress() { return _sigScanInProgress; } Wt::Signal<ScanProgressStats>& scanInProgress() { return _sigScanInProgress; }
// Called after a schedule // Called after a schedule
Wt::Signal<Wt::WDateTime>& scheduled() { return _sigScheduled; } Wt::Signal<Wt::WDateTime>& scheduled() { return _sigScheduled; }
@@ -111,36 +89,36 @@ class MediaScanner
// Update database (scheduled callback) // Update database (scheduled callback)
void scan(boost::system::error_code ec); void scan(boost::system::error_code ec);
void scanMediaDirectory( const std::filesystem::path& mediaDirectory, bool forceScan, Stats& stats); void scanMediaDirectory( const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats);
// Helpers // Helpers
void refreshScanSettings(); void refreshScanSettings();
void countAllFiles(Stats& stats); void countAllFiles(ScanStats& stats);
void removeMissingTracks(Stats& stats); void removeMissingTracks(ScanStats& stats);
void removeOrphanEntries(); void removeOrphanEntries();
void checkDuplicatedAudioFiles(Stats& stats); void checkDuplicatedAudioFiles(ScanStats& stats);
void scanAudioFile(const std::filesystem::path& file, bool forceScan, Stats& stats); void scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats);
Database::IdType doScanAudioFile(const std::filesystem::path& file, Stats& stats); Database::IdType doScanAudioFile(const std::filesystem::path& file, ScanStats& stats);
void notifyInProgressIfNeeded(Stats& stats); void notifyInProgressIfNeeded(const ScanStats& stats);
void notifyInProgress(Stats& stats); void notifyInProgress(const ScanStats& stats);
bool _running {false}; bool _running {false};
Wt::WIOService _ioService; Wt::WIOService _ioService;
boost::asio::system_timer _scheduleTimer {_ioService}; boost::asio::system_timer _scheduleTimer {_ioService};
Wt::Signal<Stats> _sigScanComplete; Wt::Signal<> _sigScanComplete;
Wt::Signal<Stats> _sigScanInProgress; Wt::Signal<ScanProgressStats> _sigScanInProgress;
std::chrono::system_clock::time_point _lastScanInProgressEmit {}; std::chrono::system_clock::time_point _lastScanInProgressEmit {};
Wt::Signal<Wt::WDateTime> _sigScheduled; Wt::Signal<Wt::WDateTime> _sigScheduled;
std::unique_ptr<Database::Session> _dbSession; std::unique_ptr<Database::Session> _dbSession;
MetaData::TagLibParser _metadataParser; MetaData::TagLibParser _metadataParser;
std::vector<MediaScannerAddon*> _addons; std::vector<MediaScannerAddon*> _addons;
std::mutex _statusMutex; std::mutex _statusMutex;
State _curState {State::NotScheduled}; State _curState {State::NotScheduled};
std::optional<Stats> _inProgressStats; std::optional<ScanStats> _lastCompleteScanStats;
std::optional<Stats> _lastScanStats; std::optional<ScanProgressStats> _inProgressScanStats;
Wt::WDateTime _nextScheduledScan; Wt::WDateTime _nextScheduledScan;
// Current scan settings // Current scan settings
std::size_t _scanVersion {}; std::size_t _scanVersion {};
src/scanner/MediaScannerStats.cpp
+56
View File
@@ -0,0 +1,56 @@
/*
* Copyright (C) 2019 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "MediaScannerStats.hpp"
namespace Scanner {
ScanError::ScanError(const std::filesystem::path& _file, ScanErrorType _error, const std::string& _systemError)
: file {_file},
error {_error},
systemError {_systemError}
{
}
std::size_t
ScanStats::nbFiles() const
{
return skips + additions + updates;
}
std::size_t
ScanStats::nbChanges() const
{
return additions + deletions + updates;
}
ScanProgressStats
ScanStats::toProgressStats() const
{
return ScanProgressStats {startTime, filesToScan, nbFiles()};
}
unsigned
ScanProgressStats::progress() const
{
return (processedFiles / static_cast<float>(filesToScan ? filesToScan : 1)) * 100;
}
} // namespace Scanner
src/scanner/MediaScannerStats.hpp
+94
View File
@@ -0,0 +1,94 @@
/*
* Copyright (C) 2019 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <Wt/WDateTime.h>
#include <filesystem>
#include <vector>
namespace Scanner {
enum class ScanErrorType
{
CannotReadFile, // cannot read file
CannotParseFile, // cannot parse file
NoAudioTrack, // no audio track found
BadDuration, // bad duration
};
enum class DuplicateReason
{
SameHash,
SameMBID,
};
struct ScanError
{
std::filesystem::path file;
ScanErrorType error;
std::string systemError;
ScanError(const std::filesystem::path& file, ScanErrorType error, const std::string& systemError = "");
};
struct ScanDuplicate
{
std::filesystem::path file;
DuplicateReason reason;
};
// reduced scan stats
struct ScanProgressStats
{
Wt::WDateTime startTime;
std::size_t filesToScan {};
std::size_t processedFiles {};
unsigned progress() const;
};
struct ScanStats
{
Wt::WDateTime startTime;
Wt::WDateTime stopTime;
std::size_t filesToScan {}; // Total number of files to be scanned (estimated)
std::size_t skips {}; // no change since last scan
std::size_t scans {}; // actually scanned filed
std::size_t additions {}; // Added in DB
std::size_t deletions {}; // removed from DB
std::size_t updates {}; // updated file in DB
std::vector<ScanError> errors;
std::vector<ScanDuplicate> duplicates;
std::size_t nbFiles() const;
std::size_t nbChanges() const;
ScanProgressStats toProgressStats() const;
};
}
src/ui/LmsApplication.cpp
+8 -6
View File
@@ -529,16 +529,16 @@ LmsApplication::createHome()
// Events from MediaScanner // Events from MediaScanner
{ {
const std::string sessionId {LmsApp->sessionId()}; const std::string sessionId {LmsApp->sessionId()};
getService<Scanner::MediaScanner>()->scanComplete().connect(this, [=] (Scanner::MediaScanner::Stats stats) getService<Scanner::MediaScanner>()->scanComplete().connect(this, [=] ()
{ {
Wt::WServer::instance()->post(sessionId, [=] Wt::WServer::instance()->post(sessionId, [=]
{ {
_events.dbScanned.emit(stats); _events.dbScanned.emit();
triggerUpdate(); triggerUpdate();
}); });
}); });
getService<Scanner::MediaScanner>()->scanInProgress().connect(this, [=] (Scanner::MediaScanner::Stats stats) getService<Scanner::MediaScanner>()->scanInProgress().connect(this, [=] (Scanner::ScanProgressStats stats)
{ {
Wt::WServer::instance()->post(sessionId, [=] Wt::WServer::instance()->post(sessionId, [=]
{ {
@@ -558,17 +558,19 @@ LmsApplication::createHome()
} }
_events.dbScanned.connect([=] (Scanner::MediaScanner::Stats stats) _events.dbScanned.connect([=] ()
{ {
if (isUserAdmin()) if (isUserAdmin())
{ {
const auto& stats {*getService<Scanner::MediaScanner>()->getStatus().lastCompleteScanStats};
notifyMsg(MsgType::Info, Wt::WString::tr("Lms.Admin.Database.scan-complete") notifyMsg(MsgType::Info, Wt::WString::tr("Lms.Admin.Database.scan-complete")
.arg(static_cast<unsigned>(stats.nbFiles())) .arg(static_cast<unsigned>(stats.nbFiles()))
.arg(static_cast<unsigned>(stats.additions)) .arg(static_cast<unsigned>(stats.additions))
.arg(static_cast<unsigned>(stats.updates)) .arg(static_cast<unsigned>(stats.updates))
.arg(static_cast<unsigned>(stats.deletions)) .arg(static_cast<unsigned>(stats.deletions))
.arg(static_cast<unsigned>(stats.nbDuplicates())) .arg(static_cast<unsigned>(stats.duplicates.size()))
.arg(static_cast<unsigned>(stats.nbErrors()))); .arg(static_cast<unsigned>(stats.errors.size())));
} }
}); });
src/ui/LmsApplication.hpp
+2 -2
View File
@@ -56,8 +56,8 @@ struct Events
Wt::Signal<> trackUnloaded; Wt::Signal<> trackUnloaded;
// Database events // Database events
Wt::Signal<Scanner::MediaScanner::Stats> dbScanned; Wt::Signal<> dbScanned;
Wt::Signal<Scanner::MediaScanner::Stats> dbScanInProgress; Wt::Signal<Scanner::ScanProgressStats> dbScanInProgress;
Wt::Signal<Wt::WDateTime> dbScanScheduled; Wt::Signal<Wt::WDateTime> dbScanScheduled;
}; };
src/ui/admin/DatabaseSettingsView.cpp
+1 -83
View File
@@ -19,14 +19,10 @@
#include "DatabaseSettingsView.hpp" #include "DatabaseSettingsView.hpp"
#include <ostream>
#include <iomanip>
#include <Wt/WComboBox.h> #include <Wt/WComboBox.h>
#include <Wt/WFormModel.h> #include <Wt/WFormModel.h>
#include <Wt/WLineEdit.h> #include <Wt/WLineEdit.h>
#include <Wt/WPushButton.h> #include <Wt/WPushButton.h>
#include <Wt/WServer.h>
#include <Wt/WString.h> #include <Wt/WString.h>
#include <Wt/WTemplateFormView.h> #include <Wt/WTemplateFormView.h>
@@ -38,6 +34,7 @@
#include "common/Validators.hpp" #include "common/Validators.hpp"
#include "common/ValueStringModel.hpp" #include "common/ValueStringModel.hpp"
#include "DatabaseStatus.hpp"
#include "LmsApplication.hpp" #include "LmsApplication.hpp"
namespace UserInterface { namespace UserInterface {
@@ -178,85 +175,6 @@ const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField = "upda
const Wt::WFormModel::Field DatabaseSettingsModel::SimilarityEngineTypeField = "similarity-engine-type"; const Wt::WFormModel::Field DatabaseSettingsModel::SimilarityEngineTypeField = "similarity-engine-type";
const Wt::WFormModel::Field DatabaseSettingsModel::TagsField = "tags"; const Wt::WFormModel::Field DatabaseSettingsModel::TagsField = "tags";
static
std::string durationToString(const Wt::WDateTime& begin, const Wt::WDateTime& end)
{
auto secs {std::chrono::duration_cast<std::chrono::seconds>(end.toTimePoint() - begin.toTimePoint()).count()};
std::ostringstream oss;
if (secs >= 3600)
oss << secs/3600 << "h";
if (secs >= 60)
oss << std::setw(2) << std::setfill('0') << (secs % 3600) / 60 << "m";
oss << std::setw(2) << std::setfill('0') << (secs % 60) << "s";
return oss.str();
}
class DatabaseStatus : public Wt::WTemplate
{
public:
DatabaseStatus(): WTemplate {Wt::WString::tr("Lms.Admin.Database.Status.template")}
{
addFunction("tr", &Wt::WTemplate::Functions::tr);
using namespace Scanner;
auto onDbEvent = [&]() { refreshContents(); };
LmsApp->getEvents().dbScanned.connect(this, onDbEvent);
LmsApp->getEvents().dbScanInProgress.connect(this, onDbEvent);
LmsApp->getEvents().dbScanScheduled.connect(this, onDbEvent);
refreshContents();
}
private:
void refreshContents()
{
using namespace Scanner;
MediaScanner::Status status {getService<MediaScanner>()->getStatus()};
if (status.lastScanStats)
{
bindString("last-scan", Wt::WString::tr("Lms.Admin.Database.Status.last-scan-status")
.arg(status.lastScanStats->totalFiles)
.arg(durationToString(status.lastScanStats->startTime, status.lastScanStats->stopTime))
.arg(status.lastScanStats->stopTime.toString())
.arg(status.lastScanStats->nbErrors())
);
}
else
{
bindString("last-scan", Wt::WString::tr("Lms.Admin.Database.Status.last-scan-not-available"));
}
switch (status.currentState)
{
case MediaScanner::State::NotScheduled:
bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-not-scheduled"));
break;
case MediaScanner::State::Scheduled:
bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-scheduled")
.arg(status.nextScheduledScan.toString()));
break;
case MediaScanner::State::InProgress:
{
std::ostringstream oss;
bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-in-progress")
.arg(status.inProgressStats->nbFiles())
.arg(status.inProgressStats->totalFiles)
.arg(static_cast<int>(status.inProgressStats->nbFiles() / static_cast<float>(status.inProgressStats->totalFiles ? status.inProgressStats->totalFiles : 1) * 100)));
}
break;
}
}
private:
};
DatabaseSettingsView::DatabaseSettingsView() DatabaseSettingsView::DatabaseSettingsView()
{ {
wApp->internalPathChanged().connect(std::bind([=] wApp->internalPathChanged().connect(std::bind([=]
src/ui/admin/DatabaseStatus.cpp
+184
View File
@@ -0,0 +1,184 @@
/*
* Copyright (C) 2019 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "DatabaseStatus.hpp"
#include <iomanip>
#include <Wt/Http/Response.h>
#include <Wt/WDateTime.h>
#include <Wt/WPushButton.h>
#include <Wt/WResource.h>
#include "main/Service.hpp"
#include "scanner/MediaScanner.hpp"
#include "LmsApplication.hpp"
namespace UserInterface {
static
std::string
durationToString(const Wt::WDateTime& begin, const Wt::WDateTime& end)
{
const auto secs {std::chrono::duration_cast<std::chrono::seconds>(end.toTimePoint() - begin.toTimePoint()).count()};
std::ostringstream oss;
if (secs >= 3600)
oss << secs/3600 << "h";
if (secs >= 60)
oss << std::setw(2) << std::setfill('0') << (secs % 3600) / 60 << "m";
oss << std::setw(2) << std::setfill('0') << (secs % 60) << "s";
return oss.str();
}
class ReportResource : public Wt::WResource
{
public:
ReportResource(const Scanner::ScanStats& stats)
: _stats {stats}
{
suggestFileName("report.txt");
}
~ReportResource()
{
beingDeleted();
}
void handleRequest(const Wt::Http::Request&, Wt::Http::Response& response)
{
response.out() << Wt::WString::tr("Lms.Admin.Database.Status.errors-header").arg(_stats.errors.size()).toUTF8() << std::endl;
for (const auto& error : _stats.errors)
{
response.out() << error.file.string() << " - " << errorTypeToWString(error.error).toUTF8();
if (!error.systemError.empty())
response.out() << ": " << error.systemError;
response.out() << std::endl;
}
response.out() << std::endl;
response.out() << Wt::WString::tr("Lms.Admin.Database.Status.duplicates-header").arg(_stats.duplicates.size()).toUTF8() << std::endl;
for (const auto& duplicate : _stats.duplicates)
response.out() << duplicate.file.string() << " - " << duplicateReasonToWString(duplicate.reason).toUTF8() << std::endl;
}
private:
static Wt::WString errorTypeToWString(Scanner::ScanErrorType error)
{
switch (error)
{
case Scanner::ScanErrorType::CannotReadFile: return Wt::WString::tr("Lms.Admin.Database.Status.cannot-read-file");
case Scanner::ScanErrorType::CannotParseFile: return Wt::WString::tr("Lms.Admin.Database.Status.cannot-parse-file");
case Scanner::ScanErrorType::NoAudioTrack: return Wt::WString::tr("Lms.Admin.Database.Status.no-audio-track");
case Scanner::ScanErrorType::BadDuration: return Wt::WString::tr("Lms.Admin.Database.Status.bad-duration");
}
return "?";
}
static Wt::WString duplicateReasonToWString(Scanner::DuplicateReason reason)
{
switch (reason)
{
case Scanner::DuplicateReason::SameHash: return Wt::WString::tr("Lms.Admin.Database.Status.same-hash");
case Scanner::DuplicateReason::SameMBID: return Wt::WString::tr("Lms.Admin.Database.Status.same-mbid");
}
return "?";
}
Scanner::ScanStats _stats;
};
DatabaseStatus::DatabaseStatus()
: WTemplate {Wt::WString::tr("Lms.Admin.Database.Status.template")}
{
addFunction("tr", &Wt::WTemplate::Functions::tr);
using namespace Scanner;
auto onDbEvent = [&]() { refreshContents(); };
LmsApp->getEvents().dbScanned.connect(this, onDbEvent);
LmsApp->getEvents().dbScanInProgress.connect(this, onDbEvent);
LmsApp->getEvents().dbScanScheduled.connect(this, onDbEvent);
refreshContents();
}
void
DatabaseStatus::refreshContents()
{
using namespace Scanner;
Wt::WPushButton* reportBtn {bindNew<Wt::WPushButton>("btn-report", Wt::WString::tr("Lms.Admin.Database.Status.get-report"))};
const MediaScanner::Status status {getService<MediaScanner>()->getStatus()};
if (status.lastCompleteScanStats)
{
bindString("last-scan", Wt::WString::tr("Lms.Admin.Database.Status.last-scan-status")
.arg(status.lastCompleteScanStats->nbFiles())
.arg(durationToString(status.lastCompleteScanStats->startTime, status.lastCompleteScanStats->stopTime))
.arg(status.lastCompleteScanStats->stopTime.toString())
.arg(status.lastCompleteScanStats->errors.size())
.arg(status.lastCompleteScanStats->duplicates.size())
);
Wt::WLink link {std::make_shared<ReportResource>(*status.lastCompleteScanStats)};
link.setTarget(Wt::LinkTarget::NewWindow);
reportBtn->setLink(link);
}
else
{
bindString("last-scan", Wt::WString::tr("Lms.Admin.Database.Status.last-scan-not-available"));
reportBtn->setEnabled(false);
}
switch (status.currentState)
{
case MediaScanner::State::NotScheduled:
bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-not-scheduled"));
break;
case MediaScanner::State::Scheduled:
bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-scheduled")
.arg(status.nextScheduledScan.toString()));
break;
case MediaScanner::State::InProgress:
{
std::ostringstream oss;
bindString("status", Wt::WString::tr("Lms.Admin.Database.Status.status-in-progress")
.arg(status.inProgressScanStats->processedFiles)
.arg(status.inProgressScanStats->filesToScan)
.arg(status.inProgressScanStats->progress()));
}
break;
}
}
} // namespace UserInterface
src/ui/admin/DatabaseStatus.hpp
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright (C) 2019 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <Wt/WTemplate.h>
namespace UserInterface {
class DatabaseStatus : public Wt::WTemplate
{
public:
DatabaseStatus();
private:
void refreshContents();
};
} // namespace DatabaseStatus