owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added more details during scan. fixed #32

Browse Source
This commit is contained in:
emeric
2020-07-25 16:42:02 +02:00
parent d0826daa41
commit 8f236c16ca
12 changed files with 136 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
approot/admin-scannercontroller.xml
+1
View File
@@ -13,6 +13,7 @@
<div class="col-lg-9">
<div class="well well-sm">
${status}
<div>${step-status}</div>
</div>
</div>
</div>
approot/messages.xml
+5 -1
View File
@@ -72,7 +72,11 @@
<message id="Lms.Admin.ScannerController.status">Status</message>
<message id="Lms.Admin.ScannerController.status-not-scheduled">Not scheduled</message>
<message id="Lms.Admin.ScannerController.status-scheduled">Scheduled on {1}</message>
<message id="Lms.Admin.ScannerController.status-in-progress">Scanning {1} of {2} files ({3} %)</message>
<message id="Lms.Admin.ScannerController.status-in-progress">Scanning: step {1}/{2}</message>
<message id="Lms.Admin.ScannerController.step-checking-removed-files">Checking removed files... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">Discovering files: {1} files</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Scanning files: {1}/{2} files ({3}%)...</message>
<!--Users-->
<message id="Lms.Admin.Users.add">New user</message>
approot/messages_fr.xml
+5 -1
View File
@@ -72,7 +72,11 @@
<message id="Lms.Admin.ScannerController.status">Statut</message>
<message id="Lms.Admin.ScannerController.status-not-scheduled">Non planifié</message>
<message id="Lms.Admin.ScannerController.status-scheduled">Planifié le {1}</message>
<message id="Lms.Admin.ScannerController.status-in-progress">Scan de {1} fichiers sur {2} ({3} %)</message>
<message id="Lms.Admin.ScannerController.status-in-progress">En cours de scan : étape {1}/{2}</message>
<message id="Lms.Admin.ScannerController.step-checking-removed-files">Vérification des fichiers... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">Découverte des fichiers: {1} fichiers</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Récupération des métadonnées AcousticBrainz: {1}/{2} fichiers ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Scan des fichiers: {1}/{2} fichiers ({3}%)...</message>
<!--Users-->
<message id="Lms.Admin.Users.add">Ajouter</message>
src/libs/scanner/impl/MediaScanner.cpp
+54 -24
View File
@@ -264,6 +264,8 @@ MediaScanner::~MediaScanner()
void
MediaScanner::start()
{
std::scoped_lock lock {_controlMutex};
scheduleNextScan();
_ioService.start();
@@ -272,6 +274,8 @@ MediaScanner::start()
void
MediaScanner::stop()
{
std::scoped_lock lock {_controlMutex};
_abortScan = true;
_scheduleTimer.cancel();
@@ -281,10 +285,15 @@ MediaScanner::stop()
void
MediaScanner::abortScan()
{
_abortScan = true;
LMS_LOG(DBUPDATER, DEBUG) << "Aborting scan...";
std::scoped_lock lock {_controlMutex};
LMS_LOG(DBUPDATER, DEBUG) << "Waiting for the scan to abort...";
_abortScan = true;
_scheduleTimer.cancel();
_ioService.stop();
LMS_LOG(DBUPDATER, DEBUG) << "Scan abort done!";
_abortScan = false;
_ioService.start();
@@ -303,6 +312,7 @@ MediaScanner::requestImmediateScan(bool force)
void
MediaScanner::requestReload()
{
abortScan();
_ioService.post([=]()
{
scheduleNextScan();
@@ -319,7 +329,7 @@ MediaScanner::getStatus() const
res.currentState = _curState;
res.nextScheduledScan = _nextScheduledScan;
res.lastCompleteScanStats = _lastCompleteScanStats;
res.inProgressScanStats = _inProgressScanStats;
res.currentScanStepStats = _currentScanStepStats;
return res;
}
@@ -382,7 +392,10 @@ MediaScanner::scheduleNextScan()
void
MediaScanner::countAllFiles(ScanStats& stats)
{
stats.filesToScan = 0;
ScanStepStats stepStats{stats.startTime, ScanProgressStep::DiscoveringFiles};
stats.filesScanned = 0;
notifyInProgress(stepStats);
exploreFilesRecursive(_mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
{
@@ -391,8 +404,9 @@ MediaScanner::countAllFiles(ScanStats& stats)
if (!ec && isFileSupported(path, _fileExtensions))
{
stats.filesToScan++;
notifyInProgressIfNeeded(stats);
stats.filesScanned++;
stepStats.processedFiles++;
notifyInProgressIfNeeded(stepStats);
}
return true;
@@ -445,11 +459,11 @@ MediaScanner::scan(bool forceScan)
refreshScanSettings();
removeMissingTracks(stats);
LMS_LOG(DBUPDATER, DEBUG) << "Counting files in media directory '" << _mediaDirectory.string() << "'...";
countAllFiles(stats);
LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesToScan;
removeMissingTracks(stats);
LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesScanned;
LMS_LOG(UI, INFO) << "Checks complete, force scan = " << forceScan;
@@ -465,7 +479,7 @@ MediaScanner::scan(bool forceScan)
// Now update all the track features if needed
fetchTrackFeatures(stats);
LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << "/" << stats.featuresToFetch <<", duplicates = " << stats.duplicates.size();
LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size();
_dbSession.optimize();
@@ -476,19 +490,22 @@ MediaScanner::scan(bool forceScan)
std::unique_lock lock {_statusMutex};
_lastCompleteScanStats = std::move(stats);
_inProgressScanStats.reset();
_currentScanStepStats.reset();
}
LMS_LOG(DBUPDATER, DEBUG) << "Scan not aborted, scheduling next scan!";
scheduleNextScan();
scanComplete().emit();
}
else
{
LMS_LOG(DBUPDATER, DEBUG) << "Scan aborted, not scheduling next scan!";
std::unique_lock lock {_statusMutex};
_curState = State::NotScheduled;
_inProgressScanStats.reset();
_currentScanStepStats.reset();
}
}
@@ -524,6 +541,8 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats)
if (_recommendationEngineType != ScanSettings::RecommendationEngineType::Features)
return;
ScanStepStats stepStats{stats.startTime, ScanProgressStep::FetchingTrackFeatures};
LMS_LOG(DBUPDATER, INFO) << "Fetching missing track features...";
struct TrackInfo
@@ -545,7 +564,8 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats)
return res;
}()};
stats.featuresToFetch = tracksToFetch.size();
stepStats.filesToProcess = tracksToFetch.size();
notifyInProgress(stepStats);
LMS_LOG(DBUPDATER, INFO) << "Found " << tracksToFetch.size() << " track(s) to fetch!";
@@ -557,6 +577,8 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats)
if (fetchTrackFeatures(trackToFetch.id, trackToFetch.mbid))
stats.featuresFetched++;
stepStats.processedFiles++;
notifyInProgressIfNeeded(stepStats);
}
LMS_LOG(DBUPDATER, INFO) << "Track features fetched!";
@@ -596,27 +618,25 @@ MediaScanner::refreshScanSettings()
}
void
MediaScanner::notifyInProgress(const ScanStats& stats)
MediaScanner::notifyInProgress(const ScanStepStats& stepStats)
{
const ScanProgressStats progressStats {stats.toProgressStats()};
{
std::unique_lock lock {_statusMutex};
_inProgressScanStats = progressStats;
_currentScanStepStats = stepStats;
}
const std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
_sigScanInProgress(progressStats);
_sigScanInProgress(stepStats);
_lastScanInProgressEmit = now;
}
void
MediaScanner::notifyInProgressIfNeeded(const ScanStats& stats)
MediaScanner::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
{
std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
if (std::chrono::duration_cast<std::chrono::seconds>(now - _lastScanInProgressEmit).count() > 2)
notifyInProgress(stats);
if (std::chrono::duration_cast<std::chrono::seconds>(now - _lastScanInProgressEmit).count() > 1)
notifyInProgress(stepStats);
}
void
@@ -777,6 +797,10 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S
void
MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats)
{
ScanStepStats stepStats{stats.startTime, ScanProgressStep::ScanningFiles};
stepStats.filesToProcess = stats.filesScanned;
notifyInProgress(stepStats);
exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
{
if (_abortScan)
@@ -791,13 +815,12 @@ MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bo
{
scanAudioFile(path, forceScan, stats );
notifyInProgressIfNeeded(stats);
stepStats.processedFiles++;
notifyInProgressIfNeeded(stepStats);
}
return true;
});
notifyInProgress(stats);
}
// Check if a file exists and is still in a media directory
@@ -842,6 +865,8 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
{
static constexpr std::size_t batchSize {50};
ScanStepStats stepStats{stats.startTime, ScanProgressStep::CheckingRemovedFiles};
LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed...";
std::size_t trackCount {};
@@ -851,6 +876,9 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
}
LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked...";
stepStats.filesToProcess = trackCount;
notifyInProgress(stepStats);
std::vector<std::pair<Database::IdType, std::filesystem::path>> trackPaths;
std::vector<IdType> tracksToRemove;
@@ -859,6 +887,8 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
trackPaths.clear();
tracksToRemove.clear();
stepStats.processedFiles++;
{
auto transaction {_dbSession.createSharedTransaction()};
trackPaths = Track::getAllPaths(_dbSession, i, batchSize);
@@ -888,7 +918,7 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
}
}
notifyInProgressIfNeeded(stats);
notifyInProgressIfNeeded(stepStats);
if (i == 0)
break;
src/libs/scanner/impl/MediaScanner.hpp
+6 -7
View File
@@ -59,7 +59,7 @@ class MediaScanner : public IMediaScanner
Wt::Signal<>& scanStarted() override { return _sigScanStarted; }
Wt::Signal<>& scanComplete() override { return _sigScanComplete; }
Wt::Signal<ScanProgressStats>& scanInProgress() override { return _sigScanInProgress; }
Wt::Signal<ScanStepStats>& scanInProgress() override { return _sigScanInProgress; }
Wt::Signal<Wt::WDateTime>& scheduled() override { return _sigScheduled; }
private:
@@ -86,17 +86,16 @@ class MediaScanner : public IMediaScanner
void checkDuplicatedAudioFiles(ScanStats& stats);
void scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats);
Database::IdType doScanAudioFile(const std::filesystem::path& file, ScanStats& stats);
void notifyInProgressIfNeeded(const ScanStats& stats);
void notifyInProgress(const ScanStats& stats);
void notifyInProgressIfNeeded(const ScanStepStats& stats);
void notifyInProgress(const ScanStepStats& stats);
std::mutex _scanInProgress {};
std::mutex _controlMutex;
std::atomic<bool> _abortScan {};
Wt::WIOService _ioService;
boost::asio::system_timer _scheduleTimer {_ioService};
Wt::Signal<> _sigScanStarted;
Wt::Signal<> _sigScanComplete;
Wt::Signal<ScanProgressStats> _sigScanInProgress;
Wt::Signal<ScanStepStats> _sigScanInProgress;
std::chrono::system_clock::time_point _lastScanInProgressEmit {};
Wt::Signal<Wt::WDateTime> _sigScheduled;
Database::Session _dbSession;
@@ -105,7 +104,7 @@ class MediaScanner : public IMediaScanner
mutable std::shared_mutex _statusMutex;
State _curState {State::NotScheduled};
std::optional<ScanStats> _lastCompleteScanStats;
std::optional<ScanProgressStats> _inProgressScanStats;
std::optional<ScanStepStats> _currentScanStepStats;
Wt::WDateTime _nextScheduledScan;
// Current scan settings
src/libs/scanner/impl/MediaScannerStats.cpp
+2 -8
View File
@@ -40,16 +40,10 @@ ScanStats::nbChanges() const
return additions + deletions + updates;
}
ScanProgressStats
ScanStats::toProgressStats() const
{
return ScanProgressStats {startTime, filesToScan, nbFiles()};
}
unsigned
ScanProgressStats::progress() const
ScanStepStats::progress() const
{
return (processedFiles / static_cast<float>(filesToScan ? filesToScan : 1)) * 100;
return (processedFiles / static_cast<float>(filesToProcess ? filesToProcess : 1)) * 100;
}
} // namespace Scanner
src/libs/scanner/include/scanner/IMediaScanner.hpp
+2 -2
View File
@@ -58,7 +58,7 @@ class IMediaScanner
State currentState {State::NotScheduled};
Wt::WDateTime nextScheduledScan;
std::optional<ScanStats> lastCompleteScanStats;
std::optional<ScanProgressStats> inProgressScanStats;
std::optional<ScanStepStats> currentScanStepStats;
};
virtual Status getStatus() const = 0;
@@ -70,7 +70,7 @@ class IMediaScanner
virtual Wt::Signal<>& scanComplete() = 0;
// Called during scan in progress
virtual Wt::Signal<ScanProgressStats>& scanInProgress() = 0;
virtual Wt::Signal<ScanStepStats>& scanInProgress() = 0;
// Called after a schedule
virtual Wt::Signal<Wt::WDateTime>& scheduled() = 0;
src/libs/scanner/include/scanner/MediaScannerStats.hpp
+19 -11
View File
@@ -56,15 +56,26 @@ namespace Scanner {
};
enum class ScanProgressStep : unsigned
{
CheckingRemovedFiles = 0,
DiscoveringFiles,
ScanningFiles,
FetchingTrackFeatures,
};
static inline constexpr unsigned ScanProgressStepCount {4};
// reduced scan stats
struct ScanProgressStats
struct ScanStepStats
{
Wt::WDateTime startTime;
std::size_t filesToScan {};
std::size_t processedFiles {};
ScanProgressStep currentStep;
unsigned progress() const;
std::size_t filesToProcess {};
std::size_t processedFiles {};
unsigned progress() const;
};
struct ScanStats
@@ -72,25 +83,22 @@ namespace Scanner {
Wt::WDateTime startTime;
Wt::WDateTime stopTime;
std::size_t filesToScan {}; // Total number of files to be scanned (estimated)
std::size_t filesScanned {}; // Total number of files scanned (estimated)
std::size_t skips {}; // no change since last scan
std::size_t scans {}; // actually scanned filed
std::size_t skips {}; // no change since last scan
std::size_t scans {}; // actually scanned filed
std::size_t additions {}; // added in DB
std::size_t deletions {}; // removed from DB
std::size_t updates {}; // updated file in DB
std::size_t updates {}; // updated file in DB
std::size_t featuresFetched {}; // features fetched in DB
std::size_t featuresToFetch {}; // features to be fetched in DB
std::vector<ScanError> errors;
std::vector<ScanDuplicate> duplicates;
std::size_t nbFiles() const;
std::size_t nbChanges() const;
ScanProgressStats toProgressStats() const;
};
}
src/libs/subsonic/impl/Scan.cpp
+9 -2
View File
@@ -35,8 +35,15 @@ namespace API::Subsonic::Scan
const IMediaScanner::Status scanStatus {ServiceProvider<IMediaScanner>::get()->getStatus()};
statusResponse.setAttribute("scanning", scanStatus.currentState == IMediaScanner::State::InProgress);
if (scanStatus.currentState == IMediaScanner::State::InProgress && scanStatus.inProgressScanStats)
statusResponse.setAttribute("count", scanStatus.inProgressScanStats->processedFiles);
if (scanStatus.currentState == IMediaScanner::State::InProgress)
{
std::size_t count{};
if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanProgressStep::ScanningFiles)
count = scanStatus.currentScanStepStats->processedFiles;
statusResponse.setAttribute("count", count);
}
return statusResponse;
}
src/lms/ui/LmsApplication.cpp
+2 -2
View File
@@ -544,11 +544,11 @@ LmsApplication::createHome()
});
});
ServiceProvider<Scanner::IMediaScanner>::get()->scanInProgress().connect(this, [=] (Scanner::ScanProgressStats stats)
ServiceProvider<Scanner::IMediaScanner>::get()->scanInProgress().connect(this, [=] (Scanner::ScanStepStats stepStats)
{
Wt::WServer::instance()->post(sessionId, [=]
{
_events.dbScanInProgress.emit(stats);
_events.dbScanInProgress.emit(stepStats);
triggerUpdate();
});
});
src/lms/ui/LmsApplication.hpp
+1 -1
View File
@@ -57,7 +57,7 @@ struct Events
// Database events
Wt::Signal<> dbScanStarted;
Wt::Signal<> dbScanned;
Wt::Signal<Scanner::ScanProgressStats> dbScanInProgress;
Wt::Signal<Scanner::ScanStepStats> dbScanInProgress;
Wt::Signal<Wt::WDateTime> dbScanScheduled;
};
src/lms/ui/admin/ScannerController.cpp
+30 -5
View File
@@ -185,18 +185,43 @@ ScannerController::refreshContents()
{
case IMediaScanner::State::NotScheduled:
bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-not-scheduled"));
bindEmpty("step-status");
break;
case IMediaScanner::State::Scheduled:
bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-scheduled")
.arg(status.nextScheduledScan.toString()));
bindEmpty("step-status");
break;
case IMediaScanner::State::InProgress:
bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress")
.arg(static_cast<int>(status.currentScanStepStats->currentStep) + 1)
.arg(Scanner::ScanProgressStepCount));
switch (status.currentScanStepStats->currentStep)
{
std::ostringstream oss;
bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress")
.arg(status.inProgressScanStats->processedFiles)
.arg(status.inProgressScanStats->filesToScan)
.arg(status.inProgressScanStats->progress()));
case Scanner::ScanProgressStep::CheckingRemovedFiles:
bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-checking-removed-files")
.arg(status.currentScanStepStats->progress()));
break;
case Scanner::ScanProgressStep::DiscoveringFiles:
bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files")
.arg(status.currentScanStepStats->processedFiles));
break;
case Scanner::ScanProgressStep::ScanningFiles:
bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files")
.arg(status.currentScanStepStats->processedFiles)
.arg(status.currentScanStepStats->filesToProcess)
.arg(status.currentScanStepStats->progress()));
break;
case Scanner::ScanProgressStep::FetchingTrackFeatures:
bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features")
.arg(status.currentScanStepStats->processedFiles)
.arg(status.currentScanStepStats->filesToProcess)
.arg(status.currentScanStepStats->progress()));
break;
}
break;
}