Refactored scanner to ease further improvements
This commit is contained in:
emeric
@@ -22,27 +22,25 @@
|
||||
#include <ctime>
|
||||
#include <boost/asio/placeholders.hpp>
|
||||
|
||||
#include "services/database/Artist.hpp"
|
||||
#include "services/database/Cluster.hpp"
|
||||
#include "services/database/Release.hpp"
|
||||
#include "services/database/ScanSettings.hpp"
|
||||
#include "services/database/Track.hpp"
|
||||
#include "services/database/TrackArtistLink.hpp"
|
||||
#include "services/database/TrackFeatures.hpp"
|
||||
#include "metadata/IParser.hpp"
|
||||
#include "services/database/ScanSettings.hpp"
|
||||
#include "services/recommendation/IRecommendationService.hpp"
|
||||
#include "utils/Exception.hpp"
|
||||
#include "utils/IConfig.hpp"
|
||||
#include "utils/Logger.hpp"
|
||||
#include "utils/Path.hpp"
|
||||
#include "utils/UUID.hpp"
|
||||
#include "utils/Tuple.hpp"
|
||||
|
||||
#include "ScanStepCheckDuplicatedDbFiles.hpp"
|
||||
#include "ScanStepDiscoverFiles.hpp"
|
||||
#include "ScanStepRemoveOrphanDbFiles.hpp"
|
||||
#include "ScanStepScanFiles.hpp"
|
||||
|
||||
using namespace Database;
|
||||
|
||||
namespace {
|
||||
|
||||
const std::filesystem::path excludeDirFileName {".lmsignore"};
|
||||
|
||||
Wt::WDate
|
||||
getNextMonday(Wt::WDate current)
|
||||
{
|
||||
@@ -65,183 +63,6 @@ getNextFirstOfMonth(Wt::WDate current)
|
||||
return current;
|
||||
}
|
||||
|
||||
bool
|
||||
isFileSupported(const std::filesystem::path& file, const std::vector<std::filesystem::path>& extensions)
|
||||
{
|
||||
const std::filesystem::path extension {StringUtils::stringToLower(file.extension().string())};
|
||||
|
||||
return (std::find(std::cbegin(extensions), std::cend(extensions), extension) != std::cend(extensions));
|
||||
}
|
||||
|
||||
bool
|
||||
isPathInMediaDirectory(const std::filesystem::path& path, const std::filesystem::path& rootPath)
|
||||
{
|
||||
std::filesystem::path curPath = path;
|
||||
|
||||
while (curPath.parent_path() != curPath)
|
||||
{
|
||||
curPath = curPath.parent_path();
|
||||
|
||||
std::error_code ec;
|
||||
if (std::filesystem::exists(curPath / excludeDirFileName, ec))
|
||||
return false;
|
||||
|
||||
if (curPath == rootPath)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static
|
||||
Artist::pointer
|
||||
createArtist(Session& session, const MetaData::Artist& artistInfo)
|
||||
{
|
||||
Artist::pointer artist {session.create<Artist>(artistInfo.name)};
|
||||
|
||||
if (artistInfo.musicBrainzArtistID)
|
||||
artist.modify()->setMBID(*artistInfo.musicBrainzArtistID);
|
||||
if (artistInfo.sortName)
|
||||
artist.modify()->setSortName(*artistInfo.sortName);
|
||||
|
||||
return artist;
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
updateArtistIfNeeded(Artist::pointer artist, const MetaData::Artist& artistInfo)
|
||||
{
|
||||
// Name may have been updated
|
||||
if (artist->getName() != artistInfo.name)
|
||||
{
|
||||
artist.modify()->setName(artistInfo.name);
|
||||
}
|
||||
|
||||
// Sortname may have been updated
|
||||
if (artistInfo.sortName && *artistInfo.sortName != artist->getSortName() )
|
||||
{
|
||||
artist.modify()->setSortName(*artistInfo.sortName);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Artist::pointer>
|
||||
getOrCreateArtists(Session& session, const std::vector<MetaData::Artist>& artistsInfo, bool allowFallbackOnMBIDEntries)
|
||||
{
|
||||
std::vector<Artist::pointer> artists;
|
||||
|
||||
for (const MetaData::Artist& artistInfo : artistsInfo)
|
||||
{
|
||||
Artist::pointer artist;
|
||||
|
||||
// First try to get by MBID
|
||||
if (artistInfo.musicBrainzArtistID)
|
||||
{
|
||||
artist = Artist::find(session, *artistInfo.musicBrainzArtistID);
|
||||
if (!artist)
|
||||
artist = createArtist(session, artistInfo);
|
||||
else
|
||||
updateArtistIfNeeded(artist, artistInfo);
|
||||
|
||||
artists.emplace_back(std::move(artist));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fall back on artist name (collisions may occur)
|
||||
if (!artistInfo.name.empty())
|
||||
{
|
||||
for (const Artist::pointer& sameNamedArtist : Artist::find(session, artistInfo.name))
|
||||
{
|
||||
// Do not fallback on artist that is correctly tagged
|
||||
if (!allowFallbackOnMBIDEntries && sameNamedArtist->getMBID())
|
||||
continue;
|
||||
|
||||
artist = sameNamedArtist;
|
||||
break;
|
||||
}
|
||||
|
||||
// No Artist found with the same name and without MBID -> creating
|
||||
if (!artist)
|
||||
artist = createArtist(session, artistInfo);
|
||||
else
|
||||
updateArtistIfNeeded(artist, artistInfo);
|
||||
|
||||
artists.emplace_back(std::move(artist));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return artists;
|
||||
}
|
||||
|
||||
Release::pointer
|
||||
getOrCreateRelease(Session& session, const MetaData::Album& album)
|
||||
{
|
||||
Release::pointer release;
|
||||
|
||||
// First try to get by MBID
|
||||
if (album.musicBrainzAlbumID)
|
||||
{
|
||||
release = Release::find(session, *album.musicBrainzAlbumID);
|
||||
if (!release)
|
||||
{
|
||||
release = session.create<Release>(album.name, album.musicBrainzAlbumID);
|
||||
}
|
||||
else if (release->getName() != album.name)
|
||||
{
|
||||
// Name may have been updated
|
||||
release.modify()->setName(album.name);
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
// Fall back on release name (collisions may occur)
|
||||
if (!album.name.empty())
|
||||
{
|
||||
for (const Release::pointer& sameNamedRelease : Release::find(session, album.name))
|
||||
{
|
||||
// do not fallback on properly tagged releases
|
||||
if (!sameNamedRelease->getMBID())
|
||||
{
|
||||
release = sameNamedRelease;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// No release found with the same name and without MBID -> creating
|
||||
if (!release)
|
||||
release = session.create<Release>(album.name);
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
return Release::pointer{};
|
||||
}
|
||||
|
||||
std::vector<Cluster::pointer>
|
||||
getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames)
|
||||
{
|
||||
std::vector< Cluster::pointer > clusters;
|
||||
|
||||
for (auto clusterNames : clustersNames)
|
||||
{
|
||||
auto clusterType = ClusterType::find(session, clusterNames.first);
|
||||
if (!clusterType)
|
||||
continue;
|
||||
|
||||
for (auto clusterName : clusterNames.second)
|
||||
{
|
||||
auto cluster = clusterType->getCluster(clusterName);
|
||||
if (!cluster)
|
||||
cluster = session.create<Cluster>(clusterType, clusterName);
|
||||
|
||||
clusters.push_back(cluster);
|
||||
}
|
||||
}
|
||||
|
||||
return clusters;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace Scanner {
|
||||
@@ -252,29 +73,11 @@ createScannerService(Db& db, Recommendation::IRecommendationService& recommendat
|
||||
return std::make_unique<ScannerService>(db, recommendationService);
|
||||
}
|
||||
|
||||
MetaData::ParserReadStyle
|
||||
getParserReadStyle()
|
||||
{
|
||||
std::string_view readStyle {Service<IConfig>::get()->getString("scanner-parser-read-style", "accurate")};
|
||||
|
||||
if (readStyle == "fast")
|
||||
return MetaData::ParserReadStyle::Fast;
|
||||
else if (readStyle == "average")
|
||||
return MetaData::ParserReadStyle::Average;
|
||||
else if (readStyle == "accurate")
|
||||
return MetaData::ParserReadStyle::Accurate;
|
||||
|
||||
throw LmsException {"Invalid value for 'scanner-parser-read-style'"};
|
||||
}
|
||||
|
||||
ScannerService::ScannerService(Db& db, Recommendation::IRecommendationService& recommendationService)
|
||||
: _recommendationService {recommendationService}
|
||||
, _skipDuplicateRecordingMBID {Service<IConfig>::get()->getBool("scanner-skip-duplicate-recording-mbid", false)}
|
||||
, _db {db}
|
||||
, _dbSession {db}
|
||||
, _metadataParser {MetaData::createParser(MetaData::ParserType::TagLib, getParserReadStyle())} // For now, always use TagLib
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "skipDuplicateRecordingMBID = " << _skipDuplicateRecordingMBID;
|
||||
|
||||
_ioService.setThreadCount(1);
|
||||
|
||||
refreshScanSettings();
|
||||
@@ -383,34 +186,34 @@ ScannerService::getStatus() const
|
||||
void
|
||||
ScannerService::scheduleNextScan()
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan";
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Scheduling next scan";
|
||||
|
||||
refreshScanSettings();
|
||||
|
||||
const Wt::WDateTime now {Wt::WDateTime::currentDateTime()};
|
||||
|
||||
Wt::WDateTime nextScanDateTime;
|
||||
switch (_updatePeriod)
|
||||
switch (_settings.updatePeriod)
|
||||
{
|
||||
case ScanSettings::UpdatePeriod::Daily:
|
||||
if (now.time() < _startTime)
|
||||
nextScanDateTime = {now.date(), _startTime};
|
||||
if (now.time() < _settings.startTime)
|
||||
nextScanDateTime = {now.date(), _settings.startTime};
|
||||
else
|
||||
nextScanDateTime = {now.date().addDays(1), _startTime};
|
||||
nextScanDateTime = {now.date().addDays(1), _settings.startTime};
|
||||
break;
|
||||
|
||||
case ScanSettings::UpdatePeriod::Weekly:
|
||||
if (now.time() < _startTime && now.date().dayOfWeek() == 1)
|
||||
nextScanDateTime = {now.date(), _startTime};
|
||||
if (now.time() < _settings.startTime && now.date().dayOfWeek() == 1)
|
||||
nextScanDateTime = {now.date(), _settings.startTime};
|
||||
else
|
||||
nextScanDateTime = {getNextMonday(now.date()), _startTime};
|
||||
nextScanDateTime = {getNextMonday(now.date()), _settings.startTime};
|
||||
break;
|
||||
|
||||
case ScanSettings::UpdatePeriod::Monthly:
|
||||
if (now.time() < _startTime && now.date().day() == 1)
|
||||
nextScanDateTime = {now.date(), _startTime};
|
||||
if (now.time() < _settings.startTime && now.date().day() == 1)
|
||||
nextScanDateTime = {now.date(), _settings.startTime};
|
||||
else
|
||||
nextScanDateTime = {getNextFirstOfMonth(now.date()), _startTime};
|
||||
nextScanDateTime = {getNextFirstOfMonth(now.date()), _settings.startTime};
|
||||
break;
|
||||
|
||||
case ScanSettings::UpdatePeriod::Hourly:
|
||||
@@ -434,31 +237,6 @@ ScannerService::scheduleNextScan()
|
||||
_events.scanScheduled.emit(_nextScheduledScan);
|
||||
}
|
||||
|
||||
void
|
||||
ScannerService::countAllFiles(ScanStats& stats)
|
||||
{
|
||||
ScanStepStats stepStats{stats.startTime, ScanProgressStep::DiscoveringFiles};
|
||||
|
||||
stats.filesScanned = 0;
|
||||
notifyInProgress(stepStats);
|
||||
|
||||
exploreFilesRecursive(_mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
|
||||
{
|
||||
if (_abortScan)
|
||||
return false;
|
||||
|
||||
if (!ec && isFileSupported(path, _fileExtensions))
|
||||
{
|
||||
stats.filesScanned++;
|
||||
stepStats.processedElems++;
|
||||
notifyInProgressIfNeeded(stepStats);
|
||||
}
|
||||
|
||||
return true;
|
||||
}, excludeDirFileName);
|
||||
notifyInProgress(stepStats);
|
||||
}
|
||||
|
||||
void
|
||||
ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime)
|
||||
{
|
||||
@@ -499,35 +277,29 @@ ScannerService::scan(bool forceScan)
|
||||
_nextScheduledScan = {};
|
||||
}
|
||||
|
||||
ScanStats stats;
|
||||
stats.startTime = Wt::WDateTime::currentDateTime();
|
||||
|
||||
LMS_LOG(UI, INFO) << "New scan started!";
|
||||
|
||||
refreshScanSettings();
|
||||
|
||||
removeMissingTracks(stats);
|
||||
IScanStep::ScanContext scanContext {_settings.mediaDirectory, forceScan, ScanStats {}, ScanStepStats {}};
|
||||
ScanStats& stats {scanContext.stats};
|
||||
stats.startTime = Wt::WDateTime::currentDateTime();
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Counting files in media directory '" << _mediaDirectory.string() << "'...";
|
||||
countAllFiles(stats);
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesScanned;
|
||||
|
||||
LMS_LOG(UI, INFO) << "Checks complete, force scan = " << forceScan;
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "scaning media directory '" << _mediaDirectory.string() << "'...";
|
||||
scanMediaDirectory(_mediaDirectory, forceScan, stats);
|
||||
LMS_LOG(DBUPDATER, INFO) << "scaning media directory '" << _mediaDirectory.string() << "' DONE";
|
||||
|
||||
removeOrphanEntries();
|
||||
|
||||
if (!_abortScan)
|
||||
for (auto& scanStep : _scanSteps)
|
||||
{
|
||||
checkDuplicatedAudioFiles(stats);
|
||||
reloadSimilarityEngine(stats);
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Starting scan step '" << scanStep->getStepName() << "'";
|
||||
scanContext.currentStepStats = ScanStepStats {Wt::WDateTime::currentDateTime(), scanStep->getStep()};
|
||||
|
||||
notifyInProgress(scanContext.currentStepStats);
|
||||
scanStep->process(scanContext);
|
||||
notifyInProgress(scanContext.currentStepStats);
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Completed scan step '" << scanStep->getStepName() << "'";
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size();
|
||||
|
||||
// TODO make it a scan step
|
||||
_dbSession.optimize();
|
||||
|
||||
if (!_abortScan)
|
||||
@@ -559,33 +331,71 @@ ScannerService::scan(bool forceScan)
|
||||
void
|
||||
ScannerService::refreshScanSettings()
|
||||
{
|
||||
auto transaction {_dbSession.createSharedTransaction()};
|
||||
ScannerSettings newSettings {readSettings()};
|
||||
if (_settings == newSettings)
|
||||
return;
|
||||
|
||||
const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)};
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Scanner settings updated";
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "skipDuplicateRecordingMBID = " << newSettings.skipDuplicateRecordingMBID;
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Using scan settings version " << newSettings.scanVersion;
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion();
|
||||
_settings = std::move(newSettings);
|
||||
|
||||
_scanVersion = scanSettings->getScanVersion();
|
||||
_startTime = scanSettings->getUpdateStartTime();
|
||||
_updatePeriod = scanSettings->getUpdatePeriod();
|
||||
auto cbFunc {[this](const ScanStepStats& stats)
|
||||
{
|
||||
notifyInProgressIfNeeded(stats);
|
||||
}};
|
||||
|
||||
ScanStepBase::InitParams params
|
||||
{
|
||||
const auto fileExtensions {scanSettings->getAudioFileExtensions()};
|
||||
_fileExtensions.clear();
|
||||
std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(_fileExtensions),
|
||||
_settings,
|
||||
cbFunc,
|
||||
_abortScan,
|
||||
_db
|
||||
};
|
||||
|
||||
_scanSteps.clear();
|
||||
_scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
|
||||
}
|
||||
|
||||
ScannerSettings
|
||||
ScannerService::readSettings()
|
||||
{
|
||||
ScannerSettings newSettings;
|
||||
|
||||
newSettings.skipDuplicateRecordingMBID = Service<IConfig>::get()->getBool("scanner-skip-duplicate-recording-mbid", false);
|
||||
{
|
||||
auto transaction {_dbSession.createSharedTransaction()};
|
||||
|
||||
const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)};
|
||||
|
||||
newSettings.scanVersion = scanSettings->getScanVersion();
|
||||
newSettings.startTime = scanSettings->getUpdateStartTime();
|
||||
newSettings.updatePeriod = scanSettings->getUpdatePeriod();
|
||||
|
||||
{
|
||||
const auto fileExtensions {scanSettings->getAudioFileExtensions()};
|
||||
newSettings.supportedExtensions.reserve(fileExtensions.size());
|
||||
std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(newSettings.supportedExtensions),
|
||||
[](const std::filesystem::path& extension) { return std::filesystem::path{ StringUtils::stringToLower(extension.string()) }; });
|
||||
}
|
||||
newSettings.recommendationServiceType = scanSettings->getRecommendationEngineType();
|
||||
newSettings.mediaDirectory = scanSettings->getMediaDirectory();
|
||||
|
||||
const auto clusterTypes = scanSettings->getClusterTypes();
|
||||
std::set<std::string> clusterTypeNames;
|
||||
|
||||
std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes),
|
||||
std::inserter(clusterTypeNames, clusterTypeNames.begin()),
|
||||
[](ClusterType::pointer clusterType) { return clusterType->getName(); });
|
||||
|
||||
newSettings.clusterTypeNames = std::move(clusterTypeNames);
|
||||
}
|
||||
_mediaDirectory = scanSettings->getMediaDirectory();
|
||||
_recommendationServiceType = scanSettings->getRecommendationEngineType();
|
||||
|
||||
const auto clusterTypes = scanSettings->getClusterTypes();
|
||||
std::set<std::string> clusterTypeNames;
|
||||
|
||||
std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes),
|
||||
std::inserter(clusterTypeNames, clusterTypeNames.begin()),
|
||||
[](ClusterType::pointer clusterType) { return clusterType->getName(); });
|
||||
|
||||
_metadataParser->setClusterTypeNames(clusterTypeNames);
|
||||
return newSettings;
|
||||
}
|
||||
|
||||
void
|
||||
@@ -610,410 +420,4 @@ ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
|
||||
notifyInProgress(stepStats);
|
||||
}
|
||||
|
||||
void
|
||||
ScannerService::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats)
|
||||
{
|
||||
Wt::WDateTime lastWriteTime;
|
||||
try
|
||||
{
|
||||
lastWriteTime = getLastWriteTime(file);
|
||||
}
|
||||
catch (LmsException& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR) << e.what();
|
||||
stats.skips++;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!forceScan)
|
||||
{
|
||||
// Skip file if last write is the same
|
||||
auto transaction {_dbSession.createSharedTransaction()};
|
||||
|
||||
const Track::pointer track {Track::findByPath(_dbSession, file)};
|
||||
|
||||
if (track && track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t()
|
||||
&& track->getScanVersion() == _scanVersion)
|
||||
{
|
||||
stats.skips++;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<MetaData::Track> trackInfo {_metadataParser->parse(file)};
|
||||
if (!trackInfo)
|
||||
{
|
||||
stats.errors.emplace_back(file, ScanErrorType::CannotParseFile);
|
||||
return;
|
||||
}
|
||||
|
||||
stats.scans++;
|
||||
|
||||
auto uniqueTransaction {_dbSession.createUniqueTransaction()};
|
||||
|
||||
Track::pointer track {Track::findByPath(_dbSession, file) };
|
||||
|
||||
// Skip duplicate recording MBID
|
||||
if (trackInfo->recordingMBID && _skipDuplicateRecordingMBID)
|
||||
{
|
||||
for (Track::pointer otherTrack : Track::findByRecordingMBID(_dbSession, *trackInfo->recordingMBID))
|
||||
{
|
||||
if (track && track->getId() == otherTrack->getId())
|
||||
continue;
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Skipped '" << file.string() << "' (similar recording MBID in '" << otherTrack->getPath().string() << "')";
|
||||
// This recording MBID already exists, just remove what we just scanned
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// We estimate this is an audio file if:
|
||||
// - we found a least one audio stream
|
||||
// - the duration is not null
|
||||
if (trackInfo->audioStreams.empty())
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (no audio stream found)";
|
||||
|
||||
// If Track exists here, delete it!
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
stats.errors.emplace_back(ScanError {file, ScanErrorType::NoAudioTrack});
|
||||
return;
|
||||
}
|
||||
if (trackInfo->duration == std::chrono::milliseconds::zero())
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (duration is 0)";
|
||||
|
||||
// If Track exists here, delete it!
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
stats.errors.emplace_back(ScanError {file, ScanErrorType::BadDuration});
|
||||
return;
|
||||
}
|
||||
|
||||
// ***** Title
|
||||
std::string title;
|
||||
if (!trackInfo->title.empty())
|
||||
title = trackInfo->title;
|
||||
else
|
||||
{
|
||||
// TODO parse file name guess track etc.
|
||||
// For now juste use file name as title
|
||||
title = file.filename().string();
|
||||
}
|
||||
|
||||
// If file already exist, update data
|
||||
// Otherwise, create it
|
||||
if (!track)
|
||||
{
|
||||
// Create a new song
|
||||
track = _dbSession.create<Track>(file);
|
||||
LMS_LOG(DBUPDATER, INFO) << "Adding '" << file.string() << "'";
|
||||
stats.additions++;
|
||||
}
|
||||
else
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Updating '" << file.string() << "'";
|
||||
|
||||
stats.updates++;
|
||||
}
|
||||
|
||||
// Track related data
|
||||
assert(track);
|
||||
|
||||
track.modify()->clearArtistLinks();
|
||||
// Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files
|
||||
for (const Artist::pointer& artist : getOrCreateArtists(_dbSession, trackInfo->artists, false))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, artist, TrackArtistLinkType::Artist));
|
||||
|
||||
for (const Artist::pointer& releaseArtist : getOrCreateArtists(_dbSession, trackInfo->albumArtists, false))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, releaseArtist, TrackArtistLinkType::ReleaseArtist));
|
||||
|
||||
// Allow fallbacks on artists with the same name even if they have MBID, since there is no tag to indicate the MBID of these artists
|
||||
// We could ask MusicBrainz to get all the information, but that would heavily slow down the import process
|
||||
for (const Artist::pointer& conductor : getOrCreateArtists(_dbSession, trackInfo->conductorArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, conductor, TrackArtistLinkType::Conductor));
|
||||
|
||||
for (const Artist::pointer& composer : getOrCreateArtists(_dbSession, trackInfo->composerArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, composer, TrackArtistLinkType::Composer));
|
||||
|
||||
for (const Artist::pointer& lyricist : getOrCreateArtists(_dbSession, trackInfo->lyricistArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, lyricist, TrackArtistLinkType::Lyricist));
|
||||
|
||||
for (const Artist::pointer& mixer : getOrCreateArtists(_dbSession, trackInfo->mixerArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, mixer, TrackArtistLinkType::Mixer));
|
||||
|
||||
for (const auto& [role, performers] : trackInfo->performerArtists)
|
||||
{
|
||||
for (const Artist::pointer& performer : getOrCreateArtists(_dbSession, performers, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, performer, TrackArtistLinkType::Performer, role));
|
||||
}
|
||||
|
||||
for (const Artist::pointer& producer : getOrCreateArtists(_dbSession, trackInfo->producerArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, producer, TrackArtistLinkType::Producer));
|
||||
|
||||
for (const Artist::pointer& remixer : getOrCreateArtists(_dbSession, trackInfo->remixerArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, remixer, TrackArtistLinkType::Remixer));
|
||||
|
||||
track.modify()->setScanVersion(_scanVersion);
|
||||
if (trackInfo->album)
|
||||
track.modify()->setRelease(getOrCreateRelease(_dbSession, *trackInfo->album));
|
||||
else
|
||||
track.modify()->setRelease({});
|
||||
track.modify()->setClusters(getOrCreateClusters(_dbSession, trackInfo->clusters));
|
||||
track.modify()->setLastWriteTime(lastWriteTime);
|
||||
track.modify()->setName(title);
|
||||
track.modify()->setDuration(trackInfo->duration);
|
||||
track.modify()->setAddedTime(Wt::WDateTime::currentDateTime());
|
||||
track.modify()->setTrackNumber(trackInfo->trackNumber ? *trackInfo->trackNumber : 0);
|
||||
track.modify()->setDiscNumber(trackInfo->discNumber ? *trackInfo->discNumber : 0);
|
||||
track.modify()->setTotalTrack(trackInfo->totalTrack);
|
||||
track.modify()->setTotalDisc(trackInfo->totalDisc);
|
||||
track.modify()->setDiscSubtitle(trackInfo->discSubtitle);
|
||||
track.modify()->setDate(trackInfo->date);
|
||||
track.modify()->setOriginalDate(trackInfo->originalDate);
|
||||
|
||||
// If a file has an OriginalYear but no Year, set it to ease filtering
|
||||
if (!trackInfo->date.isValid() && trackInfo->originalDate.isValid())
|
||||
track.modify()->setDate(trackInfo->originalDate);
|
||||
|
||||
track.modify()->setRecordingMBID(trackInfo->recordingMBID);
|
||||
track.modify()->setTrackMBID(trackInfo->trackMBID);
|
||||
if (auto trackFeatures {TrackFeatures::find(_dbSession, track->getId())})
|
||||
trackFeatures.remove(); // TODO: only if MBID changed?
|
||||
track.modify()->setHasCover(trackInfo->hasCover);
|
||||
track.modify()->setCopyright(trackInfo->copyright);
|
||||
track.modify()->setCopyrightURL(trackInfo->copyrightURL);
|
||||
track.modify()->setTrackReplayGain(trackInfo->trackReplayGain);
|
||||
track.modify()->setReleaseReplayGain(trackInfo->albumReplayGain);
|
||||
}
|
||||
|
||||
void
|
||||
ScannerService::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats)
|
||||
{
|
||||
ScanStepStats stepStats{stats.startTime, ScanProgressStep::ScanningFiles};
|
||||
stepStats.totalElems = stats.filesScanned;
|
||||
notifyInProgress(stepStats);
|
||||
|
||||
exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
|
||||
{
|
||||
if (_abortScan)
|
||||
return false;
|
||||
|
||||
if (ec)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message();
|
||||
stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()});
|
||||
}
|
||||
else if (isFileSupported(path, _fileExtensions))
|
||||
{
|
||||
scanAudioFile(path, forceScan, stats );
|
||||
|
||||
stepStats.processedElems++;
|
||||
notifyInProgressIfNeeded(stepStats);
|
||||
}
|
||||
|
||||
return true;
|
||||
}, excludeDirFileName);
|
||||
|
||||
notifyInProgress(stepStats);
|
||||
}
|
||||
|
||||
// Check if a file exists and is still in a media directory
|
||||
static bool
|
||||
checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory, const std::vector<std::filesystem::path>& extensions)
|
||||
{
|
||||
try
|
||||
{
|
||||
// For each track, make sure the the file still exists
|
||||
// and still belongs to a media directory
|
||||
if (!std::filesystem::exists( p )
|
||||
|| !std::filesystem::is_regular_file( p ) )
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!isPathInMediaDirectory(p, mediaDirectory))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': out of media directory";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!isFileSupported(p, extensions))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': file format no longer handled";
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
catch (std::filesystem::filesystem_error& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR) << "Caught exception while checking file '" << p.string() << "': " << e.what();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
ScannerService::removeMissingTracks(ScanStats& stats)
|
||||
{
|
||||
static constexpr std::size_t batchSize {50};
|
||||
|
||||
ScanStepStats stepStats{stats.startTime, ScanProgressStep::ChekingForMissingFiles};
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed...";
|
||||
std::size_t trackCount {};
|
||||
|
||||
{
|
||||
auto transaction {_dbSession.createSharedTransaction()};
|
||||
trackCount = Track::getCount(_dbSession);
|
||||
}
|
||||
LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked...";
|
||||
|
||||
stepStats.totalElems = trackCount;
|
||||
notifyInProgress(stepStats);
|
||||
|
||||
RangeResults<Track::PathResult> trackPaths;
|
||||
std::vector<TrackId> tracksToRemove;
|
||||
|
||||
for (std::size_t i {trackCount < batchSize ? 0 : trackCount - batchSize}; ; i -= (i > batchSize ? batchSize : i))
|
||||
{
|
||||
tracksToRemove.clear();
|
||||
|
||||
{
|
||||
auto transaction {_dbSession.createSharedTransaction()};
|
||||
trackPaths = Track::findPaths(_dbSession, Range {i, batchSize});
|
||||
}
|
||||
|
||||
for (const Track::PathResult& trackPath : trackPaths.results)
|
||||
{
|
||||
if (_abortScan)
|
||||
return;
|
||||
|
||||
if (!checkFile(trackPath.path, _mediaDirectory, _fileExtensions))
|
||||
tracksToRemove.push_back(trackPath.trackId);
|
||||
|
||||
stepStats.processedElems++;
|
||||
}
|
||||
|
||||
if (!tracksToRemove.empty())
|
||||
{
|
||||
auto transaction {_dbSession.createUniqueTransaction()};
|
||||
|
||||
for (const TrackId trackId : tracksToRemove)
|
||||
{
|
||||
Track::pointer track {Track::find(_dbSession, trackId)};
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
notifyInProgressIfNeeded(stepStats);
|
||||
|
||||
if (i == 0)
|
||||
break;
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks checked!";
|
||||
}
|
||||
|
||||
void
|
||||
ScannerService::removeOrphanEntries()
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters...";
|
||||
{
|
||||
auto transaction {_dbSession.createUniqueTransaction()};
|
||||
|
||||
// Now process orphan Cluster (no track)
|
||||
auto clusterIds {Cluster::findOrphans(_dbSession, Range {})};
|
||||
for (ClusterId clusterId : clusterIds.results)
|
||||
{
|
||||
Cluster::pointer cluster {Cluster::find(_dbSession, clusterId)};
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'";
|
||||
cluster.remove();
|
||||
}
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists...";
|
||||
{
|
||||
auto transaction {_dbSession.createUniqueTransaction()};
|
||||
|
||||
auto artistIds {Artist::findAllOrphans(_dbSession, Range {})};
|
||||
for (const ArtistId artistId : artistIds.results)
|
||||
{
|
||||
Artist::pointer artist {Artist::find(_dbSession, artistId)};
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'";
|
||||
artist.remove();
|
||||
}
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases...";
|
||||
{
|
||||
auto transaction {_dbSession.createUniqueTransaction()};
|
||||
|
||||
auto releases {Release::findOrphans(_dbSession, Range {})};
|
||||
for (const ReleaseId releaseId : releases.results)
|
||||
{
|
||||
Release::pointer release {Release::find(_dbSession, releaseId)};
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'";
|
||||
release.remove();
|
||||
}
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "Check audio files done!";
|
||||
}
|
||||
|
||||
void
|
||||
ScannerService::checkDuplicatedAudioFiles(ScanStats& stats)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files";
|
||||
|
||||
auto transaction {_dbSession.createSharedTransaction()};
|
||||
|
||||
const RangeResults<TrackId> tracks = Track::findTrackMBIDDuplicates(_dbSession, Range {});
|
||||
for (const TrackId trackId : tracks.results)
|
||||
{
|
||||
const Track::pointer track {Track::find(_dbSession, trackId)};
|
||||
if (auto trackMBID {track->getTrackMBID()})
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName();
|
||||
stats.duplicates.emplace_back(ScanDuplicate {track->getId(), DuplicateReason::SameTrackMBID});
|
||||
}
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!";
|
||||
}
|
||||
|
||||
void
|
||||
ScannerService::reloadSimilarityEngine(ScanStats& stats)
|
||||
{
|
||||
ScanStepStats stepStats {stats.startTime, ScanProgressStep::ReloadingSimilarityEngine};
|
||||
|
||||
auto progressCallback {[&](const Recommendation::Progress& progress)
|
||||
{
|
||||
stepStats.totalElems = progress.totalElems;
|
||||
stepStats.processedElems = progress.processedElems;
|
||||
notifyInProgressIfNeeded(stepStats);
|
||||
}};
|
||||
|
||||
notifyInProgress(stepStats);
|
||||
_recommendationService.load(stats.nbChanges() > 0, progressCallback);
|
||||
notifyInProgress(stepStats);
|
||||
}
|
||||
|
||||
} // namespace Scanner
|
||||
|
||||
Reference in New Issue
Block a user