Split the lib in smaller libs to ease unit tests
This commit is contained in:
emeric
@@ -0,0 +1,829 @@
|
||||
/*
|
||||
* Copyright (C) 2013 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "MediaScanner.hpp"
|
||||
|
||||
#include <boost/asio/placeholders.hpp>
|
||||
|
||||
#include <Wt/WLocalDateTime.h>
|
||||
|
||||
#include "database/Artist.hpp"
|
||||
#include "database/Cluster.hpp"
|
||||
#include "database/Release.hpp"
|
||||
#include "database/ScanSettings.hpp"
|
||||
#include "database/Track.hpp"
|
||||
#include "utils/Exception.hpp"
|
||||
#include "utils/Logger.hpp"
|
||||
#include "utils/Path.hpp"
|
||||
|
||||
using namespace Database;
|
||||
|
||||
namespace {
|
||||
|
||||
Wt::WDate
|
||||
getNextMonday(Wt::WDate current)
|
||||
{
|
||||
do
|
||||
{
|
||||
current = current.addDays(1);
|
||||
} while (current.dayOfWeek() != 1);
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
Wt::WDate
|
||||
getNextFirstOfMonth(Wt::WDate current)
|
||||
{
|
||||
do
|
||||
{
|
||||
current = current.addDays(1);
|
||||
} while (current.day() != 1);
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
bool
|
||||
isFileSupported(const std::filesystem::path& file, const std::set<std::filesystem::path>& extensions)
|
||||
{
|
||||
return (extensions.find(file.extension()) != extensions.end());
|
||||
}
|
||||
|
||||
bool
|
||||
isPathInParentPath(const std::filesystem::path& path, const std::filesystem::path& parentPath)
|
||||
{
|
||||
std::filesystem::path curPath = path;
|
||||
|
||||
while (curPath.parent_path() != curPath)
|
||||
{
|
||||
curPath = curPath.parent_path();
|
||||
|
||||
if (curPath == parentPath)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
std::vector<Artist::pointer>
|
||||
getOrCreateArtists(Session& session, const std::vector<MetaData::Artist>& artistsInfo)
|
||||
{
|
||||
std::vector<Artist::pointer> artists;
|
||||
|
||||
for (const MetaData::Artist& artistInfo : artistsInfo)
|
||||
{
|
||||
Artist::pointer artist;
|
||||
|
||||
// First try to get by MBID
|
||||
if (artistInfo.musicBrainzArtistID)
|
||||
{
|
||||
artist = Artist::getByMBID(session, *artistInfo.musicBrainzArtistID);
|
||||
if (!artist)
|
||||
artist = Artist::create(session, artistInfo.name, artistInfo.musicBrainzArtistID);
|
||||
|
||||
artists.emplace_back(std::move(artist));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fall back on artist name (collisions may occur)
|
||||
if (!artistInfo.name.empty())
|
||||
{
|
||||
for (const Artist::pointer& sameNamedArtist : Artist::getByName(session, artistInfo.name))
|
||||
{
|
||||
// Do not fallback on artist that is correctly tagged
|
||||
if (!sameNamedArtist->getMBID())
|
||||
{
|
||||
artist = sameNamedArtist;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// No Artist found with the same name and without MBID -> creating
|
||||
if (!artist)
|
||||
artist = Artist::create(session, artistInfo.name);
|
||||
|
||||
artists.emplace_back(std::move(artist));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return artists;
|
||||
}
|
||||
|
||||
Release::pointer
|
||||
getOrCreateRelease(Session& session, const MetaData::Album& album)
|
||||
{
|
||||
Release::pointer release;
|
||||
|
||||
// First try to get by MBID
|
||||
if (album.musicBrainzAlbumID)
|
||||
{
|
||||
release = Release::getByMBID(session, *album.musicBrainzAlbumID);
|
||||
if (!release)
|
||||
release = Release::create(session, album.name, album.musicBrainzAlbumID);
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
// Fall back on release name (collisions may occur)
|
||||
if (!album.name.empty())
|
||||
{
|
||||
for (const Release::pointer& sameNamedRelease : Release::getByName(session, album.name))
|
||||
{
|
||||
// do not fallback on properly tagged releases
|
||||
if (!sameNamedRelease->getMBID())
|
||||
{
|
||||
release = sameNamedRelease;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// No release found with the same name and without MBID -> creating
|
||||
if (!release)
|
||||
release = Release::create(session, album.name);
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
return Release::pointer{};
|
||||
}
|
||||
|
||||
std::vector<Cluster::pointer>
|
||||
getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames)
|
||||
{
|
||||
std::vector< Cluster::pointer > clusters;
|
||||
|
||||
for (auto clusterNames : clustersNames)
|
||||
{
|
||||
auto clusterType = ClusterType::getByName(session, clusterNames.first);
|
||||
if (!clusterType)
|
||||
continue;
|
||||
|
||||
for (auto clusterName : clusterNames.second)
|
||||
{
|
||||
auto cluster = clusterType->getCluster(clusterName);
|
||||
if (!cluster)
|
||||
cluster = Cluster::create(session, clusterType, clusterName);
|
||||
|
||||
clusters.push_back(cluster);
|
||||
}
|
||||
}
|
||||
|
||||
return clusters;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace Scanner {
|
||||
|
||||
std::unique_ptr<IMediaScanner>
|
||||
createMediaScanner(Database::Db& db)
|
||||
{
|
||||
return std::make_unique<MediaScanner>(db);
|
||||
}
|
||||
|
||||
MediaScanner::MediaScanner(Database::Db& db)
|
||||
: _dbSession {db}
|
||||
{
|
||||
_ioService.setThreadCount(1);
|
||||
|
||||
refreshScanSettings();
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::setAddon(MediaScannerAddon& addon)
|
||||
{
|
||||
_addons.push_back(&addon);
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::restart(void)
|
||||
{
|
||||
stop();
|
||||
start();
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::start(void)
|
||||
{
|
||||
_running = true;
|
||||
|
||||
scheduleNextScan();
|
||||
|
||||
_ioService.start();
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::stop(void)
|
||||
{
|
||||
_running = false;
|
||||
|
||||
for (auto& addon : _addons)
|
||||
addon->requestStop();
|
||||
|
||||
_scheduleTimer.cancel();
|
||||
|
||||
_ioService.stop();
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::requestImmediateScan()
|
||||
{
|
||||
_ioService.post([=]()
|
||||
{
|
||||
scheduleScan();
|
||||
});
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::requestReschedule()
|
||||
{
|
||||
_ioService.post([=]()
|
||||
{
|
||||
scheduleNextScan();
|
||||
});
|
||||
}
|
||||
|
||||
MediaScanner::Status
|
||||
MediaScanner::getStatus()
|
||||
{
|
||||
Status res;
|
||||
|
||||
std::unique_lock<std::mutex> lock {_statusMutex};
|
||||
|
||||
res.currentState = _curState;
|
||||
res.nextScheduledScan = _nextScheduledScan;
|
||||
res.lastCompleteScanStats = _lastCompleteScanStats;
|
||||
res.inProgressScanStats = _inProgressScanStats;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::scheduleNextScan()
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan";
|
||||
|
||||
refreshScanSettings();
|
||||
|
||||
Wt::WDateTime now {Wt::WLocalDateTime::currentServerDateTime().toUTC()};
|
||||
|
||||
Wt::WDate nextScanDate;
|
||||
switch (_updatePeriod)
|
||||
{
|
||||
case ScanSettings::UpdatePeriod::Daily:
|
||||
if (now.time() < _startTime)
|
||||
nextScanDate = now.date();
|
||||
else
|
||||
nextScanDate = now.date().addDays(1);
|
||||
break;
|
||||
|
||||
case ScanSettings::UpdatePeriod::Weekly:
|
||||
if (now.time() < _startTime && now.date().dayOfWeek() == 1)
|
||||
nextScanDate = now.date();
|
||||
else
|
||||
nextScanDate = getNextMonday(now.date());
|
||||
break;
|
||||
|
||||
case ScanSettings::UpdatePeriod::Monthly:
|
||||
if (now.time() < _startTime && now.date().day() == 1)
|
||||
nextScanDate = now.date();
|
||||
else
|
||||
nextScanDate = getNextFirstOfMonth(now.date());
|
||||
break;
|
||||
|
||||
case ScanSettings::UpdatePeriod::Never:
|
||||
LMS_LOG(DBUPDATER, INFO) << "Auto scan disabled!";
|
||||
break;
|
||||
}
|
||||
|
||||
Wt::WDateTime nextScanDateTime;
|
||||
|
||||
if (nextScanDate.isValid())
|
||||
{
|
||||
nextScanDateTime = Wt::WDateTime {nextScanDate, _startTime};
|
||||
scheduleScan(nextScanDateTime);
|
||||
}
|
||||
|
||||
{
|
||||
std::unique_lock<std::mutex> lock {_statusMutex};
|
||||
_curState = nextScanDateTime.isValid() ? State::Scheduled : State::NotScheduled;
|
||||
_nextScheduledScan = nextScanDateTime;
|
||||
}
|
||||
|
||||
_sigScheduled.emit(_nextScheduledScan);
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::countAllFiles(ScanStats& stats)
|
||||
{
|
||||
std::error_code ec;
|
||||
|
||||
stats.filesToScan = 0;
|
||||
|
||||
std::filesystem::recursive_directory_iterator itPath {_mediaDirectory, std::filesystem::directory_options::follow_directory_symlink, ec};
|
||||
if (ec)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR) << "Cannot iterate over '" << _mediaDirectory.string() << "': " << ec.message();
|
||||
return;
|
||||
}
|
||||
|
||||
std::filesystem::recursive_directory_iterator itEnd;
|
||||
while (_running && itPath != itEnd)
|
||||
{
|
||||
const std::filesystem::path& path {*itPath};
|
||||
|
||||
if (!ec)
|
||||
{
|
||||
if (std::filesystem::is_regular_file(path) && isFileSupported(path, _fileExtensions))
|
||||
stats.filesToScan ++;
|
||||
|
||||
if (stats.filesToScan % 250 == 0)
|
||||
notifyInProgressIfNeeded(stats);
|
||||
}
|
||||
|
||||
itPath.increment(ec);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::scheduleScan(const Wt::WDateTime& dateTime)
|
||||
{
|
||||
if (dateTime.isNull())
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan right now";
|
||||
_scheduleTimer.expires_from_now(std::chrono::seconds(0));
|
||||
_scheduleTimer.async_wait(std::bind(&MediaScanner::scan, this, std::placeholders::_1));
|
||||
}
|
||||
else
|
||||
{
|
||||
std::chrono::system_clock::time_point timePoint {dateTime.toTimePoint()};
|
||||
std::time_t t {std::chrono::system_clock::to_time_t(timePoint)};
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan at " << std::string(std::ctime(&t));
|
||||
_scheduleTimer.expires_at(timePoint);
|
||||
_scheduleTimer.async_wait(std::bind(&MediaScanner::scan, this, std::placeholders::_1));
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::scan(boost::system::error_code err)
|
||||
{
|
||||
if (err)
|
||||
return;
|
||||
|
||||
{
|
||||
std::unique_lock<std::mutex> lock {_statusMutex};
|
||||
_curState = State::InProgress;
|
||||
_nextScheduledScan = {};
|
||||
}
|
||||
|
||||
ScanStats stats;
|
||||
stats.startTime = Wt::WLocalDateTime::currentDateTime().toUTC();
|
||||
|
||||
LMS_LOG(UI, INFO) << "New scan started!";
|
||||
|
||||
refreshScanSettings();
|
||||
|
||||
bool forceScan {false};
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Counting files in media directory '" << _mediaDirectory.string() << "'...";
|
||||
countAllFiles(stats);
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesToScan;
|
||||
|
||||
removeMissingTracks(stats);
|
||||
|
||||
LMS_LOG(UI, INFO) << "Checks complete, force scan = " << forceScan;
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "scaning media directory '" << _mediaDirectory.string() << "'...";
|
||||
scanMediaDirectory(_mediaDirectory, forceScan, stats);
|
||||
LMS_LOG(DBUPDATER, INFO) << "scaning media directory '" << _mediaDirectory.string() << "' DONE";
|
||||
|
||||
removeOrphanEntries();
|
||||
|
||||
if (_running)
|
||||
checkDuplicatedAudioFiles(stats);
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), duplicates = " << stats.duplicates.size();
|
||||
|
||||
if (_running)
|
||||
{
|
||||
for (auto& addon : _addons)
|
||||
addon->preScanComplete();
|
||||
}
|
||||
|
||||
if (_running)
|
||||
{
|
||||
stats.stopTime = Wt::WLocalDateTime::currentDateTime().toUTC();
|
||||
{
|
||||
std::unique_lock<std::mutex> lock {_statusMutex};
|
||||
|
||||
_lastCompleteScanStats = std::move(stats);
|
||||
_inProgressScanStats.reset();
|
||||
}
|
||||
|
||||
scheduleNextScan();
|
||||
|
||||
scanComplete().emit();
|
||||
}
|
||||
else
|
||||
{
|
||||
std::unique_lock<std::mutex> lock {_statusMutex};
|
||||
|
||||
_curState = State::NotScheduled;
|
||||
_inProgressScanStats.reset();
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "Optimizing db...";
|
||||
_dbSession.optimize();
|
||||
LMS_LOG(DBUPDATER, INFO) << "Optimize db done!";
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::refreshScanSettings()
|
||||
{
|
||||
{
|
||||
auto transaction {_dbSession.createSharedTransaction()};
|
||||
|
||||
ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)};
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion();
|
||||
|
||||
_scanVersion = scanSettings->getScanVersion();
|
||||
_startTime = scanSettings->getUpdateStartTime();
|
||||
_updatePeriod = scanSettings->getUpdatePeriod();
|
||||
|
||||
_fileExtensions = scanSettings->getAudioFileExtensions();
|
||||
_mediaDirectory = scanSettings->getMediaDirectory();
|
||||
|
||||
auto clusterTypes = scanSettings->getClusterTypes();
|
||||
std::set<std::string> clusterTypeNames;
|
||||
|
||||
std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes),
|
||||
std::inserter(clusterTypeNames, clusterTypeNames.begin()),
|
||||
[](ClusterType::pointer clusterType) { return clusterType->getName(); });
|
||||
|
||||
_metadataParser.setClusterTypeNames(clusterTypeNames);
|
||||
}
|
||||
|
||||
for (auto& addon : _addons)
|
||||
addon->refreshSettings();
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::notifyInProgress(const ScanStats& stats)
|
||||
{
|
||||
{
|
||||
std::unique_lock<std::mutex> lock {_statusMutex};
|
||||
_inProgressScanStats = stats.toProgressStats();
|
||||
}
|
||||
|
||||
std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
|
||||
_sigScanInProgress(*_inProgressScanStats);
|
||||
_lastScanInProgressEmit = now;
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::notifyInProgressIfNeeded(const ScanStats& stats)
|
||||
{
|
||||
std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
|
||||
|
||||
if (std::chrono::duration_cast<std::chrono::seconds>(now - _lastScanInProgressEmit).count() > 2)
|
||||
notifyInProgress(stats);
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats)
|
||||
{
|
||||
notifyInProgressIfNeeded(stats);
|
||||
|
||||
Wt::WDateTime lastWriteTime;
|
||||
try
|
||||
{
|
||||
lastWriteTime = getLastWriteTime(file);
|
||||
}
|
||||
catch (LmsException& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR) << e.what();
|
||||
stats.skips++;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!forceScan)
|
||||
{
|
||||
// Skip file if last write is the same
|
||||
auto transaction {_dbSession.createSharedTransaction()};
|
||||
|
||||
const Track::pointer track {Track::getByPath(_dbSession, file)};
|
||||
|
||||
if (track && track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t()
|
||||
&& track->getScanVersion() == _scanVersion)
|
||||
{
|
||||
stats.skips++;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<MetaData::Track> trackInfo {_metadataParser.parse(file)};
|
||||
if (!trackInfo)
|
||||
{
|
||||
stats.errors.emplace_back(file, ScanErrorType::CannotParseFile);
|
||||
return;
|
||||
}
|
||||
|
||||
stats.scans++;
|
||||
|
||||
auto uniqueTransaction {_dbSession.createUniqueTransaction()};
|
||||
|
||||
Track::pointer track {Track::getByPath(_dbSession, file) };
|
||||
|
||||
// We estimate this is an audio file if:
|
||||
// - we found a least one audio stream
|
||||
// - the duration is not null
|
||||
if (trackInfo->audioStreams.empty())
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (no audio stream found)";
|
||||
|
||||
// If Track exists here, delete it!
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
stats.errors.emplace_back(ScanError {file, ScanErrorType::NoAudioTrack});
|
||||
return;
|
||||
}
|
||||
if (trackInfo->duration == std::chrono::milliseconds::zero())
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (duration is 0)";
|
||||
|
||||
// If Track exists here, delete it!
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
stats.errors.emplace_back(ScanError {file, ScanErrorType::BadDuration});
|
||||
return;
|
||||
}
|
||||
|
||||
// ***** Title
|
||||
std::string title;
|
||||
if (!trackInfo->title.empty())
|
||||
title = trackInfo->title;
|
||||
else
|
||||
{
|
||||
// TODO parse file name guess track etc.
|
||||
// For now juste use file name as title
|
||||
title = file.filename().string();
|
||||
}
|
||||
|
||||
// ***** Clusters
|
||||
std::vector<Cluster::pointer> clusters {getOrCreateClusters(_dbSession, trackInfo->clusters)};
|
||||
|
||||
// ***** Artists
|
||||
std::vector<Artist::pointer> artists {getOrCreateArtists(_dbSession, trackInfo->artists)};
|
||||
|
||||
// ***** Release artists
|
||||
std::vector<Artist::pointer> releaseArtists {getOrCreateArtists(_dbSession, trackInfo->albumArtists)};
|
||||
|
||||
// ***** Release
|
||||
Release::pointer release;
|
||||
if (trackInfo->album)
|
||||
release = getOrCreateRelease(_dbSession, *trackInfo->album);
|
||||
|
||||
// If file already exist, update data
|
||||
// Otherwise, create it
|
||||
if (!track)
|
||||
{
|
||||
// Create a new song
|
||||
track = Track::create(_dbSession, file);
|
||||
LMS_LOG(DBUPDATER, INFO) << "Adding '" << file.string() << "'";
|
||||
stats.additions++;
|
||||
}
|
||||
else
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Updating '" << file.string() << "'";
|
||||
|
||||
stats.updates++;
|
||||
}
|
||||
|
||||
// Release related data
|
||||
if (release)
|
||||
{
|
||||
release.modify()->setTotalTrackNumber(trackInfo->totalTrack ? *trackInfo->totalTrack : 0);
|
||||
release.modify()->setTotalDiscNumber(trackInfo->totalDisc ? *trackInfo->totalDisc : 0);
|
||||
}
|
||||
|
||||
// Track related data
|
||||
assert(track);
|
||||
|
||||
track.modify()->clearArtistLinks();
|
||||
for (const auto& artist : artists)
|
||||
track.modify()->addArtistLink(Database::TrackArtistLink::create(_dbSession, track, artist, Database::TrackArtistLink::Type::Artist));
|
||||
|
||||
for (const auto& releaseArtist : releaseArtists)
|
||||
track.modify()->addArtistLink(Database::TrackArtistLink::create(_dbSession, track, releaseArtist, Database::TrackArtistLink::Type::ReleaseArtist));
|
||||
|
||||
track.modify()->setScanVersion(_scanVersion);
|
||||
track.modify()->setRelease(release);
|
||||
track.modify()->setClusters(clusters);
|
||||
track.modify()->setLastWriteTime(lastWriteTime);
|
||||
track.modify()->setName(title);
|
||||
track.modify()->setDuration(trackInfo->duration);
|
||||
track.modify()->setAddedTime(Wt::WLocalDateTime::currentServerDateTime().toUTC());
|
||||
track.modify()->setTrackNumber(trackInfo->trackNumber ? *trackInfo->trackNumber : 0);
|
||||
track.modify()->setDiscNumber(trackInfo->discNumber ? *trackInfo->discNumber : 0);
|
||||
track.modify()->setYear(trackInfo->year ? *trackInfo->year : 0);
|
||||
track.modify()->setOriginalYear(trackInfo->originalYear ? *trackInfo->originalYear : 0);
|
||||
|
||||
// If a file has an OriginalYear but no Year, set it to ease filtering
|
||||
if (!trackInfo->year && trackInfo->originalYear)
|
||||
track.modify()->setYear(*trackInfo->originalYear);
|
||||
|
||||
track.modify()->setMBID(trackInfo->musicBrainzRecordID);
|
||||
track.modify()->setHasCover(trackInfo->hasCover);
|
||||
track.modify()->setCopyright(trackInfo->copyright);
|
||||
track.modify()->setCopyrightURL(trackInfo->copyrightURL);
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats)
|
||||
{
|
||||
std::error_code ec;
|
||||
|
||||
std::filesystem::recursive_directory_iterator itPath {_mediaDirectory, std::filesystem::directory_options::follow_directory_symlink, ec};
|
||||
if (ec)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR) << "Cannot iterate over '" << mediaDirectory.string() << "': " << ec.message();
|
||||
stats.errors.emplace_back(ScanError {mediaDirectory, ScanErrorType::CannotReadFile, ec.message()});
|
||||
return;
|
||||
}
|
||||
|
||||
std::filesystem::recursive_directory_iterator itEnd;
|
||||
while (_running && itPath != itEnd)
|
||||
{
|
||||
const std::filesystem::path& path {*itPath};
|
||||
|
||||
if (ec)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message();
|
||||
stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()});
|
||||
}
|
||||
else if (std::filesystem::is_regular_file(path))
|
||||
{
|
||||
if (isFileSupported(path, _fileExtensions))
|
||||
scanAudioFile(path, forceScan, stats );
|
||||
}
|
||||
|
||||
itPath.increment(ec);
|
||||
}
|
||||
|
||||
notifyInProgress(stats);
|
||||
}
|
||||
|
||||
// Check if a file exists and is still in a media directory
|
||||
static bool
|
||||
checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory, const std::set<std::filesystem::path>& extensions)
|
||||
{
|
||||
try
|
||||
{
|
||||
// For each track, make sure the the file still exists
|
||||
// and still belongs to a media directory
|
||||
if (!std::filesystem::exists( p )
|
||||
|| !std::filesystem::is_regular_file( p ) )
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!isPathInParentPath(p, mediaDirectory))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': out of media directory";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!isFileSupported(p, extensions))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': file format no longer handled";
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
}
|
||||
catch (std::filesystem::filesystem_error& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR) << "Caught exception while checking file '" << p.string() << "': " << e.what();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::removeMissingTracks(ScanStats& stats)
|
||||
{
|
||||
std::vector<std::filesystem::path> trackPaths;
|
||||
{
|
||||
auto transaction {_dbSession.createSharedTransaction()};
|
||||
trackPaths = Track::getAllPaths(_dbSession);;
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks...";
|
||||
for (const auto& trackPath : trackPaths)
|
||||
{
|
||||
if (!_running)
|
||||
return;
|
||||
|
||||
if (!checkFile(trackPath, _mediaDirectory, _fileExtensions))
|
||||
{
|
||||
auto transaction {_dbSession.createUniqueTransaction()};
|
||||
|
||||
Track::pointer track {Track::getByPath(_dbSession, trackPath)};
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::removeOrphanEntries()
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters...";
|
||||
{
|
||||
auto transaction {_dbSession.createUniqueTransaction()};
|
||||
|
||||
// Now process orphan Cluster (no track)
|
||||
auto clusters {Cluster::getAllOrphans(_dbSession)};
|
||||
for (auto& cluster : clusters)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'";
|
||||
cluster.remove();
|
||||
}
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists...";
|
||||
{
|
||||
auto transaction {_dbSession.createUniqueTransaction()};
|
||||
|
||||
auto artists {Artist::getAllOrphans(_dbSession)};
|
||||
for (auto& artist : artists)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'";
|
||||
artist.remove();
|
||||
}
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases...";
|
||||
{
|
||||
auto transaction {_dbSession.createUniqueTransaction()};
|
||||
|
||||
auto releases {Release::getAllOrphans(_dbSession)};
|
||||
for (auto& release : releases)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'";
|
||||
release.remove();
|
||||
}
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "Check audio files done!";
|
||||
}
|
||||
|
||||
void
|
||||
MediaScanner::checkDuplicatedAudioFiles(ScanStats& stats)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files";
|
||||
|
||||
auto transaction {_dbSession.createSharedTransaction()};
|
||||
|
||||
const std::vector<Track::pointer> tracks = Database::Track::getMBIDDuplicates(_dbSession);
|
||||
for (const Track::pointer& track : tracks)
|
||||
{
|
||||
if (track->getMBID())
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO) << "Found duplicated MBID [" << track->getMBID()->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName();
|
||||
stats.duplicates.emplace_back(ScanDuplicate {track->getPath(), DuplicateReason::SameMBID});
|
||||
}
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!";
|
||||
}
|
||||
|
||||
} // namespace Scanner
|
||||
@@ -0,0 +1,111 @@
|
||||
/*
|
||||
* Copyright (C) 2013 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
|
||||
#include <Wt/WDateTime.h>
|
||||
#include <Wt/WIOService.h>
|
||||
#include <Wt/WSignal.h>
|
||||
|
||||
#include <boost/asio/system_timer.hpp>
|
||||
|
||||
#include "scanner/IMediaScanner.hpp"
|
||||
#include "database/ScanSettings.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "metadata/TagLibParser.hpp"
|
||||
|
||||
|
||||
namespace Scanner {
|
||||
|
||||
class MediaScanner : public IMediaScanner
|
||||
{
|
||||
public:
|
||||
MediaScanner(Database::Db& db);
|
||||
|
||||
void setAddon(MediaScannerAddon& addon) override;
|
||||
|
||||
void start() override;
|
||||
void stop() override;
|
||||
void restart() override;
|
||||
|
||||
void requestImmediateScan() override;
|
||||
void requestReschedule() override ;
|
||||
|
||||
Status getStatus() override;
|
||||
|
||||
Wt::Signal<>& scanComplete() override { return _sigScanComplete; }
|
||||
Wt::Signal<ScanProgressStats>& scanInProgress() override { return _sigScanInProgress; }
|
||||
Wt::Signal<Wt::WDateTime>& scheduled() override { return _sigScheduled; }
|
||||
|
||||
private:
|
||||
|
||||
// Job handling
|
||||
void scheduleNextScan();
|
||||
void scheduleScan(const Wt::WDateTime& dateTime = {});
|
||||
|
||||
// Update database (scheduled callback)
|
||||
void scan(boost::system::error_code ec);
|
||||
|
||||
void scanMediaDirectory( const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats);
|
||||
|
||||
// Helpers
|
||||
void refreshScanSettings();
|
||||
|
||||
void countAllFiles(ScanStats& stats);
|
||||
void removeMissingTracks(ScanStats& stats);
|
||||
void removeOrphanEntries();
|
||||
void checkDuplicatedAudioFiles(ScanStats& stats);
|
||||
void scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats);
|
||||
Database::IdType doScanAudioFile(const std::filesystem::path& file, ScanStats& stats);
|
||||
void notifyInProgressIfNeeded(const ScanStats& stats);
|
||||
void notifyInProgress(const ScanStats& stats);
|
||||
|
||||
bool _running {false};
|
||||
Wt::WIOService _ioService;
|
||||
boost::asio::system_timer _scheduleTimer {_ioService};
|
||||
Wt::Signal<> _sigScanComplete;
|
||||
Wt::Signal<ScanProgressStats> _sigScanInProgress;
|
||||
std::chrono::system_clock::time_point _lastScanInProgressEmit {};
|
||||
Wt::Signal<Wt::WDateTime> _sigScheduled;
|
||||
Database::Session _dbSession;
|
||||
MetaData::TagLibParser _metadataParser;
|
||||
std::vector<MediaScannerAddon*> _addons;
|
||||
|
||||
std::mutex _statusMutex;
|
||||
State _curState {State::NotScheduled};
|
||||
std::optional<ScanStats> _lastCompleteScanStats;
|
||||
std::optional<ScanProgressStats> _inProgressScanStats;
|
||||
Wt::WDateTime _nextScheduledScan;
|
||||
|
||||
// Current scan settings
|
||||
std::size_t _scanVersion {};
|
||||
Wt::WTime _startTime;
|
||||
Database::ScanSettings::UpdatePeriod _updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
|
||||
std::set<std::filesystem::path> _fileExtensions;
|
||||
std::filesystem::path _mediaDirectory;
|
||||
|
||||
|
||||
}; // class MediaScanner
|
||||
|
||||
} // Scanner
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright (C) 2019 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "scanner/MediaScannerStats.hpp"
|
||||
|
||||
namespace Scanner {
|
||||
|
||||
ScanError::ScanError(const std::filesystem::path& _file, ScanErrorType _error, const std::string& _systemError)
|
||||
: file {_file},
|
||||
error {_error},
|
||||
systemError {_systemError}
|
||||
{
|
||||
}
|
||||
|
||||
std::size_t
|
||||
ScanStats::nbFiles() const
|
||||
{
|
||||
return skips + additions + updates;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
ScanStats::nbChanges() const
|
||||
{
|
||||
return additions + deletions + updates;
|
||||
}
|
||||
|
||||
ScanProgressStats
|
||||
ScanStats::toProgressStats() const
|
||||
{
|
||||
return ScanProgressStats {startTime, filesToScan, nbFiles()};
|
||||
}
|
||||
|
||||
unsigned
|
||||
ScanProgressStats::progress() const
|
||||
{
|
||||
return (processedFiles / static_cast<float>(filesToScan ? filesToScan : 1)) * 100;
|
||||
}
|
||||
|
||||
} // namespace Scanner
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
/*
|
||||
* Copyright (C) 2013 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "AvFormat.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
|
||||
#include "av/AvInfo.hpp"
|
||||
|
||||
#include "utils/Logger.hpp"
|
||||
#include "utils/String.hpp"
|
||||
|
||||
|
||||
namespace MetaData
|
||||
{
|
||||
|
||||
using MetadataMap = std::map<std::string, std::string>;
|
||||
|
||||
template <typename T>
|
||||
std::optional<T>
|
||||
findFirstValueOfAs(const MetadataMap& metadataMap, std::initializer_list<std::string> tags)
|
||||
{
|
||||
auto it = std::find_first_of(std::cbegin(metadataMap), std::cend(metadataMap), std::cbegin(tags), std::cend(tags), [](const auto& it, const auto& str) { return it.first == str; });
|
||||
if (it == std::cend(metadataMap))
|
||||
return std::nullopt;
|
||||
|
||||
return StringUtils::readAs<T>(StringUtils::stringTrim(it->second));
|
||||
}
|
||||
|
||||
template <>
|
||||
std::optional<std::vector<UUID>>
|
||||
findFirstValueOfAs(const MetadataMap& metadataMap, std::initializer_list<std::string> tags)
|
||||
{
|
||||
std::optional<std::string> str {findFirstValueOfAs<std::string>(metadataMap, tags)};
|
||||
if (!str)
|
||||
return std::nullopt;
|
||||
|
||||
std::vector<std::string> strUuids = StringUtils::splitString(*str, "/");
|
||||
std::vector<UUID> res;
|
||||
|
||||
for (const std::string strUuid : strUuids)
|
||||
{
|
||||
std::optional<UUID> uuid {UUID::fromString(strUuid)};
|
||||
if (!uuid)
|
||||
return std::nullopt;
|
||||
|
||||
res.push_back(std::move(*uuid));
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
static
|
||||
std::optional<Album>
|
||||
getAlbum(const MetadataMap& metadataMap)
|
||||
{
|
||||
std::optional<Album> res;
|
||||
|
||||
auto album {findFirstValueOfAs<std::string>(metadataMap, {"ALBUM"})};
|
||||
if (!album)
|
||||
return res;
|
||||
|
||||
auto albumMBID {findFirstValueOfAs<UUID>(metadataMap, {"MUSICBRAINZ ALBUM ID", "MUSICBRAINZ_ALBUMID", "MUSICBRAINZ/ALBUM ID"})};
|
||||
|
||||
return Album{*album, albumMBID};
|
||||
}
|
||||
|
||||
static
|
||||
std::vector<Artist>
|
||||
getAlbumArtists(const MetadataMap& metadataMap)
|
||||
{
|
||||
std::vector<Artist> res;
|
||||
|
||||
auto name {findFirstValueOfAs<std::string>(metadataMap, {"ALBUM_ARTIST"})};
|
||||
if (!name)
|
||||
return res;
|
||||
|
||||
auto mbid {findFirstValueOfAs<UUID>(metadataMap, {"MUSICBRAINZ ALBUM ARTIST ID", "MUSICBRAINZ/ALBUM ARTIST ID"})};
|
||||
|
||||
return {Artist {*name, mbid} };
|
||||
}
|
||||
|
||||
static
|
||||
std::vector<Artist>
|
||||
getArtists(const MetadataMap& metadataMap)
|
||||
{
|
||||
std::vector<Artist> artists;
|
||||
|
||||
std::vector<std::string> artistNames;
|
||||
if (metadataMap.find("ARTISTS") != metadataMap.end())
|
||||
{
|
||||
artistNames = StringUtils::splitString(metadataMap.find("ARTISTS")->second, "/;");
|
||||
}
|
||||
else if (metadataMap.find("ARTIST") != metadataMap.end())
|
||||
{
|
||||
artistNames = {metadataMap.find("ARTIST")->second};
|
||||
}
|
||||
|
||||
auto artistMBIDs {findFirstValueOfAs<std::vector<UUID>>(metadataMap, {"MUSICBRAINZ ARTIST ID", "MUSICBRAINZ_ARTISTID", "MUSICBRAINZ/ARTIST ID"})};
|
||||
|
||||
for (std::size_t i {}; i < artistNames.size(); ++i)
|
||||
{
|
||||
if (artistMBIDs && artistNames.size() == artistMBIDs->size())
|
||||
artists.emplace_back(Artist {artistNames[i], (*artistMBIDs)[i]});
|
||||
else
|
||||
artists.emplace_back(Artist {artistNames[i], {}});
|
||||
}
|
||||
|
||||
return artists;
|
||||
}
|
||||
|
||||
std::optional<Track>
|
||||
AvFormat::parse(const std::filesystem::path& p, bool debug)
|
||||
{
|
||||
Track track;
|
||||
|
||||
try
|
||||
{
|
||||
Av::MediaFile mediaFile {p};
|
||||
|
||||
// Stream info
|
||||
{
|
||||
std::vector<AudioStream> audioStreams;
|
||||
|
||||
for (auto stream : mediaFile.getStreamInfo())
|
||||
{
|
||||
MetaData::AudioStream audioStream {static_cast<unsigned>(stream.bitrate)};
|
||||
track.audioStreams.emplace_back(audioStream);
|
||||
}
|
||||
}
|
||||
|
||||
track.duration = mediaFile.getDuration();
|
||||
track.hasCover = mediaFile.hasAttachedPictures();
|
||||
|
||||
MetaData::Clusters clusters;
|
||||
|
||||
const std::map<std::string, std::string> metadataMap {mediaFile.getMetaData()};
|
||||
|
||||
for (const auto& metadata : metadataMap)
|
||||
{
|
||||
const std::string& tag {metadata.first};
|
||||
const std::string& value {metadata.second};
|
||||
|
||||
if (debug)
|
||||
std::cout << "TAG = " << tag << ", VAL = " << value << std::endl;
|
||||
|
||||
if (tag == "TITLE")
|
||||
track.title = value;
|
||||
else if (tag == "TRACK")
|
||||
{
|
||||
// Expecting 'Number/Total'
|
||||
std::vector<std::string> strings {StringUtils::splitString(value, "/") };
|
||||
|
||||
if (strings.size() > 0)
|
||||
{
|
||||
track.trackNumber = StringUtils::readAs<std::size_t>(strings[0]);
|
||||
|
||||
if (strings.size() > 1)
|
||||
track.totalTrack = StringUtils::readAs<std::size_t>(strings[1]);
|
||||
}
|
||||
}
|
||||
else if (tag == "DISC")
|
||||
{
|
||||
// Expecting 'Number/Total'
|
||||
std::vector<std::string> strings {StringUtils::splitString(value, "/")};
|
||||
|
||||
if (strings.size() > 0)
|
||||
{
|
||||
track.discNumber = StringUtils::readAs<std::size_t>(strings[0]);
|
||||
|
||||
if (strings.size() > 1)
|
||||
track.totalDisc = StringUtils::readAs<std::size_t>(strings[1]);
|
||||
}
|
||||
}
|
||||
else if (tag == "DATE"
|
||||
|| tag == "YEAR"
|
||||
|| tag == "WM/Year")
|
||||
{
|
||||
track.year = StringUtils::readAs<int>(value);
|
||||
}
|
||||
else if (tag == "TDOR" // Original release time (ID3v2 2.4)
|
||||
|| tag == "TORY") // Original release year
|
||||
{
|
||||
track.originalYear = StringUtils::readAs<int>(value);
|
||||
}
|
||||
else if (tag == "ACOUSTID ID")
|
||||
{
|
||||
track.acoustID = UUID::fromString(value);
|
||||
}
|
||||
else if (tag == "MUSICBRAINZ RELEASE TRACK ID"
|
||||
|| tag == "MUSICBRAINZ_RELEASETRACKID"
|
||||
|| tag == "MUSICBRAINZ_TRACKID"
|
||||
|| tag == "MUSICBRAINZ/TRACK ID")
|
||||
{
|
||||
track.musicBrainzTrackID = UUID::fromString(value);
|
||||
}
|
||||
else if (_clusterTypeNames.find(tag) != _clusterTypeNames.end())
|
||||
{
|
||||
std::vector<std::string> clusterNames {StringUtils::splitString(value, "/,;")};
|
||||
|
||||
if (!clusterNames.empty())
|
||||
track.clusters[tag] = std::set<std::string>{clusterNames.begin(), clusterNames.end()};
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
track.artists = getArtists(metadataMap);
|
||||
track.album = getAlbum(metadataMap);
|
||||
track.albumArtists = getAlbumArtists(metadataMap);
|
||||
}
|
||||
catch(Av::MediaFileException& e)
|
||||
{
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return track;
|
||||
}
|
||||
|
||||
} // namespace MetaData
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (C) 2018 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "MetaData.hpp"
|
||||
|
||||
namespace MetaData
|
||||
{
|
||||
|
||||
// Parse that makes use of AvFormat
|
||||
class AvFormat : public Parser
|
||||
{
|
||||
public:
|
||||
std::optional<Track> parse(const std::filesystem::path& p, bool debug = false) override;
|
||||
};
|
||||
|
||||
} // namespace MetaData
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright (C) 2018 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <filesystem>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
//#include "utils/Utils.hpp"
|
||||
#include "utils/UUID.hpp"
|
||||
|
||||
namespace MetaData
|
||||
{
|
||||
using Clusters = std::map<std::string /* type */, std::set<std::string> /* names */>;
|
||||
|
||||
struct Artist
|
||||
{
|
||||
std::string name;
|
||||
std::optional<UUID> musicBrainzArtistID;
|
||||
};
|
||||
|
||||
struct Album
|
||||
{
|
||||
std::string name;
|
||||
std::optional<UUID> musicBrainzAlbumID;
|
||||
};
|
||||
|
||||
struct AudioStream
|
||||
{
|
||||
unsigned bitRate;
|
||||
};
|
||||
|
||||
struct Track
|
||||
{
|
||||
std::vector<Artist> artists;
|
||||
std::vector<Artist> albumArtists;
|
||||
std::string title;
|
||||
std::optional<UUID> musicBrainzTrackID;
|
||||
std::optional<UUID> musicBrainzRecordID;
|
||||
std::optional<Album> album;
|
||||
Clusters clusters;
|
||||
std::chrono::milliseconds duration {};
|
||||
std::optional<std::size_t> trackNumber;
|
||||
std::optional<std::size_t> totalTrack;
|
||||
std::optional<std::size_t> discNumber;
|
||||
std::optional<std::size_t> totalDisc;
|
||||
std::optional<int> year;
|
||||
std::optional<int> originalYear;
|
||||
bool hasCover {false};
|
||||
std::vector<AudioStream> audioStreams;
|
||||
std::optional<UUID> acoustID;
|
||||
std::string copyright;
|
||||
std::string copyrightURL;
|
||||
};
|
||||
|
||||
class Parser
|
||||
{
|
||||
public:
|
||||
virtual std::optional<Track> parse(const std::filesystem::path& p, bool debug = false) = 0;
|
||||
|
||||
void setClusterTypeNames(const std::set<std::string>& clusterTypeNames) { _clusterTypeNames = clusterTypeNames; }
|
||||
|
||||
protected:
|
||||
std::set<std::string> _clusterTypeNames;
|
||||
};
|
||||
|
||||
} // namespace MetaData
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
/*
|
||||
* Copyright (C) 2016 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "TagLibParser.hpp"
|
||||
|
||||
#include <taglib/asffile.h>
|
||||
#include <taglib/id3v2tag.h>
|
||||
#include <taglib/fileref.h>
|
||||
#include <taglib/flacfile.h>
|
||||
#include <taglib/mpegfile.h>
|
||||
#include <taglib/tag.h>
|
||||
#include <taglib/tpropertymap.h>
|
||||
|
||||
#include "utils/Logger.hpp"
|
||||
#include "utils/String.hpp"
|
||||
|
||||
|
||||
namespace MetaData
|
||||
{
|
||||
|
||||
template<typename T>
|
||||
std::vector<T>
|
||||
getPropertyValuesFirstMatchAs(const TagLib::PropertyMap& properties, const std::set<std::string>& keys)
|
||||
{
|
||||
std::vector<T> res;
|
||||
|
||||
for (const std::string& key : keys)
|
||||
{
|
||||
const TagLib::StringList& values {properties[key]};
|
||||
if (values.isEmpty())
|
||||
continue;
|
||||
|
||||
res.reserve(values.size());
|
||||
|
||||
for (const auto& value : values)
|
||||
{
|
||||
auto val {StringUtils::readAs<T>(StringUtils::stringTrim(value.to8Bit(true)))};
|
||||
if (!val)
|
||||
continue;
|
||||
|
||||
res.emplace_back(std::move(*val));
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::vector<T>
|
||||
getPropertyValuesAs(const TagLib::PropertyMap& properties, const std::string& key)
|
||||
{
|
||||
return getPropertyValuesFirstMatchAs<T>(properties, {std::move(key)});
|
||||
}
|
||||
|
||||
static
|
||||
std::vector<std::string>
|
||||
splitAndTrimString(const std::string& str, const std::string& delimiters)
|
||||
{
|
||||
std::vector<std::string> res;
|
||||
|
||||
std::vector<std::string> strings {StringUtils::splitString(str, delimiters)};
|
||||
for (const std::string& s : strings)
|
||||
res.emplace_back(StringUtils::stringTrim(s));
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
static
|
||||
std::vector<Artist>
|
||||
getArtists(const TagLib::PropertyMap& properties)
|
||||
{
|
||||
std::vector<Artist> res;
|
||||
|
||||
std::vector<std::string> artistNames {getPropertyValuesAs<std::string>(properties, "ARTISTS")};
|
||||
if (artistNames.empty())
|
||||
artistNames = getPropertyValuesAs<std::string>(properties, "ARTIST");
|
||||
|
||||
if (artistNames.empty())
|
||||
return res;
|
||||
|
||||
const std::vector<UUID> artistsMBID {getPropertyValuesFirstMatchAs<UUID>(properties, {"MUSICBRAINZ_ARTISTID", "MUSICBRAINZ ARTIST ID"})};
|
||||
|
||||
if (artistNames.size() == artistsMBID.size())
|
||||
{
|
||||
std::transform(std::cbegin(artistNames), std::cend(artistNames), std::cbegin(artistsMBID), std::back_inserter(res),
|
||||
[&](const std::string& name, const UUID& mbid) { return Artist {name, mbid}; });
|
||||
}
|
||||
else
|
||||
{
|
||||
std::transform(std::cbegin(artistNames), std::cend(artistNames), std::back_inserter(res),
|
||||
[&](const std::string& name) { return Artist{name, {}}; });
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
static
|
||||
std::vector<Artist>
|
||||
getAlbumArtists(const TagLib::PropertyMap& properties)
|
||||
{
|
||||
std::vector<Artist> res;
|
||||
|
||||
std::vector<std::string> artistNames {getPropertyValuesAs<std::string>(properties, "ALBUMARTIST")};
|
||||
if (artistNames.empty())
|
||||
return res;
|
||||
|
||||
const std::vector<UUID> artistsMBID {getPropertyValuesFirstMatchAs<UUID>(properties, {"MUSICBRAINZ_ALBUMARTISTID", "MUSICBRAINZ ALBUM ARTIST ID"})};
|
||||
|
||||
if (artistNames.size() == artistsMBID.size())
|
||||
{
|
||||
std::transform(std::cbegin(artistNames), std::cend(artistNames), std::cbegin(artistsMBID), std::back_inserter(res),
|
||||
[&](const std::string& name, const UUID& mbid) { return Artist{name, mbid}; });
|
||||
}
|
||||
else
|
||||
{
|
||||
std::transform(std::cbegin(artistNames), std::cend(artistNames), std::back_inserter(res),
|
||||
[&](const std::string& name) { return Artist{name, {}}; });
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
static
|
||||
std::optional<Album>
|
||||
getAlbum(const TagLib::PropertyMap& properties)
|
||||
{
|
||||
std::vector<std::string> albumName {getPropertyValuesAs<std::string>(properties, "ALBUM")};
|
||||
if (albumName.empty())
|
||||
return std::nullopt;
|
||||
|
||||
const std::vector<UUID> albumMBID {getPropertyValuesFirstMatchAs<UUID>(properties, {"MUSICBRAINZ_ALBUMID", "MUSICBRAINZ ALBUM ID"})};
|
||||
|
||||
if (albumMBID.empty())
|
||||
return Album {std::move(albumName.front()), {}};
|
||||
else
|
||||
return Album {std::move(albumName.front()), albumMBID.front()};
|
||||
}
|
||||
|
||||
std::optional<Track>
|
||||
TagLibParser::parse(const std::filesystem::path& p, bool debug)
|
||||
{
|
||||
TagLib::FileRef f {p.string().c_str(),
|
||||
true, // read audio properties
|
||||
TagLib::AudioProperties::Fast};
|
||||
|
||||
if (f.isNull())
|
||||
{
|
||||
LMS_LOG(METADATA, ERROR) << "File '" << p.string() << "': parsing failed";
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
if (!f.audioProperties())
|
||||
{
|
||||
LMS_LOG(METADATA, INFO) << "File '" << p.string() << "': no audio properties";
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
Track track;
|
||||
|
||||
{
|
||||
const TagLib::AudioProperties *properties {f.audioProperties() };
|
||||
|
||||
track.duration = std::chrono::milliseconds {properties->length() * 1000};
|
||||
|
||||
MetaData::AudioStream audioStream {static_cast<unsigned>(properties->bitrate() * 1000)};
|
||||
track.audioStreams = {std::move(audioStream)};
|
||||
}
|
||||
|
||||
// Not that good embedded pictures handling
|
||||
|
||||
// WMA
|
||||
if (TagLib::ASF::File* asfFile {dynamic_cast<TagLib::ASF::File*>(f.file())})
|
||||
{
|
||||
const TagLib::ASF::Tag* tag {asfFile->tag()};
|
||||
if (tag && tag->attributeListMap().contains("WM/Picture"))
|
||||
track.hasCover = true;
|
||||
}
|
||||
// MP3
|
||||
else if (TagLib::MPEG::File* mp3File {dynamic_cast<TagLib::MPEG::File*>(f.file())})
|
||||
{
|
||||
if (mp3File->ID3v2Tag())
|
||||
{
|
||||
if (!mp3File->ID3v2Tag()->frameListMap()["APIC"].isEmpty())
|
||||
track.hasCover = true;
|
||||
}
|
||||
}
|
||||
// FLAC
|
||||
else if (TagLib::FLAC::File* flacFile {dynamic_cast<TagLib::FLAC::File*>(f.file())})
|
||||
{
|
||||
if (!flacFile->pictureList().isEmpty())
|
||||
track.hasCover = true;
|
||||
}
|
||||
|
||||
if (f.tag())
|
||||
{
|
||||
MetaData::Clusters clusters;
|
||||
const TagLib::PropertyMap& properties {f.file()->properties()};
|
||||
|
||||
for(const auto& property : properties)
|
||||
{
|
||||
const std::string tag {property.first.upper().to8Bit(true)};
|
||||
const TagLib::StringList& values {property.second};
|
||||
|
||||
// TODO validate MBID format
|
||||
if (debug)
|
||||
{
|
||||
std::vector<std::string> strs;
|
||||
std::transform(values.begin(), values.end(), std::back_inserter(strs), [](const auto& value) { return value.to8Bit(true); });
|
||||
|
||||
std::cout << "[" << tag << "] = " << StringUtils::joinStrings(strs, "*SEP*") << std::endl;
|
||||
}
|
||||
|
||||
if (tag.empty() || values.isEmpty() || values.front().isEmpty())
|
||||
continue;
|
||||
|
||||
std::string value {StringUtils::stringTrim(values.front().to8Bit(true))};
|
||||
|
||||
if (tag == "TITLE")
|
||||
track.title = value;
|
||||
else if (tag == "MUSICBRAINZ_RELEASETRACKID"
|
||||
|| tag == "MUSICBRAINZ RELEASE TRACK ID")
|
||||
{
|
||||
track.musicBrainzTrackID = UUID::fromString(value);
|
||||
}
|
||||
else if (tag == "MUSICBRAINZ_TRACKID"
|
||||
|| tag == "MUSICBRAINZ TRACK ID")
|
||||
track.musicBrainzRecordID = UUID::fromString(value);
|
||||
else if (tag == "ACOUSTID_ID")
|
||||
track.acoustID = UUID::fromString(value);
|
||||
else if (tag == "TRACKTOTAL")
|
||||
{
|
||||
auto totalTrack = StringUtils::readAs<std::size_t>(value);
|
||||
if (totalTrack)
|
||||
track.totalTrack = totalTrack;
|
||||
}
|
||||
else if (tag == "TRACKNUMBER")
|
||||
{
|
||||
// Expecting 'Number/Total'
|
||||
std::vector<std::string> strings {splitAndTrimString(value, "/")};
|
||||
|
||||
if (!strings.empty())
|
||||
{
|
||||
track.trackNumber = StringUtils::readAs<std::size_t>(strings[0]);
|
||||
|
||||
// Lower priority than TRACKTOTAL
|
||||
if (strings.size() > 1 && !track.totalTrack)
|
||||
track.totalTrack = StringUtils::readAs<std::size_t>(strings[1]);
|
||||
}
|
||||
}
|
||||
else if (tag == "DISCTOTAL")
|
||||
{
|
||||
auto totalDisc = StringUtils::readAs<std::size_t>(value);
|
||||
if (totalDisc)
|
||||
track.totalDisc = totalDisc;
|
||||
}
|
||||
else if (tag == "DISCNUMBER")
|
||||
{
|
||||
// Expecting 'Number/Total'
|
||||
std::vector<std::string> strings {StringUtils::splitString(value, "/")};
|
||||
|
||||
if (!strings.empty())
|
||||
{
|
||||
track.discNumber = StringUtils::readAs<std::size_t>(strings[0]);
|
||||
|
||||
// Lower priority than DISCTOTAL
|
||||
if (strings.size() > 1 && !track.totalDisc)
|
||||
track.totalDisc = StringUtils::readAs<std::size_t>(strings[1]);
|
||||
}
|
||||
}
|
||||
else if (tag == "DATE")
|
||||
track.year = StringUtils::readAs<int>(value);
|
||||
else if (tag == "ORIGINALDATE" && !track.originalYear)
|
||||
{
|
||||
// Lower priority than ORIGINALYEAR
|
||||
track.originalYear = StringUtils::readAs<int>(value);
|
||||
}
|
||||
else if (tag == "ORIGINALYEAR")
|
||||
{
|
||||
// Higher priority than ORIGINALDATE
|
||||
auto originalYear = StringUtils::readAs<int>(value);
|
||||
if (originalYear)
|
||||
track.originalYear = originalYear;
|
||||
}
|
||||
else if (tag == "METADATA_BLOCK_PICTURE")
|
||||
track.hasCover = true;
|
||||
else if (tag == "COPYRIGHT")
|
||||
track.copyright = value;
|
||||
else if (tag == "COPYRIGHTURL")
|
||||
track.copyrightURL = value;
|
||||
else if (_clusterTypeNames.find(tag) != _clusterTypeNames.end())
|
||||
{
|
||||
std::set<std::string> clusterNames;
|
||||
for (const auto& valueList : values)
|
||||
{
|
||||
auto values = splitAndTrimString(valueList.to8Bit(true), "/,;");
|
||||
|
||||
for (const auto& value : values)
|
||||
clusterNames.insert(value);
|
||||
}
|
||||
|
||||
if (!clusterNames.empty())
|
||||
track.clusters[tag] = clusterNames;
|
||||
}
|
||||
}
|
||||
|
||||
track.artists = getArtists(properties);
|
||||
track.albumArtists = getAlbumArtists(properties);
|
||||
track.album = getAlbum(properties);
|
||||
|
||||
}
|
||||
|
||||
return track;
|
||||
}
|
||||
|
||||
} // namespace MetaData
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (C) 2018 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "MetaData.hpp"
|
||||
|
||||
namespace MetaData
|
||||
{
|
||||
|
||||
// Parse that makes use of AvFormat
|
||||
class TagLibParser : public Parser
|
||||
{
|
||||
public:
|
||||
std::optional<Track> parse(const std::filesystem::path& p, bool debug = false) override;
|
||||
};
|
||||
|
||||
} // namespace MetaData
|
||||
|
||||
Reference in New Issue
Block a user