Added playlist import (and sync)from m3u files, ref #391
This commit is contained in:
emeric
@@ -1,20 +1,25 @@
|
||||
|
||||
add_library(lmsscanner SHARED
|
||||
impl/FileScanQueue.cpp
|
||||
impl/scanners/AudioFileScanner.cpp
|
||||
impl/scanners/ImageFileScanner.cpp
|
||||
impl/scanners/LyricsFileScanner.cpp
|
||||
impl/scanners/PlayListFileScanner.cpp
|
||||
impl/scanners/Utils.cpp
|
||||
impl/steps/FileScanQueue.cpp
|
||||
impl/steps/ScanStepAssociateArtistImages.cpp
|
||||
impl/steps/ScanStepAssociateExternalLyrics.cpp
|
||||
impl/steps/ScanStepAssociatePlayListTracks.cpp
|
||||
impl/steps/ScanStepAssociateReleaseImages.cpp
|
||||
impl/steps/ScanStepCheckForDuplicatedFiles.cpp
|
||||
impl/steps/ScanStepCheckForRemovedFiles.cpp
|
||||
impl/steps/ScanStepCompact.cpp
|
||||
impl/steps/ScanStepComputeClusterStats.cpp
|
||||
impl/steps/ScanStepDiscoverFiles.cpp
|
||||
impl/steps/ScanStepOptimize.cpp
|
||||
impl/steps/ScanStepRemoveOrphanedDbEntries.cpp
|
||||
impl/steps/ScanStepScanFiles.cpp
|
||||
impl/steps/ScanStepUpdateLibraryFields.cpp
|
||||
impl/ScannerService.cpp
|
||||
impl/ScannerStats.cpp
|
||||
impl/ScanStepAssociateArtistImages.cpp
|
||||
impl/ScanStepAssociateExternalLyrics.cpp
|
||||
impl/ScanStepAssociateReleaseImages.cpp
|
||||
impl/ScanStepCheckForDuplicatedFiles.cpp
|
||||
impl/ScanStepCheckForRemovedFiles.cpp
|
||||
impl/ScanStepCompact.cpp
|
||||
impl/ScanStepComputeClusterStats.cpp
|
||||
impl/ScanStepDiscoverFiles.cpp
|
||||
impl/ScanStepOptimize.cpp
|
||||
impl/ScanStepRemoveOrphanedDbEntries.cpp
|
||||
impl/ScanStepScanFiles.cpp
|
||||
impl/ScanStepUpdateLibraryFields.cpp
|
||||
)
|
||||
|
||||
target_include_directories(lmsscanner INTERFACE
|
||||
@@ -23,6 +28,7 @@ target_include_directories(lmsscanner INTERFACE
|
||||
|
||||
target_include_directories(lmsscanner PRIVATE
|
||||
include
|
||||
impl
|
||||
)
|
||||
|
||||
target_link_libraries(lmsscanner PRIVATE
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "FileScanQueue.hpp"
|
||||
|
||||
#include <fstream>
|
||||
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/ITraceLogger.hpp"
|
||||
#include "image/Exception.hpp"
|
||||
#include "image/Image.hpp"
|
||||
#include "metadata/Exception.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
FileScanQueue::FileScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort)
|
||||
: _metadataParser{ parser }
|
||||
, _scanContextRunner{ _scanContext, threadCount, "FileScan" }
|
||||
, _abort{ abort }
|
||||
{
|
||||
}
|
||||
|
||||
void FileScanQueue::pushScanRequest(const std::filesystem::path& path, ScanRequestType type)
|
||||
{
|
||||
{
|
||||
std::scoped_lock lock{ _mutex };
|
||||
_ongoingScanCount += 1;
|
||||
}
|
||||
|
||||
_scanContext.post([=, this] {
|
||||
if (_abort)
|
||||
{
|
||||
std::scoped_lock lock{ _mutex };
|
||||
_ongoingScanCount -= 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
FileScanResult result;
|
||||
result.path = path;
|
||||
|
||||
switch (type)
|
||||
{
|
||||
case ScanRequestType::AudioFile:
|
||||
result.scanData = scanAudioFile(path);
|
||||
break;
|
||||
case ScanRequestType::ImageFile:
|
||||
result.scanData = scanImageFile(path);
|
||||
break;
|
||||
case ScanRequestType::LyricsFile:
|
||||
result.scanData = scanLyricsFile(path);
|
||||
break;
|
||||
}
|
||||
|
||||
{
|
||||
std::scoped_lock lock{ _mutex };
|
||||
|
||||
_scanResults.emplace_back(std::move(result));
|
||||
_ongoingScanCount -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
_condVar.notify_all();
|
||||
});
|
||||
}
|
||||
|
||||
AudioFileScanData FileScanQueue::scanAudioFile(const std::filesystem::path& path)
|
||||
{
|
||||
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanAudioFile");
|
||||
std::unique_ptr<metadata::Track> track;
|
||||
|
||||
try
|
||||
{
|
||||
track = _metadataParser.parse(path);
|
||||
}
|
||||
catch (const metadata::Exception& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO, "Failed to parse audio file '" << path.string() << "'");
|
||||
}
|
||||
|
||||
return track;
|
||||
}
|
||||
|
||||
ImageFileScanData FileScanQueue::scanImageFile(const std::filesystem::path& path)
|
||||
{
|
||||
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanImageFile");
|
||||
|
||||
std::optional<ImageInfo> optInfo;
|
||||
|
||||
try
|
||||
{
|
||||
std::unique_ptr<image::IRawImage> rawImage{ image::decodeImage(path) };
|
||||
ImageInfo& imageInfo{ optInfo.emplace() };
|
||||
imageInfo.width = rawImage->getWidth();
|
||||
imageInfo.height = rawImage->getHeight();
|
||||
}
|
||||
catch (const image::Exception& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot read image in file '" << path.string() << "': " << e.what());
|
||||
}
|
||||
|
||||
return optInfo;
|
||||
}
|
||||
|
||||
LyricsFileScanData FileScanQueue::scanLyricsFile(const std::filesystem::path& path)
|
||||
{
|
||||
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanLyricsFile");
|
||||
|
||||
LyricsFileScanData lyrics;
|
||||
|
||||
try
|
||||
{
|
||||
std::ifstream ifs{ path.string() };
|
||||
if (!ifs)
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot open file '" << path.string() << "'");
|
||||
else
|
||||
lyrics = metadata::parseLyrics(ifs);
|
||||
}
|
||||
catch (const std::exception& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot read lyrics in file '" << path.string() << "': " << e.what());
|
||||
}
|
||||
|
||||
return lyrics;
|
||||
}
|
||||
|
||||
std::size_t FileScanQueue::getResultsCount() const
|
||||
{
|
||||
std::scoped_lock lock{ _mutex };
|
||||
return _scanResults.size();
|
||||
}
|
||||
|
||||
size_t FileScanQueue::popResults(std::vector<FileScanResult>& results, std::size_t maxCount)
|
||||
{
|
||||
results.clear();
|
||||
results.reserve(maxCount);
|
||||
|
||||
{
|
||||
std::scoped_lock lock{ _mutex };
|
||||
|
||||
while (results.size() < maxCount && !_scanResults.empty())
|
||||
{
|
||||
results.push_back(std::move(_scanResults.front()));
|
||||
_scanResults.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
return results.size();
|
||||
}
|
||||
|
||||
void FileScanQueue::wait(std::size_t maxScanRequestCount)
|
||||
{
|
||||
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "WaitParseResults");
|
||||
|
||||
std::unique_lock lock{ _mutex };
|
||||
_condVar.wait(lock, [=, this] { return _ongoingScanCount <= maxScanRequestCount; });
|
||||
}
|
||||
|
||||
} // namespace lms::scanner
|
||||
@@ -1,86 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable>
|
||||
#include <deque>
|
||||
#include <filesystem>
|
||||
#include <mutex>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include "core/IOContextRunner.hpp"
|
||||
#include "metadata/IParser.hpp"
|
||||
#include "metadata/Lyrics.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
struct ImageInfo
|
||||
{
|
||||
std::size_t height{};
|
||||
std::size_t width{};
|
||||
};
|
||||
|
||||
using AudioFileScanData = std::unique_ptr<metadata::Track>;
|
||||
using ImageFileScanData = std::optional<ImageInfo>;
|
||||
using LyricsFileScanData = std::optional<metadata::Lyrics>;
|
||||
struct FileScanResult
|
||||
{
|
||||
std::filesystem::path path;
|
||||
std::variant<std::monostate, AudioFileScanData, ImageFileScanData, LyricsFileScanData> scanData;
|
||||
};
|
||||
|
||||
class FileScanQueue
|
||||
{
|
||||
public:
|
||||
FileScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort);
|
||||
|
||||
std::size_t getThreadCount() const { return _scanContextRunner.getThreadCount(); }
|
||||
|
||||
enum ScanRequestType
|
||||
{
|
||||
AudioFile,
|
||||
ImageFile,
|
||||
LyricsFile,
|
||||
};
|
||||
void pushScanRequest(const std::filesystem::path& path, ScanRequestType type);
|
||||
|
||||
std::size_t getResultsCount() const;
|
||||
size_t popResults(std::vector<FileScanResult>& results, std::size_t maxCount);
|
||||
|
||||
void wait(std::size_t maxScanRequestCount = 0); // wait until ongoing scan request count <= maxScanRequestCount
|
||||
|
||||
private:
|
||||
AudioFileScanData scanAudioFile(const std::filesystem::path& path);
|
||||
ImageFileScanData scanImageFile(const std::filesystem::path& path);
|
||||
LyricsFileScanData scanLyricsFile(const std::filesystem::path& path);
|
||||
|
||||
metadata::IParser& _metadataParser;
|
||||
boost::asio::io_context _scanContext;
|
||||
core::IOContextRunner _scanContextRunner;
|
||||
|
||||
mutable std::mutex _mutex;
|
||||
std::size_t _ongoingScanCount{};
|
||||
std::deque<FileScanResult> _scanResults;
|
||||
std::condition_variable _condVar;
|
||||
bool& _abort;
|
||||
};
|
||||
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (C) 2023 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
#include "database/MediaLibraryId.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
struct MediaLibraryInfo
|
||||
{
|
||||
db::MediaLibraryId id;
|
||||
std::filesystem::path rootDirectory;
|
||||
|
||||
auto operator<=>(const MediaLibraryInfo& other) const = default;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "services/scanner/ScannerOptions.hpp"
|
||||
#include "services/scanner/ScannerStats.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
struct ScanContext
|
||||
{
|
||||
ScanOptions scanOptions;
|
||||
ScanStats stats;
|
||||
ScanStepStats currentStepStats;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -1,973 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2023 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "ScanStepScanFiles.hpp"
|
||||
|
||||
#include "core/Exception.hpp"
|
||||
#include "core/IConfig.hpp"
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/ITraceLogger.hpp"
|
||||
#include "core/Path.hpp"
|
||||
#include "database/Artist.hpp"
|
||||
#include "database/Cluster.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Directory.hpp"
|
||||
#include "database/Image.hpp"
|
||||
#include "database/MediaLibrary.hpp"
|
||||
#include "database/Release.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackArtistLink.hpp"
|
||||
#include "database/TrackFeatures.hpp"
|
||||
#include "database/TrackLyrics.hpp"
|
||||
#include "metadata/IParser.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
using namespace db;
|
||||
|
||||
namespace
|
||||
{
|
||||
struct FileInfo
|
||||
{
|
||||
Wt::WDateTime lastWriteTime;
|
||||
std::filesystem::path relativePath;
|
||||
std::size_t fileSize{};
|
||||
};
|
||||
|
||||
Wt::WDateTime retrieveFileGetLastWrite(const std::filesystem::path& file)
|
||||
{
|
||||
Wt::WDateTime res;
|
||||
|
||||
try
|
||||
{
|
||||
res = core::pathUtils::getLastWriteTime(file);
|
||||
}
|
||||
catch (core::LmsException& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot get last write time: " << e.what());
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::optional<FileInfo> retrieveFileInfo(const std::filesystem::path& file, const std::filesystem::path& rootPath)
|
||||
{
|
||||
std::optional<FileInfo> res;
|
||||
res.emplace();
|
||||
|
||||
res->lastWriteTime = retrieveFileGetLastWrite(file);
|
||||
if (!res->lastWriteTime.isValid())
|
||||
{
|
||||
res.reset();
|
||||
return res;
|
||||
}
|
||||
|
||||
{
|
||||
std::error_code ec;
|
||||
res->relativePath = std::filesystem::relative(file, rootPath, ec);
|
||||
if (ec)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot get relative file path for '" << file.string() << "' from '" << rootPath.string() << "': " << ec.message());
|
||||
res.reset();
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
std::error_code ec;
|
||||
res->fileSize = std::filesystem::file_size(file, ec);
|
||||
if (ec)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot get file size for '" << file.string() << "': " << ec.message());
|
||||
res.reset();
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
Directory::pointer getOrCreateDirectory(Session& session, const std::filesystem::path& path, const MediaLibrary::pointer& mediaLibrary)
|
||||
{
|
||||
Directory::pointer directory{ Directory::find(session, path) };
|
||||
if (!directory)
|
||||
{
|
||||
Directory::pointer parentDirectory;
|
||||
if (path != mediaLibrary->getPath())
|
||||
parentDirectory = getOrCreateDirectory(session, path.parent_path(), mediaLibrary);
|
||||
|
||||
directory = session.create<Directory>(path);
|
||||
directory.modify()->setParent(parentDirectory);
|
||||
directory.modify()->setMediaLibrary(mediaLibrary);
|
||||
}
|
||||
// Don't update library if it does not match, will be updated elsewhere
|
||||
|
||||
return directory;
|
||||
}
|
||||
|
||||
db::TrackLyrics::pointer createLyrics(Session& session, const metadata::Lyrics& lyricsInfo)
|
||||
{
|
||||
db::TrackLyrics::pointer lyrics{ session.create<db::TrackLyrics>() };
|
||||
|
||||
lyrics.modify()->setLanguage(!lyricsInfo.language.empty() ? lyricsInfo.language : "xxx");
|
||||
lyrics.modify()->setOffset(lyricsInfo.offset);
|
||||
lyrics.modify()->setDisplayArtist(lyricsInfo.displayArtist);
|
||||
lyrics.modify()->setDisplayTitle(lyricsInfo.displayTitle);
|
||||
if (!lyricsInfo.synchronizedLines.empty())
|
||||
lyrics.modify()->setSynchronizedLines(lyricsInfo.synchronizedLines);
|
||||
else
|
||||
lyrics.modify()->setUnsynchronizedLines(lyricsInfo.unsynchronizedLines);
|
||||
|
||||
return lyrics;
|
||||
}
|
||||
|
||||
Artist::pointer createArtist(Session& session, const metadata::Artist& artistInfo)
|
||||
{
|
||||
Artist::pointer artist{ session.create<Artist>(artistInfo.name) };
|
||||
|
||||
if (artistInfo.mbid)
|
||||
artist.modify()->setMBID(artistInfo.mbid);
|
||||
if (artistInfo.sortName)
|
||||
artist.modify()->setSortName(*artistInfo.sortName);
|
||||
|
||||
return artist;
|
||||
}
|
||||
|
||||
std::string optionalMBIDAsString(const std::optional<core::UUID>& uuid)
|
||||
{
|
||||
return uuid ? std::string{ uuid->getAsString() } : "<no MBID>";
|
||||
}
|
||||
|
||||
void updateArtistIfNeeded(Artist::pointer artist, const metadata::Artist& artistInfo)
|
||||
{
|
||||
// Name may have been updated
|
||||
if (artist->getName() != artistInfo.name)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Artist [" << optionalMBIDAsString(artist->getMBID()) << "], updated name from '" << artist->getName() << "' to '" << artistInfo.name << "'");
|
||||
artist.modify()->setName(artistInfo.name);
|
||||
}
|
||||
|
||||
// Sortname may have been updated
|
||||
// As the sort name is quite often not filled in, we update it only if already set (for now?)
|
||||
if (artistInfo.sortName && *artistInfo.sortName != artist->getSortName())
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Artist [" << optionalMBIDAsString(artist->getMBID()) << "], updated sort name from '" << artist->getSortName() << "' to '" << *artistInfo.sortName << "'");
|
||||
artist.modify()->setSortName(*artistInfo.sortName);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Artist::pointer> getOrCreateArtists(Session& session, const std::vector<metadata::Artist>& artistsInfo, bool allowFallbackOnMBIDEntries)
|
||||
{
|
||||
std::vector<Artist::pointer> artists;
|
||||
|
||||
for (const metadata::Artist& artistInfo : artistsInfo)
|
||||
{
|
||||
Artist::pointer artist;
|
||||
|
||||
// First try to get by MBID
|
||||
if (artistInfo.mbid)
|
||||
{
|
||||
artist = Artist::find(session, *artistInfo.mbid);
|
||||
if (!artist)
|
||||
artist = createArtist(session, artistInfo);
|
||||
else
|
||||
updateArtistIfNeeded(artist, artistInfo);
|
||||
|
||||
artists.emplace_back(std::move(artist));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fall back on artist name (collisions may occur)
|
||||
if (!artistInfo.name.empty())
|
||||
{
|
||||
for (const Artist::pointer& sameNamedArtist : Artist::find(session, artistInfo.name))
|
||||
{
|
||||
// Do not fallback on artist that is correctly tagged
|
||||
if (!allowFallbackOnMBIDEntries && sameNamedArtist->getMBID())
|
||||
continue;
|
||||
|
||||
artist = sameNamedArtist;
|
||||
break;
|
||||
}
|
||||
|
||||
// No Artist found with the same name and without MBID -> creating
|
||||
if (!artist)
|
||||
artist = createArtist(session, artistInfo);
|
||||
else
|
||||
updateArtistIfNeeded(artist, artistInfo);
|
||||
|
||||
artists.emplace_back(std::move(artist));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return artists;
|
||||
}
|
||||
|
||||
ReleaseType::pointer getOrCreateReleaseType(Session& session, std::string_view name)
|
||||
{
|
||||
ReleaseType::pointer releaseType{ ReleaseType::find(session, name) };
|
||||
if (!releaseType)
|
||||
releaseType = session.create<ReleaseType>(name);
|
||||
|
||||
return releaseType;
|
||||
}
|
||||
|
||||
Label::pointer getOrCreateLabel(Session& session, std::string_view name)
|
||||
{
|
||||
Label::pointer label{ Label::find(session, name) };
|
||||
if (!label)
|
||||
label = session.create<Label>(name);
|
||||
|
||||
return label;
|
||||
}
|
||||
|
||||
void updateReleaseIfNeeded(Session& session, Release::pointer release, const metadata::Release& releaseInfo)
|
||||
{
|
||||
if (release->getName() != releaseInfo.name)
|
||||
release.modify()->setName(releaseInfo.name);
|
||||
if (release->getSortName() != releaseInfo.sortName)
|
||||
release.modify()->setSortName(releaseInfo.sortName);
|
||||
if (release->getGroupMBID() != releaseInfo.groupMBID)
|
||||
release.modify()->setGroupMBID(releaseInfo.groupMBID);
|
||||
if (release->getTotalDisc() != releaseInfo.mediumCount)
|
||||
release.modify()->setTotalDisc(releaseInfo.mediumCount);
|
||||
if (release->getArtistDisplayName() != releaseInfo.artistDisplayName)
|
||||
release.modify()->setArtistDisplayName(releaseInfo.artistDisplayName);
|
||||
if (release->isCompilation() != releaseInfo.isCompilation)
|
||||
release.modify()->setCompilation(releaseInfo.isCompilation);
|
||||
if (release->getBarcode() != releaseInfo.barcode)
|
||||
release.modify()->setBarcode(releaseInfo.barcode);
|
||||
if (release->getReleaseTypeNames() != releaseInfo.releaseTypes)
|
||||
{
|
||||
release.modify()->clearReleaseTypes();
|
||||
for (std::string_view releaseType : releaseInfo.releaseTypes)
|
||||
release.modify()->addReleaseType(getOrCreateReleaseType(session, releaseType));
|
||||
}
|
||||
|
||||
if (release->getLabelNames() != releaseInfo.labels)
|
||||
{
|
||||
release.modify()->clearLabels();
|
||||
for (std::string_view label : releaseInfo.labels)
|
||||
release.modify()->addLabel(getOrCreateLabel(session, label));
|
||||
}
|
||||
}
|
||||
|
||||
// Compare release level info
|
||||
bool isReleaseMatching(const Release::pointer& candidateRelease, const metadata::Release& releaseInfo)
|
||||
{
|
||||
// TODO: add more criterias?
|
||||
return candidateRelease->getName() == releaseInfo.name
|
||||
&& candidateRelease->getSortName() == releaseInfo.sortName
|
||||
&& candidateRelease->getTotalDisc() == releaseInfo.mediumCount
|
||||
&& candidateRelease->isCompilation() == releaseInfo.isCompilation
|
||||
&& candidateRelease->getLabelNames() == releaseInfo.labels
|
||||
&& candidateRelease->getBarcode() == releaseInfo.barcode;
|
||||
}
|
||||
|
||||
Release::pointer getOrCreateRelease(Session& session, const metadata::Release& releaseInfo, const Directory::pointer& currentDirectory)
|
||||
{
|
||||
Release::pointer release;
|
||||
|
||||
// First try to get by MBID: fastest, safest
|
||||
if (releaseInfo.mbid)
|
||||
{
|
||||
release = Release::find(session, *releaseInfo.mbid);
|
||||
if (!release)
|
||||
release = session.create<Release>(releaseInfo.name, releaseInfo.mbid);
|
||||
}
|
||||
else if (releaseInfo.name.empty())
|
||||
{
|
||||
// No release name (only mbid) -> nothing to do
|
||||
return release;
|
||||
}
|
||||
|
||||
// Fall back on release name (collisions may occur)
|
||||
// First try using all sibling directories (case for Album/DiscX), only if the disc number is set
|
||||
const DirectoryId parentDirectoryId{ currentDirectory->getParentDirectoryId() };
|
||||
if (!release && releaseInfo.mediumCount && *releaseInfo.mediumCount > 1 && parentDirectoryId.isValid())
|
||||
{
|
||||
Release::FindParameters params;
|
||||
params.setParentDirectory(parentDirectoryId);
|
||||
params.setName(releaseInfo.name);
|
||||
Release::find(session, params, [&](const Release::pointer& candidateRelease) {
|
||||
// Already found a candidate
|
||||
if (release)
|
||||
return;
|
||||
|
||||
// Do not fallback on properly tagged releases
|
||||
if (candidateRelease->getMBID().has_value())
|
||||
return;
|
||||
|
||||
if (!isReleaseMatching(candidateRelease, releaseInfo))
|
||||
return;
|
||||
|
||||
release = candidateRelease;
|
||||
});
|
||||
}
|
||||
|
||||
// Lastly try in the current directory: we do this at last to have
|
||||
// opportunities to merge releases in case of migration / rescan
|
||||
if (!release)
|
||||
{
|
||||
Release::FindParameters params;
|
||||
params.setDirectory(currentDirectory->getId());
|
||||
params.setName(releaseInfo.name);
|
||||
Release::find(session, params, [&](const Release::pointer& candidateRelease) {
|
||||
// Already found a candidate
|
||||
if (release)
|
||||
return;
|
||||
|
||||
// Do not fallback on properly tagged releases
|
||||
if (candidateRelease->getMBID().has_value())
|
||||
return;
|
||||
|
||||
if (!isReleaseMatching(candidateRelease, releaseInfo))
|
||||
return;
|
||||
|
||||
release = candidateRelease;
|
||||
});
|
||||
}
|
||||
|
||||
if (!release)
|
||||
release = session.create<Release>(releaseInfo.name);
|
||||
|
||||
updateReleaseIfNeeded(session, release, releaseInfo);
|
||||
return release;
|
||||
}
|
||||
|
||||
std::vector<Cluster::pointer> getOrCreateClusters(Session& session, const metadata::Track& track)
|
||||
{
|
||||
std::vector<Cluster::pointer> clusters;
|
||||
|
||||
auto getOrCreateClusters{ [&](std::string_view tag, std::span<const std::string> values) {
|
||||
auto clusterType = ClusterType::find(session, tag);
|
||||
if (!clusterType)
|
||||
clusterType = session.create<ClusterType>(tag);
|
||||
|
||||
for (const auto& value : values)
|
||||
{
|
||||
auto cluster{ clusterType->getCluster(value) };
|
||||
if (!cluster)
|
||||
cluster = session.create<Cluster>(clusterType, value);
|
||||
|
||||
clusters.push_back(cluster);
|
||||
}
|
||||
} };
|
||||
|
||||
// TODO: migrate these fields in dedicated tables in DB
|
||||
getOrCreateClusters("GENRE", track.genres);
|
||||
getOrCreateClusters("MOOD", track.moods);
|
||||
getOrCreateClusters("LANGUAGE", track.languages);
|
||||
getOrCreateClusters("GROUPING", track.groupings);
|
||||
|
||||
for (const auto& [tag, values] : track.userExtraTags)
|
||||
getOrCreateClusters(tag, values);
|
||||
|
||||
return clusters;
|
||||
}
|
||||
|
||||
metadata::ParserReadStyle getParserReadStyle()
|
||||
{
|
||||
std::string_view readStyle{ core::Service<core::IConfig>::get()->getString("scanner-parser-read-style", "average") };
|
||||
|
||||
if (readStyle == "fast")
|
||||
return metadata::ParserReadStyle::Fast;
|
||||
if (readStyle == "average")
|
||||
return metadata::ParserReadStyle::Average;
|
||||
if (readStyle == "accurate")
|
||||
return metadata::ParserReadStyle::Accurate;
|
||||
|
||||
throw core::LmsException{ "Invalid value for 'scanner-parser-read-style'" };
|
||||
}
|
||||
|
||||
std::size_t getScanMetaDataThreadCount()
|
||||
{
|
||||
std::size_t threadCount{ core::Service<core::IConfig>::get()->getULong("scanner-metadata-thread-count", 0) };
|
||||
|
||||
if (threadCount == 0)
|
||||
threadCount = std::max<std::size_t>(std::thread::hardware_concurrency() / 2, 1);
|
||||
|
||||
return threadCount;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
|
||||
: ScanStepBase{ initParams }
|
||||
, _metadataParser{ metadata::createParser(metadata::ParserBackend::TagLib, getParserReadStyle()) } // For now, always use TagLib
|
||||
, _fileScanQueue{ *_metadataParser, getScanMetaDataThreadCount(), _abortScan }
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO, "Using " << _fileScanQueue.getThreadCount() << " thread(s) for scanning file metadata");
|
||||
}
|
||||
|
||||
void ScanStepScanFiles::process(ScanContext& context)
|
||||
{
|
||||
{
|
||||
std::vector<std::string> tagsToParse{ _extraTagsToParse };
|
||||
tagsToParse.insert(std::end(tagsToParse), std::cbegin(_settings.extraTags), std::cend(_settings.extraTags));
|
||||
_metadataParser->setUserExtraTags(tagsToParse);
|
||||
_metadataParser->setArtistTagDelimiters(_settings.artistTagDelimiters);
|
||||
_metadataParser->setDefaultTagDelimiters(_settings.defaultTagDelimiters);
|
||||
}
|
||||
|
||||
context.currentStepStats.totalElems = context.stats.totalFileCount;
|
||||
|
||||
for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
|
||||
process(context, mediaLibrary);
|
||||
}
|
||||
|
||||
void ScanStepScanFiles::process(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary)
|
||||
{
|
||||
const std::size_t scanQueueMaxScanRequestCount{ 100 * _fileScanQueue.getThreadCount() };
|
||||
const std::size_t processFileResultsBatchSize{ 5 };
|
||||
|
||||
std::vector<FileScanResult> scanResults;
|
||||
|
||||
core::pathUtils::exploreFilesRecursive(
|
||||
mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) {
|
||||
LMS_SCOPED_TRACE_DETAILED("Scanner", "OnExploreFile");
|
||||
|
||||
if (_abortScan)
|
||||
return false; // stop iterating
|
||||
|
||||
if (ec)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot scan file '" << path.string() << "': " << ec.message());
|
||||
context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() });
|
||||
}
|
||||
else
|
||||
{
|
||||
bool fileMatched{};
|
||||
if (core::pathUtils::hasFileAnyExtension(path, _settings.supportedAudioFileExtensions))
|
||||
{
|
||||
fileMatched = true;
|
||||
if (checkAudioFileNeedScan(context, path, mediaLibrary))
|
||||
_fileScanQueue.pushScanRequest(path, FileScanQueue::ScanRequestType::AudioFile);
|
||||
}
|
||||
else if (core::pathUtils::hasFileAnyExtension(path, _settings.supportedImageFileExtensions))
|
||||
{
|
||||
fileMatched = true;
|
||||
if (checkImageFileNeedScan(context, path))
|
||||
_fileScanQueue.pushScanRequest(path, FileScanQueue::ScanRequestType::ImageFile);
|
||||
}
|
||||
else if (core::pathUtils::hasFileAnyExtension(path, _settings.supportedLyricsFileExtensions))
|
||||
{
|
||||
fileMatched = true;
|
||||
if (checkLyricsFileNeedScan(context, path))
|
||||
_fileScanQueue.pushScanRequest(path, FileScanQueue::ScanRequestType::LyricsFile);
|
||||
}
|
||||
|
||||
if (fileMatched)
|
||||
{
|
||||
context.currentStepStats.processedElems++;
|
||||
_progressCallback(context.currentStepStats);
|
||||
}
|
||||
}
|
||||
|
||||
while (_fileScanQueue.getResultsCount() > (scanQueueMaxScanRequestCount / 2))
|
||||
{
|
||||
_fileScanQueue.popResults(scanResults, processFileResultsBatchSize);
|
||||
processFileScanResults(context, scanResults, mediaLibrary);
|
||||
}
|
||||
|
||||
_fileScanQueue.wait(scanQueueMaxScanRequestCount);
|
||||
|
||||
return true;
|
||||
},
|
||||
&excludeDirFileName);
|
||||
|
||||
_fileScanQueue.wait();
|
||||
|
||||
while (!_abortScan && _fileScanQueue.popResults(scanResults, processFileResultsBatchSize) > 0)
|
||||
processFileScanResults(context, scanResults, mediaLibrary);
|
||||
}
|
||||
|
||||
bool ScanStepScanFiles::checkAudioFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo)
|
||||
{
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const Wt::WDateTime lastWriteTime{ retrieveFileGetLastWrite(file) };
|
||||
// Should rarely fail as we are currently iterating it
|
||||
if (!lastWriteTime.isValid())
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (context.scanOptions.fullScan)
|
||||
return true;
|
||||
|
||||
bool needUpdateLibrary{};
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
|
||||
{
|
||||
auto transaction{ dbSession.createReadTransaction() };
|
||||
|
||||
// Skip file if last write is the same
|
||||
const Track::pointer track{ Track::findByPath(dbSession, file) };
|
||||
if (track
|
||||
&& track->getLastWriteTime() == lastWriteTime
|
||||
&& track->getScanVersion() == _settings.scanVersion)
|
||||
{
|
||||
// this file may have been moved from one library to another, then we just need to update the media library id instead of a full rescan
|
||||
const auto trackMediaLibrary{ track->getMediaLibrary() };
|
||||
if (trackMediaLibrary && trackMediaLibrary->getId() == libraryInfo.id)
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
|
||||
needUpdateLibrary = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (needUpdateLibrary)
|
||||
{
|
||||
auto transaction{ dbSession.createWriteTransaction() };
|
||||
|
||||
Track::pointer track{ Track::findByPath(dbSession, file) };
|
||||
assert(track);
|
||||
track.modify()->setMediaLibrary(db::MediaLibrary::find(dbSession, libraryInfo.id)); // may be null, will be handled in the next scan anyway
|
||||
stats.updates++;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true; // need to scan
|
||||
}
|
||||
|
||||
bool ScanStepScanFiles::checkImageFileNeedScan(ScanContext& context, const std::filesystem::path& file)
|
||||
{
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const Wt::WDateTime lastWriteTime{ retrieveFileGetLastWrite(file) };
|
||||
// Should rarely fail as we are currently iterating it
|
||||
if (!lastWriteTime.isValid())
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!context.scanOptions.fullScan)
|
||||
{
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
auto transaction{ _db.getTLSSession().createReadTransaction() };
|
||||
|
||||
const db::Image::pointer image{ db::Image::find(dbSession, file) };
|
||||
if (image && image->getLastWriteTime() == lastWriteTime)
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true; // need to scan
|
||||
}
|
||||
|
||||
bool ScanStepScanFiles::checkLyricsFileNeedScan(ScanContext& context, const std::filesystem::path& file)
|
||||
{
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const Wt::WDateTime lastWriteTime{ retrieveFileGetLastWrite(file) };
|
||||
// Should rarely fail as we are currently iterating it
|
||||
if (!lastWriteTime.isValid())
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!context.scanOptions.fullScan)
|
||||
{
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
auto transaction{ _db.getTLSSession().createReadTransaction() };
|
||||
|
||||
const db::TrackLyrics::pointer lyrics{ db::TrackLyrics::find(dbSession, file) };
|
||||
if (lyrics && lyrics->getLastWriteTime() == lastWriteTime)
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true; // need to scan
|
||||
}
|
||||
|
||||
void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span<const FileScanResult> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo)
|
||||
{
|
||||
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ProcessScanResults");
|
||||
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
auto transaction{ dbSession.createWriteTransaction() };
|
||||
|
||||
for (const FileScanResult& scanResult : scanResults)
|
||||
{
|
||||
if (_abortScan)
|
||||
return;
|
||||
|
||||
if (const AudioFileScanData * scanData{ std::get_if<AudioFileScanData>(&scanResult.scanData) })
|
||||
{
|
||||
context.stats.scans++;
|
||||
processAudioFileScanData(context, scanResult.path, scanData->get(), libraryInfo);
|
||||
}
|
||||
else if (const ImageFileScanData * scanData{ std::get_if<ImageFileScanData>(&scanResult.scanData) })
|
||||
{
|
||||
context.stats.scans++;
|
||||
processImageFileScanData(context, scanResult.path, scanData->has_value() ? &scanData->value() : nullptr, libraryInfo);
|
||||
}
|
||||
else if (const LyricsFileScanData * scanData{ std::get_if<LyricsFileScanData>(&scanResult.scanData) })
|
||||
{
|
||||
context.stats.scans++;
|
||||
processLyricsFileScanData(context, scanResult.path, scanData->has_value() ? &scanData->value() : nullptr, libraryInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ScanStepScanFiles::processAudioFileScanData(ScanContext& context, const std::filesystem::path& file, const metadata::Track* trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo)
|
||||
{
|
||||
LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessAudioScanData");
|
||||
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const std::optional<FileInfo> fileInfo{ retrieveFileInfo(file, libraryInfo.rootDirectory) };
|
||||
if (!fileInfo)
|
||||
{
|
||||
stats.skips++;
|
||||
return;
|
||||
}
|
||||
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
Track::pointer track{ Track::findByPath(dbSession, file) };
|
||||
|
||||
if (!trackMetadata)
|
||||
{
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
context.stats.errors.emplace_back(file, ScanErrorType::CannotReadAudioFile);
|
||||
return;
|
||||
}
|
||||
|
||||
if (trackMetadata->mbid && (!track || _settings.skipDuplicateMBID))
|
||||
{
|
||||
std::vector<Track::pointer> duplicateTracks{ Track::findByMBID(dbSession, *trackMetadata->mbid) };
|
||||
|
||||
// find for an existing track MBID as the file may have just been moved
|
||||
if (!track && duplicateTracks.size() == 1)
|
||||
{
|
||||
Track::pointer otherTrack{ duplicateTracks.front() };
|
||||
std::error_code ec;
|
||||
if (!std::filesystem::exists(otherTrack->getAbsoluteFilePath(), ec))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Considering track '" << file.string() << "' moved from '" << otherTrack->getAbsoluteFilePath() << "'");
|
||||
track = otherTrack;
|
||||
track.modify()->setAbsoluteFilePath(file);
|
||||
}
|
||||
}
|
||||
|
||||
// Skip duplicate track MBID
|
||||
if (_settings.skipDuplicateMBID)
|
||||
{
|
||||
for (Track::pointer& otherTrack : duplicateTracks)
|
||||
{
|
||||
// Skip ourselves
|
||||
if (track && track->getId() == otherTrack->getId())
|
||||
continue;
|
||||
|
||||
// Skip if duplicate files no longer in media root: as it will be removed later, we will end up with no file
|
||||
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
|
||||
[&](const ScannerSettings::MediaLibraryInfo& libraryInfo) {
|
||||
return core::pathUtils::isPathInRootPath(file, libraryInfo.rootDirectory, &excludeDirFileName);
|
||||
}))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Skipped '" << file.string() << "' (similar MBID in '" << otherTrack->getAbsoluteFilePath().string() << "')");
|
||||
// As this MBID already exists, just remove what we just scanned
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// We estimate this is an audio file if the duration is not null
|
||||
if (trackMetadata->audioProperties.duration == std::chrono::milliseconds::zero())
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Skipped '" << file.string() << "' (duration is 0)");
|
||||
|
||||
// If Track exists here, delete it!
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
stats.errors.emplace_back(file, ScanErrorType::BadDuration);
|
||||
return;
|
||||
}
|
||||
|
||||
// ***** Title
|
||||
std::string title;
|
||||
if (!trackMetadata->title.empty())
|
||||
title = trackMetadata->title;
|
||||
else
|
||||
{
|
||||
// TODO parse file name guess track etc.
|
||||
// For now juste use file name as title
|
||||
title = file.filename().string();
|
||||
}
|
||||
|
||||
// If file already exists, update its data
|
||||
// Otherwise, create it
|
||||
bool added{};
|
||||
if (!track)
|
||||
{
|
||||
track = dbSession.create<Track>();
|
||||
track.modify()->setAbsoluteFilePath(file);
|
||||
added = true;
|
||||
}
|
||||
|
||||
// Track related data
|
||||
assert(track);
|
||||
|
||||
// Audio properties
|
||||
track.modify()->setBitrate(trackMetadata->audioProperties.bitrate);
|
||||
track.modify()->setBitsPerSample(trackMetadata->audioProperties.bitsPerSample);
|
||||
track.modify()->setChannelCount(trackMetadata->audioProperties.channelCount);
|
||||
track.modify()->setDuration(trackMetadata->audioProperties.duration);
|
||||
track.modify()->setSampleRate(trackMetadata->audioProperties.sampleRate);
|
||||
|
||||
track.modify()->setRelativeFilePath(fileInfo->relativePath);
|
||||
track.modify()->setFileSize(fileInfo->fileSize);
|
||||
track.modify()->setLastWriteTime(fileInfo->lastWriteTime);
|
||||
|
||||
MediaLibrary::pointer mediaLibrary{ MediaLibrary::find(dbSession, libraryInfo.id) }; // may be null if settings are updated in // => next scan will correct this
|
||||
track.modify()->setMediaLibrary(mediaLibrary);
|
||||
Directory::pointer directory{ getOrCreateDirectory(dbSession, file.parent_path(), mediaLibrary) };
|
||||
track.modify()->setDirectory(directory);
|
||||
|
||||
track.modify()->clearArtistLinks();
|
||||
// Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files
|
||||
for (const Artist::pointer& artist : getOrCreateArtists(dbSession, trackMetadata->artists, false))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, artist, TrackArtistLinkType::Artist));
|
||||
|
||||
if (trackMetadata->medium && trackMetadata->medium->release)
|
||||
{
|
||||
for (const Artist::pointer& releaseArtist : getOrCreateArtists(dbSession, trackMetadata->medium->release->artists, false))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, releaseArtist, TrackArtistLinkType::ReleaseArtist));
|
||||
}
|
||||
|
||||
// Allow fallbacks on artists with the same name even if they have MBID, since there is no tag to indicate the MBID of these artists
|
||||
// We could ask MusicBrainz to get all the information, but that would heavily slow down the import process
|
||||
for (const Artist::pointer& conductor : getOrCreateArtists(dbSession, trackMetadata->conductorArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, conductor, TrackArtistLinkType::Conductor));
|
||||
|
||||
for (const Artist::pointer& composer : getOrCreateArtists(dbSession, trackMetadata->composerArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, composer, TrackArtistLinkType::Composer));
|
||||
|
||||
for (const Artist::pointer& lyricist : getOrCreateArtists(dbSession, trackMetadata->lyricistArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, lyricist, TrackArtistLinkType::Lyricist));
|
||||
|
||||
for (const Artist::pointer& mixer : getOrCreateArtists(dbSession, trackMetadata->mixerArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, mixer, TrackArtistLinkType::Mixer));
|
||||
|
||||
for (const auto& [role, performers] : trackMetadata->performerArtists)
|
||||
{
|
||||
for (const Artist::pointer& performer : getOrCreateArtists(dbSession, performers, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, performer, TrackArtistLinkType::Performer, role));
|
||||
}
|
||||
|
||||
for (const Artist::pointer& producer : getOrCreateArtists(dbSession, trackMetadata->producerArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, producer, TrackArtistLinkType::Producer));
|
||||
|
||||
for (const Artist::pointer& remixer : getOrCreateArtists(dbSession, trackMetadata->remixerArtists, true))
|
||||
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, remixer, TrackArtistLinkType::Remixer));
|
||||
|
||||
track.modify()->setScanVersion(_settings.scanVersion);
|
||||
if (trackMetadata->medium && trackMetadata->medium->release)
|
||||
track.modify()->setRelease(getOrCreateRelease(dbSession, *trackMetadata->medium->release, directory));
|
||||
else
|
||||
track.modify()->setRelease({});
|
||||
track.modify()->setTotalTrack(trackMetadata->medium ? trackMetadata->medium->trackCount : std::nullopt);
|
||||
track.modify()->setReleaseReplayGain(trackMetadata->medium ? trackMetadata->medium->replayGain : std::nullopt);
|
||||
track.modify()->setDiscSubtitle(trackMetadata->medium ? trackMetadata->medium->name : "");
|
||||
track.modify()->setClusters(getOrCreateClusters(dbSession, *trackMetadata));
|
||||
track.modify()->setName(title);
|
||||
track.modify()->setAddedTime(Wt::WDateTime::currentDateTime());
|
||||
track.modify()->setTrackNumber(trackMetadata->position);
|
||||
track.modify()->setDiscNumber(trackMetadata->medium ? trackMetadata->medium->position : std::nullopt);
|
||||
track.modify()->setDate(trackMetadata->date);
|
||||
track.modify()->setYear(trackMetadata->year);
|
||||
track.modify()->setOriginalDate(trackMetadata->originalDate);
|
||||
track.modify()->setOriginalYear(trackMetadata->originalYear);
|
||||
|
||||
// If a file has an OriginalDate but no date, set it to ease filtering
|
||||
if (!trackMetadata->date.isValid() && trackMetadata->originalDate.isValid())
|
||||
track.modify()->setDate(trackMetadata->originalDate);
|
||||
|
||||
// If a file has an OriginalYear but no Year, set it to ease filtering
|
||||
if (!trackMetadata->year && trackMetadata->originalYear)
|
||||
track.modify()->setYear(trackMetadata->originalYear);
|
||||
|
||||
track.modify()->setRecordingMBID(trackMetadata->recordingMBID);
|
||||
track.modify()->setTrackMBID(trackMetadata->mbid);
|
||||
if (auto trackFeatures{ TrackFeatures::find(dbSession, track->getId()) })
|
||||
trackFeatures.remove(); // TODO: only if MBID changed?
|
||||
track.modify()->setHasCover(trackMetadata->hasCover);
|
||||
track.modify()->setCopyright(trackMetadata->copyright);
|
||||
track.modify()->setCopyrightURL(trackMetadata->copyrightURL);
|
||||
track.modify()->setComment(!trackMetadata->comments.empty() ? trackMetadata->comments.front() : ""); // only take the first one for now
|
||||
track.modify()->setTrackReplayGain(trackMetadata->replayGain);
|
||||
track.modify()->setArtistDisplayName(trackMetadata->artistDisplayName);
|
||||
|
||||
track.modify()->clearEmbeddedLyrics();
|
||||
for (const metadata::Lyrics& lyricsInfo : trackMetadata->lyrics)
|
||||
{
|
||||
db::TrackLyrics::pointer lyrics{ createLyrics(dbSession, lyricsInfo) };
|
||||
track.modify()->addLyrics(lyrics);
|
||||
}
|
||||
|
||||
if (added)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Added audio file '" << file.string() << "'");
|
||||
stats.additions++;
|
||||
}
|
||||
else
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updated audio file '" << file.string() << "'");
|
||||
stats.updates++;
|
||||
}
|
||||
}
|
||||
|
||||
void ScanStepScanFiles::processImageFileScanData(ScanContext& context, const std::filesystem::path& file, const ImageInfo* imageInfo, const ScannerSettings::MediaLibraryInfo& libraryInfo)
|
||||
{
|
||||
LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessImageScanData");
|
||||
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const std::optional<FileInfo> fileInfo{ retrieveFileInfo(file, libraryInfo.rootDirectory) };
|
||||
if (!fileInfo)
|
||||
{
|
||||
stats.skips++;
|
||||
return;
|
||||
}
|
||||
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
db::Image::pointer image{ db::Image::find(dbSession, file) };
|
||||
|
||||
if (!imageInfo)
|
||||
{
|
||||
if (image)
|
||||
{
|
||||
image.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
context.stats.errors.emplace_back(file, ScanErrorType::CannotReadImageFile);
|
||||
return;
|
||||
}
|
||||
|
||||
const bool added{ !image };
|
||||
if (!image)
|
||||
image = dbSession.create<db::Image>(file);
|
||||
|
||||
image.modify()->setLastWriteTime(fileInfo->lastWriteTime);
|
||||
image.modify()->setFileSize(fileInfo->fileSize);
|
||||
image.modify()->setHeight(imageInfo->height);
|
||||
image.modify()->setWidth(imageInfo->width);
|
||||
MediaLibrary::pointer mediaLibrary{ MediaLibrary::find(dbSession, libraryInfo.id) }; // may be null if settings are updated in // => next scan will correct this
|
||||
image.modify()->setDirectory(getOrCreateDirectory(dbSession, file.parent_path(), mediaLibrary));
|
||||
|
||||
if (added)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Added image '" << file.string() << "'");
|
||||
stats.additions++;
|
||||
}
|
||||
else
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updated image '" << file.string() << "'");
|
||||
stats.updates++;
|
||||
}
|
||||
}
|
||||
|
||||
void ScanStepScanFiles::processLyricsFileScanData(ScanContext& context, const std::filesystem::path& file, const metadata::Lyrics* lyricsInfo, const ScannerSettings::MediaLibraryInfo& libraryInfo)
|
||||
{
|
||||
LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessImageScanData");
|
||||
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const std::optional<FileInfo> fileInfo{ retrieveFileInfo(file, libraryInfo.rootDirectory) };
|
||||
if (!fileInfo)
|
||||
{
|
||||
stats.skips++;
|
||||
return;
|
||||
}
|
||||
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
db::TrackLyrics::pointer trackLyrics{ db::TrackLyrics::find(dbSession, file) };
|
||||
|
||||
if (!lyricsInfo)
|
||||
{
|
||||
if (trackLyrics)
|
||||
{
|
||||
trackLyrics.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
context.stats.errors.emplace_back(file, ScanErrorType::CannotReadLyricsFile);
|
||||
return;
|
||||
}
|
||||
|
||||
const bool added{ !trackLyrics };
|
||||
if (!trackLyrics)
|
||||
{
|
||||
trackLyrics = dbSession.create<db::TrackLyrics>();
|
||||
trackLyrics.modify()->setAbsoluteFilePath(file);
|
||||
}
|
||||
|
||||
trackLyrics.modify()->setLastWriteTime(fileInfo->lastWriteTime);
|
||||
trackLyrics.modify()->setFileSize(fileInfo->fileSize);
|
||||
trackLyrics.modify()->setLanguage(!lyricsInfo->language.empty() ? lyricsInfo->language : "xxx");
|
||||
trackLyrics.modify()->setOffset(lyricsInfo->offset);
|
||||
trackLyrics.modify()->setDisplayTitle(lyricsInfo->displayTitle);
|
||||
trackLyrics.modify()->setDisplayArtist(lyricsInfo->displayArtist);
|
||||
if (!lyricsInfo->synchronizedLines.empty())
|
||||
trackLyrics.modify()->setSynchronizedLines(lyricsInfo->synchronizedLines);
|
||||
else
|
||||
trackLyrics.modify()->setUnsynchronizedLines(lyricsInfo->unsynchronizedLines);
|
||||
|
||||
MediaLibrary::pointer mediaLibrary{ MediaLibrary::find(dbSession, libraryInfo.id) }; // may be null if settings are updated in // => next scan will correct this
|
||||
trackLyrics.modify()->setDirectory(getOrCreateDirectory(dbSession, file.parent_path(), mediaLibrary));
|
||||
|
||||
if (added)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Added external lyrics '" << file.string() << "'");
|
||||
stats.additions++;
|
||||
}
|
||||
else
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updated external lyrics '" << file.string() << "'");
|
||||
stats.updates++;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace lms::scanner
|
||||
@@ -21,26 +21,32 @@
|
||||
|
||||
#include <ctime>
|
||||
|
||||
#include <Wt/WDate.h>
|
||||
|
||||
#include "core/IConfig.hpp"
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/ITraceLogger.hpp"
|
||||
#include "database/MediaLibrary.hpp"
|
||||
#include "database/ScanSettings.hpp"
|
||||
#include "database/TrackFeatures.hpp"
|
||||
#include "image/Image.hpp"
|
||||
|
||||
#include "ScanStepAssociateArtistImages.hpp"
|
||||
#include "ScanStepAssociateExternalLyrics.hpp"
|
||||
#include "ScanStepAssociateReleaseImages.hpp"
|
||||
#include "ScanStepCheckForDuplicatedFiles.hpp"
|
||||
#include "ScanStepCheckForRemovedFiles.hpp"
|
||||
#include "ScanStepCompact.hpp"
|
||||
#include "ScanStepComputeClusterStats.hpp"
|
||||
#include "ScanStepDiscoverFiles.hpp"
|
||||
#include "ScanStepOptimize.hpp"
|
||||
#include "ScanStepRemoveOrphanedDbEntries.hpp"
|
||||
#include "ScanStepScanFiles.hpp"
|
||||
#include "ScanStepUpdateLibraryFields.hpp"
|
||||
#include "scanners/AudioFileScanner.hpp"
|
||||
#include "scanners/ImageFileScanner.hpp"
|
||||
#include "scanners/LyricsFileScanner.hpp"
|
||||
#include "scanners/PlayListFileScanner.hpp"
|
||||
|
||||
#include "steps/ScanStepAssociateArtistImages.hpp"
|
||||
#include "steps/ScanStepAssociateExternalLyrics.hpp"
|
||||
#include "steps/ScanStepAssociatePlayListTracks.hpp"
|
||||
#include "steps/ScanStepAssociateReleaseImages.hpp"
|
||||
#include "steps/ScanStepCheckForDuplicatedFiles.hpp"
|
||||
#include "steps/ScanStepCheckForRemovedFiles.hpp"
|
||||
#include "steps/ScanStepCompact.hpp"
|
||||
#include "steps/ScanStepComputeClusterStats.hpp"
|
||||
#include "steps/ScanStepDiscoverFiles.hpp"
|
||||
#include "steps/ScanStepOptimize.hpp"
|
||||
#include "steps/ScanStepRemoveOrphanedDbEntries.hpp"
|
||||
#include "steps/ScanStepScanFiles.hpp"
|
||||
#include "steps/ScanStepUpdateLibraryFields.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
@@ -270,7 +276,7 @@ namespace lms::scanner
|
||||
|
||||
refreshScanSettings();
|
||||
|
||||
IScanStep::ScanContext scanContext;
|
||||
ScanContext scanContext;
|
||||
scanContext.scanOptions = scanOptions;
|
||||
ScanStats& stats{ scanContext.stats };
|
||||
stats.startTime = Wt::WDateTime::currentDateTime();
|
||||
@@ -341,27 +347,38 @@ namespace lms::scanner
|
||||
notifyInProgressIfNeeded(stats);
|
||||
} };
|
||||
|
||||
_fileScanners.clear();
|
||||
_fileScanners.emplace_back(std::make_unique<AudioFileScanner>(_db, _settings));
|
||||
_fileScanners.emplace_back(std::make_unique<ImageFileScanner>(_db));
|
||||
_fileScanners.emplace_back(std::make_unique<LyricsFileScanner>(_db));
|
||||
_fileScanners.emplace_back(std::make_unique<PlayListFileScanner>(_db));
|
||||
|
||||
std::vector<IFileScanner*> fileScanners;
|
||||
std::transform(std::cbegin(_fileScanners), std::cend(_fileScanners), std::back_inserter(fileScanners), [](const std::unique_ptr<IFileScanner>& scanner) { return scanner.get(); });
|
||||
|
||||
ScanStepBase::InitParams params{
|
||||
_settings,
|
||||
cbFunc,
|
||||
_abortScan,
|
||||
_db
|
||||
.settings = _settings,
|
||||
.progressCallback = cbFunc,
|
||||
.abortScan = _abortScan,
|
||||
.db = _db,
|
||||
.fileScanners = fileScanners,
|
||||
};
|
||||
|
||||
// Order is important: steps are sequential
|
||||
_scanSteps.clear();
|
||||
_scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepCheckForRemovedFiles>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepUpdateLibraryFields>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepAssociateArtistImages>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepAssociateReleaseImages>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepAssociateExternalLyrics>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanedDbEntries>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepCompact>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepOptimize>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepCheckForDuplicatedFiles>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepDiscoverFiles>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepScanFiles>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepCheckForRemovedFiles>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepAssociatePlayListTracks>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepUpdateLibraryFields>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepAssociateArtistImages>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepAssociateReleaseImages>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepAssociateExternalLyrics>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepRemoveOrphanedDbEntries>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepCompact>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepOptimize>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepComputeClusterStats>(params));
|
||||
_scanSteps.emplace_back(std::make_unique<ScanStepCheckForDuplicatedFiles>(params));
|
||||
}
|
||||
|
||||
ScannerSettings ScannerService::readSettings()
|
||||
@@ -378,29 +395,8 @@ namespace lms::scanner
|
||||
newSettings.startTime = scanSettings->getUpdateStartTime();
|
||||
newSettings.updatePeriod = scanSettings->getUpdatePeriod();
|
||||
|
||||
{
|
||||
const auto audioFileExtensions{ scanSettings->getAudioFileExtensions() };
|
||||
newSettings.supportedAudioFileExtensions.reserve(audioFileExtensions.size());
|
||||
std::transform(std::cbegin(audioFileExtensions), std::cend(audioFileExtensions), std::back_inserter(newSettings.supportedAudioFileExtensions),
|
||||
[](const std::filesystem::path& extension) { return std::filesystem::path{ core::stringUtils::stringToLower(extension.string()) }; });
|
||||
}
|
||||
|
||||
{
|
||||
const auto imageFileExtensions{ image::getSupportedFileExtensions() };
|
||||
newSettings.supportedImageFileExtensions.reserve(imageFileExtensions.size());
|
||||
std::transform(std::cbegin(imageFileExtensions), std::cend(imageFileExtensions), std::back_inserter(newSettings.supportedImageFileExtensions),
|
||||
[](const std::filesystem::path& extension) { return std::filesystem::path{ core::stringUtils::stringToLower(extension.string()) }; });
|
||||
}
|
||||
|
||||
{
|
||||
const auto lyricsFileExtensions{ metadata::getSupportedLyricsFileExtensions() };
|
||||
newSettings.supportedLyricsFileExtensions.reserve(lyricsFileExtensions.size());
|
||||
std::transform(std::cbegin(lyricsFileExtensions), std::cend(lyricsFileExtensions), std::back_inserter(newSettings.supportedLyricsFileExtensions),
|
||||
[](const std::filesystem::path& extension) { return std::filesystem::path{ core::stringUtils::stringToLower(extension.string()) }; });
|
||||
}
|
||||
|
||||
MediaLibrary::find(_db.getTLSSession(), [&](const MediaLibrary::pointer& mediaLibrary) {
|
||||
newSettings.mediaLibraries.push_back(ScannerSettings::MediaLibraryInfo{ mediaLibrary->getId(), mediaLibrary->getPath().lexically_normal() });
|
||||
newSettings.mediaLibraries.push_back(MediaLibraryInfo{ .id = mediaLibrary->getId(), .rootDirectory = mediaLibrary->getPath().lexically_normal() });
|
||||
});
|
||||
|
||||
{
|
||||
|
||||
@@ -29,14 +29,16 @@
|
||||
#include <Wt/WSignal.h>
|
||||
#include <boost/asio/system_timer.hpp>
|
||||
|
||||
#include "IScanStep.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "services/scanner/IScannerService.hpp"
|
||||
#include "steps/IScanStep.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class IFileScanner;
|
||||
|
||||
class ScannerService : public IScannerService
|
||||
{
|
||||
public:
|
||||
@@ -73,6 +75,7 @@ namespace lms::scanner
|
||||
void notifyInProgressIfNeeded(const ScanStepStats& stats);
|
||||
void notifyInProgress(const ScanStepStats& stats);
|
||||
|
||||
std::vector<std::unique_ptr<IFileScanner>> _fileScanners;
|
||||
std::vector<std::unique_ptr<IScanStep>> _scanSteps;
|
||||
|
||||
std::mutex _controlMutex;
|
||||
|
||||
@@ -25,31 +25,23 @@
|
||||
|
||||
#include <Wt/WDateTime.h>
|
||||
|
||||
#include "database/MediaLibraryId.hpp"
|
||||
#include "database/ScanSettings.hpp"
|
||||
|
||||
#include "MediaLibraryInfo.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
static inline const std::filesystem::path excludeDirFileName{ ".lmsignore" };
|
||||
|
||||
struct ScannerSettings
|
||||
{
|
||||
std::size_t scanVersion{};
|
||||
Wt::WTime startTime;
|
||||
db::ScanSettings::UpdatePeriod updatePeriod{ db::ScanSettings::UpdatePeriod::Never };
|
||||
std::vector<std::filesystem::path> supportedAudioFileExtensions;
|
||||
std::vector<std::filesystem::path> supportedImageFileExtensions;
|
||||
std::vector<std::filesystem::path> supportedLyricsFileExtensions;
|
||||
bool skipDuplicateMBID{};
|
||||
std::vector<std::string> extraTags;
|
||||
std::vector<std::string> artistTagDelimiters;
|
||||
std::vector<std::string> defaultTagDelimiters;
|
||||
|
||||
struct MediaLibraryInfo
|
||||
{
|
||||
db::MediaLibraryId id;
|
||||
std::filesystem::path rootDirectory;
|
||||
|
||||
auto operator<=>(const MediaLibraryInfo& other) const = default;
|
||||
};
|
||||
std::vector<MediaLibraryInfo> mediaLibraries;
|
||||
|
||||
bool operator==(const ScannerSettings& rhs) const = default;
|
||||
|
||||
@@ -0,0 +1,668 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "AudioFileScanner.hpp"
|
||||
|
||||
#include "core/IConfig.hpp"
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/ITraceLogger.hpp"
|
||||
#include "core/Path.hpp"
|
||||
#include "core/Service.hpp"
|
||||
#include "database/Artist.hpp"
|
||||
#include "database/Cluster.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Directory.hpp"
|
||||
#include "database/MediaLibrary.hpp"
|
||||
#include "database/Release.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackArtistLink.hpp"
|
||||
#include "database/TrackFeatures.hpp"
|
||||
#include "database/TrackLyrics.hpp"
|
||||
#include "metadata/Exception.hpp"
|
||||
#include "metadata/IParser.hpp"
|
||||
|
||||
#include "IFileScanOperation.hpp"
|
||||
#include "ScanContext.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
#include "Utils.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
namespace
|
||||
{
|
||||
db::Artist::pointer createArtist(db::Session& session, const metadata::Artist& artistInfo)
|
||||
{
|
||||
db::Artist::pointer artist{ session.create<db::Artist>(artistInfo.name) };
|
||||
|
||||
if (artistInfo.mbid)
|
||||
artist.modify()->setMBID(artistInfo.mbid);
|
||||
if (artistInfo.sortName)
|
||||
artist.modify()->setSortName(*artistInfo.sortName);
|
||||
|
||||
return artist;
|
||||
}
|
||||
|
||||
std::string optionalMBIDAsString(const std::optional<core::UUID>& uuid)
|
||||
{
|
||||
return uuid ? std::string{ uuid->getAsString() } : "<no MBID>";
|
||||
}
|
||||
|
||||
void updateArtistIfNeeded(db::Artist::pointer artist, const metadata::Artist& artistInfo)
|
||||
{
|
||||
// Name may have been updated
|
||||
if (artist->getName() != artistInfo.name)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Artist [" << optionalMBIDAsString(artist->getMBID()) << "], updated name from '" << artist->getName() << "' to '" << artistInfo.name << "'");
|
||||
artist.modify()->setName(artistInfo.name);
|
||||
}
|
||||
|
||||
// Sortname may have been updated
|
||||
// As the sort name is quite often not filled in, we update it only if already set (for now?)
|
||||
if (artistInfo.sortName && *artistInfo.sortName != artist->getSortName())
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Artist [" << optionalMBIDAsString(artist->getMBID()) << "], updated sort name from '" << artist->getSortName() << "' to '" << *artistInfo.sortName << "'");
|
||||
artist.modify()->setSortName(*artistInfo.sortName);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<db::Artist::pointer> getOrCreateArtists(db::Session& session, const std::vector<metadata::Artist>& artistsInfo, bool allowFallbackOnMBIDEntries)
|
||||
{
|
||||
std::vector<db::Artist::pointer> artists;
|
||||
|
||||
for (const metadata::Artist& artistInfo : artistsInfo)
|
||||
{
|
||||
db::Artist::pointer artist;
|
||||
|
||||
// First try to get by MBID
|
||||
if (artistInfo.mbid)
|
||||
{
|
||||
artist = db::Artist::find(session, *artistInfo.mbid);
|
||||
if (!artist)
|
||||
artist = createArtist(session, artistInfo);
|
||||
else
|
||||
updateArtistIfNeeded(artist, artistInfo);
|
||||
|
||||
artists.emplace_back(std::move(artist));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fall back on artist name (collisions may occur)
|
||||
if (!artistInfo.name.empty())
|
||||
{
|
||||
for (const db::Artist::pointer& sameNamedArtist : db::Artist::find(session, artistInfo.name))
|
||||
{
|
||||
// Do not fallback on artist that is correctly tagged
|
||||
if (!allowFallbackOnMBIDEntries && sameNamedArtist->getMBID())
|
||||
continue;
|
||||
|
||||
artist = sameNamedArtist;
|
||||
break;
|
||||
}
|
||||
|
||||
// No Artist found with the same name and without MBID -> creating
|
||||
if (!artist)
|
||||
artist = createArtist(session, artistInfo);
|
||||
else
|
||||
updateArtistIfNeeded(artist, artistInfo);
|
||||
|
||||
artists.emplace_back(std::move(artist));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return artists;
|
||||
}
|
||||
|
||||
db::ReleaseType::pointer getOrCreateReleaseType(db::Session& session, std::string_view name)
|
||||
{
|
||||
db::ReleaseType::pointer releaseType{ db::ReleaseType::find(session, name) };
|
||||
if (!releaseType)
|
||||
releaseType = session.create<db::ReleaseType>(name);
|
||||
|
||||
return releaseType;
|
||||
}
|
||||
|
||||
db::Label::pointer getOrCreateLabel(db::Session& session, std::string_view name)
|
||||
{
|
||||
db::Label::pointer label{ db::Label::find(session, name) };
|
||||
if (!label)
|
||||
label = session.create<db::Label>(name);
|
||||
|
||||
return label;
|
||||
}
|
||||
|
||||
void updateReleaseIfNeeded(db::Session& session, db::Release::pointer release, const metadata::Release& releaseInfo)
|
||||
{
|
||||
if (release->getName() != releaseInfo.name)
|
||||
release.modify()->setName(releaseInfo.name);
|
||||
if (release->getSortName() != releaseInfo.sortName)
|
||||
release.modify()->setSortName(releaseInfo.sortName);
|
||||
if (release->getGroupMBID() != releaseInfo.groupMBID)
|
||||
release.modify()->setGroupMBID(releaseInfo.groupMBID);
|
||||
if (release->getTotalDisc() != releaseInfo.mediumCount)
|
||||
release.modify()->setTotalDisc(releaseInfo.mediumCount);
|
||||
if (release->getArtistDisplayName() != releaseInfo.artistDisplayName)
|
||||
release.modify()->setArtistDisplayName(releaseInfo.artistDisplayName);
|
||||
if (release->isCompilation() != releaseInfo.isCompilation)
|
||||
release.modify()->setCompilation(releaseInfo.isCompilation);
|
||||
if (release->getBarcode() != releaseInfo.barcode)
|
||||
release.modify()->setBarcode(releaseInfo.barcode);
|
||||
if (release->getReleaseTypeNames() != releaseInfo.releaseTypes)
|
||||
{
|
||||
release.modify()->clearReleaseTypes();
|
||||
for (std::string_view releaseType : releaseInfo.releaseTypes)
|
||||
release.modify()->addReleaseType(getOrCreateReleaseType(session, releaseType));
|
||||
}
|
||||
|
||||
if (release->getLabelNames() != releaseInfo.labels)
|
||||
{
|
||||
release.modify()->clearLabels();
|
||||
for (std::string_view label : releaseInfo.labels)
|
||||
release.modify()->addLabel(getOrCreateLabel(session, label));
|
||||
}
|
||||
}
|
||||
|
||||
// Compare release level info
|
||||
bool isReleaseMatching(const db::Release::pointer& candidateRelease, const metadata::Release& releaseInfo)
|
||||
{
|
||||
// TODO: add more criterias?
|
||||
return candidateRelease->getName() == releaseInfo.name
|
||||
&& candidateRelease->getSortName() == releaseInfo.sortName
|
||||
&& candidateRelease->getTotalDisc() == releaseInfo.mediumCount
|
||||
&& candidateRelease->isCompilation() == releaseInfo.isCompilation
|
||||
&& candidateRelease->getLabelNames() == releaseInfo.labels
|
||||
&& candidateRelease->getBarcode() == releaseInfo.barcode;
|
||||
}
|
||||
|
||||
db::Release::pointer getOrCreateRelease(db::Session& session, const metadata::Release& releaseInfo, const db::Directory::pointer& currentDirectory)
|
||||
{
|
||||
db::Release::pointer release;
|
||||
|
||||
// First try to get by MBID: fastest, safest
|
||||
if (releaseInfo.mbid)
|
||||
{
|
||||
release = db::Release::find(session, *releaseInfo.mbid);
|
||||
if (!release)
|
||||
release = session.create<db::Release>(releaseInfo.name, releaseInfo.mbid);
|
||||
}
|
||||
else if (releaseInfo.name.empty())
|
||||
{
|
||||
// No release name (only mbid) -> nothing to do
|
||||
return release;
|
||||
}
|
||||
|
||||
// Fall back on release name (collisions may occur)
|
||||
// First try using all sibling directories (case for Album/DiscX), only if the disc number is set
|
||||
const db::DirectoryId parentDirectoryId{ currentDirectory->getParentDirectoryId() };
|
||||
if (!release && releaseInfo.mediumCount && *releaseInfo.mediumCount > 1 && parentDirectoryId.isValid())
|
||||
{
|
||||
db::Release::FindParameters params;
|
||||
params.setParentDirectory(parentDirectoryId);
|
||||
params.setName(releaseInfo.name);
|
||||
db::Release::find(session, params, [&](const db::Release::pointer& candidateRelease) {
|
||||
// Already found a candidate
|
||||
if (release)
|
||||
return;
|
||||
|
||||
// Do not fallback on properly tagged releases
|
||||
if (candidateRelease->getMBID().has_value())
|
||||
return;
|
||||
|
||||
if (!isReleaseMatching(candidateRelease, releaseInfo))
|
||||
return;
|
||||
|
||||
release = candidateRelease;
|
||||
});
|
||||
}
|
||||
|
||||
// Lastly try in the current directory: we do this at last to have
|
||||
// opportunities to merge releases in case of migration / rescan
|
||||
if (!release)
|
||||
{
|
||||
db::Release::FindParameters params;
|
||||
params.setDirectory(currentDirectory->getId());
|
||||
params.setName(releaseInfo.name);
|
||||
db::Release::find(session, params, [&](const db::Release::pointer& candidateRelease) {
|
||||
// Already found a candidate
|
||||
if (release)
|
||||
return;
|
||||
|
||||
// Do not fallback on properly tagged releases
|
||||
if (candidateRelease->getMBID().has_value())
|
||||
return;
|
||||
|
||||
if (!isReleaseMatching(candidateRelease, releaseInfo))
|
||||
return;
|
||||
|
||||
release = candidateRelease;
|
||||
});
|
||||
}
|
||||
|
||||
if (!release)
|
||||
release = session.create<db::Release>(releaseInfo.name);
|
||||
|
||||
updateReleaseIfNeeded(session, release, releaseInfo);
|
||||
return release;
|
||||
}
|
||||
|
||||
std::vector<db::Cluster::pointer> getOrCreateClusters(db::Session& session, const metadata::Track& track)
|
||||
{
|
||||
std::vector<db::Cluster::pointer> clusters;
|
||||
|
||||
auto getOrCreateClusters{ [&](std::string_view tag, std::span<const std::string> values) {
|
||||
auto clusterType = db::ClusterType::find(session, tag);
|
||||
if (!clusterType)
|
||||
clusterType = session.create<db::ClusterType>(tag);
|
||||
|
||||
for (const auto& value : values)
|
||||
{
|
||||
auto cluster{ clusterType->getCluster(value) };
|
||||
if (!cluster)
|
||||
cluster = session.create<db::Cluster>(clusterType, value);
|
||||
|
||||
clusters.push_back(cluster);
|
||||
}
|
||||
} };
|
||||
|
||||
// TODO: migrate these fields in dedicated tables in DB
|
||||
getOrCreateClusters("GENRE", track.genres);
|
||||
getOrCreateClusters("MOOD", track.moods);
|
||||
getOrCreateClusters("LANGUAGE", track.languages);
|
||||
getOrCreateClusters("GROUPING", track.groupings);
|
||||
|
||||
for (const auto& [tag, values] : track.userExtraTags)
|
||||
getOrCreateClusters(tag, values);
|
||||
|
||||
return clusters;
|
||||
}
|
||||
|
||||
db::TrackLyrics::pointer createLyrics(db::Session& session, const metadata::Lyrics& lyricsInfo)
|
||||
{
|
||||
db::TrackLyrics::pointer lyrics{ session.create<db::TrackLyrics>() };
|
||||
|
||||
lyrics.modify()->setLanguage(!lyricsInfo.language.empty() ? lyricsInfo.language : "xxx");
|
||||
lyrics.modify()->setOffset(lyricsInfo.offset);
|
||||
lyrics.modify()->setDisplayArtist(lyricsInfo.displayArtist);
|
||||
lyrics.modify()->setDisplayTitle(lyricsInfo.displayTitle);
|
||||
if (!lyricsInfo.synchronizedLines.empty())
|
||||
lyrics.modify()->setSynchronizedLines(lyricsInfo.synchronizedLines);
|
||||
else
|
||||
lyrics.modify()->setUnsynchronizedLines(lyricsInfo.unsynchronizedLines);
|
||||
|
||||
return lyrics;
|
||||
}
|
||||
|
||||
class AudioFileScanOperation : public IFileScanOperation
|
||||
{
|
||||
public:
|
||||
AudioFileScanOperation(const FileToScan& fileToScan, db::Db& db, metadata::IParser& parser, const ScannerSettings& settings)
|
||||
: _file{ fileToScan.file }
|
||||
, _mediaLibrary{ fileToScan.mediaLibrary }
|
||||
, _db{ db }
|
||||
, _parser{ parser }
|
||||
, _settings{ settings }
|
||||
{
|
||||
}
|
||||
~AudioFileScanOperation() override = default;
|
||||
AudioFileScanOperation(const AudioFileScanOperation&) = delete;
|
||||
AudioFileScanOperation& operator=(const AudioFileScanOperation&) = delete;
|
||||
|
||||
private:
|
||||
core::LiteralString getName() const override { return "ScanAudioFile"; }
|
||||
void scan() override;
|
||||
void processResult(ScanContext& context) override;
|
||||
|
||||
const std::filesystem::path _file;
|
||||
const MediaLibraryInfo _mediaLibrary;
|
||||
db::Db& _db;
|
||||
metadata::IParser& _parser;
|
||||
const ScannerSettings& _settings;
|
||||
std::unique_ptr<metadata::Track> _parsedTrack;
|
||||
};
|
||||
|
||||
void AudioFileScanOperation::scan()
|
||||
{
|
||||
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanAudioFile");
|
||||
std::unique_ptr<metadata::Track> track;
|
||||
|
||||
try
|
||||
{
|
||||
_parsedTrack = _parser.parse(_file);
|
||||
}
|
||||
catch (const metadata::Exception& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO, "Failed to parse audio file '" << _file.string() << "'");
|
||||
}
|
||||
}
|
||||
|
||||
void AudioFileScanOperation::processResult(ScanContext& context)
|
||||
{
|
||||
LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessAudioScanData");
|
||||
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
|
||||
if (!fileInfo)
|
||||
{
|
||||
stats.skips++;
|
||||
return;
|
||||
}
|
||||
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
db::Track::pointer track{ db::Track::findByPath(dbSession, _file) };
|
||||
|
||||
if (!_parsedTrack)
|
||||
{
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadAudioFile);
|
||||
return;
|
||||
}
|
||||
|
||||
if (_parsedTrack->mbid && (!track || _settings.skipDuplicateMBID))
|
||||
{
|
||||
std::vector<db::Track::pointer> duplicateTracks{ db::Track::findByMBID(dbSession, *_parsedTrack->mbid) };
|
||||
|
||||
// find for an existing track MBID as the file may have just been moved
|
||||
if (!track && duplicateTracks.size() == 1)
|
||||
{
|
||||
db::Track::pointer otherTrack{ duplicateTracks.front() };
|
||||
std::error_code ec;
|
||||
if (!std::filesystem::exists(otherTrack->getAbsoluteFilePath(), ec))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Considering track '" << _file.string() << "' moved from '" << otherTrack->getAbsoluteFilePath() << "'");
|
||||
track = otherTrack;
|
||||
track.modify()->setAbsoluteFilePath(_file);
|
||||
}
|
||||
}
|
||||
|
||||
// Skip duplicate track MBID
|
||||
if (_settings.skipDuplicateMBID)
|
||||
{
|
||||
for (db::Track::pointer& otherTrack : duplicateTracks)
|
||||
{
|
||||
// Skip ourselves
|
||||
if (track && track->getId() == otherTrack->getId())
|
||||
continue;
|
||||
|
||||
// Skip if duplicate files no longer in media root: as it will be removed later, we will end up with no file
|
||||
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
|
||||
[&](const MediaLibraryInfo& libraryInfo) {
|
||||
return core::pathUtils::isPathInRootPath(_file, libraryInfo.rootDirectory, &excludeDirFileName);
|
||||
}))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Skipped '" << _file.string() << "' (similar MBID in '" << otherTrack->getAbsoluteFilePath().string() << "')");
|
||||
// As this MBID already exists, just remove what we just scanned
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// We estimate this is an audio file if the duration is not null
|
||||
if (_parsedTrack->audioProperties.duration == std::chrono::milliseconds::zero())
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Skipped '" << _file.string() << "' (duration is 0)");
|
||||
|
||||
// If Track exists here, delete it!
|
||||
if (track)
|
||||
{
|
||||
track.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
stats.errors.emplace_back(_file, ScanErrorType::BadDuration);
|
||||
return;
|
||||
}
|
||||
|
||||
// ***** Title
|
||||
std::string title;
|
||||
if (!_parsedTrack->title.empty())
|
||||
title = _parsedTrack->title;
|
||||
else
|
||||
{
|
||||
// TODO parse file name guess track etc.
|
||||
// For now juste use file name as title
|
||||
title = _file.filename().string();
|
||||
}
|
||||
|
||||
// If file already exists, update its data
|
||||
// Otherwise, create it
|
||||
bool added{};
|
||||
if (!track)
|
||||
{
|
||||
track = dbSession.create<db::Track>();
|
||||
track.modify()->setAbsoluteFilePath(_file);
|
||||
added = true;
|
||||
}
|
||||
|
||||
// Track related data
|
||||
assert(track);
|
||||
|
||||
// Audio properties
|
||||
track.modify()->setBitrate(_parsedTrack->audioProperties.bitrate);
|
||||
track.modify()->setBitsPerSample(_parsedTrack->audioProperties.bitsPerSample);
|
||||
track.modify()->setChannelCount(_parsedTrack->audioProperties.channelCount);
|
||||
track.modify()->setDuration(_parsedTrack->audioProperties.duration);
|
||||
track.modify()->setSampleRate(_parsedTrack->audioProperties.sampleRate);
|
||||
|
||||
track.modify()->setRelativeFilePath(fileInfo->relativePath);
|
||||
track.modify()->setFileSize(fileInfo->fileSize);
|
||||
track.modify()->setLastWriteTime(fileInfo->lastWriteTime);
|
||||
|
||||
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
|
||||
track.modify()->setMediaLibrary(mediaLibrary);
|
||||
db::Directory::pointer directory{ utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary) };
|
||||
track.modify()->setDirectory(directory);
|
||||
|
||||
track.modify()->clearArtistLinks();
|
||||
// Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files
|
||||
for (const db::Artist::pointer& artist : getOrCreateArtists(dbSession, _parsedTrack->artists, false))
|
||||
track.modify()->addArtistLink(db::TrackArtistLink::create(dbSession, track, artist, db::TrackArtistLinkType::Artist));
|
||||
|
||||
if (_parsedTrack->medium && _parsedTrack->medium->release)
|
||||
{
|
||||
for (const db::Artist::pointer& releaseArtist : getOrCreateArtists(dbSession, _parsedTrack->medium->release->artists, false))
|
||||
track.modify()->addArtistLink(db::TrackArtistLink::create(dbSession, track, releaseArtist, db::TrackArtistLinkType::ReleaseArtist));
|
||||
}
|
||||
|
||||
// Allow fallbacks on artists with the same name even if they have MBID, since there is no tag to indicate the MBID of these artists
|
||||
// We could ask MusicBrainz to get all the information, but that would heavily slow down the import process
|
||||
for (const db::Artist::pointer& conductor : getOrCreateArtists(dbSession, _parsedTrack->conductorArtists, true))
|
||||
track.modify()->addArtistLink(db::TrackArtistLink::create(dbSession, track, conductor, db::TrackArtistLinkType::Conductor));
|
||||
|
||||
for (const db::Artist::pointer& composer : getOrCreateArtists(dbSession, _parsedTrack->composerArtists, true))
|
||||
track.modify()->addArtistLink(db::TrackArtistLink::create(dbSession, track, composer, db::TrackArtistLinkType::Composer));
|
||||
|
||||
for (const db::Artist::pointer& lyricist : getOrCreateArtists(dbSession, _parsedTrack->lyricistArtists, true))
|
||||
track.modify()->addArtistLink(db::TrackArtistLink::create(dbSession, track, lyricist, db::TrackArtistLinkType::Lyricist));
|
||||
|
||||
for (const db::Artist::pointer& mixer : getOrCreateArtists(dbSession, _parsedTrack->mixerArtists, true))
|
||||
track.modify()->addArtistLink(db::TrackArtistLink::create(dbSession, track, mixer, db::TrackArtistLinkType::Mixer));
|
||||
|
||||
for (const auto& [role, performers] : _parsedTrack->performerArtists)
|
||||
{
|
||||
for (const db::Artist::pointer& performer : getOrCreateArtists(dbSession, performers, true))
|
||||
track.modify()->addArtistLink(db::TrackArtistLink::create(dbSession, track, performer, db::TrackArtistLinkType::Performer, role));
|
||||
}
|
||||
|
||||
for (const db::Artist::pointer& producer : getOrCreateArtists(dbSession, _parsedTrack->producerArtists, true))
|
||||
track.modify()->addArtistLink(db::TrackArtistLink::create(dbSession, track, producer, db::TrackArtistLinkType::Producer));
|
||||
|
||||
for (const db::Artist::pointer& remixer : getOrCreateArtists(dbSession, _parsedTrack->remixerArtists, true))
|
||||
track.modify()->addArtistLink(db::TrackArtistLink::create(dbSession, track, remixer, db::TrackArtistLinkType::Remixer));
|
||||
|
||||
track.modify()->setScanVersion(_settings.scanVersion);
|
||||
if (_parsedTrack->medium && _parsedTrack->medium->release)
|
||||
track.modify()->setRelease(getOrCreateRelease(dbSession, *_parsedTrack->medium->release, directory));
|
||||
else
|
||||
track.modify()->setRelease({});
|
||||
track.modify()->setTotalTrack(_parsedTrack->medium ? _parsedTrack->medium->trackCount : std::nullopt);
|
||||
track.modify()->setReleaseReplayGain(_parsedTrack->medium ? _parsedTrack->medium->replayGain : std::nullopt);
|
||||
track.modify()->setDiscSubtitle(_parsedTrack->medium ? _parsedTrack->medium->name : "");
|
||||
track.modify()->setClusters(getOrCreateClusters(dbSession, *_parsedTrack));
|
||||
track.modify()->setName(title);
|
||||
track.modify()->setAddedTime(Wt::WDateTime::currentDateTime());
|
||||
track.modify()->setTrackNumber(_parsedTrack->position);
|
||||
track.modify()->setDiscNumber(_parsedTrack->medium ? _parsedTrack->medium->position : std::nullopt);
|
||||
track.modify()->setDate(_parsedTrack->date);
|
||||
track.modify()->setYear(_parsedTrack->year);
|
||||
track.modify()->setOriginalDate(_parsedTrack->originalDate);
|
||||
track.modify()->setOriginalYear(_parsedTrack->originalYear);
|
||||
|
||||
// If a file has an OriginalDate but no date, set it to ease filtering
|
||||
if (!_parsedTrack->date.isValid() && _parsedTrack->originalDate.isValid())
|
||||
track.modify()->setDate(_parsedTrack->originalDate);
|
||||
|
||||
// If a file has an OriginalYear but no Year, set it to ease filtering
|
||||
if (!_parsedTrack->year && _parsedTrack->originalYear)
|
||||
track.modify()->setYear(_parsedTrack->originalYear);
|
||||
|
||||
track.modify()->setRecordingMBID(_parsedTrack->recordingMBID);
|
||||
track.modify()->setTrackMBID(_parsedTrack->mbid);
|
||||
if (auto trackFeatures{ db::TrackFeatures::find(dbSession, track->getId()) })
|
||||
trackFeatures.remove(); // TODO: only if MBID changed?
|
||||
track.modify()->setHasCover(_parsedTrack->hasCover);
|
||||
track.modify()->setCopyright(_parsedTrack->copyright);
|
||||
track.modify()->setCopyrightURL(_parsedTrack->copyrightURL);
|
||||
track.modify()->setComment(!_parsedTrack->comments.empty() ? _parsedTrack->comments.front() : ""); // only take the first one for now
|
||||
track.modify()->setTrackReplayGain(_parsedTrack->replayGain);
|
||||
track.modify()->setArtistDisplayName(_parsedTrack->artistDisplayName);
|
||||
|
||||
track.modify()->clearEmbeddedLyrics();
|
||||
for (const metadata::Lyrics& lyricsInfo : _parsedTrack->lyrics)
|
||||
{
|
||||
db::TrackLyrics::pointer lyrics{ createLyrics(dbSession, lyricsInfo) };
|
||||
track.modify()->addLyrics(lyrics);
|
||||
}
|
||||
|
||||
if (added)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Added audio file '" << _file.string() << "'");
|
||||
stats.additions++;
|
||||
}
|
||||
else
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updated audio file '" << _file.string() << "'");
|
||||
stats.updates++;
|
||||
}
|
||||
}
|
||||
|
||||
metadata::ParserReadStyle getParserReadStyle()
|
||||
{
|
||||
std::string_view readStyle{ core::Service<core::IConfig>::get()->getString("scanner-parser-read-style", "average") };
|
||||
|
||||
if (readStyle == "fast")
|
||||
return metadata::ParserReadStyle::Fast;
|
||||
if (readStyle == "average")
|
||||
return metadata::ParserReadStyle::Average;
|
||||
if (readStyle == "accurate")
|
||||
return metadata::ParserReadStyle::Accurate;
|
||||
|
||||
throw core::LmsException{ "Invalid value for 'scanner-parser-read-style'" };
|
||||
}
|
||||
} // namespace
|
||||
|
||||
AudioFileScanner::AudioFileScanner(db::Db& db, const ScannerSettings& settings)
|
||||
: _db{ db }
|
||||
, _settings{ settings }
|
||||
, _metadataParser{ metadata::createParser(metadata::ParserBackend::TagLib, getParserReadStyle()) } // For now, always use TagLib
|
||||
{
|
||||
std::vector<std::string> tagsToParse{ _extraTagsToParse };
|
||||
tagsToParse.insert(std::end(tagsToParse), std::cbegin(settings.extraTags), std::cend(settings.extraTags));
|
||||
_metadataParser->setUserExtraTags(tagsToParse);
|
||||
_metadataParser->setArtistTagDelimiters(settings.artistTagDelimiters);
|
||||
_metadataParser->setDefaultTagDelimiters(settings.defaultTagDelimiters);
|
||||
}
|
||||
|
||||
AudioFileScanner::~AudioFileScanner() = default;
|
||||
|
||||
std::span<const std::filesystem::path> AudioFileScanner::getSupportedExtensions() const
|
||||
{
|
||||
return _metadataParser->getSupportedExtensions();
|
||||
}
|
||||
|
||||
bool AudioFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
|
||||
{
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
|
||||
// Should rarely fail as we are currently iterating it
|
||||
if (!lastWriteTime.isValid())
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (context.scanOptions.fullScan)
|
||||
return true;
|
||||
|
||||
bool needUpdateLibrary{};
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
|
||||
{
|
||||
auto transaction{ dbSession.createReadTransaction() };
|
||||
|
||||
// Skip file if last write is the same
|
||||
const db::Track::pointer track{ db::Track::findByPath(dbSession, file.file) };
|
||||
if (track
|
||||
&& track->getLastWriteTime() == lastWriteTime
|
||||
&& track->getScanVersion() == _settings.scanVersion)
|
||||
{
|
||||
// this file may have been moved from one library to another, then we just need to update the media library id instead of a full rescan
|
||||
const auto trackMediaLibrary{ track->getMediaLibrary() };
|
||||
if (trackMediaLibrary && trackMediaLibrary->getId() == file.mediaLibrary.id)
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
|
||||
needUpdateLibrary = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (needUpdateLibrary)
|
||||
{
|
||||
auto transaction{ dbSession.createWriteTransaction() };
|
||||
|
||||
db::Track::pointer track{ db::Track::findByPath(dbSession, file.file) };
|
||||
assert(track);
|
||||
track.modify()->setMediaLibrary(db::MediaLibrary::find(dbSession, file.mediaLibrary.id)); // may be null, will be handled in the next scan anyway
|
||||
stats.updates++;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true; // need to scan
|
||||
}
|
||||
|
||||
std::unique_ptr<IFileScanOperation> AudioFileScanner::createScanOperation(const FileToScan& fileToScan) const
|
||||
{
|
||||
return std::make_unique<AudioFileScanOperation>(fileToScan, _db, *_metadataParser, _settings);
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "IFileScanner.hpp"
|
||||
|
||||
namespace lms
|
||||
{
|
||||
namespace db
|
||||
{
|
||||
class Db;
|
||||
}
|
||||
|
||||
namespace metadata
|
||||
{
|
||||
class IParser;
|
||||
}
|
||||
} // namespace lms
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
struct ScannerSettings;
|
||||
|
||||
class AudioFileScanner : public IFileScanner
|
||||
{
|
||||
public:
|
||||
AudioFileScanner(db::Db& db, const ScannerSettings& settings);
|
||||
~AudioFileScanner() override;
|
||||
AudioFileScanner(const AudioFileScanner&) = delete;
|
||||
AudioFileScanner& operator=(const AudioFileScanner&) = delete;
|
||||
|
||||
private:
|
||||
std::span<const std::filesystem::path> getSupportedExtensions() const override;
|
||||
bool needsScan(ScanContext& context, const FileToScan& file) const override;
|
||||
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
|
||||
|
||||
db::Db& _db;
|
||||
const ScannerSettings& _settings;
|
||||
std::unique_ptr<metadata::IParser> _metadataParser;
|
||||
const std::vector<std::string> _extraTagsToParse;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "core/LiteralString.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class ScanContext;
|
||||
|
||||
class IFileScanOperation
|
||||
{
|
||||
public:
|
||||
virtual ~IFileScanOperation() = default;
|
||||
|
||||
virtual core::LiteralString getName() const = 0;
|
||||
virtual void scan() = 0;
|
||||
virtual void processResult(ScanContext& context) = 0;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <filesystem>
|
||||
#include <span>
|
||||
|
||||
#include "MediaLibraryInfo.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class ScanContext;
|
||||
class IFileScanOperation;
|
||||
struct ScannerSettings;
|
||||
|
||||
struct FileToScan
|
||||
{
|
||||
std::filesystem::path file;
|
||||
MediaLibraryInfo mediaLibrary;
|
||||
};
|
||||
|
||||
class IFileScanner
|
||||
{
|
||||
public:
|
||||
virtual ~IFileScanner() = default;
|
||||
|
||||
virtual std::span<const std::filesystem::path> getSupportedExtensions() const = 0;
|
||||
virtual bool needsScan(ScanContext& context, const FileToScan& file) const = 0;
|
||||
virtual std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const = 0;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,171 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "ImageFileScanner.hpp"
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "core/ILogger.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Image.hpp"
|
||||
#include "database/MediaLibrary.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "image/Exception.hpp"
|
||||
#include "image/Image.hpp"
|
||||
|
||||
#include "IFileScanOperation.hpp"
|
||||
#include "ScanContext.hpp"
|
||||
#include "Utils.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
namespace
|
||||
{
|
||||
class ImageFileScanOperation : public IFileScanOperation
|
||||
{
|
||||
public:
|
||||
ImageFileScanOperation(const FileToScan& file, db::Db& db)
|
||||
: _file{ file.file }
|
||||
, _mediaLibrary{ file.mediaLibrary }
|
||||
, _db{ db } {}
|
||||
|
||||
private:
|
||||
core::LiteralString getName() const override { return "ScanImageFile"; }
|
||||
void scan() override;
|
||||
void processResult(ScanContext& context) override;
|
||||
|
||||
const std::filesystem::path _file;
|
||||
const MediaLibraryInfo _mediaLibrary;
|
||||
db::Db& _db;
|
||||
|
||||
struct ImageInfo
|
||||
{
|
||||
std::size_t height{};
|
||||
std::size_t width{};
|
||||
};
|
||||
std::optional<ImageInfo> _parsedImageInfo;
|
||||
};
|
||||
|
||||
void ImageFileScanOperation::scan()
|
||||
{
|
||||
try
|
||||
{
|
||||
std::unique_ptr<image::IRawImage> rawImage{ image::decodeImage(_file) };
|
||||
ImageInfo& imageInfo{ _parsedImageInfo.emplace() };
|
||||
imageInfo.width = rawImage->getWidth();
|
||||
imageInfo.height = rawImage->getHeight();
|
||||
}
|
||||
catch (const image::Exception& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot read image in file '" << _file.string() << "': " << e.what());
|
||||
}
|
||||
}
|
||||
|
||||
void ImageFileScanOperation::processResult(ScanContext& context)
|
||||
{
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
|
||||
if (!fileInfo)
|
||||
{
|
||||
stats.skips++;
|
||||
return;
|
||||
}
|
||||
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
db::Image::pointer image{ db::Image::find(dbSession, _file) };
|
||||
|
||||
if (!_parsedImageInfo)
|
||||
{
|
||||
if (image)
|
||||
{
|
||||
image.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadImageFile);
|
||||
return;
|
||||
}
|
||||
|
||||
const bool added{ !image };
|
||||
if (!image)
|
||||
image = dbSession.create<db::Image>(_file);
|
||||
|
||||
image.modify()->setLastWriteTime(fileInfo->lastWriteTime);
|
||||
image.modify()->setFileSize(fileInfo->fileSize);
|
||||
image.modify()->setHeight(_parsedImageInfo->height);
|
||||
image.modify()->setWidth(_parsedImageInfo->width);
|
||||
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
|
||||
image.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary));
|
||||
|
||||
if (added)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Added image '" << _file.string() << "'");
|
||||
stats.additions++;
|
||||
}
|
||||
else
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updated image '" << _file.string() << "'");
|
||||
stats.updates++;
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
ImageFileScanner::ImageFileScanner(db::Db& db)
|
||||
: _db{ db }
|
||||
{
|
||||
}
|
||||
|
||||
std::span<const std::filesystem::path> ImageFileScanner::getSupportedExtensions() const
|
||||
{
|
||||
return image::getSupportedFileExtensions();
|
||||
}
|
||||
|
||||
bool ImageFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
|
||||
{
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
|
||||
// Should rarely fail as we are currently iterating it
|
||||
if (!lastWriteTime.isValid())
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!context.scanOptions.fullScan)
|
||||
{
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
auto transaction{ _db.getTLSSession().createReadTransaction() };
|
||||
|
||||
const db::Image::pointer image{ db::Image::find(dbSession, file.file) };
|
||||
if (image && image->getLastWriteTime() == lastWriteTime)
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true; // need to scan
|
||||
}
|
||||
|
||||
std::unique_ptr<IFileScanOperation> ImageFileScanner::createScanOperation(const FileToScan& fileToScan) const
|
||||
{
|
||||
return std::make_unique<ImageFileScanOperation>(fileToScan, _db);
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "IFileScanner.hpp"
|
||||
|
||||
namespace lms
|
||||
{
|
||||
namespace db
|
||||
{
|
||||
class Db;
|
||||
}
|
||||
} // namespace lms
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
struct ScannerSettings;
|
||||
|
||||
class ImageFileScanner : public IFileScanner
|
||||
{
|
||||
public:
|
||||
ImageFileScanner(db::Db& db);
|
||||
~ImageFileScanner() override = default;
|
||||
ImageFileScanner(const ImageFileScanner&) = delete;
|
||||
ImageFileScanner& operator=(const ImageFileScanner&) = delete;
|
||||
|
||||
private:
|
||||
std::span<const std::filesystem::path> getSupportedExtensions() const override;
|
||||
bool needsScan(ScanContext& context, const FileToScan& file) const override;
|
||||
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
|
||||
|
||||
db::Db& _db;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,177 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "LyricsFileScanner.hpp"
|
||||
|
||||
#include <fstream>
|
||||
#include <optional>
|
||||
|
||||
#include "core/ILogger.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/MediaLibrary.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "database/TrackLyrics.hpp"
|
||||
#include "metadata/Lyrics.hpp"
|
||||
|
||||
#include "IFileScanOperation.hpp"
|
||||
#include "ScanContext.hpp"
|
||||
#include "Utils.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
namespace
|
||||
{
|
||||
class LyricsFileScanOperation : public IFileScanOperation
|
||||
{
|
||||
public:
|
||||
LyricsFileScanOperation(const FileToScan& file, db::Db& db)
|
||||
: _file{ file.file }
|
||||
, _mediaLibrary{ file.mediaLibrary }
|
||||
, _db{ db } {}
|
||||
|
||||
private:
|
||||
core::LiteralString getName() const override { return "ScanLyricsFile"; }
|
||||
void scan() override;
|
||||
void processResult(ScanContext& context) override;
|
||||
|
||||
const std::filesystem::path _file;
|
||||
const MediaLibraryInfo _mediaLibrary;
|
||||
db::Db& _db;
|
||||
|
||||
std::optional<metadata::Lyrics> _parsedLyrics;
|
||||
};
|
||||
|
||||
void LyricsFileScanOperation::scan()
|
||||
{
|
||||
try
|
||||
{
|
||||
std::ifstream ifs{ _file.string() };
|
||||
if (!ifs)
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot open file '" << _file.string() << "'");
|
||||
else
|
||||
_parsedLyrics = metadata::parseLyrics(ifs);
|
||||
}
|
||||
catch (const metadata::Exception& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot read lyrics in file '" << _file.string() << "': " << e.what());
|
||||
}
|
||||
}
|
||||
|
||||
void LyricsFileScanOperation::processResult(ScanContext& context)
|
||||
{
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
|
||||
if (!fileInfo)
|
||||
{
|
||||
stats.skips++;
|
||||
return;
|
||||
}
|
||||
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
db::TrackLyrics::pointer trackLyrics{ db::TrackLyrics::find(dbSession, _file) };
|
||||
|
||||
if (!_parsedLyrics)
|
||||
{
|
||||
if (trackLyrics)
|
||||
{
|
||||
trackLyrics.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadLyricsFile);
|
||||
return;
|
||||
}
|
||||
|
||||
const bool added{ !trackLyrics };
|
||||
if (!trackLyrics)
|
||||
{
|
||||
trackLyrics = dbSession.create<db::TrackLyrics>();
|
||||
trackLyrics.modify()->setAbsoluteFilePath(_file);
|
||||
}
|
||||
|
||||
trackLyrics.modify()->setLastWriteTime(fileInfo->lastWriteTime);
|
||||
trackLyrics.modify()->setFileSize(fileInfo->fileSize);
|
||||
trackLyrics.modify()->setLanguage(!_parsedLyrics->language.empty() ? _parsedLyrics->language : "xxx");
|
||||
trackLyrics.modify()->setOffset(_parsedLyrics->offset);
|
||||
trackLyrics.modify()->setDisplayTitle(_parsedLyrics->displayTitle);
|
||||
trackLyrics.modify()->setDisplayArtist(_parsedLyrics->displayArtist);
|
||||
if (!_parsedLyrics->synchronizedLines.empty())
|
||||
trackLyrics.modify()->setSynchronizedLines(_parsedLyrics->synchronizedLines);
|
||||
else
|
||||
trackLyrics.modify()->setUnsynchronizedLines(_parsedLyrics->unsynchronizedLines);
|
||||
|
||||
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
|
||||
trackLyrics.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary));
|
||||
|
||||
if (added)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Added external lyrics '" << _file.string() << "'");
|
||||
stats.additions++;
|
||||
}
|
||||
else
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updated external lyrics '" << _file.string() << "'");
|
||||
stats.updates++;
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
LyricsFileScanner::LyricsFileScanner(db::Db& db)
|
||||
: _db{ db }
|
||||
{
|
||||
}
|
||||
|
||||
std::span<const std::filesystem::path> LyricsFileScanner::getSupportedExtensions() const
|
||||
{
|
||||
return metadata::getSupportedLyricsFileExtensions();
|
||||
}
|
||||
|
||||
bool LyricsFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
|
||||
{
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
|
||||
// Should rarely fail as we are currently iterating it
|
||||
if (!lastWriteTime.isValid())
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!context.scanOptions.fullScan)
|
||||
{
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
auto transaction{ _db.getTLSSession().createReadTransaction() };
|
||||
|
||||
const db::TrackLyrics::pointer lyrics{ db::TrackLyrics::find(dbSession, file.file) };
|
||||
if (lyrics && lyrics->getLastWriteTime() == lastWriteTime)
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true; // need to scan
|
||||
}
|
||||
|
||||
std::unique_ptr<IFileScanOperation> LyricsFileScanner::createScanOperation(const FileToScan& fileToScan) const
|
||||
{
|
||||
return std::make_unique<LyricsFileScanOperation>(fileToScan, _db);
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "IFileScanner.hpp"
|
||||
|
||||
namespace lms
|
||||
{
|
||||
namespace db
|
||||
{
|
||||
class Db;
|
||||
}
|
||||
} // namespace lms
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
struct ScannerSettings;
|
||||
|
||||
class LyricsFileScanner : public IFileScanner
|
||||
{
|
||||
public:
|
||||
LyricsFileScanner(db::Db& db);
|
||||
~LyricsFileScanner() override = default;
|
||||
LyricsFileScanner(const LyricsFileScanner&) = delete;
|
||||
LyricsFileScanner& operator=(const LyricsFileScanner&) = delete;
|
||||
|
||||
private:
|
||||
std::span<const std::filesystem::path> getSupportedExtensions() const override;
|
||||
bool needsScan(ScanContext& context, const FileToScan& file) const override;
|
||||
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
|
||||
|
||||
db::Db& _db;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,176 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "PlayListFileScanner.hpp"
|
||||
|
||||
#include <fstream>
|
||||
#include <optional>
|
||||
|
||||
#include "core/ILogger.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/MediaLibrary.hpp"
|
||||
#include "database/PlayListFile.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "metadata/Exception.hpp"
|
||||
#include "metadata/PlayList.hpp"
|
||||
|
||||
#include "IFileScanOperation.hpp"
|
||||
#include "ScanContext.hpp"
|
||||
#include "Utils.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
namespace
|
||||
{
|
||||
class PlayListFileScanOperation : public IFileScanOperation
|
||||
{
|
||||
public:
|
||||
PlayListFileScanOperation(const FileToScan& file, db::Db& db)
|
||||
: _file{ file.file }
|
||||
, _mediaLibrary{ file.mediaLibrary }
|
||||
, _db{ db } {}
|
||||
~PlayListFileScanOperation() override = default;
|
||||
PlayListFileScanOperation(const PlayListFileScanOperation&) = delete;
|
||||
PlayListFileScanOperation& operator=(const PlayListFileScanOperation&) = delete;
|
||||
|
||||
private:
|
||||
core::LiteralString getName() const override { return "ScanPlayListFile"; }
|
||||
void scan() override;
|
||||
void processResult(ScanContext& context) override;
|
||||
|
||||
const std::filesystem::path _file;
|
||||
const MediaLibraryInfo _mediaLibrary;
|
||||
db::Db& _db;
|
||||
|
||||
std::optional<metadata::PlayList> _parsedPlayList;
|
||||
};
|
||||
|
||||
void PlayListFileScanOperation::scan()
|
||||
{
|
||||
try
|
||||
{
|
||||
std::ifstream ifs{ _file.string() };
|
||||
if (!ifs)
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot open file '" << _file.string() << "'");
|
||||
else
|
||||
_parsedPlayList = metadata::parsePlayList(ifs);
|
||||
}
|
||||
catch (const metadata::Exception& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot read playlist in file '" << _file.string() << "': " << e.what());
|
||||
}
|
||||
}
|
||||
|
||||
void PlayListFileScanOperation::processResult(ScanContext& context)
|
||||
{
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
|
||||
if (!fileInfo)
|
||||
{
|
||||
stats.skips++;
|
||||
return;
|
||||
}
|
||||
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, _file) };
|
||||
|
||||
if (!_parsedPlayList)
|
||||
{
|
||||
if (playList)
|
||||
{
|
||||
playList.remove();
|
||||
stats.deletions++;
|
||||
}
|
||||
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadPlayListFile);
|
||||
return;
|
||||
}
|
||||
|
||||
const bool added{ !playList };
|
||||
if (!playList)
|
||||
playList = dbSession.create<db::PlayListFile>(_file);
|
||||
|
||||
playList.modify()->setLastWriteTime(fileInfo->lastWriteTime);
|
||||
playList.modify()->setFileSize(fileInfo->fileSize);
|
||||
if (!_parsedPlayList->name.empty())
|
||||
playList.modify()->setName(_parsedPlayList->name);
|
||||
else
|
||||
playList.modify()->setName(_file.stem().string());
|
||||
playList.modify()->setFiles(_parsedPlayList->files);
|
||||
|
||||
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
|
||||
playList.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary));
|
||||
|
||||
if (added)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Added playlist file '" << _file.string() << "'");
|
||||
LMS_LOG(DBUPDATER, DEBUG, "db playlist file = '" << playList->getAbsoluteFilePath().string() << "'");
|
||||
stats.additions++;
|
||||
}
|
||||
else
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updated playlist file '" << _file.string() << "'");
|
||||
stats.updates++;
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
PlayListFileScanner::PlayListFileScanner(db::Db& db)
|
||||
: _db{ db }
|
||||
{
|
||||
}
|
||||
|
||||
std::span<const std::filesystem::path> PlayListFileScanner::getSupportedExtensions() const
|
||||
{
|
||||
return metadata::getSupportedPlayListFileExtensions();
|
||||
}
|
||||
|
||||
bool PlayListFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
|
||||
{
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
|
||||
// Should rarely fail as we are currently iterating it
|
||||
if (!lastWriteTime.isValid())
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!context.scanOptions.fullScan)
|
||||
{
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
auto transaction{ _db.getTLSSession().createReadTransaction() };
|
||||
|
||||
const db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, file.file) };
|
||||
if (playList && playList->getLastWriteTime() == lastWriteTime)
|
||||
{
|
||||
stats.skips++;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true; // need to scan
|
||||
}
|
||||
|
||||
std::unique_ptr<IFileScanOperation> PlayListFileScanner::createScanOperation(const FileToScan& fileToScan) const
|
||||
{
|
||||
return std::make_unique<PlayListFileScanOperation>(fileToScan, _db);
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "IFileScanner.hpp"
|
||||
|
||||
namespace lms
|
||||
{
|
||||
namespace db
|
||||
{
|
||||
class Db;
|
||||
}
|
||||
} // namespace lms
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
struct ScannerSettings;
|
||||
|
||||
class PlayListFileScanner : public IFileScanner
|
||||
{
|
||||
public:
|
||||
PlayListFileScanner(db::Db& db);
|
||||
~PlayListFileScanner() override = default;
|
||||
PlayListFileScanner(const PlayListFileScanner&) = delete;
|
||||
PlayListFileScanner& operator=(const PlayListFileScanner&) = delete;
|
||||
|
||||
private:
|
||||
std::span<const std::filesystem::path> getSupportedExtensions() const override;
|
||||
bool needsScan(ScanContext& context, const FileToScan& file) const override;
|
||||
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
|
||||
|
||||
db::Db& _db;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "Utils.hpp"
|
||||
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/Path.hpp"
|
||||
#include "database/Directory.hpp"
|
||||
#include "database/MediaLibrary.hpp"
|
||||
#include "database/Session.hpp"
|
||||
|
||||
namespace lms::scanner::utils
|
||||
{
|
||||
Wt::WDateTime retrieveFileGetLastWrite(const std::filesystem::path& file)
|
||||
{
|
||||
Wt::WDateTime res;
|
||||
|
||||
try
|
||||
{
|
||||
res = core::pathUtils::getLastWriteTime(file);
|
||||
}
|
||||
catch (core::LmsException& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot get last write time: " << e.what());
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::optional<FileInfo> retrieveFileInfo(const std::filesystem::path& file, const std::filesystem::path& rootPath)
|
||||
{
|
||||
std::optional<FileInfo> res;
|
||||
res.emplace();
|
||||
|
||||
res->lastWriteTime = retrieveFileGetLastWrite(file);
|
||||
if (!res->lastWriteTime.isValid())
|
||||
{
|
||||
res.reset();
|
||||
return res;
|
||||
}
|
||||
|
||||
{
|
||||
std::error_code ec;
|
||||
res->relativePath = std::filesystem::relative(file, rootPath, ec);
|
||||
if (ec)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot get relative file path for '" << file.string() << "' from '" << rootPath.string() << "': " << ec.message());
|
||||
res.reset();
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
std::error_code ec;
|
||||
res->fileSize = std::filesystem::file_size(file, ec);
|
||||
if (ec)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot get file size for '" << file.string() << "': " << ec.message());
|
||||
res.reset();
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
db::Directory::pointer getOrCreateDirectory(db::Session& session, const std::filesystem::path& path, const db::MediaLibrary::pointer& mediaLibrary)
|
||||
{
|
||||
db::Directory::pointer directory{ db::Directory::find(session, path) };
|
||||
if (!directory)
|
||||
{
|
||||
db::Directory::pointer parentDirectory;
|
||||
if (path != mediaLibrary->getPath())
|
||||
parentDirectory = getOrCreateDirectory(session, path.parent_path(), mediaLibrary);
|
||||
|
||||
directory = session.create<db::Directory>(path);
|
||||
directory.modify()->setParent(parentDirectory);
|
||||
directory.modify()->setMediaLibrary(mediaLibrary);
|
||||
}
|
||||
// Don't update library if it does not match, will be updated elsewhere
|
||||
|
||||
return directory;
|
||||
}
|
||||
} // namespace lms::scanner::utils
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <filesystem>
|
||||
#include <optional>
|
||||
|
||||
#include <Wt/WDateTime.h>
|
||||
|
||||
#include "database/Object.hpp"
|
||||
|
||||
namespace lms::db
|
||||
{
|
||||
class Directory;
|
||||
class MediaLibrary;
|
||||
class Session;
|
||||
} // namespace lms::db
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
struct FileInfo
|
||||
{
|
||||
Wt::WDateTime lastWriteTime;
|
||||
std::filesystem::path relativePath;
|
||||
std::size_t fileSize{};
|
||||
};
|
||||
|
||||
namespace utils
|
||||
{
|
||||
Wt::WDateTime retrieveFileGetLastWrite(const std::filesystem::path& file);
|
||||
std::optional<FileInfo> retrieveFileInfo(const std::filesystem::path& file, const std::filesystem::path& rootPath);
|
||||
|
||||
db::ObjectPtr<db::Directory> getOrCreateDirectory(db::Session& session, const std::filesystem::path& path, const db::ObjectPtr<db::MediaLibrary>& mediaLibrary);
|
||||
} // namespace utils
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "FileScanQueue.hpp"
|
||||
|
||||
#include <boost/asio/post.hpp>
|
||||
|
||||
#include "core/ITraceLogger.hpp"
|
||||
|
||||
#include "scanners/IFileScanOperation.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
FileScanQueue::FileScanQueue(std::size_t threadCount, bool& abort)
|
||||
: _scanContextRunner{ _scanIoContext, threadCount, "FileScan" }
|
||||
, _abort{ abort }
|
||||
{
|
||||
}
|
||||
|
||||
FileScanQueue::~FileScanQueue() = default;
|
||||
|
||||
void FileScanQueue::pushScanRequest(std::unique_ptr<IFileScanOperation> operation)
|
||||
{
|
||||
{
|
||||
std::scoped_lock lock{ _mutex };
|
||||
_ongoingScanCount += 1;
|
||||
}
|
||||
|
||||
auto operationHandler{ [operation = std::move(operation), this]() mutable {
|
||||
if (_abort)
|
||||
{
|
||||
std::scoped_lock lock{ _mutex };
|
||||
_ongoingScanCount -= 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
{
|
||||
LMS_SCOPED_TRACE_OVERVIEW("Scanner", operation->getName());
|
||||
operation->scan();
|
||||
}
|
||||
|
||||
{
|
||||
std::scoped_lock lock{ _mutex };
|
||||
|
||||
_scanResults.emplace_back(std::move(operation));
|
||||
_ongoingScanCount -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
_condVar.notify_all();
|
||||
} };
|
||||
|
||||
boost::asio::post(_scanIoContext, std::move(operationHandler));
|
||||
}
|
||||
|
||||
std::size_t FileScanQueue::getResultsCount() const
|
||||
{
|
||||
std::scoped_lock lock{ _mutex };
|
||||
return _scanResults.size();
|
||||
}
|
||||
|
||||
size_t FileScanQueue::popResults(std::vector<std::unique_ptr<IFileScanOperation>>& results, std::size_t maxCount)
|
||||
{
|
||||
results.clear();
|
||||
results.reserve(maxCount);
|
||||
|
||||
{
|
||||
std::scoped_lock lock{ _mutex };
|
||||
|
||||
while (results.size() < maxCount && !_scanResults.empty())
|
||||
{
|
||||
results.push_back(std::move(_scanResults.front()));
|
||||
_scanResults.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
return results.size();
|
||||
}
|
||||
|
||||
void FileScanQueue::wait(std::size_t maxScanRequestCount)
|
||||
{
|
||||
if (_ongoingScanCount <= maxScanRequestCount)
|
||||
return;
|
||||
|
||||
{
|
||||
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "WaitParseResults");
|
||||
|
||||
std::unique_lock lock{ _mutex };
|
||||
_condVar.wait(lock, [=, this] { return _ongoingScanCount <= maxScanRequestCount; });
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
#include "core/IOContextRunner.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class IFileScanOperation;
|
||||
|
||||
class FileScanQueue
|
||||
{
|
||||
public:
|
||||
FileScanQueue(std::size_t threadCount, bool& abort);
|
||||
~FileScanQueue();
|
||||
FileScanQueue(const FileScanQueue&) = delete;
|
||||
FileScanQueue& operator=(const FileScanQueue&) = delete;
|
||||
|
||||
std::size_t getThreadCount() const { return _scanContextRunner.getThreadCount(); }
|
||||
|
||||
void pushScanRequest(std::unique_ptr<IFileScanOperation> operation);
|
||||
|
||||
std::size_t getResultsCount() const;
|
||||
size_t popResults(std::vector<std::unique_ptr<IFileScanOperation>>& results, std::size_t maxCount);
|
||||
|
||||
void wait(std::size_t maxScanRequestCount = 0); // wait until ongoing scan request count <= maxScanRequestCount
|
||||
|
||||
private:
|
||||
boost::asio::io_context _scanIoContext;
|
||||
core::IOContextRunner _scanContextRunner;
|
||||
|
||||
mutable std::mutex _mutex;
|
||||
std::atomic<std::size_t> _ongoingScanCount{};
|
||||
std::deque<std::unique_ptr<IFileScanOperation>> _scanResults;
|
||||
std::condition_variable _condVar;
|
||||
bool& _abort;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
+2
-9
@@ -20,8 +20,8 @@
|
||||
#pragma once
|
||||
|
||||
#include "core/LiteralString.hpp"
|
||||
#include "services/scanner/ScannerOptions.hpp"
|
||||
#include "services/scanner/ScannerStats.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
@@ -32,13 +32,6 @@ namespace lms::scanner
|
||||
|
||||
virtual ScanStep getStep() const = 0;
|
||||
virtual core::LiteralString getStepName() const = 0;
|
||||
|
||||
struct ScanContext
|
||||
{
|
||||
ScanOptions scanOptions;
|
||||
ScanStats stats;
|
||||
ScanStepStats currentStepStats;
|
||||
};
|
||||
virtual void process(ScanContext& context) = 0;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
+2
-2
@@ -59,9 +59,9 @@ namespace lms::scanner
|
||||
assert(!lyrics->getFileStem().empty());
|
||||
|
||||
params.setDirectory(lyrics->getDirectory()->getId());
|
||||
params.setStem(stem);
|
||||
params.setFileStem(stem);
|
||||
|
||||
db::Track::find(session, params, [&](const db::Track::pointer track) {
|
||||
db::Track::find(session, params, [&](const db::Track::pointer& track) {
|
||||
if (matchingTrack)
|
||||
LMS_LOG(DBUPDATER, DEBUG, "External lyrics '" << lyrics->getAbsoluteFilePath() << "' already matched with '" << matchingTrack->getAbsoluteFilePath() << "', replaced by '" << track->getAbsoluteFilePath() << "'");
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "ScanStepAssociatePlayListTracks.hpp"
|
||||
|
||||
#include <deque>
|
||||
#include <filesystem>
|
||||
|
||||
#include "core/ILogger.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Directory.hpp"
|
||||
#include "database/PlayListFile.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackList.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
namespace
|
||||
{
|
||||
constexpr std::size_t readBatchSize{ 20 };
|
||||
constexpr std::size_t writeBatchSize{ 5 };
|
||||
|
||||
struct PlayListFileAssociation
|
||||
{
|
||||
db::PlayListFileId playListFileIdId;
|
||||
std::vector<db::TrackId> trackIds;
|
||||
};
|
||||
using PlayListFileAssociationContainer = std::deque<PlayListFileAssociation>;
|
||||
|
||||
struct SearchPlayListFileContext
|
||||
{
|
||||
db::Session& session;
|
||||
db::PlayListFileId lastRetrievedPlayListFileId;
|
||||
std::size_t processedPlayListFileCount{};
|
||||
};
|
||||
|
||||
db::Track::pointer getMatchingTrack(db::Session& session, const std::filesystem::path& filePath, const db::Directory::pointer& playListDirectory)
|
||||
{
|
||||
db::Track::pointer matchingTrack;
|
||||
if (filePath.is_absolute())
|
||||
{
|
||||
matchingTrack = db::Track::findByPath(session, filePath);
|
||||
}
|
||||
else
|
||||
{
|
||||
const std::filesystem::path absolutePath{ playListDirectory->getAbsolutePath() / filePath };
|
||||
matchingTrack = db::Track::findByPath(session, absolutePath.lexically_normal());
|
||||
}
|
||||
|
||||
return matchingTrack;
|
||||
}
|
||||
|
||||
bool trackListNeedsUpdate(db::Session& session, std::string_view name, std::span<const db::TrackId> trackIds, const db::TrackList::pointer& trackList)
|
||||
{
|
||||
if (trackList->getName() != name)
|
||||
return true;
|
||||
|
||||
db::TrackListEntry::FindParameters params;
|
||||
params.setTrackList(trackList->getId());
|
||||
|
||||
bool needUpdate{};
|
||||
std::size_t currentIndex{};
|
||||
db::TrackListEntry::find(session, params, [&](const db::TrackListEntry::pointer& entry) {
|
||||
if (currentIndex > trackIds.size() || trackIds[currentIndex] != entry->getTrackId())
|
||||
needUpdate = true;
|
||||
|
||||
currentIndex += 1;
|
||||
});
|
||||
|
||||
if (currentIndex != trackIds.size())
|
||||
needUpdate = true;
|
||||
|
||||
return needUpdate;
|
||||
}
|
||||
|
||||
bool fetchNextPlayListFilesToUpdate(SearchPlayListFileContext& searchContext, PlayListFileAssociationContainer& playListFileAssociations)
|
||||
{
|
||||
const db::PlayListFileId playListFileIdId{ searchContext.lastRetrievedPlayListFileId };
|
||||
|
||||
{
|
||||
auto transaction{ searchContext.session.createReadTransaction() };
|
||||
|
||||
db::PlayListFile::find(searchContext.session, searchContext.lastRetrievedPlayListFileId, readBatchSize, [&](const db::PlayListFile::pointer& playListFile) {
|
||||
PlayListFileAssociation playListAssociation;
|
||||
|
||||
playListAssociation.playListFileIdId = playListFile->getId();
|
||||
|
||||
const auto files{ playListFile->getFiles() };
|
||||
for (const std::filesystem::path& file : files)
|
||||
{
|
||||
// TODO optim: no need to fetch the whole track
|
||||
db::Track::pointer track{ getMatchingTrack(searchContext.session, file, playListFile->getDirectory()) };
|
||||
if (track)
|
||||
playListAssociation.trackIds.push_back(track->getId());
|
||||
else
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Track '" << file.string() << "' not found in playlist '" << playListFile->getAbsoluteFilePath().string() << "'");
|
||||
}
|
||||
|
||||
bool needUpdate{ true };
|
||||
if (const db::TrackList::pointer trackList{ playListFile->getTrackList() })
|
||||
needUpdate = trackListNeedsUpdate(searchContext.session, playListFile->getName(), playListAssociation.trackIds, trackList);
|
||||
|
||||
if (needUpdate)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updating PlayList '" << playListFile->getAbsoluteFilePath().string() << "' (" << playListAssociation.trackIds.size() << " files)");
|
||||
playListFileAssociations.emplace_back(std::move(playListAssociation));
|
||||
}
|
||||
searchContext.processedPlayListFileCount++;
|
||||
});
|
||||
}
|
||||
|
||||
return playListFileIdId != searchContext.lastRetrievedPlayListFileId;
|
||||
}
|
||||
|
||||
void updatePlayListFile(db::Session& session, const PlayListFileAssociation& playListFileAssociation)
|
||||
{
|
||||
db::PlayListFile::pointer playListFile{ db::PlayListFile::find(session, playListFileAssociation.playListFileIdId) };
|
||||
assert(playListFile);
|
||||
|
||||
db::TrackList::pointer trackList{ playListFile->getTrackList() };
|
||||
if (!trackList)
|
||||
{
|
||||
trackList = session.create<db::TrackList>(playListFile->getName(), db::TrackListType::PlayList);
|
||||
playListFile.modify()->setTrackList(trackList);
|
||||
}
|
||||
|
||||
trackList.modify()->setVisibility(db::TrackList::Visibility::Public);
|
||||
trackList.modify()->setLastModifiedDateTime(playListFile->getLastWriteTime());
|
||||
trackList.modify()->setName(playListFile->getName());
|
||||
|
||||
trackList.modify()->clear();
|
||||
for (const db::TrackId trackId : playListFileAssociation.trackIds)
|
||||
{
|
||||
if (db::Track::pointer track{ db::Track::find(session, trackId) })
|
||||
session.create<db::TrackListEntry>(track, trackList, playListFile->getLastWriteTime());
|
||||
}
|
||||
}
|
||||
|
||||
void updatePlayListFiles(db::Session& session, PlayListFileAssociationContainer& playListFileAssociations)
|
||||
{
|
||||
while (!playListFileAssociations.empty())
|
||||
{
|
||||
auto transaction{ session.createWriteTransaction() };
|
||||
|
||||
for (std::size_t i{}; !playListFileAssociations.empty() && i < writeBatchSize; ++i)
|
||||
{
|
||||
updatePlayListFile(session, playListFileAssociations.front());
|
||||
playListFileAssociations.pop_front();
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void ScanStepAssociatePlayListTracks::process(ScanContext& context)
|
||||
{
|
||||
if (_abortScan)
|
||||
return;
|
||||
|
||||
if (context.stats.nbChanges() == 0)
|
||||
return;
|
||||
|
||||
auto& session{ _db.getTLSSession() };
|
||||
|
||||
{
|
||||
auto transaction{ session.createReadTransaction() };
|
||||
context.currentStepStats.totalElems = db::PlayListFile::getCount(session);
|
||||
}
|
||||
|
||||
SearchPlayListFileContext searchContext{
|
||||
.session = session,
|
||||
.lastRetrievedPlayListFileId = {},
|
||||
};
|
||||
|
||||
PlayListFileAssociationContainer playListFileAssociations;
|
||||
while (fetchNextPlayListFilesToUpdate(searchContext, playListFileAssociations))
|
||||
{
|
||||
if (_abortScan)
|
||||
return;
|
||||
|
||||
updatePlayListFiles(session, playListFileAssociations);
|
||||
context.currentStepStats.processedElems = searchContext.processedPlayListFileCount;
|
||||
_progressCallback(context.currentStepStats);
|
||||
}
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "ScanStepBase.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class ScanStepAssociatePlayListTracks : public ScanStepBase
|
||||
{
|
||||
public:
|
||||
using ScanStepBase::ScanStepBase;
|
||||
|
||||
private:
|
||||
ScanStep getStep() const override { return ScanStep::AssociatePlayListTracks; }
|
||||
core::LiteralString getStepName() const override { return "Associate playlist tracks"; }
|
||||
void process(ScanContext& context) override;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
+11
-4
@@ -20,11 +20,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <functional>
|
||||
|
||||
#include "services/scanner/ScannerStats.hpp"
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
#include "IScanStep.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
|
||||
namespace lms::db
|
||||
{
|
||||
@@ -33,10 +32,13 @@ namespace lms::db
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class IFileScanner;
|
||||
struct ScannerSettings;
|
||||
struct ScanStepStats;
|
||||
|
||||
class ScanStepBase : public IScanStep
|
||||
{
|
||||
public:
|
||||
static inline const std::filesystem::path excludeDirFileName{ ".lmsignore" };
|
||||
using ProgressCallback = std::function<void(const ScanStepStats& stats)>;
|
||||
|
||||
struct InitParams
|
||||
@@ -45,21 +47,26 @@ namespace lms::scanner
|
||||
ProgressCallback progressCallback;
|
||||
bool& abortScan;
|
||||
db::Db& db;
|
||||
std::span<IFileScanner*> fileScanners;
|
||||
};
|
||||
ScanStepBase(InitParams& initParams)
|
||||
: _settings{ initParams.settings }
|
||||
, _progressCallback{ initParams.progressCallback }
|
||||
, _abortScan{ initParams.abortScan }
|
||||
, _db{ initParams.db }
|
||||
, _fileScanners(std::cbegin(initParams.fileScanners), std::cend(initParams.fileScanners))
|
||||
{
|
||||
}
|
||||
|
||||
protected:
|
||||
~ScanStepBase() override = default;
|
||||
ScanStepBase(const ScanStepBase&) = delete;
|
||||
ScanStepBase& operator=(const ScanStepBase&) = delete;
|
||||
|
||||
const ScannerSettings& _settings;
|
||||
ProgressCallback _progressCallback;
|
||||
bool& _abortScan;
|
||||
db::Db& _db;
|
||||
std::vector<IFileScanner*> _fileScanners;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
+18
-4
@@ -19,13 +19,18 @@
|
||||
|
||||
#include "ScanStepCheckForRemovedFiles.hpp"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "ScannerSettings.hpp"
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/Path.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Image.hpp"
|
||||
#include "database/PlayListFile.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackLyrics.hpp"
|
||||
#include "scanners/IFileScanner.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
@@ -47,12 +52,21 @@ namespace lms::scanner
|
||||
context.currentStepStats.totalElems += db::Track::getCount(session);
|
||||
context.currentStepStats.totalElems += db::Image::getCount(session);
|
||||
context.currentStepStats.totalElems += db::TrackLyrics::getExternalLyricsCount(session);
|
||||
context.currentStepStats.totalElems += db::PlayListFile::getCount(session);
|
||||
}
|
||||
LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " files to be checked...");
|
||||
|
||||
checkForRemovedFiles<db::Track>(context, _settings.supportedAudioFileExtensions);
|
||||
checkForRemovedFiles<db::Image>(context, _settings.supportedImageFileExtensions);
|
||||
checkForRemovedFiles<db::TrackLyrics>(context, _settings.supportedLyricsFileExtensions);
|
||||
std::vector<std::filesystem::path> supportedFileExtensions;
|
||||
for (IFileScanner* scanner : _fileScanners)
|
||||
{
|
||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
||||
supportedFileExtensions.emplace_back(extension);
|
||||
}
|
||||
|
||||
checkForRemovedFiles<db::Track>(context, supportedFileExtensions);
|
||||
checkForRemovedFiles<db::Image>(context, supportedFileExtensions);
|
||||
checkForRemovedFiles<db::TrackLyrics>(context, supportedFileExtensions);
|
||||
checkForRemovedFiles<db::PlayListFile>(context, supportedFileExtensions);
|
||||
}
|
||||
|
||||
template<typename Object>
|
||||
@@ -124,7 +138,7 @@ namespace lms::scanner
|
||||
}
|
||||
|
||||
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
|
||||
[&](const ScannerSettings::MediaLibraryInfo& libraryInfo) {
|
||||
[&](const MediaLibraryInfo& libraryInfo) {
|
||||
return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
|
||||
}))
|
||||
{
|
||||
+12
-9
@@ -22,21 +22,24 @@
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/Path.hpp"
|
||||
|
||||
#include "MediaLibraryInfo.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
#include "scanners/IFileScanner.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
void ScanStepDiscoverFiles::process(ScanContext& context)
|
||||
{
|
||||
context.stats.totalFileCount = 0;
|
||||
|
||||
std::vector<std::filesystem::path> supportedExtensions;
|
||||
for (const auto& extension : _settings.supportedAudioFileExtensions)
|
||||
supportedExtensions.emplace_back(extension);
|
||||
for (const auto& extension : _settings.supportedImageFileExtensions)
|
||||
supportedExtensions.emplace_back(extension);
|
||||
for (const auto& extension : _settings.supportedLyricsFileExtensions)
|
||||
supportedExtensions.emplace_back(extension);
|
||||
std::vector<std::filesystem::path> supportedFileExtensions;
|
||||
for (IFileScanner* scanner : _fileScanners)
|
||||
{
|
||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
||||
supportedFileExtensions.emplace_back(extension);
|
||||
}
|
||||
|
||||
for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
|
||||
for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
|
||||
{
|
||||
std::size_t currentDirectoryProcessElemsCount{};
|
||||
core::pathUtils::exploreFilesRecursive(
|
||||
@@ -44,7 +47,7 @@ namespace lms::scanner
|
||||
if (_abortScan)
|
||||
return false;
|
||||
|
||||
if (!ec && core::pathUtils::hasFileAnyExtension(path, supportedExtensions))
|
||||
if (!ec && core::pathUtils::hasFileAnyExtension(path, supportedFileExtensions))
|
||||
{
|
||||
context.currentStepStats.processedElems++;
|
||||
currentDirectoryProcessElemsCount++;
|
||||
+1
-1
@@ -29,7 +29,7 @@ namespace lms::scanner
|
||||
{
|
||||
ScanStats& stats{ context.stats };
|
||||
|
||||
if (context.scanOptions.forceOptimize || (stats.nbChanges() > (stats.nbFiles() / 5)))
|
||||
if (context.scanOptions.forceOptimize || (stats.nbChanges() > (stats.nbFiles() / 10)))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, INFO, "Database analyze started");
|
||||
|
||||
+1
-1
@@ -84,7 +84,7 @@ namespace lms::scanner
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void ScanStepRemoveOrphanedDbEntries::removeOrphanedEntries(ScanStepRemoveOrphanedDbEntries::ScanContext& context)
|
||||
void ScanStepRemoveOrphanedDbEntries::removeOrphanedEntries(ScanContext& context)
|
||||
{
|
||||
constexpr std::size_t batchSize = 100;
|
||||
|
||||
+1
-1
@@ -42,6 +42,6 @@ namespace lms::scanner
|
||||
void removeOrphanedDirectories(ScanContext& context);
|
||||
|
||||
template<typename T>
|
||||
void removeOrphanedEntries(ScanStepRemoveOrphanedDbEntries::ScanContext& context);
|
||||
void removeOrphanedEntries(ScanContext& context);
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,146 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "ScanStepScanFiles.hpp"
|
||||
|
||||
#include "FileScanQueue.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
#include "core/IConfig.hpp"
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/ITraceLogger.hpp"
|
||||
#include "core/Path.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "scanners/IFileScanOperation.hpp"
|
||||
#include "scanners/IFileScanner.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
using namespace db;
|
||||
|
||||
namespace
|
||||
{
|
||||
std::size_t getScanMetaDataThreadCount()
|
||||
{
|
||||
std::size_t threadCount{ core::Service<core::IConfig>::get()->getULong("scanner-metadata-thread-count", 0) };
|
||||
|
||||
if (threadCount == 0)
|
||||
threadCount = std::max<std::size_t>(std::thread::hardware_concurrency() / 2, 1);
|
||||
|
||||
return threadCount;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
|
||||
: ScanStepBase{ initParams }
|
||||
, _fileScanQueue{ getScanMetaDataThreadCount(), _abortScan }
|
||||
{
|
||||
for (IFileScanner* scanner : _fileScanners)
|
||||
{
|
||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
||||
{
|
||||
[[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner) };
|
||||
assert(inserted);
|
||||
}
|
||||
}
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO, "Using " << _fileScanQueue.getThreadCount() << " thread(s) for scanning file metadata");
|
||||
}
|
||||
|
||||
void ScanStepScanFiles::process(ScanContext& context)
|
||||
{
|
||||
context.currentStepStats.totalElems = context.stats.totalFileCount;
|
||||
|
||||
for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
|
||||
process(context, mediaLibrary);
|
||||
}
|
||||
|
||||
void ScanStepScanFiles::process(ScanContext& context, const MediaLibraryInfo& mediaLibrary)
|
||||
{
|
||||
const std::size_t scanQueueMaxScanRequestCount{ 100 * _fileScanQueue.getThreadCount() };
|
||||
const std::size_t processFileResultsBatchSize{ 5 };
|
||||
|
||||
std::vector<std::unique_ptr<IFileScanOperation>> scanOperations;
|
||||
|
||||
core::pathUtils::exploreFilesRecursive(
|
||||
mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) {
|
||||
LMS_SCOPED_TRACE_DETAILED("Scanner", "OnExploreFile");
|
||||
|
||||
if (_abortScan)
|
||||
return false; // stop iterating
|
||||
|
||||
if (ec)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot scan file '" << path.string() << "': " << ec.message());
|
||||
context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() });
|
||||
}
|
||||
else
|
||||
{
|
||||
auto itScanner{ _scannerByExtension.find(core::stringUtils::stringToLower(path.extension().string())) };
|
||||
if (itScanner != std::cend(_scannerByExtension))
|
||||
{
|
||||
IFileScanner& scanner{ *itScanner->second };
|
||||
|
||||
FileToScan fileToScan{ .file = path, .mediaLibrary = mediaLibrary };
|
||||
if (scanner.needsScan(context, fileToScan))
|
||||
{
|
||||
auto scanOperation{ scanner.createScanOperation(fileToScan) };
|
||||
_fileScanQueue.pushScanRequest(std::move(scanOperation));
|
||||
}
|
||||
|
||||
context.currentStepStats.processedElems++;
|
||||
_progressCallback(context.currentStepStats);
|
||||
}
|
||||
}
|
||||
|
||||
while (_fileScanQueue.getResultsCount() > (scanQueueMaxScanRequestCount / 2))
|
||||
{
|
||||
_fileScanQueue.popResults(scanOperations, processFileResultsBatchSize);
|
||||
processFileScanResults(context, scanOperations);
|
||||
}
|
||||
|
||||
_fileScanQueue.wait(scanQueueMaxScanRequestCount);
|
||||
|
||||
return true;
|
||||
},
|
||||
&excludeDirFileName);
|
||||
|
||||
_fileScanQueue.wait();
|
||||
|
||||
while (!_abortScan && _fileScanQueue.popResults(scanOperations, processFileResultsBatchSize) > 0)
|
||||
processFileScanResults(context, scanOperations);
|
||||
}
|
||||
|
||||
void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span<std::unique_ptr<IFileScanOperation>> scanOperations)
|
||||
{
|
||||
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ProcessScanResults");
|
||||
|
||||
db::Session& dbSession{ _db.getTLSSession() };
|
||||
auto transaction{ dbSession.createWriteTransaction() };
|
||||
|
||||
for (auto& scanOperation : scanOperations)
|
||||
{
|
||||
if (_abortScan)
|
||||
return;
|
||||
|
||||
scanOperation->processResult(context);
|
||||
context.stats.scans++;
|
||||
}
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
+6
-17
@@ -21,16 +21,15 @@
|
||||
|
||||
#include <filesystem>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "metadata/IParser.hpp"
|
||||
|
||||
#include "FileScanQueue.hpp"
|
||||
#include "ScanStepBase.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class IFileScanner;
|
||||
class MediaLibraryInfo;
|
||||
|
||||
class ScanStepScanFiles : public ScanStepBase
|
||||
{
|
||||
public:
|
||||
@@ -40,20 +39,10 @@ namespace lms::scanner
|
||||
ScanStep getStep() const override { return ScanStep::ScanFiles; }
|
||||
core::LiteralString getStepName() const override { return "Scan files"; }
|
||||
void process(ScanContext& context) override;
|
||||
void process(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary);
|
||||
|
||||
bool checkAudioFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo);
|
||||
bool checkImageFileNeedScan(ScanContext& context, const std::filesystem::path& file);
|
||||
bool checkLyricsFileNeedScan(ScanContext& context, const std::filesystem::path& file);
|
||||
|
||||
void processFileScanResults(ScanContext& context, std::span<const FileScanResult> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo);
|
||||
void processAudioFileScanData(ScanContext& context, const std::filesystem::path& file, const metadata::Track* trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo);
|
||||
void processImageFileScanData(ScanContext& context, const std::filesystem::path& file, const ImageInfo* imageInfo, const ScannerSettings::MediaLibraryInfo& libraryInfo);
|
||||
void processLyricsFileScanData(ScanContext& context, const std::filesystem::path& file, const metadata::Lyrics* lyrics, const ScannerSettings::MediaLibraryInfo& libraryInfo);
|
||||
|
||||
std::unique_ptr<metadata::IParser> _metadataParser;
|
||||
const std::vector<std::string> _extraTagsToParse;
|
||||
void process(ScanContext& context, const MediaLibraryInfo& mediaLibrary);
|
||||
void processFileScanResults(ScanContext& context, std::span<std::unique_ptr<IFileScanOperation>> scanOperations);
|
||||
|
||||
FileScanQueue _fileScanQueue;
|
||||
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByExtension;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
+5
-2
@@ -24,6 +24,9 @@
|
||||
#include "database/MediaLibrary.hpp"
|
||||
#include "database/Session.hpp"
|
||||
|
||||
#include "MediaLibraryInfo.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
|
||||
@@ -34,7 +37,7 @@ namespace lms::scanner
|
||||
|
||||
void ScanStepUpdateLibraryFields::processDirectories(ScanContext& context)
|
||||
{
|
||||
for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
|
||||
for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
|
||||
{
|
||||
if (_abortScan)
|
||||
break;
|
||||
@@ -43,7 +46,7 @@ namespace lms::scanner
|
||||
}
|
||||
}
|
||||
|
||||
void ScanStepUpdateLibraryFields::processDirectory(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary)
|
||||
void ScanStepUpdateLibraryFields::processDirectory(ScanContext& context, const MediaLibraryInfo& mediaLibrary)
|
||||
{
|
||||
db::Session& session{ _db.getTLSSession() };
|
||||
|
||||
+3
-1
@@ -23,6 +23,8 @@
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class MediaLibraryInfo;
|
||||
|
||||
class ScanStepUpdateLibraryFields : public ScanStepBase
|
||||
{
|
||||
public:
|
||||
@@ -34,6 +36,6 @@ namespace lms::scanner
|
||||
void process(ScanContext& context) override;
|
||||
|
||||
void processDirectories(ScanContext& context);
|
||||
void processDirectory(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary);
|
||||
void processDirectory(ScanContext& context, const MediaLibraryInfo& mediaLibrary);
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -30,12 +30,13 @@ namespace lms::scanner
|
||||
{
|
||||
enum class ScanErrorType
|
||||
{
|
||||
CannotReadFile, // cannot read file
|
||||
CannotReadAudioFile, // cannot parse audio file
|
||||
CannotReadImageFile, // cannot parse image file
|
||||
CannotReadLyricsFile, // cannot parse lyrics file
|
||||
NoAudioTrack, // no audio track found
|
||||
BadDuration, // bad duration
|
||||
CannotReadFile,
|
||||
CannotReadAudioFile,
|
||||
CannotReadImageFile,
|
||||
CannotReadLyricsFile,
|
||||
CannotReadPlayListFile,
|
||||
NoAudioTrack,
|
||||
BadDuration,
|
||||
};
|
||||
|
||||
enum class DuplicateReason
|
||||
@@ -64,6 +65,7 @@ namespace lms::scanner
|
||||
{
|
||||
AssociateArtistImages,
|
||||
AssociateExternalLyrics,
|
||||
AssociatePlayListTracks,
|
||||
AssociateReleaseImages,
|
||||
CheckForDuplicatedFiles,
|
||||
CheckForRemovedFiles,
|
||||
|
||||
Reference in New Issue
Block a user