Scanner: parallelized processing within artwork association steps
This commit is contained in:
emeric
@@ -448,7 +448,7 @@ namespace lms::scanner
|
||||
if (!_lastScanSettings)
|
||||
_lastScanSettings = readScannerSettings(_db.getTLSSession(), lastScanSettingsName);
|
||||
|
||||
auto cbFunc{ [this](const ScanStepStats& stats) {
|
||||
auto progressFunc{ [this](const ScanStepStats& stats) {
|
||||
notifyInProgressIfNeeded(stats);
|
||||
} };
|
||||
|
||||
@@ -466,7 +466,7 @@ namespace lms::scanner
|
||||
.jobScheduler = *_jobScheduler,
|
||||
.settings = _settings,
|
||||
.lastScanSettings = _lastScanSettings.has_value() ? &(_lastScanSettings.value()) : nullptr,
|
||||
.progressCallback = cbFunc,
|
||||
.progressCallback = progressFunc,
|
||||
.abortScan = _abortScan,
|
||||
.db = _db,
|
||||
.fileScanners = fileScanners,
|
||||
@@ -500,14 +500,14 @@ namespace lms::scanner
|
||||
_currentScanStepStats = stepStats;
|
||||
}
|
||||
|
||||
const std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() };
|
||||
const auto now{ std::chrono::steady_clock::now() };
|
||||
_events.scanInProgress.emit(stepStats);
|
||||
_lastScanInProgressEmit = now;
|
||||
}
|
||||
|
||||
void ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
|
||||
{
|
||||
std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() };
|
||||
const auto now{ std::chrono::steady_clock::now() };
|
||||
|
||||
if (now - _lastScanInProgressEmit >= std::chrono::seconds{ 1 })
|
||||
notifyInProgress(stepStats);
|
||||
|
||||
@@ -96,7 +96,7 @@ namespace lms::scanner
|
||||
Wt::WIOService _ioService;
|
||||
boost::asio::system_timer _scheduleTimer{ _ioService };
|
||||
Events _events;
|
||||
std::chrono::system_clock::time_point _lastScanInProgressEmit;
|
||||
std::chrono::steady_clock::time_point _lastScanInProgressEmit;
|
||||
|
||||
mutable std::shared_mutex _statusMutex;
|
||||
State _curState{ State::NotScheduled };
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
#include "core/Path.hpp"
|
||||
#include "core/XxHash3.hpp"
|
||||
#include "database/Artist.hpp"
|
||||
#include "database/Artwork.hpp"
|
||||
#include "database/Cluster.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Directory.hpp"
|
||||
@@ -332,6 +333,8 @@ namespace lms::scanner
|
||||
image.modify()->setWidth(imageInfo.properties.width);
|
||||
image.modify()->setHeight(imageInfo.properties.height);
|
||||
image.modify()->setMimeType(imageInfo.mimeType);
|
||||
|
||||
session.create<db::Artwork>(image);
|
||||
}
|
||||
|
||||
return image;
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
#include <optional>
|
||||
|
||||
#include "core/ILogger.hpp"
|
||||
#include "database/Artwork.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Image.hpp"
|
||||
#include "database/MediaLibrary.hpp"
|
||||
@@ -84,7 +85,10 @@ namespace lms::scanner
|
||||
|
||||
const bool added{ !image };
|
||||
if (!image)
|
||||
{
|
||||
image = dbSession.create<db::Image>(getFilePath());
|
||||
dbSession.create<db::Artwork>(image);
|
||||
}
|
||||
|
||||
image.modify()->setLastWriteTime(getLastWriteTime());
|
||||
image.modify()->setFileSize(getFileSize());
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "ArtworkUtils.hpp"
|
||||
|
||||
#include "database/Artwork.hpp"
|
||||
#include "database/Image.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackEmbeddedImage.hpp"
|
||||
|
||||
namespace lms::scanner::utils
|
||||
{
|
||||
db::ObjectPtr<db::Artwork> getOrCreateArtworkFromTrackEmbeddedImage(db::Session& session, db::TrackEmbeddedImageId trackEmbeddedImageId)
|
||||
{
|
||||
assert(trackEmbeddedImageId.isValid());
|
||||
session.checkWriteTransaction();
|
||||
|
||||
db::ObjectPtr<db::Artwork> artwork{ db::Artwork::find(session, trackEmbeddedImageId) };
|
||||
if (!artwork)
|
||||
{
|
||||
db::TrackEmbeddedImage::pointer trackEmbeddedImage{ db::TrackEmbeddedImage::find(session, trackEmbeddedImageId) };
|
||||
assert(trackEmbeddedImage);
|
||||
artwork = session.create<db::Artwork>(trackEmbeddedImage);
|
||||
}
|
||||
return artwork;
|
||||
}
|
||||
|
||||
db::ObjectPtr<db::Artwork> getOrCreateArtworkFromImage(db::Session& session, db::ImageId imageId)
|
||||
{
|
||||
assert(imageId.isValid());
|
||||
session.checkWriteTransaction();
|
||||
|
||||
db::ObjectPtr<db::Artwork> artwork{ db::Artwork::find(session, imageId) };
|
||||
if (!artwork)
|
||||
{
|
||||
db::Image::pointer image{ db::Image::find(session, imageId) };
|
||||
assert(image);
|
||||
artwork = session.create<db::Artwork>(image);
|
||||
}
|
||||
return artwork;
|
||||
}
|
||||
} // namespace lms::scanner::utils
|
||||
@@ -1,39 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
#include "database/ArtworkId.hpp"
|
||||
#include "database/ImageId.hpp"
|
||||
#include "database/Object.hpp"
|
||||
#include "database/TrackEmbeddedImageId.hpp"
|
||||
|
||||
namespace lms::db
|
||||
{
|
||||
class Artwork;
|
||||
class Session;
|
||||
} // namespace lms::db
|
||||
|
||||
namespace lms::scanner::utils
|
||||
{
|
||||
db::ObjectPtr<db::Artwork> getOrCreateArtworkFromTrackEmbeddedImage(db::Session& session, db::TrackEmbeddedImageId trackEmbeddedImageId);
|
||||
db::ObjectPtr<db::Artwork> getOrCreateArtworkFromImage(db::Session& session, db::ImageId imageId);
|
||||
} // namespace lms::scanner::utils
|
||||
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright (C) 2025 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "JobQueue.hpp"
|
||||
|
||||
#include <cassert>
|
||||
|
||||
#include "core/IJob.hpp"
|
||||
#include "core/IJobScheduler.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
JobQueue::JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t batchSize, float _drainThreshold)
|
||||
: _scheduler{ scheduler }
|
||||
, _maxQueueSize{ maxQueueSize }
|
||||
, _processJobsDoneFunc{ std::move(processJobsDoneFunc) }
|
||||
, _batchSize{ batchSize }
|
||||
, _drainThreshold{ _drainThreshold }
|
||||
{
|
||||
assert(_scheduler.getJobsDoneCount() == 0);
|
||||
}
|
||||
|
||||
JobQueue::~JobQueue()
|
||||
{
|
||||
finish();
|
||||
}
|
||||
|
||||
void JobQueue::push(std::unique_ptr<core::IJob> job)
|
||||
{
|
||||
_scheduler.scheduleJob(std::move(job));
|
||||
drainIfNeeded();
|
||||
}
|
||||
|
||||
void JobQueue::finish()
|
||||
{
|
||||
_scheduler.wait();
|
||||
while (_scheduler.popJobsDone(_jobsDone, _batchSize) > 0)
|
||||
{
|
||||
_processJobsDoneFunc(std::span{ _jobsDone });
|
||||
_jobsDone.clear();
|
||||
}
|
||||
}
|
||||
|
||||
void JobQueue::drainIfNeeded()
|
||||
{
|
||||
while (_scheduler.getJobsDoneCount() > static_cast<std::size_t>(_maxQueueSize * _drainThreshold))
|
||||
{
|
||||
_scheduler.popJobsDone(_jobsDone, _batchSize);
|
||||
_processJobsDoneFunc(std::span{ _jobsDone });
|
||||
_jobsDone.clear();
|
||||
}
|
||||
_scheduler.waitUntilJobCountAtMost(_maxQueueSize);
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright (C) 2025 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
namespace lms::core
|
||||
{
|
||||
class IJob;
|
||||
class IJobScheduler;
|
||||
} // namespace lms::core
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class JobQueue
|
||||
{
|
||||
public:
|
||||
using ProcessFunction = std::function<void(std::span<std::unique_ptr<core::IJob>>)>;
|
||||
|
||||
JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t batchSize, float drainThreshold);
|
||||
~JobQueue();
|
||||
JobQueue(const JobQueue&) = delete;
|
||||
JobQueue& operator=(const JobQueue&) = delete;
|
||||
|
||||
// push can wait and invoke the supplied ProcessFunction
|
||||
void push(std::unique_ptr<core::IJob> job);
|
||||
void finish();
|
||||
|
||||
private:
|
||||
void drainIfNeeded();
|
||||
|
||||
core::IJobScheduler& _scheduler;
|
||||
const std::size_t _maxQueueSize;
|
||||
ProcessFunction _processJobsDoneFunc;
|
||||
const std::size_t _batchSize;
|
||||
const float _drainThreshold;
|
||||
std::vector<std::unique_ptr<core::IJob>> _jobsDone;
|
||||
};
|
||||
|
||||
} // namespace lms::scanner
|
||||
@@ -24,9 +24,9 @@
|
||||
#include <deque>
|
||||
#include <set>
|
||||
#include <span>
|
||||
#include <variant>
|
||||
|
||||
#include "core/IConfig.hpp"
|
||||
#include "core/IJob.hpp"
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/Path.hpp"
|
||||
#include "core/String.hpp"
|
||||
@@ -41,7 +41,7 @@
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
|
||||
#include "ArtworkUtils.hpp"
|
||||
#include "JobQueue.hpp"
|
||||
#include "ScanContext.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
|
||||
@@ -49,42 +49,24 @@ namespace lms::scanner
|
||||
{
|
||||
namespace
|
||||
{
|
||||
using ArtistArtwork = std::variant<std::monostate, db::ImageId, db::ArtworkId>; // TODO handle embedded images in tracks?
|
||||
bool isSameArtwork(ArtistArtwork preferredArtwork, const db::ObjectPtr<db::Artwork>& artwork)
|
||||
{
|
||||
if (std::holds_alternative<std::monostate>(preferredArtwork))
|
||||
return !artwork;
|
||||
|
||||
if (const db::ImageId* imageId = std::get_if<db::ImageId>(&preferredArtwork))
|
||||
return artwork && *imageId == artwork->getImageId();
|
||||
|
||||
if (const db::ArtworkId* artworkId = std::get_if<db::ArtworkId>(&preferredArtwork))
|
||||
return artwork && *artworkId == artwork->getId();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
struct ArtistArtworkAssociation
|
||||
{
|
||||
db::Artist::pointer artist;
|
||||
ArtistArtwork preferredArtwork;
|
||||
db::ArtistId artistId;
|
||||
db::ArtworkId preferredArtworkId;
|
||||
};
|
||||
using ArtistArtworkAssociationContainer = std::deque<ArtistArtworkAssociation>;
|
||||
|
||||
struct SearchArtistImageContext
|
||||
struct SearchArtistArtworkParams
|
||||
{
|
||||
db::Session& session;
|
||||
db::ArtistId lastRetrievedArtistId;
|
||||
std::size_t processedArtistCount{};
|
||||
std::span<const std::string> artistFileNames;
|
||||
const ScannerSettings& settings;
|
||||
};
|
||||
|
||||
db::Image::pointer findImageInDirectory(SearchArtistImageContext& searchContext, const std::filesystem::path& directoryPath, std::span<const std::string> fileStemsToSearch)
|
||||
db::Image::pointer findImageInDirectory(db::Session& session, const std::filesystem::path& directoryPath, std::span<const std::string> fileStemsToSearch)
|
||||
{
|
||||
db::Image::pointer image;
|
||||
|
||||
const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) };
|
||||
const db::Directory::pointer directory{ db::Directory::find(session, directoryPath) };
|
||||
if (directory) // may not exist for artists that are split on different media libraries
|
||||
{
|
||||
for (std::string_view fileStem : fileStemsToSearch)
|
||||
@@ -93,7 +75,7 @@ namespace lms::scanner
|
||||
params.setDirectory(directory->getId());
|
||||
params.setFileStem(fileStem);
|
||||
|
||||
db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) {
|
||||
db::Image::find(session, params, [&](const db::Image::pointer foundImg) {
|
||||
if (!image)
|
||||
image = foundImg;
|
||||
});
|
||||
@@ -106,38 +88,38 @@ namespace lms::scanner
|
||||
return image;
|
||||
}
|
||||
|
||||
db::ImageId getImageFromMbid(SearchArtistImageContext& searchContext, const core::UUID& mbid)
|
||||
db::Image::pointer getImageFromMbid(db::Session& session, const core::UUID& mbid)
|
||||
{
|
||||
db::Image::pointer image;
|
||||
|
||||
// Find anywhere, since it is supposed to be unique!
|
||||
db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid.getAsString()), [&](const db::Image::pointer foundImg) {
|
||||
db::Image::find(session, db::Image::FindParameters{}.setFileStem(mbid.getAsString()), [&](const db::Image::pointer foundImg) {
|
||||
if (!image)
|
||||
image = foundImg;
|
||||
});
|
||||
|
||||
return image ? image->getId() : db::ImageId{};
|
||||
return image;
|
||||
}
|
||||
|
||||
db::ImageId searchImageInArtistInfoDirectory(SearchArtistImageContext& searchContext, db::ArtistId artistId)
|
||||
db::Image::pointer searchImageInArtistInfoDirectory(db::Session& session, db::ArtistId artistId)
|
||||
{
|
||||
db::Image::pointer image;
|
||||
|
||||
std::vector<std::string> fileInfoPaths;
|
||||
db::ArtistInfo::find(searchContext.session, artistId, [&](const db::ArtistInfo::pointer& artistInfo) {
|
||||
db::ArtistInfo::find(session, artistId, [&](const db::ArtistInfo::pointer& artistInfo) {
|
||||
fileInfoPaths.push_back(artistInfo->getAbsoluteFilePath());
|
||||
|
||||
if (!image)
|
||||
image = findImageInDirectory(searchContext, artistInfo->getDirectory()->getAbsolutePath(), std::array<std::string, 2>{ "thumb", "folder" });
|
||||
image = findImageInDirectory(session, artistInfo->getDirectory()->getAbsolutePath(), std::array<std::string, 2>{ "thumb", "folder" });
|
||||
});
|
||||
|
||||
if (fileInfoPaths.size() > 1)
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Found " << fileInfoPaths.size() << " artist info files for same artist: " << core::stringUtils::joinStrings(fileInfoPaths, ", "));
|
||||
|
||||
return image ? image->getId() : db::ImageId{};
|
||||
return image;
|
||||
}
|
||||
|
||||
db::ImageId searchImageInDirectories(SearchArtistImageContext& searchContext, db::ArtistId artistId)
|
||||
db::Image::pointer searchImageInDirectories(db::Session& session, const SearchArtistArtworkParams& searchParams, db::ArtistId artistId)
|
||||
{
|
||||
db::Image::pointer image;
|
||||
|
||||
@@ -145,7 +127,7 @@ namespace lms::scanner
|
||||
db::Directory::FindParameters params;
|
||||
params.setArtist(artistId, { db::TrackArtistLinkType::ReleaseArtist });
|
||||
|
||||
db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) {
|
||||
db::Directory::find(session, params, [&](const db::Directory::pointer& directory) {
|
||||
releasePaths.insert(directory->getAbsolutePath());
|
||||
});
|
||||
|
||||
@@ -164,9 +146,9 @@ namespace lms::scanner
|
||||
std::filesystem::path directoryToInspect{ core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) };
|
||||
while (true)
|
||||
{
|
||||
image = findImageInDirectory(searchContext, directoryToInspect, searchContext.artistFileNames);
|
||||
image = findImageInDirectory(session, directoryToInspect, searchParams.artistFileNames);
|
||||
if (image)
|
||||
return image->getId();
|
||||
return image;
|
||||
|
||||
std::filesystem::path parentPath{ directoryToInspect.parent_path() };
|
||||
if (parentPath == directoryToInspect)
|
||||
@@ -181,106 +163,74 @@ namespace lms::scanner
|
||||
// /someOtherUserConfiguredArtistFile.jpg
|
||||
for (const std::filesystem::path& releasePath : releasePaths)
|
||||
{
|
||||
image = findImageInDirectory(searchContext, releasePath, searchContext.artistFileNames);
|
||||
image = findImageInDirectory(session, releasePath, searchParams.artistFileNames);
|
||||
if (image)
|
||||
return image->getId();
|
||||
return image;
|
||||
}
|
||||
}
|
||||
|
||||
return image ? image->getId() : db::ImageId{};
|
||||
return image;
|
||||
}
|
||||
|
||||
db::ArtworkId getFirstReleaseArtwork(db::Session& session, const db::Artist::pointer& artist)
|
||||
db::Artwork::pointer getFirstReleaseArtwork(db::Session& session, const db::Artist::pointer& artist)
|
||||
{
|
||||
db::ArtworkId artworkId;
|
||||
db::Artwork::pointer artwork;
|
||||
|
||||
db::Release::FindParameters params;
|
||||
params.setArtist(artist->getId(), { db::TrackArtistLinkType::ReleaseArtist });
|
||||
params.setSortMethod(db::ReleaseSortMethod::OriginalDate);
|
||||
|
||||
db::Release::find(session, params, [&](const db::Release::pointer& release) {
|
||||
if (artworkId.isValid())
|
||||
if (artwork)
|
||||
return;
|
||||
|
||||
artworkId = release->getPreferredArtworkId();
|
||||
artwork = release->getPreferredArtwork();
|
||||
});
|
||||
|
||||
return artworkId;
|
||||
return artwork;
|
||||
}
|
||||
|
||||
ArtistArtwork computePreferredArtwork(SearchArtistImageContext& searchContext, const db::Artist::pointer& artist)
|
||||
db::Artwork::pointer computePreferredArtistArtwork(db::Session& session, const SearchArtistArtworkParams& searchParams, const db::Artist::pointer& artist)
|
||||
{
|
||||
db::ImageId imageId;
|
||||
|
||||
if (const auto mbid{ artist->getMBID() })
|
||||
{
|
||||
imageId = getImageFromMbid(searchContext, *mbid);
|
||||
if (imageId.isValid())
|
||||
return ArtistArtwork{ imageId };
|
||||
const db::Image::pointer image{ getImageFromMbid(session, *mbid) };
|
||||
if (image)
|
||||
return db::Artwork::find(session, image->getId());
|
||||
}
|
||||
|
||||
imageId = searchImageInArtistInfoDirectory(searchContext, artist->getId());
|
||||
if (imageId.isValid())
|
||||
return ArtistArtwork{ imageId };
|
||||
|
||||
imageId = searchImageInDirectories(searchContext, artist->getId());
|
||||
if (imageId.isValid())
|
||||
return ArtistArtwork{ imageId };
|
||||
|
||||
if (searchContext.settings.artistImageFallbackToRelease)
|
||||
{
|
||||
db::ArtworkId artworkId{ getFirstReleaseArtwork(searchContext.session, artist) };
|
||||
if (artworkId.isValid())
|
||||
return ArtistArtwork{ artworkId };
|
||||
}
|
||||
|
||||
return ArtistArtwork{};
|
||||
}
|
||||
|
||||
bool fetchNextArtistArtworksToUpdate(SearchArtistImageContext& searchContext, ArtistArtworkAssociationContainer& ArtistArtworkAssociations)
|
||||
{
|
||||
const db::ArtistId artistId{ searchContext.lastRetrievedArtistId };
|
||||
|
||||
{
|
||||
constexpr std::size_t readBatchSize{ 100 };
|
||||
|
||||
auto transaction{ searchContext.session.createReadTransaction() };
|
||||
|
||||
db::Artist::find(searchContext.session, searchContext.lastRetrievedArtistId, readBatchSize, [&](const db::Artist::pointer& artist) {
|
||||
ArtistArtwork preferredArtwork{ computePreferredArtwork(searchContext, artist) };
|
||||
|
||||
if (!isSameArtwork(preferredArtwork, artist->getPreferredArtwork()))
|
||||
ArtistArtworkAssociations.push_back(ArtistArtworkAssociation{ artist, preferredArtwork });
|
||||
|
||||
searchContext.processedArtistCount++;
|
||||
});
|
||||
const db::Image::pointer image{ searchImageInArtistInfoDirectory(session, artist->getId()) };
|
||||
if (image)
|
||||
return db::Artwork::find(session, image->getId());
|
||||
}
|
||||
|
||||
return artistId != searchContext.lastRetrievedArtistId;
|
||||
{
|
||||
const db::Image::pointer image{ searchImageInDirectories(session, searchParams, artist->getId()) };
|
||||
if (image)
|
||||
return db::Artwork::find(session, image->getId());
|
||||
}
|
||||
|
||||
if (searchParams.settings.artistImageFallbackToRelease)
|
||||
{
|
||||
const db::Artwork::pointer artwork{ getFirstReleaseArtwork(session, artist) };
|
||||
if (artwork)
|
||||
return artwork;
|
||||
}
|
||||
|
||||
return db::Artwork::pointer{};
|
||||
}
|
||||
|
||||
void updateArtistPreferredArtwork(db::Session& session, const ArtistArtworkAssociation& ArtistArtworkAssociation)
|
||||
void updateArtistPreferredArtwork(db::Session& session, const ArtistArtworkAssociation& artistArtworkAssociation)
|
||||
{
|
||||
db::Artist::pointer artist{ ArtistArtworkAssociation.artist };
|
||||
|
||||
db::Artwork::pointer artwork;
|
||||
if (const db::ImageId * imageId{ std::get_if<db::ImageId>(&ArtistArtworkAssociation.preferredArtwork) })
|
||||
artwork = utils::getOrCreateArtworkFromImage(session, *imageId);
|
||||
else if (const db::ArtworkId * artworkId{ std::get_if<db::ArtworkId>(&ArtistArtworkAssociation.preferredArtwork) })
|
||||
artwork = db::Artwork::find(session, *artworkId);
|
||||
|
||||
artist.modify()->setPreferredArtwork(artwork);
|
||||
if (artwork)
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork for artist '" << artist->getName() << "' with image in " << artwork->getAbsoluteFilePath());
|
||||
else
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from artist '" << artist->getName() << "'");
|
||||
db::Artist::updatePreferredArtwork(session, artistArtworkAssociation.artistId, artistArtworkAssociation.preferredArtworkId);
|
||||
}
|
||||
|
||||
void updateArtistArtworks(db::Session& session, ArtistArtworkAssociationContainer& imageAssociations)
|
||||
void updateArtistPreferredArtworks(db::Session& session, ArtistArtworkAssociationContainer& imageAssociations, bool forceFullBatch)
|
||||
{
|
||||
constexpr std::size_t writeBatchSize{ 50 };
|
||||
|
||||
while (!imageAssociations.empty())
|
||||
while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty())
|
||||
{
|
||||
auto transaction{ session.createWriteTransaction() };
|
||||
|
||||
@@ -305,6 +255,62 @@ namespace lms::scanner
|
||||
return res;
|
||||
}
|
||||
|
||||
bool fetchNextArtistIdRange(db::Session& session, db::ArtistId& lastRetrievedId, db::IdRange<db::ArtistId>& idRange)
|
||||
{
|
||||
constexpr std::size_t readBatchSize{ 100 };
|
||||
|
||||
auto transaction{ session.createReadTransaction() };
|
||||
|
||||
idRange = db::Artist::findNextRange(session, lastRetrievedId, readBatchSize);
|
||||
lastRetrievedId = idRange.last;
|
||||
|
||||
return idRange.isValid();
|
||||
}
|
||||
|
||||
class ComputeArtistArtworkAssociationsJob : public core::IJob
|
||||
{
|
||||
public:
|
||||
ComputeArtistArtworkAssociationsJob(db::Db& db, const SearchArtistArtworkParams& searchParams, db::IdRange<db::ArtistId> artistIdRange)
|
||||
: _db{ db }
|
||||
, _searchParams{ searchParams }
|
||||
, _artistIdRange{ artistIdRange }
|
||||
{
|
||||
}
|
||||
|
||||
std::span<const ArtistArtworkAssociation> getAssociations() const { return _associations; }
|
||||
std::size_t getProcessedArtistCount() const { return _processedArtistCount; }
|
||||
|
||||
private:
|
||||
core::LiteralString getName() const override { return "Associate Track Artworks"; }
|
||||
void run() override
|
||||
{
|
||||
auto& session{ _db.getTLSSession() };
|
||||
auto transaction{ session.createReadTransaction() };
|
||||
|
||||
db::Artist::find(session, _artistIdRange, [this, &session](const db::Artist::pointer& artist) {
|
||||
const db::Artwork::pointer preferredArtwork{ computePreferredArtistArtwork(session, _searchParams, artist) };
|
||||
|
||||
if (artist->getPreferredArtwork() != preferredArtwork)
|
||||
{
|
||||
_associations.push_back(ArtistArtworkAssociation{ artist->getId(), preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{} });
|
||||
|
||||
if (preferredArtwork)
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork for artist '" << artist->getName() << "' with image in " << preferredArtwork->getAbsoluteFilePath());
|
||||
else
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from artist '" << artist->getName() << "'");
|
||||
}
|
||||
|
||||
_processedArtistCount++;
|
||||
});
|
||||
}
|
||||
|
||||
db::Db& _db;
|
||||
const SearchArtistArtworkParams& _searchParams;
|
||||
db::IdRange<db::ArtistId> _artistIdRange;
|
||||
std::vector<ArtistArtworkAssociation> _associations;
|
||||
std::size_t _processedArtistCount{};
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
ScanStepAssociateArtistImages::ScanStepAssociateArtistImages(InitParams& initParams)
|
||||
@@ -333,22 +339,40 @@ namespace lms::scanner
|
||||
context.currentStepStats.totalElems = db::Artist::getCount(session);
|
||||
}
|
||||
|
||||
SearchArtistImageContext searchContext{
|
||||
.session = session,
|
||||
.lastRetrievedArtistId = {},
|
||||
const SearchArtistArtworkParams searchParams{
|
||||
.artistFileNames = _artistFileNames,
|
||||
.settings = _settings,
|
||||
};
|
||||
|
||||
ArtistArtworkAssociationContainer ArtistArtworkAssociations;
|
||||
while (fetchNextArtistArtworksToUpdate(searchContext, ArtistArtworkAssociations))
|
||||
{
|
||||
ArtistArtworkAssociationContainer artistArtworkAssociations;
|
||||
auto processJobsDone = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
|
||||
if (_abortScan)
|
||||
return;
|
||||
|
||||
updateArtistArtworks(session, ArtistArtworkAssociations);
|
||||
context.currentStepStats.processedElems = searchContext.processedArtistCount;
|
||||
for (const auto& job : jobs)
|
||||
{
|
||||
const auto& associationJob{ static_cast<const ComputeArtistArtworkAssociationsJob&>(*job) };
|
||||
const auto& artistAssociations{ associationJob.getAssociations() };
|
||||
|
||||
artistArtworkAssociations.insert(std::end(artistArtworkAssociations), std::cbegin(artistAssociations), std::cend(artistAssociations));
|
||||
|
||||
context.currentStepStats.processedElems += associationJob.getProcessedArtistCount();
|
||||
}
|
||||
|
||||
updateArtistPreferredArtworks(session, artistArtworkAssociations, true);
|
||||
_progressCallback(context.currentStepStats);
|
||||
}
|
||||
};
|
||||
|
||||
JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
|
||||
|
||||
db::ArtistId lastRetrievedArtistId{};
|
||||
db::IdRange<db::ArtistId> artistIdRange;
|
||||
while (fetchNextArtistIdRange(session, lastRetrievedArtistId, artistIdRange))
|
||||
queue.push(std::make_unique<ComputeArtistArtworkAssociationsJob>(_db, searchParams, artistIdRange));
|
||||
|
||||
queue.finish();
|
||||
|
||||
// process all remaining associations
|
||||
updateArtistPreferredArtworks(session, artistArtworkAssociations, false);
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
|
||||
@@ -23,9 +23,9 @@
|
||||
#include <cassert>
|
||||
#include <deque>
|
||||
#include <set>
|
||||
#include <variant>
|
||||
|
||||
#include "core/IConfig.hpp"
|
||||
#include "core/IJob.hpp"
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/Path.hpp"
|
||||
#include "database/Artwork.hpp"
|
||||
@@ -36,92 +36,73 @@
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackEmbeddedImage.hpp"
|
||||
#include "database/TrackEmbeddedImageId.hpp"
|
||||
|
||||
#include "ArtworkUtils.hpp"
|
||||
#include "JobQueue.hpp"
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
namespace
|
||||
{
|
||||
using ReleaseArtwork = std::variant<std::monostate, db::TrackEmbeddedImageId, db::ImageId>;
|
||||
bool isSameArtwork(ReleaseArtwork preferredArtwork, const db::ObjectPtr<db::Artwork>& artwork)
|
||||
{
|
||||
if (std::holds_alternative<std::monostate>(preferredArtwork))
|
||||
return !artwork;
|
||||
|
||||
if (const db::TrackEmbeddedImageId* trackEmbeddedImageId = std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork))
|
||||
return artwork && *trackEmbeddedImageId == artwork->getTrackEmbeddedImageId();
|
||||
|
||||
if (const db::ImageId* imageId = std::get_if<db::ImageId>(&preferredArtwork))
|
||||
return artwork && *imageId == artwork->getImageId();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
struct ReleaseArtworkAssociation
|
||||
{
|
||||
db::Release::pointer release;
|
||||
ReleaseArtwork preferredArtwork;
|
||||
db::ReleaseId releaseId;
|
||||
db::ArtworkId preferredArtworkId;
|
||||
};
|
||||
using ReleaseArtworkAssociationContainer = std::deque<ReleaseArtworkAssociation>;
|
||||
|
||||
struct SearchReleaseArtworkContext
|
||||
struct SearchReleaseArtworkParams
|
||||
{
|
||||
db::Session& session;
|
||||
db::ReleaseId lastRetrievedReleaseId;
|
||||
std::size_t processedReleaseCount{};
|
||||
const std::vector<std::string>& releaseImageFileNames;
|
||||
};
|
||||
|
||||
db::Image::pointer findImageInDirectory(SearchReleaseArtworkContext& searchContext, const std::filesystem::path& directoryPath)
|
||||
db::Artwork::pointer findImageInDirectory(db::Session& session, const SearchReleaseArtworkParams& searchParams, const std::filesystem::path& directoryPath)
|
||||
{
|
||||
db::Image::pointer image;
|
||||
db::Artwork::pointer artwork;
|
||||
|
||||
const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) };
|
||||
const db::Directory::pointer directory{ db::Directory::find(session, directoryPath) };
|
||||
if (directory) // may not exist for releases that are split on different media libraries
|
||||
{
|
||||
for (std::string_view fileStem : searchContext.releaseImageFileNames)
|
||||
for (std::string_view fileStem : searchParams.releaseImageFileNames)
|
||||
{
|
||||
db::Image::FindParameters params;
|
||||
params.setDirectory(directory->getId());
|
||||
params.setFileStem(fileStem);
|
||||
|
||||
db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) {
|
||||
if (!image)
|
||||
image = foundImg;
|
||||
db::Image::find(session, params, [&](const db::Image::pointer& image) {
|
||||
if (!artwork)
|
||||
artwork = db::Artwork::find(session, image->getId());
|
||||
});
|
||||
|
||||
if (image)
|
||||
if (artwork)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return image;
|
||||
return artwork;
|
||||
}
|
||||
|
||||
db::Image::pointer computePreferredReleaseImage(SearchReleaseArtworkContext& searchContext, const db::Release::pointer& release)
|
||||
db::Artwork::pointer computePreferredReleaseImage(db::Session& session, const SearchReleaseArtworkParams& searchParams, const db::Release::pointer& release)
|
||||
{
|
||||
db::Image::pointer image;
|
||||
db::Artwork::pointer artwork;
|
||||
|
||||
const auto mbid{ release->getMBID() };
|
||||
if (mbid)
|
||||
{
|
||||
// Find anywhere, since it is suppoed to be unique!
|
||||
db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer foundImg) {
|
||||
if (!image)
|
||||
image = foundImg;
|
||||
db::Image::find(session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer& image) {
|
||||
if (!artwork)
|
||||
artwork = db::Artwork::find(session, image->getId());
|
||||
});
|
||||
}
|
||||
|
||||
if (!image)
|
||||
if (!artwork)
|
||||
{
|
||||
std::set<std::filesystem::path> releasePaths;
|
||||
db::Directory::FindParameters params;
|
||||
params.setRelease(release->getId());
|
||||
|
||||
db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) {
|
||||
db::Directory::find(session, params, [&](const db::Directory::pointer& directory) {
|
||||
releasePaths.insert(directory->getAbsolutePath());
|
||||
});
|
||||
|
||||
@@ -132,31 +113,30 @@ namespace lms::scanner
|
||||
if (releasePaths.size() > 1)
|
||||
{
|
||||
const std::filesystem::path releasePath{ core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) };
|
||||
image = findImageInDirectory(searchContext, releasePath);
|
||||
artwork = findImageInDirectory(session, searchParams, releasePath);
|
||||
}
|
||||
|
||||
if (!image)
|
||||
if (!artwork)
|
||||
{
|
||||
for (const std::filesystem::path& releasePath : releasePaths)
|
||||
{
|
||||
image = findImageInDirectory(searchContext, releasePath);
|
||||
if (image)
|
||||
artwork = findImageInDirectory(session, searchParams, releasePath);
|
||||
if (artwork)
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return image;
|
||||
return artwork;
|
||||
}
|
||||
|
||||
ReleaseArtwork computePreferredReleaseArtwork(SearchReleaseArtworkContext& searchContext, const db::Release::pointer& release)
|
||||
db::Artwork::pointer computePreferredReleaseArtwork(db::Session& session, const SearchReleaseArtworkParams& searchParams, const db::Release::pointer& release)
|
||||
{
|
||||
const db::Image::pointer image{ computePreferredReleaseImage(searchContext, release) };
|
||||
if (image)
|
||||
return ReleaseArtwork{ image->getId() };
|
||||
db::Artwork::pointer artwork{ computePreferredReleaseImage(session, searchParams, release) };
|
||||
if (artwork)
|
||||
return artwork;
|
||||
|
||||
// Fallback on embedded Front image
|
||||
db::TrackEmbeddedImageId trackEmbeddedImageId;
|
||||
{
|
||||
db::TrackEmbeddedImage::FindParameters params;
|
||||
params.setRelease(release->getId());
|
||||
@@ -164,11 +144,11 @@ namespace lms::scanner
|
||||
params.setSortMethod(db::TrackEmbeddedImageSortMethod::DiscNumberThenTrackNumberThenSizeDesc);
|
||||
params.setRange(db::Range{ .offset = 0, .size = 1 });
|
||||
|
||||
db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { trackEmbeddedImageId = image->getId(); });
|
||||
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { artwork = db::Artwork::find(session, image->getId()); });
|
||||
}
|
||||
|
||||
if (trackEmbeddedImageId.isValid())
|
||||
return ReleaseArtwork{ trackEmbeddedImageId };
|
||||
if (artwork)
|
||||
return artwork;
|
||||
|
||||
// Fallback on embedded media image
|
||||
{
|
||||
@@ -178,66 +158,28 @@ namespace lms::scanner
|
||||
params.setSortMethod(db::TrackEmbeddedImageSortMethod::DiscNumberThenTrackNumberThenSizeDesc);
|
||||
params.setRange(db::Range{ .offset = 0, .size = 1 });
|
||||
|
||||
db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { trackEmbeddedImageId = image->getId(); });
|
||||
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { artwork = db::Artwork::find(session, image->getId()); });
|
||||
}
|
||||
|
||||
if (trackEmbeddedImageId.isValid())
|
||||
return ReleaseArtwork{ trackEmbeddedImageId };
|
||||
|
||||
return ReleaseArtwork{};
|
||||
return artwork;
|
||||
}
|
||||
|
||||
bool fetchNextReleaseArtworksToUpdate(SearchReleaseArtworkContext& searchContext, ReleaseArtworkAssociationContainer& ReleaseArtworkAssociations)
|
||||
void updateReleasePreferredArtwork(db::Session& session, const ReleaseArtworkAssociation& releaseArtworkAssociation)
|
||||
{
|
||||
const db::ReleaseId releaseId{ searchContext.lastRetrievedReleaseId };
|
||||
|
||||
{
|
||||
constexpr std::size_t readBatchSize{ 100 };
|
||||
|
||||
auto transaction{ searchContext.session.createReadTransaction() };
|
||||
|
||||
db::Release::find(searchContext.session, searchContext.lastRetrievedReleaseId, readBatchSize, [&](const db::Release::pointer& release) {
|
||||
const ReleaseArtwork preferredArtwork{ computePreferredReleaseArtwork(searchContext, release) };
|
||||
const db::Artwork::pointer currentPreferredArtwork{ release->getPreferredArtwork() };
|
||||
|
||||
if (!isSameArtwork(preferredArtwork, currentPreferredArtwork))
|
||||
ReleaseArtworkAssociations.push_back(ReleaseArtworkAssociation{ release, preferredArtwork });
|
||||
searchContext.processedReleaseCount++;
|
||||
});
|
||||
}
|
||||
|
||||
return releaseId != searchContext.lastRetrievedReleaseId;
|
||||
db::Release::updatePreferredArtwork(session, releaseArtworkAssociation.releaseId, releaseArtworkAssociation.preferredArtworkId);
|
||||
}
|
||||
|
||||
void updateReleaseArtwork(db::Session& session, const ReleaseArtworkAssociation& ReleaseArtworkAssociation)
|
||||
{
|
||||
db::Release::pointer release{ ReleaseArtworkAssociation.release };
|
||||
|
||||
db::Artwork::pointer artwork;
|
||||
if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if<db::TrackEmbeddedImageId>(&ReleaseArtworkAssociation.preferredArtwork) })
|
||||
artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId);
|
||||
else if (const db::ImageId * imageId{ std::get_if<db::ImageId>(&ReleaseArtworkAssociation.preferredArtwork) })
|
||||
artwork = utils::getOrCreateArtworkFromImage(session, *imageId);
|
||||
|
||||
release.modify()->setPreferredArtwork(artwork);
|
||||
|
||||
if (artwork)
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork in release '" << release->getName() << "' with image in " << artwork->getAbsoluteFilePath());
|
||||
else
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from release '" << release->getName() << "'");
|
||||
}
|
||||
|
||||
void updateReleaseArtworks(db::Session& session, ReleaseArtworkAssociationContainer& imageAssociations)
|
||||
void updateReleasePreferredArtworks(db::Session& session, ReleaseArtworkAssociationContainer& imageAssociations, bool forceFullBatch)
|
||||
{
|
||||
constexpr std::size_t writeBatchSize{ 50 };
|
||||
|
||||
while (!imageAssociations.empty())
|
||||
while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty())
|
||||
{
|
||||
auto transaction{ session.createWriteTransaction() };
|
||||
|
||||
for (std::size_t i{}; !imageAssociations.empty() && i < writeBatchSize; ++i)
|
||||
{
|
||||
updateReleaseArtwork(session, imageAssociations.front());
|
||||
updateReleasePreferredArtwork(session, imageAssociations.front());
|
||||
imageAssociations.pop_front();
|
||||
}
|
||||
}
|
||||
@@ -256,6 +198,62 @@ namespace lms::scanner
|
||||
return res;
|
||||
}
|
||||
|
||||
bool fetchNextReleaseIdRange(db::Session& session, db::ReleaseId& lastRetrievedId, db::IdRange<db::ReleaseId>& idRange)
|
||||
{
|
||||
constexpr std::size_t readBatchSize{ 100 };
|
||||
|
||||
auto transaction{ session.createReadTransaction() };
|
||||
|
||||
idRange = db::Release::findNextRange(session, lastRetrievedId, readBatchSize);
|
||||
lastRetrievedId = idRange.last;
|
||||
|
||||
return idRange.isValid();
|
||||
}
|
||||
|
||||
class ComputeReleaseArtworkAssociationsJob : public core::IJob
|
||||
{
|
||||
public:
|
||||
ComputeReleaseArtworkAssociationsJob(db::Db& db, const SearchReleaseArtworkParams& searchParams, db::IdRange<db::ReleaseId> artistIdRange)
|
||||
: _db{ db }
|
||||
, _searchParams{ searchParams }
|
||||
, _artistIdRange{ artistIdRange }
|
||||
{
|
||||
}
|
||||
|
||||
std::span<const ReleaseArtworkAssociation> getAssociations() const { return _associations; }
|
||||
std::size_t getProcessedReleaseCount() const { return _processedReleaseCount; }
|
||||
|
||||
private:
|
||||
core::LiteralString getName() const override { return "Associate Track Artworks"; }
|
||||
void run() override
|
||||
{
|
||||
auto& session{ _db.getTLSSession() };
|
||||
auto transaction{ session.createReadTransaction() };
|
||||
|
||||
db::Release::find(session, _artistIdRange, [this, &session](const db::Release::pointer& release) {
|
||||
const db::Artwork::pointer preferredArtwork{ computePreferredReleaseArtwork(session, _searchParams, release) };
|
||||
|
||||
if (release->getPreferredArtwork() != preferredArtwork)
|
||||
{
|
||||
_associations.push_back(ReleaseArtworkAssociation{ release->getId(), preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{} });
|
||||
|
||||
if (preferredArtwork)
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork for release '" << release->getName() << "' with image in " << preferredArtwork->getAbsoluteFilePath());
|
||||
else
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from release '" << release->getName() << "'");
|
||||
}
|
||||
|
||||
_processedReleaseCount++;
|
||||
});
|
||||
}
|
||||
|
||||
db::Db& _db;
|
||||
const SearchReleaseArtworkParams& _searchParams;
|
||||
db::IdRange<db::ReleaseId> _artistIdRange;
|
||||
std::vector<ReleaseArtworkAssociation> _associations;
|
||||
std::size_t _processedReleaseCount{};
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
ScanStepAssociateReleaseImages::ScanStepAssociateReleaseImages(InitParams& initParams)
|
||||
@@ -278,21 +276,39 @@ namespace lms::scanner
|
||||
context.currentStepStats.totalElems = db::Release::getCount(session);
|
||||
}
|
||||
|
||||
SearchReleaseArtworkContext searchContext{
|
||||
.session = session,
|
||||
.lastRetrievedReleaseId = {},
|
||||
const SearchReleaseArtworkParams searchParams{
|
||||
.releaseImageFileNames = _releaseImageFileNames,
|
||||
};
|
||||
|
||||
ReleaseArtworkAssociationContainer ReleaseArtworkAssociations;
|
||||
while (fetchNextReleaseArtworksToUpdate(searchContext, ReleaseArtworkAssociations))
|
||||
{
|
||||
ReleaseArtworkAssociationContainer artistArtworkAssociations;
|
||||
auto processJobsDone = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
|
||||
if (_abortScan)
|
||||
return;
|
||||
|
||||
updateReleaseArtworks(session, ReleaseArtworkAssociations);
|
||||
context.currentStepStats.processedElems = searchContext.processedReleaseCount;
|
||||
for (const auto& job : jobs)
|
||||
{
|
||||
const auto& associationJob{ static_cast<const ComputeReleaseArtworkAssociationsJob&>(*job) };
|
||||
const auto& artistAssociations{ associationJob.getAssociations() };
|
||||
|
||||
artistArtworkAssociations.insert(std::end(artistArtworkAssociations), std::cbegin(artistAssociations), std::cend(artistAssociations));
|
||||
|
||||
context.currentStepStats.processedElems += associationJob.getProcessedReleaseCount();
|
||||
}
|
||||
|
||||
updateReleasePreferredArtworks(session, artistArtworkAssociations, true);
|
||||
_progressCallback(context.currentStepStats);
|
||||
}
|
||||
};
|
||||
|
||||
JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
|
||||
|
||||
db::ReleaseId lastRetrievedReleaseId{};
|
||||
db::IdRange<db::ReleaseId> artistIdRange;
|
||||
while (fetchNextReleaseIdRange(session, lastRetrievedReleaseId, artistIdRange))
|
||||
queue.push(std::make_unique<ComputeReleaseArtworkAssociationsJob>(_db, searchParams, artistIdRange));
|
||||
|
||||
queue.finish();
|
||||
|
||||
// process all remaining associations
|
||||
updateReleasePreferredArtworks(session, artistArtworkAssociations, false);
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
|
||||
@@ -21,20 +21,23 @@
|
||||
|
||||
#include <cassert>
|
||||
#include <deque>
|
||||
#include <variant>
|
||||
#include <optional>
|
||||
|
||||
#include "core/IJob.hpp"
|
||||
#include "core/IJobScheduler.hpp"
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/LiteralString.hpp"
|
||||
#include "database/Artwork.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Directory.hpp"
|
||||
#include "database/IdRange.hpp"
|
||||
#include "database/Image.hpp"
|
||||
#include "database/Release.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackEmbeddedImage.hpp"
|
||||
#include "database/TrackEmbeddedImageId.hpp"
|
||||
|
||||
#include "ArtworkUtils.hpp"
|
||||
#include "JobQueue.hpp"
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
@@ -42,37 +45,11 @@ namespace lms::scanner
|
||||
namespace
|
||||
{
|
||||
// May come from an embedded image in a track, or from what has been previously resolved for the release
|
||||
using TrackArtwork = std::variant<std::monostate, db::TrackEmbeddedImageId, db::ArtworkId>;
|
||||
bool isSameArtwork(TrackArtwork preferredArtwork, const db::ObjectPtr<db::Artwork>& artwork)
|
||||
{
|
||||
if (std::holds_alternative<std::monostate>(preferredArtwork))
|
||||
return !artwork;
|
||||
|
||||
if (const db::TrackEmbeddedImageId* trackEmbeddedImageId = std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork))
|
||||
{
|
||||
assert(trackEmbeddedImageId->isValid());
|
||||
return artwork && *trackEmbeddedImageId == artwork->getTrackEmbeddedImageId();
|
||||
}
|
||||
|
||||
if (const db::ArtworkId* artworkId = std::get_if<db::ArtworkId>(&preferredArtwork))
|
||||
{
|
||||
assert(artworkId->isValid());
|
||||
return artwork && *artworkId == artwork->getId();
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool isValid(const TrackArtwork& res)
|
||||
{
|
||||
return !std::holds_alternative<std::monostate>(res);
|
||||
}
|
||||
|
||||
struct TrackArtworksAssociation
|
||||
{
|
||||
db::Track::pointer track;
|
||||
TrackArtwork preferredArtwork;
|
||||
TrackArtwork preferredMediaArtwork;
|
||||
db::TrackId trackId;
|
||||
std::optional<db::ArtworkId> preferredArtworkId;
|
||||
std::optional<db::ArtworkId> preferredMediaArtworkId;
|
||||
};
|
||||
using TrackArtworksAssociationContainer = std::deque<TrackArtworksAssociation>;
|
||||
|
||||
@@ -83,11 +60,10 @@ namespace lms::scanner
|
||||
std::size_t processedTrackCount{};
|
||||
};
|
||||
|
||||
TrackArtwork computePreferredTrackArtwork(SearchTrackArtworkContext& searchContext, const db::Track::pointer& track, TrackArtwork preferredMediaArtwork)
|
||||
db::Artwork::pointer computePreferredTrackArtwork(db::Session& session, const db::Track::pointer& track, const db::Artwork::pointer& preferredMediaArtwork)
|
||||
{
|
||||
TrackArtwork res{ preferredMediaArtwork };
|
||||
|
||||
if (isValid(res))
|
||||
db::Artwork::pointer res{ preferredMediaArtwork };
|
||||
if (res)
|
||||
return res;
|
||||
|
||||
// Fallback on front cover for this track
|
||||
@@ -97,10 +73,10 @@ namespace lms::scanner
|
||||
params.setImageTypes({ db::ImageType::FrontCover });
|
||||
params.setSortMethod(db::TrackEmbeddedImageSortMethod::SizeDesc);
|
||||
params.setRange(db::Range{ .offset = 0, .size = 1 });
|
||||
db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); });
|
||||
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); });
|
||||
}
|
||||
|
||||
if (isValid(res))
|
||||
if (res)
|
||||
return res;
|
||||
|
||||
// Fallback on the artwork already resolved for the release
|
||||
@@ -108,28 +84,26 @@ namespace lms::scanner
|
||||
if (!releaseId.isValid())
|
||||
return res;
|
||||
|
||||
if (const db::Release::pointer release{ db::Release::find(searchContext.session, releaseId) })
|
||||
{
|
||||
if (const db::ArtworkId releaseArtworkId{ release->getPreferredArtworkId() }; releaseArtworkId.isValid())
|
||||
res = releaseArtworkId;
|
||||
}
|
||||
if (const db::Release::pointer release{ db::Release::find(session, releaseId) })
|
||||
res = release->getPreferredArtwork();
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
TrackArtwork computePreferredTrackMediaArtwork(SearchTrackArtworkContext& searchContext, const db::Track::pointer& track)
|
||||
db::Artwork::pointer computePreferredTrackMediaArtwork(db::Session& session, const db::Track::pointer& track)
|
||||
{
|
||||
TrackArtwork res;
|
||||
db::Artwork::pointer res;
|
||||
|
||||
{
|
||||
db::TrackEmbeddedImage::FindParameters params;
|
||||
params.setTrack(track->getId());
|
||||
params.setImageTypes({ db::ImageType::Media });
|
||||
params.setSortMethod(db::TrackEmbeddedImageSortMethod::SizeDesc);
|
||||
params.setRange(db::Range{ .offset = 0, .size = 1 });
|
||||
db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); });
|
||||
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); });
|
||||
}
|
||||
|
||||
if (isValid(res))
|
||||
if (res)
|
||||
return res;
|
||||
|
||||
// fallback on another track of the same disc
|
||||
@@ -145,95 +119,28 @@ namespace lms::scanner
|
||||
params.setImageTypes({ db::ImageType::Media });
|
||||
params.setSortMethod(db::TrackEmbeddedImageSortMethod::TrackNumberThenSizeDesc);
|
||||
params.setRange(db::Range{ .offset = 0, .size = 1 });
|
||||
db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); });
|
||||
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); });
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
bool fetchNextTrackArtworksToUpdate(SearchTrackArtworkContext& searchContext, TrackArtworksAssociationContainer& TrackArtworksAssociations)
|
||||
void updateTrackPreferredArtworks(db::Session& session, const TrackArtworksAssociation& trackArtworksAssociation)
|
||||
{
|
||||
const db::TrackId trackId{ searchContext.lastRetrievedTrackId };
|
||||
assert(trackArtworksAssociation.preferredArtworkId || trackArtworksAssociation.preferredMediaArtworkId);
|
||||
|
||||
{
|
||||
constexpr std::size_t readBatchSize{ 100 };
|
||||
if (trackArtworksAssociation.preferredArtworkId)
|
||||
db::Track::updatePreferredArtwork(session, trackArtworksAssociation.trackId, *trackArtworksAssociation.preferredArtworkId);
|
||||
|
||||
auto transaction{ searchContext.session.createReadTransaction() };
|
||||
|
||||
db::Track::find(searchContext.session, searchContext.lastRetrievedTrackId, readBatchSize, [&](const db::Track::pointer& track) {
|
||||
const TrackArtwork preferredMediaArtwork{ computePreferredTrackMediaArtwork(searchContext, track) };
|
||||
const TrackArtwork preferredArtwork{ computePreferredTrackArtwork(searchContext, track, preferredMediaArtwork) };
|
||||
|
||||
const db::Artwork::pointer currentPreferredArtwork{ track->getPreferredArtwork() };
|
||||
const db::Artwork::pointer currentPreferredMediaArtwork{ track->getPreferredMediaArtwork() };
|
||||
|
||||
if (!isSameArtwork(preferredArtwork, currentPreferredArtwork)
|
||||
|| !isSameArtwork(preferredMediaArtwork, currentPreferredMediaArtwork))
|
||||
{
|
||||
TrackArtworksAssociations.push_back(TrackArtworksAssociation{ .track = track, .preferredArtwork = preferredArtwork, .preferredMediaArtwork = preferredMediaArtwork });
|
||||
}
|
||||
|
||||
searchContext.processedTrackCount++;
|
||||
});
|
||||
}
|
||||
|
||||
return trackId != searchContext.lastRetrievedTrackId;
|
||||
if (trackArtworksAssociation.preferredMediaArtworkId)
|
||||
db::Track::updatePreferredMediaArtwork(session, trackArtworksAssociation.trackId, *trackArtworksAssociation.preferredMediaArtworkId);
|
||||
}
|
||||
|
||||
void updateTrackPreferredArtwork(db::Session& session, db::Track::pointer& track, TrackArtwork preferredArtwork)
|
||||
{
|
||||
db::Artwork::pointer artwork;
|
||||
if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork) })
|
||||
artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId);
|
||||
else if (const db::ArtworkId * artworkId{ std::get_if<db::ArtworkId>(&preferredArtwork) })
|
||||
artwork = db::Artwork::find(session, *artworkId);
|
||||
|
||||
// Using track.modify() is quite CPU intensive as the track class has too many fields
|
||||
db::Track::updatePreferredArtwork(session, track->getId(), artwork ? artwork->getId() : db::ArtworkId{});
|
||||
if (artwork)
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork in track " << track->getAbsoluteFilePath() << " with image in " << artwork->getAbsoluteFilePath());
|
||||
else
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from track " << track->getAbsoluteFilePath());
|
||||
}
|
||||
|
||||
void updateTrackPreferredMediaArtwork(db::Session& session, db::Track::pointer& track, TrackArtwork preferredArtwork)
|
||||
{
|
||||
db::Artwork::pointer artwork;
|
||||
if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork) })
|
||||
artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId);
|
||||
else if (const db::ArtworkId * artworkId{ std::get_if<db::ArtworkId>(&preferredArtwork) })
|
||||
artwork = db::Artwork::find(session, *artworkId);
|
||||
|
||||
// Using track.modify() is quite CPU intensive as the track class has too many fields
|
||||
db::Track::updatePreferredMediaArtwork(session, track->getId(), artwork ? artwork->getId() : db::ArtworkId{});
|
||||
if (artwork)
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updated preferred media artwork in track " << track->getAbsoluteFilePath() << " with image in " << artwork->getAbsoluteFilePath());
|
||||
else
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removed preferred media artwork from track " << track->getAbsoluteFilePath());
|
||||
}
|
||||
|
||||
void updateTrackPreferredArtworks(db::Session& session, const TrackArtworksAssociation& TrackArtworksAssociation)
|
||||
{
|
||||
db::Track::pointer track{ TrackArtworksAssociation.track };
|
||||
|
||||
{
|
||||
const db::Artwork::pointer currentPreferredArtwork{ track->getPreferredArtwork() };
|
||||
if (!isSameArtwork(TrackArtworksAssociation.preferredArtwork, currentPreferredArtwork))
|
||||
updateTrackPreferredArtwork(session, track, TrackArtworksAssociation.preferredArtwork);
|
||||
}
|
||||
|
||||
{
|
||||
const db::Artwork::pointer currentPreferredMediaArtwork{ track->getPreferredMediaArtwork() };
|
||||
if (!isSameArtwork(TrackArtworksAssociation.preferredMediaArtwork, currentPreferredMediaArtwork))
|
||||
updateTrackPreferredMediaArtwork(session, track, TrackArtworksAssociation.preferredMediaArtwork);
|
||||
}
|
||||
}
|
||||
|
||||
void updateTrackPreferredArtworks(db::Session& session, TrackArtworksAssociationContainer& imageAssociations)
|
||||
void updateTrackPreferredArtworks(db::Session& session, TrackArtworksAssociationContainer& imageAssociations, bool forceFullBatch)
|
||||
{
|
||||
constexpr std::size_t writeBatchSize{ 50 };
|
||||
|
||||
while (!imageAssociations.empty())
|
||||
while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty())
|
||||
{
|
||||
auto transaction{ session.createWriteTransaction() };
|
||||
|
||||
@@ -244,6 +151,76 @@ namespace lms::scanner
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool fetchNextTrackIdRange(SearchTrackArtworkContext& searchContext, db::IdRange<db::TrackId>& trackIdRange)
|
||||
{
|
||||
constexpr std::size_t readBatchSize{ 100 };
|
||||
|
||||
auto transaction{ searchContext.session.createReadTransaction() };
|
||||
|
||||
trackIdRange = db::Track::findNextRange(searchContext.session, searchContext.lastRetrievedTrackId, readBatchSize);
|
||||
searchContext.lastRetrievedTrackId = trackIdRange.last;
|
||||
|
||||
return trackIdRange.isValid();
|
||||
}
|
||||
|
||||
class ComputeTrackArtworkAssociationsJob : public core::IJob
|
||||
{
|
||||
public:
|
||||
ComputeTrackArtworkAssociationsJob(db::Db& db, db::IdRange<db::TrackId> trackIdRange)
|
||||
: _db{ db }
|
||||
, _trackIdRange{ trackIdRange }
|
||||
{
|
||||
}
|
||||
|
||||
std::span<const TrackArtworksAssociation> getTrackAssociations() const { return _trackAssociations; }
|
||||
std::size_t getProcessedTrackCount() const { return _processedTrackCount; }
|
||||
|
||||
private:
|
||||
core::LiteralString getName() const override { return "Associate Track Artworks"; }
|
||||
void run() override
|
||||
{
|
||||
auto& session{ _db.getTLSSession() };
|
||||
auto transaction{ session.createReadTransaction() };
|
||||
|
||||
db::Track::find(session, _trackIdRange, [&](const db::Track::pointer& track) {
|
||||
const db::Artwork::pointer preferredMediaArtwork{ computePreferredTrackMediaArtwork(session, track) };
|
||||
const db::Artwork::pointer preferredArtwork{ computePreferredTrackArtwork(session, track, preferredMediaArtwork) };
|
||||
|
||||
TrackArtworksAssociation artworksAssociation{ .trackId = track->getId(), .preferredArtworkId = std::nullopt, .preferredMediaArtworkId = std::nullopt };
|
||||
|
||||
if (track->getPreferredArtwork() != preferredArtwork)
|
||||
{
|
||||
artworksAssociation.preferredArtworkId = preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{};
|
||||
|
||||
if (preferredArtwork)
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork in track " << track->getAbsoluteFilePath() << " with image in " << preferredArtwork->getAbsoluteFilePath());
|
||||
else
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from track " << track->getAbsoluteFilePath());
|
||||
}
|
||||
|
||||
if (track->getPreferredMediaArtwork() != preferredMediaArtwork)
|
||||
{
|
||||
artworksAssociation.preferredMediaArtworkId = preferredMediaArtwork ? preferredMediaArtwork->getId() : db::ArtworkId{};
|
||||
|
||||
if (preferredMediaArtwork)
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Updating preferred media artwork in track " << track->getAbsoluteFilePath() << " with image in " << preferredMediaArtwork->getAbsoluteFilePath());
|
||||
else
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing preferred media artwork from track " << track->getAbsoluteFilePath());
|
||||
}
|
||||
|
||||
if (artworksAssociation.preferredArtworkId || artworksAssociation.preferredMediaArtworkId)
|
||||
_trackAssociations.push_back(artworksAssociation);
|
||||
|
||||
_processedTrackCount++;
|
||||
});
|
||||
}
|
||||
|
||||
db::Db& _db;
|
||||
db::IdRange<db::TrackId> _trackIdRange;
|
||||
std::vector<TrackArtworksAssociation> _trackAssociations;
|
||||
std::size_t _processedTrackCount{};
|
||||
};
|
||||
} // namespace
|
||||
|
||||
ScanStepAssociateTrackImages::ScanStepAssociateTrackImages(InitParams& initParams)
|
||||
@@ -270,15 +247,34 @@ namespace lms::scanner
|
||||
.lastRetrievedTrackId = {},
|
||||
};
|
||||
|
||||
TrackArtworksAssociationContainer TrackArtworksAssociations;
|
||||
while (fetchNextTrackArtworksToUpdate(searchContext, TrackArtworksAssociations))
|
||||
{
|
||||
TrackArtworksAssociationContainer trackArtworksAssociations;
|
||||
auto processTracks = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
|
||||
if (_abortScan)
|
||||
return;
|
||||
|
||||
updateTrackPreferredArtworks(session, TrackArtworksAssociations);
|
||||
context.currentStepStats.processedElems = searchContext.processedTrackCount;
|
||||
for (const auto& job : jobs)
|
||||
{
|
||||
const auto& associationJob{ static_cast<const ComputeTrackArtworkAssociationsJob&>(*job) };
|
||||
const auto& trackAssociations{ associationJob.getTrackAssociations() };
|
||||
|
||||
trackArtworksAssociations.insert(std::end(trackArtworksAssociations), std::cbegin(trackAssociations), std::cend(trackAssociations));
|
||||
|
||||
context.currentStepStats.processedElems += associationJob.getProcessedTrackCount();
|
||||
}
|
||||
|
||||
updateTrackPreferredArtworks(session, trackArtworksAssociations, true);
|
||||
_progressCallback(context.currentStepStats);
|
||||
}
|
||||
};
|
||||
|
||||
JobQueue queue{ getJobScheduler(), 20, processTracks, 1, 0.85F };
|
||||
|
||||
db::IdRange<db::TrackId> trackIdRange;
|
||||
while (fetchNextTrackIdRange(searchContext, trackIdRange))
|
||||
queue.push(std::make_unique<ComputeTrackArtworkAssociationsJob>(_db, trackIdRange));
|
||||
|
||||
queue.finish();
|
||||
|
||||
// process all remaining associations
|
||||
updateTrackPreferredArtworks(session, trackArtworksAssociations, false);
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
#include "scanners/IFileScanOperation.hpp"
|
||||
#include "scanners/IFileScanner.hpp"
|
||||
|
||||
#include "JobQueue.hpp"
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
@@ -114,11 +115,16 @@ namespace lms::scanner
|
||||
|
||||
void ScanStepScanFiles::process(ScanContext& context, const MediaLibraryInfo& mediaLibrary)
|
||||
{
|
||||
core::IJobScheduler& jobScheduler{ getJobScheduler() };
|
||||
assert(jobScheduler.getJobsDoneCount() == 0);
|
||||
|
||||
const std::size_t scanQueueMaxScanRequestCount{ 100 * jobScheduler.getThreadCount() };
|
||||
const std::size_t scanQueueMaxScanRequestCount{ 50 * getJobScheduler().getThreadCount() };
|
||||
constexpr std::size_t processFileResultsBatchSize{ 10 };
|
||||
constexpr float drainRatio{ 0.85 };
|
||||
|
||||
auto processDoneJobs = [&](std::span<std::unique_ptr<core::IJob>> jobsDone) {
|
||||
if (!_abortScan)
|
||||
processFileScanResults(context, jobsDone);
|
||||
};
|
||||
|
||||
JobQueue queue{ getJobScheduler(), scanQueueMaxScanRequestCount, processDoneJobs, processFileResultsBatchSize, drainRatio };
|
||||
|
||||
std::vector<std::unique_ptr<core::IJob>> jobsDone;
|
||||
std::vector<std::unique_ptr<IFileScanOperation>> scanOperations;
|
||||
@@ -148,7 +154,7 @@ namespace lms::scanner
|
||||
if (context.scanOptions.fullScan || scanner->needsScan(fileToScan))
|
||||
{
|
||||
auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) };
|
||||
jobScheduler.scheduleJob(std::make_unique<FileScanJob>(std::move(scanOperation)));
|
||||
queue.push(std::make_unique<FileScanJob>(std::move(scanOperation)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,30 +162,9 @@ namespace lms::scanner
|
||||
_progressCallback(context.currentStepStats);
|
||||
}
|
||||
|
||||
while (jobScheduler.getJobsDoneCount() > (scanQueueMaxScanRequestCount / 2))
|
||||
{
|
||||
jobScheduler.popJobsDone(jobsDone, processFileResultsBatchSize);
|
||||
processFileScanResults(context, jobsDone);
|
||||
jobsDone.clear();
|
||||
}
|
||||
|
||||
jobScheduler.waitUntilJobCountAtMost(scanQueueMaxScanRequestCount);
|
||||
|
||||
return true;
|
||||
},
|
||||
&excludeDirFileName);
|
||||
|
||||
jobScheduler.wait();
|
||||
|
||||
while (jobScheduler.popJobsDone(jobsDone, processFileResultsBatchSize) > 0)
|
||||
{
|
||||
if (!_abortScan)
|
||||
processFileScanResults(context, jobsDone);
|
||||
|
||||
jobsDone.clear();
|
||||
}
|
||||
|
||||
assert(jobScheduler.getJobsDoneCount() == 0);
|
||||
}
|
||||
|
||||
void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span<std::unique_ptr<core::IJob>> scanJobs)
|
||||
@@ -191,9 +176,6 @@ namespace lms::scanner
|
||||
|
||||
for (auto& scanJob : scanJobs)
|
||||
{
|
||||
if (_abortScan)
|
||||
return;
|
||||
|
||||
IFileScanOperation& scanOperation{ static_cast<FileScanJob&>(*scanJob).getScanOperation() };
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG, scanOperation.getName() << ": processing result for " << scanOperation.getFilePath());
|
||||
|
||||
Reference in New Issue
Block a user