Added an option to skip playlists that are refering to a single album + skip playlists that are empty, fixes #577

This commit is contained in:
emeric
2025-01-15 21:14:18 +01:00
parent 4a75e5220e
commit 1d15be1e6c
15 changed files with 154 additions and 35 deletions
+25 -11
View File
@@ -4,7 +4,7 @@
<!--FORMS message blocks-->
<message id="Lms.Admin.Database.template">
<form class="row g-3">
<form class="row g-3 mb-3">
<legend>${tr:Lms.Admin.Database.scan-settings}</legend>
<div class="col-sm-6">
<label class="form-label" for="${id:update-period}">
@@ -24,15 +24,7 @@
${update-start-time-info}
</div>
</div>
<div class="col-12">
<label class="form-label" for="${id:similarity-engine-type}">
${tr:Lms.Admin.Database.similarity-engine-type}
</label>
${similarity-engine-type class="form-control"}
<div class="invalid-feedback">
${similarity-engine-type-info}
</div>
</div>
<legend>${tr:Lms.Admin.Database.tag-parsing}</legend>
<div class="col-12">
<label class="form-label" for="${id:extra-tags-to-scan}">
${tr:Lms.Admin.Database.extra-tags-to-scan}
@@ -54,10 +46,32 @@
</label>
${default-tag-delimiter-container class="row gy-3"}
</div>
<legend>${tr:Lms.Admin.Database.misc}</legend>
<div class="col-12">
${save-btn class="btn btn-primary me-1"}${discard-btn class="btn btn-secondary"}
<div class="form-check">
${skip-single-release-playlists class="form-check-input"}
<label class="form-check-label" for="${id:skip-single-release-playlists}">
${tr:Lms.Admin.Database.skip-single-release-playlists}
</label>
<div class="invalid-feedback">
${skip-single-release-playlists-info}
</div>
</div>
</div>
<div class="col-12">
<label class="form-label" for="${id:similarity-engine-type}">
${tr:Lms.Admin.Database.similarity-engine-type}
</label>
${similarity-engine-type class="form-control"}
<div class="invalid-feedback">
${similarity-engine-type-info}
</div>
</div>
</form>
<hr/>
<div class="col-12">
${save-btn class="btn btn-primary me-1"}${discard-btn class="btn btn-secondary"}
</div>
</message>
<message id="Lms.Admin.Database.template.line-edit-entry">
+3
View File
@@ -78,6 +78,7 @@
<message id="Lms.Admin.Database.extra-tags-to-scan">Extra tags to scan</message>
<message id="Lms.Admin.Database.hourly">Hourly</message>
<message id="Lms.Admin.Database.immediate-scan">Scan now!</message>
<message id="Lms.Admin.Database.misc">Miscellaneous</message>
<message id="Lms.Admin.Database.monthly">Monthly</message>
<message id="Lms.Admin.Database.never">Never</message>
<message id="Lms.Admin.Database.scan-aborted">Scan aborted!</message>
@@ -87,7 +88,9 @@
<message id="Lms.Admin.Database.similarity-engine-type">Similarity engine</message>
<message id="Lms.Admin.Database.similarity-engine-type.clusters">Tag-based</message>
<message id="Lms.Admin.Database.similarity-engine-type.none">None</message>
<message id="Lms.Admin.Database.skip-single-release-playlists">Skip playlists that contain tracks from the same album</message>
<message id="Lms.Admin.Database.tag-delimiter-must-not-contain-only-spaces">The tag delimiter must not consist solely of spaces</message>
<message id="Lms.Admin.Database.tag-parsing">Tag parsing</message>
<message id="Lms.Admin.Database.update-period">Update period</message>
<message id="Lms.Admin.Database.update-start-time">Update start time</message>
<message id="Lms.Admin.Database.weekly">Weekly</message>
+3
View File
@@ -78,6 +78,7 @@
<message id="Lms.Admin.Database.extra-tags-to-scan">Tags supplémentaires à scanner</message>
<message id="Lms.Admin.Database.hourly">Toutes les heures</message>
<message id="Lms.Admin.Database.immediate-scan">Scanner maintenant !</message>
<message id="Lms.Admin.Database.misc">Divers</message>
<message id="Lms.Admin.Database.monthly">Tous les mois</message>
<message id="Lms.Admin.Database.never">Jamais</message>
<message id="Lms.Admin.Database.scan-aborted">Scan interrompu !</message>
@@ -87,7 +88,9 @@
<message id="Lms.Admin.Database.similarity-engine-type">Moteur de similarité</message>
<message id="Lms.Admin.Database.similarity-engine-type.clusters">Basé sur les tags</message>
<message id="Lms.Admin.Database.similarity-engine-type.none">Aucun</message>
<message id="Lms.Admin.Database.skip-single-release-playlists">Ignorer les playlists contenant des pistes d'un même album</message>
<message id="Lms.Admin.Database.tag-delimiter-must-not-contain-only-spaces">Le délimiteur de tag ne doit pas comporter uniquement des espaces</message>
<message id="Lms.Admin.Database.tag-parsing">Analyse des tags</message>
<message id="Lms.Admin.Database.update-period">Périodicité des mises à jour</message>
<message id="Lms.Admin.Database.update-start-time">Heure de départ de la mise à jour</message>
<message id="Lms.Admin.Database.weekly">Toutes les semaines</message>
+3
View File
@@ -78,6 +78,7 @@
<message id="Lms.Admin.Database.extra-tags-to-scan">Tag aggiuntivi da scansionare</message>
<message id="Lms.Admin.Database.hourly">Ogni ora</message>
<message id="Lms.Admin.Database.immediate-scan">Scansiona ora!</message>
<message id="Lms.Admin.Database.misc">Varie</message>
<message id="Lms.Admin.Database.monthly">Mensile</message>
<message id="Lms.Admin.Database.never">Mai</message>
<message id="Lms.Admin.Database.scan-aborted">Scansione annullata!</message>
@@ -87,7 +88,9 @@
<message id="Lms.Admin.Database.similarity-engine-type">Motore di similarità</message>
<message id="Lms.Admin.Database.similarity-engine-type.clusters">Basato su tag</message>
<message id="Lms.Admin.Database.similarity-engine-type.none">Nessuno</message>
<message id="Lms.Admin.Database.skip-single-release-playlists">Salta le playlist che contengono brani dello stesso album</message>
<message id="Lms.Admin.Database.tag-delimiter-must-not-contain-only-spaces">Il delimitatore del tag non deve consistere esclusivamente di spazi</message>
<message id="Lms.Admin.Database.tag-parsing">Analisi dei tag</message>
<message id="Lms.Admin.Database.update-period">Frequenza di aggiornamento</message>
<message id="Lms.Admin.Database.update-start-time">Orario di aggiornamento</message>
<message id="Lms.Admin.Database.weekly">Settimanale</message>
+3
View File
@@ -79,6 +79,7 @@
<message id="Lms.Admin.Database.extra-tags-to-scan">Szukaj dodatkowych znaczników</message>
<message id="Lms.Admin.Database.hourly">Co godzinę</message>
<message id="Lms.Admin.Database.immediate-scan">Skanuj teraz!</message>
<message id="Lms.Admin.Database.misc">Różne</message>
<message id="Lms.Admin.Database.monthly">Co miesiąc</message>
<message id="Lms.Admin.Database.never">Nigdy</message>
<message id="Lms.Admin.Database.scan-aborted">Skanowanie przerwane!</message>
@@ -88,7 +89,9 @@
<message id="Lms.Admin.Database.similarity-engine-type">Metoda sprawdzania podobieństwa</message>
<message id="Lms.Admin.Database.similarity-engine-type.clusters">Oparta o znaczniki</message>
<message id="Lms.Admin.Database.similarity-engine-type.none">Żadna</message>
<message id="Lms.Admin.Database.skip-single-release-playlists">Pomiń playlisty zawierające utwory z tego samego albumu</message>
<message id="Lms.Admin.Database.tag-delimiter-must-not-contain-only-spaces">Rozdzielacz nie może się składać z samych białych znaków</message>
<message id="Lms.Admin.Database.tag-parsing">Analiza tagów</message>
<message id="Lms.Admin.Database.update-period">Okres aktualizacji</message>
<message id="Lms.Admin.Database.update-start-time">Czas startu aktualizacji</message>
<message id="Lms.Admin.Database.weekly">Co tydzień</message>
+3
View File
@@ -78,6 +78,7 @@
<message id="Lms.Admin.Database.hourly">每小时</message>
<message id="Lms.Admin.Database.immediate-scan">立即扫描!</message>
<message id="Lms.Admin.Database.monthly">每月</message>
<message id="Lms.Admin.Database.never">从不</message>
@@ -88,6 +89,8 @@
<message id="Lms.Admin.Database.update-period">更新周期</message>
<message id="Lms.Admin.Database.update-start-time">更新开始时间</message>
<message id="Lms.Admin.Database.weekly">每周</message>
+8 -1
View File
@@ -35,7 +35,7 @@ namespace lms::db
{
namespace
{
static constexpr Version LMS_DATABASE_VERSION{ 77 };
static constexpr Version LMS_DATABASE_VERSION{ 78 };
}
VersionInfo::VersionInfo()
@@ -1029,6 +1029,12 @@ FROM tracklist)");
utils::executeCommand(*session.getDboSession(), "UPDATE scan_settings SET scan_version = scan_version + 1");
}
void migrateFromV77(Session& session)
{
// added new scan settings: skip single release playlists (default value is conservative, no need to rescan)
utils::executeCommand(*session.getDboSession(), "ALTER TABLE scan_settings ADD COLUMN skip_single_release_playlists BOOLEAN NOT NULL DEFAULT(FALSE)");
}
bool doDbMigration(Session& session)
{
constexpr std::string_view outdatedMsg{ "Outdated database, please rebuild it (delete the .db file and restart)" };
@@ -1082,6 +1088,7 @@ FROM tracklist)");
{ 74, migrateFromV74 },
{ 75, migrateFromV75 },
{ 76, migrateFromV76 },
{ 77, migrateFromV77 },
};
bool migrationPerformed{};
+9
View File
@@ -94,6 +94,15 @@ namespace lms::db
}
}
void ScanSettings::setSkipSingleReleasePlayLists(bool value)
{
if (_skipSingleReleasePlayLists != value)
{
_skipSingleReleasePlayLists = value;
incScanVersion();
}
}
void ScanSettings::incScanVersion()
{
_scanVersion += 1;
@@ -70,6 +70,7 @@ namespace lms::db
SimilarityEngineType getSimilarityEngineType() const { return _similarityEngineType; }
std::vector<std::string> getArtistTagDelimiters() const;
std::vector<std::string> getDefaultTagDelimiters() const;
bool getSkipSingleReleasePlayLists() const { return _skipSingleReleasePlayLists; }
// Setters
void setUpdateStartTime(Wt::WTime t) { _startTime = t; }
@@ -78,6 +79,7 @@ namespace lms::db
void setSimilarityEngineType(SimilarityEngineType type) { _similarityEngineType = type; }
void setArtistTagDelimiters(std::span<const std::string_view> delimiters);
void setDefaultTagDelimiters(std::span<const std::string_view> delimiters);
void setSkipSingleReleasePlayLists(bool value);
void incScanVersion();
template<class Action>
@@ -90,6 +92,7 @@ namespace lms::db
Wt::Dbo::field(a, _extraTagsToScan, "extra_tags_to_scan");
Wt::Dbo::field(a, _artistTagDelimiters, "artist_tag_delimiters");
Wt::Dbo::field(a, _defaultTagDelimiters, "default_tag_delimiters");
Wt::Dbo::field(a, _skipSingleReleasePlayLists, "skip_single_release_playlists");
}
private:
@@ -100,5 +103,6 @@ namespace lms::db
std::string _extraTagsToScan;
std::string _artistTagDelimiters;
std::string _defaultTagDelimiters;
bool _skipSingleReleasePlayLists{ false };
};
} // namespace lms::db
@@ -294,6 +294,7 @@ namespace lms::db
std::vector<ObjectPtr<Artist>> getArtists(core::EnumSet<TrackArtistLinkType> artistLinkTypes) const; // no type means all
std::vector<ArtistId> getArtistIds(core::EnumSet<TrackArtistLinkType> artistLinkTypes) const; // no type means all
std::vector<ObjectPtr<TrackArtistLink>> getArtistLinks() const;
ReleaseId getReleaseId() const { return _release.id(); }
ObjectPtr<Release> getRelease() const { return _release; }
std::vector<ObjectPtr<Cluster>> getClusters() const;
std::vector<ClusterId> getClusterIds() const;
@@ -338,7 +338,6 @@ namespace lms::scanner
return;
LMS_LOG(DBUPDATER, DEBUG, "Scanner settings updated");
LMS_LOG(DBUPDATER, DEBUG, "skipDuplicateMBID = " << newSettings.skipDuplicateMBID);
LMS_LOG(DBUPDATER, DEBUG, "Using scan settings version " << newSettings.scanVersion);
_settings = std::move(newSettings);
@@ -406,6 +405,8 @@ namespace lms::scanner
newSettings.artistTagDelimiters = scanSettings->getArtistTagDelimiters();
newSettings.defaultTagDelimiters = scanSettings->getDefaultTagDelimiters();
newSettings.skipSingleReleasePlayLists = scanSettings->getSkipSingleReleasePlayLists();
}
return newSettings;
@@ -42,6 +42,8 @@ namespace lms::scanner
std::vector<std::string> extraTags;
std::vector<std::string> artistTagDelimiters;
std::vector<std::string> defaultTagDelimiters;
bool skipSingleReleasePlayLists{};
std::vector<MediaLibraryInfo> mediaLibraries;
bool operator==(const ScannerSettings& rhs) const = default;
@@ -100,6 +100,7 @@ namespace lms::scanner
stats.deletions++;
}
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadPlayListFile);
LMS_LOG(DBUPDATER, DEBUG, "Removed playlist file " << _file);
return;
}
@@ -26,10 +26,13 @@
#include "database/Db.hpp"
#include "database/Directory.hpp"
#include "database/PlayListFile.hpp"
#include "database/ReleaseId.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "database/TrackList.hpp"
#include "ScannerSettings.hpp"
namespace lms::scanner
{
namespace
@@ -37,10 +40,17 @@ namespace lms::scanner
constexpr std::size_t readBatchSize{ 20 };
constexpr std::size_t writeBatchSize{ 5 };
struct TrackInfo
{
db::TrackId trackId;
db::ReleaseId releaseId;
};
struct PlayListFileAssociation
{
db::PlayListFileId playListFileIdId;
std::vector<db::TrackId> trackIds;
std::vector<TrackInfo> tracks;
};
using PlayListFileAssociationContainer = std::deque<PlayListFileAssociation>;
@@ -49,6 +59,7 @@ namespace lms::scanner
db::Session& session;
db::PlayListFileId lastRetrievedPlayListFileId;
std::size_t processedPlayListFileCount{};
const ScannerSettings& settings;
};
db::Track::pointer getMatchingTrack(db::Session& session, const std::filesystem::path& filePath, const db::Directory::pointer& playListDirectory)
@@ -67,7 +78,19 @@ namespace lms::scanner
return matchingTrack;
}
bool trackListNeedsUpdate(db::Session& session, std::string_view name, std::span<const db::TrackId> trackIds, const db::TrackList::pointer& trackList)
bool isSingleReleasePlayList(std::span<const TrackInfo> tracks)
{
if (tracks.empty())
return true;
const db::ReleaseId releaseId{ tracks.front().releaseId };
if (std::all_of(std::cbegin(tracks) + 1, std::cend(tracks), [=](const TrackInfo& trackInfo) { return trackInfo.releaseId == releaseId; }))
return true;
return false;
}
bool trackListNeedsUpdate(db::Session& session, std::string_view name, std::span<const TrackInfo> tracks, const db::TrackList::pointer& trackList)
{
if (trackList->getName() != name)
return true;
@@ -78,13 +101,13 @@ namespace lms::scanner
bool needUpdate{};
std::size_t currentIndex{};
db::TrackListEntry::find(session, params, [&](const db::TrackListEntry::pointer& entry) {
if (currentIndex > trackIds.size() || trackIds[currentIndex] != entry->getTrackId())
if (currentIndex > tracks.size() || tracks[currentIndex].trackId != entry->getTrackId())
needUpdate = true;
currentIndex += 1;
});
if (currentIndex != trackIds.size())
if (currentIndex != tracks.size())
needUpdate = true;
return needUpdate;
@@ -108,20 +131,27 @@ namespace lms::scanner
// TODO optim: no need to fetch the whole track
db::Track::pointer track{ getMatchingTrack(searchContext.session, file, playListFile->getDirectory()) };
if (track)
playListAssociation.trackIds.push_back(track->getId());
playListAssociation.tracks.push_back(TrackInfo{ .trackId = track->getId(), .releaseId = track->getReleaseId() });
else
LMS_LOG(DBUPDATER, DEBUG, "Track '" << file.string() << "' not found in playlist '" << playListFile->getAbsoluteFilePath().string() << "'");
LMS_LOG(DBUPDATER, DEBUG, "Track " << file << " not found in playlist " << playListFile->getAbsoluteFilePath());
}
if (playListAssociation.tracks.empty()
|| (searchContext.settings.skipSingleReleasePlayLists && isSingleReleasePlayList(playListAssociation.tracks)))
{
playListAssociation.tracks.clear();
}
bool needUpdate{ true };
if (const db::TrackList::pointer trackList{ playListFile->getTrackList() })
needUpdate = trackListNeedsUpdate(searchContext.session, playListFile->getName(), playListAssociation.trackIds, trackList);
{
if (!playListAssociation.tracks.empty())
needUpdate = trackListNeedsUpdate(searchContext.session, playListFile->getName(), playListAssociation.tracks, trackList);
}
if (needUpdate)
{
LMS_LOG(DBUPDATER, DEBUG, "Updating PlayList '" << playListFile->getAbsoluteFilePath().string() << "' (" << playListAssociation.trackIds.size() << " files)");
playListFileAssociations.emplace_back(std::move(playListAssociation));
}
searchContext.processedPlayListFileCount++;
});
}
@@ -135,7 +165,19 @@ namespace lms::scanner
assert(playListFile);
db::TrackList::pointer trackList{ playListFile->getTrackList() };
if (!trackList)
if (playListFileAssociation.tracks.empty())
{
if (trackList)
{
LMS_LOG(DBUPDATER, DEBUG, "Removed associated tracklist for " << playListFile->getAbsoluteFilePath() << "");
trackList.remove();
}
return;
}
const bool createTrackList{ !trackList };
if (createTrackList)
{
trackList = session.create<db::TrackList>(playListFile->getName(), db::TrackListType::PlayList);
playListFile.modify()->setTrackList(trackList);
@@ -146,11 +188,13 @@ namespace lms::scanner
trackList.modify()->setName(playListFile->getName());
trackList.modify()->clear();
for (const db::TrackId trackId : playListFileAssociation.trackIds)
for (const TrackInfo trackInfo : playListFileAssociation.tracks)
{
if (db::Track::pointer track{ db::Track::find(session, trackId) })
if (db::Track::pointer track{ db::Track::find(session, trackInfo.trackId) })
session.create<db::TrackListEntry>(track, trackList, playListFile->getLastWriteTime());
}
LMS_LOG(DBUPDATER, DEBUG, std::string_view{ createTrackList ? "Created" : "Updated" } << " associated tracklist for " << playListFile->getAbsoluteFilePath() << " (" << playListFileAssociation.tracks.size() << " tracks)");
}
void updatePlayListFiles(db::Session& session, PlayListFileAssociationContainer& playListFileAssociations)
@@ -186,6 +230,7 @@ namespace lms::scanner
SearchPlayListFileContext searchContext{
.session = session,
.lastRetrievedPlayListFileId = {},
.settings = _settings,
};
PlayListFileAssociationContainer playListFileAssociations;
+29 -9
View File
@@ -19,6 +19,7 @@
#include "ScanSettingsView.hpp"
#include <Wt/WCheckBox.h>
#include <Wt/WComboBox.h>
#include <Wt/WFormModel.h>
#include <Wt/WLineEdit.h>
@@ -69,6 +70,7 @@ namespace lms::ui
static inline constexpr Field UpdatePeriodField{ "update-period" };
static inline constexpr Field UpdateStartTimeField{ "update-start-time" };
static inline constexpr Field SimilarityEngineTypeField{ "similarity-engine-type" };
static inline constexpr Field SkipSingleReleasePlayLists{ "skip-single-release-playlists" };
using UpdatePeriodModel = ValueStringModel<ScanSettings::UpdatePeriod>;
@@ -79,10 +81,12 @@ namespace lms::ui
addField(UpdatePeriodField);
addField(UpdateStartTimeField);
addField(SimilarityEngineTypeField);
addField(SkipSingleReleasePlayLists);
setValidator(UpdatePeriodField, createMandatoryValidator());
setValidator(UpdateStartTimeField, createMandatoryValidator());
setValidator(SimilarityEngineTypeField, createMandatoryValidator());
setValidator(SkipSingleReleasePlayLists, createMandatoryValidator());
}
std::shared_ptr<UpdatePeriodModel> updatePeriodModel() { return _updatePeriodModel; }
@@ -109,6 +113,8 @@ namespace lms::ui
setReadOnly(DatabaseSettingsModel::UpdateStartTimeField, true);
}
setValue(SkipSingleReleasePlayLists, scanSettings->getSkipSingleReleasePlayLists());
auto similarityEngineTypeRow{ _similarityEngineTypeModel->getRowFromValue(scanSettings->getSimilarityEngineType()) };
if (similarityEngineTypeRow)
setValue(SimilarityEngineTypeField, _similarityEngineTypeModel->getString(*similarityEngineTypeRow));
@@ -126,17 +132,28 @@ namespace lms::ui
ScanSettings::pointer scanSettings{ ScanSettings::get(LmsApp->getDbSession()) };
auto updatePeriodRow{ _updatePeriodModel->getRowFromString(valueText(UpdatePeriodField)) };
if (updatePeriodRow)
scanSettings.modify()->setUpdatePeriod(_updatePeriodModel->getValue(*updatePeriodRow));
{
const auto updatePeriodRow{ _updatePeriodModel->getRowFromString(valueText(UpdatePeriodField)) };
if (updatePeriodRow)
scanSettings.modify()->setUpdatePeriod(_updatePeriodModel->getValue(*updatePeriodRow));
}
auto startTimeRow{ _updateStartTimeModel->getRowFromString(valueText(UpdateStartTimeField)) };
if (startTimeRow)
scanSettings.modify()->setUpdateStartTime(_updateStartTimeModel->getValue(*startTimeRow));
{
const auto startTimeRow{ _updateStartTimeModel->getRowFromString(valueText(UpdateStartTimeField)) };
if (startTimeRow)
scanSettings.modify()->setUpdateStartTime(_updateStartTimeModel->getValue(*startTimeRow));
}
auto similarityEngineTypeRow{ _similarityEngineTypeModel->getRowFromString(valueText(SimilarityEngineTypeField)) };
if (similarityEngineTypeRow)
scanSettings.modify()->setSimilarityEngineType(_similarityEngineTypeModel->getValue(*similarityEngineTypeRow));
{
const bool skipSingleReleasePlayLists{ Wt::asNumber(value(SkipSingleReleasePlayLists)) != 0 };
scanSettings.modify()->setSkipSingleReleasePlayLists(skipSingleReleasePlayLists);
}
{
const auto similarityEngineTypeRow{ _similarityEngineTypeModel->getRowFromString(valueText(SimilarityEngineTypeField)) };
if (similarityEngineTypeRow)
scanSettings.modify()->setSimilarityEngineType(_similarityEngineTypeModel->getValue(*similarityEngineTypeRow));
}
scanSettings.modify()->setExtraTagsToScan(extraTagsToScan);
scanSettings.modify()->setArtistTagDelimiters(artistDelimiters);
@@ -318,6 +335,9 @@ namespace lms::ui
updateStartTime->setModel(model->updateStartTimeModel());
t->setFormWidget(DatabaseSettingsModel::UpdateStartTimeField, std::move(updateStartTime));
// Skip playlists
t->setFormWidget(DatabaseSettingsModel::SkipSingleReleasePlayLists, std::make_unique<Wt::WCheckBox>());
// Similarity engine type
auto similarityEngineType{ std::make_unique<Wt::WComboBox>() };
similarityEngineType->setModel(model->similarityEngineTypeModel());