owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prevent the media scanner from never quitting

Browse Source
This commit is contained in:
emeric
2019-06-13 14:29:39 +02:00
parent c288e439b3
commit 08988b17a2
6 changed files with 66 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/scanner/MediaScanner.cpp
+12
View File
@@ -420,7 +420,10 @@ MediaScanner::scan(boost::system::error_code err)
{
for (auto& addon : _addons)
addon->preScanComplete();
}
if (_running)
{
stats.stopTime = Wt::WLocalDateTime::currentDateTime().toUTC();
{
std::unique_lock<std::mutex> lock {_statusMutex};
@@ -471,6 +474,13 @@ MediaScanner::refreshScanSettings()
addon->refreshSettings();
}
void MediaScanner::notifyInProgress(Stats& stats)
{
std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
_sigScanInProgress(stats);
_lastScanInProgressEmit = now;
}
void MediaScanner::notifyInProgressIfNeeded(Stats& stats)
{
std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
@@ -674,6 +684,8 @@ MediaScanner::scanMediaDirectory(boost::filesystem::path mediaDirectory, bool fo
itPath.increment(ec);
}
notifyInProgress(stats);
}
// Check if a file exists and is still in a media directory
src/scanner/MediaScanner.hpp
+1
View File
@@ -121,6 +121,7 @@ class MediaScanner
void checkDuplicatedAudioFiles(Stats& stats);
void scanAudioFile(const boost::filesystem::path& file, bool forceScan, Stats& stats);
void notifyInProgressIfNeeded(Stats& stats);
void notifyInProgress(Stats& stats);
bool _running {false};
Wt::WIOService _ioService;
src/similarity/features/SimilarityFeaturesCache.cpp
+6 -2
View File
@@ -95,6 +95,8 @@ createNetworkFromCacheFile(boost::filesystem::path path)
{
try
{
LMS_LOG(SIMILARITY, INFO) << "Reading network from cache...";
boost::property_tree::ptree root;
boost::property_tree::read_xml(path.string(), root);
@@ -127,7 +129,7 @@ createNetworkFromCacheFile(boost::filesystem::path path)
res.setRefVector({x, y}, refVector);
}
LMS_LOG(SIMILARITY, DEBUG) << "Successfully read network from cache";
LMS_LOG(SIMILARITY, INFO) << "Successfully read network from cache";
return res;
}
@@ -180,6 +182,8 @@ createObjectPositionsFromCacheFile(boost::filesystem::path path)
{
try
{
LMS_LOG(SIMILARITY, INFO) << "Reading object position from cache...";
boost::property_tree::ptree root;
boost::property_tree::read_xml(path.string(), root);
@@ -198,7 +202,7 @@ createObjectPositionsFromCacheFile(boost::filesystem::path path)
}
}
LMS_LOG(SIMILARITY, DEBUG) << "Successfully read object position from cache";
LMS_LOG(SIMILARITY, INFO) << "Successfully read object position from cache";
return res;
}
src/similarity/features/SimilarityFeaturesScannerAddon.cpp
+18 -7
View File
@@ -59,7 +59,7 @@ FeaturesScannerAddon::FeaturesScannerAddon(Wt::Dbo::SqlConnectionPool& connectio
boost::optional<Similarity::FeaturesCache> cache {Similarity::FeaturesCache::read()};
if (cache)
{
auto searcher {std::make_shared<Similarity::FeaturesSearcher>(_db.getSession(), *cache)};
auto searcher {std::make_shared<Similarity::FeaturesSearcher>(_db.getSession(), *cache, [&]() { return _stopRequested; })};
if (searcher->isValid())
std::atomic_store(&_searcher, searcher);
}
@@ -107,7 +107,12 @@ FeaturesScannerAddon::preScanComplete()
LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features DONE (found " << tracksInfo.size() << ")";
for (const TrackInfo& trackInfo : tracksInfo)
{
if (_stopRequested)
return;
fetchFeatures(trackInfo.id, trackInfo.mbid);
}
Similarity::FeaturesCache::invalidate();
updateSearcher();
@@ -116,9 +121,13 @@ FeaturesScannerAddon::preScanComplete()
void
FeaturesScannerAddon::updateSearcher()
{
Wt::Dbo::Transaction transaction {_db.getSession()};
std::vector<Database::IdType> trackIds {Database::Track::getAllIdsWithFeatures(_db.getSession())};
transaction.commit();
LMS_LOG(SIMILARITY, INFO) << "Updating searcher...";
std::vector<Database::IdType> trackIds;
{
Wt::Dbo::Transaction transaction {_db.getSession()};
trackIds = Database::Track::getAllIdsWithFeatures(_db.getSession());
}
if (trackIds.empty())
{
@@ -127,7 +136,7 @@ FeaturesScannerAddon::updateSearcher()
return;
}
auto searcher {std::make_shared<Similarity::FeaturesSearcher>(_db.getSession(), _stopRequested)};
auto searcher {std::make_shared<Similarity::FeaturesSearcher>(_db.getSession(), [&]() { return _stopRequested; })};
if (searcher->isValid())
{
std::atomic_store(&_searcher, searcher);
@@ -137,8 +146,10 @@ FeaturesScannerAddon::updateSearcher()
LMS_LOG(DBUPDATER, INFO) << "New features similarity searcher instanciated";
}
else
{
LMS_LOG(DBUPDATER, ERROR) << "Cannot set up a valid features similarity searcher!";
std::atomic_store(&_searcher, std::shared_ptr<FeaturesSearcher>{});
}
}
bool
@@ -150,7 +161,7 @@ FeaturesScannerAddon::fetchFeatures(Database::IdType trackId, const std::string&
std::string data {AcousticBrainz::extractLowLevelFeatures(MBID)};
if (data.empty())
{
LMS_LOG(DBUPDATER, ERROR) << "Cannot extract features using AcousticBrainz!";
LMS_LOG(DBUPDATER, ERROR) << "Track " << trackId << ", MBID = '" << MBID << "': cannot extract features using AcousticBrainz";
return false;
}
src/similarity/features/SimilarityFeaturesSearcher.cpp
+25 -16
View File
@@ -119,8 +119,10 @@ getInputVectorWeights(const FeatureInfoMap& featuresInfo, std::size_t nbDimensio
return weights;
}
FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequested)
FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, std::function<bool()> stopRequested)
{
LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher...";
Wt::Dbo::Transaction transaction {session};
FeatureInfoMap featuresInfo {getFeatureInfoMap(session)};
@@ -143,7 +145,7 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
LMS_LOG(SIMILARITY, DEBUG) << "Extracting features...";
for (Database::IdType trackId : trackIds)
{
if (stopRequested)
if (stopRequested())
return;
boost::optional<SOM::InputVector> inputVector {getInputVectorFromTrack(session, trackId, featuresInfo, nbDimensions)};
@@ -181,20 +183,18 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
LMS_LOG(SIMILARITY, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount;
}};
auto stopper{[&]() { return stopRequested; }};
LMS_LOG(SIMILARITY, DEBUG) << "Training network...";
network.train(samples, 10, progressIndicator, stopper);
network.train(samples, 10, progressIndicator, stopRequested);
LMS_LOG(SIMILARITY, DEBUG) << "Training network DONE";
if (stopRequested)
if (stopRequested())
return;
LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks...";
std::map<Database::IdType, std::set<SOM::Position>> trackPositions;
for (std::size_t i {}; i < samples.size(); ++i)
{
if (stopRequested)
if (stopRequested())
return;
const SOM::Position position {network.getClosestRefVectorPosition(samples[i])};
@@ -204,15 +204,18 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE";
init(session, std::move(network), std::move(trackPositions));
init(session, std::move(network), std::move(trackPositions), stopRequested);
LMS_LOG(SIMILARITY, INFO) << "Successfully constructed features searcher";
}
FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache)
FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache, std::function<bool()> stopRequested)
{
init(session, std::move(cache._network), std::move(cache._trackPositions));
LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher from cache...";
LMS_LOG(SIMILARITY, DEBUG) << "Init from cache DONE";
init(session, std::move(cache._network), std::move(cache._trackPositions), stopRequested);
LMS_LOG(SIMILARITY, INFO) << "Successfully constructed features searcher from cache";
}
bool
@@ -318,14 +321,15 @@ FeaturesSearcher::toCache() const
void
FeaturesSearcher::init(Wt::Dbo::Session& session,
SOM::Network network,
std::map<Database::IdType, std::set<SOM::Position>> tracksPosition)
std::map<Database::IdType,
std::set<SOM::Position>> tracksPosition,
std::function<bool()> stopRequested)
{
_network = std::make_unique<SOM::Network>(std::move(network));
_networkRefVectorsDistanceMedian = _network->computeRefVectorsDistanceMedian();
_networkRefVectorsDistanceMedian = network.computeRefVectorsDistanceMedian();
LMS_LOG(SIMILARITY, DEBUG) << "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian;
SOM::Coordinate width {_network->getWidth()};
SOM::Coordinate height {_network->getHeight()};
SOM::Coordinate width {network.getWidth()};
SOM::Coordinate height {network.getHeight()};
_artistsMap = SOM::Matrix<std::set<Database::IdType>>{width, height};
_releasesMap = SOM::Matrix<std::set<Database::IdType>>{width, height};
@@ -335,6 +339,9 @@ FeaturesSearcher::init(Wt::Dbo::Session& session,
for (auto itTrackCoord : tracksPosition)
{
if (stopRequested())
return;
Wt::Dbo::Transaction transaction {session};
Database::IdType trackId {itTrackCoord.first};
@@ -362,6 +369,8 @@ FeaturesSearcher::init(Wt::Dbo::Session& session,
}
}
_network = std::make_unique<SOM::Network>(std::move(network));
LMS_LOG(SIMILARITY, DEBUG) << "Constructing maps... DONE";
}
src/similarity/features/SimilarityFeaturesSearcher.hpp
+4 -3
View File
@@ -36,10 +36,10 @@ class FeaturesSearcher
public:
// Use cache
FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache);
FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache, std::function<bool()> stopRequested);
// Use training (may be very slow)
FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequested);
FeaturesSearcher(Wt::Dbo::Session& session, std::function<bool()> stopRequested);
bool isValid() const;
@@ -61,7 +61,8 @@ class FeaturesSearcher
void init(Wt::Dbo::Session& session,
SOM::Network network,
ObjectPositions tracksPosition);
ObjectPositions tracksPosition,
std::function<bool()> stopRequested);
std::vector<Database::IdType> getSimilarObjects(const std::set<Database::IdType>& ids,
const SOM::Matrix<std::set<Database::IdType>>& objectsMap,