WIP. Database updater now has its own namespace and dedicated directory

This commit is contained in:
emeric
2014-07-01 23:43:49 +02:00
parent 7fd546eb44
commit 16d1194d61
35 changed files with 320 additions and 224 deletions
-44
View File
@@ -1,44 +0,0 @@
#include <fstream>
#include <stdexcept>
#include <boost/crc.hpp> // for boost::crc_32_type
#include "Checksum.hpp"
typedef boost::crc_32_type crc_type;
void computeCrc(const boost::filesystem::path& p, std::vector<unsigned char>& crc)
{
crc_type result;
std::ifstream ifs( p.string().c_str(), std::ios_base::binary );
if (ifs)
{
do
{
std::array<char,1024> buffer;
ifs.read( buffer.data(), buffer.size() );
result.process_bytes( buffer.data(), ifs.gcount() );
}
while ( ifs );
}
else
{
std::cerr << "Failed to open file '" << p << "'" << std::endl;
throw std::runtime_error("Failed to open file '" + p.string() + "'" );
}
// Copy back result into the vector
// Copy the result into a vector of unsigned char
const crc_type::value_type checksum = result.checksum();
for (std::size_t i = 0; (i+1)*8 <= crc_type::bit_count; i++)
{
const unsigned char* data = reinterpret_cast<const unsigned char*>( &checksum );
crc.push_back(data[i]);
}
}
-6
View File
@@ -1,6 +0,0 @@
#include <vector>
#include <boost/filesystem.hpp>
void computeCrc(const boost::filesystem::path& p, std::vector<unsigned char>& checksum);
-433
View File
@@ -1,433 +0,0 @@
#include <boost/filesystem.hpp>
#include <boost/foreach.hpp>
#include <boost/thread.hpp>
#include "Database.hpp"
#include "Checksum.hpp"
#include "AudioTypes.hpp"
#include "transcode/InputMediaFile.hpp"
namespace Database {
Database::Database(boost::filesystem::path dbPath, MetaData::Parser& parser)
: _db(dbPath),
_metadataParser(parser)
{
}
void
Database::watchDirectory(WatchedDirectory directory)
{
_directories.push_back(directory);
}
void
Database::unwatchDirectory(WatchedDirectory directory)
{
std::list<WatchedDirectory>::iterator it = std::find(_directories.begin(), _directories.end(), directory);
if (it != _directories.end())
_directories.erase(it);
}
void
Database::refresh(void)
{
removeMissingAudioFiles();
BOOST_FOREACH( const WatchedDirectory& directory, _directories) {
switch (directory.getType()) {
case WatchedDirectory::Audio:
refreshAudioDirectory(directory.getPath());
break;
case WatchedDirectory::Video:
refreshVideoDirectory(directory.getPath());
break;
default:
assert(0);
}
}
}
void
Database::processAudioFile( const boost::filesystem::path& file)
{
try {
// Check last update time
boost::posix_time::ptime lastWriteTime (boost::posix_time::from_time_t( boost::filesystem::last_write_time( file ) ) );
Wt::Dbo::Transaction transaction(_db.getSession());
// Skip file if last write is the same
Wt::Dbo::ptr<Track> track = Track::getByPath(_db.getSession(), file);
if (track && track->getLastWriteTime() == lastWriteTime)
{
std::cerr << "Skipped '" << file << "' (last write time match)" << std::endl;
return;
}
std::vector<unsigned char> checksum;
/* Compute CRC */
computeCrc( file, checksum );
// Skip file if its checksum is still the same
if (track && track->getChecksum() == checksum) {
std::cerr << "Skipped '" << file << "' (checksum match)" << std::endl;
return;
}
std::cout << "parsing file " << file << std::endl;
MetaData::Items items;
_metadataParser.parse(file, items);
// We estimate this is a audio file if:
// - we found a least one audio
// - there is no video stream
// - the duration is not null
Transcode::InputMediaFile mediaFile(file);
/* if (!mediaFile.getStreams( Transcode::Stream::Video ).empty())
{
std::cerr << "Skipped '" << file << "' (estimated video)" << std::endl;
return;
}
else */if (mediaFile.getStreams( Transcode::Stream::Audio ).empty())
{
std::cerr << "Skipped '" << file << "' (no audio stream found)" << std::endl;
return;
}
else if (mediaFile.getDuration().total_seconds() == 0)
{
std::cerr << "Skipped '" << file << "' (duration null!)" << std::endl;
// If Track exists here, delete it!
if (track)
track.remove();
}
std::string title;
if (items.find(MetaData::Title) != items.end()) {
title = boost::any_cast<std::string>(items[MetaData::Title]);
}
else
{
// TODO parse file name guess track etc.
// For now juste use file name as title
title = file.filename().string();
}
// ***** Artist
Wt::Dbo::ptr<Artist> artist;
if (items.find(MetaData::Artist) != items.end())
{
const std::string artistName (boost::any_cast<std::string>(items[MetaData::Artist]));
artist = Artist::getByName(_db.getSession(), artistName );
if (!artist)
artist = Artist::create( _db.getSession(), artistName );
}
else
artist = Artist::getNone(_db.getSession());
assert(artist);
// ***** Release
Wt::Dbo::ptr<Release> release;
if (items.find(MetaData::Album) != items.end())
{
const std::string albumName (boost::any_cast<std::string>(items[MetaData::Album]));
release = Release::getByName(_db.getSession(), albumName);
if (!release)
release = Release::create( _db.getSession(), albumName );
}
else
release = Release::getNone( _db.getSession() );
assert(release);
// ***** Genres
typedef std::list<std::string> GenreList;
GenreList genreList;
std::vector< Genre::pointer > genres;
if (items.find(MetaData::Genre) != items.end())
{
genreList = (boost::any_cast<GenreList>(items[MetaData::Genre]));
BOOST_FOREACH(const std::string& genre, genreList) {
Genre::pointer dbGenre ( Genre::getByName(_db.getSession(), genre) );
if (!dbGenre)
dbGenre = Genre::create(_db.getSession(), genre);
genres.push_back( dbGenre );
}
}
if (genres.empty())
genres.push_back( Genre::getNone( _db.getSession() ));
assert( !genres.empty() );
// If file already exist, update data
// Otherwise, create it
if (!track)
{
// Create a new song
track = Track::create(_db.getSession(), file, artist, release);
std::cout << "Adding '" << file << "'" << std::endl;
}
else
{
std::cout << "Updating '" << file << "'" << std::endl;
}
assert(track);
track.modify()->setChecksum(checksum);
track.modify()->setLastWriteTime(lastWriteTime);
track.modify()->setName(title);
{
std::string trackGenreList;
// Product genre list
BOOST_FOREACH(const std::string& genre, genreList) {
if (!trackGenreList.empty())
trackGenreList += ", ";
trackGenreList += genre;
}
std::cout << "Genre list = " << trackGenreList << std::endl;
track.modify()->setGenres( trackGenreList );
}
track.modify()->setGenres( genres );
track.modify()->setArtist( artist );
track.modify()->setRelease( release );
if (items.find(MetaData::TrackNumber) != items.end())
track.modify()->setTrackNumber( boost::any_cast<std::size_t>(items[MetaData::TrackNumber]) );
if (items.find(MetaData::DiscNumber) != items.end())
track.modify()->setDiscNumber( boost::any_cast<std::size_t>(items[MetaData::DiscNumber]) );
if (items.find(MetaData::Duration) != items.end())
track.modify()->setDuration( boost::any_cast<boost::posix_time::time_duration>(items[MetaData::Duration]) );
if (items.find(MetaData::CreationTime) != items.end())
track.modify()->setCreationTime( boost::any_cast<boost::posix_time::ptime>(items[MetaData::CreationTime]) );
transaction.commit();
}
catch( std::exception& e ) {
std::cerr << "Exception while parsing audio file : '" << file << "': '" << e.what() << "' => skipping!" << std::endl;
}
}
void
Database::refreshAudioDirectory( const boost::filesystem::path& p)
{
std::cout << "Refreshing audio directory " << p << std::endl;
if (boost::filesystem::exists(p) && boost::filesystem::is_directory(p)) {
typedef std::vector<boost::filesystem::path> Paths; // store paths,
Paths files;
std::copy(boost::filesystem::directory_iterator(p), boost::filesystem::directory_iterator(), std::back_inserter(files));
BOOST_FOREACH(const boost::filesystem::path& file, files) {
boost::this_thread::interruption_point();
try {
if (boost::filesystem::is_directory(file)) {
refreshAudioDirectory( file );
}
else if (boost::filesystem::is_regular(file)) {
processAudioFile( file );
}
else {
std::cout << "Skipped '" << file << "' (not regular)" << std::endl;
}
}
catch(std::exception& e) {
std::cerr << "Exception while accessing '" << file << ": " << e.what() << std::endl;
}
}
}
std::cout << "Refreshing audio directory " << p << ": DONE" << std::endl;
}
void
Database::removeMissingAudioFiles( void )
{
std::cerr << "Removing missing files..." << std::endl;
Wt::Dbo::Transaction transaction(_db.getSession());
typedef Wt::Dbo::collection< Wt::Dbo::ptr<Track> > Tracks;
Tracks tracks = Track::getAll(_db.getSession());
for (Tracks::iterator i = tracks.begin(); i != tracks.end(); ++i)
{
const boost::filesystem::path p ((*i)->getPath() );
if (!boost::filesystem::exists( p )
|| !boost::filesystem::is_regular( p ) )
{
(*i).remove();
std::cerr << "Removing file '" << p << "'" << std::endl;
}
}
transaction.commit();
std::cerr << "Refrshing missing files done!" << std::endl;
}
Path::pointer
Database::getAddPath(const boost::filesystem::path& path)
{
Path::pointer res;
Path::pointer parentDirectory;
if (path.has_parent_path())
parentDirectory = Path::getByPath(_db.getSession(), path.parent_path());
res = Path::getByPath(_db.getSession(), path);
if (!res)
res = Path::create(_db.getSession(), path, parentDirectory);
else
{
// Make sure the parent directory owns the child
if (parentDirectory && !res->getParent())
parentDirectory.modify()->addChild( res );
}
return res;
}
void
Database::refreshVideoDirectory( const boost::filesystem::path& path)
{
std::cout << "Refreshing video directory " << path << std::endl;
if (boost::filesystem::exists(path) && boost::filesystem::is_directory(path))
{
// Add this directory in the database
{
Wt::Dbo::Transaction transaction(_db.getSession());
Path::pointer pathDirectory = getAddPath(path);
assert( pathDirectory->isDirectory() );
transaction.commit();
}
// Now process all files/dirs in directory
typedef std::vector<boost::filesystem::path> Paths; // store paths,
Paths pathChildren;
std::copy(boost::filesystem::directory_iterator(path), boost::filesystem::directory_iterator(), std::back_inserter(pathChildren));
BOOST_FOREACH(const boost::filesystem::path& pathChild, pathChildren) {
if (boost::filesystem::is_directory(pathChild)) {
refreshVideoDirectory( pathChild );
}
else if (boost::filesystem::is_regular(pathChild)) {
processVideoFile( pathChild );
}
else {
std::cout << "Skipped '" << pathChild << "' (not regular)" << std::endl;
}
}
}
std::cout << "Refreshing video directory " << path << ": DONE" << std::endl;
}
void
Database::processVideoFile( const boost::filesystem::path& file)
{
try {
// Check last update time
boost::posix_time::ptime lastWriteTime (boost::posix_time::from_time_t( boost::filesystem::last_write_time( file ) ) );
Wt::Dbo::Transaction transaction(_db.getSession());
// Skip file if last write is the same
Path::pointer dbPath = Path::getByPath(_db.getSession(), file);
if (dbPath && dbPath->getLastWriteTime() == lastWriteTime)
{
std::cerr << "Skipped '" << file << "' (last write time match)" << std::endl;
return;
}
std::cout << "Video, parsing file " << file << std::endl;
// TODO try to open the video file and get info on it
Transcode::InputMediaFile mediaFile(file);
// We estimate this is a video if:
// - we found a least one video stream
// - the duration is not null
std::vector<Transcode::Stream> videoStreams(mediaFile.getStreams( Transcode::Stream::Video ));
if (videoStreams.empty())
{
std::cerr << "Skipped '" << file << "' (no video stream found!)" << std::endl;
// If Path exist, delete it!
if (dbPath)
dbPath.remove();
}
else if (mediaFile.getDuration().total_seconds() == 0)
{
std::cerr << "Skipped '" << file << "' (duration null!)" << std::endl;
// If Path exist, delete it!
if (dbPath)
dbPath.remove();
}
else
{
// add Path if needed
if (!dbPath)
dbPath = getAddPath( file );
dbPath.modify()->setLastWriteTime( lastWriteTime );
assert(dbPath);
// Valid video here
// Today we are very aggressive, but we could also guess names from path, etc.
Video::pointer video = dbPath.modify()->getVideo();
if (!video) {
video = Video::create(_db.getSession(), dbPath);
std::cout << "Adding '" << file << "'" << std::endl;
}
else
std::cout << "Updating '" << file << "'" << std::endl;
assert(video);
video.modify()->setName( file.filename().string() );
video.modify()->setDuration( mediaFile.getDuration() );
}
transaction.commit();
}
catch( std::exception& e ) {
std::cerr << "Exception while parsing video file : '" << file << "': '" << e.what() << "' => skipping!" << std::endl;
}
}
} // namespace Database
-69
View File
@@ -1,69 +0,0 @@
#include "metadata/MetaData.hpp"
#include "DatabaseHandler.hpp"
#include "FileTypes.hpp"
namespace Database {
class WatchedDirectory {
public:
enum Type {
Audio,
Video,
};
WatchedDirectory( boost::filesystem::path path, Type type) : _path(path), _type(type) {
if (!boost::filesystem::is_directory(path))
throw std::runtime_error( "path '" + path.string() + "' is not a directory!");
}
bool operator==(WatchedDirectory const& other) { return _path == other._path && _type == other._type; }
const boost::filesystem::path& getPath() const { return _path; }
Type getType() const { return _type; }
private:
boost::filesystem::path _path;
Type _type;
};
class Database
{
public:
Database(boost::filesystem::path db, MetaData::Parser& parser);
void watchDirectory(WatchedDirectory dir);
void unwatchDirectory(WatchedDirectory dir);
// Rescan media folders
void refresh();
private:
void refresh(const WatchedDirectory& directory);
// Video
void refreshVideoDirectory( const boost::filesystem::path& directory );
void processVideoFile( const boost::filesystem::path& file);
// Audio
void removeMissingAudioFiles( void );
void refreshAudioDirectory( const boost::filesystem::path& directory);
void processAudioFile( const boost::filesystem::path& file);
Path::pointer getAddPath(const boost::filesystem::path& path);
DatabaseHandler _db;
std::list<WatchedDirectory> _directories;
MetaData::Parser& _metadataParser;
};
} // Database
+6 -1
View File
@@ -2,9 +2,11 @@
#include "AudioTypes.hpp"
#include "FileTypes.hpp"
#include "MediaDirectory.hpp"
namespace Database {
DatabaseHandler::DatabaseHandler(boost::filesystem::path db)
Handler::Handler(boost::filesystem::path db)
:
_path(db),
_dbBackend( db.string() )
@@ -17,6 +19,8 @@ _dbBackend( db.string() )
_session.mapClass<Database::Release>("release");
_session.mapClass<Database::Path>("path");
_session.mapClass<Database::Video>("video");
_session.mapClass<Database::MediaDirectory>("media_directory");
_session.mapClass<Database::MediaDirectorySettings>("media_directory_settings");
try {
_session.createTables();
@@ -28,3 +32,4 @@ _dbBackend( db.string() )
_dbBackend.executeSql("pragma journal_mode=WAL");
}
} // namespace Database
+5 -2
View File
@@ -6,11 +6,13 @@
#include <Wt/Dbo/Dbo>
#include <Wt/Dbo/backend/Sqlite3>
namespace Database {
// Long living class handling the database
class DatabaseHandler
class Handler
{
public:
DatabaseHandler(boost::filesystem::path db);
Handler(boost::filesystem::path db);
Wt::Dbo::Session& getSession() { return _session; }
@@ -25,6 +27,7 @@ class DatabaseHandler
};
} // namespace Database
#endif
+26
View File
@@ -0,0 +1,26 @@
#include "MediaDirectory.hpp"
namespace Database {
MediaDirectorySettings::pointer
MediaDirectorySettings::get(Wt::Dbo::Session& session)
{
MediaDirectorySettings::pointer res;
res = session.find<MediaDirectorySettings>().where("id = ?").bind(1);
if (!res)
res = session.add( new MediaDirectorySettings());
return res;
}
std::vector<MediaDirectory::pointer>
MediaDirectory::getAll(Wt::Dbo::Session& session)
{
Wt::Dbo::collection< MediaDirectory::pointer > res = session.find<MediaDirectory>();
return std::vector<MediaDirectory::pointer>(res.begin(), res.end());
}
} // namespace Database
+91
View File
@@ -0,0 +1,91 @@
#ifndef DATABASE_MEDIA_DIRECTORY_HPP
#define DATABASE_MEDIA_DIRECTORY_HPP
#include <vector>
#include <boost/filesystem/path.hpp>
#include <Wt/Dbo/Dbo>
#include <Wt/Dbo/WtSqlTraits>
namespace Database {
class MediaDirectory;
class MediaDirectorySettings
{
public:
typedef Wt::Dbo::ptr<MediaDirectorySettings> pointer;
MediaDirectorySettings() {}
// accessors
static pointer get(Wt::Dbo::Session& session);
// write accessors
void addMediaDirectory(Wt::Dbo::ptr<MediaDirectory> mediaDirectory);
void setLastUpdate(boost::posix_time::ptime time) { _lastUpdate = time; }
void setLastScan(boost::posix_time::ptime time) { _lastScan = time; }
// Read accessors
boost::posix_time::ptime getLastUpdated(void) const { return _lastUpdate; }
template<class Action>
void persist(Action& a)
{
Wt::Dbo::field(a, _lastUpdate, "last_update");
Wt::Dbo::field(a, _lastScan, "last_scan");
Wt::Dbo::hasMany(a, _mediaDirectories, Wt::Dbo::ManyToOne, "media_directory_settings");
}
private:
boost::posix_time::time_duration _updatePeriod; // TODO
boost::posix_time::ptime _lastUpdate; // last time the database has changed
boost::posix_time::ptime _lastScan; // last time the database has been scanned
Wt::Dbo::collection< Wt::Dbo::ptr<MediaDirectory> > _mediaDirectories; // list of media directories
};
class MediaDirectory
{
public:
typedef Wt::Dbo::ptr<MediaDirectory> pointer;
enum Type {
Audio = 1,
Video = 2,
};
MediaDirectory() {}
MediaDirectory(boost::filesystem::path p, Type type);
// Accessors
static std::vector<MediaDirectory::pointer> getAll(Wt::Dbo::Session& session);
Type getType(void) const { return _type; }
boost::filesystem::path getPath(void) const { return boost::filesystem::path(_path); }
template<class Action>
void persist(Action& a)
{
Wt::Dbo::field(a, _type, "type");
Wt::Dbo::field(a, _path, "path");
Wt::Dbo::belongsTo(a, _settings, "media_directory_settings", Wt::Dbo::OnDeleteCascade);
}
private:
Type _type;
std::string _path;
MediaDirectorySettings::pointer _settings; // back pointer
};
} // namespace Database
#endif