diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0d730c78..82119510 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -30,7 +30,7 @@ jobs: name: Install dependencies (cpp) run: | sudo apt-get update - sudo apt-get install --yes build-essential cmake libboost-all-dev libconfig++-dev libavcodec-dev libavutil-dev libavformat-dev libstb-dev libtag1-dev libpam0g-dev libgtest-dev + sudo apt-get install --yes build-essential cmake libboost-all-dev libconfig++-dev libavcodec-dev libavutil-dev libavformat-dev libstb-dev libtag1-dev libpam0g-dev libgtest-dev libarchive-dev export WT_VERSION=4.9.0 export WT_INSTALL_PREFIX=/usr git clone https://github.com/emweb/wt.git /tmp/wt diff --git a/CMakeLists.txt b/CMakeLists.txt index b13f35b0..a48f53d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ pkg_check_modules(Taglib REQUIRED IMPORTED_TARGET taglib) pkg_check_modules(Config++ REQUIRED IMPORTED_TARGET libconfig++) pkg_check_modules(GraphicsMagick++ IMPORTED_TARGET GraphicsMagick++) pkg_check_modules(LIBAV IMPORTED_TARGET libavcodec libavutil libavformat) +pkg_check_modules(Archive REQUIRED IMPORTED_TARGET libarchive) find_package(PAM) find_package(STB) diff --git a/Dockerfile-build-alpine b/Dockerfile-build-alpine index 80734408..ed896dfa 100644 --- a/Dockerfile-build-alpine +++ b/Dockerfile-build-alpine @@ -17,6 +17,7 @@ ARG LMS_BUILD_PACKAGES=" \ musl-dev \ boost-dev \ ffmpeg-dev \ + libarchive-dev \ libconfig-dev \ taglib-dev \ stb \ diff --git a/Dockerfile-build-arch b/Dockerfile-build-arch index 1d8ecd92..14eeac08 100644 --- a/Dockerfile-build-arch +++ b/Dockerfile-build-arch @@ -7,6 +7,7 @@ ARG BUILD_PACKAGES="\ ffmpeg \ gtest \ graphicsmagick \ + libarchive \ libconfig \ make \ pkgconfig \ diff --git a/Dockerfile-release b/Dockerfile-release index ad9a1193..77d6b2af 100644 --- a/Dockerfile-release +++ b/Dockerfile-release @@ -28,6 +28,7 @@ ARG BUILD_PACKAGES=" \ zlib-dev \ openssl-dev \ boost-dev \ + libarchive-dev \ libconfig-dev \ taglib-dev \ gtest-dev" @@ -149,6 +150,7 @@ ARG RUNTIME_PACKAGES=" \ boost-program_options \ boost-system \ boost-thread \ + libarchive \ libconfig++ \ taglib" diff --git a/INSTALL.md b/INSTALL.md index 4190eb08..96f92310 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,6 +1,6 @@ - [Installation](#installation) * [Docker](#docker) - * [Debian Bullseye packages](#debian-bullseye-packages) + * [Debian packages](#debian-packages) * [From source](#from-source) + [Build dependencies](#build-dependencies) + [Build](#build) @@ -15,12 +15,12 @@ # Installation ## Docker _Docker_ images are available, please see detailed instructions on https://hub.docker.com/r/epoupon/lms. -## Debian Bullseye packages -_Bullseye_ packages are provided for _amd64_ and _armhf_ architectures. +## Debian packages +_Bookworm_ packages are provided for _amd64_ architectures. As root, trust the following debian package provider and add it in your list of repositories: ```sh -wget -O - https://debian.poupon.dev/apt/debian/epoupon.gpg.key | apt-key add - -echo "deb https://debian.poupon.dev/apt/debian bullseye main" > /etc/apt/sources.list.d/epoupon.list +wget https://debian.poupon.dev/apt/debian/epoupon.gpg -P /usr/share/keyrings +echo "deb [signed-by=/usr/share/keyrings/epoupon.gpg] https://debian.poupon.dev/apt/debian bookworm main" > /etc/apt/sources.list.d/epoupon.list ``` To install or upgrade _LMS_: ```sh @@ -30,7 +30,7 @@ apt install lms The _lms_ service is started just after the package installation, run by a dedicated _lms_ system user.
Please refer to [Deployment](#deployment) for further configuration options. ## From source -__Note__: this installation process and the default values of the configuration files have been written for _Debian Bullseye_. Therefore, you may have to adapt commands and/or paths in order to fit to your distribution. +__Note__: this installation process and the default values of the configuration files have been written for _Debian Bookworm_. Therefore, you may have to adapt commands and/or paths in order to fit to your distribution. ### Build dependencies __Notes__: * a C++17 compiler is needed diff --git a/approot/release.xml b/approot/release.xml index f67237ec..e836e793 100644 --- a/approot/release.xml +++ b/approot/release.xml @@ -15,7 +15,7 @@

${name}

${}${artists}${} ${}
${tr:Lms.Explore.various-artists}
${} -
${}${year}${} (${orig-year})${} · ${}${duration}
+
${}${year} · ${}${duration}
${clusters class="mb-2"}
${play-btn class="btn btn-primary"} diff --git a/src/libs/av/impl/TranscodeResourceHandler.hpp b/src/libs/av/impl/TranscodeResourceHandler.hpp index 47d76f36..ff254b63 100644 --- a/src/libs/av/impl/TranscodeResourceHandler.hpp +++ b/src/libs/av/impl/TranscodeResourceHandler.hpp @@ -36,6 +36,7 @@ namespace Av private: Wt::Http::ResponseContinuation* processRequest(const Wt::Http::Request& request, Wt::Http::Response& reponse) override; + void abort() override {}; static constexpr std::size_t _chunkSize {32768}; std::optional _estimatedContentLength; diff --git a/src/libs/services/auth/CMakeLists.txt b/src/libs/services/auth/CMakeLists.txt index 07400a6b..0a2eb6a5 100644 --- a/src/libs/services/auth/CMakeLists.txt +++ b/src/libs/services/auth/CMakeLists.txt @@ -24,7 +24,6 @@ target_link_libraries(lmsauth PRIVATE ) target_link_libraries(lmsauth PUBLIC - pthread Boost::system Wt::Wt ) diff --git a/src/libs/services/database/impl/Release.cpp b/src/libs/services/database/impl/Release.cpp index d26bfc40..781a3017 100644 --- a/src/libs/services/database/impl/Release.cpp +++ b/src/libs/services/database/impl/Release.cpp @@ -41,6 +41,9 @@ createQuery(Session& session, const Release::FindParameters& params) auto query {session.getDboSession().query("SELECT DISTINCT r.id from release r")}; if (params.sortMethod == ReleaseSortMethod::LastWritten + || params.sortMethod == ReleaseSortMethod::Date + || params.sortMethod == ReleaseSortMethod::OriginalDate + || params.sortMethod == ReleaseSortMethod::OriginalDateDesc || params.writtenAfter.isValid() || params.dateRange || params.artist.isValid()) @@ -160,8 +163,11 @@ createQuery(Session& session, const Release::FindParameters& params) case ReleaseSortMethod::Date: query.orderBy("t.date, r.name COLLATE NOCASE"); break; - case ReleaseSortMethod::DateDesc: - query.orderBy("t.date DESC, r.name COLLATE NOCASE"); + case ReleaseSortMethod::OriginalDate: + query.orderBy("CASE WHEN t.original_date IS NULL THEN t.date ELSE t.original_date END, t.date, r.name COLLATE NOCASE"); + break; + case ReleaseSortMethod::OriginalDateDesc: + query.orderBy("CASE WHEN t.original_date IS NULL THEN t.date ELSE t.original_date END DESC, t.date, r.name COLLATE NOCASE"); break; case ReleaseSortMethod::StarredDateDesc: assert(params.starringUser.isValid()); diff --git a/src/libs/services/database/include/services/database/Types.hpp b/src/libs/services/database/include/services/database/Types.hpp index 57085732..0e50cb39 100644 --- a/src/libs/services/database/include/services/database/Types.hpp +++ b/src/libs/services/database/include/services/database/Types.hpp @@ -98,7 +98,8 @@ namespace Database None, Name, Date, - DateDesc, + OriginalDate, + OriginalDateDesc, Random, LastWritten, StarredDateDesc, diff --git a/src/libs/services/database/test/Release.cpp b/src/libs/services/database/test/Release.cpp index f0d90f70..573ed9c7 100644 --- a/src/libs/services/database/test/Release.cpp +++ b/src/libs/services/database/test/Release.cpp @@ -530,3 +530,73 @@ TEST_F(DatabaseFixture, Release_releaseType) EXPECT_TRUE(release.get()->getSecondaryTypes().contains(ReleaseTypeSecondary::Compilation)); } } + +TEST_F(DatabaseFixture, ReleaseSortOrder) +{ + ScopedRelease release1 {session, "MyRelease1"}; + const Wt::WDate release1Date {Wt::WDate {2000, 2, 3}}; + const Wt::WDate release1OriginalDate {Wt::WDate {1993, 4, 5}}; + + ScopedRelease release2 {session, "MyRelease2"}; + const Wt::WDate release2Date {Wt::WDate {1994, 2, 3}}; + + ScopedTrack track1 {session, "MyTrack1"}; + ScopedTrack track2 {session, "MyTrack2"}; + + ASSERT_LT(release2Date, release1Date); + ASSERT_GT(release2Date, release1OriginalDate); + + { + auto transaction {session.createUniqueTransaction()}; + + track1.get().modify()->setRelease(release1.get()); + track1.get().modify()->setOriginalDate(release1OriginalDate); + track1.get().modify()->setDate(release1Date); + + track2.get().modify()->setRelease(release2.get()); + track2.get().modify()->setDate(release2Date); + } + + { + auto transaction {session.createSharedTransaction()}; + + const auto releases {Release::find(session, Release::FindParameters {}.setSortMethod(ReleaseSortMethod::Name) )}; + ASSERT_EQ(releases.results.size(), 2); + EXPECT_EQ(releases.results.front(), release1.getId()); + EXPECT_EQ(releases.results.back(), release2.getId()); + } + + { + auto transaction {session.createSharedTransaction()}; + + const auto releases {Release::find(session, Release::FindParameters {}.setSortMethod(ReleaseSortMethod::Random) )}; + ASSERT_EQ(releases.results.size(), 2); + } + + { + auto transaction {session.createSharedTransaction()}; + + const auto releases {Release::find(session, Release::FindParameters {}.setSortMethod(ReleaseSortMethod::Date) )}; + ASSERT_EQ(releases.results.size(), 2); + EXPECT_EQ(releases.results.front(), release2.getId()); + EXPECT_EQ(releases.results.back(), release1.getId()); + } + + { + auto transaction {session.createSharedTransaction()}; + + const auto releases {Release::find(session, Release::FindParameters {}.setSortMethod(ReleaseSortMethod::OriginalDate) )}; + ASSERT_EQ(releases.results.size(), 2); + EXPECT_EQ(releases.results.front(), release1.getId()); + EXPECT_EQ(releases.results.back(), release2.getId()); + } + { + auto transaction {session.createSharedTransaction()}; + + const auto releases {Release::find(session, Release::FindParameters {}.setSortMethod(ReleaseSortMethod::OriginalDateDesc) )}; + ASSERT_EQ(releases.results.size(), 2); + EXPECT_EQ(releases.results.front(), release2.getId()); + EXPECT_EQ(releases.results.back(), release1.getId()); + } +} + diff --git a/src/libs/services/scrobbling/impl/listenbrainz/ListensParser.cpp b/src/libs/services/scrobbling/impl/listenbrainz/ListensParser.cpp index 765b5b5c..3b2787fd 100644 --- a/src/libs/services/scrobbling/impl/listenbrainz/ListensParser.cpp +++ b/src/libs/services/scrobbling/impl/listenbrainz/ListensParser.cpp @@ -55,7 +55,8 @@ namespace listen.recordingMBID = UUID::fromString(additionalInfo.get("recording_mbid").orIfNull("")); listen.releaseMBID = UUID::fromString(additionalInfo.get("release_mbid").orIfNull("")); - int trackNumber {additionalInfo.get("tracknumber").orIfNull(-1)}; + // tracknumber should be an integer but some players encode as strings + int trackNumber {additionalInfo.get("tracknumber").toNumber().orIfNull(-1)}; if (trackNumber > 0) listen.trackNumber = trackNumber; } diff --git a/src/libs/services/scrobbling/test/Listenbrainz.cpp b/src/libs/services/scrobbling/test/Listenbrainz.cpp index 4d0684c9..565f06f8 100644 --- a/src/libs/services/scrobbling/test/Listenbrainz.cpp +++ b/src/libs/services/scrobbling/test/Listenbrainz.cpp @@ -107,6 +107,7 @@ TEST(Listenbrainz, parseListens_multiArtists) ListensParser::Result result {ListensParser::parse(R"({"payload":{"count":1,"latest_listen_ts":1664106427,"listens":[{"inserted_at":1664106427,"listened_at":1664106427,"recording_msid":"b1dad0df-329b-443d-bacf-cdbebdddbfd0","track_metadata":{"additional_info":{"artist_mbids":["04ce0202-043d-4cbe-8f09-8abaf3b80c71","79311c51-9748-49df-baa1-d925fd29f4e8"],"artist_msid":null,"listening_from":"LMS","recording_mbid":"a5f380bc-0a85-4a9f-88db-d41bb9aa2a4b","recording_msid":"b1dad0df-329b-443d-bacf-cdbebdddbfd0","release_mbid":"147b4669-3d20-43f8-89c0-ba1da8b87dd3","release_msid":null,"track_mbid":"a20dd067-29b6-3d38-a0be-eeb86b4671c1","tracknumber":1},"artist_name":"Gloom","release_name":"Demovibes 9: Party, people going","track_name":"Stargazer of Disgrace"},"user_name":"epoupon"}],"user_id":"epoupon"}})")}; EXPECT_EQ(result.listenCount, 1); ASSERT_EQ(result.listens.size(), 1); + EXPECT_EQ(result.listens[0].trackNumber, 1); } TEST(Listenbrainz, parseListens_minPayload) @@ -120,4 +121,15 @@ TEST(Listenbrainz, parseListens_minPayload) EXPECT_EQ(result.listens[0].releaseName, ""); EXPECT_FALSE(result.listens[0].recordingMBID.has_value()); EXPECT_FALSE(result.listens[0].releaseMBID.has_value()); + EXPECT_EQ(result.listens[0].trackNumber, std::nullopt); } + +TEST(Listenbrainz, parseListens_stringInsteadOfInt) +{ + ListensParser::Result result {ListensParser::parse(R"({"payload":{"count":1,"latest_listen_ts":1683298766,"listens":[{"inserted_at":1683299849,"listened_at":1683298766,"recording_msid":"3841ad1c-f674-491c-b855-f10a82ea38fe","track_metadata":{"additional_info":{"albumartist":"Various Artists","artist_mbids":["80ccfede-c258-4575-a7ad-c982e9932e0f"],"date":"2002-06-10","discnumber":"1","duration":190,"isrc":"USAM10200204","media_player":"foobar2000","media_player_version":"1.6.16","recording_msid":"3841ad1c-f674-491c-b855-f10a82ea38fe","release_group_mbid":"25f44677-2ecd-33fb-8454-9a9776496e3c","release_mbid":"bddad4c6-e9e1-4218-b9d2-b02a5a778a7d","submission_client":"foobar2000","submission_client_version":"1.3.2","totaldiscs":"1","totaltracks":"18","track_mbid":"08b1a5c1-c845-3d3b-9aa4-560a1af7f28c","tracknumber":"4"},"artist_name":"Sheryl Crow","mbid_mapping":{"artist_mbids":["80ccfede-c258-4575-a7ad-c982e9932e0f"],"artists":[{"artist_credit_name":"Sheryl Crow","artist_mbid":"80ccfede-c258-4575-a7ad-c982e9932e0f","join_phrase":""}],"caa_id":22039936498,"caa_release_mbid":"f10b4b40-4db1-4b9a-a223-06915fa06b3c","recording_mbid":"7ca21c82-1345-44df-b7ac-035a6e05d4ed","recording_name":"Strong Enough","release_mbid":"f10b4b40-4db1-4b9a-a223-06915fa06b3c"},"release_name":"The Very Best of MTV Unplugged","track_name":"Strong Enough"},"user_name":"awesomeuser"}],"user_id":"awesomeuser"}})")}; + + EXPECT_EQ(result.listenCount, 1); + ASSERT_EQ(result.listens.size(), 1); + EXPECT_EQ(result.listens[0].trackNumber, 4); +} + diff --git a/src/libs/utils/CMakeLists.txt b/src/libs/utils/CMakeLists.txt index 261ec14c..a1f3590d 100644 --- a/src/libs/utils/CMakeLists.txt +++ b/src/libs/utils/CMakeLists.txt @@ -1,6 +1,7 @@ add_library(lmsutils SHARED impl/http/Client.cpp impl/http/SendQueue.cpp + impl/ArchiveZipper.cpp impl/ChildProcess.cpp impl/ChildProcessManager.cpp impl/Config.cpp @@ -15,7 +16,6 @@ add_library(lmsutils SHARED impl/String.cpp impl/UUID.cpp impl/WtLogger.cpp - impl/Zipper.cpp ) target_include_directories(lmsutils INTERFACE @@ -28,6 +28,7 @@ target_include_directories(lmsutils PRIVATE target_link_libraries(lmsutils PRIVATE PkgConfig::Config++ + PkgConfig::Archive ) target_link_libraries(lmsutils PUBLIC diff --git a/src/libs/utils/impl/ArchiveZipper.cpp b/src/libs/utils/impl/ArchiveZipper.cpp new file mode 100644 index 00000000..e4d8c850 --- /dev/null +++ b/src/libs/utils/impl/ArchiveZipper.cpp @@ -0,0 +1,309 @@ +/* + * Copyright (C) 2020 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ArchiveZipper.hpp" + +#include +#include +#include // strerror +#include +#include +#include + +#include "utils/Logger.hpp" + +namespace Zip +{ + std::unique_ptr + createArchiveZipper(const EntryContainer& entries) + { + return std::make_unique(entries); + } + + class FileException : public Exception + { + public: + FileException(const std::filesystem::path& p, std::string_view message) + : Exception {"File '" + p.string() + "': " + std::string {message}} + {} + + FileException(const std::filesystem::path& p, std::string_view message, int err) + : Exception {"File '" + p.string() + "': " + std::string {message} + ": " + ::strerror(err)} + {} + }; + + class ArchiveException : public Exception + { + public: + ArchiveException(struct ::archive* arch) + : Exception {getError(arch)} + {} + + static std::string_view getError(struct ::archive* arch) + { + const char* str {archive_error_string(arch)}; + if (!str) + { + static std::string unknownError {"Unknown archive error"}; + return unknownError; + } + + return str; + } + }; + + void + ArchiveZipper::ArchiveDeleter::operator()(struct ::archive* arch) + { + const int res {::archive_write_free(arch)}; + if (res != ARCHIVE_OK) + LMS_LOG(UTILS, ERROR) << "Failure while freeing archive control struct: " << std::string {::strerror(res)}; + } + + void + ArchiveZipper::ArchiveEntryDeleter::operator()(struct ::archive_entry* archEntry) + { + ::archive_entry_free(archEntry); + } + + ArchiveZipper::ArchiveZipper(const EntryContainer& entries) + : _entries {entries} + , _readBuffer(_readBufferSize, {}) + , _currentEntry {std::cbegin(_entries)} + { + _archive = ArchivePtr {::archive_write_new()}; + if (!_archive) + throw Exception {"Cannot create archive control struct"}; + + auto archiveOpen {[](struct ::archive*, void*) + { + return ARCHIVE_OK; + }}; + + auto archiveWrite {[](struct ::archive*, void* clientData, const void* buff, ::size_t n) -> la_ssize_t + { + ArchiveZipper* zipper {static_cast(clientData)}; + return zipper->onWriteCallback(static_cast(buff), n); + }}; + + auto archiveClose {[](struct ::archive*, void*) + { + return ARCHIVE_OK; + }}; + + if (::archive_write_set_bytes_per_block(_archive.get(), _writeBlockSize) != ARCHIVE_OK) + throw ArchiveException {_archive.get()}; + + // 1 => no padding for last block + if (::archive_write_set_bytes_in_last_block(_archive.get(), 1) != ARCHIVE_OK) + throw ArchiveException {_archive.get()}; + + if (::archive_write_set_format_zip(_archive.get()) != ARCHIVE_OK) + throw ArchiveException {_archive.get()}; + + if (::archive_write_set_option(_archive.get(), "zip", "compression", "deflate") != ARCHIVE_OK) + throw ArchiveException {_archive.get()}; + + int res {::archive_write_open(_archive.get(), this, archiveOpen, archiveWrite, archiveClose)}; + if (res != ARCHIVE_OK) + throw ArchiveException {_archive.get()}; + } + + std::uint64_t + ArchiveZipper::writeSome(std::ostream& output) + { + assert(!_currentOutputStream); + + _currentOutputStream = &output; + _bytesWrittenInCurrentOutputStream = 0; + + while (_bytesWrittenInCurrentOutputStream == 0) + { + if (!_currentArchiveEntry) + { + if (_currentEntry == std::cend(_entries)) + { + if (::archive_write_close(_archive.get()) != ARCHIVE_OK) + throw ArchiveException {_archive.get()}; + + _archive.reset(); + break; + } + + _currentArchiveEntry = createArchiveEntry(*_currentEntry); + _currentEntryOffset = 0; + if (::archive_write_header(_archive.get(), _currentArchiveEntry.get()) != ARCHIVE_OK) + throw ArchiveException {_archive.get()}; + } + + if (writeSomeCurrentFileData()) + { + // entry complete + if (::archive_write_finish_entry(_archive.get()) != ARCHIVE_OK) + throw ArchiveException {_archive.get()}; + + _currentArchiveEntry.reset(); + _currentEntry++; + } + } + + _currentOutputStream = nullptr; + return _bytesWrittenInCurrentOutputStream; + } + + bool + ArchiveZipper::isComplete() const + { + return !_archive; + } + + void + ArchiveZipper::abort() + { + LMS_LOG(UTILS, DEBUG) << "Aborting zip creation"; + if (_archive) + { + ::archive_write_fail(_archive.get()); + _archive.reset(); + } + } + + static + ::mode_t + permsToMode(const std::filesystem::perms p) + { + using std::filesystem::perms; + ::mode_t mode{}; + + auto testPerm {[](perms p, perms permToTest) + { + return (p & permToTest) == permToTest; + }}; + + if (testPerm(p, perms::owner_read)) + mode |= S_IRUSR; + if (testPerm(p, perms::owner_write)) + mode |= S_IWUSR; + if (testPerm(p, perms::owner_exec)) + mode |= S_IXUSR; + if (testPerm(p, perms::group_read)) + mode |= S_IRGRP; + if (testPerm(p, perms::group_write)) + mode |= S_IWGRP; + if (testPerm(p, perms::group_exec)) + mode |= S_IXGRP; + if (testPerm(p, perms::others_read)) + mode |= S_IROTH; + if (testPerm(p, perms::others_write)) + mode |= S_IWOTH; + if (testPerm(p, perms::others_exec)) + mode |= S_IXOTH; + + return mode; + } + + ArchiveZipper::ArchiveEntryPtr + ArchiveZipper::createArchiveEntry(const Entry& entry) + { + try + { + if (!std::filesystem::is_regular_file(entry.filePath)) + throw FileException {entry.filePath, "not a regular file"}; + + ArchiveEntryPtr archiveEntry {archive_entry_new()}; + if (!archiveEntry) + throw Exception {"Cannot create archive entry control struct"}; + + archive_entry_set_pathname(archiveEntry.get(), entry.fileName.c_str()); + archive_entry_set_size(archiveEntry.get(), std::filesystem::file_size(entry.filePath)); + archive_entry_set_mode(archiveEntry.get(), permsToMode(std::filesystem::status(entry.filePath).permissions())); + archive_entry_set_filetype(archiveEntry.get(), AE_IFREG); + + return archiveEntry; + } + catch (const std::filesystem::filesystem_error& error) + { + throw FileException {entry.filePath, error.what()}; + } + } + + bool + ArchiveZipper::writeSomeCurrentFileData() + { + assert(_currentEntry != std::cend(_entries)); + + std::ifstream ifs {_currentEntry->filePath.c_str(), std::ios_base::binary}; + if (!ifs) + throw FileException {_currentEntry->filePath, "cannot open file", errno}; + + ifs.seekg(0, std::ios::end); + const std::uint64_t fileSize {static_cast(ifs.tellg())}; + ifs.seekg(0, std::ios::beg); + + // TODO: store file size? + if (fileSize < _currentEntryOffset) + throw FileException {_currentEntry->filePath, "size changed?"}; + + const std::uint64_t bytesToRead {std::min(fileSize - _currentEntryOffset, static_cast(_readBufferSize))}; + + // read from file + if (!ifs.seekg(_currentEntryOffset, std::ios::beg)) + throw FileException {_currentEntry->filePath, "seek failed", errno}; + + if (!ifs.read(reinterpret_cast(&_readBuffer[0]), bytesToRead)) + throw FileException {_currentEntry->filePath, "read failed", errno}; + + const std::uint64_t actualBytesRead {static_cast(ifs.gcount())}; + + // write to archive + { + std::uint64_t remainingBytesToWrite {actualBytesRead}; + while (remainingBytesToWrite > 0) + { + const auto writtenBytes {archive_write_data(_archive.get(), &_readBuffer[actualBytesRead - remainingBytesToWrite], remainingBytesToWrite)}; + if (writtenBytes < 0) + throw ArchiveException {_archive.get()}; + + assert(static_cast(writtenBytes) <= remainingBytesToWrite); + remainingBytesToWrite -= writtenBytes; + } + } + + _currentEntryOffset += actualBytesRead; + return (_currentEntryOffset >= fileSize); + } + + std::int64_t + ArchiveZipper::onWriteCallback(const std::byte* buffer, std::size_t bufferSize) + { + if (!_currentOutputStream) + { + archive_set_error(_archive.get(), EIO, "IO error: operation cancelled"); + return -1; + } + + _currentOutputStream->write(reinterpret_cast(buffer), bufferSize); + if (!*_currentOutputStream) + throw Exception {"Failed to write " + std::to_string(bufferSize) + " bytes in final archive output!"}; + + _bytesWrittenInCurrentOutputStream += bufferSize; + + return bufferSize; + } +} // namespace Zip diff --git a/src/libs/utils/impl/ArchiveZipper.hpp b/src/libs/utils/impl/ArchiveZipper.hpp new file mode 100644 index 00000000..15b8b7af --- /dev/null +++ b/src/libs/utils/impl/ArchiveZipper.hpp @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include "utils/IZipper.hpp" + +extern "C" +{ + struct archive; + struct archive_entry; +} + +namespace Zip +{ + class ArchiveZipper : public IZipper + { + public: + ArchiveZipper(const EntryContainer& files); + ArchiveZipper(const ArchiveZipper&) = delete; + ArchiveZipper& operator=(const ArchiveZipper&) = delete; + + private: + std::uint64_t writeSome(std::ostream& output) override; + bool isComplete() const override; + void abort() override; + + class ArchiveDeleter + { + public: + void operator()(struct ::archive* arch); + }; + using ArchivePtr = std::unique_ptr; + + class ArchiveEntryDeleter + { + public: + void operator()(struct ::archive_entry* archEntry); + }; + using ArchiveEntryPtr = std::unique_ptr; + + void prepareCurrentEntry(); + static ArchiveEntryPtr createArchiveEntry(const Entry& entry); + bool writeSomeCurrentFileData(); + std::int64_t onWriteCallback(const std::byte* buff, std::size_t size); + + const EntryContainer _entries; + ArchivePtr _archive; + + static inline constexpr std::size_t _writeBlockSize {65536}; + static inline constexpr std::size_t _readBufferSize {65536}; + std::vector _readBuffer; + + EntryContainer::const_iterator _currentEntry; + ArchiveEntryPtr _currentArchiveEntry; + + std::uint64_t _currentEntryOffset {}; + std::ostream* _currentOutputStream {}; + std::uint64_t _bytesWrittenInCurrentOutputStream {}; + }; + +} // namespace Zip + diff --git a/src/libs/utils/impl/FileResourceHandler.cpp b/src/libs/utils/impl/FileResourceHandler.cpp index c9105630..8e3a6eb0 100644 --- a/src/libs/utils/impl/FileResourceHandler.cpp +++ b/src/libs/utils/impl/FileResourceHandler.cpp @@ -48,7 +48,6 @@ FileResourceHandler::processRequest(const Wt::Http::Request& request, Wt::Http:: { LMS_LOG(UTILS, ERROR) << "Cannot open file stream for '" << _path.string() << "'"; response.setStatus(404); - _isFinished = true; return {}; } else @@ -71,7 +70,6 @@ FileResourceHandler::processRequest(const Wt::Http::Request& request, Wt::Http:: response.addHeader("Content-Range", contentRange.str()); LMS_LOG(UTILS, DEBUG) << "Range not satisfiable"; - _isFinished = true; return {}; } @@ -101,7 +99,6 @@ FileResourceHandler::processRequest(const Wt::Http::Request& request, Wt::Http:: else if (!ifs) { LMS_LOG(UTILS, ERROR) << "Cannot reopen file stream for '" << _path.string() << "'"; - _isFinished = true; return {}; } @@ -128,11 +125,7 @@ FileResourceHandler::processRequest(const Wt::Http::Request& request, Wt::Http:: return response.createContinuation(); } - _isFinished = true; LMS_LOG(UTILS, DEBUG) << "Job complete!"; - - return {}; + return nullptr; } - - diff --git a/src/libs/utils/impl/FileResourceHandler.hpp b/src/libs/utils/impl/FileResourceHandler.hpp index e6666a59..8293f587 100644 --- a/src/libs/utils/impl/FileResourceHandler.hpp +++ b/src/libs/utils/impl/FileResourceHandler.hpp @@ -29,13 +29,12 @@ class FileResourceHandler final : public IResourceHandler private: Wt::Http::ResponseContinuation* processRequest(const Wt::Http::Request& request, Wt::Http::Response& response) override; + void abort() override {}; static constexpr std::size_t _chunkSize {65536}; std::filesystem::path _path; - ::uint64_t _beyondLastByte {}; - ::uint64_t _offset {}; - bool _isFinished {}; - + ::uint64_t _beyondLastByte {}; + ::uint64_t _offset {}; }; diff --git a/src/libs/utils/impl/Zipper.cpp b/src/libs/utils/impl/Zipper.cpp deleted file mode 100644 index 3e11d0ed..00000000 --- a/src/libs/utils/impl/Zipper.cpp +++ /dev/null @@ -1,644 +0,0 @@ -/* - * Copyright (C) 2020 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "utils/Zipper.hpp" - -#include -#include - -#include -#include - -#include "utils/Path.hpp" - -// Done using specs from https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT - -namespace Zip -{ - - class ZipHeader - { - public: - constexpr ZipHeader(std::byte* buffer, SizeType) - : _buffer {buffer} - {} - - enum GeneralPurposeFlag : std::uint16_t - { - UseDataDescriptor = 1 << 3, - LanguageEncoding = 1 << 11, - }; - - enum CompressionMethod : std::uint16_t - { - NoCompression = 0, - }; - - static constexpr std::uint32_t UnknownCrc32 {0}; - static constexpr SizeType UnknownFileSize {0}; - - struct Version - { - unsigned major; - unsigned minor; - }; - static constexpr Version VersionMadeBy {4, 5}; - static constexpr Version VersionNeededToExtract {4, 5}; - - protected: - void write8(SizeType offset, std::uint8_t value); - void write16(SizeType offset, std::uint16_t value); - void write32(SizeType offset, std::uint32_t value); - void write64(SizeType offset, std::uint64_t value); - void writeDateTime(SizeType offset, const Wt::WDateTime& time); - - private: - std::byte* _buffer {}; - }; - - void - ZipHeader::write8(SizeType offset, std::uint8_t value) - { - _buffer[offset] = static_cast(value); - } - - void - ZipHeader::write16(SizeType offset, std::uint16_t value) - { - _buffer[offset] = static_cast(value & 0xff); - _buffer[offset + 1] = static_cast(value >> 8); - } - - void - ZipHeader::write32(SizeType offset, std::uint32_t value) - { - _buffer[offset] = static_cast(value & 0xff); - _buffer[offset + 1] = static_cast((value >> 8) & 0xff); - _buffer[offset + 2] = static_cast((value >> 16) & 0xff); - _buffer[offset + 3] = static_cast(value >> 24); - } - - void - ZipHeader::write64(SizeType offset, std::uint64_t value) - { - _buffer[offset] = static_cast(value & 0xff); - _buffer[offset + 1] = static_cast((value >> 8) & 0xff); - _buffer[offset + 2] = static_cast((value >> 16) & 0xff); - _buffer[offset + 3] = static_cast((value >> 24) & 0xff); - _buffer[offset + 4] = static_cast((value >> 32) & 0xff); - _buffer[offset + 5] = static_cast((value >> 40) & 0xff); - _buffer[offset + 6] = static_cast((value >> 48) & 0xff); - _buffer[offset + 7] = static_cast(value >> 56); - } - - void - ZipHeader::writeDateTime(SizeType offset, const Wt::WDateTime& dateTime) - { - std::uint32_t encodedDateTime{}; - - // Date - encodedDateTime |= ((dateTime.date().year() - 1980) << 25); - encodedDateTime |= (dateTime.date().month() << 21); - encodedDateTime |= (dateTime.date().day() << 16); - - // Time - encodedDateTime |= (dateTime.time().hour() << 11); - encodedDateTime |= (dateTime.time().minute() << 5); - encodedDateTime |= (dateTime.time().second() << 1); - - write32(offset, encodedDateTime); - } - - class LocalFileHeader : public ZipHeader - { - public: - using ZipHeader::ZipHeader; - - void setSignature() { write32(0, 0x04034b50); } - void setVersionNeededToExtract(Version version) { assert(version.minor < 10); write16(4, version.major*10 + version.minor); } - void setGeneralPurposeFlags(std::uint16_t flags) { write16(6, flags); } - void setCompressionMethod(CompressionMethod compressionMethod) { write16(8, compressionMethod); } - void setLastModifiedDateTime(const Wt::WDateTime& dateTime) { writeDateTime(10, dateTime); } - void setCrc32UncompressedData(std::uint32_t crc) { write32(14, crc); } - void setCompressedSize(std::uint32_t size = UINT32_MAX) { write32(18, size); } - void setUncompressedSize(std::uint32_t size = UINT32_MAX) { write32(22, size); } - void setFileNameLength(SizeType size) { write16(26, size); } - void setExtraFieldLength(SizeType size) { write16(28, size); } - static constexpr SizeType getHeaderSize() { return 30; } - }; - - class Zip64ExtendedInformationExtraField : public ZipHeader - { - public: - using ZipHeader::ZipHeader; - - struct WithFileOffset {}; - constexpr Zip64ExtendedInformationExtraField(std::byte* buffer, SizeType bufferSize, WithFileOffset) - : ZipHeader {buffer, bufferSize} - , _withFileOffset {true} - {} - - void setTag() { write16(0, 0x0001); } - void setSize() { write16(2, (_withFileOffset ? getHeaderSize(WithFileOffset {}) : getHeaderSize()) - 4); } - void setUncompressedSize(SizeType size) { write64(4, size); } - void setCompressedSize(SizeType size) { write64(12, size); } - void setFileOffset(SizeType size) { assert(_withFileOffset); write64(20, size); } - - static constexpr SizeType getHeaderSize() { return 20; } - static constexpr SizeType getHeaderSize(WithFileOffset) { return 28; } - - private: - const bool _withFileOffset {}; - }; - - class DataDescriptor : public ZipHeader - { - public: - using ZipHeader::ZipHeader; - - void setSignature() { write32(0, 0x08074b50 ); } - void setCrc32UncompressedData(std::uint32_t crc32) { write32(4, crc32); } - void setCompressedSize(SizeType size) { write64(8, size); } - void setUncompressedSize(SizeType size) { write64(16, size); } - static constexpr SizeType getHeaderSize() { return 24; } - }; - - class CentralDirectoryHeader : public ZipHeader - { - public: - using ZipHeader::ZipHeader; - - void setSignature() { write32(0, 0x02014b50); } - void setVersionMadeBy(Version version) { assert(version.minor < 10); write16(4, version.major * 10 + version.minor); } - void setVersionNeededToExtract(Version version) { assert(version.minor < 10); write16(6, version.major*10 + version.minor); } - void setGeneralPurposeFlags(std::uint16_t flags) { write16(8, flags); } - void setCompressionMethod(CompressionMethod method) { write16(10, method); } - void setLastModifiedDateTime(const Wt::WDateTime& dateTime) { writeDateTime(12, dateTime); } - void setCrc32UncompressedData(std::uint32_t crc32) { write32(16, crc32); } - void setCompressedSize(SizeType size = UINT32_MAX) { write32(20, size); } - void setUncompressedSize(SizeType size = UINT32_MAX) { write32(24, size); } - void setFileNameLength(SizeType size) { write16(28, size); } - void setExtraFieldLength(SizeType size) { write16(30, size); } - void setFileCommentLength(SizeType size) { write16(32, size); } - void setDiskNumber(SizeType number) { write16(34, number); } - void setInternalFileAttributes(std::uint16_t attributes) { write16(36, attributes); } - void setExternalFileAttributes(std::uint16_t attributes) { write32(38, attributes); } - void setRelativeFileHeaderOffset(SizeType offset = UINT32_MAX) { write32(42, offset); } - static constexpr SizeType getHeaderSize() { return 46; } - }; - - class Zip64EndOfCentralDirectoryRecord : public ZipHeader - { - public: - using ZipHeader::ZipHeader; - - void setSignature() { write32(0, 0x06064b50); } - void setSize() { write64(4, 56 - 12); } - void setVersionMadeBy(Version version) { assert(version.minor < 10); write16(12, version.major * 10 + version.minor); } - void setVersionNeededToExtract(Version version) { assert(version.minor < 10); write16(14, version.major*10 + version.minor); } - void setDiskNumber(SizeType number) { write32(16, number); } - void setCentralDirectoryDiskNumber(unsigned number) { write32(20, number); } - void setNbDiskCentralDirectoryRecords(unsigned number) { write64(24, number); } - void setNbCentralDirectoryRecords(unsigned number) { write64(32, number); } - void setCentralDirectorySize(SizeType size) { write64(40, size); } - void setCentralDirectoryOffset(SizeType offset) { write64(48, offset); } - - static constexpr SizeType getHeaderSize() { return 56; } - }; - - class Zip64EndOfCentralDirectoryLocator : public ZipHeader - { - public: - using ZipHeader::ZipHeader; - - void setSignature() { write32(0, 0x07064b50); } - void setCentralDirectoryDiskNumber(unsigned number) { write32(4, number); } - void setZip64EndOfCentralDirectoryOffset(SizeType offset) { write64(8, offset); } - void setTotalNumberOfDisks(unsigned number) { write32(16, number); }; - - static constexpr SizeType getHeaderSize() { return 20; } - }; - - class EndOfCentralDirectoryRecord : public ZipHeader - { - public: - using ZipHeader::ZipHeader; - - void setSignature() { write32(0, 0x06054b50); } - void setDiskNumber(std::uint16_t number = UINT16_MAX) { write16(4, number); } - void setCentralDirectoryDiskNumber(std::uint16_t number = UINT16_MAX) { write16(6, number); } - void setNbDiskCentralDirectoryRecords(std::uint16_t number = UINT16_MAX) { write16(8, number); } - void setNbCentralDirectoryRecords(std::uint16_t number = UINT16_MAX) { write16(10, number); } - void setCentralDirectorySize(std::uint32_t size = UINT32_MAX) { write32(12, size); } - void setCentralDirectoryOffset(std::uint32_t offset = UINT32_MAX) { write32(16, offset); } - void setCommentLength(SizeType length) { write16(20, length); } - static constexpr SizeType getHeaderSize() { return 22; } - }; - - Zipper::Zipper(const std::map& files, const Wt::WDateTime& lastModifiedTime) - { - for (const auto& [filename, filePath] : files) - { - FileContext fileContext; - fileContext.filePath = filePath; - - std::error_code ec; - fileContext.fileSize = std::filesystem::file_size(filePath, ec); - if (ec) - throw ZipperException {"Cannot get file size for '" + filePath.string() + "': " + ec.message()}; - - if (lastModifiedTime.isValid()) - fileContext.lastModifiedTime = lastModifiedTime; - else - fileContext.lastModifiedTime = PathUtils::getLastWriteTime(filePath); - - _files[filename] = std::move(fileContext); - - _totalZipSize += LocalFileHeader::getHeaderSize(); - _totalZipSize += filename.size(); - _totalZipSize += Zip64ExtendedInformationExtraField::getHeaderSize(); - _totalZipSize += fileContext.fileSize; - _totalZipSize += DataDescriptor::getHeaderSize(); - _totalZipSize += CentralDirectoryHeader::getHeaderSize(); - _totalZipSize += filename.size(); - _totalZipSize += Zip64ExtendedInformationExtraField::getHeaderSize(Zip64ExtendedInformationExtraField::WithFileOffset {}); - } - - _totalZipSize += Zip64EndOfCentralDirectoryRecord::getHeaderSize(); - _totalZipSize += Zip64EndOfCentralDirectoryLocator::getHeaderSize(); - _totalZipSize += EndOfCentralDirectoryRecord::getHeaderSize(); - - _currentFile = std::begin(_files); - } - - SizeType - Zipper::writeSome(std::byte* buffer, SizeType bufferSize) - { - // make sure we have some room for the headers - assert(bufferSize >= minOutputBufferSize); - - SizeType nbTotalWrittenBytes {}; - - while (!isComplete() && (bufferSize >= minOutputBufferSize)) - { - SizeType nbWrittenBytes {}; - - switch (_writeState) - { - case WriteState::LocalFileHeader: - nbWrittenBytes = writeLocalFileHeader(buffer, bufferSize); - break; - - case WriteState::LocalFileHeaderFileName: - nbWrittenBytes = writeLocalFileHeaderFileName(buffer, bufferSize); - break; - - case WriteState::LocalFileHeaderExtraFields: - nbWrittenBytes = writeLocalFileHeaderExtraFields(buffer, bufferSize); - break; - - case WriteState::FileData: - nbWrittenBytes = writeFileData(buffer, bufferSize); - break; - - case WriteState::DataDescriptor: - nbWrittenBytes = writeDataDescriptor(buffer, bufferSize); - break; - - case WriteState::CentralDirectoryHeader: - nbWrittenBytes = writeCentralDirectoryHeader(buffer, bufferSize); - break; - - case WriteState::CentralDirectoryHeaderFileName: - nbWrittenBytes = writeCentralDirectoryHeaderFileName(buffer, bufferSize); - break; - - case WriteState::CentralDirectoryHeaderExtraFields: - nbWrittenBytes = writeCentralDirectoryHeaderExtraFields(buffer, bufferSize); - break; - - case WriteState::Zip64EndOfCentralDirectoryRecord: - nbWrittenBytes = writeZip64EndOfCentralDirectoryRecord(buffer, bufferSize); - break; - - case WriteState::Zip64EndOfCentralDirectoryLocator: - nbWrittenBytes = writeZip64EndOfCentralDirectoryLocator(buffer, bufferSize); - break; - - case WriteState::EndOfCentralDirectoryRecord: - nbWrittenBytes = writeEndOfCentralDirectoryRecord(buffer, bufferSize); - break; - - case WriteState::Complete: - break; - } - - buffer += nbWrittenBytes; - bufferSize -= nbWrittenBytes; - _currentZipOffset += nbWrittenBytes; - nbTotalWrittenBytes += nbWrittenBytes ; - } - - return nbTotalWrittenBytes; - } - - bool - Zipper::isComplete() const - { - return _writeState == WriteState::Complete; - } - - SizeType - Zipper::writeLocalFileHeader(std::byte* buffer, SizeType bufferSize) - { - static_assert(LocalFileHeader::getHeaderSize() <= minOutputBufferSize); - - assert(bufferSize >= minOutputBufferSize); - - if (_currentFile == std::cend(_files)) - { - _currentFile = std::begin(_files); - _writeState = WriteState::CentralDirectoryHeader; - return 0; - } - - LocalFileHeader header {buffer, bufferSize}; - - header.setSignature(); - header.setVersionNeededToExtract(ZipHeader::VersionNeededToExtract); - header.setGeneralPurposeFlags(ZipHeader::GeneralPurposeFlag::LanguageEncoding | ZipHeader::GeneralPurposeFlag::UseDataDescriptor); - header.setCompressionMethod(ZipHeader::CompressionMethod::NoCompression); - header.setCrc32UncompressedData(ZipHeader::UnknownCrc32); - header.setCompressedSize(); - header.setUncompressedSize(); - header.setLastModifiedDateTime(_currentFile->second.lastModifiedTime); - header.setFileNameLength(_currentFile->first.size()); - header.setExtraFieldLength(Zip64ExtendedInformationExtraField::getHeaderSize()); - - _writeState = WriteState::LocalFileHeaderFileName; - _currentFile->second.localFileHeaderOffset = _currentZipOffset; - - return header.getHeaderSize(); - } - - SizeType - Zipper::writeLocalFileHeaderFileName(std::byte* buffer, SizeType bufferSize) - { - assert(_currentFile != std::end(_files)); - - const std::string& fileName {_currentFile->first}; - - assert(_currentOffset <= fileName.size()); - if (_currentOffset == fileName.size()) - { - _writeState = WriteState::LocalFileHeaderExtraFields; - _currentOffset = 0; - return 0; - } - - const SizeType nbBytesToCopy {std::min(fileName.size() - _currentOffset, bufferSize)}; - - std::copy(std::next(std::begin(fileName), _currentOffset), std::next(std::begin(fileName), _currentOffset + nbBytesToCopy), reinterpret_cast(buffer)); - - _currentOffset += nbBytesToCopy; - return nbBytesToCopy; - } - - SizeType - Zipper::writeLocalFileHeaderExtraFields(std::byte* buffer, SizeType bufferSize) - { - assert(_currentFile != std::end(_files)); - static_assert(Zip64ExtendedInformationExtraField::getHeaderSize() <= minOutputBufferSize); - - Zip64ExtendedInformationExtraField header {buffer, bufferSize}; - - header.setTag(); - header.setSize(); - header.setUncompressedSize(ZipHeader::UnknownFileSize); - header.setCompressedSize(ZipHeader::UnknownFileSize); - - _writeState = WriteState::FileData; - return header.getHeaderSize(); - } - - SizeType - Zipper::writeFileData(std::byte* buffer, SizeType bufferSize) - { - assert(_currentFile != std::end(_files)); - - if (_currentOffset == _currentFile->second.fileSize) - { - _currentOffset = 0; - _writeState = WriteState::DataDescriptor; - return 0; - } - - const std::string filePath {_currentFile->second.filePath.string()}; - - std::ifstream ifs {filePath.c_str(), std::ios_base::binary}; - if (!ifs) - throw ZipperException {"File '" + filePath + "' does no longer exist!"}; - - ifs.seekg(0, std::ios::end); - const ::uint64_t fileSize {static_cast<::uint64_t>(ifs.tellg())}; - ifs.seekg(0, std::ios::beg); - - if (fileSize != _currentFile->second.fileSize) - throw ZipperException {"File '" + filePath + "': size mismatch!"}; - - const SizeType nbBytesToRead {std::min(static_cast(fileSize) - _currentOffset, bufferSize)}; - - ifs.seekg(_currentOffset, std::ios::beg); - ifs.read(reinterpret_cast(buffer), nbBytesToRead ); - const ::uint64_t actualReadSize {static_cast<::uint64_t>(ifs.gcount())}; - - _currentFile->second.fileCrc32.processBytes(buffer, actualReadSize); - _currentOffset += actualReadSize; - - return actualReadSize; - } - - SizeType - Zipper::writeDataDescriptor(std::byte* buffer, SizeType bufferSize) - { - assert(bufferSize >= minOutputBufferSize); - static_assert(DataDescriptor::getHeaderSize() <= minOutputBufferSize); - - assert(_currentFile != std::end(_files)); - - DataDescriptor desc {buffer, bufferSize}; - desc.setSignature(); - desc.setCrc32UncompressedData(_currentFile->second.fileCrc32.getResult()); - desc.setCompressedSize(_currentFile->second.fileSize); - desc.setUncompressedSize(_currentFile->second.fileSize); - - ++_currentFile; - _writeState = WriteState::LocalFileHeader; - - return desc.getHeaderSize(); - } - - SizeType - Zipper::writeCentralDirectoryHeader(std::byte* buffer, SizeType bufferSize) - { - assert(bufferSize >= minOutputBufferSize); - static_assert(CentralDirectoryHeader::getHeaderSize() <= minOutputBufferSize); - - if (_currentFile == std::begin(_files)) - _centralDirectoryOffset = _currentZipOffset; - - if (_currentFile == std::end(_files)) - { - _writeState = WriteState::Zip64EndOfCentralDirectoryRecord; - _currentFile = std::begin(_files); - return 0; - } - - CentralDirectoryHeader header {buffer, bufferSize}; - header.setSignature(); - header.setVersionMadeBy(ZipHeader::VersionMadeBy); - header.setVersionNeededToExtract(ZipHeader::VersionNeededToExtract); - header.setGeneralPurposeFlags(ZipHeader::GeneralPurposeFlag::LanguageEncoding | ZipHeader::GeneralPurposeFlag::UseDataDescriptor); - header.setCompressionMethod(ZipHeader::CompressionMethod::NoCompression); - header.setCompressedSize(); - header.setUncompressedSize(); - header.setLastModifiedDateTime(_currentFile->second.lastModifiedTime); - header.setCrc32UncompressedData(_currentFile->second.fileCrc32.getResult()); - header.setFileNameLength(_currentFile->first.size()); - header.setExtraFieldLength(Zip64ExtendedInformationExtraField::getHeaderSize(Zip64ExtendedInformationExtraField::WithFileOffset {})); - header.setFileCommentLength(0); - header.setDiskNumber(0); - header.setInternalFileAttributes(0); - header.setExternalFileAttributes(0); - header.setRelativeFileHeaderOffset(); - - _writeState = WriteState::CentralDirectoryHeaderFileName; - _centralDirectorySize += header.getHeaderSize(); - - return header.getHeaderSize(); - } - - SizeType - Zipper::writeCentralDirectoryHeaderFileName(std::byte* buffer, SizeType bufferSize) - { - const std::string& fileName {_currentFile->first}; - - assert(_currentOffset <= fileName.size()); - if (_currentOffset == fileName.size()) - { - _currentOffset = 0; - _writeState = WriteState::CentralDirectoryHeaderExtraFields; - - return 0; - } - - const SizeType nbBytesToCopy {std::min(fileName.size() - _currentOffset, bufferSize)}; - - std::copy(std::next(std::begin(fileName), _currentOffset), std::next(std::begin(fileName), _currentOffset + nbBytesToCopy), reinterpret_cast(buffer)); - - _currentOffset += nbBytesToCopy; - _centralDirectorySize += nbBytesToCopy; - return nbBytesToCopy; - } - - SizeType - Zipper::writeCentralDirectoryHeaderExtraFields(std::byte* buffer, SizeType bufferSize) - { - assert(bufferSize >= minOutputBufferSize); - assert(_currentFile != std::cend(_files)); - static_assert(Zip64ExtendedInformationExtraField::getHeaderSize(Zip64ExtendedInformationExtraField::WithFileOffset {}) <= minOutputBufferSize); - - Zip64ExtendedInformationExtraField header {buffer, bufferSize, Zip64ExtendedInformationExtraField::WithFileOffset {}}; - - header.setTag(); - header.setSize(); - header.setUncompressedSize(_currentFile->second.fileSize); - header.setCompressedSize(_currentFile->second.fileSize); - header.setFileOffset(_currentFile->second.localFileHeaderOffset); - - ++_currentFile; - _writeState = WriteState::CentralDirectoryHeader; - _centralDirectorySize += header.getHeaderSize(Zip64ExtendedInformationExtraField::WithFileOffset {}); - - return header.getHeaderSize(Zip64ExtendedInformationExtraField::WithFileOffset {}); - } - - SizeType - Zipper::writeZip64EndOfCentralDirectoryRecord(std::byte* buffer, SizeType bufferSize) - { - assert(bufferSize >= minOutputBufferSize); - static_assert(Zip64EndOfCentralDirectoryRecord::getHeaderSize() <= minOutputBufferSize); - - Zip64EndOfCentralDirectoryRecord record {buffer, bufferSize}; - - record.setSignature(); - record.setSize(); - record.setVersionMadeBy(ZipHeader::VersionNeededToExtract); - record.setVersionNeededToExtract(ZipHeader::VersionNeededToExtract); - record.setDiskNumber(0); - record.setCentralDirectoryDiskNumber(0); - record.setNbDiskCentralDirectoryRecords(_files.size()); - record.setNbCentralDirectoryRecords(_files.size()); - record.setCentralDirectorySize(_centralDirectorySize); - record.setCentralDirectoryOffset(_centralDirectoryOffset); - - _zip64EndOfCentralDirectoryRecordOffset = _currentZipOffset; - _writeState = WriteState::Zip64EndOfCentralDirectoryLocator; - return record.getHeaderSize(); - } - - SizeType - Zipper::writeZip64EndOfCentralDirectoryLocator(std::byte* buffer, SizeType bufferSize) - { - assert(bufferSize >= minOutputBufferSize); - static_assert(Zip64EndOfCentralDirectoryLocator::getHeaderSize() <= minOutputBufferSize); - - Zip64EndOfCentralDirectoryLocator locator {buffer, bufferSize}; - - locator.setSignature(); - locator.setCentralDirectoryDiskNumber(0); - locator.setZip64EndOfCentralDirectoryOffset(_zip64EndOfCentralDirectoryRecordOffset); - locator.setTotalNumberOfDisks(1); - - _writeState = WriteState::EndOfCentralDirectoryRecord; - return locator.getHeaderSize(); - } - - SizeType - Zipper::writeEndOfCentralDirectoryRecord(std::byte* buffer, SizeType bufferSize) - { - assert(bufferSize >= minOutputBufferSize); - static_assert(EndOfCentralDirectoryRecord::getHeaderSize() <= minOutputBufferSize); - - EndOfCentralDirectoryRecord record {buffer, bufferSize}; - - record.setSignature(); - record.setDiskNumber(0); - record.setCentralDirectoryDiskNumber(0); - record.setNbDiskCentralDirectoryRecords(); - record.setNbCentralDirectoryRecords(); - record.setCentralDirectorySize(); - record.setCentralDirectoryOffset(); - record.setCommentLength(0); - - _writeState = WriteState::Complete; - return record.getHeaderSize(); - } - -} // namespace Zip diff --git a/src/libs/utils/include/utils/Exception.hpp b/src/libs/utils/include/utils/Exception.hpp index 1d238809..fe9182f6 100644 --- a/src/libs/utils/include/utils/Exception.hpp +++ b/src/libs/utils/include/utils/Exception.hpp @@ -21,10 +21,11 @@ #include #include +#include class LmsException : public std::runtime_error { public: - LmsException(const std::string& error = "") : std::runtime_error {error} {} + LmsException(std::string_view error = "") : std::runtime_error {std::string {error}} {} }; diff --git a/src/libs/utils/include/utils/IResourceHandler.hpp b/src/libs/utils/include/utils/IResourceHandler.hpp index 6518a810..6dfbd255 100644 --- a/src/libs/utils/include/utils/IResourceHandler.hpp +++ b/src/libs/utils/include/utils/IResourceHandler.hpp @@ -29,5 +29,6 @@ class IResourceHandler virtual ~IResourceHandler() = default; [[nodiscard]] virtual Wt::Http::ResponseContinuation* processRequest(const Wt::Http::Request& request, Wt::Http::Response& response) = 0; + virtual void abort() = 0; }; diff --git a/src/libs/utils/include/utils/IZipper.hpp b/src/libs/utils/include/utils/IZipper.hpp new file mode 100644 index 00000000..813df2d7 --- /dev/null +++ b/src/libs/utils/include/utils/IZipper.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2020 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include "Exception.hpp" + +namespace Zip +{ + struct Entry + { + std::string fileName; + std::filesystem::path filePath; + }; + using EntryContainer = std::vector; + + class Exception : public LmsException + { + using LmsException::LmsException; + }; + + class IZipper + { + public: + virtual ~IZipper() = default; + + virtual std::uint64_t writeSome(std::ostream& output) = 0; + virtual bool isComplete() const = 0; + virtual void abort() = 0; + }; + + std::unique_ptr createArchiveZipper(const EntryContainer& entries); +} // namespace Zip + diff --git a/src/libs/utils/include/utils/Zipper.hpp b/src/libs/utils/include/utils/Zipper.hpp deleted file mode 100644 index 288799fa..00000000 --- a/src/libs/utils/include/utils/Zipper.hpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (C) 2020 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include -#include - -#include - -#include "Exception.hpp" -#include "utils/Crc32Calculator.hpp" - -namespace Zip -{ - using SizeType = std::uint64_t; - - class ZipperException : public LmsException - { - using LmsException::LmsException; - }; - - // Very simple on-the-fly zip creator, "store" method only - class Zipper - { - public: - Zipper(const std::map& files, const Wt::WDateTime& lastModifiedTime = {}); - - static constexpr SizeType minOutputBufferSize {64}; - SizeType writeSome(std::byte* buffer, SizeType bufferSize); - bool isComplete() const; - - SizeType getTotalZipFile() const { return _totalZipSize; } - - private: - void setComplete(); - - SizeType writeLocalFileHeader(std::byte* buffer, SizeType bufferSize); - SizeType writeLocalFileHeaderFileName(std::byte* buffer, SizeType bufferSize); - SizeType writeLocalFileHeaderExtraFields(std::byte* buffer, SizeType bufferSize); - SizeType writeFileData(std::byte* buffer, SizeType bufferSize); - SizeType writeDataDescriptor(std::byte* buffer, SizeType bufferSize); - SizeType writeCentralDirectoryHeader(std::byte* buffer, SizeType bufferSize); - SizeType writeCentralDirectoryHeaderFileName(std::byte* buffer, SizeType bufferSize); - SizeType writeCentralDirectoryHeaderExtraFields(std::byte* buffer, SizeType bufferSize); - SizeType writeZip64EndOfCentralDirectoryRecord(std::byte* buffer, SizeType bufferSize); - SizeType writeZip64EndOfCentralDirectoryLocator(std::byte* buffer, SizeType bufferSize); - SizeType writeEndOfCentralDirectoryRecord(std::byte* buffer, SizeType bufferSize); - - struct FileContext - { - std::filesystem::path filePath; - SizeType fileSize; - Wt::WDateTime lastModifiedTime; - Utils::Crc32Calculator fileCrc32; - SizeType localFileHeaderOffset {}; - }; - - using FileContainer = std::map; - FileContainer _files; - - enum class WriteState - { - LocalFileHeader, - LocalFileHeaderFileName, - LocalFileHeaderExtraFields, - FileData, - DataDescriptor, - CentralDirectoryHeader, - CentralDirectoryHeaderFileName, - CentralDirectoryHeaderExtraFields, - Zip64EndOfCentralDirectoryRecord, - Zip64EndOfCentralDirectoryLocator, - EndOfCentralDirectoryRecord, - Complete, - }; - - SizeType _totalZipSize {}; - WriteState _writeState {WriteState::LocalFileHeader}; - FileContainer::iterator _currentFile; - SizeType _currentOffset {}; - SizeType _currentZipOffset {}; - SizeType _centralDirectoryOffset {}; - SizeType _centralDirectorySize {}; - SizeType _zip64EndOfCentralDirectoryRecordOffset {}; - }; - -} // namespace Zip - diff --git a/src/libs/utils/include/utils/ZipperResourceHandlerCreator.hpp b/src/libs/utils/include/utils/ZipperResourceHandlerCreator.hpp new file mode 100644 index 00000000..d8e40157 --- /dev/null +++ b/src/libs/utils/include/utils/ZipperResourceHandlerCreator.hpp @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2020 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +#include "utils/IResourceHandler.hpp" +#include "utils/IZipper.hpp" + +std::unique_ptr createZipperResourceHandler(std::unique_ptr zipper); + diff --git a/src/libs/utils/test/String.cpp b/src/libs/utils/test/String.cpp index 52f4a562..ccf02829 100644 --- a/src/libs/utils/test/String.cpp +++ b/src/libs/utils/test/String.cpp @@ -119,7 +119,7 @@ TEST(StringUtils, readAs) EXPECT_EQ(StringUtils::readAs(""), std::nullopt); } -TEST(StrinUtils, capitalize) +TEST(StringUtils, capitalize) { struct TestCase { diff --git a/src/lms/main.cpp b/src/lms/main.cpp index 185fc208..4cb86c98 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -207,6 +207,12 @@ int main(int argc, char* argv[]) Service config {createConfig(configFilePath)}; Service logger {std::make_unique()}; + // use system locale. libarchive relies on this to write filenames + if (char *locale {::setlocale(LC_ALL, "")}) + LMS_LOG(MAIN, INFO) << "locale set to '" << locale << "'"; + else + LMS_LOG(MAIN, WARNING) << "Cannot set locale from system"; + // Make sure the working directory exists std::filesystem::create_directories(config->getPath("working-dir")); std::filesystem::create_directories(config->getPath("working-dir") / "cache"); diff --git a/src/lms/ui/explore/ArtistView.cpp b/src/lms/ui/explore/ArtistView.cpp index cd1f6323..13369377 100644 --- a/src/lms/ui/explore/ArtistView.cpp +++ b/src/lms/ui/explore/ArtistView.cpp @@ -212,7 +212,7 @@ Artist::refreshReleases() Release::FindParameters params; params.setClusters(_filters.getClusterIds()); params.setArtist(_artistId, {TrackArtistLinkType::ReleaseArtist}, {}); - params.setSortMethod(ReleaseSortMethod::DateDesc); + params.setSortMethod(ReleaseSortMethod::OriginalDateDesc); const auto releases {Release::find(LmsApp->getDbSession(), params)}; if (!releases.results.empty()) @@ -272,7 +272,7 @@ Artist::refreshAppearsOnReleases() Release::FindParameters params; params.setClusters(_filters.getClusterIds()); params.setArtist(_artistId, types, {TrackArtistLinkType::ReleaseArtist}); - params.setSortMethod(ReleaseSortMethod::DateDesc); + params.setSortMethod(ReleaseSortMethod::OriginalDateDesc); const auto releases {Release::find(LmsApp->getDbSession(), params)}; if (!releases.results.empty()) diff --git a/src/lms/ui/explore/ReleaseHelpers.cpp b/src/lms/ui/explore/ReleaseHelpers.cpp index a1db7549..c4b9fa32 100644 --- a/src/lms/ui/explore/ReleaseHelpers.cpp +++ b/src/lms/ui/explore/ReleaseHelpers.cpp @@ -66,19 +66,11 @@ namespace UserInterface::ReleaseListHelpers if (showYear) { - if (std::optional year {release->getReleaseYear()}) + Wt::WString year {ReleaseHelpers::buildReleaseYearString(release->getReleaseYear(), release->getReleaseYear(true))}; + if (!year.empty()) { entry->setCondition("if-has-year", true); - - std::string strYear {std::to_string(*year)}; - - std::optional originalYear {release->getReleaseYear(true)}; - if (originalYear && *originalYear != *year) - { - strYear += " (" + std::to_string(*originalYear) + ")"; - } - - entry->bindString("year", strYear, Wt::TextFormat::Plain); + entry->bindString("year", year, Wt::TextFormat::Plain); } } @@ -142,4 +134,20 @@ namespace UserInterface::ReleaseHelpers return res; } + + Wt::WString buildReleaseYearString(std::optional year, std::optional originalYear) + { + Wt::WString res; + + // Year can be here, but originalYear can't be here without year (enforced by scanner) + if (!year) + return res; + + if (originalYear && *originalYear != *year) + res = std::to_string(*originalYear) + " (" + std::to_string(*year) + ")"; + else + res = std::to_string(*year); + + return res; + } } // namespace UserInterface::ReleaseHelpers} diff --git a/src/lms/ui/explore/ReleaseHelpers.hpp b/src/lms/ui/explore/ReleaseHelpers.hpp index b0c61911..ebc424aa 100644 --- a/src/lms/ui/explore/ReleaseHelpers.hpp +++ b/src/lms/ui/explore/ReleaseHelpers.hpp @@ -20,7 +20,9 @@ #pragma once #include +#include +#include #include #include "services/database/Object.hpp" #include "services/database/Types.hpp" @@ -41,4 +43,5 @@ namespace UserInterface::ReleaseListHelpers namespace UserInterface::ReleaseHelpers { Wt::WString buildReleaseTypeString(Database::ReleaseTypePrimary primaryType, EnumSet secondaryTypes); + Wt::WString buildReleaseYearString(std::optional year, std::optional originalYear); } diff --git a/src/lms/ui/explore/ReleaseView.cpp b/src/lms/ui/explore/ReleaseView.cpp index 816cd0c4..1060c7fc 100644 --- a/src/lms/ui/explore/ReleaseView.cpp +++ b/src/lms/ui/explore/ReleaseView.cpp @@ -246,18 +246,11 @@ Release::refreshView() bindString("name", Wt::WString::fromUTF8(release->getName()), Wt::TextFormat::Plain); - std::optional year {release->getReleaseYear()}; - if (year) + Wt::WString year {ReleaseHelpers::buildReleaseYearString(release->getReleaseYear(), release->getReleaseYear(true))}; + if (!year.empty()) { setCondition("if-has-year", true); - bindInt("year", *year); - - std::optional originalYear {release->getReleaseYear(true)}; - if (originalYear && *originalYear != *year) - { - setCondition("if-has-orig-year", true); - bindInt("orig-year", *originalYear); - } + bindString("year", year, Wt::TextFormat::Plain); } bindString("duration", Utils::durationToString(release->getDuration()), Wt::TextFormat::Plain); diff --git a/src/lms/ui/resource/DownloadResource.cpp b/src/lms/ui/resource/DownloadResource.cpp index 898e861a..4f6f4b77 100644 --- a/src/lms/ui/resource/DownloadResource.cpp +++ b/src/lms/ui/resource/DownloadResource.cpp @@ -33,7 +33,6 @@ #include "services/database/TrackList.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" -#include "utils/Zipper.hpp" #include "LmsApplication.hpp" @@ -51,34 +50,25 @@ DownloadResource::handleRequest(const Wt::Http::Request& request, Wt::Http::Resp { try { - std::shared_ptr zipper; + std::shared_ptr zipper; // First, see if this request is for a continuation if (Wt::Http::ResponseContinuation *continuation {request.continuation()}) - zipper = Wt::cpp17::any_cast>(continuation->data()); + zipper = Wt::cpp17::any_cast>(continuation->data()); else { zipper = createZipper(); - if (zipper) - response.setContentLength(zipper->getTotalZipFile()); response.setMimeType("application/zip"); } - if (!zipper) - return; - - std::array buffer; - Zip::SizeType nbWrittenBytes {zipper->writeSome(buffer.data(), buffer.size())}; - - response.out().write(reinterpret_cast(buffer.data()), nbWrittenBytes); - + zipper->writeSome(response.out()); if (!zipper->isComplete()) { auto* continuation {response.createContinuation()}; continuation->setData(zipper); } } - catch (Zip::ZipperException& exception) + catch (Zip::Exception& exception) { LOG(ERROR) << "Zipper exception: " << exception.what(); } @@ -126,59 +116,61 @@ getReleasePathName(Database::Release::pointer release) return releaseName; } -static -std::string -getTrackPathName(Database::Track::pointer track) +namespace details { - std::ostringstream fileName; - - if (auto discNumber {track->getDiscNumber()}) - fileName << *discNumber << "."; - if (auto trackNumber {track->getTrackNumber()}) - fileName << std::setw(2) << std::setfill('0') << *trackNumber << " - "; - - fileName << StringUtils::replaceInString(track->getName(), "/", "_") << track->getPath().filename().extension().string(); - - return fileName.str(); -} - -static -std::string -getTrackListPathName(Database::TrackList::pointer trackList) -{ - return StringUtils::replaceInString(trackList->getName(), "/", "_"); -} - -static -std::unique_ptr -createZipper(const std::vector& tracks) -{ - if (tracks.empty()) - return {}; - - std::map files; - - for (const Database::Track::pointer& track : tracks) + static + std::string + getTrackPathName(Database::Track::pointer track) { - std::string releaseName; - std::string releaseArtistName; - if (auto release {track->getRelease()}) - { - releaseName = getReleasePathName(release); - releaseArtistName = getReleaseArtistPathName(release); - } + std::ostringstream fileName; - std::string fileName; - if (!releaseArtistName.empty()) - fileName += releaseArtistName + "/"; - if (!releaseName.empty()) - fileName += releaseName + "/"; - fileName += getTrackPathName(track); + if (auto discNumber {track->getDiscNumber()}) + fileName << *discNumber << "."; + if (auto trackNumber {track->getTrackNumber()}) + fileName << std::setw(2) << std::setfill('0') << *trackNumber << " - "; - files.emplace(fileName, track->getPath()); + fileName << StringUtils::replaceInString(track->getName(), "/", "_") << track->getPath().filename().extension().string(); + + return fileName.str(); } - return std::make_unique(files, Wt::WDateTime::currentDateTime()); + static + std::string + getTrackListPathName(Database::TrackList::pointer trackList) + { + return StringUtils::replaceInString(trackList->getName(), "/", "_"); + } + + static + std::unique_ptr + createZipper(const std::vector& tracks) + { + if (tracks.empty()) + return {}; + + Zip::EntryContainer files; + for (const Database::Track::pointer& track : tracks) + { + std::string releaseName; + std::string releaseArtistName; + if (auto release {track->getRelease()}) + { + releaseName = getReleasePathName(release); + releaseArtistName = getReleaseArtistPathName(release); + } + + std::string fileName; + if (!releaseArtistName.empty()) + fileName += releaseArtistName + "/"; + if (!releaseName.empty()) + fileName += releaseName + "/"; + fileName += getTrackPathName(track); + + files.emplace_back(Zip::Entry {fileName, track->getPath()}); + } + + return Zip::createArchiveZipper(files); + } } DownloadArtistResource::DownloadArtistResource(Database::ArtistId artistId) @@ -191,7 +183,7 @@ DownloadArtistResource::DownloadArtistResource(Database::ArtistId artistId) suggestFileName(getArtistPathName(artist) + ".zip"); } -std::unique_ptr +std::unique_ptr DownloadArtistResource::createZipper() { auto transaction {LmsApp->getDbSession().createSharedTransaction()}; @@ -203,7 +195,7 @@ DownloadArtistResource::createZipper() for (const Database::TrackId trackId : trackResults.results) tracks.push_back(Database::Track::find(LmsApp->getDbSession(), trackId)); - return UserInterface::createZipper(tracks); + return details::createZipper(tracks); } DownloadReleaseResource::DownloadReleaseResource(Database::ReleaseId releaseId) @@ -217,7 +209,7 @@ DownloadReleaseResource::DownloadReleaseResource(Database::ReleaseId releaseId) } -std::unique_ptr +std::unique_ptr DownloadReleaseResource::createZipper() { using namespace Database; @@ -230,7 +222,7 @@ DownloadReleaseResource::createZipper() tracks.reserve(trackResults.results.size()); std::transform(std::cbegin(trackResults.results), std::cend(trackResults.results), std::back_inserter(tracks), [](TrackId trackId){ return Track::find(LmsApp->getDbSession(), trackId); }); - return UserInterface::createZipper(tracks); + return details::createZipper(tracks); } DownloadTrackResource::DownloadTrackResource(Database::TrackId trackId) @@ -240,10 +232,10 @@ DownloadTrackResource::DownloadTrackResource(Database::TrackId trackId) Database::Track::pointer track {Database::Track::find(LmsApp->getDbSession(), trackId)}; if (track) - suggestFileName(getTrackPathName(track) + ".zip"); + suggestFileName(details::getTrackPathName(track) + ".zip"); } -std::unique_ptr +std::unique_ptr DownloadTrackResource::createZipper() { auto transaction {LmsApp->getDbSession().createSharedTransaction()}; @@ -255,7 +247,7 @@ DownloadTrackResource::createZipper() return {}; } - return UserInterface::createZipper({track}); + return details::createZipper({track}); } DownloadTrackListResource::DownloadTrackListResource(Database::TrackListId trackListId) @@ -265,10 +257,10 @@ DownloadTrackListResource::DownloadTrackListResource(Database::TrackListId track const Database::TrackList::pointer trackList {Database::TrackList::find(LmsApp->getDbSession(), trackListId)}; if (trackList) - suggestFileName(getTrackListPathName(trackList) + ".zip"); + suggestFileName(details::getTrackListPathName(trackList) + ".zip"); } -std::unique_ptr +std::unique_ptr DownloadTrackListResource::createZipper() { using namespace Database; @@ -282,7 +274,7 @@ DownloadTrackListResource::createZipper() tracks.reserve(trackResults.results.size()); std::transform(std::cbegin(trackResults.results), std::cend(trackResults.results), std::back_inserter(tracks), [](TrackId trackId){ return Track::find(LmsApp->getDbSession(), trackId); }); - return UserInterface::createZipper(tracks); + return details::createZipper(tracks); } } // namespace UserInterface diff --git a/src/lms/ui/resource/DownloadResource.hpp b/src/lms/ui/resource/DownloadResource.hpp index 16e9edd2..d0099fa9 100644 --- a/src/lms/ui/resource/DownloadResource.hpp +++ b/src/lms/ui/resource/DownloadResource.hpp @@ -26,7 +26,8 @@ #include "services/database/ReleaseId.hpp" #include "services/database/TrackId.hpp" #include "services/database/TrackListId.hpp" -#include "utils/Zipper.hpp" +#include "utils/IZipper.hpp" +#include "utils/ZipperResourceHandlerCreator.hpp" namespace UserInterface { @@ -39,7 +40,7 @@ namespace UserInterface private: void handleRequest(const Wt::Http::Request& request, Wt::Http::Response& response) override; - virtual std::unique_ptr createZipper() = 0; + virtual std::unique_ptr createZipper() = 0; }; class DownloadArtistResource : public DownloadResource @@ -48,7 +49,7 @@ namespace UserInterface DownloadArtistResource(Database::ArtistId artistId); private: - std::unique_ptr createZipper() override; + std::unique_ptr createZipper() override; Database::ArtistId _artistId; }; @@ -58,7 +59,7 @@ namespace UserInterface DownloadReleaseResource(Database::ReleaseId releaseId); private: - std::unique_ptr createZipper() override; + std::unique_ptr createZipper() override; Database::ReleaseId _releaseId; }; @@ -68,7 +69,7 @@ namespace UserInterface DownloadTrackResource(Database::TrackId trackId); private: - std::unique_ptr createZipper() override; + std::unique_ptr createZipper() override; Database::TrackId _trackId; }; @@ -78,7 +79,7 @@ namespace UserInterface DownloadTrackListResource(Database::TrackListId trackListId); private: - std::unique_ptr createZipper() override; + std::unique_ptr createZipper() override; Database::TrackListId _trackListId; }; } // namespace UserInterface diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 08707a6a..8b305be3 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -1,4 +1,3 @@ add_subdirectory(cover) add_subdirectory(metadata) add_subdirectory(recommendation) -add_subdirectory(zipper) diff --git a/src/tools/zipper/CMakeLists.txt b/src/tools/zipper/CMakeLists.txt deleted file mode 100644 index 8a588796..00000000 --- a/src/tools/zipper/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ - -add_executable(lms-zipper - LmsZipper.cpp - ) - -target_link_libraries(lms-zipper PRIVATE - lmsutils - ) - diff --git a/src/tools/zipper/LmsZipper.cpp b/src/tools/zipper/LmsZipper.cpp deleted file mode 100644 index 51a494f3..00000000 --- a/src/tools/zipper/LmsZipper.cpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2020 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include - -#include -#include -#include - -#include "utils/Service.hpp" -#include "utils/StreamLogger.hpp" -#include "utils/Zipper.hpp" - -int main(int argc, char* argv[]) -{ - // log to stdout - Service logger {std::make_unique(std::cout)}; - - if (argc < 2) - { - std::cerr << "Usage: [...]" << std::endl; - return EXIT_FAILURE; - } - - std::filesystem::path zipPath {argv[1]}; - - std::map files; - for (int i {2}; i < argc; ++i) - { - std::filesystem::path path {argv[i]}; - files.emplace(path.relative_path(), path); - } - - - std::cout << "Compressing " << files.size() << " files..." << std::endl; - - using namespace Zip; - - std::ofstream ofs {zipPath.string().c_str(), std::ios_base::binary}; - if (!ofs) - { - std::cerr << "Cannot open file '" << zipPath.string() << "' for writing"; - return EXIT_FAILURE; - } - - try - { - Zipper zipper {files}; - - Zip::SizeType nbTotalWrittenBytes {}; - while (!zipper.isComplete()) - { - std::array buffer; - - const Zip::SizeType nbWrittenBytes {zipper.writeSome(buffer.data(), buffer.size())}; - ofs.write(reinterpret_cast(buffer.data()), nbWrittenBytes); - nbTotalWrittenBytes += nbWrittenBytes; - } - - if (nbTotalWrittenBytes != zipper.getTotalZipFile()) - std::cerr << "ERROR: actual size mismatch!" << std::endl; - - std::cout << "Total zip size = " << zipper.getTotalZipFile() << std::endl; - } - catch (const ZipperException& e) - { - std::cerr << "Caught Zipper exception: " << e.what() << std::endl; - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; -}