From 90b1646876007b2972c5ffd0e23e0ae0d2dbe2ee Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 13 May 2020 11:32:08 +0200 Subject: [PATCH] A lot of file transfer updates --- file/CMakeLists.txt | 2 + file/include/files/ExecuteResponse.h | 77 ++ file/include/files/FileServer.h | 296 +++-- file/local_server/LocalFileProvider.cpp | 46 +- file/local_server/LocalFileProvider.h | 174 +-- file/local_server/LocalFileSystem.cpp | 188 ++- file/local_server/LocalFileTransfer.cpp | 180 ++- .../LocalFileTransferClientWorker.cpp | 20 +- file/local_server/LocalFileTransferDisk.cpp | 42 +- .../local_server/LocalFileTransferNetwork.cpp | 67 +- file/local_server/NetTools.cpp | 9 + file/local_server/NetTools.h | 164 +++ file/todo.txt | 1 + license/LicenseServerMain.cpp | 15 +- server/CMakeLists.txt | 5 +- server/main.cpp | 1 - server/src/Configuration.cpp | 2 + server/src/FileServerHandler.cpp | 50 + server/src/FileServerHandler.h | 31 + server/src/Group.cpp | 3 +- server/src/InstanceHandler.cpp | 43 +- server/src/InstanceHandler.h | 9 +- server/src/InstanceHandlerSetup.cpp | 1 - server/src/VirtualServer.cpp | 49 +- server/src/VirtualServerManager.cpp | 24 +- server/src/channel/ServerChannel.cpp | 3 +- server/src/client/ConnectedClient.cpp | 5 +- server/src/client/ConnectedClient.h | 19 +- .../client/ConnectedClientNotifyHandler.cpp | 1 - server/src/client/DataClient.cpp | 3 +- server/src/client/command_handler/channel.cpp | 3 +- server/src/client/command_handler/client.cpp | 27 - server/src/client/command_handler/file.cpp | 1167 ++++++++++++----- server/src/client/command_handler/helpers.h | 4 +- server/src/client/command_handler/misc.cpp | 5 +- server/src/client/command_handler/music.cpp | 2 - server/src/client/command_handler/server.cpp | 2 - server/src/client/file/FileClient.cpp | 719 ---------- server/src/client/file/FileClient.h | 133 -- server/src/client/file/FileClientIO.cpp | 248 ---- .../channel_replay/ChannelProvider.cpp | 10 +- server/src/lincense/LicenseService.cpp | 3 + server/src/server/file/LocalFileServer.cpp | 777 ----------- server/src/server/file/LocalFileServer.h | 163 --- shared | 2 +- 45 files changed, 1989 insertions(+), 2806 deletions(-) create mode 100644 file/include/files/ExecuteResponse.h create mode 100644 file/local_server/NetTools.cpp create mode 100644 file/local_server/NetTools.h create mode 100644 file/todo.txt create mode 100644 server/src/FileServerHandler.cpp create mode 100644 server/src/FileServerHandler.h delete mode 100644 server/src/client/file/FileClient.cpp delete mode 100644 server/src/client/file/FileClient.h delete mode 100644 server/src/client/file/FileClientIO.cpp delete mode 100644 server/src/server/file/LocalFileServer.cpp delete mode 100644 server/src/server/file/LocalFileServer.h diff --git a/file/CMakeLists.txt b/file/CMakeLists.txt index 72b0365..4b506af 100644 --- a/file/CMakeLists.txt +++ b/file/CMakeLists.txt @@ -12,6 +12,7 @@ add_library(TeaSpeak-FileServer STATIC local_server/LocalFileTransferDisk.cpp local_server/LocalFileTransferNetwork.cpp local_server/clnpath.cpp + local_server/NetTools.cpp ) target_link_libraries(TeaSpeak-FileServer PUBLIC TeaSpeak ${StringVariable_LIBRARIES_STATIC} stdc++fs @@ -22,6 +23,7 @@ target_link_libraries(TeaSpeak-FileServer PUBLIC TeaSpeak ${StringVariable_LIBRA ) target_include_directories(TeaSpeak-FileServer PUBLIC include/) +target_compile_options(TeaSpeak-FileServer PUBLIC "-Wswitch-enum") add_executable(TeaSpeak-FileServerTest test/main.cpp) target_link_libraries(TeaSpeak-FileServerTest PUBLIC TeaSpeak-FileServer diff --git a/file/include/files/ExecuteResponse.h b/file/include/files/ExecuteResponse.h new file mode 100644 index 0000000..8c4207c --- /dev/null +++ b/file/include/files/ExecuteResponse.h @@ -0,0 +1,77 @@ +#pragma once + +namespace ts::server::file { + enum struct ExecuteStatus { + UNKNOWN, + WAITING, + SUCCESS, + ERROR + }; + + template + constexpr std::size_t variant_index() { + if constexpr (index == std::variant_size_v) { + return index; + } else if constexpr (std::is_same_v, T>) { + return index; + } else { + return variant_index(); + } + } + + struct EmptyExecuteResponse { }; + template + class ExecuteResponse { + typedef std::variant variant_t; + public: + ExecuteStatus status{ExecuteStatus::WAITING}; + + [[nodiscard]] inline auto response() const -> const response_t& { return std::get(this->response_); } + + template ::value>> + [[nodiscard]] inline const error_t& error() const { return std::get(this->response_); } + + inline void wait() const { + std::unique_lock nlock{this->notify_mutex}; + this->notify_cv.wait(nlock, [&]{ return this->status != ExecuteStatus::WAITING; }); + } + + template + [[nodiscard]] inline bool wait_for(const std::chrono::duration<_Rep, _Period>& time) const { + std::unique_lock nlock{this->notify_mutex}; + return this->notify_cv.wait_for(nlock, time, [&]{ return this->status != ExecuteStatus::WAITING; }); + } + + template + inline void emplace_success(Args&&... args) { + constexpr auto success_index = variant_index(); + + std::lock_guard rlock{this->notify_mutex}; + this->response_.template emplace(std::forward(args)...); + this->status = ExecuteStatus::SUCCESS; + this->notify_cv.notify_all(); + } + + template + inline void emplace_fail(Args&&... args) { + constexpr auto error_index = variant_index(); + + std::lock_guard rlock{this->notify_mutex}; + this->response_.template emplace(std::forward(args)...); + this->status = ExecuteStatus::ERROR; + this->notify_cv.notify_all(); + } + + [[nodiscard]] inline bool succeeded() const { + return this->status == ExecuteStatus::SUCCESS; + } + + ExecuteResponse(std::mutex& notify_mutex, std::condition_variable& notify_cv) + : notify_mutex{notify_mutex}, notify_cv{notify_cv} {} + private: + variant_t response_{}; /* void* as default value so we don't initialize error_t or response_t */ + + std::mutex& notify_mutex; + std::condition_variable& notify_cv; + }; +} \ No newline at end of file diff --git a/file/include/files/FileServer.h b/file/include/files/FileServer.h index 5d2dcaa..9302e34 100644 --- a/file/include/files/FileServer.h +++ b/file/include/files/FileServer.h @@ -4,86 +4,18 @@ #include #include #include +#include #include #include #include +#include + +#include "./ExecuteResponse.h" #define TRANSFER_KEY_LENGTH (32) namespace ts::server::file { - enum struct ExecuteStatus { - UNKNOWN, - WAITING, - SUCCESS, - ERROR - }; - - template - constexpr std::size_t variant_index() { - if constexpr (index == std::variant_size_v) { - return index; - } else if constexpr (std::is_same_v, T>) { - return index; - } else { - return variant_index(); - } - } - - struct EmptyExecuteResponse { }; - template - class ExecuteResponse { - typedef std::variant variant_t; - public: - ExecuteStatus status{ExecuteStatus::WAITING}; - - [[nodiscard]] inline const auto& response() const { return std::get(this->response_); } - - template ::value>> - [[nodiscard]] inline const error_t& error() const { return std::get(this->response_); } - - inline void wait() const { - std::unique_lock nlock{this->notify_mutex}; - this->notify_cv.wait(nlock, [&]{ return this->status != ExecuteStatus::WAITING; }); - } - - template - [[nodiscard]] inline bool wait_for(const std::chrono::duration<_Rep, _Period>& time) const { - std::unique_lock nlock{this->notify_mutex}; - return this->notify_cv.wait_for(nlock, time, [&]{ return this->status != ExecuteStatus::WAITING; }); - } - - template - inline void emplace_success(Args&&... args) { - constexpr auto success_index = variant_index(); - - std::lock_guard rlock{this->notify_mutex}; - this->response_.template emplace(std::forward(args)...); - this->status = ExecuteStatus::SUCCESS; - this->notify_cv.notify_all(); - } - - template - inline void emplace_fail(Args&&... args) { - constexpr auto error_index = variant_index(); - - std::lock_guard rlock{this->notify_mutex}; - this->response_.template emplace(std::forward(args)...); - this->status = ExecuteStatus::ERROR; - this->notify_cv.notify_all(); - } - - [[nodiscard]] inline bool succeeded() const { - return this->status == ExecuteStatus::SUCCESS; - } - - ExecuteResponse(std::mutex& notify_mutex, std::condition_variable& notify_cv) - : notify_mutex{notify_mutex}, notify_cv{notify_cv} {} - private: - variant_t response_{}; /* void* as default value so we don't initialize error_t or response_t */ - - std::mutex& notify_mutex; - std::condition_variable& notify_cv; - }; + class VirtualFileServer; namespace filesystem { template @@ -99,11 +31,9 @@ namespace ts::server::file { PATH_EXCEEDS_ROOT_PATH, PATH_IS_A_FILE, PATH_DOES_NOT_EXISTS, - FAILED_TO_LIST_FILES, - - MAX + FAILED_TO_LIST_FILES }; - constexpr std::array directory_query_error_messages = { + constexpr std::array directory_query_error_messages = { "unknown error", "path exceeds base path", "path is a file", @@ -143,11 +73,38 @@ namespace ts::server::file { TARGET_PATH_ALREADY_EXISTS, FAILED_TO_DELETE_FILES, FAILED_TO_RENAME_FILE, + FAILED_TO_CREATE_DIRECTORIES, SOME_FILES_ARE_LOCKED }; typedef DetailedError FileModifyError; + enum struct FileDeleteErrorType { + UNKNOWN, + }; + typedef DetailedError FileDeleteError; + + struct FileDeleteResponse { + enum struct StatusType { + SUCCESS, + + PATH_EXCEEDS_ROOT_PATH, + PATH_DOES_NOT_EXISTS, + FAILED_TO_DELETE_FILES, + SOME_FILES_ARE_LOCKED + }; + + struct DeleteResult { + StatusType status{StatusType::SUCCESS}; + std::string error_detail{}; + + DeleteResult(StatusType status, std::string errorDetail) : status{status}, + error_detail{std::move(errorDetail)} {} + }; + + std::vector delete_results{}; + }; + enum struct ServerCommandErrorType { UNKNOWN, FAILED_TO_CREATE_DIRECTORIES, @@ -155,27 +112,59 @@ namespace ts::server::file { }; typedef DetailedError ServerCommandError; + struct FileInfoResponse { + enum struct StatusType { + SUCCESS, + + PATH_EXCEEDS_ROOT_PATH, + PATH_DOES_NOT_EXISTS, + + FAILED_TO_QUERY_INFO, + UNKNOWN_FILE_TYPE + }; + + struct FileInfo { + StatusType status{StatusType::SUCCESS}; + std::string error_detail{}; + + DirectoryEntry info{}; + + FileInfo(StatusType status, std::string errorDetail, DirectoryEntry info) : status{status}, + error_detail{std::move(errorDetail)}, info{std::move(info)} {} + }; + + std::vector file_info{}; + }; + + enum struct FileInfoErrorType { + UNKNOWN, + }; + typedef DetailedError FileInfoError; + class AbstractProvider { public: typedef ExecuteResponse> directory_query_response_t; /* server */ - [[nodiscard]] virtual std::shared_ptr> initialize_server(ServerId /* server */) = 0; - [[nodiscard]] virtual std::shared_ptr> delete_server(ServerId /* server */) = 0; + [[nodiscard]] virtual std::shared_ptr> initialize_server(const std::shared_ptr &/* server */) = 0; + [[nodiscard]] virtual std::shared_ptr> delete_server(const std::shared_ptr &/* server */) = 0; /* channels */ - [[nodiscard]] virtual std::shared_ptr query_channel_directory(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */) = 0; - [[nodiscard]] virtual std::shared_ptr> create_channel_directory(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */) = 0; - [[nodiscard]] virtual std::shared_ptr> delete_channel_file(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */) = 0; - [[nodiscard]] virtual std::shared_ptr> rename_channel_file(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */, const std::string& /* target */) = 0; + [[nodiscard]] virtual std::shared_ptr> query_channel_info(const std::shared_ptr &/* server */, ChannelId /* channel */, const std::vector& /* names */) = 0; + [[nodiscard]] virtual std::shared_ptr query_channel_directory(const std::shared_ptr &/* server */, ChannelId /* channel */, const std::string& /* path */) = 0; + [[nodiscard]] virtual std::shared_ptr> create_channel_directory(const std::shared_ptr &/* server */, ChannelId /* channel */, const std::string& /* path */) = 0; + [[nodiscard]] virtual std::shared_ptr> delete_channel_files(const std::shared_ptr &/* server */, ChannelId /* channel */, const std::vector& /* paths */) = 0; + [[nodiscard]] virtual std::shared_ptr> rename_channel_file(const std::shared_ptr &/* server */, ChannelId /* channel */, const std::string& /* path */, ChannelId /* target channel */, const std::string& /* target */) = 0; /* icons */ - [[nodiscard]] virtual std::shared_ptr query_icon_directory(ServerId /* server */) = 0; - [[nodiscard]] virtual std::shared_ptr> delete_icon(ServerId /* server */, const std::string& /* name */) = 0; + [[nodiscard]] virtual std::shared_ptr> query_icon_info(const std::shared_ptr &/* server */, const std::vector& /* names */) = 0; + [[nodiscard]] virtual std::shared_ptr query_icon_directory(const std::shared_ptr &/* server */) = 0; + [[nodiscard]] virtual std::shared_ptr> delete_icons(const std::shared_ptr &/* server */, const std::vector& /* names */) = 0; /* avatars */ - [[nodiscard]] virtual std::shared_ptr query_avatar_directory(ServerId /* server */) = 0; - [[nodiscard]] virtual std::shared_ptr> delete_avatar(ServerId /* server */, const std::string& /* name */) = 0; + [[nodiscard]] virtual std::shared_ptr> query_avatar_info(const std::shared_ptr &/* server */, const std::vector& /* names */) = 0; + [[nodiscard]] virtual std::shared_ptr query_avatar_directory(const std::shared_ptr &/* server */) = 0; + [[nodiscard]] virtual std::shared_ptr> delete_avatars(const std::shared_ptr &/* server */, const std::vector& /* names */) = 0; private: }; } @@ -187,10 +176,12 @@ namespace ts::server::file { transfer_id server_transfer_id{0}; transfer_id client_transfer_id{0}; - ServerId server_id{0}; - ClientId client_id{0}; + std::shared_ptr server{nullptr}; ChannelId channel_id{0}; + ClientId client_id{0}; + std::string client_unique_id{}; + char transfer_key[TRANSFER_KEY_LENGTH]{}; std::chrono::system_clock::time_point initialized_timestamp{}; enum Direction { @@ -212,6 +203,10 @@ namespace ts::server::file { TARGET_TYPE_AVATAR } target_type{TARGET_TYPE_UNKNOWN}; std::string target_file_path{}; + std::string absolute_file_path{}; + + std::string relative_file_path{}; + std::string file_name{}; int64_t max_bandwidth{-1}; size_t expected_file_size{0}; /* incl. the offset! */ @@ -236,9 +231,24 @@ namespace ts::server::file { struct TransferInitError { enum Type { - UNKNOWN + UNKNOWN, + + INVALID_FILE_TYPE, + FILE_DOES_NOT_EXISTS, + FILE_IS_NOT_A_FILE, + + CLIENT_TOO_MANY_TRANSFERS, + SERVER_TOO_MANY_TRANSFERS, + + SERVER_QUOTA_EXCEEDED, + CLIENT_QUOTA_EXCEEDED, + + IO_ERROR } error_type{UNKNOWN}; std::string error_message{}; + + TransferInitError(Type errorType, std::string errorMessage) : error_type{errorType}, + error_message{std::move(errorMessage)} {} }; struct TransferActionError { @@ -268,22 +278,69 @@ namespace ts::server::file { std::string error_message{}; }; + struct ActiveFileTransfer { + transfer_id client_transfer_id{0}; + transfer_id server_transfer_id{0}; + + Transfer::Direction direction{Transfer::DIRECTION_UNKNOWN}; + + ClientId client_id{}; + std::string client_unique_id{}; + + std::string file_path{}; + std::string file_name{}; + + size_t expected_size{}; + size_t size_done{}; + + enum Status { + NOT_STARTED, + RUNNING, + INACTIVE /* (not used yet) */ + } status{Status::NOT_STARTED}; + + std::chrono::milliseconds runtime{}; + + double average_speed{0}; + double current_speed{0}; + }; + + enum struct TransferListError { + UNKNOWN + }; + class AbstractProvider { public: struct TransferInfo { std::string file_path{}; + std::string client_unique_id{}; + ClientId client_id{}; bool override_exiting{false}; /* only for upload valid */ + size_t file_offset{0}; size_t expected_file_size{0}; + int64_t max_bandwidth{-1}; + int64_t max_concurrent_transfers{-1}; + + /* only used for upload, for download the quotas could be checked before */ + size_t download_server_quota_limit{(size_t) -1}; + size_t download_client_quota_limit{(size_t) -1}; }; - virtual std::shared_ptr>> initialize_channel_transfer(Transfer::Direction /* direction */, ServerId /* server */, ChannelId /* channel */, const TransferInfo& /* info */) = 0; - virtual std::shared_ptr>> initialize_icon_transfer(Transfer::Direction /* direction */, ServerId /* server */, const TransferInfo& /* info */) = 0; - virtual std::shared_ptr>> initialize_avatar_transfer(Transfer::Direction /* direction */, ServerId /* server */, const TransferInfo& /* info */) = 0; + virtual std::shared_ptr>> + initialize_channel_transfer(Transfer::Direction /* direction */, const std::shared_ptr& /* server */, ChannelId /* channel */, const TransferInfo& /* info */) = 0; - virtual std::shared_ptr> stop_transfer(transfer_id /* id */, bool /* flush */) = 0; + virtual std::shared_ptr>> + initialize_icon_transfer(Transfer::Direction /* direction */, const std::shared_ptr& /* server */, const TransferInfo& /* info */) = 0; + + virtual std::shared_ptr>> + initialize_avatar_transfer(Transfer::Direction /* direction */, const std::shared_ptr& /* server */, const TransferInfo& /* info */) = 0; + + virtual std::shared_ptr>> list_transfer() = 0; + + virtual std::shared_ptr> stop_transfer(const std::shared_ptr& /* server */, transfer_id /* id */, bool /* flush */) = 0; std::function&)> callback_transfer_registered{}; /* transfer has been registered */ std::function&)> callback_transfer_started{}; /* transfer has been started */ @@ -293,11 +350,60 @@ namespace ts::server::file { }; } + class VirtualFileServer { + public: + explicit VirtualFileServer(ServerId server_id, std::string unique_id) : server_id_{server_id}, unique_id_{std::move(unique_id)} {} + + [[nodiscard]] inline auto unique_id() const -> const std::string& { return this->unique_id_; } + [[nodiscard]] inline auto server_id() const -> ServerId { return this->server_id_; } + + [[nodiscard]] inline auto max_networking_upload_bandwidth() const -> int64_t { return this->max_networking_upload_bandwidth_; } + virtual void max_networking_upload_bandwidth(int64_t value) { + this->max_networking_upload_bandwidth_ = value; + } + + [[nodiscard]] inline auto max_networking_download_bandwidth() const -> int64_t { return this->max_networking_download_bandwidth_; } + virtual void max_networking_download_bandwidth(int64_t value) { + this->max_networking_download_bandwidth_ = value; + } + + [[nodiscard]] inline auto generate_transfer_id() { + return ++this->current_transfer_id; + } + private: + ServerId server_id_; + std::string unique_id_; + + int64_t max_networking_upload_bandwidth_{-1}; + int64_t max_networking_download_bandwidth_{-1}; + + std::atomic current_transfer_id{0}; + }; + class AbstractFileServer { public: + [[nodiscard]] virtual std::string file_base_path() const = 0; [[nodiscard]] virtual filesystem::AbstractProvider& file_system() = 0; [[nodiscard]] virtual transfer::AbstractProvider& file_transfer() = 0; - private: + + [[nodiscard]] inline auto virtual_servers() const -> std::deque> { + std::lock_guard slock{this->servers_mutex}; + return this->servers_; + } + + [[nodiscard]] inline auto find_virtual_server(ServerId server_id) const -> std::shared_ptr { + std::lock_guard slock{this->servers_mutex}; + auto it = std::find_if(this->servers_.begin(), this->servers_.end(), [&](const std::shared_ptr& server) { + return server->server_id() == server_id; + }); + return it == this->servers_.end() ? nullptr : *it; + } + + virtual std::shared_ptr register_server(ServerId /* server id */) = 0; + virtual void unregister_server(ServerId /* server id */) = 0; + protected: + mutable std::mutex servers_mutex{}; + std::deque> servers_{}; }; extern bool initialize(std::string& /* error */); diff --git a/file/local_server/LocalFileProvider.cpp b/file/local_server/LocalFileProvider.cpp index 85e29dd..c145840 100644 --- a/file/local_server/LocalFileProvider.cpp +++ b/file/local_server/LocalFileProvider.cpp @@ -7,6 +7,7 @@ using namespace ts::server; using LocalFileServer = file::LocalFileProvider; +using LocalVirtualFileServer = file::LocalVirtualFileServer; EVP_PKEY* ssl_generate_key() { auto key = std::unique_ptr(EVP_PKEY_new(), ::EVP_PKEY_free); @@ -81,7 +82,7 @@ LocalFileServer::LocalFileProvider() : file_system_{}, file_transfer_{this->file LocalFileServer::~LocalFileProvider() {} bool LocalFileServer::initialize(std::string &error, const std::shared_ptr& ssl_options) { - if(!this->file_system_.initialize(error, "file-root/")) + if(!this->file_system_.initialize(error, "files/")) return false; @@ -89,7 +90,7 @@ bool LocalFileServer::initialize(std::string &error, const std::shared_ptr(); - binding->hostname = "localhost"; + binding->hostname = "0.0.0.0"; auto& iaddr = *(sockaddr_in*) &binding->address; iaddr.sin_family = AF_INET; @@ -116,6 +117,45 @@ file::filesystem::AbstractProvider &LocalFileServer::file_system() { return this->file_system_; } -file::transfer::AbstractProvider & LocalFileServer::file_transfer() { +file::transfer::AbstractProvider &LocalFileServer::file_transfer() { return this->file_transfer_; +} + +std::string file::LocalFileProvider::file_base_path() const { + return this->file_system_.root_path(); +} + +std::shared_ptr LocalFileServer::register_server(ServerId server_id) { + auto server = this->find_virtual_server(server_id); + if(server) return server; + + server = std::make_shared(server_id, std::to_string(server_id)); + { + std::lock_guard slock{this->servers_mutex}; + this->servers_.push_back(server); + } + + return server; +} + +void LocalFileServer::unregister_server(ServerId server_id) { + auto server_unique_id = std::to_string(server_id); + + std::lock_guard slock{this->servers_mutex}; + auto it = std::find_if(this->servers_.begin(), this->servers_.end(), [&](const std::shared_ptr& server) { + return server->unique_id() == server_unique_id; + }); + + if(it == this->servers_.end()) return; + this->servers_.erase(it); +} + +void LocalVirtualFileServer::max_networking_upload_bandwidth(int64_t value) { + VirtualFileServer::max_networking_upload_bandwidth(value); + this->upload_throttle.set_max_bandwidth(value); +} + +void LocalVirtualFileServer::max_networking_download_bandwidth(int64_t value) { + VirtualFileServer::max_networking_download_bandwidth(value); + this->download_throttle.set_max_bandwidth(value); } \ No newline at end of file diff --git a/file/local_server/LocalFileProvider.h b/file/local_server/LocalFileProvider.h index a0eb6d9..998ed25 100644 --- a/file/local_server/LocalFileProvider.h +++ b/file/local_server/LocalFileProvider.h @@ -9,7 +9,9 @@ #include #include #include +#include #include +#include "./NetTools.h" #define TRANSFER_MAX_CACHED_BYTES (1024 * 1024 * 1) // Buffer up to 1mb @@ -39,54 +41,66 @@ namespace ts::server::file { [[nodiscard]] inline const auto &root_path() const { return this->root_path_; } - [[nodiscard]] std::string absolute_avatar_path(ServerId, const std::string&); - [[nodiscard]] std::string absolute_icon_path(ServerId, const std::string&); - [[nodiscard]] std::string absolute_channel_path(ServerId, ChannelId, const std::string&); + [[nodiscard]] std::string absolute_avatar_path(const std::shared_ptr &, const std::string&); + [[nodiscard]] std::string absolute_icon_path(const std::shared_ptr &, const std::string&); + [[nodiscard]] std::string absolute_channel_path(const std::shared_ptr &, ChannelId, const std::string&); - std::shared_ptr> initialize_server(ServerId /* server */) override; - std::shared_ptr> delete_server(ServerId /* server */) override; + std::shared_ptr> initialize_server(const std::shared_ptr & /* server */) override; + std::shared_ptr> delete_server(const std::shared_ptr & /* server */) override; + + std::shared_ptr> + query_channel_info(const std::shared_ptr & /* server */, ChannelId /* channel */, const std::vector& /* names */) override; std::shared_ptr - query_channel_directory(ServerId id, ChannelId channelId, const std::string &string) override; + query_channel_directory(const std::shared_ptr & id, ChannelId channelId, const std::string &string) override; std::shared_ptr> - create_channel_directory(ServerId id, ChannelId channelId, const std::string &string) override; + create_channel_directory(const std::shared_ptr & id, ChannelId channelId, const std::string &string) override; + + std::shared_ptr> + delete_channel_files(const std::shared_ptr & id, ChannelId channelId, const std::vector &string) override; std::shared_ptr> - delete_channel_file(ServerId id, ChannelId channelId, const std::string &string) override; + rename_channel_file(const std::shared_ptr & id, ChannelId channelId, const std::string &, ChannelId, const std::string &) override; - std::shared_ptr> - rename_channel_file(ServerId id, ChannelId channelId, const std::string &, const std::string &) override; + std::shared_ptr> + query_icon_info(const std::shared_ptr & /* server */, const std::vector& /* names */) override; - std::shared_ptr query_icon_directory(ServerId id) override; + std::shared_ptr query_icon_directory(const std::shared_ptr & id) override; - std::shared_ptr> - delete_icon(ServerId id, const std::string &string) override; + std::shared_ptr> + delete_icons(const std::shared_ptr & id, const std::vector &string) override; - std::shared_ptr query_avatar_directory(ServerId id) override; + std::shared_ptr> + query_avatar_info(const std::shared_ptr & /* server */, const std::vector& /* names */) override; - std::shared_ptr> - delete_avatar(ServerId id, const std::string &string) override; + std::shared_ptr query_avatar_directory(const std::shared_ptr & id) override; + + std::shared_ptr> + delete_avatars(const std::shared_ptr & id, const std::vector &string) override; private: #ifdef FS_INCLUDED - [[nodiscard]] fs::path server_path(ServerId); - [[nodiscard]] fs::path server_channel_path(ServerId, ChannelId); + [[nodiscard]] fs::path server_path(const std::shared_ptr &); + [[nodiscard]] fs::path server_channel_path(const std::shared_ptr &, ChannelId); [[nodiscard]] static bool exceeds_base_path(const fs::path& /* base */, const fs::path& /* target */); [[nodiscard]] bool is_any_file_locked(const fs::path& /* base */, const std::string& /* path */, std::string& /* file (relative to the base) */); - [[nodiscard]] std::shared_ptr> - delete_file(const fs::path& /* base */, const std::string &string); + [[nodiscard]] std::shared_ptr> + delete_files(const fs::path& /* base */, const std::vector &string); [[nodiscard]] std::shared_ptr query_directory(const fs::path& /* base */, const std::string &string, bool); + + [[nodiscard]] std::shared_ptr> + query_file_info(const fs::path& /* base */, const std::vector &string); #endif template std::shared_ptr> create_execute_response() { return std::make_shared>(this->result_notify_mutex, this->result_notify_cv); } - std::string target_file_path(FileCategory type, ts::ServerId sid, ts::ChannelId cid, const std::string &path); + std::string target_file_path(FileCategory type, const std::shared_ptr &sid, ts::ChannelId cid, const std::string &path); std::mutex result_notify_mutex{}; std::condition_variable result_notify_cv{}; @@ -113,65 +127,6 @@ namespace ts::server::file { [[nodiscard]] extern Buffer* allocate_buffer(size_t); extern void free_buffer(Buffer*); - struct NetworkThrottle { - constexpr static auto kThrottleTimespanMs{250}; - typedef uint8_t span_t; - - ssize_t max_bytes{0}; - - span_t current_index{0}; - size_t bytes_send{0}; - - inline bool increase_bytes(size_t bytes) { - auto current_ms = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - auto current_span = (span_t) (current_ms / kThrottleTimespanMs); - if(this->current_index != current_span) { - this->current_index = current_span; - this->bytes_send = bytes; - } else { - this->bytes_send += bytes; - } - return this->max_bytes > 0 && this->bytes_send >= this->max_bytes; - } - - inline void set_max_bandwidth(ssize_t bytes_per_second) { - if(bytes_per_second <= 0) - this->max_bytes = -1; - else - this->max_bytes = bytes_per_second * kThrottleTimespanMs / 1000; - } - - [[nodiscard]] inline bool should_throttle(timeval& next_timestamp) { - if(this->max_bytes <= 0) return false; - - auto current_ms = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - auto current_span = (span_t) (current_ms / kThrottleTimespanMs); - if(this->current_index != current_span) return false; - if(this->bytes_send < this->max_bytes) return false; - - next_timestamp.tv_usec = (kThrottleTimespanMs - current_ms % kThrottleTimespanMs) * 1000; - next_timestamp.tv_sec = next_timestamp.tv_usec / 1000000; - next_timestamp.tv_usec -= next_timestamp.tv_sec * 1000000; - return true; - } - - [[nodiscard]] inline size_t bytes_left() const { - if(this->max_bytes <= 0) return (size_t) -1; - - auto current_ms = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - auto current_span = (span_t) (current_ms / kThrottleTimespanMs); - if(this->current_index != current_span) return this->max_bytes; - if(this->bytes_send < this->max_bytes) return this->max_bytes - this->bytes_send; - return 0; - } - - [[nodiscard]] inline std::chrono::milliseconds expected_writing_time(size_t bytes) const { - if(this->max_bytes <= 0) return std::chrono::milliseconds{0}; - - return std::chrono::seconds{bytes / (this->max_bytes * (1000 / kThrottleTimespanMs))}; - } - }; - /* all variables are locked via the state_mutex */ struct FileClient : std::enable_shared_from_this { LocalFileTransfer* handle; @@ -200,8 +155,6 @@ namespace ts::server::file { struct { bool file_locked{false}; - std::string absolute_path{}; - int file_descriptor{0}; bool currently_processing{false}; @@ -238,7 +191,9 @@ namespace ts::server::file { std::chrono::system_clock::time_point disconnect_timeout{}; - NetworkThrottle throttle; + networking::NetworkThrottle client_throttle{}; + /* the right side is the server throttle */ + networking::DualNetworkThrottle throttle{&client_throttle, &networking::NetworkThrottle::kNoThrottle}; pipes::SSL pipe_ssl{}; bool pipe_ssl_init{false}; @@ -250,20 +205,13 @@ namespace ts::server::file { } networking{}; struct { - size_t network_bytes_send{0}; - size_t network_bytes_received{0}; + networking::TransferStatistics network_send{}; + networking::TransferStatistics network_received{}; - size_t file_bytes_transferred{0}; + networking::TransferStatistics file_transferred{}; - /* used for delta statistics */ - size_t last_network_bytes_send{0}; - size_t last_network_bytes_received{0}; - - /* used for delta statistics */ - size_t last_file_bytes_transferred{0}; - - size_t disk_bytes_read{0}; - size_t disk_bytes_write{0}; + networking::TransferStatistics disk_bytes_read{}; + networking::TransferStatistics disk_bytes_write{}; } statistics{}; struct { @@ -327,8 +275,8 @@ namespace ts::server::file { FILE_TOO_LARGE, DISK_IS_READ_ONLY, - FILE_SIZE_MISMATCH, FILE_SEEK_FAILED, + FILE_SIZE_MISMATCH, FILE_IS_NOT_ACCESSIBLE, @@ -352,9 +300,8 @@ namespace ts::server::file { /* FILE_TOO_LARGE */ "file is too large", /* DISK_IS_READ_ONLY */ "disk is in read only mode", - /* FILE_SIZE_MISMATCH */ "file size mismatch", /* FILE_SEEK_FAILED */ "failed to seek to target file offset", - + /* FILE_SIZE_MISMATCH */ "file size miss match", /* FILE_IS_NOT_ACCESSIBLE */ "file is not accessible" }; @@ -413,16 +360,18 @@ namespace ts::server::file { } std::shared_ptr>> - initialize_channel_transfer(Transfer::Direction direction, ServerId id, ChannelId channelId, + initialize_channel_transfer(Transfer::Direction direction, const std::shared_ptr& server, ChannelId channelId, const TransferInfo &info) override; std::shared_ptr>> - initialize_icon_transfer(Transfer::Direction direction, ServerId id, const TransferInfo &info) override; + initialize_icon_transfer(Transfer::Direction direction, const std::shared_ptr& server, const TransferInfo &info) override; std::shared_ptr>> - initialize_avatar_transfer(Transfer::Direction direction, ServerId id, const TransferInfo &info) override; + initialize_avatar_transfer(Transfer::Direction direction, const std::shared_ptr& server, const TransferInfo &info) override; - std::shared_ptr> stop_transfer(transfer_id id, bool) override; + std::shared_ptr>> list_transfer() override; + + std::shared_ptr> stop_transfer(const std::shared_ptr& /* server */, transfer_id id, bool) override; private: enum struct DiskIOLoopState { STOPPED, @@ -433,13 +382,14 @@ namespace ts::server::file { }; filesystem::LocalFileSystem& file_system_; - std::atomic current_transfer_id{0}; + size_t max_concurrent_transfers{1024}; std::mt19937 transfer_random_token_generator{std::random_device{}()}; std::mutex result_notify_mutex{}; std::condition_variable result_notify_cv{}; std::mutex transfers_mutex{}; + std::mutex transfer_create_mutex{}; std::deque> transfers_{}; std::deque> pending_transfers{}; @@ -483,7 +433,7 @@ namespace ts::server::file { } std::shared_ptr>> - initialize_transfer(Transfer::Direction, ServerId, ChannelId, Transfer::TargetType, const TransferInfo &info); + initialize_transfer(Transfer::Direction, const std::shared_ptr &, ChannelId, Transfer::TargetType, const TransferInfo &info); [[nodiscard]] NetworkingStartResult start_networking(); [[nodiscard]] DiskIOStartResult start_disk_io(); @@ -529,6 +479,17 @@ namespace ts::server::file { }; } + class LocalVirtualFileServer : public VirtualFileServer { + public: + explicit LocalVirtualFileServer(ServerId server_id, std::string unique_id) : VirtualFileServer{server_id, std::move(unique_id)} {} + + void max_networking_upload_bandwidth(int64_t value) override; + void max_networking_download_bandwidth(int64_t value) override; + + networking::NetworkThrottle upload_throttle{}; + networking::NetworkThrottle download_throttle{}; + }; + class LocalFileProvider : public AbstractFileServer { public: LocalFileProvider(); @@ -537,9 +498,14 @@ namespace ts::server::file { [[nodiscard]] bool initialize(std::string& /* error */, const std::shared_ptr& /* ssl options */); void finalize(); + [[nodiscard]] std::string file_base_path() const override; + filesystem::AbstractProvider &file_system() override; transfer::AbstractProvider &file_transfer() override; + + std::shared_ptr register_server(ServerId /* server id */) override; + void unregister_server(ServerId /* server id */) override; private: filesystem::LocalFileSystem file_system_; transfer::LocalFileTransfer file_transfer_; diff --git a/file/local_server/LocalFileSystem.cpp b/file/local_server/LocalFileSystem.cpp index 2295fd1..49ec00f 100644 --- a/file/local_server/LocalFileSystem.cpp +++ b/file/local_server/LocalFileSystem.cpp @@ -36,12 +36,12 @@ bool LocalFileSystem::initialize(std::string &error_message, const std::string & void LocalFileSystem::finalize() {} -fs::path LocalFileSystem::server_path(ts::ServerId id) { - return fs::u8path(this->root_path_) / fs::u8path("server_" + std::to_string(id)); +fs::path LocalFileSystem::server_path(const std::shared_ptr &server) { + return fs::u8path(this->root_path_) / fs::u8path("server_" + std::to_string(server->server_id())); } -fs::path LocalFileSystem::server_channel_path(ts::ServerId sid, ts::ChannelId cid) { - return this->server_path(sid) / fs::u8path("channel_" + std::to_string(cid)); +fs::path LocalFileSystem::server_channel_path(const std::shared_ptr &server, ts::ChannelId cid) { + return this->server_path(server) / fs::u8path("channel_" + std::to_string(cid)); } bool LocalFileSystem::exceeds_base_path(const fs::path &base, const fs::path &target) { @@ -67,32 +67,32 @@ bool LocalFileSystem::is_any_file_locked(const fs::path &base, const std::string return false; } -std::string LocalFileSystem::target_file_path(FileCategory type, ts::ServerId sid, ts::ChannelId cid, const std::string &path) { +std::string LocalFileSystem::target_file_path(FileCategory type, const std::shared_ptr &server, ts::ChannelId cid, const std::string &path) { fs::path target_path{}; switch (type) { case FileCategory::CHANNEL: - target_path = this->server_channel_path(sid, cid) / path; + target_path = this->server_channel_path(server, cid) / path; break; case FileCategory::ICON: - target_path = this->server_path(sid) / "icons" / path; + target_path = this->server_path(server) / "icons" / path; break; case FileCategory::AVATAR: - target_path = this->server_path(sid) / "avatars" / path; + target_path = this->server_path(server) / "avatars" / path; break; } return clnpath(fs::absolute(target_path).string()); } -std::string LocalFileSystem::absolute_avatar_path(ServerId sid, const std::string &path) { +std::string LocalFileSystem::absolute_avatar_path(const std::shared_ptr &sid, const std::string &path) { return this->target_file_path(FileCategory::AVATAR, sid, 0, path); } -std::string LocalFileSystem::absolute_icon_path(ServerId sid, const std::string &path) { +std::string LocalFileSystem::absolute_icon_path(const std::shared_ptr &sid, const std::string &path) { return this->target_file_path(FileCategory::ICON, sid, 0, path); } -std::string LocalFileSystem::absolute_channel_path(ServerId sid, ChannelId cid, const std::string &path) { +std::string LocalFileSystem::absolute_channel_path(const std::shared_ptr &sid, ChannelId cid, const std::string &path) { return this->target_file_path(FileCategory::CHANNEL, sid, cid, path); } @@ -107,7 +107,7 @@ void LocalFileSystem::unlock_file(const std::string &c_path) { this->locked_files_.erase(std::remove_if(this->locked_files_.begin(), this->locked_files_.end(), [&](const auto& p) { return p == c_path; }), this->locked_files_.end()); } -std::shared_ptr> LocalFileSystem::initialize_server(ServerId id) { +std::shared_ptr> LocalFileSystem::initialize_server(const std::shared_ptr &id) { auto path = this->server_path(id); std::error_code error{}; @@ -126,7 +126,7 @@ std::shared_ptr> LocalFileSystem::initialize return response; } -std::shared_ptr> LocalFileSystem::delete_server(ServerId id) { +std::shared_ptr> LocalFileSystem::delete_server(const std::shared_ptr &id) { auto path = this->server_path(id); std::error_code error{}; @@ -217,19 +217,19 @@ std::shared_ptr LocalFileSystem::query_directory(con return response; } -std::shared_ptr LocalFileSystem::query_icon_directory(ServerId id) { +std::shared_ptr LocalFileSystem::query_icon_directory(const std::shared_ptr &id) { return this->query_directory(this->server_path(id) / fs::u8path("icons"), "/", true); } -std::shared_ptr LocalFileSystem::query_avatar_directory(ServerId id) { +std::shared_ptr LocalFileSystem::query_avatar_directory(const std::shared_ptr &id) { return this->query_directory(this->server_path(id) / fs::u8path("avatars"), "/", true); } -std::shared_ptr LocalFileSystem::query_channel_directory(ServerId id, ChannelId channelId, const std::string &path) { +std::shared_ptr LocalFileSystem::query_channel_directory(const std::shared_ptr &id, ChannelId channelId, const std::string &path) { return this->query_directory(this->server_channel_path(id, channelId), path, false); } -std::shared_ptr> LocalFileSystem::create_channel_directory(ServerId id, ChannelId channelId, const std::string &path) { +std::shared_ptr> LocalFileSystem::create_channel_directory(const std::shared_ptr &id, ChannelId channelId, const std::string &path) { auto channel_path_root = this->server_channel_path(id, channelId); std::error_code error{}; @@ -256,20 +256,21 @@ std::shared_ptr> LocalFileSystem::create_c return response; } -std::shared_ptr> LocalFileSystem::rename_channel_file(ServerId id, ChannelId channelId, const std::string ¤t_path_string, const std::string &new_path_string) { +std::shared_ptr> LocalFileSystem::rename_channel_file(const std::shared_ptr &id, ChannelId channelId, const std::string ¤t_path_string, ChannelId targetChannelId, const std::string &new_path_string) { auto channel_path_root = this->server_channel_path(id, channelId); + auto target_path_root = this->server_channel_path(id, targetChannelId); std::error_code error{}; std::string locked_file{}; auto response = this->create_execute_response(); auto current_path = channel_path_root / fs::u8path(current_path_string); - auto target_path = channel_path_root / fs::u8path(new_path_string); + auto target_path = target_path_root / fs::u8path(new_path_string); if(this->exceeds_base_path(channel_path_root, current_path)) { response->emplace_fail(FileModifyErrorType::PATH_EXCEEDS_ROOT_PATH, ""); return response; } - if(this->exceeds_base_path(channel_path_root, target_path)) { + if(this->exceeds_base_path(target_path_root, target_path)) { response->emplace_fail(FileModifyErrorType::TARGET_PATH_EXCEEDS_ROOT_PATH, ""); return response; } @@ -283,6 +284,15 @@ std::shared_ptr> LocalFileSystem::rename_channe return response; } + if(!fs::exists(target_path.parent_path(), error)) { + if(!fs::create_directories(target_path.parent_path(), error)) { + response->emplace_fail(FileModifyErrorType::FAILED_TO_CREATE_DIRECTORIES, std::to_string(error.value()) + "/" + error.message()); + return response; + } + } else if(error) { + logWarning(LOG_FT, "Failed to test for target directory existence for {}: {}/{}. Assuming it does exists", target_path.parent_path().string(), error.value(), error.message()); + } + if(fs::exists(target_path, error)) { response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, ""); return response; @@ -307,44 +317,126 @@ std::shared_ptr> LocalFileSystem::rename_channe return response; } -std::shared_ptr> LocalFileSystem::delete_file(const fs::path &base, - const std::string &path) { +std::shared_ptr> LocalFileSystem::delete_files(const fs::path &base, + const std::vector &paths) { std::error_code error{}; std::string locked_file{}; - auto response = this->create_execute_response(); - auto target_path = base / fs::u8path(path); + auto response = this->create_execute_response(); - if(fs::exists(target_path, error)) { - response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, ""); - return response; - } else if(error) { - logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does exists.", target_path.string(), error.value(), error.message()); - response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, ""); - return response; + std::vector delete_results{}; + for(const auto& path : paths) { + auto target_path = base / fs::u8path(path); + + if(!fs::exists(target_path, error)) { + delete_results.emplace_back(FileDeleteResponse::StatusType::PATH_DOES_NOT_EXISTS, ""); + continue; + } else if(error) { + logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does exists.", target_path.string(), error.value(), error.message()); + delete_results.emplace_back(FileDeleteResponse::StatusType::PATH_DOES_NOT_EXISTS, ""); + continue; + } + + if(this->is_any_file_locked(base, path, locked_file)) { + delete_results.emplace_back(FileDeleteResponse::StatusType::SOME_FILES_ARE_LOCKED, locked_file); + continue; + } + + if(!fs::remove_all(target_path, error) || error) { + delete_results.emplace_back(FileDeleteResponse::StatusType::FAILED_TO_DELETE_FILES, std::to_string(error.value()) + "/" + error.message()); + continue; + } + + delete_results.emplace_back(FileDeleteResponse::StatusType::SUCCESS, ""); } - if(this->is_any_file_locked(base, path, locked_file)) { - response->emplace_fail(FileModifyErrorType::SOME_FILES_ARE_LOCKED, locked_file); - return response; - } - - if(!fs::remove(target_path, error) || error) { - response->emplace_fail(FileModifyErrorType::FAILED_TO_DELETE_FILES, std::to_string(error.value()) + "/" + error.message()); - return response; - } - - response->emplace_success(); + response->emplace_success(FileDeleteResponse{delete_results}); return response; } -std::shared_ptr> LocalFileSystem::delete_channel_file(ServerId id, ChannelId channelId, const std::string &path) { - return this->delete_file(this->server_channel_path(id, channelId), path); +std::shared_ptr> LocalFileSystem::delete_channel_files(const std::shared_ptr &id, ChannelId channelId, const std::vector &path) { + return this->delete_files(this->server_channel_path(id, channelId), path); } -std::shared_ptr> LocalFileSystem::delete_icon(ServerId id, const std::string &icon) { - return this->delete_file(this->server_path(id) / fs::u8path("icons"), icon); +std::shared_ptr> LocalFileSystem::delete_icons(const std::shared_ptr &id, const std::vector &icon) { + return this->delete_files(this->server_path(id) / fs::u8path("icons"), icon); } -std::shared_ptr> LocalFileSystem::delete_avatar(ServerId id, const std::string &avatar) { - return this->delete_file(this->server_path(id) / fs::u8path("avatars"), avatar); +std::shared_ptr> LocalFileSystem::delete_avatars(const std::shared_ptr &id, const std::vector &avatar) { + return this->delete_files(this->server_path(id) / fs::u8path("avatars"), avatar); +} + +std::shared_ptr> LocalFileSystem::query_file_info(const fs::path &base, + const std::vector &paths) { + std::error_code error{}; + auto response = this->create_execute_response(); + std::vector file_infos{}; + file_infos.reserve(paths.size()); + + for(const auto& path : paths) { + auto target_path = base / fs::u8path(path); + if(this->exceeds_base_path(base, target_path)) { + file_infos.emplace_back(FileInfoResponse::StatusType::PATH_EXCEEDS_ROOT_PATH, "", DirectoryEntry{}); + continue; + } + + if(!fs::exists(target_path, error)) { + file_infos.emplace_back(FileInfoResponse::StatusType::PATH_DOES_NOT_EXISTS, "", DirectoryEntry{}); + continue; + } else if(error) { + logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", target_path.string(), error.value(), error.message()); + + file_infos.emplace_back(FileInfoResponse::StatusType::PATH_DOES_NOT_EXISTS, "", DirectoryEntry{}); + continue; + } + + auto status = fs::status(target_path, error); + if(error) { + logWarning(LOG_FT, "Failed to query file status for {} ({}/{}). Skipping entry for file info query.", target_path.string(), error.value(), error.message()); + + file_infos.emplace_back(FileInfoResponse::StatusType::FAILED_TO_QUERY_INFO, "", DirectoryEntry{}); + continue; + } + + if(status.type() == fs::file_type::directory) { + DirectoryEntry dentry{}; + dentry.type = DirectoryEntry::DIRECTORY; + dentry.name = target_path.filename(); + + dentry.modified_at = fs::last_write_time(target_path, error); + if(error) + logWarning(LOG_FT, "Failed to query last write time for file {} ({}/{})", target_path.string(), error.value(), error.message()); + dentry.size = 0; + file_infos.emplace_back(FileInfoResponse::StatusType::SUCCESS, "", dentry); + } else if(status.type() == fs::file_type::regular) { + DirectoryEntry dentry{}; + dentry.type = DirectoryEntry::FILE; + dentry.name = target_path.filename(); + + dentry.modified_at = fs::last_write_time(target_path, error); + if(error) + logWarning(LOG_FT, "Failed to query last write time for file {} ({}/{}).", target_path.string(), error.value(), error.message()); + dentry.size = fs::file_size(target_path, error); + if(error) + logWarning(LOG_FT, "Failed to query size for file {} ({}/{}).", target_path.string(), error.value(), error.message()); + file_infos.emplace_back(FileInfoResponse::StatusType::SUCCESS, "", dentry); + } else { + logWarning(LOG_FT, "File info query contains an unknown file type for file {} ({}).", target_path.string(), (int) status.type()); + file_infos.emplace_back(FileInfoResponse::StatusType::UNKNOWN_FILE_TYPE, "", DirectoryEntry{}); + } + } + + response->emplace_success(FileInfoResponse{file_infos}); + return response; +} + +std::shared_ptr> LocalFileSystem::query_channel_info(const std::shared_ptr &sid, ChannelId cid, const std::vector &paths) { + return this->query_file_info(this->server_channel_path(sid, cid), paths); +} + +std::shared_ptr> LocalFileSystem::query_icon_info(const std::shared_ptr &sid, const std::vector &paths) { + return this->query_file_info(this->server_path(sid) / fs::u8path("icons"), paths); +} + +std::shared_ptr > LocalFileSystem::query_avatar_info(const std::shared_ptr &sid, const std::vector &paths) { + return this->query_file_info(this->server_path(sid) / fs::u8path("avatars"), paths); } \ No newline at end of file diff --git a/file/local_server/LocalFileTransfer.cpp b/file/local_server/LocalFileTransfer.cpp index 8904305..3ab049f 100644 --- a/file/local_server/LocalFileTransfer.cpp +++ b/file/local_server/LocalFileTransfer.cpp @@ -8,7 +8,9 @@ #include #include "./LocalFileProvider.h" #include "LocalFileProvider.h" +#include +namespace fs = std::experimental::filesystem; using namespace ts::server::file; using namespace ts::server::file::transfer; @@ -101,29 +103,59 @@ void LocalFileTransfer::stop() { this->shutdown_client_worker(); } -std::shared_ptr>> LocalFileTransfer::initialize_icon_transfer(Transfer::Direction direction, ServerId sid, const TransferInfo &info) { - return this->initialize_transfer(direction, sid, 0, Transfer::TARGET_TYPE_ICON, info); +std::shared_ptr>> + LocalFileTransfer::initialize_icon_transfer(Transfer::Direction direction, const std::shared_ptr &server, const TransferInfo &info) { + return this->initialize_transfer(direction, server, 0, Transfer::TARGET_TYPE_ICON, info); } -std::shared_ptr>> LocalFileTransfer::initialize_avatar_transfer(Transfer::Direction direction, ServerId sid, const TransferInfo &info) { - return this->initialize_transfer(direction, sid, 0, Transfer::TARGET_TYPE_AVATAR, info); +std::shared_ptr>> + LocalFileTransfer::initialize_avatar_transfer(Transfer::Direction direction, const std::shared_ptr &server, const TransferInfo &info) { + return this->initialize_transfer(direction, server, 0, Transfer::TARGET_TYPE_AVATAR, info); } -std::shared_ptr>> LocalFileTransfer::initialize_channel_transfer(Transfer::Direction direction, ServerId sid, ChannelId cid, const TransferInfo &info) { - return this->initialize_transfer(direction, sid, cid, Transfer::TARGET_TYPE_CHANNEL_FILE, info); +std::shared_ptr>> + LocalFileTransfer::initialize_channel_transfer(Transfer::Direction direction, const std::shared_ptr &server, ChannelId cid, const TransferInfo &info) { + return this->initialize_transfer(direction, server, cid, Transfer::TARGET_TYPE_CHANNEL_FILE, info); } std::shared_ptr>> LocalFileTransfer::initialize_transfer( - Transfer::Direction direction, ServerId sid, ChannelId cid, + Transfer::Direction direction, const std::shared_ptr &server, ChannelId cid, Transfer::TargetType ttype, const TransferInfo &info) { auto response = this->create_execute_response>(); - /* TODO: test for a transfer limit */ + std::lock_guard clock{this->transfer_create_mutex}; + if(info.max_concurrent_transfers > 0) { + std::unique_lock tlock{this->transfers_mutex}; + { + auto transfers = std::count_if(this->transfers_.begin(), this->transfers_.end(), [&](const std::shared_ptr& client) { + return client->transfer && client->transfer->client_unique_id == info.client_unique_id; + }); + transfers += std::count_if(this->pending_transfers.begin(), this->pending_transfers.end(), [&](const std::shared_ptr& transfer) { + return transfer->client_unique_id == info.client_unique_id; + }); + + if(transfers >= info.max_concurrent_transfers) { + response->emplace_fail(TransferInitError::CLIENT_TOO_MANY_TRANSFERS, std::to_string(transfers)); + return response; + } + } + + { + auto server_transfers = this->pending_transfers.size(); + server_transfers += std::count_if(this->transfers_.begin(), this->transfers_.end(), [&](const std::shared_ptr& client) { + return client->transfer; + }); + if(server_transfers >= this->max_concurrent_transfers) { + response->emplace_fail(TransferInitError::SERVER_TOO_MANY_TRANSFERS, std::to_string(server_transfers)); + return response; + } + } + } auto transfer = std::make_shared(); - transfer->server_transfer_id = ++this->current_transfer_id; - transfer->server_id = sid; + transfer->server_transfer_id = server->generate_transfer_id(); + transfer->server = server; transfer->channel_id = cid; transfer->target_type = ttype; transfer->direction = direction; @@ -142,6 +174,8 @@ std::shared_ptr>> L transfer->file_offset = info.file_offset; transfer->expected_file_size = info.expected_file_size; transfer->max_bandwidth = info.max_bandwidth; + transfer->client_unique_id = info.client_unique_id; + transfer->client_id = info.client_id; constexpr static std::string_view kTokenCharacters{"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"}; for(auto& c : transfer->transfer_key) @@ -152,6 +186,70 @@ std::shared_ptr>> L transfer->initialized_timestamp = std::chrono::system_clock::now(); + { + std::string absolute_path{}; + switch (transfer->target_type) { + case Transfer::TARGET_TYPE_AVATAR: + absolute_path = this->file_system_.absolute_avatar_path(transfer->server, transfer->target_file_path); + logMessage(LOG_FT, "Initialized avatar transfer for avatar \"{}\" ({} bytes, transferring {} bytes).", transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset); + break; + case Transfer::TARGET_TYPE_ICON: + absolute_path = this->file_system_.absolute_icon_path(transfer->server, transfer->target_file_path); + logMessage(LOG_FT, "Initialized icon transfer for icon \"{}\" ({} bytes, transferring {} bytes).", + transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset); + break; + case Transfer::TARGET_TYPE_CHANNEL_FILE: + absolute_path = this->file_system_.absolute_channel_path(transfer->server, transfer->channel_id, transfer->target_file_path); + logMessage(LOG_FT, "Initialized channel transfer for file \"{}/{}\" ({} bytes, transferring {} bytes).", + transfer->channel_id, transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset); + break; + case Transfer::TARGET_TYPE_UNKNOWN: + default: + response->emplace_fail(TransferInitError::INVALID_FILE_TYPE, ""); + return response; + } + transfer->absolute_file_path = absolute_path; + + const auto root_path_length = this->file_system_.root_path().size(); + if(root_path_length < absolute_path.size()) + transfer->relative_file_path = absolute_path.substr(root_path_length); + else + transfer->relative_file_path = "error"; + transfer->file_name = fs::u8path(absolute_path).filename(); + } + + if(direction == Transfer::DIRECTION_DOWNLOAD) { + auto path = fs::u8path(transfer->absolute_file_path); + std::error_code error{}; + if(!fs::exists(path, error)) { + response->emplace_fail(TransferInitError::FILE_DOES_NOT_EXISTS, ""); + return response; + } else if(error) { + logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", transfer->absolute_file_path, error.value(), error.message()); + response->emplace_fail(TransferInitError::FILE_DOES_NOT_EXISTS, ""); + return response; + } + + auto status = fs::status(path, error); + if(error) { + logWarning(LOG_FT, "Failed to status for file at {}: {}. Ignoring file transfer.", transfer->absolute_file_path, error.value(), error.message()); + response->emplace_fail(TransferInitError::IO_ERROR, "stat"); + return response; + } + + if(status.type() != fs::file_type::regular) { + response->emplace_fail(TransferInitError::FILE_IS_NOT_A_FILE, ""); + return response; + } + + transfer->expected_file_size = fs::file_size(path, error); + if(error) { + logWarning(LOG_FT, "Failed to get file size for file at {}: {}. Ignoring file transfer.", transfer->absolute_file_path, error.value(), error.message()); + response->emplace_fail(TransferInitError::IO_ERROR, "file_size"); + return response; + } + } + { std::lock_guard tlock{this->transfers_mutex}; this->pending_transfers.push_back(transfer); @@ -164,7 +262,7 @@ std::shared_ptr>> L return response; } -std::shared_ptr> LocalFileTransfer::stop_transfer(transfer_id id, bool flush) { +std::shared_ptr> LocalFileTransfer::stop_transfer(const std::shared_ptr& server, transfer_id id, bool flush) { auto response = this->create_execute_response(); std::shared_ptr transfer{}; @@ -174,13 +272,13 @@ std::shared_ptr> LocalFileTransfer::stop_tr std::lock_guard tlock{this->transfers_mutex}; auto ct_it = std::find_if(this->transfers_.begin(), this->transfers_.end(), [&](const std::shared_ptr& t) { - return t->transfer && t->transfer->server_transfer_id == id; + return t->transfer && t->transfer->server_transfer_id == id && t->transfer->server == server; }); if(ct_it != this->transfers_.end()) connected_transfer = *ct_it; else { auto t_it = std::find_if(this->pending_transfers.begin(), this->pending_transfers.end(), [&](const std::shared_ptr& t) { - return t->server_transfer_id == id; + return t->server_transfer_id == id && t->server == server; }); if(t_it != this->pending_transfers.end()) { transfer = *t_it; @@ -209,4 +307,60 @@ std::shared_ptr> LocalFileTransfer::stop_tr response->emplace_success(); return response; +} + +inline void apply_transfer_info(const std::shared_ptr& transfer, ActiveFileTransfer& info) { + info.server_transfer_id = transfer->server_transfer_id; + info.client_transfer_id = transfer->client_transfer_id; + info.direction = transfer->direction; + info.client_id = transfer->client_id; + info.client_unique_id = transfer->client_unique_id; + + info.file_path = transfer->relative_file_path; + info.file_name = transfer->file_name; + + info.expected_size = transfer->expected_file_size; +} + +std::shared_ptr>> LocalFileTransfer::list_transfer() { + std::vector transfer_infos{}; + auto response = this->create_execute_response>(); + + std::unique_lock tlock{this->transfers_mutex}; + auto awaiting_transfers = this->pending_transfers; + auto running_transfers = this->transfers_; + tlock.unlock(); + + transfer_infos.reserve(awaiting_transfers.size() + running_transfers.size()); + for(const auto& transfer : awaiting_transfers) { + ActiveFileTransfer info{}; + apply_transfer_info(transfer, info); + info.size_done = transfer->file_offset; + + info.status = ActiveFileTransfer::NOT_STARTED; + info.runtime = std::chrono::milliseconds{0}; + + info.average_speed = 0; + info.current_speed = 0; + transfer_infos.push_back(info); + } + + for(const auto& client : running_transfers) { + auto transfer = client->transfer; + if(!transfer) continue; + + ActiveFileTransfer info{}; + apply_transfer_info(transfer, info); + info.size_done = transfer->file_offset + client->statistics.file_transferred.total_bytes; + + info.status = ActiveFileTransfer::RUNNING; + info.runtime = std::chrono::floor(std::chrono::system_clock::now() - client->timings.key_received); + + info.average_speed = client->statistics.file_transferred.average_bandwidth(); + info.current_speed = client->statistics.file_transferred.current_bandwidth(); + transfer_infos.push_back(info); + } + + response->emplace_success(std::move(transfer_infos)); + return response; } \ No newline at end of file diff --git a/file/local_server/LocalFileTransferClientWorker.cpp b/file/local_server/LocalFileTransferClientWorker.cpp index 2a354d0..1dffb80 100644 --- a/file/local_server/LocalFileTransferClientWorker.cpp +++ b/file/local_server/LocalFileTransferClientWorker.cpp @@ -46,7 +46,7 @@ void LocalFileTransfer::disconnect_client(const std::shared_ptr &cli client->state = flush ? FileClient::STATE_DISCONNECTING : FileClient::STATE_DISCONNECTED; client->timings.disconnecting = std::chrono::system_clock::now(); if(flush) { - const auto network_flush_time = client->networking.throttle.expected_writing_time(client->buffer.bytes) + std::chrono::seconds{10}; + const auto network_flush_time = client->networking.client_throttle.expected_writing_time(client->buffer.bytes) + std::chrono::seconds{10}; if(!client->transfer) { del_ev_noblock(client->networking.event_read); @@ -104,6 +104,9 @@ void LocalFileTransfer::dispatch_loop_client_worker(void *ptr_transfer) { case FileClient::STATE_DISCONNECTING: if(transfer->transfer && transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) break; /* we're still transferring (sending data) */ + continue; + case FileClient::STATE_AWAITING_KEY: + case FileClient::STATE_DISCONNECTED: default: continue; } @@ -182,6 +185,7 @@ void LocalFileTransfer::dispatch_loop_client_worker(void *ptr_transfer) { case FileClient::STATE_DISCONNECTING: logMessage(LOG_FT, "{} Failed to flush connection. Dropping client", client->log_prefix()); break; + case FileClient::STATE_DISCONNECTED: default: break; } @@ -205,17 +209,17 @@ void LocalFileTransfer::report_transfer_statistics(const std::shared_ptrstatistics.network_bytes_send; - stats.network_bytes_received = client->statistics.network_bytes_received; - stats.file_bytes_transferred = client->statistics.file_bytes_transferred; + stats.network_bytes_send = client->statistics.network_send.total_bytes; + stats.network_bytes_received = client->statistics.network_received.total_bytes; + stats.file_bytes_transferred = client->statistics.file_transferred.total_bytes; - stats.delta_network_bytes_received = stats.network_bytes_received - std::exchange(client->statistics.last_network_bytes_received, stats.network_bytes_received); - stats.delta_network_bytes_send = stats.network_bytes_send - std::exchange(client->statistics.last_network_bytes_send, stats.network_bytes_send); + stats.delta_network_bytes_received = client->statistics.network_received.take_delta(); + stats.delta_network_bytes_send = client->statistics.network_received.take_delta(); - stats.delta_file_bytes_transferred = stats.file_bytes_transferred - std::exchange(client->statistics.last_file_bytes_transferred, stats.file_bytes_transferred); + stats.delta_file_bytes_transferred = client->statistics.file_transferred.take_delta(); stats.file_start_offset = client->transfer->file_offset; - stats.file_current_offset = client->statistics.file_bytes_transferred + client->transfer->file_offset; + stats.file_current_offset = client->statistics.file_transferred.total_bytes + client->transfer->file_offset; stats.file_total_size = client->transfer->expected_file_size; callback(client->transfer, stats); diff --git a/file/local_server/LocalFileTransferDisk.cpp b/file/local_server/LocalFileTransferDisk.cpp index c1ba09c..9bcc6f1 100644 --- a/file/local_server/LocalFileTransferDisk.cpp +++ b/file/local_server/LocalFileTransferDisk.cpp @@ -107,17 +107,18 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr assert(!file_data.file_descriptor); assert(!file_data.next_client); + auto absolute_path = transfer->transfer->absolute_file_path; { unsigned int open_flags{0}; if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) { open_flags = O_RDONLY; std::error_code fs_error{}; - if(file_data.absolute_path.empty() || !fs::exists(file_data.absolute_path, fs_error)) { + if(absolute_path.empty() || !fs::exists(absolute_path, fs_error)) { result = FileInitializeResult::FILE_DOES_NOT_EXISTS; goto error_exit; } else if(fs_error) { - logWarning(LOG_FT, "{} Failed to check for file existence of {}: {}/{}", transfer->log_prefix(), file_data.absolute_path, fs_error.value(), fs_error.message()); + logWarning(LOG_FT, "{} Failed to check for file existence of {}: {}/{}", transfer->log_prefix(), absolute_path, fs_error.value(), fs_error.message()); result = FileInitializeResult::FILE_SYSTEM_ERROR; goto error_exit; } @@ -129,7 +130,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr return FileInitializeResult::INVALID_TRANSFER_DIRECTION; } - file_data.file_descriptor = open(file_data.absolute_path.c_str(), (int) open_flags, 0644); + file_data.file_descriptor = open(absolute_path.c_str(), (int) open_flags, 0644); if(file_data.file_descriptor <= 0) { const auto errno_ = errno; switch (errno_) { @@ -137,7 +138,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE; break; case EDQUOT: - logWarning(LOG_FT, "{} Disk inode limit has been reached. Failed to start file transfer for file {}", transfer->log_prefix(), file_data.absolute_path); + logWarning(LOG_FT, "{} Disk inode limit has been reached. Failed to start file transfer for file {}", transfer->log_prefix(), absolute_path); result = FileInitializeResult::FILE_SYSTEM_ERROR; break; case EISDIR: @@ -156,7 +157,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr result = FileInitializeResult::DISK_IS_READ_ONLY; break; default: - logWarning(LOG_FT, "{} Failed to start file transfer for file {}: {}/{}", transfer->log_prefix(), file_data.absolute_path, errno_, strerror(errno_)); + logWarning(LOG_FT, "{} Failed to start file transfer for file {}: {}/{}", transfer->log_prefix(), absolute_path, errno_, strerror(errno_)); result = FileInitializeResult::FILE_SYSTEM_ERROR; break; } @@ -164,7 +165,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr } } - this->file_system_.lock_file(transfer->file.absolute_path); + this->file_system_.lock_file(absolute_path); transfer->file.file_locked = true; if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) { @@ -172,7 +173,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr const auto errno_ = errno; switch (errno_) { case EACCES: - logWarning(LOG_FT, "{} File {} got inaccessible on truncating, but not on opening.", transfer->log_prefix(), file_data.absolute_path); + logWarning(LOG_FT, "{} File {} got inaccessible on truncating, but not on opening.", transfer->log_prefix(), absolute_path); result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE; goto error_exit; @@ -181,16 +182,16 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr goto error_exit; case EIO: - logWarning(LOG_FT, "{} A disk IO error occurred while resizing file {}.", transfer->log_prefix(), file_data.absolute_path); + logWarning(LOG_FT, "{} A disk IO error occurred while resizing file {}.", transfer->log_prefix(), absolute_path); result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE; goto error_exit; case EROFS: - logWarning(LOG_FT, "{} Failed to resize file {} because disk is in read only mode.", transfer->log_prefix(), file_data.absolute_path); + logWarning(LOG_FT, "{} Failed to resize file {} because disk is in read only mode.", transfer->log_prefix(), absolute_path); result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE; goto error_exit; default: - debugMessage(LOG_FT, "{} Failed to truncate file {}: {}/{}. Trying to upload file anyways.", transfer->log_prefix(), file_data.absolute_path, errno_, strerror(errno_)); + debugMessage(LOG_FT, "{} Failed to truncate file {}: {}/{}. Trying to upload file anyways.", transfer->log_prefix(), absolute_path, errno_, strerror(errno_)); } } } else if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) { @@ -218,7 +219,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr return FileInitializeResult::SUCCESS; error_exit: if(std::exchange(transfer->file.file_locked, false)) - this->file_system_.unlock_file(transfer->file.absolute_path); + this->file_system_.unlock_file(absolute_path); if(file_data.file_descriptor > 0) ::close(file_data.file_descriptor); @@ -230,6 +231,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr void LocalFileTransfer::finalize_file_io(const std::shared_ptr &transfer, std::unique_lock &state_lock) { assert(state_lock.owns_lock()); + auto absolute_path = transfer->transfer->absolute_file_path; auto& file_data = transfer->file; @@ -268,7 +270,7 @@ void LocalFileTransfer::finalize_file_io(const std::shared_ptr &tran state_lock.lock(); if(std::exchange(file_data.file_locked, false)) - this->file_system_.unlock_file(file_data.absolute_path); + this->file_system_.unlock_file(absolute_path); if(file_data.file_descriptor > 0) ::close(file_data.file_descriptor); @@ -329,7 +331,7 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr &clien if(written <= 0) { if(written == 0) { /* EOF, how the hell is this event possible?! */ - auto offset_written = client->statistics.disk_bytes_write + client->transfer->file_offset; + auto offset_written = client->statistics.disk_bytes_write.total_bytes + client->transfer->file_offset; auto aoffset = lseek(client->file.file_descriptor, 0, SEEK_CUR); logError(LOG_FT, "{} Received unexpected file write EOF. EOF received at {} but expected {}. Actual file offset: {}. Closing transfer.", client->log_prefix(), offset_written, client->transfer->expected_file_size, aoffset); @@ -349,7 +351,7 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr &clien break; } - auto offset_written = client->statistics.disk_bytes_write + client->transfer->file_offset; + auto offset_written = client->statistics.disk_bytes_write.total_bytes + client->transfer->file_offset; auto aoffset = lseek(client->file.file_descriptor, 0, SEEK_CUR); logError(LOG_FT, "{} Received write to disk IO error. Write pointer is at {} of {}. Actual file offset: {}. Closing transfer.", client->log_prefix(), offset_written, client->transfer->expected_file_size, aoffset); @@ -389,7 +391,7 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr &clien (void) buffer_left_size; /* trick my IDE here a bit */ } - client->statistics.disk_bytes_write += written; + client->statistics.disk_bytes_write.increase_bytes(written); } } @@ -418,7 +420,7 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr &clien if(read <= 0) { if(read == 0) { /* EOF */ - auto offset_send = client->statistics.disk_bytes_read + client->transfer->file_offset; + auto offset_send = client->statistics.disk_bytes_read.total_bytes + client->transfer->file_offset; if(client->transfer->expected_file_size == offset_send) { debugMessage(LOG_FT, "{} Finished file reading. Flushing and disconnecting transfer. Reading took {} seconds.", client->log_prefix(), duration_to_string(std::chrono::system_clock::now() - client->timings.key_received)); @@ -442,7 +444,7 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr &clien return; } - logWarning(LOG_FT, "{} Failed to read from file {} ({}/{}). Aborting transfer.", client->log_prefix(), client->file.absolute_path, errno, strerror(errno)); + logWarning(LOG_FT, "{} Failed to read from file {} ({}/{}). Aborting transfer.", client->log_prefix(), client->transfer->absolute_file_path, errno, strerror(errno)); this->report_transfer_statistics(client); if(auto callback{client->handle->callback_transfer_aborted}; callback) @@ -456,12 +458,12 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr &clien return; } else { auto buffer_full = client->send_file_bytes(buffer, read); - client->statistics.disk_bytes_read += read; - client->statistics.file_bytes_transferred += read; + client->statistics.disk_bytes_read.increase_bytes(read); + client->statistics.file_transferred.increase_bytes(read); std::shared_lock slock{client->state_mutex}; if(buffer_full) { - logMessage(LOG_FT, "{} Stopping buffering from disk. Buffer full ({}bytes)", client->log_prefix(), client->buffer.bytes); + //logTrace(LOG_FT, "{} Stopping buffering from disk. Buffer full ({}bytes)", client->log_prefix(), client->buffer.bytes); break; } diff --git a/file/local_server/LocalFileTransferNetwork.cpp b/file/local_server/LocalFileTransferNetwork.cpp index 86baaaf..aae2a1a 100644 --- a/file/local_server/LocalFileTransferNetwork.cpp +++ b/file/local_server/LocalFileTransferNetwork.cpp @@ -452,6 +452,7 @@ void LocalFileTransfer::callback_transfer_network_read(int fd, short events, voi const auto max_read_buffer = transfer->networking.throttle.bytes_left(); if(!max_read_buffer) break; /* network throttle */ + errno = 0; auto read = ::recv(fd, buffer, std::min(buffer_size, std::min(max_read_buffer, transfer->networking.max_read_buffer_size)), MSG_NOSIGNAL | MSG_DONTWAIT); //logTrace(0, "Read {}, max {} | {}", read, std::min(buffer_size, std::min(max_read_buffer, transfer->networking.max_read_buffer_size)), max_read_buffer); if(read <= 0) { @@ -471,10 +472,10 @@ void LocalFileTransfer::callback_transfer_network_read(int fd, short events, voi /* since we're counting the bytes directly on read, a disconnect would mean that not all data has been transferred */ if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) { logMessage(LOG_FT, "{} Disconnected while receiving file. Received {} out of {} bytes", - transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset); + transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset); } else if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) { logMessage(LOG_FT, "{} Disconnected while sending file. Send {} out of {} bytes", - transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset); + transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset); } transfer->handle->report_transfer_statistics(transfer->shared_from_this()); @@ -482,6 +483,8 @@ void LocalFileTransfer::callback_transfer_network_read(int fd, short events, voi callback(transfer->transfer, { TransferError::UNEXPECTED_CLIENT_DISCONNECT, "" }); break; } + case FileClient::STATE_DISCONNECTING: + case FileClient::STATE_DISCONNECTED: default: break; } @@ -506,16 +509,18 @@ void LocalFileTransfer::callback_transfer_network_read(int fd, short events, voi /* since we're counting the bytes directly on read, a disconnect would mean that not all data has been transferred */ if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) { logMessage(LOG_FT, "{} Received read error while receiving file. Received {} out of {} bytes: {}/{}", - transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset, errno, strerror(errno)); + transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset, errno, strerror(errno)); } else if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) { logMessage(LOG_FT, "{} Received read error while sending file. Send {} out of {} bytes: {}/{}", - transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset, errno, strerror(errno)); + transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset, errno, strerror(errno)); } transfer->handle->report_transfer_statistics(transfer->shared_from_this()); if(auto callback{transfer->handle->callback_transfer_aborted}; callback) callback(transfer->transfer, { TransferError::NETWORK_IO_ERROR, strerror(errno) }); break; + case FileClient::STATE_DISCONNECTING: + case FileClient::STATE_DISCONNECTED: default: break; } @@ -524,7 +529,7 @@ void LocalFileTransfer::callback_transfer_network_read(int fd, short events, voi return; } else { transfer->timings.last_read = std::chrono::system_clock::now(); - transfer->statistics.network_bytes_received += read; + transfer->statistics.network_received.increase_bytes(read); bool throttle_read = transfer->networking.throttle.increase_bytes(read); if(transfer->state != FileClient::STATE_AWAITING_KEY && !(transfer->state == FileClient::STATE_TRANSFERRING && transfer->transfer->direction == Transfer::DIRECTION_UPLOAD)) { @@ -616,7 +621,7 @@ void LocalFileTransfer::callback_transfer_network_write(int fd, short events, vo if(written == 0) { /* EOF, how the hell is this event possible?! (Read should already catch it) */ logError(LOG_FT, "{} Client disconnected unexpectedly on write. Send {} bytes out of {}.", - transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset); + transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset); transfer->handle->report_transfer_statistics(transfer->shared_from_this()); if(auto callback{transfer->handle->callback_transfer_aborted}; callback) @@ -633,7 +638,7 @@ void LocalFileTransfer::callback_transfer_network_write(int fd, short events, vo } logError(LOG_FT, "{} Received network write error. Send {} bytes out of {}. Closing transfer.", - transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset); + transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset); transfer->handle->report_transfer_statistics(transfer->shared_from_this()); if(auto callback{transfer->handle->callback_transfer_aborted}; callback) @@ -668,7 +673,7 @@ void LocalFileTransfer::callback_transfer_network_write(int fd, short events, vo } transfer->timings.last_write = std::chrono::system_clock::now(); - transfer->statistics.network_bytes_send += written; + transfer->statistics.network_send.increase_bytes(written); if(transfer->networking.throttle.increase_bytes(written)) break; /* we've to slow down */ @@ -678,7 +683,7 @@ void LocalFileTransfer::callback_transfer_network_write(int fd, short events, vo if(buffer_left_size > 0) transfer->add_network_write_event(false); else if(transfer->state == FileClient::STATE_DISCONNECTING) { - if(transfer->transfer && transfer->statistics.file_bytes_transferred + transfer->transfer->file_offset == transfer->transfer->expected_file_size) { + if(transfer->transfer && transfer->statistics.file_transferred.total_bytes + transfer->transfer->file_offset == transfer->transfer->expected_file_size) { logMessage(LOG_FT, "{} Finished file transfer within {}. Closing connection.", transfer->log_prefix(), duration_to_string(std::chrono::system_clock::now() - transfer->timings.key_received)); transfer->handle->report_transfer_statistics(transfer->shared_from_this()); if(auto callback{transfer->handle->callback_transfer_finished}; callback) @@ -783,6 +788,7 @@ size_t LocalFileTransfer::handle_transfer_read_raw(const std::shared_ptrlog_prefix()); break; + case TransferKeyApplyResult::INTERNAL_ERROR: default: this->report_transfer_statistics(client); if(auto callback{this->callback_transfer_aborted}; client->transfer && callback) @@ -881,6 +887,7 @@ size_t LocalFileTransfer::handle_transfer_read(const std::shared_ptr response.setHeader("x-error-message", { "unknown key" }); goto send_response_exit; + case TransferKeyApplyResult::INTERNAL_ERROR: default: this->report_transfer_statistics(client); if(auto callback{this->callback_transfer_aborted}; client->transfer && callback) @@ -1044,37 +1051,19 @@ TransferKeyApplyResult LocalFileTransfer::handle_transfer_key_provided(const std if(!client->transfer) return TransferKeyApplyResult::UNKNOWN_KEY; - { - std::string absolute_path{}; - auto transfer = client->transfer; - switch (transfer->target_type) { - case Transfer::TARGET_TYPE_AVATAR: - absolute_path = this->file_system_.absolute_avatar_path(transfer->server_id, transfer->target_file_path); - logMessage(LOG_FT, "{} Initialized avatar transfer for avatar \"{}\" ({} bytes, transferring {} bytes).", - client->log_prefix(), transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset); - break; - case Transfer::TARGET_TYPE_ICON: - absolute_path = this->file_system_.absolute_icon_path(transfer->server_id, transfer->target_file_path); - logMessage(LOG_FT, "{} Initialized icon transfer for icon \"{}\" ({} bytes, transferring {} bytes).", - client->log_prefix(), transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset); - break; - case Transfer::TARGET_TYPE_CHANNEL_FILE: - absolute_path = this->file_system_.absolute_channel_path(transfer->server_id, transfer->channel_id, transfer->target_file_path); - logMessage(LOG_FT, "{} Initialized channel transfer for file \"{}/{}\" ({} bytes, transferring {} bytes).", - client->log_prefix(), transfer->channel_id, transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset); - break; - default: - logError(LOG_FT, "{} Tried to initialize client with unknown file target type ({}). Dropping transfer.", client->log_prefix(), (int) transfer->target_type); - error_detail = "invalid transfer target type"; - return TransferKeyApplyResult::INTERNAL_ERROR; - } - debugMessage(LOG_FT, "{} Absolute file path: {}", client->log_prefix(), absolute_path); - client->file.absolute_path = absolute_path; + if(client->transfer->direction == Transfer::DIRECTION_UPLOAD) { + auto server = dynamic_pointer_cast(client->transfer->server); + assert(server); + client->networking.throttle.right = &server->upload_throttle; + } else if(client->transfer->direction == Transfer::DIRECTION_DOWNLOAD) { + auto server = dynamic_pointer_cast(client->transfer->server); + assert(server); + client->networking.throttle.right = &server->download_throttle; } if(client->transfer->max_bandwidth > 0) { - debugMessage(LOG_FT, "{} Limit network bandwidth to {} bytes/second", client->log_prefix(), client->transfer->max_bandwidth); - client->networking.throttle.set_max_bandwidth(client->transfer->max_bandwidth); + debugMessage(LOG_FT, "{} Limit network bandwidth especially for the client to {} bytes/second", client->log_prefix(), client->transfer->max_bandwidth); + client->networking.client_throttle.set_max_bandwidth(client->transfer->max_bandwidth); } client->networking.max_read_buffer_size = (size_t) -1; /* limit max bandwidth via throttle */ @@ -1102,10 +1091,10 @@ TransferKeyApplyResult LocalFileTransfer::handle_transfer_key_provided(const std } TransferUploadRawResult LocalFileTransfer::handle_transfer_upload_raw(const std::shared_ptr &client, const char *buffer, size_t length) { - client->statistics.file_bytes_transferred += length; + client->statistics.file_transferred.increase_bytes(length); bool transfer_finished{false}; - auto writte_offset = client->statistics.file_bytes_transferred + client->transfer->file_offset; + auto writte_offset = client->statistics.file_transferred.total_bytes + client->transfer->file_offset; if(writte_offset > client->transfer->expected_file_size) { logMessage(LOG_FT, "{} Client send {} too many bytes (Transfer length was {}). Dropping them, flushing the disk IO and closing the transfer.", client->log_prefix(), writte_offset - client->transfer->expected_file_size, duration_to_string(std::chrono::system_clock::now() - client->timings.key_received)); length -= writte_offset - client->transfer->expected_file_size; diff --git a/file/local_server/NetTools.cpp b/file/local_server/NetTools.cpp new file mode 100644 index 0000000..a35013b --- /dev/null +++ b/file/local_server/NetTools.cpp @@ -0,0 +1,9 @@ +// +// Created by WolverinDEV on 12/05/2020. +// + +#include "./NetTools.h" + +using namespace ts::server::file::networking; + +NetworkThrottle NetworkThrottle::kNoThrottle{-1}; \ No newline at end of file diff --git a/file/local_server/NetTools.h b/file/local_server/NetTools.h new file mode 100644 index 0000000..f1e2be9 --- /dev/null +++ b/file/local_server/NetTools.h @@ -0,0 +1,164 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace ts::server::file::networking { + struct NetworkThrottle { + constexpr static auto kThrottleTimespanMs{250}; + typedef uint8_t span_t; + + static NetworkThrottle kNoThrottle; + + ssize_t max_bytes{0}; + + span_t current_index{0}; + size_t bytes_send{0}; + + mutable spin_mutex mutex{}; + + inline bool increase_bytes(size_t bytes) { + auto current_ms = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + auto current_span = (span_t) (current_ms / kThrottleTimespanMs); + + std::lock_guard slock{this->mutex}; + if(this->current_index != current_span) { + this->current_index = current_span; + this->bytes_send = bytes; + } else { + this->bytes_send += bytes; + } + return this->max_bytes > 0 && this->bytes_send >= this->max_bytes; + } + + inline void set_max_bandwidth(ssize_t bytes_per_second) { + std::lock_guard slock{this->mutex}; + if(bytes_per_second <= 0) + this->max_bytes = -1; + else + this->max_bytes = bytes_per_second * kThrottleTimespanMs / 1000; + } + + [[nodiscard]] inline bool should_throttle(timeval& next_timestamp) { + if(this->max_bytes <= 0) return false; + + auto current_ms = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + auto current_span = (span_t) (current_ms / kThrottleTimespanMs); + + std::lock_guard slock{this->mutex}; + if(this->max_bytes <= 0) return false; /* we've to test here again, else out arithmetic will fail */ + if(this->current_index != current_span) return false; + if(this->bytes_send < this->max_bytes) return false; + + next_timestamp.tv_usec = (kThrottleTimespanMs - current_ms % kThrottleTimespanMs) * 1000; + next_timestamp.tv_sec = next_timestamp.tv_usec / 1000000; + next_timestamp.tv_usec -= next_timestamp.tv_sec * 1000000; + return true; + } + + [[nodiscard]] inline size_t bytes_left() const { + if(this->max_bytes <= 0) return (size_t) -1; + + auto current_ms = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + auto current_span = (span_t) (current_ms / kThrottleTimespanMs); + + std::lock_guard slock{this->mutex}; + if(this->max_bytes <= 0) return false; /* we've to test here again, else out arithmetic will fail */ + if(this->current_index != current_span) return this->max_bytes; + if(this->bytes_send < this->max_bytes) return this->max_bytes - this->bytes_send; + return 0; + } + + [[nodiscard]] inline std::chrono::milliseconds expected_writing_time(size_t bytes) const { + std::lock_guard slock{this->mutex}; + + if(this->max_bytes <= 0) return std::chrono::milliseconds{0}; + return std::chrono::seconds{bytes / (this->max_bytes * (1000 / kThrottleTimespanMs))}; + } + }; + + struct DualNetworkThrottle { + NetworkThrottle *left, *right; + + explicit DualNetworkThrottle(NetworkThrottle* left, NetworkThrottle* right) : left{left}, right{right} { + assert(left); + assert(right); + } + + [[nodiscard]] inline size_t bytes_left() const { + return std::min(this->left->bytes_left(), this->right->bytes_left()); + } + + [[nodiscard]] inline std::chrono::milliseconds expected_writing_time(size_t bytes) const { + return std::max(this->left->expected_writing_time(bytes), this->right->expected_writing_time(bytes)); + } + + [[nodiscard]] inline bool should_throttle(timeval& next_timestamp) const { + bool throttle = false; + timeval right_timestamp{}; + throttle |= this->left->should_throttle(next_timestamp); + throttle |= this->right->should_throttle(right_timestamp); + if(!throttle) return false; + + if(right_timestamp.tv_sec > next_timestamp.tv_sec || (right_timestamp.tv_sec == next_timestamp.tv_sec && right_timestamp.tv_usec > next_timestamp.tv_usec)) + next_timestamp = right_timestamp; + return true; + } + + inline bool increase_bytes(size_t bytes) { + bool result = false; + result |= this->left->increase_bytes(bytes); + result |= this->right->increase_bytes(bytes); + return result; + } + }; + + struct TransferStatistics { + constexpr static auto kMeasureTimespanMs{1000}; + constexpr static auto kAverageTimeCount{60}; + typedef uint8_t span_t; + + size_t total_bytes{0}; + size_t delta_bytes{0}; /* used for statistics propagation */ + + span_t span_index{0}; + size_t span_bytes{0}; + std::array history{}; + + spin_mutex mutex{}; + + inline void increase_bytes(size_t bytes) { + auto current_ms = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + auto current_span = (span_t) (current_ms / kMeasureTimespanMs); + + std::lock_guard slock{this->mutex}; + this->total_bytes += bytes; + + if(this->span_index != current_span) + this->history[this->span_index % kAverageTimeCount] = std::exchange(this->span_bytes, 0); + this->span_index = current_span; + this->span_bytes += bytes; + } + + [[nodiscard]] inline size_t take_delta() { + std::lock_guard slock{this->mutex}; + assert(this->delta_bytes <= this->total_bytes); + auto delta = this->total_bytes - this->delta_bytes; + this->delta_bytes = this->total_bytes; + return delta; + } + + [[nodiscard]] inline double current_bandwidth() const { + return (this->history[(this->span_index - 1) % kAverageTimeCount] * (double) 1000) / (double) kMeasureTimespanMs; + } + + [[nodiscard]] inline double average_bandwidth() const { + return (std::accumulate(this->history.begin(), this->history.end(), 0UL) * (double) 1000) / (double) (kMeasureTimespanMs * kAverageTimeCount); + } + }; +} \ No newline at end of file diff --git a/file/todo.txt b/file/todo.txt new file mode 100644 index 0000000..f920cb2 --- /dev/null +++ b/file/todo.txt @@ -0,0 +1 @@ +Test HTTPS file upload! \ No newline at end of file diff --git a/license/LicenseServerMain.cpp b/license/LicenseServerMain.cpp index 13b4bf9..ab3ecf8 100644 --- a/license/LicenseServerMain.cpp +++ b/license/LicenseServerMain.cpp @@ -18,7 +18,7 @@ using namespace license; /* * Requests/license: SELECT `tmp`.`keyId`, `tmp`.`key`, `tmp`.`ip`, `tmp`.`count`, `license_info`.`username`, `tmp`.`type` FROM (SELECT DISTINCT `license_request`.`keyId`, `key`, `license_request`.`ip`, COUNT(`license_request`.`keyId`) AS `count`, `license`.`type` FROM `license_request` INNER JOIN `license` ON `license`.`keyId` = `license_request`.`keyId` GROUP BY (`license_request`.`ip`)) AS `tmp` INNER JOIN `license_info` ON `license_info`.`keyId` = `tmp`.`keyId` - * Different IP's: SELECT `tmp`.`keyId`, `license_info`.`username`, COUNT(`ip`) FROM (SELECT DISTINCT `ip`, `keyId` FROM `license_request`) AS `tmp` LEFT JOIN `license_info` ON `license_info`.`keyId` = `tmp`.`keyId` GROUP BY (`tmp`.`keyId`) + * Different IP's: SELECT `tmp`.`keyId`, `license_info`.`username`, COUNT(`ip`) FROM (SELECT DISTINCT `ip`, `keyId` FROM `license_request`) AS `tmp` LEFT JOIN `license_info` ON `license_info`.`keyId` = `tmp`.`keyId` GROUP BY (`tmp`.`keyId`) ORDER BY COUNT(`ip`) DESC * * Requests (license) / ip: SELECT DISTINCT `ip`, COUNT(`ip`) FROM `license_request` WHERE `keyId` = ? GROUP BY `ip` * @@ -26,6 +26,8 @@ using namespace license; * //462 * * SELECT DISTINCT(`ip`), `license_request`.`keyId`, `license_info`.`username` FROM `license_request` LEFT JOIN `license_info` ON `license_info`.`keyId` = `license_request`.`keyId` WHERE `timestamp` > (UNIX_TIMESTAMP() - 2 * 60 * 60) * 1000 + * License too many request looking: SELECT `keyId`, `username`, COUNT(`ip`) FROM `unique_license_requests` GROUP BY `keyId` ORDER BY COUNT(`ip`) DESC + * * * * Online clients: SELECT SUM(`clients`) FROM (SELECT DISTINCT(`ip`), `clients` FROM `history_online` WHERE `timestamp` > (UNIX_TIMESTAMP() - 60 * 60 * 2) * 1000) AS `a` @@ -34,6 +36,17 @@ using namespace license; * * Empty instances: curl -ik "https://stats.teaspeak.de:27788?type=request&request_type=general" -X GET 2>&1 | grep "data:" */ +//SELECT SUM(`clients`) FROM `history_online` WHERE `keyId` = 701 OR `keyId` = 795 OR `keyId` = 792 OR `keyId` = 582 OR `keyId` = 753 OR `keyId` = 764 OR `keyId` = 761 OR `keyId` = 796 WHERE `timestamp` > (UNIX_TIMESTAMP() - 2.1 * 60 * 60) * 1000 + +/* + * Extra users: + * - ServerSponsoring + * - Davide550 + * - xDeyego? + * - latters + * - Pamonha + */ + bool handle_command(string& line); shared_ptr license_manager; diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 0a01a9a..b5cd879 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -52,6 +52,7 @@ set(SERVER_SOURCE_FILES src/client/voice/PacketStatistics.cpp src/TS3ServerClientManager.cpp src/VirtualServer.cpp + src/FileServerHandler.cpp src/TS3ServerHeartbeat.cpp src/SignalHandler.cpp src/server/VoiceServer.cpp @@ -66,11 +67,8 @@ set(SERVER_SOURCE_FILES src/client/ConnectedClientNotifyHandler.cpp src/VirtualServerManager.cpp - src/server/file/LocalFileServer.cpp src/channel/ServerChannel.cpp src/channel/ClientChannelView.cpp - src/client/file/FileClient.cpp - src/client/file/FileClientIO.cpp src/Group.cpp src/manager/BanManager.cpp src/client/InternalClient.cpp @@ -265,6 +263,7 @@ target_link_libraries(TeaSpeakServer TeaLicenseHelper #Static TeaMusic #Static CXXTerminal::static #Static + TeaSpeak-FileServer ${StringVariable_LIBRARIES_STATIC} ${YAML_CPP_LIBRARIES} pthread diff --git a/server/main.cpp b/server/main.cpp index 2b468e9..112af8c 100644 --- a/server/main.cpp +++ b/server/main.cpp @@ -10,7 +10,6 @@ #include "src/VirtualServer.h" #include "src/InstanceHandler.h" #include "src/server/QueryServer.h" -#include "src/server/file/LocalFileServer.h" #include "src/terminal/CommandHandler.h" #include "src/client/InternalClient.h" #include "src/SignalHandler.h" diff --git a/server/src/Configuration.cpp b/server/src/Configuration.cpp index bbfe4cd..4f0803c 100644 --- a/server/src/Configuration.cpp +++ b/server/src/Configuration.cpp @@ -1249,11 +1249,13 @@ std::deque> config::create_bindings() { BIND_BOOL(config::server::delete_old_bans, true); ADD_DESCRIPTION("Enable/disable the deletion of old bans within the database"); } +#if 0 { CREATE_BINDING("delete_missing_icon_permissions", 0); BIND_BOOL(config::server::delete_missing_icon_permissions, true); ADD_DESCRIPTION("Enable/disable the deletion of invalid icon id permissions"); } +#endif { CREATE_BINDING("allow_weblist", 0); BIND_BOOL(config::server::enable_teamspeak_weblist, true); diff --git a/server/src/FileServerHandler.cpp b/server/src/FileServerHandler.cpp new file mode 100644 index 0000000..3419d12 --- /dev/null +++ b/server/src/FileServerHandler.cpp @@ -0,0 +1,50 @@ +// +// Created by WolverinDEV on 12/05/2020. +// + +#include +#include +#include "FileServerHandler.h" + +using namespace ts::server::file; + +bool FileServerHandler::initialize(std::string &error) { + /* + * FIXME: Ports etc! + * + auto bindings_string = this->properties()[property::SERVERINSTANCE_FILETRANSFER_HOST].as(); + auto port = this->properties()[property::SERVERINSTANCE_FILETRANSFER_PORT].as(); + */ + /* TODO: Callback handler */ + if(!file::initialize(error)) + return false; + + auto server = file::server(); + assert(server); + + auto& transfer = server->file_transfer(); + transfer.callback_transfer_registered = std::bind(&FileServerHandler::callback_transfer_registered, this, std::placeholders::_1); + transfer.callback_transfer_started = std::bind(&FileServerHandler::callback_transfer_started, this, std::placeholders::_1); + transfer.callback_transfer_finished = std::bind(&FileServerHandler::callback_transfer_finished, this, std::placeholders::_1); + + transfer.callback_transfer_aborted = std::bind(&FileServerHandler::callback_transfer_aborted, this, std::placeholders::_1, std::placeholders::_2); + transfer.callback_transfer_statistics = std::bind(&FileServerHandler::callback_transfer_statistics, this, std::placeholders::_1, std::placeholders::_2); +} + +void FileServerHandler::callback_transfer_registered(const std::shared_ptr &transfer) { + auto server = this->instance_->getVoiceServerManager()->findServerById(transfer->server->server_id()); + if(!server) return; /* well that's bad */ + + if(transfer->direction == transfer::Transfer::DIRECTION_UPLOAD) { + server->properties()[property::VIRTUALSERVER_TOTAL_BYTES_UPLOADED] += transfer->expected_file_size - transfer->file_offset; + server->properties()[property::VIRTUALSERVER_MONTH_BYTES_UPLOADED] += transfer->expected_file_size - transfer->file_offset; + } else { + server->properties()[property::VIRTUALSERVER_TOTAL_BYTES_DOWNLOADED] += transfer->expected_file_size - transfer->file_offset; + server->properties()[property::VIRTUALSERVER_MONTH_BYTES_DOWNLOADED] += transfer->expected_file_size - transfer->file_offset; + } +} + +void FileServerHandler::callback_transfer_aborted(const std::shared_ptr &transfer, + const ts::server::file::transfer::TransferError &error) { + /* TODO: Remove not used quota from server & client */ +} \ No newline at end of file diff --git a/server/src/FileServerHandler.h b/server/src/FileServerHandler.h new file mode 100644 index 0000000..36a1ed2 --- /dev/null +++ b/server/src/FileServerHandler.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include "./InstanceHandler.h" + +namespace ts::server::file { + class FileServerHandler { + public: + explicit FileServerHandler(InstanceHandler*); + + bool initialize(std::string& /* error */); + void finalize(); + private: + InstanceHandler* instance_; + +#if 0 + std::function&)> callback_transfer_registered{}; /* transfer has been registered */ + std::function&)> callback_transfer_started{}; /* transfer has been started */ + std::function&)> callback_transfer_finished{}; /* transfer has been finished */ + std::function&, const TransferError&)> callback_transfer_aborted{}; /* an error happened while transferring the data */ + std::function&, const TransferStatistics&)> callback_transfer_statistics{}; +#endif + void callback_transfer_registered(const std::shared_ptr&); + void callback_transfer_started(const std::shared_ptr&); + void callback_transfer_finished(const std::shared_ptr&); + + void callback_transfer_aborted(const std::shared_ptr&, const transfer::TransferError&); + void callback_transfer_statistics(const std::shared_ptr&, const transfer::TransferStatistics&); + }; +} \ No newline at end of file diff --git a/server/src/Group.cpp b/server/src/Group.cpp index 7042f45..7815d82 100644 --- a/server/src/Group.cpp +++ b/server/src/Group.cpp @@ -6,7 +6,6 @@ #include "VirtualServer.h" #include "src/client/ConnectedClient.h" #include "InstanceHandler.h" -#include "src/server/file/LocalFileServer.h" using namespace std; using namespace std::chrono; @@ -216,11 +215,13 @@ int GroupManager::insertGroupFromDb(int count, char **values, char **column) { debugMessage(this->getServerId(), "Push back group -> " + to_string(group->groupId()) + " - " + group->name()); this->groups.push_back(group); +#if 0 auto iconId = (IconId) group->icon_id(); if(iconId != 0 && serverInstance && serverInstance->getFileServer() && !serverInstance->getFileServer()->iconExists(this->server.lock(), iconId)) { logMessage(this->getServerId(), "[FILE] Missing group icon (" + to_string(iconId) + ")."); if(config::server::delete_missing_icon_permissions) group->permissions()->set_permission(permission::i_icon_id, {0, 0}, permission::v2::delete_value, permission::v2::do_nothing); } +#endif return 0; } diff --git a/server/src/InstanceHandler.cpp b/server/src/InstanceHandler.cpp index ffbadf3..3bbc0f8 100644 --- a/server/src/InstanceHandler.cpp +++ b/server/src/InstanceHandler.cpp @@ -3,15 +3,13 @@ #define XFREE undefined_free #define XREALLOC undefined_realloc -#include #include "src/weblist/WebListManager.h" #include #include "InstanceHandler.h" #include "src/client/InternalClient.h" #include "src/server/QueryServer.h" -#include "src/server/file/LocalFileServer.h" -#include "SignalHandler.h" #include "src/manager/PermissionNameMapper.h" +#include "./FileServerHandler.h" #include #include "ShutdownHelper.h" #include @@ -21,13 +19,14 @@ #include #include #include -#include #include #ifndef _POSIX_SOURCE #define _POSIX_SOURCE #endif #include +#include + #undef _POSIX_SOURCE using namespace std; @@ -281,31 +280,11 @@ bool InstanceHandler::startInstance() { } this->loadWebCertificate(); - fileServer = new ts::server::LocalFileServer(); - { - auto bindings_string = this->properties()[property::SERVERINSTANCE_FILETRANSFER_HOST].as(); - auto port = this->properties()[property::SERVERINSTANCE_FILETRANSFER_PORT].as(); - auto ft_bindings = net::resolve_bindings(bindings_string, port); - deque> bindings; - for(auto& binding : ft_bindings) { - if(!get<2>(binding).empty()) { - logError(LOG_FT, "Failed to resolve binding for {}: {}", get<0>(binding), get<2>(binding)); - continue; - } - auto entry = make_shared(); - memcpy(&entry->address, &get<1>(binding), sizeof(sockaddr_storage)); - - entry->file_descriptor = -1; - entry->event_accept = nullptr; - bindings.push_back(entry); - } - - logMessage(LOG_FT, "Starting server on {}:{}", bindings_string, port); - if(!fileServer->start(bindings, errorMessage)) { - logCritical(LOG_FT, "Failed to start server: {}", errorMessage); - return false; - } + this->file_server_handler_ = new file::FileServerHandler{this}; + if(!this->file_server_handler_->initialize(errorMessage)) { + logCritical(LOG_FT, "Failed to initialize server: {}", errorMessage); + return false; } if(config::query::sslMode > 0) { @@ -443,9 +422,7 @@ void InstanceHandler::stopInstance() { debugMessage(LOG_QUERY, "Query server stopped"); debugMessage(LOG_FT, "Stopping file server"); - if (this->fileServer) this->fileServer->stop(); - delete this->fileServer; - this->fileServer = nullptr; + file::finalize(); debugMessage(LOG_FT, "File server stopped"); this->save_channel_permissions(); @@ -519,10 +496,6 @@ void InstanceHandler::tickInstance() { } } } - { - ALARM_TIMER(t, "InstanceHandler::tickInstance -> fileserver tick", milliseconds(5)); - if(this->fileServer) this->fileServer->instanceTick(); - } { ALARM_TIMER(t, "InstanceHandler::tickInstance -> sql_test tick", milliseconds(5)); if(this->sql && this->active) { diff --git a/server/src/InstanceHandler.h b/server/src/InstanceHandler.h index 6a3e54c..dbdc2b6 100644 --- a/server/src/InstanceHandler.h +++ b/server/src/InstanceHandler.h @@ -23,6 +23,10 @@ namespace ts { class LicenseService; } + namespace file { + class FileServerHandler; + } + class InstanceHandler { public: explicit InstanceHandler(SqlDataManager*); @@ -42,12 +46,12 @@ namespace ts { std::shared_mutex& getChannelTreeLock() { return this->default_tree_lock; } VirtualServerManager* getVoiceServerManager(){ return this->voiceServerManager; } - LocalFileServer* getFileServer(){ return fileServer; } QueryServer* getQueryServer(){ return queryServer; } DatabaseHelper* databaseHelper(){ return this->dbHelper; } BanManager* banManager(){ return this->banMgr; } ssl::SSLManager* sslManager(){ return this->sslMgr; } sql::SqlManager* getSql(){ return sql->sql(); } + file::FileServerHandler* getFileServerHandler() { return this->file_server_handler_; } std::chrono::time_point getStartTimestamp(){ return startTimestamp; } @@ -110,12 +114,13 @@ namespace ts { std::chrono::system_clock::time_point memcleanTimestamp; SqlDataManager* sql; - LocalFileServer* fileServer = nullptr; QueryServer* queryServer = nullptr; VirtualServerManager* voiceServerManager = nullptr; DatabaseHelper* dbHelper = nullptr; BanManager* banMgr = nullptr; ssl::SSLManager* sslMgr = nullptr; + file::FileServerHandler* file_server_handler_{nullptr}; + ts::Properties* _properties = nullptr; diff --git a/server/src/InstanceHandlerSetup.cpp b/server/src/InstanceHandlerSetup.cpp index d1f10d6..b883f36 100644 --- a/server/src/InstanceHandlerSetup.cpp +++ b/server/src/InstanceHandlerSetup.cpp @@ -6,7 +6,6 @@ #include "InstanceHandler.h" #include "src/client/InternalClient.h" #include "src/server/QueryServer.h" -#include "src/server/file/LocalFileServer.h" using namespace std; using namespace std::chrono; diff --git a/server/src/VirtualServer.cpp b/server/src/VirtualServer.cpp index 2c37f74..acbaa50 100644 --- a/server/src/VirtualServer.cpp +++ b/server/src/VirtualServer.cpp @@ -10,6 +10,8 @@ #include #include +#include + #include "weblist/WebListManager.h" #include "./client/web/WebClient.h" #include "./client/voice/VoiceClient.h" @@ -18,7 +20,6 @@ #include "./client/query/QueryClient.h" #include "music/MusicBotManager.h" #include "server/VoiceServer.h" -#include "src/server/file/LocalFileServer.h" #include "server/QueryServer.h" #include "InstanceHandler.h" #include "Configuration.h" @@ -52,10 +53,17 @@ bool VirtualServer::initialize(bool test_properties) { if(prop.type() == property::VIRTUALSERVER_DISABLE_IP_SAVING) { this->_disable_ip_saving = prop.as(); return; - } - if(prop.type() == property::VIRTUALSERVER_CODEC_ENCRYPTION_MODE) { + } else if(prop.type() == property::VIRTUALSERVER_CODEC_ENCRYPTION_MODE) { this->_voice_encryption_mode = prop.as(); return; + } else if(prop.type() == property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH) { + auto file_vs = file::server()->find_virtual_server(this->getServerId()); + if(!file_vs) return; + file_vs->max_networking_upload_bandwidth(prop.as_save([]{ return -1; })); + } else if(prop.type() == property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH) { + auto file_vs = file::server()->find_virtual_server(this->getServerId()); + if(!file_vs) return; + file_vs->max_networking_download_bandwidth(prop.as_save([]{ return -1; })); } std::string sql{}; if(prop.type() == property::VIRTUALSERVER_HOST) @@ -196,8 +204,32 @@ bool VirtualServer::initialize(bool test_properties) { this->serverAdmin->server = nullptr; this->registerInternalClient(this->serverAdmin); /* lets assign server id 0 */ - if(serverInstance->getFileServer()) - serverInstance->getFileServer()->setupServer(self.lock()); + { + using ErrorType = file::filesystem::ServerCommandErrorType; + + auto file_vs = file::server()->register_server(this->getServerId()); + auto initialize_result = file::server()->file_system().initialize_server(file_vs); + if(!initialize_result->wait_for(std::chrono::seconds{5})) { + logError(this->getServerId(), "Failed to wait for file directory initialisation."); + } else if(!initialize_result->succeeded()) { + switch (initialize_result->error().error_type) { + case ErrorType::FAILED_TO_CREATE_DIRECTORIES: + logError(this->getServerId(), "Failed to create server file directories ({}).", initialize_result->error().error_message); + break; + + case ErrorType::UNKNOWN: + case ErrorType::FAILED_TO_DELETE_DIRECTORIES: + logError(this->getServerId(), "Failed to initialize server file directory due to an unknown error: {}/{}", + (int) initialize_result->error().error_type, initialize_result->error().error_message); + break; + } + } + + + this->properties()[property::VIRTUALSERVER_FILEBASE] = file::server()->file_base_path(); + file_vs->max_networking_download_bandwidth(this->properties()[property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH].as_save([]{ return -1; })); + file_vs->max_networking_upload_bandwidth(this->properties()[property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH].as_save([]{ return -1; })); + } this->channelTree->printChannelTree([&](std::string msg){ debugMessage(this->serverId, msg); }); this->musicManager = make_shared(self.lock()); @@ -205,11 +237,13 @@ bool VirtualServer::initialize(bool test_properties) { this->musicManager->load_playlists(); this->musicManager->load_bots(); +#if 0 if(this->properties()[property::VIRTUALSERVER_ICON_ID] != (IconId) 0) if(!serverInstance->getFileServer()->iconExists(self.lock(), this->properties()[property::VIRTUALSERVER_ICON_ID])) { debugMessage(this->getServerId(), "Removing invalid icon id of server"); this->properties()[property::VIRTUALSERVER_ICON_ID] = 0; - } + } +#endif for(const auto& type : vector{ property::VIRTUALSERVER_DOWNLOAD_QUOTA, @@ -227,9 +261,6 @@ bool VirtualServer::initialize(bool test_properties) { } } - if(this->properties()[property::VIRTUALSERVER_FILEBASE].value().empty()) - this->properties()[property::VIRTUALSERVER_FILEBASE] = serverInstance->getFileServer()->server_file_base(self.lock()); - /* lets cleanup the conversations for not existent channels */ this->_conversation_manager->synchronize_channels(); return true; diff --git a/server/src/VirtualServerManager.cpp b/server/src/VirtualServerManager.cpp index 6941f9e..e436406 100644 --- a/server/src/VirtualServerManager.cpp +++ b/server/src/VirtualServerManager.cpp @@ -4,9 +4,9 @@ #include "src/server/VoiceServer.h" #include "src/client/query/QueryClient.h" #include "InstanceHandler.h" -#include "src/server/file/LocalFileServer.h" #include "src/client/ConnectedClient.h" #include +#include using namespace std; using namespace std::chrono; @@ -419,7 +419,27 @@ bool VirtualServerManager::deleteServer(shared_ptr server) { this->handle->properties()[property::SERVERINSTANCE_SPOKEN_TIME_DELETED] += server->properties()[property::VIRTUALSERVER_SPOKEN_TIME].as(); this->delete_server_in_db(server->serverId); - this->handle->getFileServer()->deleteServer(server); + + if(auto file_vs = file::server()->find_virtual_server(server->getServerId()); file_vs) { + using ErrorType = file::filesystem::ServerCommandErrorType; + + auto delete_result = file::server()->file_system().delete_server(file_vs); + if(!delete_result->wait_for(std::chrono::seconds{5})) { + logError(LOG_INSTANCE, "Failed to wait for file directory initialisation."); + } else if(!delete_result->succeeded()) { + switch (delete_result->error().error_type) { + case ErrorType::FAILED_TO_DELETE_DIRECTORIES: + logError(LOG_INSTANCE, "Failed to delete server {} file directories ({}).", server->getServerId(), delete_result->error().error_message); + break; + + case ErrorType::UNKNOWN: + case ErrorType::FAILED_TO_CREATE_DIRECTORIES: + logError(LOG_INSTANCE, "Failed to delete server {} file directory due to an unknown error: {}/{}", + server->getServerId(), (int) delete_result->error().error_type, delete_result->error().error_message); + break; + } + } + } return true; } diff --git a/server/src/channel/ServerChannel.cpp b/server/src/channel/ServerChannel.cpp index 198ddbd..0d3b72f 100644 --- a/server/src/channel/ServerChannel.cpp +++ b/server/src/channel/ServerChannel.cpp @@ -5,7 +5,6 @@ #include "misc/rnd.h" #include "src/VirtualServer.h" #include "src/client/ConnectedClient.h" -#include "src/server/file/LocalFileServer.h" #include "src/InstanceHandler.h" #include "../manager/ConversationManager.h" @@ -481,6 +480,7 @@ bool ServerChannelTree::validateChannelNames() { bool ServerChannelTree::validateChannelIcons() { for(const auto &channel : this->channels()) { auto iconId = (IconId) channel->properties()[property::CHANNEL_ICON_ID]; +#if 0 if(iconId != 0 && !serverInstance->getFileServer()->iconExists(this->server.lock(), iconId)) { logMessage(this->getServerId(), "[FILE] Missing channel icon (" + to_string(iconId) + ")."); if(config::server::delete_missing_icon_permissions) { @@ -488,6 +488,7 @@ bool ServerChannelTree::validateChannelIcons() { channel->permissions()->set_permission(permission::i_icon_id, {0, 0}, permission::v2::PermissionUpdateType::set_value, permission::v2::PermissionUpdateType::do_nothing); } } +#endif } return true; } diff --git a/server/src/client/ConnectedClient.cpp b/server/src/client/ConnectedClient.cpp index d22947f..f4d3dfb 100644 --- a/server/src/client/ConnectedClient.cpp +++ b/server/src/client/ConnectedClient.cpp @@ -11,7 +11,6 @@ #include "src/VirtualServer.h" #include "voice/VoiceClient.h" #include "../server/VoiceServer.h" -#include "src/server/file/LocalFileServer.h" #include "../InstanceHandler.h" #include "ConnectedClient.h" @@ -143,7 +142,8 @@ void ConnectedClient::updateChannelClientProperties(bool lock_channel_tree, bool iconId = value.value; } logTrace(this->getServerId(), "[CLIENT] Updating client icon from " + to_string(this->properties()[property::CLIENT_ICON_ID].as()) + " to " + to_string(iconId)); - if(this->properties()[property::CLIENT_ICON_ID].as() != iconId){ + if(this->properties()[property::CLIENT_ICON_ID].as() != iconId) { +#if 0 if(server_ref && iconId != 0) { auto dir = serverInstance->getFileServer()->iconDirectory(server_ref); if(!serverInstance->getFileServer()->iconExists(server_ref, iconId)) { @@ -151,6 +151,7 @@ void ConnectedClient::updateChannelClientProperties(bool lock_channel_tree, bool iconId = 0; } } +#endif if(this->properties()[property::CLIENT_ICON_ID].as() != iconId) { this->properties()[property::CLIENT_ICON_ID] = (IconId) iconId; notifyList.emplace_back(property::CLIENT_ICON_ID); diff --git a/server/src/client/ConnectedClient.h b/server/src/client/ConnectedClient.h index 3f25858..f97d35d 100644 --- a/server/src/client/ConnectedClient.h +++ b/server/src/client/ConnectedClient.h @@ -13,7 +13,11 @@ #define CLIENT_STR_LOG_PREFIX CLIENT_STR_LOG_PREFIX_(this) #define CMD_REQ_SERVER \ -if(!this->server) return command_result{error::server_invalid_id}; +do { \ + if(!this->server) { \ + return command_result{error::server_invalid_id}; \ + } \ +} while(0) /* TODO: Play lock the server here with read? So the client dosn't get kicked within that moment */ #define CMD_REF_SERVER(variable_name) \ @@ -33,8 +37,10 @@ if(!cmd[0].has(parm)) return command_result{error::parameter_not_found}; //the message here is show to the manager! #define CMD_CHK_AND_INC_FLOOD_POINTS(num) \ -this->increaseFloodPoints(num); \ -if(this->shouldFloodBlock()) return command_result{error::ban_flooding}; +do {\ + this->increaseFloodPoints(num); \ + if(this->shouldFloodBlock()) return command_result{error::ban_flooding}; \ +} while(0) #define CMD_CHK_PARM_COUNT(count) \ if(cmd.bulkCount() != count) return command_result{error::parameter_invalid_count}; @@ -453,10 +459,11 @@ namespace ts { command_result handleCommandFTDeleteFile(Command&); command_result handleCommandFTInitUpload(Command&); command_result handleCommandFTInitDownload(Command&); - command_result handleCommandFTGetFileInfo(Command&); - //CMD_TODO handleCommandFTGetFileInfo -> 5 points + command_result handleCommandFTGetFileInfo(Command&); + command_result handleCommandFTRenameFile(Command&); + command_result handleCommandFTList(Command&); + command_result handleCommandFTStop(Command&); //CMD_TODO handleCommandFTStop -> 5 points - //CMD_TODO handleCommandFTRenameFile -> 5 points //CMD_TODO handleCommandFTList -> 5 points command_result handleCommandBanList(Command&); diff --git a/server/src/client/ConnectedClientNotifyHandler.cpp b/server/src/client/ConnectedClientNotifyHandler.cpp index d289d5a..368ec02 100644 --- a/server/src/client/ConnectedClientNotifyHandler.cpp +++ b/server/src/client/ConnectedClientNotifyHandler.cpp @@ -3,7 +3,6 @@ #include #include "ConnectedClient.h" #include "voice/VoiceClient.h" -#include "src/server/file/LocalFileServer.h" #include "../server/VoiceServer.h" #include "../InstanceHandler.h" #include "../server/QueryServer.h" diff --git a/server/src/client/DataClient.cpp b/server/src/client/DataClient.cpp index ea9e734..59fc330 100644 --- a/server/src/client/DataClient.cpp +++ b/server/src/client/DataClient.cpp @@ -4,7 +4,6 @@ #include #include "DataClient.h" #include "ConnectedClient.h" -#include "src/server/file/LocalFileServer.h" #include "src/InstanceHandler.h" #include "misc/base64.h" @@ -115,6 +114,7 @@ bool DataClient::loadDataForCurrentServer() { //TODO for query this->clientPermissions = serverInstance->databaseHelper()->loadClientPermissionManager(ref_server, this->getClientDatabaseId()); //Setup / fix stuff +#if 0 if(!this->properties()[property::CLIENT_FLAG_AVATAR].as().empty()){ if( !ref_server || @@ -123,6 +123,7 @@ bool DataClient::loadDataForCurrentServer() { //TODO for query this->properties()[property::CLIENT_FLAG_AVATAR] = ""; } } +#endif if(ref_server) this->properties()[property::CLIENT_UNREAD_MESSAGES] = ref_server->letters->unread_letter_count(this->getUid()); diff --git a/server/src/client/command_handler/channel.cpp b/server/src/client/command_handler/channel.cpp index 3c3c89f..2177688 100644 --- a/server/src/client/command_handler/channel.cpp +++ b/server/src/client/command_handler/channel.cpp @@ -5,13 +5,11 @@ #include "../../build.h" #include "../ConnectedClient.h" #include "../InternalClient.h" -#include "src/server/file/LocalFileServer.h" #include "../../server/VoiceServer.h" #include "../voice/VoiceClient.h" #include "PermissionManager.h" #include "../../InstanceHandler.h" #include "../../server/QueryServer.h" -#include "../file/FileClient.h" #include "../music/MusicClient.h" #include "../query/QueryClient.h" #include "../../weblist/WebListManager.h" @@ -182,6 +180,7 @@ command_result ConnectedClient::handleCommandChannelGroupCopy(Command &cmd) { return permission::b_serverinstance_modify_querygroup; break; + case GroupType::GROUP_TYPE_NORMAL: default: break; } diff --git a/server/src/client/command_handler/client.cpp b/server/src/client/command_handler/client.cpp index 1881c28..064454e 100644 --- a/server/src/client/command_handler/client.cpp +++ b/server/src/client/command_handler/client.cpp @@ -6,12 +6,10 @@ #include "../../build.h" #include "../ConnectedClient.h" #include "../InternalClient.h" -#include "src/server/file/LocalFileServer.h" #include "../voice/VoiceClient.h" #include "PermissionManager.h" #include "../../InstanceHandler.h" #include "../../server/QueryServer.h" -#include "../file/FileClient.h" #include "../music/MusicClient.h" #include "../query/QueryClient.h" #include "../../weblist/WebListManager.h" @@ -349,31 +347,6 @@ command_result ConnectedClient::handleCommandClientChatClosed(Command &cmd) { return command_result{error::ok}; } -//ftgetfilelist cid=1 cpw path=\/ return_code=1:x -//Answer: -//1 .. n -// notifyfilelist cid=1 path=\/ return_code=1:x name=testFile size=35256 datetime=1509459767 type=1|name=testDir size=0 datetime=1509459741 type=0|name=testDir_2 size=0 datetime=1509459763 type=0 -//notifyfilelistfinished cid=1 path=\/ -inline void cmd_filelist_append_files(ServerId sid, Command &command, vector> files) { - int index = 0; - - logTrace(sid, "Sending file list for path {}", command["path"].string()); - for (const auto& fileEntry : files) { - logTrace(sid, " - {} ({})", fileEntry->name, fileEntry->type == file::FileType::FILE ? "file" : "directory"); - - command[index]["name"] = fileEntry->name; - command[index]["datetime"] = std::chrono::duration_cast(fileEntry->lastChanged.time_since_epoch()).count(); - command[index]["type"] = fileEntry->type; - if (fileEntry->type == file::FileType::FILE) - command[index]["size"] = static_pointer_cast(fileEntry)->fileSize; - else - command[index]["size"] = 0; - index++; - } -} - -#define CMD_REQ_FSERVER if(!serverInstance->getFileServer()) return command_result{error::vs_critical, "file server not started yet!"} - //start=0 duration=10 //pattern=%asd% diff --git a/server/src/client/command_handler/file.cpp b/server/src/client/command_handler/file.cpp index a743519..bb1b2a3 100644 --- a/server/src/client/command_handler/file.cpp +++ b/server/src/client/command_handler/file.cpp @@ -4,52 +4,41 @@ #include -#include - -#include #include #include #include #include "../../build.h" #include "../ConnectedClient.h" #include "../InternalClient.h" -#include "src/server/file/LocalFileServer.h" #include "../../server/VoiceServer.h" #include "../voice/VoiceClient.h" -#include "PermissionManager.h" #include "../../InstanceHandler.h" #include "../../server/QueryServer.h" -#include "../file/FileClient.h" #include "../music/MusicClient.h" #include "../query/QueryClient.h" #include "../../weblist/WebListManager.h" #include "../../manager/ConversationManager.h" #include "../../manager/PermissionNameMapper.h" -#include -#include -#include - #include "helpers.h" -#include +#include + #include #include #include #include -#include #include -#include #include -#include #include -namespace fs = std::experimental::filesystem; using namespace std::chrono; using namespace std; using namespace ts; using namespace ts::server; using namespace ts::token; +constexpr static auto kFileAPITimeout = std::chrono::milliseconds{500}; +constexpr static auto kMaxClientTransfers = 10; #define QUERY_PASSWORD_LENGTH 12 //ftgetfilelist cid=1 cpw path=\/ return_code=1:x @@ -57,338 +46,336 @@ using namespace ts::token; //1 .. n // notifyfilelist cid=1 path=\/ return_code=1:x name=testFile size=35256 datetime=1509459767 type=1|name=testDir size=0 datetime=1509459741 type=0|name=testDir_2 size=0 datetime=1509459763 type=0 //notifyfilelistfinished cid=1 path=\/ -inline void cmd_filelist_append_files(ServerId sid, Command &command, vector> files) { - int index = 0; - - logTrace(sid, "Sending file list for path {}", command["path"].string()); - for (const auto& fileEntry : files) { - logTrace(sid, " - {} ({})", fileEntry->name, fileEntry->type == file::FileType::FILE ? "file" : "directory"); - - command[index]["name"] = fileEntry->name; - command[index]["datetime"] = std::chrono::duration_cast(fileEntry->lastChanged.time_since_epoch()).count(); - command[index]["type"] = fileEntry->type; - if (fileEntry->type == file::FileType::FILE) - command[index]["size"] = static_pointer_cast(fileEntry)->fileSize; - else - command[index]["size"] = 0; - index++; - } -} - -#define CMD_REQ_FSERVER if(!serverInstance->getFileServer()) return command_result{error::vs_critical, "file server not started yet!"} command_result ConnectedClient::handleCommandFTGetFileList(Command &cmd) { + using directory_query_response_t = file::filesystem::AbstractProvider::directory_query_response_t; + using DirectoryEntry = file::filesystem::DirectoryEntry; + CMD_REQ_SERVER; CMD_RESET_IDLE; - CMD_REQ_FSERVER; - std::string code = cmd["return_code"].size() > 0 ? cmd["return_code"].string() : ""; + auto virtual_file_server = file::server()->find_virtual_server(this->getServerId()); + if(!virtual_file_server) return command_result{error::file_virtual_server_not_registered}; - Command fileList(this->getExternalType() == CLIENT_TEAMSPEAK ? "notifyfilelist" : ""); - Command fileListFinished("notifyfilelistfinished"); - - fileList["path"] = cmd["path"].as(); - if(!code.empty()) fileList["return_code"] = code; - fileListFinished["path"] = cmd["path"].as(); - fileList["cid"] = cmd["cid"].as(); - fileListFinished["cid"] = cmd["cid"].as(); + auto& file_system = file::server()->file_system(); + auto directory_path = cmd["path"].string(); + std::shared_ptr query_result{}; if (cmd[0].has("cid") && cmd["cid"] != 0) { //Channel auto channel = this->server->channelTree->findChannel(cmd["cid"].as()); - if (!channel) return command_result{error::channel_invalid_id, "Cant resolve channel"}; + if (!channel) + return command_result{error::channel_invalid_id}; + if (!channel->passwordMatch(cmd["cpw"]) && !permission::v2::permission_granted(1, this->calculate_permission(permission::b_ft_ignore_password, channel->channelId()))) return cmd["cpw"].string().empty() ? command_result{permission::b_ft_ignore_password} : command_result{error::channel_invalid_password}; - ACTION_REQUIRES_CHANNEL_PERMISSION(channel, permission::i_ft_needed_file_browse_power, permission::i_ft_file_browse_power, true); - cmd_filelist_append_files( - this->getServerId(), - fileList, - serverInstance->getFileServer()->listFiles(serverInstance->getFileServer()->resolveDirectory(this->server, channel, cmd["path"].as())) - ); + if(!channel->permission_granted(permission::i_ft_needed_file_browse_power, this->calculate_permission(permission::i_ft_file_browse_power, channel->channelId()), true)) + return command_result{permission::i_ft_file_browse_power}; + + query_result = file_system.query_channel_directory(virtual_file_server, cmd["cid"].as(), cmd["path"].string()); } else { - if (cmd["path"].as() == "/icons" || cmd["path"].as() == "/icons/") { - ACTION_REQUIRES_GLOBAL_PERMISSION(permission::b_icon_manage, 1); - cmd_filelist_append_files(this->getServerId(), fileList, serverInstance->getFileServer()->listFiles(serverInstance->getFileServer()->iconDirectory(this->server))); - } else if (cmd["path"].as() == "/") { - ACTION_REQUIRES_GLOBAL_PERMISSION(permission::b_icon_manage, 1); - cmd_filelist_append_files(this->getServerId(), fileList, serverInstance->getFileServer()->listFiles(serverInstance->getFileServer()->avatarDirectory(this->server))); + if (directory_path == "/icons" || directory_path == "/icons/") { + if(!permission::v2::permission_granted(1, this->calculate_permission(permission::b_icon_manage, 0))) + return command_result{permission::b_icon_manage}; + + query_result = file_system.query_icon_directory(virtual_file_server); + } else if (directory_path == "/") { + if(!permission::v2::permission_granted(1, this->calculate_permission(permission::b_icon_manage, 0))) + return command_result{permission::b_icon_manage}; + + query_result = file_system.query_avatar_directory(virtual_file_server); } else { - logMessage(this->getServerId(), "{} Requested file list for unknown path/name: path: {} name: {}", cmd["path"].string(), cmd["name"].string()); - return command_result{error::not_implemented}; + debugMessage(this->getServerId(), "{} Requested file list for unknown path/name: path: {} name: {}", cmd["path"].string(), cmd["name"].string()); + return command_result{error::parameter_invalid, "path"}; } } - if (fileList[0].has("name")) { - this->sendCommand(fileList); - if(this->getType() != CLIENT_QUERY) - this->sendCommand(fileListFinished); - return command_result{error::ok}; - } else { - return command_result{error::database_empty_result}; + if(!query_result->wait_for(kFileAPITimeout)) + return command_result{error::file_api_timeout}; + + if(!query_result->succeeded()) { + debugMessage(this->getServerId(), "{} Failed to query directory: {} / {}", CLIENT_STR_LOG_PREFIX, file::filesystem::directory_query_error_messages[(int) query_result->error().error_type], query_result->error().error_message); + using ErrorType = file::filesystem::DirectoryQueryErrorType; + switch(query_result->error().error_type) { + case ErrorType::UNKNOWN: + case ErrorType::FAILED_TO_LIST_FILES: + return command_result{error::vs_critical, query_result->error().error_message}; + + case ErrorType::PATH_IS_A_FILE: + case ErrorType::PATH_EXCEEDS_ROOT_PATH: + return command_result{error::file_invalid_path}; + + case ErrorType::PATH_DOES_NOT_EXISTS: + /* directory has not been created because there are no files */ + if(directory_path.empty() || directory_path == "/") + return command_result{error::database_empty_result}; + return command_result{error::file_not_found}; + + default: + assert(false); + return command_result{error::vs_critical}; + } } + + const auto& files = query_result->response(); + if(files.empty()) + return command_result{error::database_empty_result}; + + auto return_code = cmd["return_code"].size() > 0 ? cmd["return_code"].string() : ""; + + { + ts::command_builder notify_file_list{this->notify_response_command("notifyfilelist")}; + size_t bulk_index{0}; + for(const auto& file : files) { + if(bulk_index == 0) { + notify_file_list.reset(); + notify_file_list.put_unchecked(0, "path", cmd["path"].string()); + notify_file_list.put_unchecked(0, "cid", cmd["cid"].string()); + if(!return_code.empty()) + notify_file_list.put_unchecked(0, "return_code", return_code); + } + auto bulk = notify_file_list.bulk(bulk_index++); + + switch(file.type) { + case DirectoryEntry::DIRECTORY: + bulk.put_unchecked("type", "0"); + break; + + case DirectoryEntry::FILE: + bulk.put_unchecked("type", "1"); + break; + + case DirectoryEntry::UNKNOWN: + bulk.reset(); + bulk_index--; + continue; + } + + bulk.put_unchecked("name", file.name); + bulk.put_unchecked("size", file.size); + bulk.put_unchecked("datetime", std::chrono::duration_cast(file.modified_at.time_since_epoch()).count()); + if(bulk_index >= 16) { + this->sendCommand(notify_file_list); + bulk_index = 0; + } + } + if(bulk_index > 0) + this->sendCommand(notify_file_list); + } + + if(this->getExternalType() != ClientType::CLIENT_QUERY) { + ts::command_builder notify_file_list_finished{this->notify_response_command("notifyfilelistfinished")}; + notify_file_list_finished.put_unchecked(0, "path", cmd["path"].string()); + notify_file_list_finished.put_unchecked(0, "cid", cmd["cid"].string()); + if(!return_code.empty()) + notify_file_list_finished.put_unchecked(0, "return_code", return_code); + this->sendCommand(notify_file_list_finished); + } + return command_result{error::ok}; } //ftcreatedir cid=4 cpw dirname=\/TestDir return_code=1:17 command_result ConnectedClient::handleCommandFTCreateDir(Command &cmd) { + using ErrorType = file::filesystem::DirectoryModifyErrorType; + CMD_REQ_SERVER; CMD_CHK_AND_INC_FLOOD_POINTS(5); - CMD_REQ_FSERVER; + + auto virtual_file_server = file::server()->find_virtual_server(this->getServerId()); + if(!virtual_file_server) return command_result{error::file_virtual_server_not_registered}; + auto& file_system = file::server()->file_system(); auto channel = this->server->channelTree->findChannel(cmd["cid"].as()); - if (!channel) return command_result{error::channel_invalid_id, "Cant resolve channel"}; + if (!channel) + return command_result{error::channel_invalid_id}; + if (!channel->passwordMatch(cmd["cpw"]) && !permission::v2::permission_granted(1, this->calculate_permission(permission::b_ft_ignore_password, channel->channelId()))) return cmd["cpw"].string().empty() ? command_result{permission::b_ft_ignore_password} : command_result{error::channel_invalid_password}; - ACTION_REQUIRES_CHANNEL_PERMISSION(channel, permission::i_ft_needed_directory_create_power, permission::i_ft_directory_create_power, true); - auto dir = serverInstance->getFileServer()->createDirectory(cmd["dirname"], serverInstance->getFileServer()->resolveDirectory(this->server, channel, cmd["path"].as())); - if (!dir) return command_result{error::file_invalid_path, "could not create dir"}; + if(!channel->permission_granted(permission::i_ft_needed_directory_create_power, this->calculate_permission(permission::i_ft_directory_create_power, channel->channelId()), true)) + return command_result{permission::i_ft_directory_create_power}; + + auto create_result = file_system.create_channel_directory(virtual_file_server, channel->channelId(), cmd["dirname"].string()); + if(!create_result->wait_for(kFileAPITimeout)) + return command_result{error::file_api_timeout}; + + if(!create_result->succeeded()) { + debugMessage(this->getServerId(), "{} Failed to create channel directory: {} / {}", CLIENT_STR_LOG_PREFIX, (int) create_result->error().error_type, create_result->error().error_message); + switch(create_result->error().error_type) { + case ErrorType::UNKNOWN: + case ErrorType::FAILED_TO_CREATE_DIRECTORIES: { + auto error_detail = std::to_string((int) create_result->error().error_type); + if(!create_result->error().error_message.empty()) + error_detail += "/" + create_result->error().error_message; + return command_result{error::file_io_error, error_detail}; + } + case ErrorType::PATH_ALREADY_EXISTS: + return command_result{error::file_already_exists}; + + case ErrorType::PATH_EXCEEDS_ROOT_PATH: + return command_result{error::file_invalid_path}; + } + } return command_result{error::ok}; } command_result ConnectedClient::handleCommandFTDeleteFile(Command &cmd) { + using ErrorType = file::filesystem::FileDeleteErrorType; + CMD_REQ_SERVER; CMD_CHK_AND_INC_FLOOD_POINTS(5); - CMD_REQ_FSERVER; - std::vector> files; + auto virtual_file_server = file::server()->find_virtual_server(this->getServerId()); + if(!virtual_file_server) return command_result{error::file_virtual_server_not_registered}; + auto& file_system = file::server()->file_system(); - if (cmd[0].has("cid") && cmd["cid"] != 0) { //Channel + ts::command_result_bulk response{}; + response.emplace_result_n(cmd.bulkCount(), error::ok); + + auto file_path = cmd["path"].string(); + std::shared_ptr> delete_response{}; + if (cmd[0].has("cid") && cmd["cid"] != 0) { auto channel = this->server->channelTree->findChannel(cmd["cid"].as()); - if (!channel) return command_result{error::channel_invalid_id, "Cant resolve channel"}; + if (!channel) + return command_result{error::channel_invalid_id}; + if (!channel->passwordMatch(cmd["cpw"]) && !permission::v2::permission_granted(1, this->calculate_permission(permission::b_ft_ignore_password, channel->channelId()))) return cmd["cpw"].string().empty() ? command_result{permission::b_ft_ignore_password} : command_result{error::channel_invalid_password}; - ACTION_REQUIRES_CHANNEL_PERMISSION(channel, permission::i_ft_needed_file_delete_power, permission::i_ft_file_delete_power, true); - for (int index = 0; index < cmd.bulkCount(); index++) - files.push_back(serverInstance->getFileServer()->findFile(cmd[index]["name"].as(), serverInstance->getFileServer()->resolveDirectory(this->server, channel))); + if(!channel->permission_granted(permission::i_ft_needed_file_delete_power, this->calculate_permission(permission::i_ft_file_delete_power, channel->channelId()), true)) + return command_result{permission::i_ft_file_delete_power}; + + std::vector delete_files{}; + delete_files.reserve(cmd.bulkCount()); + for(size_t index{0}; index < cmd.bulkCount(); index++) + delete_files.push_back(cmd[index]["name"].string()); + + delete_response = file_system.delete_channel_files(virtual_file_server, channel->channelId(), delete_files); } else { - if (cmd["name"].string().find("/icon_") == 0 && cmd["path"].string().empty()) { - ACTION_REQUIRES_GLOBAL_PERMISSION(permission::b_icon_manage, 1); - files.push_back(serverInstance->getFileServer()->findFile(cmd["name"].string(), serverInstance->getFileServer()->iconDirectory(this->server))); - } else if (cmd["name"].string().find("/avatar_") == 0 && cmd["path"].string().empty()) { - if (cmd["name"].string() != "/avatar_") { - ACTION_REQUIRES_GLOBAL_PERMISSION(permission::b_client_avatar_delete_other, 1); + auto first_entry_name = cmd["name"].string(); + if (first_entry_name.find("/icon_") == 0 && file_path.empty()) { + if(!permission::v2::permission_granted(1, this->calculate_permission(permission::b_icon_manage, 0))) + return command_result{permission::b_icon_manage}; - auto uid = cmd["name"].string().substr(strlen("/avatar_")); - auto avId = hex::hex(base64::decode(uid), 'a', 'q'); - - auto cls = this->server->findClientsByUid(uid); - for (const auto &cl : cls) { - cl->properties()[property::CLIENT_FLAG_AVATAR] = ""; - this->server->notifyClientPropertyUpdates(cl, deque{property::CLIENT_FLAG_AVATAR}); + std::vector delete_files{}; + delete_files.reserve(cmd.bulkCount()); + for(size_t index{0}; index < cmd.bulkCount(); index++) { + auto file_name = cmd[index]["name"].string(); + if(!file_name.starts_with("/icon_")) { + response.set_result(index, ts::command_result{error::parameter_constraint_violation}); + continue; } - cmd["name"] = "/avatar_" + avId; - } else { - cmd["name"] = "/avatar_" + this->getAvatarId(); - this->properties()[property::CLIENT_FLAG_AVATAR] = ""; - this->server->notifyClientPropertyUpdates(_this.lock(), deque{property::CLIENT_FLAG_AVATAR}); + delete_files.push_back(file_name); } - files.push_back(serverInstance->getFileServer()->findFile(cmd["name"].string(), serverInstance->getFileServer()->avatarDirectory(this->server))); + + delete_response = file_system.delete_icons(virtual_file_server, delete_files); + } else if (first_entry_name.starts_with("/avatar_") && file_path.empty()) { + enum PermissionTestState { + SUCCEEDED, + FAILED, + UNSET + } permission_delete_other{PermissionTestState::UNSET}; + + std::vector delete_files{}; + delete_files.reserve(cmd.bulkCount()); + for(size_t index{0}; index < cmd.bulkCount(); index++) { + auto file_name = cmd[index]["name"].string(); + if(!file_name.starts_with("/avatar_")) { + response.set_result(index, ts::command_result{error::parameter_constraint_violation}); + continue; + } + + if (file_name != "/avatar_") { + if(permission_delete_other == PermissionTestState::UNSET) { + if(permission::v2::permission_granted(1, this->calculate_permission(permission::b_client_avatar_delete_other, 0))) + permission_delete_other = PermissionTestState::SUCCEEDED; + else + permission_delete_other = PermissionTestState::FAILED; + } + + if(permission_delete_other != PermissionTestState::SUCCEEDED) { + response.set_result(index, ts::command_result{permission::b_client_avatar_delete_other}); + continue; + } + + auto uid = file_name.substr(strlen("/avatar_")); + auto avId = hex::hex(base64::decode(uid), 'a', 'q'); + + auto cls = this->server->findClientsByUid(uid); + for (const auto &cl : cls) { + cl->properties()[property::CLIENT_FLAG_AVATAR] = ""; + this->server->notifyClientPropertyUpdates(cl, deque{property::CLIENT_FLAG_AVATAR}); + } + + delete_files.push_back("/avatar_" + avId); + } else { + this->properties()[property::CLIENT_FLAG_AVATAR] = ""; + this->server->notifyClientPropertyUpdates(_this.lock(), deque{property::CLIENT_FLAG_AVATAR}); + } + } + + delete_response = file_system.delete_avatars(virtual_file_server, delete_files); } else { logError(this->getServerId(), "Unknown requested file to delete: {}", cmd["path"].as()); return command_result{error::not_implemented}; } } - for (const auto &file : files) { - if (!file) continue; - if (!serverInstance->getFileServer()->deleteFile(file)) { - logCritical(this->getServerId(), "Could not delete file {}/{}", file->path, file->name); - } - } + if(!delete_response->wait_for(kFileAPITimeout)) + return command_result{error::file_api_timeout}; - return command_result{error::ok}; -} - -command_result ConnectedClient::handleCommandFTInitUpload(Command &cmd) { - CMD_REQ_SERVER; - CMD_REQ_FSERVER; - - std::shared_ptr directory = nullptr; - - if (cmd[0].has("cid") && cmd["cid"] != 0) { //Channel - auto channel = this->server->channelTree->findChannel(cmd["cid"].as()); - if (!channel) return command_result{error::channel_invalid_id, "Cant resolve channel"}; - if (!channel->passwordMatch(cmd["cpw"]) && !permission::v2::permission_granted(1, this->calculate_permission(permission::b_ft_ignore_password, channel->channelId()))) - return cmd["cpw"].string().empty() ? command_result{permission::b_ft_ignore_password} : command_result{error::channel_invalid_password}; - ACTION_REQUIRES_CHANNEL_PERMISSION(channel, permission::i_ft_needed_file_upload_power, permission::i_ft_file_upload_power, true); - directory = serverInstance->getFileServer()->resolveDirectory(this->server, channel); - } else { - if (cmd["path"].as().empty() && cmd["name"].as().find("/icon_") == 0) { - auto max_size = this->calculate_permission(permission::i_max_icon_filesize, 0); - if(max_size.has_value && !max_size.has_infinite_power() && (max_size.value < 0 || max_size.value < cmd["size"].as())) - return command_result{permission::i_max_icon_filesize}; - directory = serverInstance->getFileServer()->iconDirectory(this->server); - } else if (cmd["path"].as().empty() && cmd["name"].string() == "/avatar") { - auto max_size = this->calculate_permission(permission::i_client_max_avatar_filesize, 0); - if(max_size.has_value && !max_size.has_infinite_power() && (max_size.value < 0 || max_size.value < cmd["size"].as())) - return command_result{permission::i_client_max_avatar_filesize}; - - directory = serverInstance->getFileServer()->avatarDirectory(this->server); - cmd["name"] = "/avatar_" + this->getAvatarId(); - } else { - return command_result{error::not_implemented}; - } - } - - if (!directory || directory->type != file::FileType::DIRECTORY) { //Should not happen - return command_result{error::file_invalid_path, "could not resolve directory"}; - } - - { - auto server_quota = this->server->properties()[property::VIRTUALSERVER_UPLOAD_QUOTA].as(); - auto server_used_quota = this->server->properties()[property::VIRTUALSERVER_MONTH_BYTES_UPLOADED].as(); - server_used_quota += cmd["size"].as(); - for(const auto& trans : serverInstance->getFileServer()->pending_file_transfers()) - server_used_quota += trans->size; - for(const auto& trans : serverInstance->getFileServer()->running_file_transfers()) - server_used_quota += trans->remaining_bytes(); - if(server_quota >= 0 && server_quota * 1024 * 1024 < (int64_t) server_used_quota) return command_result{error::file_transfer_server_quota_exceeded}; - - auto client_quota = this->calculate_permission(permission::i_ft_quota_mb_upload_per_client, 0); - auto client_used_quota = this->properties()[property::CLIENT_MONTH_BYTES_UPLOADED].as(); - client_used_quota += cmd["size"].as(); - for(const auto& trans : serverInstance->getFileServer()->pending_file_transfers(_this.lock())) - client_used_quota += trans->size; - for(const auto& trans : serverInstance->getFileServer()->running_file_transfers(_this.lock())) - client_used_quota += trans->remaining_bytes(); - - if(client_quota.has_value && !client_quota.has_infinite_power() && (client_quota.value < 0 || client_quota.value * 1024 * 1024 < (int64_t) client_used_quota)) - return command_result{error::file_transfer_client_quota_exceeded}; - } - - if (cmd["overwrite"].as() && cmd["resume"].as()) return command_result{error::file_overwrite_excludes_resume}; - if (serverInstance->getFileServer()->findFile(cmd["name"].as(), directory) && !(cmd["overwrite"].as() || cmd["resume"].as())) - return command_result{error::file_already_exists, "file already exists"}; - - //Request: clientftfid=1 serverftfid=6 ftkey=itRNdsIOvcBiBg\/Xj4Ge51ZSrsShHuid port=30033 seekpos=0 - //Reqpose: notifystartupload clientftfid=4079 serverftfid=1 ftkey=aX9CFQbfaddHpOYxhQiSLu\/BumfVtPyP port=30033 seekpos=0 proto=1 - string error = "success"; - auto key = serverInstance->getFileServer()->generateUploadTransferKey(error, directory->path + "/" + directory->name + cmd["name"].string(), cmd["size"].as(), 0, _this.lock()); //TODO implement resume! - if (!key) return command_result{error::vs_critical}; - - Command result(this->getExternalType() == CLIENT_TEAMSPEAK ? "notifystartupload" : ""); - result["clientftfid"] = cmd["clientftfid"].as(); - result["ftkey"] = key->key; - - auto bindings = serverInstance->getFileServer()->list_bindings(); - if(!bindings.empty()) { - result["port"] = net::port(bindings[0]->address); - string ip = ""; - for(auto& entry : bindings) { - if(net::is_anybind(entry->address)) { - ip = ""; - break; + if(!delete_response->succeeded()) { + debugMessage(this->getServerId(), "{} Failed to create channel directory: {} / {}", CLIENT_STR_LOG_PREFIX, (int) delete_response->error().error_type, delete_response->error().error_message); + switch(delete_response->error().error_type) { + case ErrorType::UNKNOWN: { + auto error_detail = std::to_string((int) delete_response->error().error_type); + if(!delete_response->error().error_message.empty()) + error_detail += "/" + delete_response->error().error_message; + return command_result{error::vs_critical, error_detail}; } - ip += net::to_string(entry->address, false) + ","; - } - if(!ip.empty()) - result["ip"] = ip; - } else { - return command_result{error::server_is_not_running, "file server is not bound to any address"}; - } - result["seekpos"] = 0; - result["proto"] = 1; - result["serverftfid"] = key->key_id; //TODO generate! - this->sendCommand(result); - - return command_result{error::ok}; -} - -command_result ConnectedClient::handleCommandFTInitDownload(Command &cmd) { - CMD_REQ_SERVER; - CMD_REQ_FSERVER; - - std::shared_ptr directory = nullptr; - - if(!cmd[0].has("path")) cmd["path"] = ""; - - if (cmd[0].has("cid") && cmd["cid"] != (ChannelId) 0) { //Channel - auto channel = this->server->channelTree->findChannel(cmd["cid"].as()); - if (!channel) return command_result{error::channel_invalid_id, "Cant resolve channel"}; - - if (!channel->passwordMatch(cmd["cpw"]) && !permission::v2::permission_granted(1, this->calculate_permission(permission::b_ft_ignore_password, channel->channelId()))) - return cmd["cpw"].string().empty() ? command_result{permission::b_ft_ignore_password} : command_result{error::channel_invalid_password}; - ACTION_REQUIRES_CHANNEL_PERMISSION(channel, permission::i_ft_needed_file_download_power, permission::i_ft_file_download_power, true); - - directory = serverInstance->getFileServer()->resolveDirectory(this->server, channel); - } else { - if (cmd["path"].as().empty() && cmd["name"].as().find("/icon_") == 0) { - directory = serverInstance->getFileServer()->iconDirectory(this->server); - } else if (cmd["path"].as().empty() && cmd["name"].as().find("/avatar_") == 0) { //TODO fix avatar download not working - directory = serverInstance->getFileServer()->avatarDirectory(this->server); - } else { - cerr << "Unknown requested directory: " << cmd["path"].as() << endl; - return command_result{error::not_implemented}; } } - if (!directory || directory->type != file::FileType::DIRECTORY) { //Should not happen - cerr << "Invalid download file path!" << endl; - return command_result{error::file_invalid_path, "could not resolve directory"}; - } + const auto& file_status = delete_response->response(); + size_t bulk_index{0}; + for(const auto& file : file_status.delete_results) { + while(response.response(bulk_index).error_code() != error::ok) + bulk_index++; - if (!serverInstance->getFileServer()->findFile(cmd["name"].as(), directory)) - return command_result{error::file_not_found, "file not exists"}; - - string error = "success"; - auto key = serverInstance->getFileServer()->generateDownloadTransferKey(error, directory->path + "/" + directory->name + cmd["name"].as(), 0, _this.lock()); //TODO implement resume! - if (!key) return command_result{error::vs_critical, "cant generate key (" + error + ")"}; - - { - auto server_quota = this->server->properties()[property::VIRTUALSERVER_DOWNLOAD_QUOTA].as(); - auto server_used_quota = this->server->properties()[property::VIRTUALSERVER_MONTH_BYTES_DOWNLOADED].as(); - server_used_quota += key->size; - for(const auto& trans : serverInstance->getFileServer()->pending_file_transfers()) - server_used_quota += trans->size; - for(const auto& trans : serverInstance->getFileServer()->running_file_transfers()) - server_used_quota += trans->remaining_bytes(); - if(server_quota >= 0 && server_quota * 1024 * 1024 < (int64_t) server_used_quota) return command_result{error::file_transfer_server_quota_exceeded}; - - - auto client_quota = this->calculate_permission(permission::i_ft_quota_mb_download_per_client, 0); - auto client_used_quota = this->properties()[property::CLIENT_MONTH_BYTES_DOWNLOADED].as(); - client_used_quota += key->size; - for(const auto& trans : serverInstance->getFileServer()->pending_file_transfers(_this.lock())) - client_used_quota += trans->size; - for(const auto& trans : serverInstance->getFileServer()->running_file_transfers(_this.lock())) - client_used_quota += trans->remaining_bytes(); - - if(client_quota.has_value && !client_quota.has_infinite_power() && (client_quota.value < 0 || client_quota.value * 1024 * 1024 < (int64_t) client_used_quota)) - return command_result{error::file_transfer_client_quota_exceeded}; - } - - Command result(this->getExternalType() == CLIENT_TEAMSPEAK ? "notifystartdownload" : ""); - result["clientftfid"] = cmd["clientftfid"].as(); - result["proto"] = 1; - result["serverftfid"] = key->key_id; - result["ftkey"] = key->key; - - auto bindings = serverInstance->getFileServer()->list_bindings(); - if(!bindings.empty()) { - result["port"] = net::port(bindings[0]->address); - string ip = ""; - for(auto& entry : bindings) { - if(net::is_anybind(entry->address)) { - ip = ""; + using Status = file::filesystem::FileDeleteResponse::StatusType; + switch (file.status) { + case Status::SUCCESS: + /* we already emplaced success */ + break; + + case Status::PATH_EXCEEDS_ROOT_PATH: + response.set_result(bulk_index, ts::command_result{error::file_invalid_path}); + break; + + case Status::PATH_DOES_NOT_EXISTS: + response.set_result(bulk_index, ts::command_result{error::file_not_found}); + break; + + case Status::SOME_FILES_ARE_LOCKED: + response.set_result(bulk_index, ts::command_result{error::file_already_in_use, file.error_detail}); + break; + + case Status::FAILED_TO_DELETE_FILES: + response.set_result(bulk_index, ts::command_result{error::file_io_error, file.error_detail}); break; - } - ip += net::to_string(entry->address, false) + ","; } - if(!ip.empty()) - result["ip"] = ip; - } else { - return command_result{error::server_is_not_running, "file server is not bound to any address"}; + bulk_index++; } - result["size"] = key->size; - this->sendCommand(result); - - return command_result{error::ok}; + while(response.length() > bulk_index && response.response(bulk_index).type() == command_result_type::error && response.response(bulk_index).error_code() != error::ok) + bulk_index++; + assert(bulk_index == cmd.bulkCount()); + return command_result{std::move(response)}; } + /* * Usage: ftgetfileinfo cid={channelID} cpw={channelPassword} name={filePath}... @@ -408,72 +395,590 @@ Example: */ command_result ConnectedClient::handleCommandFTGetFileInfo(ts::Command &cmd) { - CMD_REQ_SERVER; - CMD_REQ_FSERVER; + using ErrorType = file::filesystem::FileInfoErrorType; + using DirectoryEntry = file::filesystem::DirectoryEntry; + CMD_RESET_IDLE; + CMD_REQ_SERVER; + CMD_CHK_AND_INC_FLOOD_POINTS(5); - Command result(this->getExternalType() == CLIENT_TEAMSPEAK ? "notifyfileinfo" : ""); + auto virtual_file_server = file::server()->find_virtual_server(this->getServerId()); + if(!virtual_file_server) return command_result{error::file_virtual_server_not_registered}; + auto& file_system = file::server()->file_system(); - int result_index = 0; - for(int index = 0; index < cmd.bulkCount(); index++) { - auto& request = cmd[index]; - std::shared_ptr file; - if (request.has("cid") && request["cid"].as() != 0) { //Channel - auto channel = this->server->channelTree->findChannel(request["cid"].as()); - if (!channel) return command_result{error::channel_invalid_id, "Cant resolve channel"}; - if (!channel->passwordMatch(cmd["cpw"]) && !permission::v2::permission_granted(1, this->calculate_permission(permission::b_ft_ignore_password, channel->channelId()))) - return cmd["cpw"].string().empty() ? command_result{permission::b_ft_ignore_password} : command_result{error::channel_invalid_password}; + ts::command_result_bulk response{}; + response.emplace_result_n(cmd.bulkCount(), error::ok); - ACTION_REQUIRES_CHANNEL_PERMISSION(channel, permission::i_ft_needed_file_browse_power, permission::i_ft_file_browse_power, true); - file = serverInstance->getFileServer()->findFile(request["name"],serverInstance->getFileServer()->resolveDirectory(this->server, channel, request["path"])); - } else { - std::shared_ptr directory; - if (!request.has("path") || request["path"].as() == "/") { - directory = serverInstance->getFileServer()->avatarDirectory(this->server); - } else if (request["path"].as() == "/icons" || request["path"].as() == "/icons/") { - directory = serverInstance->getFileServer()->iconDirectory(this->server); - } else { - cerr << "Unknown requested directory: '" << request["path"].as() << "'" << endl; - return command_result{error::not_implemented}; + auto file_path = cmd["path"].string(); + std::shared_ptr> info_response{}; + if (cmd[0].has("cid") && cmd["cid"] != 0) { + auto channel = this->server->channelTree->findChannel(cmd["cid"].as()); + if (!channel) + return command_result{error::channel_invalid_id}; + + if (!channel->passwordMatch(cmd["cpw"]) && !permission::v2::permission_granted(1, this->calculate_permission(permission::b_ft_ignore_password, channel->channelId()))) + return cmd["cpw"].string().empty() ? command_result{permission::b_ft_ignore_password} : command_result{error::channel_invalid_password}; + + if(!channel->permission_granted(permission::i_ft_needed_file_browse_power, this->calculate_permission(permission::i_ft_file_browse_power, channel->channelId()), true)) + return command_result{permission::i_ft_file_browse_power}; + + std::vector delete_files{}; + delete_files.reserve(cmd.bulkCount()); + for(size_t index{0}; index < cmd.bulkCount(); index++) + delete_files.push_back(cmd[index]["name"].string()); + + info_response = file_system.query_channel_info(virtual_file_server, channel->channelId(), delete_files); + } else { + auto first_entry_name = cmd["name"].string(); + if (first_entry_name.find("/icon_") == 0 && file_path.empty()) { + if(!permission::v2::permission_granted(1, this->calculate_permission(permission::b_icon_manage, 0))) + return command_result{permission::b_icon_manage}; + + std::vector delete_files{}; + delete_files.reserve(cmd.bulkCount()); + for(size_t index{0}; index < cmd.bulkCount(); index++) { + auto file_name = cmd[index]["name"].string(); + if(!file_name.starts_with("/icon_")) { + response.set_result(index, ts::command_result{error::parameter_constraint_violation}); + continue; + } + + delete_files.push_back(file_name); } - if(!directory) continue; - file = serverInstance->getFileServer()->findFile(cmd["name"].as(), directory); - } - if(!file) continue; + info_response = file_system.query_icon_info(virtual_file_server, delete_files); + } else if (first_entry_name.starts_with("/avatar_") && file_path.empty()) { + if(!permission::v2::permission_granted(1, this->calculate_permission(permission::b_icon_manage, 0))) + return command_result{permission::b_icon_manage}; - result[result_index]["cid"] = request["cid"].as(); - result[result_index]["name"] = request["name"]; - result[result_index]["path"] = request["path"]; - result[result_index]["type"] = file->type; - result[result_index]["datetime"] = duration_cast(file->lastChanged.time_since_epoch()).count(); - if (file->type == file::FileType::FILE) - result[result_index]["size"] = static_pointer_cast(file)->fileSize; - else - result[result_index]["size"] = 0; - result_index++; + std::vector delete_files{}; + delete_files.reserve(cmd.bulkCount()); + for(size_t index{0}; index < cmd.bulkCount(); index++) { + auto file_name = cmd[index]["name"].string(); + if(!file_name.starts_with("/avatar_")) { + response.set_result(index, ts::command_result{error::parameter_constraint_violation}); + continue; + } + + if (file_name != "/avatar_") { + auto uid = file_name.substr(strlen("/avatar_")); + auto avId = hex::hex(base64::decode(uid), 'a', 'q'); + delete_files.push_back("/avatar_" + avId); + } else { + this->properties()[property::CLIENT_FLAG_AVATAR] = ""; + this->server->notifyClientPropertyUpdates(_this.lock(), deque{property::CLIENT_FLAG_AVATAR}); + } + } + + info_response = file_system.query_avatar_info(virtual_file_server, delete_files); + } else { + logError(this->getServerId(), "Unknown requested file to query info: {}", cmd["path"].as()); + return command_result{error::parameter_invalid}; + } } - if(result_index == 0) - return command_result{error::database_empty_result}; + if(!info_response->wait_for(kFileAPITimeout)) + return command_result{error::file_api_timeout}; + + if(!info_response->succeeded()) { + debugMessage(this->getServerId(), "{} Failed to execute file info query: {} / {}", CLIENT_STR_LOG_PREFIX, (int) info_response->error().error_type, info_response->error().error_message); + switch(info_response->error().error_type) { + case ErrorType::UNKNOWN: { + auto error_detail = std::to_string((int) info_response->error().error_type); + if(!info_response->error().error_message.empty()) + error_detail += "/" + info_response->error().error_message; + return command_result{error::vs_critical, error_detail}; + } + } + } + + const auto& file_status = info_response->response(); + size_t bulk_index{0}; + + ts::command_builder notify_file_info{this->notify_response_command("notifyfileinfo")}; + size_t notify_index{0}; + for(const auto& file : file_status.file_info) { + while(response.response(bulk_index).error_code() != error::ok) + bulk_index++; + + using Status = file::filesystem::FileInfoResponse::StatusType; + switch (file.status) { + case Status::SUCCESS: + /* we already emplaced success */ + break; + + case Status::PATH_EXCEEDS_ROOT_PATH: + response.set_result(bulk_index, ts::command_result{error::file_invalid_path}); + break; + + case Status::PATH_DOES_NOT_EXISTS: + case Status::UNKNOWN_FILE_TYPE: + response.set_result(bulk_index, ts::command_result{error::file_not_found}); + break; + + case Status::FAILED_TO_QUERY_INFO: + response.set_result(bulk_index, ts::command_result{error::file_io_error, file.error_detail}); + break; + } + bulk_index++; + + auto bulk = notify_file_info.bulk(notify_index++); + switch(file.info.type) { + case DirectoryEntry::DIRECTORY: + bulk.put_unchecked("type", "0"); + break; + + case DirectoryEntry::FILE: + bulk.put_unchecked("type", "1"); + break; + + case DirectoryEntry::UNKNOWN: + bulk.reset(); + notify_index--; + continue; + } + + bulk.put_unchecked("name", file.info.name); + bulk.put_unchecked("size", file.info.size); + bulk.put_unchecked("datetime", std::chrono::duration_cast(file.info.modified_at.time_since_epoch()).count()); + } + if(notify_index > 0) + this->sendCommand(notify_file_info); + + while(response.length() > bulk_index && response.response(bulk_index).type() == command_result_type::error && response.response(bulk_index).error_code() != error::ok) + bulk_index++; + + assert(bulk_index == cmd.bulkCount()); + return command_result{std::move(response)}; +} + +/* +ftinitupload clientftfid={clientFileTransferID} name={filePath} + cid={channelID} cpw={channelPassword} size={fileSize} overwrite={1|0} + resume={1|0} [proto=0-1] + */ +command_result ConnectedClient::handleCommandFTInitUpload(ts::Command &cmd) { + CMD_REQ_SERVER; + + auto virtual_file_server = file::server()->find_virtual_server(this->getServerId()); + if(!virtual_file_server) return command_result{error::file_virtual_server_not_registered}; + + if(!cmd[0].has("path")) cmd["path"] = ""; + + file::transfer::AbstractProvider::TransferInfo info{}; + std::shared_ptr>> transfer_response{}; + + info.max_bandwidth = -1; + info.file_offset = 0; + info.expected_file_size = cmd["size"].as(); + info.override_exiting = cmd["overwrite"].as(); + info.file_path = cmd["name"].string(); + info.client_unique_id = this->getUid(); + info.client_id = this->getClientId(); + info.max_concurrent_transfers = kMaxClientTransfers; + + /* TODO: Save last file offset and resume */ + if(cmd["resume"].as() && info.override_exiting) + return ts::command_result{error::file_overwrite_excludes_resume}; + + { + auto server_quota = this->server->properties()[property::VIRTUALSERVER_UPLOAD_QUOTA].as(); + auto server_used_quota = this->server->properties()[property::VIRTUALSERVER_MONTH_BYTES_UPLOADED].as(); + server_used_quota += cmd["size"].as(); + if(server_quota >= 0 && server_quota * 1024 * 1024 < (int64_t) server_used_quota) return command_result{error::file_transfer_server_quota_exceeded}; + + auto client_quota = this->calculate_permission(permission::i_ft_quota_mb_upload_per_client, 0); + auto client_used_quota = this->properties()[property::CLIENT_MONTH_BYTES_UPLOADED].as(); + client_used_quota += cmd["size"].as(); + if(client_quota.has_value && !client_quota.has_infinite_power() && (client_quota.value < 0 || client_quota.value * 1024 * 1024 < (int64_t) client_used_quota)) + return command_result{error::file_transfer_client_quota_exceeded}; + } + + + if(cmd[0].has("cid") && cmd["cid"] != 0) { //Channel + auto channel = this->server->channelTree->findChannel(cmd["cid"].as()); + if (!channel) + return command_result{error::channel_invalid_id, "Cant resolve channel"}; + + if (!channel->passwordMatch(cmd["cpw"]) && !permission::v2::permission_granted(1, this->calculate_permission(permission::b_ft_ignore_password, channel->channelId()))) + return cmd["cpw"].string().empty() ? command_result{permission::b_ft_ignore_password} : command_result{error::channel_invalid_password}; + + ACTION_REQUIRES_CHANNEL_PERMISSION(channel, permission::i_ft_needed_file_upload_power, permission::i_ft_file_upload_power, true); + transfer_response = file::server()->file_transfer().initialize_channel_transfer(file::transfer::Transfer::DIRECTION_UPLOAD, virtual_file_server, channel->channelId(), info); + } else { + if (cmd["path"].string().empty() && cmd["name"].string().starts_with("/icon_") == 0) { + auto max_size = this->calculate_permission(permission::i_max_icon_filesize, 0); + if(max_size.has_value && !max_size.has_infinite_power() && (max_size.value < 0 || max_size.value < cmd["size"].as())) + return command_result{permission::i_max_icon_filesize}; + + transfer_response = file::server()->file_transfer().initialize_icon_transfer(file::transfer::Transfer::DIRECTION_UPLOAD, virtual_file_server, info); + } else if (cmd["path"].as().empty() && cmd["name"].string() == "/avatar") { + auto max_size = this->calculate_permission(permission::i_client_max_avatar_filesize, 0); + if(max_size.has_value && !max_size.has_infinite_power() && (max_size.value < 0 || max_size.value < cmd["size"].as())) + return command_result{permission::i_client_max_avatar_filesize}; + + info.file_path = "/avatar_" + this->getAvatarId(); + transfer_response = file::server()->file_transfer().initialize_avatar_transfer(file::transfer::Transfer::DIRECTION_UPLOAD, virtual_file_server, info); + } else { + return command_result{error::parameter_invalid, "name"}; + } + } + + if(!transfer_response->wait_for(kFileAPITimeout)) + return command_result{error::file_api_timeout}; + + if(!transfer_response->succeeded()) { + using ErrorType = file::transfer::TransferInitError; + + debugMessage(this->getServerId(), "{} Failed to initialize file upload: {} / {}", CLIENT_STR_LOG_PREFIX, (int) transfer_response->error().error_type, transfer_response->error().error_message); + switch(transfer_response->error().error_type) { + case ErrorType::UNKNOWN: { + auto error_detail = std::to_string((int) transfer_response->error().error_type); + if(!transfer_response->error().error_message.empty()) + error_detail += "/" + transfer_response->error().error_message; + return command_result{error::vs_critical, error_detail}; + } + case ErrorType::IO_ERROR: + return command_result{error::file_io_error, transfer_response->error().error_message}; + + case ErrorType::FILE_IS_NOT_A_FILE: + case ErrorType::INVALID_FILE_TYPE: + case ErrorType::FILE_DOES_NOT_EXISTS: + return command_result{error::file_not_found}; + + case ErrorType::SERVER_QUOTA_EXCEEDED: + return command_result{error::file_transfer_server_quota_exceeded}; + + case ErrorType::CLIENT_QUOTA_EXCEEDED: + return command_result{error::file_transfer_client_quota_exceeded}; + + case ErrorType::SERVER_TOO_MANY_TRANSFERS: + return command_result{error::file_server_transfer_limit_reached}; + + case ErrorType::CLIENT_TOO_MANY_TRANSFERS: + return command_result{error::file_client_transfer_limit_reached}; + } + } + + auto transfer = transfer_response->response(); + if(transfer->server_addresses.empty()) { + logError(0, "{} Received transfer without any addresses!", CLIENT_STR_LOG_PREFIX); + return ts::command_result{error::vs_critical}; + } + transfer->client_transfer_id = cmd["clientftfid"]; + + ts::command_builder result{this->notify_response_command("notifystartupload")}; + result.put_unchecked(0, "clientftfid", cmd["clientftfid"].string()); + result.put_unchecked(0, "serverftfid", transfer->server_transfer_id); + + auto used_address = transfer->server_addresses[0]; + result.put_unchecked(0, "ip", used_address.hostname); + result.put_unchecked(0, "port", used_address.port); + + result.put_unchecked(0, "ftkey", std::string_view{transfer->transfer_key, TRANSFER_KEY_LENGTH}); + result.put_unchecked(0, "seekpos", transfer->file_offset); + result.put_unchecked(0, "proto", "1"); this->sendCommand(result); return command_result{error::ok}; } +/* +ftinitdownload clientftfid={clientFileTransferID} name={filePath} + cid={channelID} cpw={channelPassword} seekpos={seekPosition} [proto=0-1] + */ +command_result ConnectedClient::handleCommandFTInitDownload(ts::Command &cmd) { + CMD_REQ_SERVER; + + auto virtual_file_server = file::server()->find_virtual_server(this->getServerId()); + if(!virtual_file_server) return command_result{error::file_virtual_server_not_registered}; + if(!cmd[0].has("path")) cmd["path"] = ""; + + file::transfer::AbstractProvider::TransferInfo info{}; + std::shared_ptr>> transfer_response{}; + + { + auto server_quota = this->server->properties()[property::VIRTUALSERVER_DOWNLOAD_QUOTA].as(); + auto server_used_quota = this->server->properties()[property::VIRTUALSERVER_MONTH_BYTES_DOWNLOADED].as(); + if(server_quota >= 0) { + if((size_t) server_quota * 1024 * 1024 <= server_used_quota) + return command_result{error::file_transfer_server_quota_exceeded}; + info.download_server_quota_limit = server_quota * 1024 * 1024 - server_used_quota; + } + + + auto client_quota = this->calculate_permission(permission::i_ft_quota_mb_download_per_client, 0); + auto client_used_quota = this->properties()[property::CLIENT_MONTH_BYTES_DOWNLOADED].as(); + if(client_quota.has_value) { + if(client_quota.value > 0) { + if((size_t) client_quota.value * 1024 * 1024 <= client_used_quota) + return command_result{error::file_transfer_server_quota_exceeded}; + info.download_client_quota_limit = client_quota.value * 1024 * 1024 - client_used_quota; + } else if(client_quota.value != -1) { + return command_result{error::file_transfer_client_quota_exceeded}; + } + } + } + + info.max_bandwidth = -1; + info.file_offset = cmd["seekpos"].as(); + info.override_exiting = false; + info.file_path = cmd["name"].string(); + info.client_unique_id = this->getUid(); + info.client_id = this->getClientId(); + info.max_concurrent_transfers = kMaxClientTransfers; + + if(cmd[0].has("cid") && cmd["cid"] != 0) { //Channel + auto channel = this->server->channelTree->findChannel(cmd["cid"].as()); + if (!channel) + return command_result{error::channel_invalid_id}; + + if (!channel->passwordMatch(cmd["cpw"]) && !permission::v2::permission_granted(1, this->calculate_permission(permission::b_ft_ignore_password, channel->channelId()))) + return cmd["cpw"].string().empty() ? command_result{permission::b_ft_ignore_password} : command_result{error::channel_invalid_password}; + + ACTION_REQUIRES_CHANNEL_PERMISSION(channel, permission::i_ft_needed_file_download_power, permission::i_ft_file_download_power, true); + transfer_response = file::server()->file_transfer().initialize_channel_transfer(file::transfer::Transfer::DIRECTION_DOWNLOAD, virtual_file_server, channel->channelId(), info); + } else { + if (cmd["path"].as().empty() && cmd["name"].string().find("/icon_") == 0) { + transfer_response = file::server()->file_transfer().initialize_icon_transfer(file::transfer::Transfer::DIRECTION_DOWNLOAD, virtual_file_server, info); + } else if (cmd["path"].as().empty() && cmd["name"].string() == "/avatar") { + info.file_path = "/avatar_" + this->getAvatarId(); + transfer_response = file::server()->file_transfer().initialize_avatar_transfer(file::transfer::Transfer::DIRECTION_DOWNLOAD, virtual_file_server, info); + } else { + return command_result{error::parameter_invalid, "name"}; + } + } + + if(!transfer_response->wait_for(kFileAPITimeout)) + return command_result{error::file_api_timeout}; + + if(!transfer_response->succeeded()) { + using ErrorType = file::transfer::TransferInitError; + + debugMessage(this->getServerId(), "{} Failed to initialize file download: {} / {}", CLIENT_STR_LOG_PREFIX, (int) transfer_response->error().error_type, transfer_response->error().error_message); + switch(transfer_response->error().error_type) { + case ErrorType::UNKNOWN: { + auto error_detail = std::to_string((int) transfer_response->error().error_type); + if(!transfer_response->error().error_message.empty()) + error_detail += "/" + transfer_response->error().error_message; + return command_result{error::vs_critical, error_detail}; + } + case ErrorType::IO_ERROR: + return command_result{error::file_io_error, transfer_response->error().error_message}; + + case ErrorType::FILE_IS_NOT_A_FILE: + case ErrorType::INVALID_FILE_TYPE: + case ErrorType::FILE_DOES_NOT_EXISTS: + return command_result{error::file_not_found}; + + case ErrorType::SERVER_QUOTA_EXCEEDED: + return command_result{error::file_transfer_server_quota_exceeded}; + + case ErrorType::CLIENT_QUOTA_EXCEEDED: + return command_result{error::file_transfer_client_quota_exceeded}; + + case ErrorType::SERVER_TOO_MANY_TRANSFERS: + return command_result{error::file_server_transfer_limit_reached}; + + case ErrorType::CLIENT_TOO_MANY_TRANSFERS: + return command_result{error::file_client_transfer_limit_reached}; + } + } + + auto transfer = transfer_response->response(); + if(transfer->server_addresses.empty()) { + logError(0, "{} Received transfer without any addresses!", CLIENT_STR_LOG_PREFIX); + return ts::command_result{error::vs_critical}; + } + transfer->client_transfer_id = cmd["clientftfid"]; + + ts::command_builder result{this->notify_response_command("notifystartdownload")}; + result.put_unchecked(0, "clientftfid", cmd["clientftfid"].string()); + result.put_unchecked(0, "serverftfid", transfer->server_transfer_id); + + auto used_address = transfer->server_addresses[0]; + result.put_unchecked(0, "ip", used_address.hostname); + result.put_unchecked(0, "port", used_address.port); + + result.put_unchecked(0, "ftkey", std::string_view{transfer->transfer_key, TRANSFER_KEY_LENGTH}); + result.put_unchecked(0, "proto", "1"); + result.put_unchecked(0, "size", transfer->expected_file_size); + + result.put_unchecked(0, "seekpos", transfer->file_offset); + this->sendCommand(result); + + return command_result{error::ok}; +} + +/* + * Usage: ftrenamefile cid={channelID} cpw={channelPassword} + [tcid={targetChannelID}] [tcpw={targetChannelPassword}] + oldname={oldFilePath} newname={newFilePath} + + i_ft_file_rename_power + i_ft_needed_file_rename_power + */ +command_result ConnectedClient::handleCommandFTRenameFile(ts::Command &cmd) { + CMD_RESET_IDLE; + CMD_REQ_SERVER; + CMD_CHK_AND_INC_FLOOD_POINTS(5); + + auto virtual_file_server = file::server()->find_virtual_server(this->getServerId()); + if(!virtual_file_server) return command_result{error::file_virtual_server_not_registered}; + + auto channel_id = cmd["cid"].as(); + auto target_channel_id = cmd[0].has("tcid") ? cmd["tcid"].as() : channel_id; + + auto channel = this->server->channelTree->findChannel(channel_id); + if (!channel) + return command_result{error::channel_invalid_id}; + + if (!channel->passwordMatch(cmd["cpw"]) && !permission::v2::permission_granted(1, this->calculate_permission(permission::b_ft_ignore_password, channel->channelId()))) + return cmd["cpw"].string().empty() ? command_result{permission::b_ft_ignore_password} : command_result{error::channel_invalid_password}; + + ACTION_REQUIRES_CHANNEL_PERMISSION(channel, permission::i_ft_needed_file_rename_power, permission::i_ft_file_rename_power, true); + + if(target_channel_id != channel_id) { + auto targetChannel = this->server->channelTree->findChannel(target_channel_id); + if (!targetChannel) + return command_result{error::channel_invalid_id}; + + if (!channel->passwordMatch(cmd["tcpw"]) && !permission::v2::permission_granted(1, this->calculate_permission(permission::b_ft_ignore_password, channel->channelId()))) + return cmd["tcpw"].string().empty() ? command_result{permission::b_ft_ignore_password} : command_result{error::channel_invalid_password}; + + ACTION_REQUIRES_CHANNEL_PERMISSION(targetChannel, permission::i_ft_needed_file_rename_power, permission::i_ft_file_rename_power, true); + } + + + auto rename_response = file::server()->file_system().rename_channel_file(virtual_file_server, channel_id, cmd["oldname"].string(), target_channel_id, cmd["newname"].string()); + if(!rename_response->wait_for(kFileAPITimeout)) + return command_result{error::file_api_timeout}; + + if(!rename_response->succeeded()) { + using ErrorType = file::filesystem::FileModifyErrorType; + + debugMessage(this->getServerId(), "{} Failed to rename file: {} / {}", CLIENT_STR_LOG_PREFIX, (int) rename_response->error().error_type, rename_response->error().error_message); + switch(rename_response->error().error_type) { + case ErrorType::UNKNOWN: + case ErrorType::FAILED_TO_RENAME_FILE: + case ErrorType::FAILED_TO_DELETE_FILES: + case ErrorType::FAILED_TO_CREATE_DIRECTORIES: { + auto error_detail = std::to_string((int) rename_response->error().error_type); + if(!rename_response->error().error_message.empty()) + error_detail += "/" + rename_response->error().error_message; + return command_result{error::vs_critical, error_detail}; + } + case ErrorType::PATH_EXCEEDS_ROOT_PATH: + case ErrorType::TARGET_PATH_EXCEEDS_ROOT_PATH: + case ErrorType::PATH_DOES_NOT_EXISTS: + return command_result{error::file_invalid_path}; + + case ErrorType::TARGET_PATH_ALREADY_EXISTS: + return command_result{error::file_already_exists}; + + case ErrorType::SOME_FILES_ARE_LOCKED: + return command_result{error::file_already_in_use, rename_response->error().error_message}; + } + } + + return command_result{error::ok}; +} + +//clid=2 path=files\/virtualserver_1\/channel_5 name=image.iso +// size=673460224 sizedone=450756 clientftfid=2 +// serverftfid=6 sender=0 status=1 current_speed=60872.8 average_speed runtime +command_result ConnectedClient::handleCommandFTList(ts::Command &cmd) { + CMD_RESET_IDLE; + CMD_REQ_SERVER; + CMD_CHK_AND_INC_FLOOD_POINTS(25); + ACTION_REQUIRES_PERMISSION(permission::b_ft_transfer_list, 1, 0); + + auto virtual_file_server = file::server()->find_virtual_server(this->getServerId()); + if(!virtual_file_server) return command_result{error::file_virtual_server_not_registered}; + + auto list_response = file::server()->file_transfer().list_transfer(); //FIXME: Only for the appropriate servers! + if(!list_response->wait_for(kFileAPITimeout)) + return command_result{error::file_api_timeout}; + + if(!list_response->succeeded()) { + using ErrorType = file::transfer::TransferListError; + + debugMessage(this->getServerId(), "{} Failed to list current transfers: {}", CLIENT_STR_LOG_PREFIX, (int) list_response->error()); + switch(list_response->error()) { + case ErrorType::UNKNOWN: { + auto error_detail = std::to_string((int) list_response->error()); + return command_result{error::vs_critical, error_detail}; + } + } + } + + + const auto& transfers = list_response->response(); + if(transfers.empty()) + return command_result{error::database_empty_result}; + + ts::command_builder notify{this->notify_response_command("notifyftlist")}; + size_t bulk_index{0}; + for(const auto& transfer : transfers) { + auto bulk = notify.bulk(bulk_index++); + + bulk.put_unchecked("clientftfid", transfer.client_transfer_id); + bulk.put_unchecked("serverftfid", transfer.server_transfer_id); + bulk.put_unchecked("sender", transfer.direction == file::transfer::Transfer::DIRECTION_DOWNLOAD); + + bulk.put_unchecked("clid", transfer.client_id); + bulk.put_unchecked("cluid", transfer.client_unique_id); + bulk.put_unchecked("path", transfer.file_path); + bulk.put_unchecked("name", transfer.file_name); + + bulk.put_unchecked("size", transfer.expected_size); + bulk.put_unchecked("sizedone", transfer.size_done); + + bulk.put_unchecked("status", (int) transfer.status); + + bulk.put_unchecked("runtime", transfer.runtime.count()); + bulk.put_unchecked("current_speed", transfer.current_speed); + bulk.put_unchecked("average_speed", transfer.average_speed); + } + this->sendCommand(notify); + + return command_result{error::ok}; +} + +//ftstop serverftfid='2' clientftfid='4096' delete='0' +command_result ConnectedClient::handleCommandFTStop(ts::Command &cmd) { + CMD_RESET_IDLE; + CMD_REQ_SERVER; + CMD_CHK_AND_INC_FLOOD_POINTS(25); + + auto virtual_file_server = file::server()->find_virtual_server(this->getServerId()); + if(!virtual_file_server) return command_result{error::file_virtual_server_not_registered}; + + auto stop_response = file::server()->file_transfer().stop_transfer(virtual_file_server, cmd["serverftfid"], false); + if(!stop_response->wait_for(kFileAPITimeout)) + return command_result{error::file_api_timeout}; + + if(!stop_response->succeeded()) { + using ErrorType = file::transfer::TransferActionError::Type; + switch (stop_response->error().error_type) { + case ErrorType::UNKNOWN_TRANSFER: + /* not known, so not stopped but it has the same result as it would have been stopped */ + return command_result{error::ok}; + + case ErrorType::UNKNOWN: { + auto error_detail = std::to_string((int) stop_response->error().error_type); + if(!stop_response->error().error_message.empty()) + error_detail += "/" + stop_response->error().error_message; + return command_result{error::vs_critical, error_detail}; + } + } + } + return command_result{error::ok}; +} + - - - - - - - - - - diff --git a/server/src/client/command_handler/helpers.h b/server/src/client/command_handler/helpers.h index 48dece6..f5e9326 100644 --- a/server/src/client/command_handler/helpers.h +++ b/server/src/client/command_handler/helpers.h @@ -79,7 +79,8 @@ if (permission::resolvePermissionData(permType)->type == permission::PermissionT } - +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" inline bool permission_require_granted_value(ts::permission::PermissionType type) { using namespace ts; /* @@ -192,6 +193,7 @@ inline bool permission_is_client_property(ts::permission::PermissionType type) { return false; } } +#pragma GCC diagnostic pop inline ssize_t count_characters(const std::string& in) { diff --git a/server/src/client/command_handler/misc.cpp b/server/src/client/command_handler/misc.cpp index d98e01e..9854766 100644 --- a/server/src/client/command_handler/misc.cpp +++ b/server/src/client/command_handler/misc.cpp @@ -13,13 +13,11 @@ #include "../../build.h" #include "../ConnectedClient.h" #include "../InternalClient.h" -#include "src/server/file/LocalFileServer.h" #include "../../server/VoiceServer.h" #include "../voice/VoiceClient.h" #include "PermissionManager.h" #include "../../InstanceHandler.h" #include "../../server/QueryServer.h" -#include "../file/FileClient.h" #include "../music/MusicClient.h" #include "../query/QueryClient.h" #include "../../weblist/WebListManager.h" @@ -136,6 +134,9 @@ command_result ConnectedClient::handleCommand(Command &cmd) { else if (command == "ftinitupload") return this->handleCommandFTInitUpload(cmd); else if (command == "ftinitdownload") return this->handleCommandFTInitDownload(cmd); else if (command == "ftgetfileinfo") return this->handleCommandFTGetFileInfo(cmd); + else if (command == "ftrenamefile") return this->handleCommandFTRenameFile(cmd); + else if (command == "ftlist") return this->handleCommandFTList(cmd); + else if (command == "ftstop") return this->handleCommandFTStop(cmd); //Banlist else if (command == "banlist") return this->handleCommandBanList(cmd); else if (command == "banadd") return this->handleCommandBanAdd(cmd); diff --git a/server/src/client/command_handler/music.cpp b/server/src/client/command_handler/music.cpp index 231b876..f2a3efc 100644 --- a/server/src/client/command_handler/music.cpp +++ b/server/src/client/command_handler/music.cpp @@ -13,13 +13,11 @@ #include "../../build.h" #include "../ConnectedClient.h" #include "../InternalClient.h" -#include "src/server/file/LocalFileServer.h" #include "../../server/VoiceServer.h" #include "../voice/VoiceClient.h" #include "PermissionManager.h" #include "../../InstanceHandler.h" #include "../../server/QueryServer.h" -#include "../file/FileClient.h" #include "../music/MusicClient.h" #include "../query/QueryClient.h" #include "../../weblist/WebListManager.h" diff --git a/server/src/client/command_handler/server.cpp b/server/src/client/command_handler/server.cpp index 7401b05..1bfe5db 100644 --- a/server/src/client/command_handler/server.cpp +++ b/server/src/client/command_handler/server.cpp @@ -10,12 +10,10 @@ #include "../../build.h" #include "../ConnectedClient.h" #include "../InternalClient.h" -#include "src/server/file/LocalFileServer.h" #include "../../server/VoiceServer.h" #include "../voice/VoiceClient.h" #include "../../InstanceHandler.h" #include "../../server/QueryServer.h" -#include "../file/FileClient.h" #include "../music/MusicClient.h" #include "../query/QueryClient.h" #include "../../weblist/WebListManager.h" diff --git a/server/src/client/file/FileClient.cpp b/server/src/client/file/FileClient.cpp deleted file mode 100644 index 4f2d8ed..0000000 --- a/server/src/client/file/FileClient.cpp +++ /dev/null @@ -1,719 +0,0 @@ -#include -#include -#include -#include "FileClient.h" -#include -#include -#include -#include -#include "src/client/ConnectedClient.h" -#include - -using namespace std; -using namespace std::chrono; -using namespace ts::server; -namespace fs = std::experimental::filesystem; - -#define BUFFER_SIZE (size_t) 2048 -FileClient::FileClient(LocalFileServer* handle, int socketFd) : handle(handle), clientFd(socketFd) { - memtrack::allocated(this); - this->last_io_action = system_clock::now(); - - int enabled = 1; - if(setsockopt(socketFd, IPPROTO_TCP, TCP_NODELAY, &enabled, sizeof enabled) < 0) - logError(LOG_FT, "{} Cant enable TCP no delay for socket {}. Error: {}/{}", this->client_prefix(), socketFd, errno, strerror(errno)); - - this->readEvent = event_new(this->handle->ioLoop, socketFd, EV_READ|EV_PERSIST, [](int a, short b, void* c){ ((FileClient*) c)->handleMessageRead(a, b, c); }, this); - this->writeEvent = event_new(this->handle->ioLoop, socketFd, EV_TIMEOUT | EV_WRITE, [](int a, short b, void* c){ ((FileClient*) c)->handleMessageWrite(a, b, c); }, this); - - - this->ssl_handler.direct_process(pipes::PROCESS_DIRECTION_OUT, true); - this->ssl_handler.direct_process(pipes::PROCESS_DIRECTION_IN, true); - this->ssl_handler.callback_write(bind(&FileClient::sendRawMessage, this, placeholders::_1)); - this->ssl_handler.callback_data(bind(&FileClient::handle_ssl_message, this, placeholders::_1)); - /* - this->ssl_handler.callback_data([&](const pipes::buffer_view& msg) { - if(this->ftType == FTType::TeaWeb_HTTPS) { - this->handle_http_message(msg); - } else if(this->ftType == FTType::TeaWeb_SSL_WS) { - this->ws_handler.process_incoming_data(msg); - } else { - logError(LOG_FT, "{} Decoded SSL packet, but transfer type isn't SSL!", this->client_prefix()); - } - }); - */ - //FIXME init ssl error handler? - - this->ws_handler.direct_process(pipes::PROCESS_DIRECTION_OUT, true); - this->ws_handler.direct_process(pipes::PROCESS_DIRECTION_IN, true); - - /* - this->ws_handler.callback_data([&](const pipes::WSMessage& message) { - if(this->state_transfere == T_INITIALIZE) { - this->applyKey(message.data.string()); //Got the key :) - } else if(this->state_transfere == T_TRANSFER) { - if(this->pendingKey->upload) - this->uploadWriteBytes(message.data.string()); - else { - logError(LOG_FT, "{} Invalid write (Just download)", this->client_prefix()); - this->disconnect(); - } - } - }); - */ - this->ws_handler.callback_data(bind(&FileClient::handle_ws_message,this, placeholders::_1)); - this->ws_handler.callback_write([&](const pipes::buffer_view& data) { - this->ssl_handler.send(data); - }); - this->ws_handler.on_connect = [&]() {}; - this->ws_handler.on_disconnect = [&](const std::string&) {}; -} - -FileClient::~FileClient() { - if(this->thread_flush.joinable()){ - this->state_connection = C_DISCONNECTED; - - if(this->thread_flush.get_id() != this_thread::get_id()) - this->thread_flush.join(); - else - this->thread_flush.detach(); - } - memtrack::freed(this); -} - -size_t FileClient::used_bandwidth() { - auto now = system_clock::now(); - - size_t tranfarred_bytes = 0; - auto timeout = now - seconds(1); - { - lock_guard lock(this->bandwidth_lock); - for(auto it = this->bandwidth.end(); it != this->bandwidth.begin();) { - --it; - - if((*it)->timestamp < timeout) { - this->bandwidth.erase(this->bandwidth.begin(), it); - break; - } - tranfarred_bytes += (*it)->length; - } - } - return tranfarred_bytes; -} - -std::string FileClient::client_prefix() { - bool hide_ip = config::server::disable_ip_saving; - if(!hide_ip) { - auto client = this->client; - if(client) { - auto server = client->getServer(); - if(server) { - hide_ip = server->disable_ip_saving(); - } - } - } - std::string address = ""; - if(hide_ip) - address = "X.X.X.X:" + to_string(net::port(this->remote_address)); - else - address = net::to_string(this->remote_address); - if(this->client) return "[" + to_string(this->client->getServerId()) + "|" + address + "| " + this->client->getDisplayName() + "]"; - return "[0|" + address + "|unconnected]"; -} - -size_t FileClient::transferred_bytes() { - return this->bytesHandled; -} - -size_t FileClient::remaining_bytes() { - threads::MutexLock lock(this->tickLock); - if(!this->pendingKey) return 0; - return this->pendingKey->size - this->bytesHandled; -} - -void FileClient::disconnect(std::chrono::milliseconds timeout) { - auto own_lock = _this.lock(); - if(!own_lock) return; - - bool apply_flush = timeout.count() > 0; - debugMessage(LOG_FT, "{} Disconnecting client. Connection state: {} Flush IO: {}", this->client_prefix(), (int) this->state_connection, apply_flush); - unique_lock tick_lock(this->tickLock); - - if(this->state_connection == C_DISCONNECTED) return; /* we're already disconnected */ - - if(apply_flush && this->state_connection == C_DISCONNECTING) return; /* we're already flushing the IO */ - - if(this->ftType == FTType::TeaWeb_SSL_WS) { - this->ws_handler.disconnect(1000, "disconnected"); - }; - - this->state_connection = apply_flush ? C_DISCONNECTING : C_DISCONNECTED; - if(this->readEvent) - event_del_noblock(this->readEvent); - - if(apply_flush){ - lock_guard flush_lock(this->thread_flush_lock); - assert(!this->thread_flush.joinable()); - - this->thread_flush = std::thread([own_lock, timeout]{ - auto beg = system_clock::now(); - while(own_lock->state_connection == C_DISCONNECTING && beg + timeout > system_clock::now()){ - { - lock_guard buffer_lock(own_lock->bufferLock); - if(own_lock->read_queue.empty() && own_lock->write_queue.empty()) break; - } - usleep(10 * 1000); - } - unique_lock l(own_lock->tickLock); - if(own_lock->state_connection != C_DISCONNECTING) return; - own_lock->disconnectFinal(l, true); - }); - { - auto native_handle = this->thread_flush.native_handle(); - pthread_setname_np(native_handle, "FileClient flush"); - } - } else { - unique_lock flush_lock(this->thread_flush_lock); - if(this->thread_flush.joinable()) { - flush_lock.unlock(); - this->thread_flush.join(); - flush_lock.lock(); - } - disconnectFinal(tick_lock, false); - } -} - -void FileClient::disconnectFinal(unique_lock& l, bool lock_flush_thread) { - auto ownLock = _this.lock(); - unique_lock l1(this->bufferLock); - - this->state_connection = C_DISCONNECTED; - { - unique_lock flush_lock(this->thread_flush_lock, try_to_lock); - if(flush_lock.owns_lock() || !lock_flush_thread) { - if(this->thread_flush.joinable()) { - if(this->thread_flush.get_id() == this_thread::get_id()) - this->thread_flush.detach(); - else - this->thread_flush.join(); - } - } - } - - if(this->readEvent){ - auto event = this->readEvent; - this->readEvent = nullptr; - l1.unlock(); - l.unlock(); - event_del_block(event); - event_free(event); - l.lock(); - l1.lock(); - } - - if(this->writeEvent){ - auto event = this->writeEvent; - this->writeEvent = nullptr; - l1.unlock(); - l.unlock(); - event_del_block(event); - event_free(event); - l.lock(); - l1.lock(); - } - - if(this->clientFd > 0) { - shutdown(this->clientFd, SHUT_RDWR); - close(this->clientFd); - } - this->clientFd = -1; - - { - threads::MutexLock l2(this->handle->clientLock); - auto& clList = this->handle->connectedClients; - auto elm = find(clList.begin(), clList.end(), _this.lock()); - if(elm != clList.end()) clList.erase(elm); - else logError(LOG_FT, "{} Invalid ft client list!", this->client_prefix()); - } - - this->read_queue.clear(); - this->write_queue.clear(); - - if(this->fstream){ - this->fstream->flush(); - this->fstream->close(); - delete this->fstream; - this->fstream = nullptr; - } - - this->pendingKey = nullptr; -} - -bool FileClient::tick() { - lock_guard l(this->tickLock); - if(this->state_connection == C_DISCONNECTED) return false; - - if(this->state_connection != C_DISCONNECTING) { - if(last_io_action.time_since_epoch().count() == 0) - last_io_action = system_clock::now(); - else if(last_io_action + minutes(1) < system_clock::now()) { - logMessage(LOG_FT, "{} Timed out after one minute of silence!", this->client_prefix()); - this->disconnect(); - return false; - } - } - - /* decode incomming stuff */ - bool flag_reexecute = false; - { - /* types which require an extra layer of decode */ - if(this->ftType == FTType::TeaWeb_SSL || this->ftType == FTType::TeaWeb_SSL_HTTP || this->ftType == FTType::TeaWeb_SSL_WS || this->ftType == FTType::TeaWeb_HTTP) { - pipes::buffer buffer; - { - lock_guard buffer_lock(this->bufferLock); - if(this->read_queue.empty()) { - flag_reexecute = false; - } else { - flag_reexecute = true; - - buffer = this->read_queue.front(); - this->read_queue.pop_front(); - } - } - - if(flag_reexecute) { - if(this->ftType == FTType::TeaWeb_HTTP) - this->handle_http_message(buffer); - else - this->ssl_handler.process_incoming_data(buffer); - } - } else if(this->ftType == FTType::TeamSpeak) { - flag_reexecute |= this->handle_ts_message(); - } else if(this->ftType == FTType::Unknown) { - /* we need at least 16 bytes to detect any type */ - if(this->availableBytes() >= 16) { - auto header = this->peekBytes(16); - if(header.find("GET") != -1 || header.find("POST") != -1 || header.find("OPTIONS") != -1) { - debugMessage(LOG_FT, "{} Using HTTP only!", this->client_prefix()); - this->ftType = FTType::TeaWeb_HTTP; - return true; - } else if(pipes::SSL::isSSLHeader(header) && serverInstance->sslManager()->web_ssl_options()) { - debugMessage(LOG_FT, "{} Encrypting pipe with SSL", this->client_prefix()); - this->ftType = FTType::TeaWeb_SSL; - this->ssl_handler.initialize(serverInstance->sslManager()->web_ssl_options()); - return true; - } else { - debugMessage(LOG_FT, "{} Transferring data with the TS protocol", this->client_prefix()); - this->ftType = FTType::TeamSpeak; - return true; - } - } - } else { - logError(LOG_FT, "{} Ticked client with unknown protocol type. Closing connection.", this->client_prefix()); - this->disconnect(); - return false; - } - } - - if(this->state_transfer == T_TRANSFER) { - if(this->pendingKey) { - /* test for download */ - if(!this->pendingKey->upload) { - /* lets send some data :) */ - - if(!fstream) { - logError(LOG_FT, "{} Missing file stream! Disconnecting client...", this->client_prefix()); - this->disconnect(); - return false; - } - size_t count = 0; - { - lock_guard lock(this->bufferLock); - count = this->write_queue.size(); - } - - if(count <= (1024 * 512) / BUFFER_SIZE){ //Max buffer 500Kb - if(!fstream->good()){ - logError(LOG_FT, "{} Cant finish file download. File isn't good anymore!", this->client_prefix()); - this->disconnect(); - return false; - } - - pipes::buffer writeBuffer(BUFFER_SIZE); - auto read = fstream->readsome((char*) writeBuffer.data_ptr(), writeBuffer.length()); - if(read < 0){ - logError(LOG_FT, "{} Invalid file read. Read {} bytes of max {}. Index {}/{}", this->client_prefix(), read, writeBuffer.length(), this->bytesHandled, this->pendingKey->size); - this->disconnect(); - return false; - } else if(read == 0){ - if(this->bytesHandled != this->pendingKey->size){ - logError(LOG_FT, "{} Invalid end of file. Expected {} bytes and attempted to read at {}", this->client_prefix(), this->pendingKey->size, this->bytesHandled); - this->disconnect(); - return false; - } - } - - this->bytesHandled += read; - this->client->getConnectionStatistics()->logFileTransferOut(read); - if(read > 0){ - writeBuffer.resize(read); - this->sendMessage(writeBuffer); - flag_reexecute |= true; - } - } - } - - - /* lets test if we're done */ - if(this->bytesHandled == this->pendingKey->size){ - auto time = duration_cast(system_clock::now() - this->connect_timestamp).count(); - logMessage(LOG_FT, "{} File transfer completed. Transferred {} bytes in {} milliseconds. Waiting for disconnect.", this->client_prefix(), this->bytesHandled, time); - - this->close_file_handle(); - this->pendingKey.reset(); - this->state_transfer = T_DONE; - this->finished_timestamp = system_clock::now(); - - /* we expect TS3 to hangup the connection */ - if(this->ftType != FTType::TeamSpeak) - this->disconnect(seconds(5)); - return false; - } - } - } else if(this->state_transfer == T_DONE) { - if(this->finished_timestamp + seconds(2) < system_clock::now() && this->state_connection == C_CONNECTED) { - debugMessage(LOG_FT, "{} Disconnecting client after 2 seconds after finish!", this->client_prefix()); - this->disconnect(seconds(5)); - } - } - - return flag_reexecute; -} - -bool FileClient::applyKey(const string &key) { - { - threads::MutexLock lock(this->handle->keylock); - for(const auto& pkey : this->handle->pendingKeys) //needs a copy - if(pkey && pkey->key == key){ - this->pendingKey = pkey; - this->handle->pendingKeys.erase(std::find(this->handle->pendingKeys.begin(), this->handle->pendingKeys.end(), pkey)); - break; - } - } - if(!this->pendingKey){ - logError(LOG_FT, "{} Tried to apply an non existing key! (Key: {})", this->client_prefix(), key); - this->disconnect(); - return false; - } - - this->client = this->pendingKey->owner.lock(); - if(!this->client) { - logError(LOG_FT, "{} Tried connect with an invalid key (client offline)! (Key: {})", this->client_prefix(), key); - this->disconnect(); - return false; - } - - debugMessage(LOG_FT, "{} Initialized file transfer for file '{}' (Type: {})", this->client_prefix(), this->pendingKey->targetFile, pendingKey->upload ? "upload" : "download"); - if(pendingKey->offset == 0 && pendingKey->upload) { - try { - fs::remove(fs::u8path(pendingKey->targetFile)); - } catch (std::exception& e) {} - } - - fstream = new std::fstream(); - fstream->open(pendingKey->targetFile, (pendingKey->upload ? ios::out : ios::in) | ios::binary | ios::app); - if(!*fstream) { - logError(LOG_FT, "{} Failed to open target file {} for {}", this->client_prefix(), this->pendingKey->targetFile, pendingKey->upload ? "upload" : "download"); - delete fstream; - this->fstream = nullptr; - return false; - } - fstream->seekg(0, std::ios::beg); - auto fsize = fstream->tellg(); - fstream->seekg(0, std::ios::end); - fsize = fstream->tellg() - fsize; - fstream->seekg(this->pendingKey->offset, std::ios::beg); - - if(!*fstream) { - logError(LOG_FT, "{} Failed to seek within file {} for {}", this->client_prefix(), this->pendingKey->targetFile, pendingKey->upload ? "upload" : "download"); - delete fstream; - this->fstream = nullptr; - return false; - } - debugMessage(LOG_FT, "{} Received local file size {}. Target size if {}", this->client_prefix(), fsize, pendingKey->size); - if(this->state_transfer == T_INITIALIZE) - this->state_transfer = T_TRANSFER; - - this->connect_timestamp = system_clock::now(); - return true; -} - - -bool FileClient::uploadWriteBytes(const pipes::buffer_view& message) { - this->client->getConnectionStatistics()->logFileTransferIn(message.length()); - if(!fstream->good()){ - logError(LOG_FT, "{} uploadWriteBytes(...) called with invalid fstream!", this->client_prefix()); - return false; - } - - fstream->write(message.data_ptr(), message.length()); - if(!fstream->good()){ - logError(LOG_FT, "{} Invalid file write! ({})", this->client_prefix(), message.length()); - this->disconnect(); - return false; - } - - this->bytesHandled += message.length(); - return true; -} - -void FileClient::close_file_handle() { - if(this->fstream) { - this->fstream->flush(); - this->fstream->close(); - delete this->fstream; - this->fstream = nullptr; - } -} - -void FileClient::sendMessage(const pipes::buffer_view& message) { - switch(this->ftType) { - case FTType::TeamSpeak: - case FTType::TeaWeb_HTTP: - this->sendRawMessage(message); - break; - case FTType::TeaWeb_SSL: - case FTType::TeaWeb_SSL_HTTP: - this->ssl_handler.send(message); - break; - case FTType::TeaWeb_SSL_WS: - this->ws_handler.send({pipes::BINARY, message.own_buffer()}); - break; - default: - /* Dont log an error because the timeout disconnect function uses this without knowing which proto */ - __asm__("nop"); - return; - } -} - -void FileClient::handle_ssl_message(const pipes::buffer_view &buffer) { - if(this->ftType == FTType::TeaWeb_SSL_HTTP) - this->handle_http_message(buffer); - else if(this->ftType == FTType::TeaWeb_SSL_WS) - this->ws_handler.process_incoming_data(buffer); - else if(this->ftType == FTType::TeaWeb_SSL) { /* lets detect if we have HTTP or WebSocket */ - this->read_buffer += buffer; - this->handle_http_header(); - } -} - -void FileClient::handle_http_header() { - auto header_end = this->read_buffer.find("\r\n\r\n"); - if(header_end == string::npos) { - if(this->read_buffer.length() > 1024 * 1024 * 4) { - this->read_buffer = pipes::buffer{}; - logMessage(LOG_FT, "{} Client tried to fillup server memory. Disconnecting client.", this->client_prefix()); - this->disconnect(); - return; - } - return; - } - - auto raw_request = this->read_buffer.view(0, header_end).string(); - http::HttpRequest request{}; - if(!http::parse_request(raw_request, request)) { - logError(LOG_FT, "{} Failed to parse HTTP request. Disconnecting client.", this->client_prefix()); - this->disconnect(); - return; - } - - auto header_upgrade = request.findHeader("Upgrade"); - if(header_upgrade && header_upgrade.values[0] == "websocket") { - debugMessage(LOG_FT, "{} Received WebSocket upgrade. Upgrading connection.", this->client_prefix(), raw_request); - if(this->ftType == FTType::TeaWeb_SSL) - this->ftType = FTType::TeaWeb_SSL_WS; - else { - //TODO: WebSocket only! - } - this->ws_handler.initialize(); - this->ws_handler.process_incoming_data(this->read_buffer); - this->read_buffer = pipes::buffer{}; - this->handle->tickFileClient(_this.lock()); /* we require a manual reticking */ - return; - } else { - /* process the http request header */ - if(this->ftType == FTType::TeaWeb_SSL) - this->ftType = FTType::TeaWeb_SSL_HTTP; - - http::HttpResponse response{}; - response.setHeader("Connection", {"close"}); /* close the connection instance, we dont want multiple requests */ - response.setHeader("Access-Control-Allow-Methods", {"GET, POST"}); - response.setHeader("Access-Control-Allow-Origin", {"*"}); - response.setHeader("Access-Control-Allow-Headers", request.findHeader("Access-Control-Request-Headers").values); //access-control-allow-headers - response.setHeader("Access-Control-Max-Age", {"86400"}); - response.setHeader("Access-Control-Expose-Headers", {"*, Content-Length, X-media-bytes, Content-Disposition"}); - - /* test for a preflight request: https://developer.mozilla.org/en-US/docs/Glossary/preflight_request */ - auto request_method = request.findHeader("Access-Control-Request-Method"); - if(request_method) { - debugMessage(LOG_FT, "{} Received preflight request. Sending close response with allow and let the client reconnect.", this->client_prefix()); - - response.code = http::code::_200; - - this->read_buffer = this->read_buffer.range(header_end + 4); - auto raw_response = response.build(); - this->sendMessage(pipes::buffer_view{raw_response.data(), raw_response.length()}); - this->disconnect(seconds(5)); /* write our response & flush */ - return; - } else { - auto transfer_key = request.findHeader("transfer-key"); - auto download_name = request.findHeader("download-name"); - - if(!transfer_key) { - response.code = http::code::code(400, "Bad Request"); - debugMessage(LOG_FT, "{} Received invalid HTTP request (Missing transfer key).", this->client_prefix()); - } else { - if(!this->applyKey(transfer_key.values[0]) || !this->pendingKey) { - response.code = http::code::code(404, "Not Found"); - debugMessage(LOG_FT, "{} Received invalid HTTP request (Invalid transfer key: {}).", this->client_prefix(), transfer_key.values[0]); - } else { - response.code = http::code::code(200, "OK"); - response.setHeader("Content-Length", {to_string(this->pendingKey->size)}); - - if(!this->pendingKey->upload) { - response.setHeader("Content-type", {"application/octet-stream; "}); - response.setHeader("Content-Transfer-Encoding", {"binary"}); - response.setHeader("Content-Disposition", {"attachment; filename=\"" + http::encode_url(download_name ? download_name.values[0] : "TeaWeb Download") + "\""}); - - if(this->pendingKey->size > 1) { - char header_buffer[64]; - auto read = fstream->readsome(header_buffer, 64); - if(read > 0) - response.setHeader("X-media-bytes", {base64::encode(header_buffer, read)}); - fstream->seekg(this->pendingKey->offset, std::ios::beg); - } - } - } - } - - if(!this->pendingKey || !this->pendingKey->upload) { - auto raw_response = response.build(); - this->sendMessage(pipes::buffer_view{raw_response.data(), raw_response.length()}); - } - if(response.code->code != 200) { - this->disconnect(seconds(5)) /* write our response & flush */; - return; - } - - this->http_init = true; - auto overhead = this->read_buffer.range(header_end + 4); - this->read_buffer = pipes::buffer{}; /* reset the read buffer */ - if(overhead.length() > 0) - this->handle_http_message(overhead); - this->handle->tickFileClient(_this.lock()); /* we require a manual reticking */ - } - } -} - -bool FileClient::handle_ts_message() { - if(this->state_transfer == T_INITIALIZE) { - if(this->availableBytes() >= 16) { - return this->applyKey(this->getBytes(16)); - } else - return false; - } else if(this->state_transfer == T_TRANSFER) { - if(!this->pendingKey) - return false; - - if(!this->pendingKey->upload) - return false; /* should never happen! */ - - bool reexecute = false; - pipes::buffer buffer; - { - lock_guard buffer_lock(this->bufferLock); - if(this->read_queue.empty()) - return false; /* nothing to upload */ - - buffer = this->read_queue.front(); - this->read_queue.pop_front(); - reexecute |= !this->read_queue.empty(); - } - - if(!this->uploadWriteBytes(buffer)) - return false; /* error already handeled by uploadWriteBytes(...) */ - - return reexecute; - } - - return false; -} - -void FileClient::handle_ws_message(const pipes::WSMessage &message) { - if(this->state_transfer == T_INITIALIZE) { - debugMessage(LOG_FT, "{} Received transfer key: {}", this->client_prefix(), message.data.string()); - if(this->applyKey(message.data.string())) - this->handle->tickFileClient(_this.lock()); /* we require a manual reticking */ - return; - } else if(this->state_transfer == T_TRANSFER) { - if(this->pendingKey->upload) - this->uploadWriteBytes(message.data); - else { - logError(LOG_FT, "{} Invalid write (Just download)", this->client_prefix()); - this->disconnect(); - } - } -} - -void FileClient::handle_http_message(const pipes::buffer_view &message) { - if(!this->http_init) { - this->read_buffer += message; - this->handle_http_header(); - return; - } - if(!https_upload_init) { - //------WebKitFormBoundaryaWP8XAzMBnMOJznv\r\nContent-Disposition: form-data; name=\"file\"; filename=\"blob\"\r\nContent-Type: application/octet-stream\r\n\r\n - this->read_buffer += message; - - auto header_end = this->read_buffer.find("\r\n\r\n"); - if(header_end == string::npos) { - if(this->read_buffer.length() > 1024 * 1024 * 4) { - this->read_buffer = pipes::buffer{}; - logMessage(LOG_FT, "{} Client tried to fillup server memory. Disconnecting client.", this->client_prefix()); - this->disconnect(); - return; - } - return; - } - - https_upload_init = true; - auto overhead = this->read_buffer.view(header_end + 4); - this->read_buffer = pipes::buffer{}; - if(!overhead.empty()) - this->handle_http_message(overhead); - return; - } - if(!this->pendingKey || !this->pendingKey->upload) { - logError(LOG_FT, "{} HTTP Invalid request", this->client_prefix()); - return; - } - - auto bytes_to_write = this->pendingKey->size - this->bytesHandled; /* ignore boundaries */ - if(bytes_to_write < message.length()) - this->uploadWriteBytes(message.view(0, bytes_to_write)); - else - this->uploadWriteBytes(message); - - if(this->bytesHandled == this->pendingKey->size){ - http::HttpResponse response{}; - response.setHeader("Connection", {"close"}); /* close the connection instance, we dont want multiple requests */ - response.setHeader("Access-Control-Allow-Methods", {"GET, POST"}); - response.setHeader("Access-Control-Allow-Origin", {"*"}); - response.setHeader("Access-Control-Allow-Headers", {"*"}); - response.setHeader("Access-Control-Max-Age", {"86400"}); - response.setHeader("Access-Control-Expose-Headers", {"*"}); - - auto raw_response = response.build(); - this->sendMessage(pipes::buffer_view{raw_response.data(), raw_response.length()}); - } -} \ No newline at end of file diff --git a/server/src/client/file/FileClient.h b/server/src/client/file/FileClient.h deleted file mode 100644 index e845d76..0000000 --- a/server/src/client/file/FileClient.h +++ /dev/null @@ -1,133 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include "src/VirtualServer.h" - -namespace ts { - namespace server { - class ConnectedClient; - - class ConnectedClient; - class LocalFileServer; - - enum FTType { - Unknown, - TeamSpeak, - - TeaWeb_SSL, /* not yet decided if the protocol is HTTP or WebSocket */ - TeaWeb_SSL_WS, - TeaWeb_SSL_HTTP, - - TeaWeb_HTTP - }; - - class FileClient { - friend class LocalFileServer; - public: - enum TransferState { - T_INITIALIZE, - T_TRANSFER, - T_DONE - }; - enum ConnectionState { - C_CONNECTED, - C_DISCONNECTING, - C_DISCONNECTED - }; - struct BandwidthEntry { - std::chrono::system_clock::time_point timestamp; - uint16_t length = 0; - }; - - FileClient(LocalFileServer* handle, int socketFd); - ~FileClient(); - - void disconnect(std::chrono::milliseconds = std::chrono::milliseconds(5000)); - FTType getFTType(){ return this->ftType; } - - size_t used_bandwidth(); - - std::string client_prefix(); - - size_t transferred_bytes(); - size_t remaining_bytes(); - protected: - void disconnectFinal(std::unique_lock& /* tick lock */, bool /* handle flush thread */); - - bool tick(); - - bool uploadWriteBytes(const pipes::buffer_view&); - void close_file_handle(); - - bool applyKey(const std::string &); - void sendMessage(const pipes::buffer_view&); - - //Direct methods & IO stuff - void sendRawMessage(const pipes::buffer_view&); - void handleMessageRead(int, short, void*); - void handleMessageWrite(int, short, void*); - - void handle_ssl_message(const pipes::buffer_view&); /* handeles all decoded SSL messages */ - - /* http header parser. header must be stored with read buffer! */ - void handle_http_header(); - - /* Final protocol handlers */ - void handle_http_message(const pipes::buffer_view&); - void handle_ws_message(const pipes::WSMessage&); - bool handle_ts_message(); - private: - LocalFileServer* handle; - std::weak_ptr _this; - - std::recursive_mutex bandwidth_lock; - std::deque> bandwidth; - - sockaddr_storage remote_address; - int clientFd; - - bool event_read_hold = false; - ::event* readEvent = nullptr; - - bool event_write_hold = false; - ::event* writeEvent = nullptr; - threads::Mutex tickLock; - - std::recursive_timed_mutex bufferLock; - std::deque write_queue; - std::deque read_queue; - pipes::buffer read_buffer; /* buffer which contains fragments of decoded data, e.g. HTTP request. Access only within tickLock locked */ - FTType ftType = FTType::Unknown; - - pipes::WebSocket ws_handler; - pipes::SSL ssl_handler; - bool https_upload_init = false; - bool http_init = false; /* general flag if the HTTP header has been parsed */ - - std::shared_ptr client; - std::shared_ptr pendingKey = nullptr; - std::chrono::time_point last_io_action; - std::chrono::time_point connect_timestamp; - std::chrono::time_point finished_timestamp; - - std::fstream* fstream = nullptr; - size_t bytesHandled = 0; - - ConnectionState state_connection = ConnectionState::C_CONNECTED; - TransferState state_transfer = TransferState::T_INITIALIZE; - - size_t availableBytes(); - std::string getBytes(size_t); - std::string peekBytes(size_t); - - std::mutex thread_flush_lock; - std::thread thread_flush; - }; - } -} \ No newline at end of file diff --git a/server/src/client/file/FileClientIO.cpp b/server/src/client/file/FileClientIO.cpp deleted file mode 100644 index 5e47a2b..0000000 --- a/server/src/client/file/FileClientIO.cpp +++ /dev/null @@ -1,248 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include "FileClient.h" - -using namespace std; -using namespace std::chrono; -using namespace ts::server; - -void FileClient::sendRawMessage(const pipes::buffer_view &message) { - lock_guard lock(this->bufferLock); - this->write_queue.push_back(message.own_buffer()); - - if(!this->event_write_hold) - event_add(this->writeEvent, nullptr); - else { - __asm__("nop"); - } -} - -void FileClient::handleMessageWrite(int fd, short, void *) { - auto self = this->_this.lock(); - if(!self) return; - - decltype(this->pendingKey) pending_key; - { - - lock_guard l(this->tickLock); - pending_key = this->pendingKey; - } - - unique_lock lock(this->bufferLock, defer_lock); - if(!lock.try_lock_for(milliseconds(5))) { - logWarning(LOG_FT, "{} Buffer lock locked, could not write data!", this->client_prefix()); - return; /* somebody else doing a action and will (hopefully) readd the event */ - } - if(self->state_connection == C_DISCONNECTED) return; - - auto used = this->used_bandwidth(); - if(pending_key) { //Delay the write <3 :P - if(pending_key->max_bandwhidth >= 0 && used > pending_key->max_bandwhidth) { - event_write_hold = true; - logTrace(LOG_FT, "{} Exceeded bandwidth limit of {} bytes (Used {} bytes). Temporary skipping write event!", this->client_prefix(), pending_key->max_bandwhidth, this->used_bandwidth()); - return; - } - } - event_write_hold = false; - - pipes::buffer* buffer = nullptr; - while(true) { - if(this->write_queue.empty()) { - lock.unlock(); /* unlock write buffer because we're ticking */ - if(pending_key && !pending_key->upload) - this->handle->tickFileClient(_this.lock()); //We have to fill up again - return; - } - - buffer = &this->write_queue.front(); - if(buffer->empty()) { - this->write_queue.pop_front(); - buffer = nullptr; - continue; - } - break; - } - - ssize_t length = buffer->length(); - if(pending_key && pending_key->max_bandwhidth >= 0) { - if(pending_key->max_bandwhidth < length && pending_key->max_bandwhidth > used) length = pending_key->max_bandwhidth - used; - } - - length = send(fd, buffer->data_ptr(), length, MSG_NOSIGNAL); - //logTrace(LOG_FT, "{} Wrote {} bytes", this->client_prefix(), length); - - if(length == -1) { - if (errno == EINTR || errno == EAGAIN) { - event_add(this->writeEvent, nullptr); - return; - } - else { - logError(LOG_FT, "{} Failed to write some data! ({}/{})", this->client_prefix(), errno, strerror(errno)); - lock.unlock(); - self->disconnect(); - return; - } - } else { - *buffer = buffer->range(length); - - auto entry = make_unique(); - entry->length = length ; - entry->timestamp = system_clock::now(); - { - lock_guard b_lock(this->bandwidth_lock); - this->bandwidth.push_back(move(entry)); - } - } - last_io_action = system_clock::now(); - - if(buffer->empty()) { - this->write_queue.pop_front(); - buffer = nullptr; - } - - if(pending_key && this->bytesHandled != pending_key->size) { - if(this->write_queue.size() < 4) this->handle->tickFileClient(_this.lock()); - } - - if(!this->write_queue.empty()) { - event_add(this->writeEvent, nullptr); - } - self.reset(); -} - -void FileClient::handleMessageRead(int fd, short, void *) { - auto self = this->_this.lock(); - if(self->state_connection != C_CONNECTED) return; - - decltype(this->pendingKey) pending_key; - { - - lock_guard l(this->tickLock); - pending_key = this->pendingKey; - } - size_t buffer_length = 1024; - if(pending_key && pending_key->max_bandwhidth >= 0) { - auto used = this->used_bandwidth(); - if(used < pending_key->max_bandwhidth) { - buffer_length = pending_key->max_bandwhidth - used; - if(buffer_length > 1024) buffer_length = 1024; - } else { - logTrace(LOG_FT, "{} Exceeded bandwidth limit of {} bytes (Used {} bytes). Temporary removing read event!", this->client_prefix(), pending_key->max_bandwhidth, used); - { - lock_guard lock(this->bufferLock); - if(this->readEvent) - event_del_noblock(this->readEvent); - } - this->event_read_hold = true; - return; - } - } - pipes::buffer buffer(buffer_length); - auto length = recv(fd, buffer.data_ptr(), buffer.length(), 0); - if(length < 0){ - if(errno == EINTR || errno == EAGAIN) - ;//event_add(this->readEvent, nullptr); - else { - { - lock_guard lock(this->bufferLock); - if(this->readEvent) - event_del_noblock(this->readEvent); - } - if(this->state_connection == C_CONNECTED) { - logError(LOG_FT, "{} Failed to read some data! ({}/{})", this->client_prefix(), errno, strerror(errno)); - self->disconnect(); - } - } - return; - } else if(length == 0){ - { - lock_guard lock(this->bufferLock); - if(this->readEvent) - event_del_noblock(this->readEvent); - } - if(this->state_connection == C_CONNECTED) { - if(this->state_transfer == T_TRANSFER) - logWarning(LOG_FT, "{} Transfer hang up. Remote peer closed the connection.", this->client_prefix()); - else - logMessage(LOG_FT, "{} Remote peer has closed the connection before initializing a transfer.", this->client_prefix()); - self->disconnect(seconds(3)); - } - return; - } - last_io_action = system_clock::now(); - - buffer.resize(length); - { - lock_guard lock(this->bufferLock); - if(self->state_connection != C_CONNECTED) return; /* drop the buffer because we're not connected anymore */ - - this->read_queue.push_back(std::move(buffer)); - } - - this->handle->tickFileClient(_this.lock()); - - auto entry = make_unique(); - entry->length = length ; - entry->timestamp = system_clock::now(); - //logTrace(LOG_FT, "{} Readed {} bytes", this->client_prefix(), length); - { - lock_guard lock(this->bandwidth_lock); - this->bandwidth.push_back(move(entry)); - } -} - - -size_t FileClient::availableBytes() { - lock_guard lock(this->bufferLock); - size_t available = 0; - for(const auto& buf : this->read_queue) - available += buf.length(); - return available; -} - -std::string FileClient::peekBytes(size_t size) { - ssize_t required = size; - lock_guard lock(this->bufferLock); - - string result; - result.reserve(size); - - for(pipes::buffer& buf : this->read_queue) { - if(required <= 0) break; - if(buf.length() > required) { - result += string(buf.data_ptr(), required); - required = 0; - } else { - result += string(buf.data_ptr(), buf.length()); - required -= buf.length(); - } - } - return result; -} - -std::string FileClient::getBytes(size_t size) { - lock_guard lock(this->bufferLock); - - string result; - result.reserve(size); - - while(!this->read_queue.empty()) { - if(size <= 0) break; - - auto& buf = this->read_queue.front(); - if(buf.length() > size) { - result += string((char*) buf.pipes::buffer_view::data_ptr(), size); - buf = buf.range(size); - size = 0; - } else { - result += string((char*) buf.data_ptr(), buf.length()); - size -= buf.length(); - this->read_queue.pop_front(); - } - } - return result; -} \ No newline at end of file diff --git a/server/src/client/music/internal_provider/channel_replay/ChannelProvider.cpp b/server/src/client/music/internal_provider/channel_replay/ChannelProvider.cpp index d0480ea..128e9bd 100644 --- a/server/src/client/music/internal_provider/channel_replay/ChannelProvider.cpp +++ b/server/src/client/music/internal_provider/channel_replay/ChannelProvider.cpp @@ -1,7 +1,6 @@ #include "ChannelProvider.h" #include "../../MusicClient.h" #include "../../../../InstanceHandler.h" -#include "src/server/file/LocalFileServer.h" #include "../../../../../../music/providers/ffmpeg/FFMpegProvider.h" @@ -22,6 +21,7 @@ threads::Future> ChannelProvider::createPl auto server = ((VirtualServer*) ptr_server)->ref(); threads::Future> future; +#if 0 if(server) { std::thread([future, server, url, ptr_server]{ auto f_server = serverInstance->getFileServer(); @@ -116,7 +116,9 @@ threads::Future> ChannelProvider::createPl } else { future.executionFailed("invalid bot"); } - +#else + future.executionFailed("channel file playback is currently not supported"); +#endif return future; } @@ -165,6 +167,7 @@ threads::Future> ChannelProvider::query_info(const std::stri auto server = ((VirtualServer*) ptr_server)->ref(); threads::Future> future; +#if 0 if(server) { std::thread([future, server, url, ptr_server]{ auto f_server = serverInstance->getFileServer(); @@ -264,6 +267,9 @@ threads::Future> ChannelProvider::query_info(const std::stri future.executionFailed("invalid bot"); } +#else + future.executionFailed("channel file playback is currently not supported"); +#endif return future; } \ No newline at end of file diff --git a/server/src/lincense/LicenseService.cpp b/server/src/lincense/LicenseService.cpp index 7d3777e..ae8ce15 100644 --- a/server/src/lincense/LicenseService.cpp +++ b/server/src/lincense/LicenseService.cpp @@ -267,6 +267,8 @@ void LicenseService::handle_client_connected() { this->send_license_validate_request(); } +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" void LicenseService::handle_message(::license::protocol::PacketType type, const void *buffer, size_t size) { switch (type) { case ::license::protocol::PACKET_SERVER_VALIDATION_RESPONSE: @@ -286,6 +288,7 @@ void LicenseService::handle_message(::license::protocol::PacketType type, const return; } } +#pragma GCC diagnostic pop void LicenseService::handle_client_disconnected(const std::string& message) { std::lock_guard rlock{this->request_lock}; diff --git a/server/src/server/file/LocalFileServer.cpp b/server/src/server/file/LocalFileServer.cpp deleted file mode 100644 index 16ca101..0000000 --- a/server/src/server/file/LocalFileServer.cpp +++ /dev/null @@ -1,777 +0,0 @@ -#include "LocalFileServer.h" -#include "src/client/file/FileClient.h" -#include "src/client/ConnectedClient.h" -#include -#include -#include -#include -#include "src/InstanceHandler.h" - -using namespace std; -using namespace std::chrono; -using namespace ts; -using namespace ts::file; -using namespace ts::server; -namespace fs = std::experimental::filesystem; - -#if defined(TCP_CORK) && !defined(TCP_NOPUSH) - #define TCP_NOPUSH TCP_CORK -#endif - -extern InstanceHandler* serverInstance; - -LocalFileServer::LocalFileServer() {} - -LocalFileServer::~LocalFileServer() { - stop(); -} - -inline fs::path buildPath(std::string rootPath, std::shared_ptr directory){ - auto strPath = directory ? directory->path + "/" + directory->name : ""; - if(strPath.find(rootPath) == 0) return fs::u8path(strPath); - return fs::u8path(rootPath + strPath); -} - -std::shared_ptr LocalFileServer::createDirectory(std::string name, std::shared_ptr parent) { - auto path = buildPath(this->rootPath, parent); - path += name; - std::error_code code{}; - if(!fs::exists(path, code)) - if(!fs::create_directories(path, code)) return nullptr; - else ; - else if(!fs::is_directory(path, code)) return nullptr; - - return static_pointer_cast(this->findFile(path.string())); -} - -bool LocalFileServer::fileExists(std::shared_ptr dir) { - std::error_code code{}; - return fs::exists(buildPath(this->rootPath, dir), code); -} - -bool LocalFileServer::fileExists(std::shared_ptr file) { - std::error_code code{}; - return fs::exists(buildPath(this->rootPath, file), code); -} - -std::shared_ptr LocalFileServer::findFile(std::string path, std::shared_ptr parent) { - if(path.find(this->rootPath) != 0) { - string strPath = (parent ? parent->path + "/" + parent->name : this->rootPath) + "/"; - if(strPath.find(this->rootPath) != 0 && rootPath != strPath) - path = this->rootPath + strPath; - if(path.find(strPath) == 0 && !strPath.empty()) - ; - else path = strPath + path; - } - - std::error_code code{}; - fs::path absPath = fs::u8path(path); - if(!fs::is_regular_file(absPath, code) && !fs::is_directory(absPath, code)){ - debugMessage(LOG_FT, "Could not find requested file. Abs path: {} | {}. (path={}, parent={})", absPath.string(), path, path, (parent ? parent->path + "/" + parent->name : "./")); - return nullptr; - } - - std::shared_ptr entry; - if(fs::is_directory(absPath, code)) - entry = std::make_shared(); - else entry = std::make_shared(); - - entry->name = absPath.filename(); - entry->type = fs::is_directory(absPath, code) ? FileType::DIRECTORY : FileType::FILE; - entry->path = absPath.parent_path().string(); - entry->lastChanged = fs::last_write_time(absPath, code); - if(entry->type == FileType::FILE) - static_pointer_cast(entry)->fileSize = entry->type == FileType::FILE ? fs::file_size(absPath, code) : 0; - return entry; -} - -std::vector> LocalFileServer::listFiles(std::shared_ptr dir) { - if(!dir) return {}; - auto directory = buildPath(this->rootPath, dir); - - std::error_code code{}; - if(!fs::exists(directory, code) || code) return {}; - if(!fs::is_directory(directory, code) || code) return {}; - - std::vector> result; - deque files; - for(const auto& elm : fs::directory_iterator(directory, code)) { - files.push_back(elm); - } - if(code) { - logWarning(LOG_FT, "Failed to iterate over directory {}: {}", directory.string(), code.message()); - return {}; - } - std::stable_sort(files.begin(), files.end(), [&](const fs::directory_entry& a, const fs::directory_entry& b) { - return fs::last_write_time(a.path(), code) > fs::last_write_time(b.path(), code); - }); - for(const auto& elm : files) { - if(fs::is_regular_file(elm.path(), code)){ - auto entry = make_shared(); - entry->name = elm.path().filename(); - entry->type = FileType::FILE; - entry->path = elm.path().parent_path().string(); - entry->lastChanged = fs::last_write_time(elm.path(), code); - entry->fileSize = fs::file_size(elm.path(), code); - result.push_back(entry); - } else if(fs::is_directory(elm.path(), code)){ - auto entry = make_shared(); - entry->name = elm.path().filename(); - entry->type = FileType::DIRECTORY; - entry->path = elm.path().parent_path().string(); - entry->lastChanged = fs::last_write_time(elm.path(), code); - result.push_back(entry); - } else { - logError(LOG_FT, "Invalid file in file tree. File path: " + elm.path().string()); - } - } - - return result; -} - -bool LocalFileServer::deleteFile(std::shared_ptr file) { - std::error_code code{}; - return fs::remove_all(fs::u8path(file->path + "/" + file->name), code) > 0 && !code; -} - -inline std::string randomString(uint length = 15, std::string charIndex = "abcdefghijklmnaoqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") -{ - std::string rs = ""; - for (uint i = 0; i < length; ++i) - rs += charIndex[rand() % charIndex.length()]; - return rs; -} - -std::shared_ptr LocalFileServer::generateDownloadTransferKey(std::string &errorMessage, std::string targetFile, size_t offset, const shared_ptr& client) { - auto file = this->findFile(targetFile, nullptr); - if(!file){ - errorMessage = "file does not exists"; - return nullptr; - } - if(file->type != FileType::FILE) { - errorMessage = "file is a directory"; - return nullptr; - } - - auto result = make_shared(); - result->owner = client; - result->server = client->getServerId(); - - result->key_id = (uint16_t) (rand() & 0xFFFF); - result->createTimestamp = std::chrono::system_clock::now(); - result->key = randomString(16); - result->offset = offset; - result->size = static_pointer_cast(file)->fileSize; - result->upload = false; - result->targetFile = targetFile; - - { //Test for server? - auto bandwidth_server = client->getServer()->properties()[property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH].as(); - auto bandwidth_instance = serverInstance->properties()[property::SERVERINSTANCE_MAX_UPLOAD_TOTAL_BANDWIDTH].as(); - - result->max_server_bandwidth = bandwidth_server; - result->max_bandwhidth = bandwidth_server; - - if(bandwidth_instance != -1) { - if(bandwidth_instance < bandwidth_server) - result->max_bandwhidth = bandwidth_instance; //We want to limit that already until exact calculations was made - } - } - - if(client->getServer()) - result->max_server_bandwidth = client->getServer()->properties()[property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH]; - - { - threads::MutexLock lock(this->keylock); - pendingKeys.push_back(result); - } - return result; -} - -std::shared_ptr LocalFileServer::generateUploadTransferKey(std::string &errorMessage, std::string targetFile, size_t size, size_t offset, const shared_ptr& client) { - shared_ptr result = make_shared(); - result->owner = client; - result->server = client->getServerId(); - - result->key_id = (uint16_t) (rand() & 0xFFFF); - result->createTimestamp = std::chrono::system_clock::now(); - result->key = randomString(16); - result->offset = offset; - result->size = size; - result->upload = true; - result->targetFile = targetFile; - - { //Test for server? - auto bandwidth_server = client->getServer()->properties()[property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH].as(); - auto bandwidth_instance = serverInstance->properties()[property::SERVERINSTANCE_MAX_DOWNLOAD_TOTAL_BANDWIDTH].as(); - - result->max_server_bandwidth = bandwidth_server; - result->max_bandwhidth = bandwidth_server; - - if(bandwidth_instance != -1) { - if(bandwidth_instance < bandwidth_server) - result->max_bandwhidth = bandwidth_instance; //We want to limit that already until exact calculations was made - } - } - { - threads::MutexLock lock(this->keylock); - pendingKeys.push_back(result); - } - debugMessage(LOG_FT, "Created file upload key=" + result->key + " for " + targetFile + " (" + to_string(size) + " bytes)"); - return result; -} - -std::shared_ptr LocalFileServer::resolveDirectory(const shared_ptr &server, std::shared_ptr channel, std::string subPath) { - fs::path path = fs::u8path("server_" + to_string(server ? server->getServerId() : 0) + "/channel_" + to_string(channel->channelId())); - if(!findFile(path)) - this->createDirectory(path.string(), nullptr); - path += subPath; - - auto ffile = findFile(path.string()); - debugMessage(LOG_FT, "Resolve {} => {} -> {}", path.string(), (void*) ffile.get(), typeid(ffile).name()); - return static_pointer_cast(ffile); -} - -std::shared_ptr LocalFileServer::iconDirectory(const shared_ptr &server) { - fs::path root = fs::u8path(this->rootPath); - fs::path path = fs::u8path("server_" + to_string(server ? server->getServerId() : 0) + "/icons"); - std::error_code code{}; - if(!fs::exists(root / path, code)) { - if(!fs::create_directories(root / path, code) || code) - return nullptr; - } - return static_pointer_cast(findFile(path.string())); -} - -bool LocalFileServer::iconExists(const shared_ptr &server, IconId icon) { - if(icon == 0) return false; - if(icon < 1000) return true; - return this->findFile("icon_" + to_string(icon), this->iconDirectory(server)) != nullptr; -} - -std::shared_ptr LocalFileServer::avatarDirectory(const shared_ptr &server) { - fs::path path = fs::u8path("server_" + to_string(server ? server->getServerId() : 0) + "/avatars"); - if(!findFile(path)) - this->createDirectory(path.string(), nullptr); - return static_pointer_cast(findFile(path.string())); -} - -#define TS3_ICON_HASH "icon_1001" -void LocalFileServer::setupServer(const shared_ptr &server) { - auto dir = iconDirectory(server); - if(!dir) { - logError(LOG_FT,"Failed to find icon directory for server {}", server ? server->getServerId() : 0); - } else { - std::error_code code{}; - if(!fs::exists(fs::u8path(dir->path + "/" + dir->name + "/" TS3_ICON_HASH), code)) { - try { - fs::copy(fs::u8path("resources/teaspeak16px.png"), fs::u8path(dir->path + "/" + dir->name + "/" + TS3_ICON_HASH), code); - } catch(std::exception& ex) { - logError(LOG_FT, "Failed to copy default path: {}", ex.what()); - } - } - } -} - -std::string LocalFileServer::server_file_base(const std::shared_ptr &server) { - return rootPath + "/server_" + to_string(server->getServerId()); -} - -void LocalFileServer::deleteServer(const shared_ptr &server) { - fs::path path = fs::u8path(rootPath + "/server_" + to_string(server ? server->getServerId() : 0)); - std::error_code code{}; - if(fs::exists(path, code) && !code) { - if(fs::remove_all(path, code) == 0) - logError(LOG_FT, "Could not delete server directory {} ({} | {})", path.string(), code.value(), code.message()); - } else { - logError(LOG_FT, "Could not delete missing server directory (" + path.string() + ")"); - } -} - -//The actual server! -bool LocalFileServer::start(const std::deque>& bindings, std::string& error) { - if(this->running()) { - error = "server already running"; - return false; - } - this->active = true; - - /* reserve backup file descriptor in case that the max file descriptors have been reached */ - { - this->server_reserve_fd = dup(1); - if(this->server_reserve_fd < 0) - logWarning(LOG_FT, "Failed to reserve a backup accept file descriptor. ({} | {})", errno, strerror(errno)); - } - - /* setup event bases */ - { - this->ioLoop = event_base_new(); - this->ioThread = new threads::Thread(THREAD_SAVE_OPERATIONS | THREAD_EXECUTE_LATER, [&]{ - while(this->active) { - debugMessage(LOG_FT, "Entering event loop ({})", (void*) this->ioLoop); - event_base_loop(this->ioLoop, EVLOOP_NO_EXIT_ON_EMPTY); - if(this->active) { - debugMessage(LOG_FT, "Event loop exited ({}). No active events. Sleeping 1 seconds", (void*) this->ioLoop); - this_thread::sleep_for(seconds(1)); - } else { - debugMessage(LOG_FT, "Event loop exited ({})", (void*) this->ioLoop); - } - } - }); - this->ioThread->name("File IO #1").execute(); - } - - { - for(auto& binding : bindings) { - binding->file_descriptor = socket(binding->address.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0); - if(binding->file_descriptor < 0) { - logError(LOG_FT, "Failed to bind server to {}. (Failed to create socket: {} | {})", binding->as_string(), errno, strerror(errno)); - continue; - } - - int enable = 1, disabled = 0; - - if (setsockopt(binding->file_descriptor, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int)) < 0) - logWarning(LOG_FT, "Failed to activate SO_REUSEADDR for binding {} ({} | {})", binding->as_string(), errno, strerror(errno)); - if(setsockopt(binding->file_descriptor, IPPROTO_TCP, TCP_NOPUSH, &disabled, sizeof disabled) < 0) - logWarning(LOG_FT, "Failed to deactivate TCP_NOPUSH for binding {} ({} | {})", binding->as_string(), errno, strerror(errno)); - if(binding->address.ss_family == AF_INET6) { - if(setsockopt(binding->file_descriptor, IPPROTO_IPV6, IPV6_V6ONLY, &enable, sizeof(int)) < 0) - logWarning(LOG_FT, "Failed to activate IPV6_V6ONLY for IPv6 binding {} ({} | {})", binding->as_string(), errno, strerror(errno)); - } - if(fcntl(binding->file_descriptor, F_SETFD, FD_CLOEXEC) < 0) - logWarning(LOG_FT, "Failed to set flag FD_CLOEXEC for binding {} ({} | {})", binding->as_string(), errno, strerror(errno)); - - - if (bind(binding->file_descriptor, (struct sockaddr *) &binding->address, sizeof(binding->address)) < 0) { - logError(LOG_FT, "Failed to bind server to {}. (Failed to bind socket: {} | {})", binding->as_string(), errno, strerror(errno)); - close(binding->file_descriptor); - continue; - } - - if (listen(binding->file_descriptor, SOMAXCONN) < 0) { - logError(LOG_FT, "Failed to bind server to {}. (Failed to listen: {} | {})", binding->as_string(), errno, strerror(errno)); - close(binding->file_descriptor); - continue; - } - - binding->event_accept = event_new(this->ioLoop, binding->file_descriptor, EV_READ | EV_PERSIST, [](int a, short b, void* c){ ((LocalFileServer *) c)->on_client_accept(a, b, c); }, this); - event_add(binding->event_accept, nullptr); - this->bindings.push_back(binding); - } - - if(this->bindings.empty()) { - this->stop(); - error = "failed to bind to any address"; - return false; - } - } - - for(int index = 0; index < 2; index++){ - auto th = new threads::Thread(THREAD_SAVE_OPERATIONS | THREAD_EXECUTE_LATER, &LocalFileServer::clientTickingExecutor, this); - th->name("Ticking FT #" + to_string(index)).execute(); - this->tickingThreads.push_back(th); - } - return true; -} - -void LocalFileServer::stop() { - if(!this->running()) return; - active = false; - { - lock_guard lock(this->tickingLock); - this->tickingCon.notify_all(); - } - - for(auto& binding : this->bindings) { - if(binding->event_accept) { - event_del_block(binding->event_accept); - event_free(binding->event_accept); - binding->event_accept = nullptr; - } - if(binding->file_descriptor > 0) { - if(shutdown(binding->file_descriptor, SHUT_RDWR) < 0) - logWarning(LOG_FT, "Failed to shutdown socket for binding {} ({} | {}).", binding->as_string(), errno, strerror(errno)); - if(close(binding->file_descriptor) < 0) - logError(LOG_FT, "Failed to close socket for binding {} ({} | {}).", binding->as_string(), errno, strerror(errno)); - binding->file_descriptor = -1; - } - } - this->bindings.clear(); - - auto clClone = this->connectedClients; - for(const auto& cl : clClone) { - cl->disconnect(chrono::milliseconds(1)); - } - - if(this->ioLoop) { - event_base_loopbreak(this->ioLoop); - event_base_loopexit(this->ioLoop, nullptr); - } - - for(const auto& thread : this->tickingThreads){ - if(thread->join(chrono::seconds(1)) != 0) logCritical(LOG_FT, "Failed to terminal file server tick thread!"); - delete thread; - } - this->tickingThreads.clear(); - - if(this->ioThread) - if(this->ioThread->join(chrono::system_clock::now() + chrono::seconds(3)) != 0) { - logCritical(LOG_FT, "Could not terminate the event base dispatch thread of the file server"); - this->ioThread->detach(); - } - delete this->ioThread; - this->ioThread = nullptr; - - if(this->ioLoop) { - event_base_free(this->ioLoop); - this->ioLoop = nullptr; - } - - if(this->server_reserve_fd > 0) { - if(close(this->server_reserve_fd) < 0) - logError(LOG_FT, "Failed to close backup file descriptor ({} | {})", errno, strerror(errno)); - } - this->server_reserve_fd = -1; -} - -inline std::string logging_address(const sockaddr_storage& address) { - if(config::server::disable_ip_saving) - return "[0|X.X.X.X" + to_string(net::port(address)) + "|unconnected]"; - return "[0|X.X.X.X" + net::to_string(address, true) + "|unconnected]"; -} - -#define CLOSE_CONNECTION \ -if(shutdown(file_descriptor, SHUT_RDWR) < 0) { \ - debugMessage(LOG_FT, "[{}] Failed to shutdown socket ({} | {}).", logging_address(remote_address), errno, strerror(errno)); \ -} \ -if(close(file_descriptor) < 0) { \ - debugMessage(LOG_FT, "[{}] Failed to close socket ({} | {}).", logging_address(remote_address), errno, strerror(errno)); \ -} - -void LocalFileServer::on_client_accept(int _server_file_descriptor, short ev, void *arg) { - sockaddr_storage remote_address{}; - memset(&remote_address, 0, sizeof(remote_address)); - socklen_t address_length = sizeof(remote_address); - - int file_descriptor = accept(_server_file_descriptor, (struct sockaddr *) &remote_address, &address_length); - if (file_descriptor < 0) { - if(errno == EAGAIN) - return; - - if(errno == EMFILE || errno == ENFILE) { - if(errno == EMFILE) - logError(LOG_FT, "Server ran out file descriptors. Please increase the process file descriptor limit or decrease the instance variable 'serverinstance_filetransfer_max_connections'"); - else - logError(LOG_FT, "Server ran out file descriptors. Please increase the process and system-wide file descriptor limit or decrease the instance variable 'serverinstance_filetransfer_max_connections'"); - - bool tmp_close_success = false; - { - lock_guard reserve_fd_lock(server_reserve_fd_lock); - if(this->server_reserve_fd > 0) { - debugMessage(LOG_FT, "Trying to accept client with the reserved file descriptor to close the incomming connection."); - auto _ = [&]{ - if(close(this->server_reserve_fd) < 0) { - debugMessage(LOG_FT, "Failed to close reserved file descriptor"); - tmp_close_success = false; - return; - } - this->server_reserve_fd = 0; - - errno = 0; - file_descriptor = accept(_server_file_descriptor, (struct sockaddr *) &remote_address, &address_length); - if(file_descriptor < 0) { - if(errno == EMFILE || errno == ENFILE) - debugMessage(LOG_FT, "[{}] Even with freeing the reserved descriptor accept failed. Attempting to reclaim reserved file descriptor", logging_address(remote_address)); - else if(errno == EAGAIN); - else { - debugMessage(LOG_FT, "[{}] Failed to accept client with reserved file descriptor. ({} | {})", logging_address(remote_address), errno, strerror(errno)); - } - this->server_reserve_fd = dup(1); - if(this->server_reserve_fd < 0) - debugMessage(LOG_FT, "[{}] Failed to reclaim reserved file descriptor. Future clients cant be accepted!", logging_address(remote_address)); - else - tmp_close_success = true; - return; - } - debugMessage(LOG_FT, "[{}] Successfully accepted client via reserved descriptor (fd: {}). Disconnecting client.", logging_address(remote_address), file_descriptor); - - CLOSE_CONNECTION - this->server_reserve_fd = dup(1); - if(this->server_reserve_fd < 0) - debugMessage(LOG_FT, "Failed to reclaim reserved file descriptor. Future clients cant be accepted!"); - else - tmp_close_success = true; - logMessage(LOG_FT, "[{}] Dropping file transfer connection attempt because of too many open file descriptors.", logging_address(remote_address)); - }; - _(); - } - } - - if(!tmp_close_success) { - debugMessage(LOG_FT, "Sleeping two seconds because we're currently having no resources for this user. (Removing the accept event)"); - for(auto& binding : this->bindings) - event_del_noblock(binding->event_accept); - accept_event_deleted = system_clock::now(); - return; - } - return; - } - logMessage(LOG_FT, "Got an error while accepting a new client. (errno: {}, message: {})", errno, strerror(errno)); - return; - } - { - unique_lock lock(this->clientLock); - auto max_connections = serverInstance->properties()[property::SERVERINSTANCE_FILETRANSFER_MAX_CONNECTIONS].as(); - if(max_connections > 0 && max_connections <= this->connectedClients.size()) { - lock.unlock(); - logMessage(LOG_FT, "[{}] Dropping new connection attempt because of too many connected clients.", logging_address(remote_address)); - CLOSE_CONNECTION - return; - } - - auto max_ip_connections = serverInstance->properties()[property::SERVERINSTANCE_FILETRANSFER_MAX_CONNECTIONS_PER_IP].as(); - if(max_ip_connections > 0) { - size_t connection_count = 0; - for(auto& client : this->connectedClients) { - if(net::address_equal(client->remote_address, remote_address)) - connection_count++; - } - - if(connection_count >= max_ip_connections) { - lock.unlock(); - logMessage(LOG_FT, "[{}] Dropping new connection attempt because of too many simultaneously connected session from this ip.", logging_address(remote_address)); - CLOSE_CONNECTION - return; - } - } - } - - - shared_ptr client = std::make_shared(this, file_descriptor); - client->_this = client; - memcpy(&client->remote_address, &remote_address, sizeof(remote_address)); - - this->clientLock.lock(); - this->connectedClients.push_back(client); - this->clientLock.unlock(); - event_add(client->readEvent, nullptr); - logMessage(LOG_FT, "[{}] Remote peer connected. Initializing session.", logging_address(remote_address)); -} - -void LocalFileServer::clientTickingExecutor() { - while(this->running()){ - shared_ptr client; - { - unique_lock lock(this->tickingLock); - this->tickingCon.wait(lock, [&](){ return !this->running() || !this->tickQueue.empty(); }); - if(!this->running()) return; - - client = this->tickQueue.front(); - this->tickQueue.pop_front(); - } - - if(client->tick()) this->tickFileClient(client); //Client needs more ticking - } -} - -std::deque> LocalFileServer::running_file_transfers(const std::shared_ptr &client) { - std::deque> result; - - { - threads::MutexLock lock(this->clientLock); - for(const auto& c : this->connectedClients) { - if(!client || c->client == client) result.push_back(c); - } - } - - return result; -} - -std::deque> LocalFileServer::pending_file_transfers(const std::shared_ptr &client) { - std::deque> result; - for(const auto& key : this->pending_keys()) - if(!client || key->owner.lock() == client) - result.push_back(key); - return result; -} - -void LocalFileServer::tickFileClient(std::shared_ptr cl) { - lock_guard lock(this->tickingLock); - this->tickQueue.push_back(cl); - this->tickingCon.notify_one(); -} - -struct TransfareGroup { - shared_ptr server; - deque> clients; - enum { - upload, - download - } direction = upload; - size_t used_bandwidth = 0; - ssize_t max_bandwidth = -1; -}; - -void LocalFileServer::instanceTick() { - { - //tickQueue - auto client = this->connected_clients(); - lock_guard tick_lock(this->tickingLock); - this->tickQueue.insert(this->tickQueue.end(), client.begin(), client.end()); //Tick all clients :) - this->tickingCon.notify_all(); - } - if(this->accept_event_deleted.time_since_epoch().count() != 0 && accept_event_deleted + seconds(5) < system_clock::now()) { - debugMessage(LOG_FT, "Readding accept event and try again if we have enough resources again."); - for(auto& binding : this->bindings) - event_add(binding->event_accept, nullptr); - accept_event_deleted = system_clock::time_point{}; - } - - auto now = system_clock::now(); - if(timestamp_bandwidth_update + seconds(1) < now) { - timestamp_bandwidth_update = now; - - for(const auto& entry : this->pending_keys()) { - if(!entry) continue; - if(entry->createTimestamp + minutes(1) < now) { - logMessage(entry->server, "[FILE] Timeout file transfer for file " + entry->targetFile + "! Dropping key"); - - { - threads::MutexLock lock(this->keylock); - auto index = find(this->pendingKeys.begin(), this->pendingKeys.end(), entry); - if(index != this->pendingKeys.end()) - this->pendingKeys.erase(index); - } - } - } - - deque> groups; - { - for(const auto& client : this->connected_clients()) { - if(!client || !client->client) continue; - - lock_guard l(client->tickLock); //Instance hangup here - if(!client->pendingKey) continue; - - auto used_band = client->used_bandwidth(); - if(client->pendingKey && client->state_connection != FileClient::C_DISCONNECTED && (client->pendingKey->max_bandwhidth < 0 || used_band < client->pendingKey->max_bandwhidth)) { - - if(client->event_read_hold) { //Reactivate read! - event_add(client->readEvent, nullptr); - client->event_read_hold = false; - debugMessage(LOG_FT, "{} Reattaching client read event! (Current bandwidth {})", client->client_prefix(), used_band); - } - if(client->event_write_hold) { //Reactivate read! - event_add(client->writeEvent, nullptr); - client->event_write_hold = false; - debugMessage(LOG_FT, "{} Reattaching client write event! (Current bandwidth {})", client->client_prefix(), used_band); - } - } - if(!client->client->getServer()) continue; - - bool found = false; - for(auto& group : groups) { - if (group->server == client->client->getServer() && (group->direction == TransfareGroup::upload) == client->pendingKey->upload) { - group->clients.push_back(client); - group->used_bandwidth += client->used_bandwidth(); - found = true; - break; - } - } - if(found) continue; - - auto entry = make_shared(); - entry->direction = client->pendingKey->upload ? TransfareGroup::upload : TransfareGroup::download; - entry->used_bandwidth = client->used_bandwidth(); - entry->clients.push_back(client); - entry->server = client->client->getServer(); - groups.push_back(entry); - } - } - - { //Instance limit upload - auto limit_upload = serverInstance->properties()[property::SERVERINSTANCE_MAX_UPLOAD_TOTAL_BANDWIDTH].as(); - if(limit_upload >= 0) { - deque> elements; - size_t used_bandwidth = 0; - for(const auto& entry : groups) { - if (entry->direction == TransfareGroup::upload) { - elements.push_back(entry); - used_bandwidth += entry->used_bandwidth; - } - } - if(!elements.empty() && used_bandwidth > limit_upload) { - for(auto& entry : elements) { - entry->max_bandwidth = ssize_t(ceil(double(entry->used_bandwidth) * double(limit_upload) / double(used_bandwidth))); //Adjust the total - } - } - } - } - - { //Instance limit download - auto limit_download = serverInstance->properties()[property::SERVERINSTANCE_MAX_DOWNLOAD_TOTAL_BANDWIDTH].as(); - if(limit_download >= 0) { - deque> elements; - size_t used_bandwidth = 0; - for(const auto& entry : groups) { - if (entry->direction == TransfareGroup::download) { - elements.push_back(entry); - used_bandwidth += entry->used_bandwidth; - } - } - if(!elements.empty() && used_bandwidth > limit_download) { - for(auto& entry : elements) { - entry->max_bandwidth = ssize_t(ceil(double(entry->used_bandwidth) * double(limit_download) / double(used_bandwidth))); //Adjust the total - } - } - } - } - - map>> servers; - for(const auto& entry : groups) { - servers[entry->server->getServerId()].push_back(entry); - } - - for(const auto& server : servers) { - if(server.second.empty()) continue; - - { //Adjust server upload - auto max_bandwidth = server.second.front()->server->properties()[property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH].as(); - if(max_bandwidth >= 0) { - deque> tranfares; - size_t bandwidth = 0; - for(const auto& trans : server.second) { - if(trans->direction == TransfareGroup::upload) { - tranfares.push_back(trans); - bandwidth += trans->max_bandwidth; - } - } - - if(bandwidth > max_bandwidth) { - for(auto& trans : tranfares) - trans->max_bandwidth = ssize_t(ceil(double(trans->max_bandwidth) * double(max_bandwidth) / double(bandwidth))); //Adjust the total - } - } - } - - { //Adjust server download - auto max_bandwidth = server.second.front()->server->properties()[property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH].as(); - if(max_bandwidth >= 0) { - deque> tranfares; - size_t bandwidth = 0; - for(const auto& trans : server.second) - if(trans->direction == TransfareGroup::download) { - tranfares.push_back(trans); - bandwidth += trans->max_bandwidth; - } - - if(bandwidth > max_bandwidth) { - for(auto& trans : tranfares) - trans->max_bandwidth = ssize_t(ceil(double(trans->max_bandwidth) * double(max_bandwidth) / double(bandwidth))); //Adjust the total - } - } - } - } - } -} \ No newline at end of file diff --git a/server/src/server/file/LocalFileServer.h b/server/src/server/file/LocalFileServer.h deleted file mode 100644 index 805aafb..0000000 --- a/server/src/server/file/LocalFileServer.h +++ /dev/null @@ -1,163 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "Variable.h" -#include -#include - -namespace ts { - namespace file { - namespace FileType { - enum FileType { - DIRECTORY, - FILE - }; - } - } -} -DEFINE_VARIABLE_TRANSFORM(ts::file::FileType::FileType, VARTYPE_INT, std::to_string((uint8_t) in), static_cast(in.as())); -namespace ts { - class BasicChannel; - - namespace server { - class ConnectedClient; - } - - namespace file { - struct FileEntry { - FileType::FileType type; - std::string path; - std::string name; - std::chrono::time_point lastChanged; - }; - - struct File : public FileEntry { - int64_t fileSize; - }; - - struct Directory : public FileEntry { }; - - - struct SymLink : public FileEntry { - std::string targetPath; - std::string targetName; - }; - - struct FileTransfereKey { - uint16_t key_id = 0; - ServerId server; - std::weak_ptr owner; - - std::chrono::time_point createTimestamp; - std::string key; - std::string targetFile; //Relative - int64_t size; - size_t offset; - - bool upload = false; - - int64_t max_bandwhidth = -1; //Get calculated each time - int64_t max_server_bandwidth = -1; - }; - } - - - namespace server { - class VirtualServer; - class FileClient; - - //FIXME Valid path - class LocalFileServer { - friend class FileClient; - public: - struct Binding { - sockaddr_storage address{}; - int file_descriptor = 0; - ::event* event_accept = nullptr; - - inline std::string as_string() { return net::to_string(address, true); } - }; - - LocalFileServer(); - ~LocalFileServer(); - - bool start(const std::deque>& /* bindings */, std::string& /* error */); - void stop(); - ts_always_inline bool running(){ return active; } - ts_always_inline std::deque> list_bindings() { return this->bindings; } - - std::shared_ptr createDirectory(std::string name, std::shared_ptr parent); - bool fileExists(std::shared_ptr); - bool fileExists(std::shared_ptr); - std::shared_ptr findFile(std::string, std::shared_ptr = nullptr); - std::vector> listFiles(std::shared_ptr directory = nullptr); - bool deleteFile(std::shared_ptr); - - - std::shared_ptr generateDownloadTransferKey(std::string &errorMessage, std::string targetFile, size_t offset, const std::shared_ptr&); - std::shared_ptr generateUploadTransferKey(std::string &errorMessage, std::string targetFile, size_t size, size_t offset, const std::shared_ptr&); - - std::shared_ptr resolveDirectory(const std::shared_ptr &, std::shared_ptr, std::string = ""); - std::shared_ptr iconDirectory(const std::shared_ptr &); - bool iconExists(const std::shared_ptr &, IconId); - std::shared_ptr avatarDirectory(const std::shared_ptr &); - - std::string server_file_base(const std::shared_ptr &); - void setupServer(const std::shared_ptr &); - void deleteServer(const std::shared_ptr &); - - void tickFileClient(std::shared_ptr); - void instanceTick(); - - std::deque> running_file_transfers(const std::shared_ptr & /* client */ = nullptr); - std::deque> pending_file_transfers(const std::shared_ptr & /* client */ = nullptr); - private: - bool active = false; - std::deque> bindings; - std::string rootPath = "./files/"; - - //IO stuff - event_base* ioLoop = nullptr; - std::chrono::system_clock::time_point accept_event_deleted; - - std::mutex server_reserve_fd_lock; - int server_reserve_fd = -1; /* -1 = unset | 0 = in use | > 0 ready to use */ - - threads::Mutex clientLock; - std::deque> connectedClients; - inline std::deque> connected_clients(){ - threads::MutexLock lock(clientLock); - return connectedClients; - } - - threads::Thread* ioThread = nullptr; - void on_client_accept(int fd, short ev, void *arg); - - std::deque> tickQueue; - std::deque tickingThreads; - std::mutex tickingLock; - std::condition_variable tickingCon; - - std::chrono::system_clock::time_point timestamp_bandwidth_update; - - //file management - threads::Mutex keylock; - std::deque> pendingKeys; - inline std::deque> pending_keys(){ - threads::MutexLock lock(keylock); - return pendingKeys; - } - - void clientTickingExecutor(); - }; - } -} \ No newline at end of file diff --git a/shared b/shared index b60608f..a4febf7 160000 --- a/shared +++ b/shared @@ -1 +1 @@ -Subproject commit b60608ff94b06145bc808426392871ebd95fe9d3 +Subproject commit a4febf7b5af191d41c566292958c55155128f16f