TeaSpeak/Teaspeak-Server
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

A lot of file transfer updates

Browse Source
This commit is contained in:
WolverinDEV 2020-05-13 11:32:08 +02:00
parent 1a2dd4a008
commit 90b1646876
45 changed files with 1989 additions and 2806 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
file/CMakeLists.txt
View File

@ -12,6 +12,7 @@ add_library(TeaSpeak-FileServer STATIC
local_server/LocalFileTransferDisk.cpp
local_server/LocalFileTransferNetwork.cpp
local_server/clnpath.cpp
local_server/NetTools.cpp
)
target_link_libraries(TeaSpeak-FileServer PUBLIC TeaSpeak ${StringVariable_LIBRARIES_STATIC} stdc++fs
@ -22,6 +23,7 @@ target_link_libraries(TeaSpeak-FileServer PUBLIC TeaSpeak ${StringVariable_LIBRA
)
target_include_directories(TeaSpeak-FileServer PUBLIC include/)
target_compile_options(TeaSpeak-FileServer PUBLIC "-Wswitch-enum")
add_executable(TeaSpeak-FileServerTest test/main.cpp)
target_link_libraries(TeaSpeak-FileServerTest PUBLIC TeaSpeak-FileServer

77
file/include/files/ExecuteResponse.h Normal file
View File

@ -0,0 +1,77 @@
#pragma once
namespace ts::server::file {
enum struct ExecuteStatus {
UNKNOWN,
WAITING,
SUCCESS,
ERROR
};
template<typename VariantType, typename T, std::size_t index = 0>
constexpr std::size_t variant_index() {
if constexpr (index == std::variant_size_v<VariantType>) {
return index;
} else if constexpr (std::is_same_v<std::variant_alternative_t<index, VariantType>, T>) {
return index;
} else {
return variant_index<VariantType, T, index + 1>();
}
}
struct EmptyExecuteResponse { };
template <class error_t, class response_t = EmptyExecuteResponse>
class ExecuteResponse {
typedef std::variant<EmptyExecuteResponse, error_t, response_t> variant_t;
public:
ExecuteStatus status{ExecuteStatus::WAITING};
[[nodiscard]] inline auto response() const -> const response_t& { return std::get<response_t>(this->response_); }
template <typename = std::enable_if_t<!std::is_void<error_t>::value>>
[[nodiscard]] inline const error_t& error() const { return std::get<error_t>(this->response_); }
inline void wait() const {
std::unique_lock nlock{this->notify_mutex};
this->notify_cv.wait(nlock, [&]{ return this->status != ExecuteStatus::WAITING; });
}
template<typename _Rep, typename _Period>
[[nodiscard]] inline bool wait_for(const std::chrono::duration<_Rep, _Period>& time) const {
std::unique_lock nlock{this->notify_mutex};
return this->notify_cv.wait_for(nlock, time, [&]{ return this->status != ExecuteStatus::WAITING; });
}
template <typename... Args>
inline void emplace_success(Args&&... args) {
constexpr auto success_index = variant_index<variant_t, response_t>();
std::lock_guard rlock{this->notify_mutex};
this->response_.template emplace<success_index, Args...>(std::forward<Args>(args)...);
this->status = ExecuteStatus::SUCCESS;
this->notify_cv.notify_all();
}
template <typename... Args>
inline void emplace_fail(Args&&... args) {
constexpr auto error_index = variant_index<variant_t, error_t>();
std::lock_guard rlock{this->notify_mutex};
this->response_.template emplace<error_index, Args...>(std::forward<Args>(args)...);
this->status = ExecuteStatus::ERROR;
this->notify_cv.notify_all();
}
[[nodiscard]] inline bool succeeded() const {
return this->status == ExecuteStatus::SUCCESS;
}
ExecuteResponse(std::mutex& notify_mutex, std::condition_variable& notify_cv)
: notify_mutex{notify_mutex}, notify_cv{notify_cv} {}
private:
variant_t response_{}; /* void* as default value so we don't initialize error_t or response_t */
std::mutex& notify_mutex;
std::condition_variable& notify_cv;
};
}

296
file/include/files/FileServer.h
View File

@ -4,86 +4,18 @@
#include <chrono>
#include <Definitions.h>
#include <condition_variable>
#include <utility>
#include <variant>
#include <deque>
#include <functional>
#include <atomic>
#include "./ExecuteResponse.h"
#define TRANSFER_KEY_LENGTH (32)
namespace ts::server::file {
enum struct ExecuteStatus {
UNKNOWN,
WAITING,
SUCCESS,
ERROR
};
template<typename VariantType, typename T, std::size_t index = 0>
constexpr std::size_t variant_index() {
if constexpr (index == std::variant_size_v<VariantType>) {
return index;
} else if constexpr (std::is_same_v<std::variant_alternative_t<index, VariantType>, T>) {
return index;
} else {
return variant_index<VariantType, T, index + 1>();
}
}
struct EmptyExecuteResponse { };
template <class error_t, class response_t = EmptyExecuteResponse>
class ExecuteResponse {
typedef std::variant<EmptyExecuteResponse, error_t, response_t> variant_t;
public:
ExecuteStatus status{ExecuteStatus::WAITING};
[[nodiscard]] inline const auto& response() const { return std::get<response_t>(this->response_); }
template <typename = std::enable_if_t<!std::is_void<error_t>::value>>
[[nodiscard]] inline const error_t& error() const { return std::get<error_t>(this->response_); }
inline void wait() const {
std::unique_lock nlock{this->notify_mutex};
this->notify_cv.wait(nlock, [&]{ return this->status != ExecuteStatus::WAITING; });
}
template<typename _Rep, typename _Period>
[[nodiscard]] inline bool wait_for(const std::chrono::duration<_Rep, _Period>& time) const {
std::unique_lock nlock{this->notify_mutex};
return this->notify_cv.wait_for(nlock, time, [&]{ return this->status != ExecuteStatus::WAITING; });
}
template <typename... Args>
inline void emplace_success(Args&&... args) {
constexpr auto success_index = variant_index<variant_t, response_t>();
std::lock_guard rlock{this->notify_mutex};
this->response_.template emplace<success_index, Args...>(std::forward<Args>(args)...);
this->status = ExecuteStatus::SUCCESS;
this->notify_cv.notify_all();
}
template <typename... Args>
inline void emplace_fail(Args&&... args) {
constexpr auto error_index = variant_index<variant_t, error_t>();
std::lock_guard rlock{this->notify_mutex};
this->response_.template emplace<error_index, Args...>(std::forward<Args>(args)...);
this->status = ExecuteStatus::ERROR;
this->notify_cv.notify_all();
}
[[nodiscard]] inline bool succeeded() const {
return this->status == ExecuteStatus::SUCCESS;
}
ExecuteResponse(std::mutex& notify_mutex, std::condition_variable& notify_cv)
: notify_mutex{notify_mutex}, notify_cv{notify_cv} {}
private:
variant_t response_{}; /* void* as default value so we don't initialize error_t or response_t */
std::mutex& notify_mutex;
std::condition_variable& notify_cv;
};
class VirtualFileServer;
namespace filesystem {
template <typename ErrorCodes>
@ -99,11 +31,9 @@ namespace ts::server::file {
PATH_EXCEEDS_ROOT_PATH,
PATH_IS_A_FILE,
PATH_DOES_NOT_EXISTS,
FAILED_TO_LIST_FILES,
MAX
FAILED_TO_LIST_FILES
};
constexpr std::array<std::string_view, (int) DirectoryQueryErrorType::MAX> directory_query_error_messages = {
constexpr std::array<std::string_view, 5> directory_query_error_messages = {
"unknown error",
"path exceeds base path",
"path is a file",
@ -143,11 +73,38 @@ namespace ts::server::file {
TARGET_PATH_ALREADY_EXISTS,
FAILED_TO_DELETE_FILES,
FAILED_TO_RENAME_FILE,
FAILED_TO_CREATE_DIRECTORIES,
SOME_FILES_ARE_LOCKED
};
typedef DetailedError<FileModifyErrorType> FileModifyError;
enum struct FileDeleteErrorType {
UNKNOWN,
};
typedef DetailedError<FileDeleteErrorType> FileDeleteError;
struct FileDeleteResponse {
enum struct StatusType {
SUCCESS,
PATH_EXCEEDS_ROOT_PATH,
PATH_DOES_NOT_EXISTS,
FAILED_TO_DELETE_FILES,
SOME_FILES_ARE_LOCKED
};
struct DeleteResult {
StatusType status{StatusType::SUCCESS};
std::string error_detail{};
DeleteResult(StatusType status, std::string errorDetail) : status{status},
error_detail{std::move(errorDetail)} {}
};
std::vector<DeleteResult> delete_results{};
};
enum struct ServerCommandErrorType {
UNKNOWN,
FAILED_TO_CREATE_DIRECTORIES,
@ -155,27 +112,59 @@ namespace ts::server::file {
};
typedef DetailedError<ServerCommandErrorType> ServerCommandError;
struct FileInfoResponse {
enum struct StatusType {
SUCCESS,
PATH_EXCEEDS_ROOT_PATH,
PATH_DOES_NOT_EXISTS,
FAILED_TO_QUERY_INFO,
UNKNOWN_FILE_TYPE
};
struct FileInfo {
StatusType status{StatusType::SUCCESS};
std::string error_detail{};
DirectoryEntry info{};
FileInfo(StatusType status, std::string errorDetail, DirectoryEntry info) : status{status},
error_detail{std::move(errorDetail)}, info{std::move(info)} {}
};
std::vector<FileInfo> file_info{};
};
enum struct FileInfoErrorType {
UNKNOWN,
};
typedef DetailedError<FileInfoErrorType> FileInfoError;
class AbstractProvider {
public:
typedef ExecuteResponse<DirectoryQueryError, std::deque<DirectoryEntry>> directory_query_response_t;
/* server */
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<ServerCommandError>> initialize_server(ServerId /* server */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<ServerCommandError>> delete_server(ServerId /* server */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<ServerCommandError>> initialize_server(const std::shared_ptr<VirtualFileServer> &/* server */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<ServerCommandError>> delete_server(const std::shared_ptr<VirtualFileServer> &/* server */) = 0;
/* channels */
[[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_channel_directory(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<DirectoryModifyError>> create_channel_directory(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileModifyError>> delete_channel_file(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileModifyError>> rename_channel_file(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */, const std::string& /* target */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> query_channel_info(const std::shared_ptr<VirtualFileServer> &/* server */, ChannelId /* channel */, const std::vector<std::string>& /* names */) = 0;
[[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_channel_directory(const std::shared_ptr<VirtualFileServer> &/* server */, ChannelId /* channel */, const std::string& /* path */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<DirectoryModifyError>> create_channel_directory(const std::shared_ptr<VirtualFileServer> &/* server */, ChannelId /* channel */, const std::string& /* path */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> delete_channel_files(const std::shared_ptr<VirtualFileServer> &/* server */, ChannelId /* channel */, const std::vector<std::string>& /* paths */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileModifyError>> rename_channel_file(const std::shared_ptr<VirtualFileServer> &/* server */, ChannelId /* channel */, const std::string& /* path */, ChannelId /* target channel */, const std::string& /* target */) = 0;
/* icons */
[[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_icon_directory(ServerId /* server */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileModifyError>> delete_icon(ServerId /* server */, const std::string& /* name */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> query_icon_info(const std::shared_ptr<VirtualFileServer> &/* server */, const std::vector<std::string>& /* names */) = 0;
[[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_icon_directory(const std::shared_ptr<VirtualFileServer> &/* server */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> delete_icons(const std::shared_ptr<VirtualFileServer> &/* server */, const std::vector<std::string>& /* names */) = 0;
/* avatars */
[[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_avatar_directory(ServerId /* server */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileModifyError>> delete_avatar(ServerId /* server */, const std::string& /* name */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> query_avatar_info(const std::shared_ptr<VirtualFileServer> &/* server */, const std::vector<std::string>& /* names */) = 0;
[[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_avatar_directory(const std::shared_ptr<VirtualFileServer> &/* server */) = 0;
[[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> delete_avatars(const std::shared_ptr<VirtualFileServer> &/* server */, const std::vector<std::string>& /* names */) = 0;
private:
};
}
@ -187,10 +176,12 @@ namespace ts::server::file {
transfer_id server_transfer_id{0};
transfer_id client_transfer_id{0};
ServerId server_id{0};
ClientId client_id{0};
std::shared_ptr<VirtualFileServer> server{nullptr};
ChannelId channel_id{0};
ClientId client_id{0};
std::string client_unique_id{};
char transfer_key[TRANSFER_KEY_LENGTH]{};
std::chrono::system_clock::time_point initialized_timestamp{};
enum Direction {
@ -212,6 +203,10 @@ namespace ts::server::file {
TARGET_TYPE_AVATAR
} target_type{TARGET_TYPE_UNKNOWN};
std::string target_file_path{};
std::string absolute_file_path{};
std::string relative_file_path{};
std::string file_name{};
int64_t max_bandwidth{-1};
size_t expected_file_size{0}; /* incl. the offset! */
@ -236,9 +231,24 @@ namespace ts::server::file {
struct TransferInitError {
enum Type {
UNKNOWN
UNKNOWN,
INVALID_FILE_TYPE,
FILE_DOES_NOT_EXISTS,
FILE_IS_NOT_A_FILE,
CLIENT_TOO_MANY_TRANSFERS,
SERVER_TOO_MANY_TRANSFERS,
SERVER_QUOTA_EXCEEDED,
CLIENT_QUOTA_EXCEEDED,
IO_ERROR
} error_type{UNKNOWN};
std::string error_message{};
TransferInitError(Type errorType, std::string errorMessage) : error_type{errorType},
error_message{std::move(errorMessage)} {}
};
struct TransferActionError {
@ -268,22 +278,69 @@ namespace ts::server::file {
std::string error_message{};
};
struct ActiveFileTransfer {
transfer_id client_transfer_id{0};
transfer_id server_transfer_id{0};
Transfer::Direction direction{Transfer::DIRECTION_UNKNOWN};
ClientId client_id{};
std::string client_unique_id{};
std::string file_path{};
std::string file_name{};
size_t expected_size{};
size_t size_done{};
enum Status {
NOT_STARTED,
RUNNING,
INACTIVE /* (not used yet) */
} status{Status::NOT_STARTED};
std::chrono::milliseconds runtime{};
double average_speed{0};
double current_speed{0};
};
enum struct TransferListError {
UNKNOWN
};
class AbstractProvider {
public:
struct TransferInfo {
std::string file_path{};
std::string client_unique_id{};
ClientId client_id{};
bool override_exiting{false}; /* only for upload valid */
size_t file_offset{0};
size_t expected_file_size{0};
int64_t max_bandwidth{-1};
int64_t max_concurrent_transfers{-1};
/* only used for upload, for download the quotas could be checked before */
size_t download_server_quota_limit{(size_t) -1};
size_t download_client_quota_limit{(size_t) -1};
};
virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> initialize_channel_transfer(Transfer::Direction /* direction */, ServerId /* server */, ChannelId /* channel */, const TransferInfo& /* info */) = 0;
virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> initialize_icon_transfer(Transfer::Direction /* direction */, ServerId /* server */, const TransferInfo& /* info */) = 0;
virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> initialize_avatar_transfer(Transfer::Direction /* direction */, ServerId /* server */, const TransferInfo& /* info */) = 0;
virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
initialize_channel_transfer(Transfer::Direction /* direction */, const std::shared_ptr<VirtualFileServer>& /* server */, ChannelId /* channel */, const TransferInfo& /* info */) = 0;
virtual std::shared_ptr<ExecuteResponse<TransferActionError>> stop_transfer(transfer_id /* id */, bool /* flush */) = 0;
virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
initialize_icon_transfer(Transfer::Direction /* direction */, const std::shared_ptr<VirtualFileServer>& /* server */, const TransferInfo& /* info */) = 0;
virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
initialize_avatar_transfer(Transfer::Direction /* direction */, const std::shared_ptr<VirtualFileServer>& /* server */, const TransferInfo& /* info */) = 0;
virtual std::shared_ptr<ExecuteResponse<TransferListError, std::vector<ActiveFileTransfer>>> list_transfer() = 0;
virtual std::shared_ptr<ExecuteResponse<TransferActionError>> stop_transfer(const std::shared_ptr<VirtualFileServer>& /* server */, transfer_id /* id */, bool /* flush */) = 0;
std::function<void(const std::shared_ptr<Transfer>&)> callback_transfer_registered{}; /* transfer has been registered */
std::function<void(const std::shared_ptr<Transfer>&)> callback_transfer_started{}; /* transfer has been started */
@ -293,11 +350,60 @@ namespace ts::server::file {
};
}
class VirtualFileServer {
public:
explicit VirtualFileServer(ServerId server_id, std::string unique_id) : server_id_{server_id}, unique_id_{std::move(unique_id)} {}
[[nodiscard]] inline auto unique_id() const -> const std::string& { return this->unique_id_; }
[[nodiscard]] inline auto server_id() const -> ServerId { return this->server_id_; }
[[nodiscard]] inline auto max_networking_upload_bandwidth() const -> int64_t { return this->max_networking_upload_bandwidth_; }
virtual void max_networking_upload_bandwidth(int64_t value) {
this->max_networking_upload_bandwidth_ = value;
}
[[nodiscard]] inline auto max_networking_download_bandwidth() const -> int64_t { return this->max_networking_download_bandwidth_; }
virtual void max_networking_download_bandwidth(int64_t value) {
this->max_networking_download_bandwidth_ = value;
}
[[nodiscard]] inline auto generate_transfer_id() {
return ++this->current_transfer_id;
}
private:
ServerId server_id_;
std::string unique_id_;
int64_t max_networking_upload_bandwidth_{-1};
int64_t max_networking_download_bandwidth_{-1};
std::atomic<transfer::transfer_id> current_transfer_id{0};
};
class AbstractFileServer {
public:
[[nodiscard]] virtual std::string file_base_path() const = 0;
[[nodiscard]] virtual filesystem::AbstractProvider& file_system() = 0;
[[nodiscard]] virtual transfer::AbstractProvider& file_transfer() = 0;
private:
[[nodiscard]] inline auto virtual_servers() const -> std::deque<std::shared_ptr<VirtualFileServer>> {
std::lock_guard slock{this->servers_mutex};
return this->servers_;
}
[[nodiscard]] inline auto find_virtual_server(ServerId server_id) const -> std::shared_ptr<VirtualFileServer> {
std::lock_guard slock{this->servers_mutex};
auto it = std::find_if(this->servers_.begin(), this->servers_.end(), [&](const std::shared_ptr<VirtualFileServer>& server) {
return server->server_id() == server_id;
});
return it == this->servers_.end() ? nullptr : *it;
}
virtual std::shared_ptr<VirtualFileServer> register_server(ServerId /* server id */) = 0;
virtual void unregister_server(ServerId /* server id */) = 0;
protected:
mutable std::mutex servers_mutex{};
std::deque<std::shared_ptr<VirtualFileServer>> servers_{};
};
extern bool initialize(std::string& /* error */);

46
file/local_server/LocalFileProvider.cpp
View File

@ -7,6 +7,7 @@
using namespace ts::server;
using LocalFileServer = file::LocalFileProvider;
using LocalVirtualFileServer = file::LocalVirtualFileServer;
EVP_PKEY* ssl_generate_key() {
auto key = std::unique_ptr<EVP_PKEY, decltype(&EVP_PKEY_free)>(EVP_PKEY_new(), ::EVP_PKEY_free);
@ -81,7 +82,7 @@ LocalFileServer::LocalFileProvider() : file_system_{}, file_transfer_{this->file
LocalFileServer::~LocalFileProvider() {}
bool LocalFileServer::initialize(std::string &error, const std::shared_ptr<pipes::SSL::Options>& ssl_options) {
if(!this->file_system_.initialize(error, "file-root/"))
if(!this->file_system_.initialize(error, "files/"))
return false;
@ -89,7 +90,7 @@ bool LocalFileServer::initialize(std::string &error, const std::shared_ptr<pipes
{
auto binding = std::make_shared<transfer::NetworkBinding>();
binding->hostname = "localhost";
binding->hostname = "0.0.0.0";
auto& iaddr = *(sockaddr_in*) &binding->address;
iaddr.sin_family = AF_INET;
@ -116,6 +117,45 @@ file::filesystem::AbstractProvider &LocalFileServer::file_system() {
return this->file_system_;
}
file::transfer::AbstractProvider & LocalFileServer::file_transfer() {
file::transfer::AbstractProvider &LocalFileServer::file_transfer() {
return this->file_transfer_;
}
std::string file::LocalFileProvider::file_base_path() const {
return this->file_system_.root_path();
}
std::shared_ptr<file::VirtualFileServer> LocalFileServer::register_server(ServerId server_id) {
auto server = this->find_virtual_server(server_id);
if(server) return server;
server = std::make_shared<file::LocalVirtualFileServer>(server_id, std::to_string(server_id));
{
std::lock_guard slock{this->servers_mutex};
this->servers_.push_back(server);
}
return server;
}
void LocalFileServer::unregister_server(ServerId server_id) {
auto server_unique_id = std::to_string(server_id);
std::lock_guard slock{this->servers_mutex};
auto it = std::find_if(this->servers_.begin(), this->servers_.end(), [&](const std::shared_ptr<VirtualFileServer>& server) {
return server->unique_id() == server_unique_id;
});
if(it == this->servers_.end()) return;
this->servers_.erase(it);
}
void LocalVirtualFileServer::max_networking_upload_bandwidth(int64_t value) {
VirtualFileServer::max_networking_upload_bandwidth(value);
this->upload_throttle.set_max_bandwidth(value);
}
void LocalVirtualFileServer::max_networking_download_bandwidth(int64_t value) {
VirtualFileServer::max_networking_download_bandwidth(value);
this->download_throttle.set_max_bandwidth(value);
}

174
file/local_server/LocalFileProvider.h
View File

@ -9,7 +9,9 @@
#include <pipes/ws.h>
#include <pipes/ssl.h>
#include <misc/net.h>
#include <misc/spin_mutex.h>
#include <random>
#include "./NetTools.h"
#define TRANSFER_MAX_CACHED_BYTES (1024 * 1024 * 1) // Buffer up to 1mb
@ -39,54 +41,66 @@ namespace ts::server::file {
[[nodiscard]] inline const auto &root_path() const { return this->root_path_; }
[[nodiscard]] std::string absolute_avatar_path(ServerId, const std::string&);
[[nodiscard]] std::string absolute_icon_path(ServerId, const std::string&);
[[nodiscard]] std::string absolute_channel_path(ServerId, ChannelId, const std::string&);
[[nodiscard]] std::string absolute_avatar_path(const std::shared_ptr<VirtualFileServer> &, const std::string&);
[[nodiscard]] std::string absolute_icon_path(const std::shared_ptr<VirtualFileServer> &, const std::string&);
[[nodiscard]] std::string absolute_channel_path(const std::shared_ptr<VirtualFileServer> &, ChannelId, const std::string&);
std::shared_ptr<ExecuteResponse<ServerCommandError>> initialize_server(ServerId /* server */) override;
std::shared_ptr<ExecuteResponse<ServerCommandError>> delete_server(ServerId /* server */) override;
std::shared_ptr<ExecuteResponse<ServerCommandError>> initialize_server(const std::shared_ptr<VirtualFileServer> & /* server */) override;
std::shared_ptr<ExecuteResponse<ServerCommandError>> delete_server(const std::shared_ptr<VirtualFileServer> & /* server */) override;
std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>>
query_channel_info(const std::shared_ptr<VirtualFileServer> & /* server */, ChannelId /* channel */, const std::vector<std::string>& /* names */) override;
std::shared_ptr<directory_query_response_t>
query_channel_directory(ServerId id, ChannelId channelId, const std::string &string) override;
query_channel_directory(const std::shared_ptr<VirtualFileServer> & id, ChannelId channelId, const std::string &string) override;
std::shared_ptr<ExecuteResponse<DirectoryModifyError>>
create_channel_directory(ServerId id, ChannelId channelId, const std::string &string) override;
create_channel_directory(const std::shared_ptr<VirtualFileServer> & id, ChannelId channelId, const std::string &string) override;
std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>>
delete_channel_files(const std::shared_ptr<VirtualFileServer> & id, ChannelId channelId, const std::vector<std::string> &string) override;
std::shared_ptr<ExecuteResponse<FileModifyError>>
delete_channel_file(ServerId id, ChannelId channelId, const std::string &string) override;
rename_channel_file(const std::shared_ptr<VirtualFileServer> & id, ChannelId channelId, const std::string &, ChannelId, const std::string &) override;
std::shared_ptr<ExecuteResponse<FileModifyError>>
rename_channel_file(ServerId id, ChannelId channelId, const std::string &, const std::string &) override;
std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>>
query_icon_info(const std::shared_ptr<VirtualFileServer> & /* server */, const std::vector<std::string>& /* names */) override;
std::shared_ptr<directory_query_response_t> query_icon_directory(ServerId id) override;
std::shared_ptr<directory_query_response_t> query_icon_directory(const std::shared_ptr<VirtualFileServer> & id) override;
std::shared_ptr<ExecuteResponse<FileModifyError>>
delete_icon(ServerId id, const std::string &string) override;
std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>>
delete_icons(const std::shared_ptr<VirtualFileServer> & id, const std::vector<std::string> &string) override;
std::shared_ptr<directory_query_response_t> query_avatar_directory(ServerId id) override;
std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>>
query_avatar_info(const std::shared_ptr<VirtualFileServer> & /* server */, const std::vector<std::string>& /* names */) override;
std::shared_ptr<ExecuteResponse<FileModifyError>>
delete_avatar(ServerId id, const std::string &string) override;
std::shared_ptr<directory_query_response_t> query_avatar_directory(const std::shared_ptr<VirtualFileServer> & id) override;
std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>>
delete_avatars(const std::shared_ptr<VirtualFileServer> & id, const std::vector<std::string> &string) override;
private:
#ifdef FS_INCLUDED
[[nodiscard]] fs::path server_path(ServerId);
[[nodiscard]] fs::path server_channel_path(ServerId, ChannelId);
[[nodiscard]] fs::path server_path(const std::shared_ptr<VirtualFileServer> &);
[[nodiscard]] fs::path server_channel_path(const std::shared_ptr<VirtualFileServer> &, ChannelId);
[[nodiscard]] static bool exceeds_base_path(const fs::path& /* base */, const fs::path& /* target */);
[[nodiscard]] bool is_any_file_locked(const fs::path& /* base */, const std::string& /* path */, std::string& /* file (relative to the base) */);
[[nodiscard]] std::shared_ptr<ExecuteResponse<FileModifyError>>
delete_file(const fs::path& /* base */, const std::string &string);
[[nodiscard]] std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>>
delete_files(const fs::path& /* base */, const std::vector<std::string> &string);
[[nodiscard]] std::shared_ptr<directory_query_response_t>
query_directory(const fs::path& /* base */, const std::string &string, bool);
[[nodiscard]] std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>>
query_file_info(const fs::path& /* base */, const std::vector<std::string> &string);
#endif
template <typename error_t, typename result_t = EmptyExecuteResponse>
std::shared_ptr<ExecuteResponse<error_t, result_t>> create_execute_response() {
return std::make_shared<ExecuteResponse<error_t, result_t>>(this->result_notify_mutex, this->result_notify_cv);
}
std::string target_file_path(FileCategory type, ts::ServerId sid, ts::ChannelId cid, const std::string &path);
std::string target_file_path(FileCategory type, const std::shared_ptr<VirtualFileServer> &sid, ts::ChannelId cid, const std::string &path);
std::mutex result_notify_mutex{};
std::condition_variable result_notify_cv{};
@ -113,65 +127,6 @@ namespace ts::server::file {
[[nodiscard]] extern Buffer* allocate_buffer(size_t);
extern void free_buffer(Buffer*);
struct NetworkThrottle {
constexpr static auto kThrottleTimespanMs{250};
typedef uint8_t span_t;
ssize_t max_bytes{0};
span_t current_index{0};
size_t bytes_send{0};
inline bool increase_bytes(size_t bytes) {
auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
if(this->current_index != current_span) {
this->current_index = current_span;
this->bytes_send = bytes;
} else {
this->bytes_send += bytes;
}
return this->max_bytes > 0 && this->bytes_send >= this->max_bytes;
}
inline void set_max_bandwidth(ssize_t bytes_per_second) {
if(bytes_per_second <= 0)
this->max_bytes = -1;
else
this->max_bytes = bytes_per_second * kThrottleTimespanMs / 1000;
}
[[nodiscard]] inline bool should_throttle(timeval& next_timestamp) {
if(this->max_bytes <= 0) return false;
auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
if(this->current_index != current_span) return false;
if(this->bytes_send < this->max_bytes) return false;
next_timestamp.tv_usec = (kThrottleTimespanMs - current_ms % kThrottleTimespanMs) * 1000;
next_timestamp.tv_sec = next_timestamp.tv_usec / 1000000;
next_timestamp.tv_usec -= next_timestamp.tv_sec * 1000000;
return true;
}
[[nodiscard]] inline size_t bytes_left() const {
if(this->max_bytes <= 0) return (size_t) -1;
auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
if(this->current_index != current_span) return this->max_bytes;
if(this->bytes_send < this->max_bytes) return this->max_bytes - this->bytes_send;
return 0;
}
[[nodiscard]] inline std::chrono::milliseconds expected_writing_time(size_t bytes) const {
if(this->max_bytes <= 0) return std::chrono::milliseconds{0};
return std::chrono::seconds{bytes / (this->max_bytes * (1000 / kThrottleTimespanMs))};
}
};
/* all variables are locked via the state_mutex */
struct FileClient : std::enable_shared_from_this<FileClient> {
LocalFileTransfer* handle;
@ -200,8 +155,6 @@ namespace ts::server::file {
struct {
bool file_locked{false};
std::string absolute_path{};
int file_descriptor{0};
bool currently_processing{false};
@ -238,7 +191,9 @@ namespace ts::server::file {
std::chrono::system_clock::time_point disconnect_timeout{};
NetworkThrottle throttle;
networking::NetworkThrottle client_throttle{};
/* the right side is the server throttle */
networking::DualNetworkThrottle throttle{&client_throttle, &networking::NetworkThrottle::kNoThrottle};
pipes::SSL pipe_ssl{};
bool pipe_ssl_init{false};
@ -250,20 +205,13 @@ namespace ts::server::file {
} networking{};
struct {
size_t network_bytes_send{0};
size_t network_bytes_received{0};
networking::TransferStatistics network_send{};
networking::TransferStatistics network_received{};
size_t file_bytes_transferred{0};
networking::TransferStatistics file_transferred{};
/* used for delta statistics */
size_t last_network_bytes_send{0};
size_t last_network_bytes_received{0};
/* used for delta statistics */
size_t last_file_bytes_transferred{0};
size_t disk_bytes_read{0};
size_t disk_bytes_write{0};
networking::TransferStatistics disk_bytes_read{};
networking::TransferStatistics disk_bytes_write{};
} statistics{};
struct {
@ -327,8 +275,8 @@ namespace ts::server::file {
FILE_TOO_LARGE,
DISK_IS_READ_ONLY,
FILE_SIZE_MISMATCH,
FILE_SEEK_FAILED,
FILE_SIZE_MISMATCH,
FILE_IS_NOT_ACCESSIBLE,
@ -352,9 +300,8 @@ namespace ts::server::file {
/* FILE_TOO_LARGE */ "file is too large",
/* DISK_IS_READ_ONLY */ "disk is in read only mode",
/* FILE_SIZE_MISMATCH */ "file size mismatch",
/* FILE_SEEK_FAILED */ "failed to seek to target file offset",
/* FILE_SIZE_MISMATCH */ "file size miss match",
/* FILE_IS_NOT_ACCESSIBLE */ "file is not accessible"
};
@ -413,16 +360,18 @@ namespace ts::server::file {
}
std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
initialize_channel_transfer(Transfer::Direction direction, ServerId id, ChannelId channelId,
initialize_channel_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer>& server, ChannelId channelId,
const TransferInfo &info) override;
std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
initialize_icon_transfer(Transfer::Direction direction, ServerId id, const TransferInfo &info) override;
initialize_icon_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer>& server, const TransferInfo &info) override;
std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
initialize_avatar_transfer(Transfer::Direction direction, ServerId id, const TransferInfo &info) override;
initialize_avatar_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer>& server, const TransferInfo &info) override;
std::shared_ptr<ExecuteResponse<TransferActionError>> stop_transfer(transfer_id id, bool) override;
std::shared_ptr<ExecuteResponse<TransferListError, std::vector<ActiveFileTransfer>>> list_transfer() override;
std::shared_ptr<ExecuteResponse<TransferActionError>> stop_transfer(const std::shared_ptr<VirtualFileServer>& /* server */, transfer_id id, bool) override;
private:
enum struct DiskIOLoopState {
STOPPED,
@ -433,13 +382,14 @@ namespace ts::server::file {
};
filesystem::LocalFileSystem& file_system_;
std::atomic<transfer_id> current_transfer_id{0};
size_t max_concurrent_transfers{1024};
std::mt19937 transfer_random_token_generator{std::random_device{}()};
std::mutex result_notify_mutex{};
std::condition_variable result_notify_cv{};
std::mutex transfers_mutex{};
std::mutex transfer_create_mutex{};
std::deque<std::shared_ptr<FileClient>> transfers_{};
std::deque<std::shared_ptr<Transfer>> pending_transfers{};
@ -483,7 +433,7 @@ namespace ts::server::file {
}
std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
initialize_transfer(Transfer::Direction, ServerId, ChannelId, Transfer::TargetType, const TransferInfo &info);
initialize_transfer(Transfer::Direction, const std::shared_ptr<VirtualFileServer> &, ChannelId, Transfer::TargetType, const TransferInfo &info);
[[nodiscard]] NetworkingStartResult start_networking();
[[nodiscard]] DiskIOStartResult start_disk_io();
@ -529,6 +479,17 @@ namespace ts::server::file {
};
}
class LocalVirtualFileServer : public VirtualFileServer {
public:
explicit LocalVirtualFileServer(ServerId server_id, std::string unique_id) : VirtualFileServer{server_id, std::move(unique_id)} {}
void max_networking_upload_bandwidth(int64_t value) override;
void max_networking_download_bandwidth(int64_t value) override;
networking::NetworkThrottle upload_throttle{};
networking::NetworkThrottle download_throttle{};
};
class LocalFileProvider : public AbstractFileServer {
public:
LocalFileProvider();
@ -537,9 +498,14 @@ namespace ts::server::file {
[[nodiscard]] bool initialize(std::string& /* error */, const std::shared_ptr<pipes::SSL::Options>& /* ssl options */);
void finalize();
[[nodiscard]] std::string file_base_path() const override;
filesystem::AbstractProvider &file_system() override;
transfer::AbstractProvider &file_transfer() override;
std::shared_ptr<VirtualFileServer> register_server(ServerId /* server id */) override;
void unregister_server(ServerId /* server id */) override;
private:
filesystem::LocalFileSystem file_system_;
transfer::LocalFileTransfer file_transfer_;

188
file/local_server/LocalFileSystem.cpp
View File

@ -36,12 +36,12 @@ bool LocalFileSystem::initialize(std::string &error_message, const std::string &
void LocalFileSystem::finalize() {}
fs::path LocalFileSystem::server_path(ts::ServerId id) {
return fs::u8path(this->root_path_) / fs::u8path("server_" + std::to_string(id));
fs::path LocalFileSystem::server_path(const std::shared_ptr<VirtualFileServer> &server) {
return fs::u8path(this->root_path_) / fs::u8path("server_" + std::to_string(server->server_id()));
}
fs::path LocalFileSystem::server_channel_path(ts::ServerId sid, ts::ChannelId cid) {
return this->server_path(sid) / fs::u8path("channel_" + std::to_string(cid));
fs::path LocalFileSystem::server_channel_path(const std::shared_ptr<VirtualFileServer> &server, ts::ChannelId cid) {
return this->server_path(server) / fs::u8path("channel_" + std::to_string(cid));
}
bool LocalFileSystem::exceeds_base_path(const fs::path &base, const fs::path &target) {
@ -67,32 +67,32 @@ bool LocalFileSystem::is_any_file_locked(const fs::path &base, const std::string
return false;
}
std::string LocalFileSystem::target_file_path(FileCategory type, ts::ServerId sid, ts::ChannelId cid, const std::string &path) {
std::string LocalFileSystem::target_file_path(FileCategory type, const std::shared_ptr<VirtualFileServer> &server, ts::ChannelId cid, const std::string &path) {
fs::path target_path{};
switch (type) {
case FileCategory::CHANNEL:
target_path = this->server_channel_path(sid, cid) / path;
target_path = this->server_channel_path(server, cid) / path;
break;
case FileCategory::ICON:
target_path = this->server_path(sid) / "icons" / path;
target_path = this->server_path(server) / "icons" / path;
break;
case FileCategory::AVATAR:
target_path = this->server_path(sid) / "avatars" / path;
target_path = this->server_path(server) / "avatars" / path;
break;
}
return clnpath(fs::absolute(target_path).string());
}
std::string LocalFileSystem::absolute_avatar_path(ServerId sid, const std::string &path) {
std::string LocalFileSystem::absolute_avatar_path(const std::shared_ptr<VirtualFileServer> &sid, const std::string &path) {
return this->target_file_path(FileCategory::AVATAR, sid, 0, path);
}
std::string LocalFileSystem::absolute_icon_path(ServerId sid, const std::string &path) {
std::string LocalFileSystem::absolute_icon_path(const std::shared_ptr<VirtualFileServer> &sid, const std::string &path) {
return this->target_file_path(FileCategory::ICON, sid, 0, path);
}
std::string LocalFileSystem::absolute_channel_path(ServerId sid, ChannelId cid, const std::string &path) {
std::string LocalFileSystem::absolute_channel_path(const std::shared_ptr<VirtualFileServer> &sid, ChannelId cid, const std::string &path) {
return this->target_file_path(FileCategory::CHANNEL, sid, cid, path);
}
@ -107,7 +107,7 @@ void LocalFileSystem::unlock_file(const std::string &c_path) {
this->locked_files_.erase(std::remove_if(this->locked_files_.begin(), this->locked_files_.end(), [&](const auto& p) { return p == c_path; }), this->locked_files_.end());
}
std::shared_ptr<ExecuteResponse<ServerCommandError>> LocalFileSystem::initialize_server(ServerId id) {
std::shared_ptr<ExecuteResponse<ServerCommandError>> LocalFileSystem::initialize_server(const std::shared_ptr<VirtualFileServer> &id) {
auto path = this->server_path(id);
std::error_code error{};
@ -126,7 +126,7 @@ std::shared_ptr<ExecuteResponse<ServerCommandError>> LocalFileSystem::initialize
return response;
}
std::shared_ptr<ExecuteResponse<ServerCommandError>> LocalFileSystem::delete_server(ServerId id) {
std::shared_ptr<ExecuteResponse<ServerCommandError>> LocalFileSystem::delete_server(const std::shared_ptr<VirtualFileServer> &id) {
auto path = this->server_path(id);
std::error_code error{};
@ -217,19 +217,19 @@ std::shared_ptr<directory_query_response_t> LocalFileSystem::query_directory(con
return response;
}
std::shared_ptr<directory_query_response_t> LocalFileSystem::query_icon_directory(ServerId id) {
std::shared_ptr<directory_query_response_t> LocalFileSystem::query_icon_directory(const std::shared_ptr<VirtualFileServer> &id) {
return this->query_directory(this->server_path(id) / fs::u8path("icons"), "/", true);
}
std::shared_ptr<directory_query_response_t> LocalFileSystem::query_avatar_directory(ServerId id) {
std::shared_ptr<directory_query_response_t> LocalFileSystem::query_avatar_directory(const std::shared_ptr<VirtualFileServer> &id) {
return this->query_directory(this->server_path(id) / fs::u8path("avatars"), "/", true);
}
std::shared_ptr<directory_query_response_t> LocalFileSystem::query_channel_directory(ServerId id, ChannelId channelId, const std::string &path) {
std::shared_ptr<directory_query_response_t> LocalFileSystem::query_channel_directory(const std::shared_ptr<VirtualFileServer> &id, ChannelId channelId, const std::string &path) {
return this->query_directory(this->server_channel_path(id, channelId), path, false);
}
std::shared_ptr<ExecuteResponse<DirectoryModifyError>> LocalFileSystem::create_channel_directory(ServerId id, ChannelId channelId, const std::string &path) {
std::shared_ptr<ExecuteResponse<DirectoryModifyError>> LocalFileSystem::create_channel_directory(const std::shared_ptr<VirtualFileServer> &id, ChannelId channelId, const std::string &path) {
auto channel_path_root = this->server_channel_path(id, channelId);
std::error_code error{};
@ -256,20 +256,21 @@ std::shared_ptr<ExecuteResponse<DirectoryModifyError>> LocalFileSystem::create_c
return response;
}
std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::rename_channel_file(ServerId id, ChannelId channelId, const std::string &current_path_string, const std::string &new_path_string) {
std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::rename_channel_file(const std::shared_ptr<VirtualFileServer> &id, ChannelId channelId, const std::string &current_path_string, ChannelId targetChannelId, const std::string &new_path_string) {
auto channel_path_root = this->server_channel_path(id, channelId);
auto target_path_root = this->server_channel_path(id, targetChannelId);
std::error_code error{};
std::string locked_file{};
auto response = this->create_execute_response<FileModifyError>();
auto current_path = channel_path_root / fs::u8path(current_path_string);
auto target_path = channel_path_root / fs::u8path(new_path_string);
auto target_path = target_path_root / fs::u8path(new_path_string);
if(this->exceeds_base_path(channel_path_root, current_path)) {
response->emplace_fail(FileModifyErrorType::PATH_EXCEEDS_ROOT_PATH, "");
return response;
}
if(this->exceeds_base_path(channel_path_root, target_path)) {
if(this->exceeds_base_path(target_path_root, target_path)) {
response->emplace_fail(FileModifyErrorType::TARGET_PATH_EXCEEDS_ROOT_PATH, "");
return response;
}
@ -283,6 +284,15 @@ std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::rename_channe
return response;
}
if(!fs::exists(target_path.parent_path(), error)) {
if(!fs::create_directories(target_path.parent_path(), error)) {
response->emplace_fail(FileModifyErrorType::FAILED_TO_CREATE_DIRECTORIES, std::to_string(error.value()) + "/" + error.message());
return response;
}
} else if(error) {
logWarning(LOG_FT, "Failed to test for target directory existence for {}: {}/{}. Assuming it does exists", target_path.parent_path().string(), error.value(), error.message());
}
if(fs::exists(target_path, error)) {
response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, "");
return response;
@ -307,44 +317,126 @@ std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::rename_channe
return response;
}
std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::delete_file(const fs::path &base,
const std::string &path) {
std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> LocalFileSystem::delete_files(const fs::path &base,
const std::vector<std::string> &paths) {
std::error_code error{};
std::string locked_file{};
auto response = this->create_execute_response<FileModifyError>();
auto target_path = base / fs::u8path(path);
auto response = this->create_execute_response<FileDeleteError, FileDeleteResponse>();
if(fs::exists(target_path, error)) {
response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, "");
return response;
} else if(error) {
logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does exists.", target_path.string(), error.value(), error.message());
response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, "");
return response;
std::vector<FileDeleteResponse::DeleteResult> delete_results{};
for(const auto& path : paths) {
auto target_path = base / fs::u8path(path);
if(!fs::exists(target_path, error)) {
delete_results.emplace_back(FileDeleteResponse::StatusType::PATH_DOES_NOT_EXISTS, "");
continue;
} else if(error) {
logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does exists.", target_path.string(), error.value(), error.message());
delete_results.emplace_back(FileDeleteResponse::StatusType::PATH_DOES_NOT_EXISTS, "");
continue;
}
if(this->is_any_file_locked(base, path, locked_file)) {
delete_results.emplace_back(FileDeleteResponse::StatusType::SOME_FILES_ARE_LOCKED, locked_file);
continue;
}
if(!fs::remove_all(target_path, error) || error) {
delete_results.emplace_back(FileDeleteResponse::StatusType::FAILED_TO_DELETE_FILES, std::to_string(error.value()) + "/" + error.message());
continue;
}
delete_results.emplace_back(FileDeleteResponse::StatusType::SUCCESS, "");
}
if(this->is_any_file_locked(base, path, locked_file)) {
response->emplace_fail(FileModifyErrorType::SOME_FILES_ARE_LOCKED, locked_file);
return response;
}
if(!fs::remove(target_path, error) || error) {
response->emplace_fail(FileModifyErrorType::FAILED_TO_DELETE_FILES, std::to_string(error.value()) + "/" + error.message());
return response;
}
response->emplace_success();
response->emplace_success(FileDeleteResponse{delete_results});
return response;
}
std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::delete_channel_file(ServerId id, ChannelId channelId, const std::string &path) {
return this->delete_file(this->server_channel_path(id, channelId), path);
std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> LocalFileSystem::delete_channel_files(const std::shared_ptr<VirtualFileServer> &id, ChannelId channelId, const std::vector<std::string> &path) {
return this->delete_files(this->server_channel_path(id, channelId), path);
}
std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::delete_icon(ServerId id, const std::string &icon) {
return this->delete_file(this->server_path(id) / fs::u8path("icons"), icon);
std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> LocalFileSystem::delete_icons(const std::shared_ptr<VirtualFileServer> &id, const std::vector<std::string> &icon) {
return this->delete_files(this->server_path(id) / fs::u8path("icons"), icon);
}
std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::delete_avatar(ServerId id, const std::string &avatar) {
return this->delete_file(this->server_path(id) / fs::u8path("avatars"), avatar);
std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> LocalFileSystem::delete_avatars(const std::shared_ptr<VirtualFileServer> &id, const std::vector<std::string> &avatar) {
return this->delete_files(this->server_path(id) / fs::u8path("avatars"), avatar);
}
std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> LocalFileSystem::query_file_info(const fs::path &base,
const std::vector<std::string> &paths) {
std::error_code error{};
auto response = this->create_execute_response<FileInfoError, FileInfoResponse>();
std::vector<FileInfoResponse::FileInfo> file_infos{};
file_infos.reserve(paths.size());
for(const auto& path : paths) {
auto target_path = base / fs::u8path(path);
if(this->exceeds_base_path(base, target_path)) {
file_infos.emplace_back(FileInfoResponse::StatusType::PATH_EXCEEDS_ROOT_PATH, "", DirectoryEntry{});
continue;
}
if(!fs::exists(target_path, error)) {
file_infos.emplace_back(FileInfoResponse::StatusType::PATH_DOES_NOT_EXISTS, "", DirectoryEntry{});
continue;
} else if(error) {
logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", target_path.string(), error.value(), error.message());
file_infos.emplace_back(FileInfoResponse::StatusType::PATH_DOES_NOT_EXISTS, "", DirectoryEntry{});
continue;
}
auto status = fs::status(target_path, error);
if(error) {
logWarning(LOG_FT, "Failed to query file status for {} ({}/{}). Skipping entry for file info query.", target_path.string(), error.value(), error.message());
file_infos.emplace_back(FileInfoResponse::StatusType::FAILED_TO_QUERY_INFO, "", DirectoryEntry{});
continue;
}
if(status.type() == fs::file_type::directory) {
DirectoryEntry dentry{};
dentry.type = DirectoryEntry::DIRECTORY;
dentry.name = target_path.filename();
dentry.modified_at = fs::last_write_time(target_path, error);
if(error)
logWarning(LOG_FT, "Failed to query last write time for file {} ({}/{})", target_path.string(), error.value(), error.message());
dentry.size = 0;
file_infos.emplace_back(FileInfoResponse::StatusType::SUCCESS, "", dentry);
} else if(status.type() == fs::file_type::regular) {
DirectoryEntry dentry{};
dentry.type = DirectoryEntry::FILE;
dentry.name = target_path.filename();
dentry.modified_at = fs::last_write_time(target_path, error);
if(error)
logWarning(LOG_FT, "Failed to query last write time for file {} ({}/{}).", target_path.string(), error.value(), error.message());
dentry.size = fs::file_size(target_path, error);
if(error)
logWarning(LOG_FT, "Failed to query size for file {} ({}/{}).", target_path.string(), error.value(), error.message());
file_infos.emplace_back(FileInfoResponse::StatusType::SUCCESS, "", dentry);
} else {
logWarning(LOG_FT, "File info query contains an unknown file type for file {} ({}).", target_path.string(), (int) status.type());
file_infos.emplace_back(FileInfoResponse::StatusType::UNKNOWN_FILE_TYPE, "", DirectoryEntry{});
}
}
response->emplace_success(FileInfoResponse{file_infos});
return response;
}
std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> LocalFileSystem::query_channel_info(const std::shared_ptr<VirtualFileServer> &sid, ChannelId cid, const std::vector<std::string> &paths) {
return this->query_file_info(this->server_channel_path(sid, cid), paths);
}
std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> LocalFileSystem::query_icon_info(const std::shared_ptr<VirtualFileServer> &sid, const std::vector<std::string> &paths) {
return this->query_file_info(this->server_path(sid) / fs::u8path("icons"), paths);
}
std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse> > LocalFileSystem::query_avatar_info(const std::shared_ptr<VirtualFileServer> &sid, const std::vector<std::string> &paths) {
return this->query_file_info(this->server_path(sid) / fs::u8path("avatars"), paths);
}

180
file/local_server/LocalFileTransfer.cpp
View File

@ -8,7 +8,9 @@
#include <random>
#include "./LocalFileProvider.h"
#include "LocalFileProvider.h"
#include <experimental/filesystem>
namespace fs = std::experimental::filesystem;
using namespace ts::server::file;
using namespace ts::server::file::transfer;
@ -101,29 +103,59 @@ void LocalFileTransfer::stop() {
this->shutdown_client_worker();
}
std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> LocalFileTransfer::initialize_icon_transfer(Transfer::Direction direction, ServerId sid, const TransferInfo &info) {
return this->initialize_transfer(direction, sid, 0, Transfer::TARGET_TYPE_ICON, info);
std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
LocalFileTransfer::initialize_icon_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer> &server, const TransferInfo &info) {
return this->initialize_transfer(direction, server, 0, Transfer::TARGET_TYPE_ICON, info);
}
std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> LocalFileTransfer::initialize_avatar_transfer(Transfer::Direction direction, ServerId sid, const TransferInfo &info) {
return this->initialize_transfer(direction, sid, 0, Transfer::TARGET_TYPE_AVATAR, info);
std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
LocalFileTransfer::initialize_avatar_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer> &server, const TransferInfo &info) {
return this->initialize_transfer(direction, server, 0, Transfer::TARGET_TYPE_AVATAR, info);
}
std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> LocalFileTransfer::initialize_channel_transfer(Transfer::Direction direction, ServerId sid, ChannelId cid, const TransferInfo &info) {
return this->initialize_transfer(direction, sid, cid, Transfer::TARGET_TYPE_CHANNEL_FILE, info);
std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
LocalFileTransfer::initialize_channel_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer> &server, ChannelId cid, const TransferInfo &info) {
return this->initialize_transfer(direction, server, cid, Transfer::TARGET_TYPE_CHANNEL_FILE, info);
}
std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> LocalFileTransfer::initialize_transfer(
Transfer::Direction direction, ServerId sid, ChannelId cid,
Transfer::Direction direction, const std::shared_ptr<VirtualFileServer> &server, ChannelId cid,
Transfer::TargetType ttype,
const TransferInfo &info) {
auto response = this->create_execute_response<TransferInitError, std::shared_ptr<Transfer>>();
/* TODO: test for a transfer limit */
std::lock_guard clock{this->transfer_create_mutex};
if(info.max_concurrent_transfers > 0) {
std::unique_lock tlock{this->transfers_mutex};
{
auto transfers = std::count_if(this->transfers_.begin(), this->transfers_.end(), [&](const std::shared_ptr<FileClient>& client) {
return client->transfer && client->transfer->client_unique_id == info.client_unique_id;
});
transfers += std::count_if(this->pending_transfers.begin(), this->pending_transfers.end(), [&](const std::shared_ptr<Transfer>& transfer) {
return transfer->client_unique_id == info.client_unique_id;
});
if(transfers >= info.max_concurrent_transfers) {
response->emplace_fail(TransferInitError::CLIENT_TOO_MANY_TRANSFERS, std::to_string(transfers));
return response;
}
}
{
auto server_transfers = this->pending_transfers.size();
server_transfers += std::count_if(this->transfers_.begin(), this->transfers_.end(), [&](const std::shared_ptr<FileClient>& client) {
return client->transfer;
});
if(server_transfers >= this->max_concurrent_transfers) {
response->emplace_fail(TransferInitError::SERVER_TOO_MANY_TRANSFERS, std::to_string(server_transfers));
return response;
}
}
}
auto transfer = std::make_shared<Transfer>();
transfer->server_transfer_id = ++this->current_transfer_id;
transfer->server_id = sid;
transfer->server_transfer_id = server->generate_transfer_id();
transfer->server = server;
transfer->channel_id = cid;
transfer->target_type = ttype;
transfer->direction = direction;
@ -142,6 +174,8 @@ std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> L
transfer->file_offset = info.file_offset;
transfer->expected_file_size = info.expected_file_size;
transfer->max_bandwidth = info.max_bandwidth;
transfer->client_unique_id = info.client_unique_id;
transfer->client_id = info.client_id;
constexpr static std::string_view kTokenCharacters{"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"};
for(auto& c : transfer->transfer_key)
@ -152,6 +186,70 @@ std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> L
transfer->initialized_timestamp = std::chrono::system_clock::now();
{
std::string absolute_path{};
switch (transfer->target_type) {
case Transfer::TARGET_TYPE_AVATAR:
absolute_path = this->file_system_.absolute_avatar_path(transfer->server, transfer->target_file_path);
logMessage(LOG_FT, "Initialized avatar transfer for avatar \"{}\" ({} bytes, transferring {} bytes).", transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset);
break;
case Transfer::TARGET_TYPE_ICON:
absolute_path = this->file_system_.absolute_icon_path(transfer->server, transfer->target_file_path);
logMessage(LOG_FT, "Initialized icon transfer for icon \"{}\" ({} bytes, transferring {} bytes).",
transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset);
break;
case Transfer::TARGET_TYPE_CHANNEL_FILE:
absolute_path = this->file_system_.absolute_channel_path(transfer->server, transfer->channel_id, transfer->target_file_path);
logMessage(LOG_FT, "Initialized channel transfer for file \"{}/{}\" ({} bytes, transferring {} bytes).",
transfer->channel_id, transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset);
break;
case Transfer::TARGET_TYPE_UNKNOWN:
default:
response->emplace_fail(TransferInitError::INVALID_FILE_TYPE, "");
return response;
}
transfer->absolute_file_path = absolute_path;
const auto root_path_length = this->file_system_.root_path().size();
if(root_path_length < absolute_path.size())
transfer->relative_file_path = absolute_path.substr(root_path_length);
else
transfer->relative_file_path = "error";
transfer->file_name = fs::u8path(absolute_path).filename();
}
if(direction == Transfer::DIRECTION_DOWNLOAD) {
auto path = fs::u8path(transfer->absolute_file_path);
std::error_code error{};
if(!fs::exists(path, error)) {
response->emplace_fail(TransferInitError::FILE_DOES_NOT_EXISTS, "");
return response;
} else if(error) {
logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", transfer->absolute_file_path, error.value(), error.message());
response->emplace_fail(TransferInitError::FILE_DOES_NOT_EXISTS, "");
return response;
}
auto status = fs::status(path, error);
if(error) {
logWarning(LOG_FT, "Failed to status for file at {}: {}. Ignoring file transfer.", transfer->absolute_file_path, error.value(), error.message());
response->emplace_fail(TransferInitError::IO_ERROR, "stat");
return response;
}
if(status.type() != fs::file_type::regular) {
response->emplace_fail(TransferInitError::FILE_IS_NOT_A_FILE, "");
return response;
}
transfer->expected_file_size = fs::file_size(path, error);
if(error) {
logWarning(LOG_FT, "Failed to get file size for file at {}: {}. Ignoring file transfer.", transfer->absolute_file_path, error.value(), error.message());
response->emplace_fail(TransferInitError::IO_ERROR, "file_size");
return response;
}
}
{
std::lock_guard tlock{this->transfers_mutex};
this->pending_transfers.push_back(transfer);
@ -164,7 +262,7 @@ std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> L
return response;
}
std::shared_ptr<ExecuteResponse<TransferActionError>> LocalFileTransfer::stop_transfer(transfer_id id, bool flush) {
std::shared_ptr<ExecuteResponse<TransferActionError>> LocalFileTransfer::stop_transfer(const std::shared_ptr<VirtualFileServer>& server, transfer_id id, bool flush) {
auto response = this->create_execute_response<TransferActionError>();
std::shared_ptr<Transfer> transfer{};
@ -174,13 +272,13 @@ std::shared_ptr<ExecuteResponse<TransferActionError>> LocalFileTransfer::stop_tr
std::lock_guard tlock{this->transfers_mutex};
auto ct_it = std::find_if(this->transfers_.begin(), this->transfers_.end(), [&](const std::shared_ptr<FileClient>& t) {
return t->transfer && t->transfer->server_transfer_id == id;
return t->transfer && t->transfer->server_transfer_id == id && t->transfer->server == server;
});
if(ct_it != this->transfers_.end())
connected_transfer = *ct_it;
else {
auto t_it = std::find_if(this->pending_transfers.begin(), this->pending_transfers.end(), [&](const std::shared_ptr<Transfer>& t) {
return t->server_transfer_id == id;
return t->server_transfer_id == id && t->server == server;
});
if(t_it != this->pending_transfers.end()) {
transfer = *t_it;
@ -209,4 +307,60 @@ std::shared_ptr<ExecuteResponse<TransferActionError>> LocalFileTransfer::stop_tr
response->emplace_success();
return response;
}
inline void apply_transfer_info(const std::shared_ptr<Transfer>& transfer, ActiveFileTransfer& info) {
info.server_transfer_id = transfer->server_transfer_id;
info.client_transfer_id = transfer->client_transfer_id;
info.direction = transfer->direction;
info.client_id = transfer->client_id;
info.client_unique_id = transfer->client_unique_id;
info.file_path = transfer->relative_file_path;
info.file_name = transfer->file_name;
info.expected_size = transfer->expected_file_size;
}
std::shared_ptr<ExecuteResponse<TransferListError, std::vector<ActiveFileTransfer>>> LocalFileTransfer::list_transfer() {
std::vector<ActiveFileTransfer> transfer_infos{};
auto response = this->create_execute_response<TransferListError, std::vector<ActiveFileTransfer>>();
std::unique_lock tlock{this->transfers_mutex};
auto awaiting_transfers = this->pending_transfers;
auto running_transfers = this->transfers_;
tlock.unlock();
transfer_infos.reserve(awaiting_transfers.size() + running_transfers.size());
for(const auto& transfer : awaiting_transfers) {
ActiveFileTransfer info{};
apply_transfer_info(transfer, info);
info.size_done = transfer->file_offset;
info.status = ActiveFileTransfer::NOT_STARTED;
info.runtime = std::chrono::milliseconds{0};
info.average_speed = 0;
info.current_speed = 0;
transfer_infos.push_back(info);
}
for(const auto& client : running_transfers) {
auto transfer = client->transfer;
if(!transfer) continue;
ActiveFileTransfer info{};
apply_transfer_info(transfer, info);
info.size_done = transfer->file_offset + client->statistics.file_transferred.total_bytes;
info.status = ActiveFileTransfer::RUNNING;
info.runtime = std::chrono::floor<std::chrono::milliseconds>(std::chrono::system_clock::now() - client->timings.key_received);
info.average_speed = client->statistics.file_transferred.average_bandwidth();
info.current_speed = client->statistics.file_transferred.current_bandwidth();
transfer_infos.push_back(info);
}
response->emplace_success(std::move(transfer_infos));
return response;
}

20
file/local_server/LocalFileTransferClientWorker.cpp
View File

@ -46,7 +46,7 @@ void LocalFileTransfer::disconnect_client(const std::shared_ptr<FileClient> &cli
client->state = flush ? FileClient::STATE_DISCONNECTING : FileClient::STATE_DISCONNECTED;
client->timings.disconnecting = std::chrono::system_clock::now();
if(flush) {
const auto network_flush_time = client->networking.throttle.expected_writing_time(client->buffer.bytes) + std::chrono::seconds{10};
const auto network_flush_time = client->networking.client_throttle.expected_writing_time(client->buffer.bytes) + std::chrono::seconds{10};
if(!client->transfer) {
del_ev_noblock(client->networking.event_read);
@ -104,6 +104,9 @@ void LocalFileTransfer::dispatch_loop_client_worker(void *ptr_transfer) {
case FileClient::STATE_DISCONNECTING:
if(transfer->transfer && transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD)
break; /* we're still transferring (sending data) */
continue;
case FileClient::STATE_AWAITING_KEY:
case FileClient::STATE_DISCONNECTED:
default:
continue;
}
@ -182,6 +185,7 @@ void LocalFileTransfer::dispatch_loop_client_worker(void *ptr_transfer) {
case FileClient::STATE_DISCONNECTING:
logMessage(LOG_FT, "{} Failed to flush connection. Dropping client", client->log_prefix());
break;
case FileClient::STATE_DISCONNECTED:
default:
break;
}
@ -205,17 +209,17 @@ void LocalFileTransfer::report_transfer_statistics(const std::shared_ptr<FileCli
TransferStatistics stats{};
stats.network_bytes_send = client->statistics.network_bytes_send;
stats.network_bytes_received = client->statistics.network_bytes_received;
stats.file_bytes_transferred = client->statistics.file_bytes_transferred;
stats.network_bytes_send = client->statistics.network_send.total_bytes;
stats.network_bytes_received = client->statistics.network_received.total_bytes;
stats.file_bytes_transferred = client->statistics.file_transferred.total_bytes;
stats.delta_network_bytes_received = stats.network_bytes_received - std::exchange(client->statistics.last_network_bytes_received, stats.network_bytes_received);
stats.delta_network_bytes_send = stats.network_bytes_send - std::exchange(client->statistics.last_network_bytes_send, stats.network_bytes_send);
stats.delta_network_bytes_received = client->statistics.network_received.take_delta();
stats.delta_network_bytes_send = client->statistics.network_received.take_delta();
stats.delta_file_bytes_transferred = stats.file_bytes_transferred - std::exchange(client->statistics.last_file_bytes_transferred, stats.file_bytes_transferred);
stats.delta_file_bytes_transferred = client->statistics.file_transferred.take_delta();
stats.file_start_offset = client->transfer->file_offset;
stats.file_current_offset = client->statistics.file_bytes_transferred + client->transfer->file_offset;
stats.file_current_offset = client->statistics.file_transferred.total_bytes + client->transfer->file_offset;
stats.file_total_size = client->transfer->expected_file_size;
callback(client->transfer, stats);

42
file/local_server/LocalFileTransferDisk.cpp
View File

@ -107,17 +107,18 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr
assert(!file_data.file_descriptor);
assert(!file_data.next_client);
auto absolute_path = transfer->transfer->absolute_file_path;
{
unsigned int open_flags{0};
if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
open_flags = O_RDONLY;
std::error_code fs_error{};
if(file_data.absolute_path.empty() || !fs::exists(file_data.absolute_path, fs_error)) {
if(absolute_path.empty() || !fs::exists(absolute_path, fs_error)) {
result = FileInitializeResult::FILE_DOES_NOT_EXISTS;
goto error_exit;
} else if(fs_error) {
logWarning(LOG_FT, "{} Failed to check for file existence of {}: {}/{}", transfer->log_prefix(), file_data.absolute_path, fs_error.value(), fs_error.message());
logWarning(LOG_FT, "{} Failed to check for file existence of {}: {}/{}", transfer->log_prefix(), absolute_path, fs_error.value(), fs_error.message());
result = FileInitializeResult::FILE_SYSTEM_ERROR;
goto error_exit;
}
@ -129,7 +130,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr
return FileInitializeResult::INVALID_TRANSFER_DIRECTION;
}
file_data.file_descriptor = open(file_data.absolute_path.c_str(), (int) open_flags, 0644);
file_data.file_descriptor = open(absolute_path.c_str(), (int) open_flags, 0644);
if(file_data.file_descriptor <= 0) {
const auto errno_ = errno;
switch (errno_) {
@ -137,7 +138,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr
result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
break;
case EDQUOT:
logWarning(LOG_FT, "{} Disk inode limit has been reached. Failed to start file transfer for file {}", transfer->log_prefix(), file_data.absolute_path);
logWarning(LOG_FT, "{} Disk inode limit has been reached. Failed to start file transfer for file {}", transfer->log_prefix(), absolute_path);
result = FileInitializeResult::FILE_SYSTEM_ERROR;
break;
case EISDIR:
@ -156,7 +157,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr
result = FileInitializeResult::DISK_IS_READ_ONLY;
break;
default:
logWarning(LOG_FT, "{} Failed to start file transfer for file {}: {}/{}", transfer->log_prefix(), file_data.absolute_path, errno_, strerror(errno_));
logWarning(LOG_FT, "{} Failed to start file transfer for file {}: {}/{}", transfer->log_prefix(), absolute_path, errno_, strerror(errno_));
result = FileInitializeResult::FILE_SYSTEM_ERROR;
break;
}
@ -164,7 +165,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr
}
}
this->file_system_.lock_file(transfer->file.absolute_path);
this->file_system_.lock_file(absolute_path);
transfer->file.file_locked = true;
if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) {
@ -172,7 +173,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr
const auto errno_ = errno;
switch (errno_) {
case EACCES:
logWarning(LOG_FT, "{} File {} got inaccessible on truncating, but not on opening.", transfer->log_prefix(), file_data.absolute_path);
logWarning(LOG_FT, "{} File {} got inaccessible on truncating, but not on opening.", transfer->log_prefix(), absolute_path);
result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
goto error_exit;
@ -181,16 +182,16 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr
goto error_exit;
case EIO:
logWarning(LOG_FT, "{} A disk IO error occurred while resizing file {}.", transfer->log_prefix(), file_data.absolute_path);
logWarning(LOG_FT, "{} A disk IO error occurred while resizing file {}.", transfer->log_prefix(), absolute_path);
result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
goto error_exit;
case EROFS:
logWarning(LOG_FT, "{} Failed to resize file {} because disk is in read only mode.", transfer->log_prefix(), file_data.absolute_path);
logWarning(LOG_FT, "{} Failed to resize file {} because disk is in read only mode.", transfer->log_prefix(), absolute_path);
result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
goto error_exit;
default:
debugMessage(LOG_FT, "{} Failed to truncate file {}: {}/{}. Trying to upload file anyways.", transfer->log_prefix(), file_data.absolute_path, errno_, strerror(errno_));
debugMessage(LOG_FT, "{} Failed to truncate file {}: {}/{}. Trying to upload file anyways.", transfer->log_prefix(), absolute_path, errno_, strerror(errno_));
}
}
} else if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
@ -218,7 +219,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr
return FileInitializeResult::SUCCESS;
error_exit:
if(std::exchange(transfer->file.file_locked, false))
this->file_system_.unlock_file(transfer->file.absolute_path);
this->file_system_.unlock_file(absolute_path);
if(file_data.file_descriptor > 0)
::close(file_data.file_descriptor);
@ -230,6 +231,7 @@ FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr
void LocalFileTransfer::finalize_file_io(const std::shared_ptr<FileClient> &transfer,
std::unique_lock<std::shared_mutex> &state_lock) {
assert(state_lock.owns_lock());
auto absolute_path = transfer->transfer->absolute_file_path;
auto& file_data = transfer->file;
@ -268,7 +270,7 @@ void LocalFileTransfer::finalize_file_io(const std::shared_ptr<FileClient> &tran
state_lock.lock();
if(std::exchange(file_data.file_locked, false))
this->file_system_.unlock_file(file_data.absolute_path);
this->file_system_.unlock_file(absolute_path);
if(file_data.file_descriptor > 0)
::close(file_data.file_descriptor);
@ -329,7 +331,7 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr<FileClient> &clien
if(written <= 0) {
if(written == 0) {
/* EOF, how the hell is this event possible?! */
auto offset_written = client->statistics.disk_bytes_write + client->transfer->file_offset;
auto offset_written = client->statistics.disk_bytes_write.total_bytes + client->transfer->file_offset;
auto aoffset = lseek(client->file.file_descriptor, 0, SEEK_CUR);
logError(LOG_FT, "{} Received unexpected file write EOF. EOF received at {} but expected {}. Actual file offset: {}. Closing transfer.",
client->log_prefix(), offset_written, client->transfer->expected_file_size, aoffset);
@ -349,7 +351,7 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr<FileClient> &clien
break;
}
auto offset_written = client->statistics.disk_bytes_write + client->transfer->file_offset;
auto offset_written = client->statistics.disk_bytes_write.total_bytes + client->transfer->file_offset;
auto aoffset = lseek(client->file.file_descriptor, 0, SEEK_CUR);
logError(LOG_FT, "{} Received write to disk IO error. Write pointer is at {} of {}. Actual file offset: {}. Closing transfer.",
client->log_prefix(), offset_written, client->transfer->expected_file_size, aoffset);
@ -389,7 +391,7 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr<FileClient> &clien
(void) buffer_left_size; /* trick my IDE here a bit */
}
client->statistics.disk_bytes_write += written;
client->statistics.disk_bytes_write.increase_bytes(written);
}
}
@ -418,7 +420,7 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr<FileClient> &clien
if(read <= 0) {
if(read == 0) {
/* EOF */
auto offset_send = client->statistics.disk_bytes_read + client->transfer->file_offset;
auto offset_send = client->statistics.disk_bytes_read.total_bytes + client->transfer->file_offset;
if(client->transfer->expected_file_size == offset_send) {
debugMessage(LOG_FT, "{} Finished file reading. Flushing and disconnecting transfer. Reading took {} seconds.",
client->log_prefix(), duration_to_string(std::chrono::system_clock::now() - client->timings.key_received));
@ -442,7 +444,7 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr<FileClient> &clien
return;
}
logWarning(LOG_FT, "{} Failed to read from file {} ({}/{}). Aborting transfer.", client->log_prefix(), client->file.absolute_path, errno, strerror(errno));
logWarning(LOG_FT, "{} Failed to read from file {} ({}/{}). Aborting transfer.", client->log_prefix(), client->transfer->absolute_file_path, errno, strerror(errno));
this->report_transfer_statistics(client);
if(auto callback{client->handle->callback_transfer_aborted}; callback)
@ -456,12 +458,12 @@ void LocalFileTransfer::execute_disk_io(const std::shared_ptr<FileClient> &clien
return;
} else {
auto buffer_full = client->send_file_bytes(buffer, read);
client->statistics.disk_bytes_read += read;
client->statistics.file_bytes_transferred += read;
client->statistics.disk_bytes_read.increase_bytes(read);
client->statistics.file_transferred.increase_bytes(read);
std::shared_lock slock{client->state_mutex};
if(buffer_full) {
logMessage(LOG_FT, "{} Stopping buffering from disk. Buffer full ({}bytes)", client->log_prefix(), client->buffer.bytes);
//logTrace(LOG_FT, "{} Stopping buffering from disk. Buffer full ({}bytes)", client->log_prefix(), client->buffer.bytes);
break;
}

67
file/local_server/LocalFileTransferNetwork.cpp
View File

@ -452,6 +452,7 @@ void LocalFileTransfer::callback_transfer_network_read(int fd, short events, voi
const auto max_read_buffer = transfer->networking.throttle.bytes_left();
if(!max_read_buffer) break; /* network throttle */
errno = 0;
auto read = ::recv(fd, buffer, std::min(buffer_size, std::min(max_read_buffer, transfer->networking.max_read_buffer_size)), MSG_NOSIGNAL | MSG_DONTWAIT);
//logTrace(0, "Read {}, max {} | {}", read, std::min(buffer_size, std::min(max_read_buffer, transfer->networking.max_read_buffer_size)), max_read_buffer);
if(read <= 0) {
@ -471,10 +472,10 @@ void LocalFileTransfer::callback_transfer_network_read(int fd, short events, voi
/* since we're counting the bytes directly on read, a disconnect would mean that not all data has been transferred */
if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) {
logMessage(LOG_FT, "{} Disconnected while receiving file. Received {} out of {} bytes",
transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset);
transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset);
} else if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
logMessage(LOG_FT, "{} Disconnected while sending file. Send {} out of {} bytes",
transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset);
transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset);
}
transfer->handle->report_transfer_statistics(transfer->shared_from_this());
@ -482,6 +483,8 @@ void LocalFileTransfer::callback_transfer_network_read(int fd, short events, voi
callback(transfer->transfer, { TransferError::UNEXPECTED_CLIENT_DISCONNECT, "" });
break;
}
case FileClient::STATE_DISCONNECTING:
case FileClient::STATE_DISCONNECTED:
default:
break;
}
@ -506,16 +509,18 @@ void LocalFileTransfer::callback_transfer_network_read(int fd, short events, voi
/* since we're counting the bytes directly on read, a disconnect would mean that not all data has been transferred */
if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) {
logMessage(LOG_FT, "{} Received read error while receiving file. Received {} out of {} bytes: {}/{}",
transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset, errno, strerror(errno));
transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset, errno, strerror(errno));
} else if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
logMessage(LOG_FT, "{} Received read error while sending file. Send {} out of {} bytes: {}/{}",
transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset, errno, strerror(errno));
transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset, errno, strerror(errno));
}
transfer->handle->report_transfer_statistics(transfer->shared_from_this());
if(auto callback{transfer->handle->callback_transfer_aborted}; callback)
callback(transfer->transfer, { TransferError::NETWORK_IO_ERROR, strerror(errno) });
break;
case FileClient::STATE_DISCONNECTING:
case FileClient::STATE_DISCONNECTED:
default:
break;
}
@ -524,7 +529,7 @@ void LocalFileTransfer::callback_transfer_network_read(int fd, short events, voi
return;
} else {
transfer->timings.last_read = std::chrono::system_clock::now();
transfer->statistics.network_bytes_received += read;
transfer->statistics.network_received.increase_bytes(read);
bool throttle_read = transfer->networking.throttle.increase_bytes(read);
if(transfer->state != FileClient::STATE_AWAITING_KEY && !(transfer->state == FileClient::STATE_TRANSFERRING && transfer->transfer->direction == Transfer::DIRECTION_UPLOAD)) {
@ -616,7 +621,7 @@ void LocalFileTransfer::callback_transfer_network_write(int fd, short events, vo
if(written == 0) {
/* EOF, how the hell is this event possible?! (Read should already catch it) */
logError(LOG_FT, "{} Client disconnected unexpectedly on write. Send {} bytes out of {}.",
transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset);
transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset);
transfer->handle->report_transfer_statistics(transfer->shared_from_this());
if(auto callback{transfer->handle->callback_transfer_aborted}; callback)
@ -633,7 +638,7 @@ void LocalFileTransfer::callback_transfer_network_write(int fd, short events, vo
}
logError(LOG_FT, "{} Received network write error. Send {} bytes out of {}. Closing transfer.",
transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset);
transfer->log_prefix(), transfer->statistics.file_transferred.total_bytes, transfer->transfer->expected_file_size - transfer->transfer->file_offset);
transfer->handle->report_transfer_statistics(transfer->shared_from_this());
if(auto callback{transfer->handle->callback_transfer_aborted}; callback)
@ -668,7 +673,7 @@ void LocalFileTransfer::callback_transfer_network_write(int fd, short events, vo
}
transfer->timings.last_write = std::chrono::system_clock::now();
transfer->statistics.network_bytes_send += written;
transfer->statistics.network_send.increase_bytes(written);
if(transfer->networking.throttle.increase_bytes(written))
break; /* we've to slow down */
@ -678,7 +683,7 @@ void LocalFileTransfer::callback_transfer_network_write(int fd, short events, vo
if(buffer_left_size > 0)
transfer->add_network_write_event(false);
else if(transfer->state == FileClient::STATE_DISCONNECTING) {
if(transfer->transfer && transfer->statistics.file_bytes_transferred + transfer->transfer->file_offset == transfer->transfer->expected_file_size) {
if(transfer->transfer && transfer->statistics.file_transferred.total_bytes + transfer->transfer->file_offset == transfer->transfer->expected_file_size) {
logMessage(LOG_FT, "{} Finished file transfer within {}. Closing connection.", transfer->log_prefix(), duration_to_string(std::chrono::system_clock::now() - transfer->timings.key_received));
transfer->handle->report_transfer_statistics(transfer->shared_from_this());
if(auto callback{transfer->handle->callback_transfer_finished}; callback)
@ -783,6 +788,7 @@ size_t LocalFileTransfer::handle_transfer_read_raw(const std::shared_ptr<FileCli
logMessage(LOG_FT, "{} Disconnecting client because we failed to open the target file.", client->log_prefix());
break;
case TransferKeyApplyResult::INTERNAL_ERROR:
default:
this->report_transfer_statistics(client);
if(auto callback{this->callback_transfer_aborted}; client->transfer && callback)
@ -881,6 +887,7 @@ size_t LocalFileTransfer::handle_transfer_read(const std::shared_ptr<FileClient>
response.setHeader("x-error-message", { "unknown key" });
goto send_response_exit;
case TransferKeyApplyResult::INTERNAL_ERROR:
default:
this->report_transfer_statistics(client);
if(auto callback{this->callback_transfer_aborted}; client->transfer && callback)
@ -1044,37 +1051,19 @@ TransferKeyApplyResult LocalFileTransfer::handle_transfer_key_provided(const std
if(!client->transfer)
return TransferKeyApplyResult::UNKNOWN_KEY;
{
std::string absolute_path{};
auto transfer = client->transfer;
switch (transfer->target_type) {
case Transfer::TARGET_TYPE_AVATAR:
absolute_path = this->file_system_.absolute_avatar_path(transfer->server_id, transfer->target_file_path);
logMessage(LOG_FT, "{} Initialized avatar transfer for avatar \"{}\" ({} bytes, transferring {} bytes).",
client->log_prefix(), transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset);
break;
case Transfer::TARGET_TYPE_ICON:
absolute_path = this->file_system_.absolute_icon_path(transfer->server_id, transfer->target_file_path);
logMessage(LOG_FT, "{} Initialized icon transfer for icon \"{}\" ({} bytes, transferring {} bytes).",
client->log_prefix(), transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset);
break;
case Transfer::TARGET_TYPE_CHANNEL_FILE:
absolute_path = this->file_system_.absolute_channel_path(transfer->server_id, transfer->channel_id, transfer->target_file_path);
logMessage(LOG_FT, "{} Initialized channel transfer for file \"{}/{}\" ({} bytes, transferring {} bytes).",
client->log_prefix(), transfer->channel_id, transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset);
break;
default:
logError(LOG_FT, "{} Tried to initialize client with unknown file target type ({}). Dropping transfer.", client->log_prefix(), (int) transfer->target_type);
error_detail = "invalid transfer target type";
return TransferKeyApplyResult::INTERNAL_ERROR;
}
debugMessage(LOG_FT, "{} Absolute file path: {}", client->log_prefix(), absolute_path);
client->file.absolute_path = absolute_path;
if(client->transfer->direction == Transfer::DIRECTION_UPLOAD) {
auto server = dynamic_pointer_cast<LocalVirtualFileServer>(client->transfer->server);
assert(server);
client->networking.throttle.right = &server->upload_throttle;
} else if(client->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
auto server = dynamic_pointer_cast<LocalVirtualFileServer>(client->transfer->server);
assert(server);
client->networking.throttle.right = &server->download_throttle;
}
if(client->transfer->max_bandwidth > 0) {
debugMessage(LOG_FT, "{} Limit network bandwidth to {} bytes/second", client->log_prefix(), client->transfer->max_bandwidth);
client->networking.throttle.set_max_bandwidth(client->transfer->max_bandwidth);
debugMessage(LOG_FT, "{} Limit network bandwidth especially for the client to {} bytes/second", client->log_prefix(), client->transfer->max_bandwidth);
client->networking.client_throttle.set_max_bandwidth(client->transfer->max_bandwidth);
}
client->networking.max_read_buffer_size = (size_t) -1; /* limit max bandwidth via throttle */
@ -1102,10 +1091,10 @@ TransferKeyApplyResult LocalFileTransfer::handle_transfer_key_provided(const std
}
TransferUploadRawResult LocalFileTransfer::handle_transfer_upload_raw(const std::shared_ptr<FileClient> &client, const char *buffer, size_t length) {
client->statistics.file_bytes_transferred += length;
client->statistics.file_transferred.increase_bytes(length);
bool transfer_finished{false};
auto writte_offset = client->statistics.file_bytes_transferred + client->transfer->file_offset;
auto writte_offset = client->statistics.file_transferred.total_bytes + client->transfer->file_offset;
if(writte_offset > client->transfer->expected_file_size) {
logMessage(LOG_FT, "{} Client send {} too many bytes (Transfer length was {}). Dropping them, flushing the disk IO and closing the transfer.", client->log_prefix(), writte_offset - client->transfer->expected_file_size, duration_to_string(std::chrono::system_clock::now() - client->timings.key_received));
length -= writte_offset - client->transfer->expected_file_size;

9
file/local_server/NetTools.cpp Normal file
View File

@ -0,0 +1,9 @@
//
// Created by WolverinDEV on 12/05/2020.
//
#include "./NetTools.h"
using namespace ts::server::file::networking;
NetworkThrottle NetworkThrottle::kNoThrottle{-1};

164
file/local_server/NetTools.h Normal file
View File

@ -0,0 +1,164 @@
#pragma once
#include <cstddef>
#include <chrono>
#include <mutex>
#include <cassert>
#include <misc/spin_mutex.h>
#include <numeric>
namespace ts::server::file::networking {
struct NetworkThrottle {
constexpr static auto kThrottleTimespanMs{250};
typedef uint8_t span_t;
static NetworkThrottle kNoThrottle;
ssize_t max_bytes{0};
span_t current_index{0};
size_t bytes_send{0};
mutable spin_mutex mutex{};
inline bool increase_bytes(size_t bytes) {
auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
std::lock_guard slock{this->mutex};
if(this->current_index != current_span) {
this->current_index = current_span;
this->bytes_send = bytes;
} else {
this->bytes_send += bytes;
}
return this->max_bytes > 0 && this->bytes_send >= this->max_bytes;
}
inline void set_max_bandwidth(ssize_t bytes_per_second) {
std::lock_guard slock{this->mutex};
if(bytes_per_second <= 0)
this->max_bytes = -1;
else
this->max_bytes = bytes_per_second * kThrottleTimespanMs / 1000;
}
[[nodiscard]] inline bool should_throttle(timeval& next_timestamp) {
if(this->max_bytes <= 0) return false;
auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
std::lock_guard slock{this->mutex};
if(this->max_bytes <= 0) return false; /* we've to test here again, else out arithmetic will fail */
if(this->current_index != current_span) return false;
if(this->bytes_send < this->max_bytes) return false;
next_timestamp.tv_usec = (kThrottleTimespanMs - current_ms % kThrottleTimespanMs) * 1000;
next_timestamp.tv_sec = next_timestamp.tv_usec / 1000000;
next_timestamp.tv_usec -= next_timestamp.tv_sec * 1000000;
return true;
}
[[nodiscard]] inline size_t bytes_left() const {
if(this->max_bytes <= 0) return (size_t) -1;
auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
std::lock_guard slock{this->mutex};
if(this->max_bytes <= 0) return false; /* we've to test here again, else out arithmetic will fail */
if(this->current_index != current_span) return this->max_bytes;
if(this->bytes_send < this->max_bytes) return this->max_bytes - this->bytes_send;
return 0;
}
[[nodiscard]] inline std::chrono::milliseconds expected_writing_time(size_t bytes) const {
std::lock_guard slock{this->mutex};
if(this->max_bytes <= 0) return std::chrono::milliseconds{0};
return std::chrono::seconds{bytes / (this->max_bytes * (1000 / kThrottleTimespanMs))};
}
};
struct DualNetworkThrottle {
NetworkThrottle *left, *right;
explicit DualNetworkThrottle(NetworkThrottle* left, NetworkThrottle* right) : left{left}, right{right} {
assert(left);
assert(right);
}
[[nodiscard]] inline size_t bytes_left() const {
return std::min(this->left->bytes_left(), this->right->bytes_left());
}
[[nodiscard]] inline std::chrono::milliseconds expected_writing_time(size_t bytes) const {
return std::max(this->left->expected_writing_time(bytes), this->right->expected_writing_time(bytes));
}
[[nodiscard]] inline bool should_throttle(timeval& next_timestamp) const {
bool throttle = false;
timeval right_timestamp{};
throttle |= this->left->should_throttle(next_timestamp);
throttle |= this->right->should_throttle(right_timestamp);
if(!throttle) return false;
if(right_timestamp.tv_sec > next_timestamp.tv_sec || (right_timestamp.tv_sec == next_timestamp.tv_sec && right_timestamp.tv_usec > next_timestamp.tv_usec))
next_timestamp = right_timestamp;
return true;
}
inline bool increase_bytes(size_t bytes) {
bool result = false;
result |= this->left->increase_bytes(bytes);
result |= this->right->increase_bytes(bytes);
return result;
}
};
struct TransferStatistics {
constexpr static auto kMeasureTimespanMs{1000};
constexpr static auto kAverageTimeCount{60};
typedef uint8_t span_t;
size_t total_bytes{0};
size_t delta_bytes{0}; /* used for statistics propagation */
span_t span_index{0};
size_t span_bytes{0};
std::array<size_t, kAverageTimeCount> history{};
spin_mutex mutex{};
inline void increase_bytes(size_t bytes) {
auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
auto current_span = (span_t) (current_ms / kMeasureTimespanMs);
std::lock_guard slock{this->mutex};
this->total_bytes += bytes;
if(this->span_index != current_span)
this->history[this->span_index % kAverageTimeCount] = std::exchange(this->span_bytes, 0);
this->span_index = current_span;
this->span_bytes += bytes;
}
[[nodiscard]] inline size_t take_delta() {
std::lock_guard slock{this->mutex};
assert(this->delta_bytes <= this->total_bytes);
auto delta = this->total_bytes - this->delta_bytes;
this->delta_bytes = this->total_bytes;
return delta;
}
[[nodiscard]] inline double current_bandwidth() const {
return (this->history[(this->span_index - 1) % kAverageTimeCount] * (double) 1000) / (double) kMeasureTimespanMs;
}
[[nodiscard]] inline double average_bandwidth() const {
return (std::accumulate(this->history.begin(), this->history.end(), 0UL) * (double) 1000) / (double) (kMeasureTimespanMs * kAverageTimeCount);
}
};
}

1
file/todo.txt Normal file
View File

@ -0,0 +1 @@
Test HTTPS file upload!

15
license/LicenseServerMain.cpp
View File

@ -18,7 +18,7 @@ using namespace license;
/*
* Requests/license: SELECT `tmp`.`keyId`, `tmp`.`key`, `tmp`.`ip`, `tmp`.`count`, `license_info`.`username`, `tmp`.`type` FROM (SELECT DISTINCT `license_request`.`keyId`, `key`, `license_request`.`ip`, COUNT(`license_request`.`keyId`) AS `count`, `license`.`type` FROM `license_request` INNER JOIN `license` ON `license`.`keyId` = `license_request`.`keyId` GROUP BY (`license_request`.`ip`)) AS `tmp` INNER JOIN `license_info` ON `license_info`.`keyId` = `tmp`.`keyId`
* Different IP's: SELECT `tmp`.`keyId`, `license_info`.`username`, COUNT(`ip`) FROM (SELECT DISTINCT `ip`, `keyId` FROM `license_request`) AS `tmp` LEFT JOIN `license_info` ON `license_info`.`keyId` = `tmp`.`keyId` GROUP BY (`tmp`.`keyId`)
* Different IP's: SELECT `tmp`.`keyId`, `license_info`.`username`, COUNT(`ip`) FROM (SELECT DISTINCT `ip`, `keyId` FROM `license_request`) AS `tmp` LEFT JOIN `license_info` ON `license_info`.`keyId` = `tmp`.`keyId` GROUP BY (`tmp`.`keyId`) ORDER BY COUNT(`ip`) DESC
*
* Requests (license) / ip: SELECT DISTINCT `ip`, COUNT(`ip`) FROM `license_request` WHERE `keyId` = ? GROUP BY `ip`
*
@ -26,6 +26,8 @@ using namespace license;
* //462
*
* SELECT DISTINCT(`ip`), `license_request`.`keyId`, `license_info`.`username` FROM `license_request` LEFT JOIN `license_info` ON `license_info`.`keyId` = `license_request`.`keyId` WHERE `timestamp` > (UNIX_TIMESTAMP() - 2 * 60 * 60) * 1000
* License too many request looking: SELECT `keyId`, `username`, COUNT(`ip`) FROM `unique_license_requests` GROUP BY `keyId` ORDER BY COUNT(`ip`) DESC
*
*
*
* Online clients: SELECT SUM(`clients`) FROM (SELECT DISTINCT(`ip`), `clients` FROM `history_online` WHERE `timestamp` > (UNIX_TIMESTAMP() - 60 * 60 * 2) * 1000) AS `a`
@ -34,6 +36,17 @@ using namespace license;
*
* Empty instances: curl -ik "https://stats.teaspeak.de:27788?type=request&request_type=general" -X GET 2>&1 | grep "data:"
*/
//SELECT SUM(`clients`) FROM `history_online` WHERE `keyId` = 701 OR `keyId` = 795 OR `keyId` = 792 OR `keyId` = 582 OR `keyId` = 753 OR `keyId` = 764 OR `keyId` = 761 OR `keyId` = 796 WHERE `timestamp` > (UNIX_TIMESTAMP() - 2.1 * 60 * 60) * 1000
/*
* Extra users:
* - ServerSponsoring
* - Davide550
* - xDeyego?
* - latters
* - Pamonha
*/
bool handle_command(string& line);
shared_ptr<server::database::DatabaseHandler> license_manager;

5
server/CMakeLists.txt
View File

@ -52,6 +52,7 @@ set(SERVER_SOURCE_FILES
src/client/voice/PacketStatistics.cpp
src/TS3ServerClientManager.cpp
src/VirtualServer.cpp
src/FileServerHandler.cpp
src/TS3ServerHeartbeat.cpp
src/SignalHandler.cpp
src/server/VoiceServer.cpp
@ -66,11 +67,8 @@ set(SERVER_SOURCE_FILES
src/client/ConnectedClientNotifyHandler.cpp
src/VirtualServerManager.cpp
src/server/file/LocalFileServer.cpp
src/channel/ServerChannel.cpp
src/channel/ClientChannelView.cpp
src/client/file/FileClient.cpp
src/client/file/FileClientIO.cpp
src/Group.cpp
src/manager/BanManager.cpp
src/client/InternalClient.cpp
@ -265,6 +263,7 @@ target_link_libraries(TeaSpeakServer
TeaLicenseHelper #Static
TeaMusic #Static
CXXTerminal::static #Static
TeaSpeak-FileServer
${StringVariable_LIBRARIES_STATIC}
${YAML_CPP_LIBRARIES}
pthread

1
server/main.cpp
View File

@ -10,7 +10,6 @@
#include "src/VirtualServer.h"
#include "src/InstanceHandler.h"
#include "src/server/QueryServer.h"
#include "src/server/file/LocalFileServer.h"
#include "src/terminal/CommandHandler.h"
#include "src/client/InternalClient.h"
#include "src/SignalHandler.h"

2
server/src/Configuration.cpp
View File

@ -1249,11 +1249,13 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
BIND_BOOL(config::server::delete_old_bans, true);
ADD_DESCRIPTION("Enable/disable the deletion of old bans within the database");
}
#if 0
{
CREATE_BINDING("delete_missing_icon_permissions", 0);
BIND_BOOL(config::server::delete_missing_icon_permissions, true);
ADD_DESCRIPTION("Enable/disable the deletion of invalid icon id permissions");
}
#endif
{
CREATE_BINDING("allow_weblist", 0);
BIND_BOOL(config::server::enable_teamspeak_weblist, true);

50
server/src/FileServerHandler.cpp Normal file
View File

@ -0,0 +1,50 @@
//
// Created by WolverinDEV on 12/05/2020.
//
#include <files/FileServer.h>
#include <log/LogUtils.h>
#include "FileServerHandler.h"
using namespace ts::server::file;
bool FileServerHandler::initialize(std::string &error) {
/*
* FIXME: Ports etc!
*
auto bindings_string = this->properties()[property::SERVERINSTANCE_FILETRANSFER_HOST].as<string>();
auto port = this->properties()[property::SERVERINSTANCE_FILETRANSFER_PORT].as<uint16_t>();
*/
/* TODO: Callback handler */
if(!file::initialize(error))
return false;
auto server = file::server();
assert(server);
auto& transfer = server->file_transfer();
transfer.callback_transfer_registered = std::bind(&FileServerHandler::callback_transfer_registered, this, std::placeholders::_1);
transfer.callback_transfer_started = std::bind(&FileServerHandler::callback_transfer_started, this, std::placeholders::_1);
transfer.callback_transfer_finished = std::bind(&FileServerHandler::callback_transfer_finished, this, std::placeholders::_1);
transfer.callback_transfer_aborted = std::bind(&FileServerHandler::callback_transfer_aborted, this, std::placeholders::_1, std::placeholders::_2);
transfer.callback_transfer_statistics = std::bind(&FileServerHandler::callback_transfer_statistics, this, std::placeholders::_1, std::placeholders::_2);
}
void FileServerHandler::callback_transfer_registered(const std::shared_ptr<transfer::Transfer> &transfer) {
auto server = this->instance_->getVoiceServerManager()->findServerById(transfer->server->server_id());
if(!server) return; /* well that's bad */
if(transfer->direction == transfer::Transfer::DIRECTION_UPLOAD) {
server->properties()[property::VIRTUALSERVER_TOTAL_BYTES_UPLOADED] += transfer->expected_file_size - transfer->file_offset;
server->properties()[property::VIRTUALSERVER_MONTH_BYTES_UPLOADED] += transfer->expected_file_size - transfer->file_offset;
} else {
server->properties()[property::VIRTUALSERVER_TOTAL_BYTES_DOWNLOADED] += transfer->expected_file_size - transfer->file_offset;
server->properties()[property::VIRTUALSERVER_MONTH_BYTES_DOWNLOADED] += transfer->expected_file_size - transfer->file_offset;
}
}
void FileServerHandler::callback_transfer_aborted(const std::shared_ptr<transfer::Transfer> &transfer,
const ts::server::file::transfer::TransferError &error) {
/* TODO: Remove not used quota from server & client */
}

31
server/src/FileServerHandler.h Normal file
View File

@ -0,0 +1,31 @@
#pragma once
#include <string>
#include "./InstanceHandler.h"
namespace ts::server::file {
class FileServerHandler {
public:
explicit FileServerHandler(InstanceHandler*);
bool initialize(std::string& /* error */);
void finalize();
private:
InstanceHandler* instance_;
#if 0
std::function<void(const std::shared_ptr<Transfer>&)> callback_transfer_registered{}; /* transfer has been registered */
std::function<void(const std::shared_ptr<Transfer>&)> callback_transfer_started{}; /* transfer has been started */
std::function<void(const std::shared_ptr<Transfer>&)> callback_transfer_finished{}; /* transfer has been finished */
std::function<void(const std::shared_ptr<Transfer>&, const TransferError&)> callback_transfer_aborted{}; /* an error happened while transferring the data */
std::function<void(const std::shared_ptr<Transfer>&, const TransferStatistics&)> callback_transfer_statistics{};
#endif
void callback_transfer_registered(const std::shared_ptr<transfer::Transfer>&);
void callback_transfer_started(const std::shared_ptr<transfer::Transfer>&);
void callback_transfer_finished(const std::shared_ptr<transfer::Transfer>&);
void callback_transfer_aborted(const std::shared_ptr<transfer::Transfer>&, const transfer::TransferError&);
void callback_transfer_statistics(const std::shared_ptr<transfer::Transfer>&, const transfer::TransferStatistics&);
};
}

3
server/src/Group.cpp
View File

@ -6,7 +6,6 @@
#include "VirtualServer.h"
#include "src/client/ConnectedClient.h"
#include "InstanceHandler.h"
#include "src/server/file/LocalFileServer.h"
using namespace std;
using namespace std::chrono;
@ -216,11 +215,13 @@ int GroupManager::insertGroupFromDb(int count, char **values, char **column) {
debugMessage(this->getServerId(), "Push back group -> " + to_string(group->groupId()) + " - " + group->name());
this->groups.push_back(group);
#if 0
auto iconId = (IconId) group->icon_id();
if(iconId != 0 && serverInstance && serverInstance->getFileServer() && !serverInstance->getFileServer()->iconExists(this->server.lock(), iconId)) {
logMessage(this->getServerId(), "[FILE] Missing group icon (" + to_string(iconId) + ").");
if(config::server::delete_missing_icon_permissions) group->permissions()->set_permission(permission::i_icon_id, {0, 0}, permission::v2::delete_value, permission::v2::do_nothing);
}
#endif
return 0;
}

43
server/src/InstanceHandler.cpp
View File

@ -3,15 +3,13 @@
#define XFREE undefined_free
#define XREALLOC undefined_realloc
#include <netdb.h>
#include "src/weblist/WebListManager.h"
#include <log/LogUtils.h>
#include "InstanceHandler.h"
#include "src/client/InternalClient.h"
#include "src/server/QueryServer.h"
#include "src/server/file/LocalFileServer.h"
#include "SignalHandler.h"
#include "src/manager/PermissionNameMapper.h"
#include "./FileServerHandler.h"
#include <ThreadPool/Timer.h>
#include "ShutdownHelper.h"
#include <sys/utsname.h>
@ -21,13 +19,14 @@
#include <misc/hex.h>
#include <misc/rnd.h>
#include <misc/strobf.h>
#include <jemalloc/jemalloc.h>
#include <protocol/buffers.h>
#ifndef _POSIX_SOURCE
#define _POSIX_SOURCE
#endif
#include <unistd.h>
#include <files/FileServer.h>
#undef _POSIX_SOURCE
using namespace std;
@ -281,31 +280,11 @@ bool InstanceHandler::startInstance() {
}
this->loadWebCertificate();
fileServer = new ts::server::LocalFileServer();
{
auto bindings_string = this->properties()[property::SERVERINSTANCE_FILETRANSFER_HOST].as<string>();
auto port = this->properties()[property::SERVERINSTANCE_FILETRANSFER_PORT].as<uint16_t>();
auto ft_bindings = net::resolve_bindings(bindings_string, port);
deque<shared_ptr<LocalFileServer::Binding>> bindings;
for(auto& binding : ft_bindings) {
if(!get<2>(binding).empty()) {
logError(LOG_FT, "Failed to resolve binding for {}: {}", get<0>(binding), get<2>(binding));
continue;
}
auto entry = make_shared<LocalFileServer::Binding>();
memcpy(&entry->address, &get<1>(binding), sizeof(sockaddr_storage));
entry->file_descriptor = -1;
entry->event_accept = nullptr;
bindings.push_back(entry);
}
logMessage(LOG_FT, "Starting server on {}:{}", bindings_string, port);
if(!fileServer->start(bindings, errorMessage)) {
logCritical(LOG_FT, "Failed to start server: {}", errorMessage);
return false;
}
this->file_server_handler_ = new file::FileServerHandler{this};
if(!this->file_server_handler_->initialize(errorMessage)) {
logCritical(LOG_FT, "Failed to initialize server: {}", errorMessage);
return false;
}
if(config::query::sslMode > 0) {
@ -443,9 +422,7 @@ void InstanceHandler::stopInstance() {
debugMessage(LOG_QUERY, "Query server stopped");
debugMessage(LOG_FT, "Stopping file server");
if (this->fileServer) this->fileServer->stop();
delete this->fileServer;
this->fileServer = nullptr;
file::finalize();
debugMessage(LOG_FT, "File server stopped");
this->save_channel_permissions();
@ -519,10 +496,6 @@ void InstanceHandler::tickInstance() {
}
}
}
{
ALARM_TIMER(t, "InstanceHandler::tickInstance -> fileserver tick", milliseconds(5));
if(this->fileServer) this->fileServer->instanceTick();
}
{
ALARM_TIMER(t, "InstanceHandler::tickInstance -> sql_test tick", milliseconds(5));
if(this->sql && this->active) {

9
server/src/InstanceHandler.h
View File

@ -23,6 +23,10 @@ namespace ts {
class LicenseService;
}
namespace file {
class FileServerHandler;
}
class InstanceHandler {
public:
explicit InstanceHandler(SqlDataManager*);
@ -42,12 +46,12 @@ namespace ts {
std::shared_mutex& getChannelTreeLock() { return this->default_tree_lock; }
VirtualServerManager* getVoiceServerManager(){ return this->voiceServerManager; }
LocalFileServer* getFileServer(){ return fileServer; }
QueryServer* getQueryServer(){ return queryServer; }
DatabaseHelper* databaseHelper(){ return this->dbHelper; }
BanManager* banManager(){ return this->banMgr; }
ssl::SSLManager* sslManager(){ return this->sslMgr; }
sql::SqlManager* getSql(){ return sql->sql(); }
file::FileServerHandler* getFileServerHandler() { return this->file_server_handler_; }
std::chrono::time_point<std::chrono::system_clock> getStartTimestamp(){ return startTimestamp; }
@ -110,12 +114,13 @@ namespace ts {
std::chrono::system_clock::time_point memcleanTimestamp;
SqlDataManager* sql;
LocalFileServer* fileServer = nullptr;
QueryServer* queryServer = nullptr;
VirtualServerManager* voiceServerManager = nullptr;
DatabaseHelper* dbHelper = nullptr;
BanManager* banMgr = nullptr;
ssl::SSLManager* sslMgr = nullptr;
file::FileServerHandler* file_server_handler_{nullptr};
ts::Properties* _properties = nullptr;

1
server/src/InstanceHandlerSetup.cpp
View File

@ -6,7 +6,6 @@
#include "InstanceHandler.h"
#include "src/client/InternalClient.h"
#include "src/server/QueryServer.h"
#include "src/server/file/LocalFileServer.h"
using namespace std;
using namespace std::chrono;

49
server/src/VirtualServer.cpp
View File

@ -10,6 +10,8 @@
#include <misc/digest.h>
#include <misc/base64.h>
#include <files/FileServer.h>
#include "weblist/WebListManager.h"
#include "./client/web/WebClient.h"
#include "./client/voice/VoiceClient.h"
@ -18,7 +20,6 @@
#include "./client/query/QueryClient.h"
#include "music/MusicBotManager.h"
#include "server/VoiceServer.h"
#include "src/server/file/LocalFileServer.h"
#include "server/QueryServer.h"
#include "InstanceHandler.h"
#include "Configuration.h"
@ -52,10 +53,17 @@ bool VirtualServer::initialize(bool test_properties) {
if(prop.type() == property::VIRTUALSERVER_DISABLE_IP_SAVING) {
this->_disable_ip_saving = prop.as<bool>();
return;
}
if(prop.type() == property::VIRTUALSERVER_CODEC_ENCRYPTION_MODE) {
} else if(prop.type() == property::VIRTUALSERVER_CODEC_ENCRYPTION_MODE) {
this->_voice_encryption_mode = prop.as<int>();
return;
} else if(prop.type() == property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH) {
auto file_vs = file::server()->find_virtual_server(this->getServerId());
if(!file_vs) return;
file_vs->max_networking_upload_bandwidth(prop.as_save<int64_t>([]{ return -1; }));
} else if(prop.type() == property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH) {
auto file_vs = file::server()->find_virtual_server(this->getServerId());
if(!file_vs) return;
file_vs->max_networking_download_bandwidth(prop.as_save<int64_t>([]{ return -1; }));
}
std::string sql{};
if(prop.type() == property::VIRTUALSERVER_HOST)
@ -196,8 +204,32 @@ bool VirtualServer::initialize(bool test_properties) {
this->serverAdmin->server = nullptr;
this->registerInternalClient(this->serverAdmin); /* lets assign server id 0 */
if(serverInstance->getFileServer())
serverInstance->getFileServer()->setupServer(self.lock());
{
using ErrorType = file::filesystem::ServerCommandErrorType;
auto file_vs = file::server()->register_server(this->getServerId());
auto initialize_result = file::server()->file_system().initialize_server(file_vs);
if(!initialize_result->wait_for(std::chrono::seconds{5})) {
logError(this->getServerId(), "Failed to wait for file directory initialisation.");
} else if(!initialize_result->succeeded()) {
switch (initialize_result->error().error_type) {
case ErrorType::FAILED_TO_CREATE_DIRECTORIES:
logError(this->getServerId(), "Failed to create server file directories ({}).", initialize_result->error().error_message);
break;
case ErrorType::UNKNOWN:
case ErrorType::FAILED_TO_DELETE_DIRECTORIES:
logError(this->getServerId(), "Failed to initialize server file directory due to an unknown error: {}/{}",
(int) initialize_result->error().error_type, initialize_result->error().error_message);
break;
}
}
this->properties()[property::VIRTUALSERVER_FILEBASE] = file::server()->file_base_path();
file_vs->max_networking_download_bandwidth(this->properties()[property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH].as_save<int64_t>([]{ return -1; }));
file_vs->max_networking_upload_bandwidth(this->properties()[property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH].as_save<int64_t>([]{ return -1; }));
}
this->channelTree->printChannelTree([&](std::string msg){ debugMessage(this->serverId, msg); });
this->musicManager = make_shared<music::MusicBotManager>(self.lock());
@ -205,11 +237,13 @@ bool VirtualServer::initialize(bool test_properties) {
this->musicManager->load_playlists();
this->musicManager->load_bots();
#if 0
if(this->properties()[property::VIRTUALSERVER_ICON_ID] != (IconId) 0)
if(!serverInstance->getFileServer()->iconExists(self.lock(), this->properties()[property::VIRTUALSERVER_ICON_ID])) {
debugMessage(this->getServerId(), "Removing invalid icon id of server");
this->properties()[property::VIRTUALSERVER_ICON_ID] = 0;
}
}
#endif
for(const auto& type : vector<property::VirtualServerProperties>{
property::VIRTUALSERVER_DOWNLOAD_QUOTA,
@ -227,9 +261,6 @@ bool VirtualServer::initialize(bool test_properties) {
}
}
if(this->properties()[property::VIRTUALSERVER_FILEBASE].value().empty())
this->properties()[property::VIRTUALSERVER_FILEBASE] = serverInstance->getFileServer()->server_file_base(self.lock());
/* lets cleanup the conversations for not existent channels */
this->_conversation_manager->synchronize_channels();
return true;

24
server/src/VirtualServerManager.cpp
View File

@ -4,9 +4,9 @@
#include "src/server/VoiceServer.h"
#include "src/client/query/QueryClient.h"
#include "InstanceHandler.h"
#include "src/server/file/LocalFileServer.h"
#include "src/client/ConnectedClient.h"
#include <ThreadPool/ThreadHelper.h>
#include <files/FileServer.h>
using namespace std;
using namespace std::chrono;
@ -419,7 +419,27 @@ bool VirtualServerManager::deleteServer(shared_ptr<VirtualServer> server) {
this->handle->properties()[property::SERVERINSTANCE_SPOKEN_TIME_DELETED] += server->properties()[property::VIRTUALSERVER_SPOKEN_TIME].as<uint64_t>();
this->delete_server_in_db(server->serverId);
this->handle->getFileServer()->deleteServer(server);
if(auto file_vs = file::server()->find_virtual_server(server->getServerId()); file_vs) {
using ErrorType = file::filesystem::ServerCommandErrorType;
auto delete_result = file::server()->file_system().delete_server(file_vs);
if(!delete_result->wait_for(std::chrono::seconds{5})) {
logError(LOG_INSTANCE, "Failed to wait for file directory initialisation.");
} else if(!delete_result->succeeded()) {
switch (delete_result->error().error_type) {
case ErrorType::FAILED_TO_DELETE_DIRECTORIES:
logError(LOG_INSTANCE, "Failed to delete server {} file directories ({}).", server->getServerId(), delete_result->error().error_message);
break;
case ErrorType::UNKNOWN:
case ErrorType::FAILED_TO_CREATE_DIRECTORIES:
logError(LOG_INSTANCE, "Failed to delete server {} file directory due to an unknown error: {}/{}",
server->getServerId(), (int) delete_result->error().error_type, delete_result->error().error_message);
break;
}
}
}
return true;
}

3
server/src/channel/ServerChannel.cpp
View File

@ -5,7 +5,6 @@
#include "misc/rnd.h"
#include "src/VirtualServer.h"
#include "src/client/ConnectedClient.h"
#include "src/server/file/LocalFileServer.h"
#include "src/InstanceHandler.h"
#include "../manager/ConversationManager.h"
@ -481,6 +480,7 @@ bool ServerChannelTree::validateChannelNames() {
bool ServerChannelTree::validateChannelIcons() {
for(const auto &channel : this->channels()) {
auto iconId = (IconId) channel->properties()[property::CHANNEL_ICON_ID];
#if 0
if(iconId != 0 && !serverInstance->getFileServer()->iconExists(this->server.lock(), iconId)) {
logMessage(this->getServerId(), "[FILE] Missing channel icon (" + to_string(iconId) + ").");
if(config::server::delete_missing_icon_permissions) {
@ -488,6 +488,7 @@ bool ServerChannelTree::validateChannelIcons() {
channel->permissions()->set_permission(permission::i_icon_id, {0, 0}, permission::v2::PermissionUpdateType::set_value, permission::v2::PermissionUpdateType::do_nothing);
}
}
#endif
}
return true;
}

5
server/src/client/ConnectedClient.cpp
View File

@ -11,7 +11,6 @@
#include "src/VirtualServer.h"
#include "voice/VoiceClient.h"
#include "../server/VoiceServer.h"
#include "src/server/file/LocalFileServer.h"
#include "../InstanceHandler.h"
#include "ConnectedClient.h"
@ -143,7 +142,8 @@ void ConnectedClient::updateChannelClientProperties(bool lock_channel_tree, bool
iconId = value.value;
}
logTrace(this->getServerId(), "[CLIENT] Updating client icon from " + to_string(this->properties()[property::CLIENT_ICON_ID].as<IconId>()) + " to " + to_string(iconId));
if(this->properties()[property::CLIENT_ICON_ID].as<IconId>() != iconId){
if(this->properties()[property::CLIENT_ICON_ID].as<IconId>() != iconId) {
#if 0
if(server_ref && iconId != 0) {
auto dir = serverInstance->getFileServer()->iconDirectory(server_ref);
if(!serverInstance->getFileServer()->iconExists(server_ref, iconId)) {
@ -151,6 +151,7 @@ void ConnectedClient::updateChannelClientProperties(bool lock_channel_tree, bool
iconId = 0;
}
}
#endif
if(this->properties()[property::CLIENT_ICON_ID].as<IconId>() != iconId) {
this->properties()[property::CLIENT_ICON_ID] = (IconId) iconId;
notifyList.emplace_back(property::CLIENT_ICON_ID);

19
server/src/client/ConnectedClient.h
View File

@ -13,7 +13,11 @@
#define CLIENT_STR_LOG_PREFIX CLIENT_STR_LOG_PREFIX_(this)
#define CMD_REQ_SERVER \
if(!this->server) return command_result{error::server_invalid_id};
do { \
if(!this->server) { \
return command_result{error::server_invalid_id}; \
} \
} while(0)
/* TODO: Play lock the server here with read? So the client dosn't get kicked within that moment */
#define CMD_REF_SERVER(variable_name) \
@ -33,8 +37,10 @@ if(!cmd[0].has(parm)) return command_result{error::parameter_not_found};
//the message here is show to the manager!
#define CMD_CHK_AND_INC_FLOOD_POINTS(num) \
this->increaseFloodPoints(num); \
if(this->shouldFloodBlock()) return command_result{error::ban_flooding};
do {\
this->increaseFloodPoints(num); \
if(this->shouldFloodBlock()) return command_result{error::ban_flooding}; \
} while(0)
#define CMD_CHK_PARM_COUNT(count) \
if(cmd.bulkCount() != count) return command_result{error::parameter_invalid_count};
@ -453,10 +459,11 @@ namespace ts {
command_result handleCommandFTDeleteFile(Command&);
command_result handleCommandFTInitUpload(Command&);
command_result handleCommandFTInitDownload(Command&);
command_result handleCommandFTGetFileInfo(Command&);
//CMD_TODO handleCommandFTGetFileInfo -> 5 points
command_result handleCommandFTGetFileInfo(Command&);
command_result handleCommandFTRenameFile(Command&);
command_result handleCommandFTList(Command&);
command_result handleCommandFTStop(Command&);
//CMD_TODO handleCommandFTStop -> 5 points
//CMD_TODO handleCommandFTRenameFile -> 5 points
//CMD_TODO handleCommandFTList -> 5 points
command_result handleCommandBanList(Command&);

1
server/src/client/ConnectedClientNotifyHandler.cpp
View File

@ -3,7 +3,6 @@
#include <algorithm>
#include "ConnectedClient.h"
#include "voice/VoiceClient.h"
#include "src/server/file/LocalFileServer.h"
#include "../server/VoiceServer.h"
#include "../InstanceHandler.h"
#include "../server/QueryServer.h"

3
server/src/client/DataClient.cpp
View File

@ -4,7 +4,6 @@
#include <misc/hex.h>
#include "DataClient.h"
#include "ConnectedClient.h"
#include "src/server/file/LocalFileServer.h"
#include "src/InstanceHandler.h"
#include "misc/base64.h"
@ -115,6 +114,7 @@ bool DataClient::loadDataForCurrentServer() { //TODO for query
this->clientPermissions = serverInstance->databaseHelper()->loadClientPermissionManager(ref_server, this->getClientDatabaseId());
//Setup / fix stuff
#if 0
if(!this->properties()[property::CLIENT_FLAG_AVATAR].as<string>().empty()){
if(
!ref_server ||
@ -123,6 +123,7 @@ bool DataClient::loadDataForCurrentServer() { //TODO for query
this->properties()[property::CLIENT_FLAG_AVATAR] = "";
}
}
#endif
if(ref_server)
this->properties()[property::CLIENT_UNREAD_MESSAGES] = ref_server->letters->unread_letter_count(this->getUid());

3
server/src/client/command_handler/channel.cpp
View File

@ -5,13 +5,11 @@
#include "../../build.h"
#include "../ConnectedClient.h"
#include "../InternalClient.h"
#include "src/server/file/LocalFileServer.h"
#include "../../server/VoiceServer.h"
#include "../voice/VoiceClient.h"
#include "PermissionManager.h"
#include "../../InstanceHandler.h"
#include "../../server/QueryServer.h"
#include "../file/FileClient.h"
#include "../music/MusicClient.h"
#include "../query/QueryClient.h"
#include "../../weblist/WebListManager.h"
@ -182,6 +180,7 @@ command_result ConnectedClient::handleCommandChannelGroupCopy(Command &cmd) {
return permission::b_serverinstance_modify_querygroup;
break;
case GroupType::GROUP_TYPE_NORMAL:
default:
break;
}

27
server/src/client/command_handler/client.cpp
View File

@ -6,12 +6,10 @@
#include "../../build.h"
#include "../ConnectedClient.h"
#include "../InternalClient.h"
#include "src/server/file/LocalFileServer.h"
#include "../voice/VoiceClient.h"
#include "PermissionManager.h"
#include "../../InstanceHandler.h"
#include "../../server/QueryServer.h"
#include "../file/FileClient.h"
#include "../music/MusicClient.h"
#include "../query/QueryClient.h"
#include "../../weblist/WebListManager.h"
@ -349,31 +347,6 @@ command_result ConnectedClient::handleCommandClientChatClosed(Command &cmd) {
return command_result{error::ok};
}
//ftgetfilelist cid=1 cpw path=\/ return_code=1:x
//Answer:
//1 .. n
// notifyfilelist cid=1 path=\/ return_code=1:x name=testFile size=35256 datetime=1509459767 type=1|name=testDir size=0 datetime=1509459741 type=0|name=testDir_2 size=0 datetime=1509459763 type=0
//notifyfilelistfinished cid=1 path=\/
inline void cmd_filelist_append_files(ServerId sid, Command &command, vector<std::shared_ptr<file::FileEntry>> files) {
int index = 0;
logTrace(sid, "Sending file list for path {}", command["path"].string());
for (const auto& fileEntry : files) {
logTrace(sid, " - {} ({})", fileEntry->name, fileEntry->type == file::FileType::FILE ? "file" : "directory");
command[index]["name"] = fileEntry->name;
command[index]["datetime"] = std::chrono::duration_cast<std::chrono::seconds>(fileEntry->lastChanged.time_since_epoch()).count();
command[index]["type"] = fileEntry->type;
if (fileEntry->type == file::FileType::FILE)
command[index]["size"] = static_pointer_cast<file::File>(fileEntry)->fileSize;
else
command[index]["size"] = 0;
index++;
}
}
#define CMD_REQ_FSERVER if(!serverInstance->getFileServer()) return command_result{error::vs_critical, "file server not started yet!"}
//start=0 duration=10
//pattern=%asd%

1167
server/src/client/command_handler/file.cpp
View File

File diff suppressed because it is too large Load Diff

4
server/src/client/command_handler/helpers.h
View File

@ -79,7 +79,8 @@ if (permission::resolvePermissionData(permType)->type == permission::PermissionT
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
inline bool permission_require_granted_value(ts::permission::PermissionType type) {
using namespace ts;
/*
@ -192,6 +193,7 @@ inline bool permission_is_client_property(ts::permission::PermissionType type) {
return false;
}
}
#pragma GCC diagnostic pop
inline ssize_t count_characters(const std::string& in) {

5
server/src/client/command_handler/misc.cpp
View File

@ -13,13 +13,11 @@
#include "../../build.h"
#include "../ConnectedClient.h"
#include "../InternalClient.h"
#include "src/server/file/LocalFileServer.h"
#include "../../server/VoiceServer.h"
#include "../voice/VoiceClient.h"
#include "PermissionManager.h"
#include "../../InstanceHandler.h"
#include "../../server/QueryServer.h"
#include "../file/FileClient.h"
#include "../music/MusicClient.h"
#include "../query/QueryClient.h"
#include "../../weblist/WebListManager.h"
@ -136,6 +134,9 @@ command_result ConnectedClient::handleCommand(Command &cmd) {
else if (command == "ftinitupload") return this->handleCommandFTInitUpload(cmd);
else if (command == "ftinitdownload") return this->handleCommandFTInitDownload(cmd);
else if (command == "ftgetfileinfo") return this->handleCommandFTGetFileInfo(cmd);
else if (command == "ftrenamefile") return this->handleCommandFTRenameFile(cmd);
else if (command == "ftlist") return this->handleCommandFTList(cmd);
else if (command == "ftstop") return this->handleCommandFTStop(cmd);
//Banlist
else if (command == "banlist") return this->handleCommandBanList(cmd);
else if (command == "banadd") return this->handleCommandBanAdd(cmd);

2
server/src/client/command_handler/music.cpp
View File

@ -13,13 +13,11 @@
#include "../../build.h"
#include "../ConnectedClient.h"
#include "../InternalClient.h"
#include "src/server/file/LocalFileServer.h"
#include "../../server/VoiceServer.h"
#include "../voice/VoiceClient.h"
#include "PermissionManager.h"
#include "../../InstanceHandler.h"
#include "../../server/QueryServer.h"
#include "../file/FileClient.h"
#include "../music/MusicClient.h"
#include "../query/QueryClient.h"
#include "../../weblist/WebListManager.h"

2
server/src/client/command_handler/server.cpp
View File

@ -10,12 +10,10 @@
#include "../../build.h"
#include "../ConnectedClient.h"
#include "../InternalClient.h"
#include "src/server/file/LocalFileServer.h"
#include "../../server/VoiceServer.h"
#include "../voice/VoiceClient.h"
#include "../../InstanceHandler.h"
#include "../../server/QueryServer.h"
#include "../file/FileClient.h"
#include "../music/MusicClient.h"
#include "../query/QueryClient.h"
#include "../../weblist/WebListManager.h"

719
server/src/client/file/FileClient.cpp
View File

@ -1,719 +0,0 @@
#include <algorithm>
#include <src/server/file/LocalFileServer.h>
#include <log/LogUtils.h>
#include "FileClient.h"
#include <src/InstanceHandler.h>
#include <experimental/filesystem>
#include <misc/memtracker.h>
#include <misc/base64.h>
#include "src/client/ConnectedClient.h"
#include <netinet/tcp.h>
using namespace std;
using namespace std::chrono;
using namespace ts::server;
namespace fs = std::experimental::filesystem;
#define BUFFER_SIZE (size_t) 2048
FileClient::FileClient(LocalFileServer* handle, int socketFd) : handle(handle), clientFd(socketFd) {
memtrack::allocated<FileClient>(this);
this->last_io_action = system_clock::now();
int enabled = 1;
if(setsockopt(socketFd, IPPROTO_TCP, TCP_NODELAY, &enabled, sizeof enabled) < 0)
logError(LOG_FT, "{} Cant enable TCP no delay for socket {}. Error: {}/{}", this->client_prefix(), socketFd, errno, strerror(errno));
this->readEvent = event_new(this->handle->ioLoop, socketFd, EV_READ|EV_PERSIST, [](int a, short b, void* c){ ((FileClient*) c)->handleMessageRead(a, b, c); }, this);
this->writeEvent = event_new(this->handle->ioLoop, socketFd, EV_TIMEOUT | EV_WRITE, [](int a, short b, void* c){ ((FileClient*) c)->handleMessageWrite(a, b, c); }, this);
this->ssl_handler.direct_process(pipes::PROCESS_DIRECTION_OUT, true);
this->ssl_handler.direct_process(pipes::PROCESS_DIRECTION_IN, true);
this->ssl_handler.callback_write(bind(&FileClient::sendRawMessage, this, placeholders::_1));
this->ssl_handler.callback_data(bind(&FileClient::handle_ssl_message, this, placeholders::_1));
/*
this->ssl_handler.callback_data([&](const pipes::buffer_view& msg) {
if(this->ftType == FTType::TeaWeb_HTTPS) {
this->handle_http_message(msg);
} else if(this->ftType == FTType::TeaWeb_SSL_WS) {
this->ws_handler.process_incoming_data(msg);
} else {
logError(LOG_FT, "{} Decoded SSL packet, but transfer type isn't SSL!", this->client_prefix());
}
});
*/
//FIXME init ssl error handler?
this->ws_handler.direct_process(pipes::PROCESS_DIRECTION_OUT, true);
this->ws_handler.direct_process(pipes::PROCESS_DIRECTION_IN, true);
/*
this->ws_handler.callback_data([&](const pipes::WSMessage& message) {
if(this->state_transfere == T_INITIALIZE) {
this->applyKey(message.data.string()); //Got the key :)
} else if(this->state_transfere == T_TRANSFER) {
if(this->pendingKey->upload)
this->uploadWriteBytes(message.data.string());
else {
logError(LOG_FT, "{} Invalid write (Just download)", this->client_prefix());
this->disconnect();
}
}
});
*/
this->ws_handler.callback_data(bind(&FileClient::handle_ws_message,this, placeholders::_1));
this->ws_handler.callback_write([&](const pipes::buffer_view& data) {
this->ssl_handler.send(data);
});
this->ws_handler.on_connect = [&]() {};
this->ws_handler.on_disconnect = [&](const std::string&) {};
}
FileClient::~FileClient() {
if(this->thread_flush.joinable()){
this->state_connection = C_DISCONNECTED;
if(this->thread_flush.get_id() != this_thread::get_id())
this->thread_flush.join();
else
this->thread_flush.detach();
}
memtrack::freed<FileClient>(this);
}
size_t FileClient::used_bandwidth() {
auto now = system_clock::now();
size_t tranfarred_bytes = 0;
auto timeout = now - seconds(1);
{
lock_guard<recursive_mutex> lock(this->bandwidth_lock);
for(auto it = this->bandwidth.end(); it != this->bandwidth.begin();) {
--it;
if((*it)->timestamp < timeout) {
this->bandwidth.erase(this->bandwidth.begin(), it);
break;
}
tranfarred_bytes += (*it)->length;
}
}
return tranfarred_bytes;
}
std::string FileClient::client_prefix() {
bool hide_ip = config::server::disable_ip_saving;
if(!hide_ip) {
auto client = this->client;
if(client) {
auto server = client->getServer();
if(server) {
hide_ip = server->disable_ip_saving();
}
}
}
std::string address = "";
if(hide_ip)
address = "X.X.X.X:" + to_string(net::port(this->remote_address));
else
address = net::to_string(this->remote_address);
if(this->client) return "[" + to_string(this->client->getServerId()) + "|" + address + "| " + this->client->getDisplayName() + "]";
return "[0|" + address + "|unconnected]";
}
size_t FileClient::transferred_bytes() {
return this->bytesHandled;
}
size_t FileClient::remaining_bytes() {
threads::MutexLock lock(this->tickLock);
if(!this->pendingKey) return 0;
return this->pendingKey->size - this->bytesHandled;
}
void FileClient::disconnect(std::chrono::milliseconds timeout) {
auto own_lock = _this.lock();
if(!own_lock) return;
bool apply_flush = timeout.count() > 0;
debugMessage(LOG_FT, "{} Disconnecting client. Connection state: {} Flush IO: {}", this->client_prefix(), (int) this->state_connection, apply_flush);
unique_lock<threads::Mutex> tick_lock(this->tickLock);
if(this->state_connection == C_DISCONNECTED) return; /* we're already disconnected */
if(apply_flush && this->state_connection == C_DISCONNECTING) return; /* we're already flushing the IO */
if(this->ftType == FTType::TeaWeb_SSL_WS) {
this->ws_handler.disconnect(1000, "disconnected");
};
this->state_connection = apply_flush ? C_DISCONNECTING : C_DISCONNECTED;
if(this->readEvent)
event_del_noblock(this->readEvent);
if(apply_flush){
lock_guard flush_lock(this->thread_flush_lock);
assert(!this->thread_flush.joinable());
this->thread_flush = std::thread([own_lock, timeout]{
auto beg = system_clock::now();
while(own_lock->state_connection == C_DISCONNECTING && beg + timeout > system_clock::now()){
{
lock_guard buffer_lock(own_lock->bufferLock);
if(own_lock->read_queue.empty() && own_lock->write_queue.empty()) break;
}
usleep(10 * 1000);
}
unique_lock<threads::Mutex> l(own_lock->tickLock);
if(own_lock->state_connection != C_DISCONNECTING) return;
own_lock->disconnectFinal(l, true);
});
{
auto native_handle = this->thread_flush.native_handle();
pthread_setname_np(native_handle, "FileClient flush");
}
} else {
unique_lock flush_lock(this->thread_flush_lock);
if(this->thread_flush.joinable()) {
flush_lock.unlock();
this->thread_flush.join();
flush_lock.lock();
}
disconnectFinal(tick_lock, false);
}
}
void FileClient::disconnectFinal(unique_lock<threads::Mutex>& l, bool lock_flush_thread) {
auto ownLock = _this.lock();
unique_lock l1(this->bufferLock);
this->state_connection = C_DISCONNECTED;
{
unique_lock flush_lock(this->thread_flush_lock, try_to_lock);
if(flush_lock.owns_lock() || !lock_flush_thread) {
if(this->thread_flush.joinable()) {
if(this->thread_flush.get_id() == this_thread::get_id())
this->thread_flush.detach();
else
this->thread_flush.join();
}
}
}
if(this->readEvent){
auto event = this->readEvent;
this->readEvent = nullptr;
l1.unlock();
l.unlock();
event_del_block(event);
event_free(event);
l.lock();
l1.lock();
}
if(this->writeEvent){
auto event = this->writeEvent;
this->writeEvent = nullptr;
l1.unlock();
l.unlock();
event_del_block(event);
event_free(event);
l.lock();
l1.lock();
}
if(this->clientFd > 0) {
shutdown(this->clientFd, SHUT_RDWR);
close(this->clientFd);
}
this->clientFd = -1;
{
threads::MutexLock l2(this->handle->clientLock);
auto& clList = this->handle->connectedClients;
auto elm = find(clList.begin(), clList.end(), _this.lock());
if(elm != clList.end()) clList.erase(elm);
else logError(LOG_FT, "{} Invalid ft client list!", this->client_prefix());
}
this->read_queue.clear();
this->write_queue.clear();
if(this->fstream){
this->fstream->flush();
this->fstream->close();
delete this->fstream;
this->fstream = nullptr;
}
this->pendingKey = nullptr;
}
bool FileClient::tick() {
lock_guard<threads::Mutex> l(this->tickLock);
if(this->state_connection == C_DISCONNECTED) return false;
if(this->state_connection != C_DISCONNECTING) {
if(last_io_action.time_since_epoch().count() == 0)
last_io_action = system_clock::now();
else if(last_io_action + minutes(1) < system_clock::now()) {
logMessage(LOG_FT, "{} Timed out after one minute of silence!", this->client_prefix());
this->disconnect();
return false;
}
}
/* decode incomming stuff */
bool flag_reexecute = false;
{
/* types which require an extra layer of decode */
if(this->ftType == FTType::TeaWeb_SSL || this->ftType == FTType::TeaWeb_SSL_HTTP || this->ftType == FTType::TeaWeb_SSL_WS || this->ftType == FTType::TeaWeb_HTTP) {
pipes::buffer buffer;
{
lock_guard buffer_lock(this->bufferLock);
if(this->read_queue.empty()) {
flag_reexecute = false;
} else {
flag_reexecute = true;
buffer = this->read_queue.front();
this->read_queue.pop_front();
}
}
if(flag_reexecute) {
if(this->ftType == FTType::TeaWeb_HTTP)
this->handle_http_message(buffer);
else
this->ssl_handler.process_incoming_data(buffer);
}
} else if(this->ftType == FTType::TeamSpeak) {
flag_reexecute |= this->handle_ts_message();
} else if(this->ftType == FTType::Unknown) {
/* we need at least 16 bytes to detect any type */
if(this->availableBytes() >= 16) {
auto header = this->peekBytes(16);
if(header.find("GET") != -1 || header.find("POST") != -1 || header.find("OPTIONS") != -1) {
debugMessage(LOG_FT, "{} Using HTTP only!", this->client_prefix());
this->ftType = FTType::TeaWeb_HTTP;
return true;
} else if(pipes::SSL::isSSLHeader(header) && serverInstance->sslManager()->web_ssl_options()) {
debugMessage(LOG_FT, "{} Encrypting pipe with SSL", this->client_prefix());
this->ftType = FTType::TeaWeb_SSL;
this->ssl_handler.initialize(serverInstance->sslManager()->web_ssl_options());
return true;
} else {
debugMessage(LOG_FT, "{} Transferring data with the TS protocol", this->client_prefix());
this->ftType = FTType::TeamSpeak;
return true;
}
}
} else {
logError(LOG_FT, "{} Ticked client with unknown protocol type. Closing connection.", this->client_prefix());
this->disconnect();
return false;
}
}
if(this->state_transfer == T_TRANSFER) {
if(this->pendingKey) {
/* test for download */
if(!this->pendingKey->upload) {
/* lets send some data :) */
if(!fstream) {
logError(LOG_FT, "{} Missing file stream! Disconnecting client...", this->client_prefix());
this->disconnect();
return false;
}
size_t count = 0;
{
lock_guard lock(this->bufferLock);
count = this->write_queue.size();
}
if(count <= (1024 * 512) / BUFFER_SIZE){ //Max buffer 500Kb
if(!fstream->good()){
logError(LOG_FT, "{} Cant finish file download. File isn't good anymore!", this->client_prefix());
this->disconnect();
return false;
}
pipes::buffer writeBuffer(BUFFER_SIZE);
auto read = fstream->readsome((char*) writeBuffer.data_ptr(), writeBuffer.length());
if(read < 0){
logError(LOG_FT, "{} Invalid file read. Read {} bytes of max {}. Index {}/{}", this->client_prefix(), read, writeBuffer.length(), this->bytesHandled, this->pendingKey->size);
this->disconnect();
return false;
} else if(read == 0){
if(this->bytesHandled != this->pendingKey->size){
logError(LOG_FT, "{} Invalid end of file. Expected {} bytes and attempted to read at {}", this->client_prefix(), this->pendingKey->size, this->bytesHandled);
this->disconnect();
return false;
}
}
this->bytesHandled += read;
this->client->getConnectionStatistics()->logFileTransferOut(read);
if(read > 0){
writeBuffer.resize(read);
this->sendMessage(writeBuffer);
flag_reexecute |= true;
}
}
}
/* lets test if we're done */
if(this->bytesHandled == this->pendingKey->size){
auto time = duration_cast<milliseconds>(system_clock::now() - this->connect_timestamp).count();
logMessage(LOG_FT, "{} File transfer completed. Transferred {} bytes in {} milliseconds. Waiting for disconnect.", this->client_prefix(), this->bytesHandled, time);
this->close_file_handle();
this->pendingKey.reset();
this->state_transfer = T_DONE;
this->finished_timestamp = system_clock::now();
/* we expect TS3 to hangup the connection */
if(this->ftType != FTType::TeamSpeak)
this->disconnect(seconds(5));
return false;
}
}
} else if(this->state_transfer == T_DONE) {
if(this->finished_timestamp + seconds(2) < system_clock::now() && this->state_connection == C_CONNECTED) {
debugMessage(LOG_FT, "{} Disconnecting client after 2 seconds after finish!", this->client_prefix());
this->disconnect(seconds(5));
}
}
return flag_reexecute;
}
bool FileClient::applyKey(const string &key) {
{
threads::MutexLock lock(this->handle->keylock);
for(const auto& pkey : this->handle->pendingKeys) //needs a copy
if(pkey && pkey->key == key){
this->pendingKey = pkey;
this->handle->pendingKeys.erase(std::find(this->handle->pendingKeys.begin(), this->handle->pendingKeys.end(), pkey));
break;
}
}
if(!this->pendingKey){
logError(LOG_FT, "{} Tried to apply an non existing key! (Key: {})", this->client_prefix(), key);
this->disconnect();
return false;
}
this->client = this->pendingKey->owner.lock();
if(!this->client) {
logError(LOG_FT, "{} Tried connect with an invalid key (client offline)! (Key: {})", this->client_prefix(), key);
this->disconnect();
return false;
}
debugMessage(LOG_FT, "{} Initialized file transfer for file '{}' (Type: {})", this->client_prefix(), this->pendingKey->targetFile, pendingKey->upload ? "upload" : "download");
if(pendingKey->offset == 0 && pendingKey->upload) {
try {
fs::remove(fs::u8path(pendingKey->targetFile));
} catch (std::exception& e) {}
}
fstream = new std::fstream();
fstream->open(pendingKey->targetFile, (pendingKey->upload ? ios::out : ios::in) | ios::binary | ios::app);
if(!*fstream) {
logError(LOG_FT, "{} Failed to open target file {} for {}", this->client_prefix(), this->pendingKey->targetFile, pendingKey->upload ? "upload" : "download");
delete fstream;
this->fstream = nullptr;
return false;
}
fstream->seekg(0, std::ios::beg);
auto fsize = fstream->tellg();
fstream->seekg(0, std::ios::end);
fsize = fstream->tellg() - fsize;
fstream->seekg(this->pendingKey->offset, std::ios::beg);
if(!*fstream) {
logError(LOG_FT, "{} Failed to seek within file {} for {}", this->client_prefix(), this->pendingKey->targetFile, pendingKey->upload ? "upload" : "download");
delete fstream;
this->fstream = nullptr;
return false;
}
debugMessage(LOG_FT, "{} Received local file size {}. Target size if {}", this->client_prefix(), fsize, pendingKey->size);
if(this->state_transfer == T_INITIALIZE)
this->state_transfer = T_TRANSFER;
this->connect_timestamp = system_clock::now();
return true;
}
bool FileClient::uploadWriteBytes(const pipes::buffer_view& message) {
this->client->getConnectionStatistics()->logFileTransferIn(message.length());
if(!fstream->good()){
logError(LOG_FT, "{} uploadWriteBytes(...) called with invalid fstream!", this->client_prefix());
return false;
}
fstream->write(message.data_ptr<char>(), message.length());
if(!fstream->good()){
logError(LOG_FT, "{} Invalid file write! ({})", this->client_prefix(), message.length());
this->disconnect();
return false;
}
this->bytesHandled += message.length();
return true;
}
void FileClient::close_file_handle() {
if(this->fstream) {
this->fstream->flush();
this->fstream->close();
delete this->fstream;
this->fstream = nullptr;
}
}
void FileClient::sendMessage(const pipes::buffer_view& message) {
switch(this->ftType) {
case FTType::TeamSpeak:
case FTType::TeaWeb_HTTP:
this->sendRawMessage(message);
break;
case FTType::TeaWeb_SSL:
case FTType::TeaWeb_SSL_HTTP:
this->ssl_handler.send(message);
break;
case FTType::TeaWeb_SSL_WS:
this->ws_handler.send({pipes::BINARY, message.own_buffer()});
break;
default:
/* Dont log an error because the timeout disconnect function uses this without knowing which proto */
__asm__("nop");
return;
}
}
void FileClient::handle_ssl_message(const pipes::buffer_view &buffer) {
if(this->ftType == FTType::TeaWeb_SSL_HTTP)
this->handle_http_message(buffer);
else if(this->ftType == FTType::TeaWeb_SSL_WS)
this->ws_handler.process_incoming_data(buffer);
else if(this->ftType == FTType::TeaWeb_SSL) { /* lets detect if we have HTTP or WebSocket */
this->read_buffer += buffer;
this->handle_http_header();
}
}
void FileClient::handle_http_header() {
auto header_end = this->read_buffer.find("\r\n\r\n");
if(header_end == string::npos) {
if(this->read_buffer.length() > 1024 * 1024 * 4) {
this->read_buffer = pipes::buffer{};
logMessage(LOG_FT, "{} Client tried to fillup server memory. Disconnecting client.", this->client_prefix());
this->disconnect();
return;
}
return;
}
auto raw_request = this->read_buffer.view(0, header_end).string();
http::HttpRequest request{};
if(!http::parse_request(raw_request, request)) {
logError(LOG_FT, "{} Failed to parse HTTP request. Disconnecting client.", this->client_prefix());
this->disconnect();
return;
}
auto header_upgrade = request.findHeader("Upgrade");
if(header_upgrade && header_upgrade.values[0] == "websocket") {
debugMessage(LOG_FT, "{} Received WebSocket upgrade. Upgrading connection.", this->client_prefix(), raw_request);
if(this->ftType == FTType::TeaWeb_SSL)
this->ftType = FTType::TeaWeb_SSL_WS;
else {
//TODO: WebSocket only!
}
this->ws_handler.initialize();
this->ws_handler.process_incoming_data(this->read_buffer);
this->read_buffer = pipes::buffer{};
this->handle->tickFileClient(_this.lock()); /* we require a manual reticking */
return;
} else {
/* process the http request header */
if(this->ftType == FTType::TeaWeb_SSL)
this->ftType = FTType::TeaWeb_SSL_HTTP;
http::HttpResponse response{};
response.setHeader("Connection", {"close"}); /* close the connection instance, we dont want multiple requests */
response.setHeader("Access-Control-Allow-Methods", {"GET, POST"});
response.setHeader("Access-Control-Allow-Origin", {"*"});
response.setHeader("Access-Control-Allow-Headers", request.findHeader("Access-Control-Request-Headers").values); //access-control-allow-headers
response.setHeader("Access-Control-Max-Age", {"86400"});
response.setHeader("Access-Control-Expose-Headers", {"*, Content-Length, X-media-bytes, Content-Disposition"});
/* test for a preflight request: https://developer.mozilla.org/en-US/docs/Glossary/preflight_request */
auto request_method = request.findHeader("Access-Control-Request-Method");
if(request_method) {
debugMessage(LOG_FT, "{} Received preflight request. Sending close response with allow and let the client reconnect.", this->client_prefix());
response.code = http::code::_200;
this->read_buffer = this->read_buffer.range(header_end + 4);
auto raw_response = response.build();
this->sendMessage(pipes::buffer_view{raw_response.data(), raw_response.length()});
this->disconnect(seconds(5)); /* write our response & flush */
return;
} else {
auto transfer_key = request.findHeader("transfer-key");
auto download_name = request.findHeader("download-name");
if(!transfer_key) {
response.code = http::code::code(400, "Bad Request");
debugMessage(LOG_FT, "{} Received invalid HTTP request (Missing transfer key).", this->client_prefix());
} else {
if(!this->applyKey(transfer_key.values[0]) || !this->pendingKey) {
response.code = http::code::code(404, "Not Found");
debugMessage(LOG_FT, "{} Received invalid HTTP request (Invalid transfer key: {}).", this->client_prefix(), transfer_key.values[0]);
} else {
response.code = http::code::code(200, "OK");
response.setHeader("Content-Length", {to_string(this->pendingKey->size)});
if(!this->pendingKey->upload) {
response.setHeader("Content-type", {"application/octet-stream; "});
response.setHeader("Content-Transfer-Encoding", {"binary"});
response.setHeader("Content-Disposition", {"attachment; filename=\"" + http::encode_url(download_name ? download_name.values[0] : "TeaWeb Download") + "\""});
if(this->pendingKey->size > 1) {
char header_buffer[64];
auto read = fstream->readsome(header_buffer, 64);
if(read > 0)
response.setHeader("X-media-bytes", {base64::encode(header_buffer, read)});
fstream->seekg(this->pendingKey->offset, std::ios::beg);
}
}
}
}
if(!this->pendingKey || !this->pendingKey->upload) {
auto raw_response = response.build();
this->sendMessage(pipes::buffer_view{raw_response.data(), raw_response.length()});
}
if(response.code->code != 200) {
this->disconnect(seconds(5)) /* write our response & flush */;
return;
}
this->http_init = true;
auto overhead = this->read_buffer.range(header_end + 4);
this->read_buffer = pipes::buffer{}; /* reset the read buffer */
if(overhead.length() > 0)
this->handle_http_message(overhead);
this->handle->tickFileClient(_this.lock()); /* we require a manual reticking */
}
}
}
bool FileClient::handle_ts_message() {
if(this->state_transfer == T_INITIALIZE) {
if(this->availableBytes() >= 16) {
return this->applyKey(this->getBytes(16));
} else
return false;
} else if(this->state_transfer == T_TRANSFER) {
if(!this->pendingKey)
return false;
if(!this->pendingKey->upload)
return false; /* should never happen! */
bool reexecute = false;
pipes::buffer buffer;
{
lock_guard buffer_lock(this->bufferLock);
if(this->read_queue.empty())
return false; /* nothing to upload */
buffer = this->read_queue.front();
this->read_queue.pop_front();
reexecute |= !this->read_queue.empty();
}
if(!this->uploadWriteBytes(buffer))
return false; /* error already handeled by uploadWriteBytes(...) */
return reexecute;
}
return false;
}
void FileClient::handle_ws_message(const pipes::WSMessage &message) {
if(this->state_transfer == T_INITIALIZE) {
debugMessage(LOG_FT, "{} Received transfer key: {}", this->client_prefix(), message.data.string());
if(this->applyKey(message.data.string()))
this->handle->tickFileClient(_this.lock()); /* we require a manual reticking */
return;
} else if(this->state_transfer == T_TRANSFER) {
if(this->pendingKey->upload)
this->uploadWriteBytes(message.data);
else {
logError(LOG_FT, "{} Invalid write (Just download)", this->client_prefix());
this->disconnect();
}
}
}
void FileClient::handle_http_message(const pipes::buffer_view &message) {
if(!this->http_init) {
this->read_buffer += message;
this->handle_http_header();
return;
}
if(!https_upload_init) {
//------WebKitFormBoundaryaWP8XAzMBnMOJznv\r\nContent-Disposition: form-data; name=\"file\"; filename=\"blob\"\r\nContent-Type: application/octet-stream\r\n\r\n
this->read_buffer += message;
auto header_end = this->read_buffer.find("\r\n\r\n");
if(header_end == string::npos) {
if(this->read_buffer.length() > 1024 * 1024 * 4) {
this->read_buffer = pipes::buffer{};
logMessage(LOG_FT, "{} Client tried to fillup server memory. Disconnecting client.", this->client_prefix());
this->disconnect();
return;
}
return;
}
https_upload_init = true;
auto overhead = this->read_buffer.view(header_end + 4);
this->read_buffer = pipes::buffer{};
if(!overhead.empty())
this->handle_http_message(overhead);
return;
}
if(!this->pendingKey || !this->pendingKey->upload) {
logError(LOG_FT, "{} HTTP Invalid request", this->client_prefix());
return;
}
auto bytes_to_write = this->pendingKey->size - this->bytesHandled; /* ignore boundaries */
if(bytes_to_write < message.length())
this->uploadWriteBytes(message.view(0, bytes_to_write));
else
this->uploadWriteBytes(message);
if(this->bytesHandled == this->pendingKey->size){
http::HttpResponse response{};
response.setHeader("Connection", {"close"}); /* close the connection instance, we dont want multiple requests */
response.setHeader("Access-Control-Allow-Methods", {"GET, POST"});
response.setHeader("Access-Control-Allow-Origin", {"*"});
response.setHeader("Access-Control-Allow-Headers", {"*"});
response.setHeader("Access-Control-Max-Age", {"86400"});
response.setHeader("Access-Control-Expose-Headers", {"*"});
auto raw_response = response.build();
this->sendMessage(pipes::buffer_view{raw_response.data(), raw_response.length()});
}
}

133
server/src/client/file/FileClient.h
View File

@ -1,133 +0,0 @@
#pragma once
#include <protocol/buffers.h>
#include <poll.h>
#include <fstream>
#include <src/server/file/LocalFileServer.h>
#include <event.h>
#include <pipes/ws.h>
#include <pipes/ssl.h>
#include "src/VirtualServer.h"
namespace ts {
namespace server {
class ConnectedClient;
class ConnectedClient;
class LocalFileServer;
enum FTType {
Unknown,
TeamSpeak,
TeaWeb_SSL, /* not yet decided if the protocol is HTTP or WebSocket */
TeaWeb_SSL_WS,
TeaWeb_SSL_HTTP,
TeaWeb_HTTP
};
class FileClient {
friend class LocalFileServer;
public:
enum TransferState {
T_INITIALIZE,
T_TRANSFER,
T_DONE
};
enum ConnectionState {
C_CONNECTED,
C_DISCONNECTING,
C_DISCONNECTED
};
struct BandwidthEntry {
std::chrono::system_clock::time_point timestamp;
uint16_t length = 0;
};
FileClient(LocalFileServer* handle, int socketFd);
~FileClient();
void disconnect(std::chrono::milliseconds = std::chrono::milliseconds(5000));
FTType getFTType(){ return this->ftType; }
size_t used_bandwidth();
std::string client_prefix();
size_t transferred_bytes();
size_t remaining_bytes();
protected:
void disconnectFinal(std::unique_lock<threads::Mutex>& /* tick lock */, bool /* handle flush thread */);
bool tick();
bool uploadWriteBytes(const pipes::buffer_view&);
void close_file_handle();
bool applyKey(const std::string &);
void sendMessage(const pipes::buffer_view&);
//Direct methods & IO stuff
void sendRawMessage(const pipes::buffer_view&);
void handleMessageRead(int, short, void*);
void handleMessageWrite(int, short, void*);
void handle_ssl_message(const pipes::buffer_view&); /* handeles all decoded SSL messages */
/* http header parser. header must be stored with read buffer! */
void handle_http_header();
/* Final protocol handlers */
void handle_http_message(const pipes::buffer_view&);
void handle_ws_message(const pipes::WSMessage&);
bool handle_ts_message();
private:
LocalFileServer* handle;
std::weak_ptr<FileClient> _this;
std::recursive_mutex bandwidth_lock;
std::deque<std::unique_ptr<BandwidthEntry>> bandwidth;
sockaddr_storage remote_address;
int clientFd;
bool event_read_hold = false;
::event* readEvent = nullptr;
bool event_write_hold = false;
::event* writeEvent = nullptr;
threads::Mutex tickLock;
std::recursive_timed_mutex bufferLock;
std::deque<pipes::buffer> write_queue;
std::deque<pipes::buffer> read_queue;
pipes::buffer read_buffer; /* buffer which contains fragments of decoded data, e.g. HTTP request. Access only within tickLock locked */
FTType ftType = FTType::Unknown;
pipes::WebSocket ws_handler;
pipes::SSL ssl_handler;
bool https_upload_init = false;
bool http_init = false; /* general flag if the HTTP header has been parsed */
std::shared_ptr<ConnectedClient> client;
std::shared_ptr<file::FileTransfereKey> pendingKey = nullptr;
std::chrono::time_point<std::chrono::system_clock> last_io_action;
std::chrono::time_point<std::chrono::system_clock> connect_timestamp;
std::chrono::time_point<std::chrono::system_clock> finished_timestamp;
std::fstream* fstream = nullptr;
size_t bytesHandled = 0;
ConnectionState state_connection = ConnectionState::C_CONNECTED;
TransferState state_transfer = TransferState::T_INITIALIZE;
size_t availableBytes();
std::string getBytes(size_t);
std::string peekBytes(size_t);
std::mutex thread_flush_lock;
std::thread thread_flush;
};
}
}

248
server/src/client/file/FileClientIO.cpp
View File

@ -1,248 +0,0 @@
#include <algorithm>
#include <memory>
#include <src/server/file/LocalFileServer.h>
#include <log/LogUtils.h>
#include <misc/std_unique_ptr.h>
#include <pipes/buffer.h>
#include "FileClient.h"
using namespace std;
using namespace std::chrono;
using namespace ts::server;
void FileClient::sendRawMessage(const pipes::buffer_view &message) {
lock_guard lock(this->bufferLock);
this->write_queue.push_back(message.own_buffer());
if(!this->event_write_hold)
event_add(this->writeEvent, nullptr);
else {
__asm__("nop");
}
}
void FileClient::handleMessageWrite(int fd, short, void *) {
auto self = this->_this.lock();
if(!self) return;
decltype(this->pendingKey) pending_key;
{
lock_guard<threads::Mutex> l(this->tickLock);
pending_key = this->pendingKey;
}
unique_lock lock(this->bufferLock, defer_lock);
if(!lock.try_lock_for(milliseconds(5))) {
logWarning(LOG_FT, "{} Buffer lock locked, could not write data!", this->client_prefix());
return; /* somebody else doing a action and will (hopefully) readd the event */
}
if(self->state_connection == C_DISCONNECTED) return;
auto used = this->used_bandwidth();
if(pending_key) { //Delay the write <3 :P
if(pending_key->max_bandwhidth >= 0 && used > pending_key->max_bandwhidth) {
event_write_hold = true;
logTrace(LOG_FT, "{} Exceeded bandwidth limit of {} bytes (Used {} bytes). Temporary skipping write event!", this->client_prefix(), pending_key->max_bandwhidth, this->used_bandwidth());
return;
}
}
event_write_hold = false;
pipes::buffer* buffer = nullptr;
while(true) {
if(this->write_queue.empty()) {
lock.unlock(); /* unlock write buffer because we're ticking */
if(pending_key && !pending_key->upload)
this->handle->tickFileClient(_this.lock()); //We have to fill up again
return;
}
buffer = &this->write_queue.front();
if(buffer->empty()) {
this->write_queue.pop_front();
buffer = nullptr;
continue;
}
break;
}
ssize_t length = buffer->length();
if(pending_key && pending_key->max_bandwhidth >= 0) {
if(pending_key->max_bandwhidth < length && pending_key->max_bandwhidth > used) length = pending_key->max_bandwhidth - used;
}
length = send(fd, buffer->data_ptr(), length, MSG_NOSIGNAL);
//logTrace(LOG_FT, "{} Wrote {} bytes", this->client_prefix(), length);
if(length == -1) {
if (errno == EINTR || errno == EAGAIN) {
event_add(this->writeEvent, nullptr);
return;
}
else {
logError(LOG_FT, "{} Failed to write some data! ({}/{})", this->client_prefix(), errno, strerror(errno));
lock.unlock();
self->disconnect();
return;
}
} else {
*buffer = buffer->range(length);
auto entry = make_unique<BandwidthEntry>();
entry->length = length ;
entry->timestamp = system_clock::now();
{
lock_guard<recursive_mutex> b_lock(this->bandwidth_lock);
this->bandwidth.push_back(move(entry));
}
}
last_io_action = system_clock::now();
if(buffer->empty()) {
this->write_queue.pop_front();
buffer = nullptr;
}
if(pending_key && this->bytesHandled != pending_key->size) {
if(this->write_queue.size() < 4) this->handle->tickFileClient(_this.lock());
}
if(!this->write_queue.empty()) {
event_add(this->writeEvent, nullptr);
}
self.reset();
}
void FileClient::handleMessageRead(int fd, short, void *) {
auto self = this->_this.lock();
if(self->state_connection != C_CONNECTED) return;
decltype(this->pendingKey) pending_key;
{
lock_guard<threads::Mutex> l(this->tickLock);
pending_key = this->pendingKey;
}
size_t buffer_length = 1024;
if(pending_key && pending_key->max_bandwhidth >= 0) {
auto used = this->used_bandwidth();
if(used < pending_key->max_bandwhidth) {
buffer_length = pending_key->max_bandwhidth - used;
if(buffer_length > 1024) buffer_length = 1024;
} else {
logTrace(LOG_FT, "{} Exceeded bandwidth limit of {} bytes (Used {} bytes). Temporary removing read event!", this->client_prefix(), pending_key->max_bandwhidth, used);
{
lock_guard lock(this->bufferLock);
if(this->readEvent)
event_del_noblock(this->readEvent);
}
this->event_read_hold = true;
return;
}
}
pipes::buffer buffer(buffer_length);
auto length = recv(fd, buffer.data_ptr(), buffer.length(), 0);
if(length < 0){
if(errno == EINTR || errno == EAGAIN)
;//event_add(this->readEvent, nullptr);
else {
{
lock_guard lock(this->bufferLock);
if(this->readEvent)
event_del_noblock(this->readEvent);
}
if(this->state_connection == C_CONNECTED) {
logError(LOG_FT, "{} Failed to read some data! ({}/{})", this->client_prefix(), errno, strerror(errno));
self->disconnect();
}
}
return;
} else if(length == 0){
{
lock_guard lock(this->bufferLock);
if(this->readEvent)
event_del_noblock(this->readEvent);
}
if(this->state_connection == C_CONNECTED) {
if(this->state_transfer == T_TRANSFER)
logWarning(LOG_FT, "{} Transfer hang up. Remote peer closed the connection.", this->client_prefix());
else
logMessage(LOG_FT, "{} Remote peer has closed the connection before initializing a transfer.", this->client_prefix());
self->disconnect(seconds(3));
}
return;
}
last_io_action = system_clock::now();
buffer.resize(length);
{
lock_guard lock(this->bufferLock);
if(self->state_connection != C_CONNECTED) return; /* drop the buffer because we're not connected anymore */
this->read_queue.push_back(std::move(buffer));
}
this->handle->tickFileClient(_this.lock());
auto entry = make_unique<BandwidthEntry>();
entry->length = length ;
entry->timestamp = system_clock::now();
//logTrace(LOG_FT, "{} Readed {} bytes", this->client_prefix(), length);
{
lock_guard<recursive_mutex> lock(this->bandwidth_lock);
this->bandwidth.push_back(move(entry));
}
}
size_t FileClient::availableBytes() {
lock_guard lock(this->bufferLock);
size_t available = 0;
for(const auto& buf : this->read_queue)
available += buf.length();
return available;
}
std::string FileClient::peekBytes(size_t size) {
ssize_t required = size;
lock_guard lock(this->bufferLock);
string result;
result.reserve(size);
for(pipes::buffer& buf : this->read_queue) {
if(required <= 0) break;
if(buf.length() > required) {
result += string(buf.data_ptr<const char>(), required);
required = 0;
} else {
result += string(buf.data_ptr<const char>(), buf.length());
required -= buf.length();
}
}
return result;
}
std::string FileClient::getBytes(size_t size) {
lock_guard lock(this->bufferLock);
string result;
result.reserve(size);
while(!this->read_queue.empty()) {
if(size <= 0) break;
auto& buf = this->read_queue.front();
if(buf.length() > size) {
result += string((char*) buf.pipes::buffer_view::data_ptr(), size);
buf = buf.range(size);
size = 0;
} else {
result += string((char*) buf.data_ptr(), buf.length());
size -= buf.length();
this->read_queue.pop_front();
}
}
return result;
}

10
server/src/client/music/internal_provider/channel_replay/ChannelProvider.cpp
View File

@ -1,7 +1,6 @@
#include "ChannelProvider.h"
#include "../../MusicClient.h"
#include "../../../../InstanceHandler.h"
#include "src/server/file/LocalFileServer.h"
#include "../../../../../../music/providers/ffmpeg/FFMpegProvider.h"
@ -22,6 +21,7 @@ threads::Future<std::shared_ptr<::music::MusicPlayer>> ChannelProvider::createPl
auto server = ((VirtualServer*) ptr_server)->ref();
threads::Future<std::shared_ptr<::music::MusicPlayer>> future;
#if 0
if(server) {
std::thread([future, server, url, ptr_server]{
auto f_server = serverInstance->getFileServer();
@ -116,7 +116,9 @@ threads::Future<std::shared_ptr<::music::MusicPlayer>> ChannelProvider::createPl
} else {
future.executionFailed("invalid bot");
}
#else
future.executionFailed("channel file playback is currently not supported");
#endif
return future;
}
@ -165,6 +167,7 @@ threads::Future<shared_ptr<UrlInfo>> ChannelProvider::query_info(const std::stri
auto server = ((VirtualServer*) ptr_server)->ref();
threads::Future<shared_ptr<UrlInfo>> future;
#if 0
if(server) {
std::thread([future, server, url, ptr_server]{
auto f_server = serverInstance->getFileServer();
@ -264,6 +267,9 @@ threads::Future<shared_ptr<UrlInfo>> ChannelProvider::query_info(const std::stri
future.executionFailed("invalid bot");
}
#else
future.executionFailed("channel file playback is currently not supported");
#endif
return future;
}

3
server/src/lincense/LicenseService.cpp
View File

@ -267,6 +267,8 @@ void LicenseService::handle_client_connected() {
this->send_license_validate_request();
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
void LicenseService::handle_message(::license::protocol::PacketType type, const void *buffer, size_t size) {
switch (type) {
case ::license::protocol::PACKET_SERVER_VALIDATION_RESPONSE:
@ -286,6 +288,7 @@ void LicenseService::handle_message(::license::protocol::PacketType type, const
return;
}
}
#pragma GCC diagnostic pop
void LicenseService::handle_client_disconnected(const std::string& message) {
std::lock_guard rlock{this->request_lock};

777
server/src/server/file/LocalFileServer.cpp
View File

@ -1,777 +0,0 @@
#include "LocalFileServer.h"
#include "src/client/file/FileClient.h"
#include "src/client/ConnectedClient.h"
#include <netinet/tcp.h>
#include <experimental/filesystem>
#include <log/LogUtils.h>
#include <misc/std_unique_ptr.h>
#include "src/InstanceHandler.h"
using namespace std;
using namespace std::chrono;
using namespace ts;
using namespace ts::file;
using namespace ts::server;
namespace fs = std::experimental::filesystem;
#if defined(TCP_CORK) && !defined(TCP_NOPUSH)
#define TCP_NOPUSH TCP_CORK
#endif
extern InstanceHandler* serverInstance;
LocalFileServer::LocalFileServer() {}
LocalFileServer::~LocalFileServer() {
stop();
}
inline fs::path buildPath(std::string rootPath, std::shared_ptr<file::FileEntry> directory){
auto strPath = directory ? directory->path + "/" + directory->name : "";
if(strPath.find(rootPath) == 0) return fs::u8path(strPath);
return fs::u8path(rootPath + strPath);
}
std::shared_ptr<file::Directory> LocalFileServer::createDirectory(std::string name, std::shared_ptr<file::Directory> parent) {
auto path = buildPath(this->rootPath, parent);
path += name;
std::error_code code{};
if(!fs::exists(path, code))
if(!fs::create_directories(path, code)) return nullptr;
else ;
else if(!fs::is_directory(path, code)) return nullptr;
return static_pointer_cast<file::Directory>(this->findFile(path.string()));
}
bool LocalFileServer::fileExists(std::shared_ptr<file::Directory> dir) {
std::error_code code{};
return fs::exists(buildPath(this->rootPath, dir), code);
}
bool LocalFileServer::fileExists(std::shared_ptr<file::File> file) {
std::error_code code{};
return fs::exists(buildPath(this->rootPath, file), code);
}
std::shared_ptr<file::FileEntry> LocalFileServer::findFile(std::string path, std::shared_ptr<file::Directory> parent) {
if(path.find(this->rootPath) != 0) {
string strPath = (parent ? parent->path + "/" + parent->name : this->rootPath) + "/";
if(strPath.find(this->rootPath) != 0 && rootPath != strPath)
path = this->rootPath + strPath;
if(path.find(strPath) == 0 && !strPath.empty())
;
else path = strPath + path;
}
std::error_code code{};
fs::path absPath = fs::u8path(path);
if(!fs::is_regular_file(absPath, code) && !fs::is_directory(absPath, code)){
debugMessage(LOG_FT, "Could not find requested file. Abs path: {} | {}. (path={}, parent={})", absPath.string(), path, path, (parent ? parent->path + "/" + parent->name : "./"));
return nullptr;
}
std::shared_ptr<file::FileEntry> entry;
if(fs::is_directory(absPath, code))
entry = std::make_shared<file::Directory>();
else entry = std::make_shared<file::File>();
entry->name = absPath.filename();
entry->type = fs::is_directory(absPath, code) ? FileType::DIRECTORY : FileType::FILE;
entry->path = absPath.parent_path().string();
entry->lastChanged = fs::last_write_time(absPath, code);
if(entry->type == FileType::FILE)
static_pointer_cast<file::File>(entry)->fileSize = entry->type == FileType::FILE ? fs::file_size(absPath, code) : 0;
return entry;
}
std::vector<std::shared_ptr<file::FileEntry>> LocalFileServer::listFiles(std::shared_ptr<file::Directory> dir) {
if(!dir) return {};
auto directory = buildPath(this->rootPath, dir);
std::error_code code{};
if(!fs::exists(directory, code) || code) return {};
if(!fs::is_directory(directory, code) || code) return {};
std::vector<std::shared_ptr<file::FileEntry>> result;
deque<fs::directory_entry> files;
for(const auto& elm : fs::directory_iterator(directory, code)) {
files.push_back(elm);
}
if(code) {
logWarning(LOG_FT, "Failed to iterate over directory {}: {}", directory.string(), code.message());
return {};
}
std::stable_sort(files.begin(), files.end(), [&](const fs::directory_entry& a, const fs::directory_entry& b) {
return fs::last_write_time(a.path(), code) > fs::last_write_time(b.path(), code);
});
for(const auto& elm : files) {
if(fs::is_regular_file(elm.path(), code)){
auto entry = make_shared<file::File>();
entry->name = elm.path().filename();
entry->type = FileType::FILE;
entry->path = elm.path().parent_path().string();
entry->lastChanged = fs::last_write_time(elm.path(), code);
entry->fileSize = fs::file_size(elm.path(), code);
result.push_back(entry);
} else if(fs::is_directory(elm.path(), code)){
auto entry = make_shared<file::Directory>();
entry->name = elm.path().filename();
entry->type = FileType::DIRECTORY;
entry->path = elm.path().parent_path().string();
entry->lastChanged = fs::last_write_time(elm.path(), code);
result.push_back(entry);
} else {
logError(LOG_FT, "Invalid file in file tree. File path: " + elm.path().string());
}
}
return result;
}
bool LocalFileServer::deleteFile(std::shared_ptr<file::FileEntry> file) {
std::error_code code{};
return fs::remove_all(fs::u8path(file->path + "/" + file->name), code) > 0 && !code;
}
inline std::string randomString(uint length = 15, std::string charIndex = "abcdefghijklmnaoqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")
{
std::string rs = "";
for (uint i = 0; i < length; ++i)
rs += charIndex[rand() % charIndex.length()];
return rs;
}
std::shared_ptr<file::FileTransfereKey> LocalFileServer::generateDownloadTransferKey(std::string &errorMessage, std::string targetFile, size_t offset, const shared_ptr<ConnectedClient>& client) {
auto file = this->findFile(targetFile, nullptr);
if(!file){
errorMessage = "file does not exists";
return nullptr;
}
if(file->type != FileType::FILE) {
errorMessage = "file is a directory";
return nullptr;
}
auto result = make_shared<file::FileTransfereKey>();
result->owner = client;
result->server = client->getServerId();
result->key_id = (uint16_t) (rand() & 0xFFFF);
result->createTimestamp = std::chrono::system_clock::now();
result->key = randomString(16);
result->offset = offset;
result->size = static_pointer_cast<file::File>(file)->fileSize;
result->upload = false;
result->targetFile = targetFile;
{ //Test for server?
auto bandwidth_server = client->getServer()->properties()[property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH].as<int64_t>();
auto bandwidth_instance = serverInstance->properties()[property::SERVERINSTANCE_MAX_UPLOAD_TOTAL_BANDWIDTH].as<int64_t>();
result->max_server_bandwidth = bandwidth_server;
result->max_bandwhidth = bandwidth_server;
if(bandwidth_instance != -1) {
if(bandwidth_instance < bandwidth_server)
result->max_bandwhidth = bandwidth_instance; //We want to limit that already until exact calculations was made
}
}
if(client->getServer())
result->max_server_bandwidth = client->getServer()->properties()[property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH];
{
threads::MutexLock lock(this->keylock);
pendingKeys.push_back(result);
}
return result;
}
std::shared_ptr<file::FileTransfereKey> LocalFileServer::generateUploadTransferKey(std::string &errorMessage, std::string targetFile, size_t size, size_t offset, const shared_ptr<ConnectedClient>& client) {
shared_ptr<file::FileTransfereKey> result = make_shared<file::FileTransfereKey>();
result->owner = client;
result->server = client->getServerId();
result->key_id = (uint16_t) (rand() & 0xFFFF);
result->createTimestamp = std::chrono::system_clock::now();
result->key = randomString(16);
result->offset = offset;
result->size = size;
result->upload = true;
result->targetFile = targetFile;
{ //Test for server?
auto bandwidth_server = client->getServer()->properties()[property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH].as<int64_t>();
auto bandwidth_instance = serverInstance->properties()[property::SERVERINSTANCE_MAX_DOWNLOAD_TOTAL_BANDWIDTH].as<int64_t>();
result->max_server_bandwidth = bandwidth_server;
result->max_bandwhidth = bandwidth_server;
if(bandwidth_instance != -1) {
if(bandwidth_instance < bandwidth_server)
result->max_bandwhidth = bandwidth_instance; //We want to limit that already until exact calculations was made
}
}
{
threads::MutexLock lock(this->keylock);
pendingKeys.push_back(result);
}
debugMessage(LOG_FT, "Created file upload key=" + result->key + " for " + targetFile + " (" + to_string(size) + " bytes)");
return result;
}
std::shared_ptr<file::Directory> LocalFileServer::resolveDirectory(const shared_ptr<VirtualServer> &server, std::shared_ptr<BasicChannel> channel, std::string subPath) {
fs::path path = fs::u8path("server_" + to_string(server ? server->getServerId() : 0) + "/channel_" + to_string(channel->channelId()));
if(!findFile(path))
this->createDirectory(path.string(), nullptr);
path += subPath;
auto ffile = findFile(path.string());
debugMessage(LOG_FT, "Resolve {} => {} -> {}", path.string(), (void*) ffile.get(), typeid(ffile).name());
return static_pointer_cast<file::Directory>(ffile);
}
std::shared_ptr<file::Directory> LocalFileServer::iconDirectory(const shared_ptr<VirtualServer> &server) {
fs::path root = fs::u8path(this->rootPath);
fs::path path = fs::u8path("server_" + to_string(server ? server->getServerId() : 0) + "/icons");
std::error_code code{};
if(!fs::exists(root / path, code)) {
if(!fs::create_directories(root / path, code) || code)
return nullptr;
}
return static_pointer_cast<file::Directory>(findFile(path.string()));
}
bool LocalFileServer::iconExists(const shared_ptr<VirtualServer> &server, IconId icon) {
if(icon == 0) return false;
if(icon < 1000) return true;
return this->findFile("icon_" + to_string(icon), this->iconDirectory(server)) != nullptr;
}
std::shared_ptr<file::Directory> LocalFileServer::avatarDirectory(const shared_ptr<VirtualServer> &server) {
fs::path path = fs::u8path("server_" + to_string(server ? server->getServerId() : 0) + "/avatars");
if(!findFile(path))
this->createDirectory(path.string(), nullptr);
return static_pointer_cast<file::Directory>(findFile(path.string()));
}
#define TS3_ICON_HASH "icon_1001"
void LocalFileServer::setupServer(const shared_ptr<VirtualServer> &server) {
auto dir = iconDirectory(server);
if(!dir) {
logError(LOG_FT,"Failed to find icon directory for server {}", server ? server->getServerId() : 0);
} else {
std::error_code code{};
if(!fs::exists(fs::u8path(dir->path + "/" + dir->name + "/" TS3_ICON_HASH), code)) {
try {
fs::copy(fs::u8path("resources/teaspeak16px.png"), fs::u8path(dir->path + "/" + dir->name + "/" + TS3_ICON_HASH), code);
} catch(std::exception& ex) {
logError(LOG_FT, "Failed to copy default path: {}", ex.what());
}
}
}
}
std::string LocalFileServer::server_file_base(const std::shared_ptr<ts::server::VirtualServer> &server) {
return rootPath + "/server_" + to_string(server->getServerId());
}
void LocalFileServer::deleteServer(const shared_ptr<VirtualServer> &server) {
fs::path path = fs::u8path(rootPath + "/server_" + to_string(server ? server->getServerId() : 0));
std::error_code code{};
if(fs::exists(path, code) && !code) {
if(fs::remove_all(path, code) == 0)
logError(LOG_FT, "Could not delete server directory {} ({} | {})", path.string(), code.value(), code.message());
} else {
logError(LOG_FT, "Could not delete missing server directory (" + path.string() + ")");
}
}
//The actual server!
bool LocalFileServer::start(const std::deque<std::shared_ptr<LocalFileServer::Binding>>& bindings, std::string& error) {
if(this->running()) {
error = "server already running";
return false;
}
this->active = true;
/* reserve backup file descriptor in case that the max file descriptors have been reached */
{
this->server_reserve_fd = dup(1);
if(this->server_reserve_fd < 0)
logWarning(LOG_FT, "Failed to reserve a backup accept file descriptor. ({} | {})", errno, strerror(errno));
}
/* setup event bases */
{
this->ioLoop = event_base_new();
this->ioThread = new threads::Thread(THREAD_SAVE_OPERATIONS | THREAD_EXECUTE_LATER, [&]{
while(this->active) {
debugMessage(LOG_FT, "Entering event loop ({})", (void*) this->ioLoop);
event_base_loop(this->ioLoop, EVLOOP_NO_EXIT_ON_EMPTY);
if(this->active) {
debugMessage(LOG_FT, "Event loop exited ({}). No active events. Sleeping 1 seconds", (void*) this->ioLoop);
this_thread::sleep_for(seconds(1));
} else {
debugMessage(LOG_FT, "Event loop exited ({})", (void*) this->ioLoop);
}
}
});
this->ioThread->name("File IO #1").execute();
}
{
for(auto& binding : bindings) {
binding->file_descriptor = socket(binding->address.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0);
if(binding->file_descriptor < 0) {
logError(LOG_FT, "Failed to bind server to {}. (Failed to create socket: {} | {})", binding->as_string(), errno, strerror(errno));
continue;
}
int enable = 1, disabled = 0;
if (setsockopt(binding->file_descriptor, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int)) < 0)
logWarning(LOG_FT, "Failed to activate SO_REUSEADDR for binding {} ({} | {})", binding->as_string(), errno, strerror(errno));
if(setsockopt(binding->file_descriptor, IPPROTO_TCP, TCP_NOPUSH, &disabled, sizeof disabled) < 0)
logWarning(LOG_FT, "Failed to deactivate TCP_NOPUSH for binding {} ({} | {})", binding->as_string(), errno, strerror(errno));
if(binding->address.ss_family == AF_INET6) {
if(setsockopt(binding->file_descriptor, IPPROTO_IPV6, IPV6_V6ONLY, &enable, sizeof(int)) < 0)
logWarning(LOG_FT, "Failed to activate IPV6_V6ONLY for IPv6 binding {} ({} | {})", binding->as_string(), errno, strerror(errno));
}
if(fcntl(binding->file_descriptor, F_SETFD, FD_CLOEXEC) < 0)
logWarning(LOG_FT, "Failed to set flag FD_CLOEXEC for binding {} ({} | {})", binding->as_string(), errno, strerror(errno));
if (bind(binding->file_descriptor, (struct sockaddr *) &binding->address, sizeof(binding->address)) < 0) {
logError(LOG_FT, "Failed to bind server to {}. (Failed to bind socket: {} | {})", binding->as_string(), errno, strerror(errno));
close(binding->file_descriptor);
continue;
}
if (listen(binding->file_descriptor, SOMAXCONN) < 0) {
logError(LOG_FT, "Failed to bind server to {}. (Failed to listen: {} | {})", binding->as_string(), errno, strerror(errno));
close(binding->file_descriptor);
continue;
}
binding->event_accept = event_new(this->ioLoop, binding->file_descriptor, EV_READ | EV_PERSIST, [](int a, short b, void* c){ ((LocalFileServer *) c)->on_client_accept(a, b, c); }, this);
event_add(binding->event_accept, nullptr);
this->bindings.push_back(binding);
}
if(this->bindings.empty()) {
this->stop();
error = "failed to bind to any address";
return false;
}
}
for(int index = 0; index < 2; index++){
auto th = new threads::Thread(THREAD_SAVE_OPERATIONS | THREAD_EXECUTE_LATER, &LocalFileServer::clientTickingExecutor, this);
th->name("Ticking FT #" + to_string(index)).execute();
this->tickingThreads.push_back(th);
}
return true;
}
void LocalFileServer::stop() {
if(!this->running()) return;
active = false;
{
lock_guard<mutex> lock(this->tickingLock);
this->tickingCon.notify_all();
}
for(auto& binding : this->bindings) {
if(binding->event_accept) {
event_del_block(binding->event_accept);
event_free(binding->event_accept);
binding->event_accept = nullptr;
}
if(binding->file_descriptor > 0) {
if(shutdown(binding->file_descriptor, SHUT_RDWR) < 0)
logWarning(LOG_FT, "Failed to shutdown socket for binding {} ({} | {}).", binding->as_string(), errno, strerror(errno));
if(close(binding->file_descriptor) < 0)
logError(LOG_FT, "Failed to close socket for binding {} ({} | {}).", binding->as_string(), errno, strerror(errno));
binding->file_descriptor = -1;
}
}
this->bindings.clear();
auto clClone = this->connectedClients;
for(const auto& cl : clClone) {
cl->disconnect(chrono::milliseconds(1));
}
if(this->ioLoop) {
event_base_loopbreak(this->ioLoop);
event_base_loopexit(this->ioLoop, nullptr);
}
for(const auto& thread : this->tickingThreads){
if(thread->join(chrono::seconds(1)) != 0) logCritical(LOG_FT, "Failed to terminal file server tick thread!");
delete thread;
}
this->tickingThreads.clear();
if(this->ioThread)
if(this->ioThread->join(chrono::system_clock::now() + chrono::seconds(3)) != 0) {
logCritical(LOG_FT, "Could not terminate the event base dispatch thread of the file server");
this->ioThread->detach();
}
delete this->ioThread;
this->ioThread = nullptr;
if(this->ioLoop) {
event_base_free(this->ioLoop);
this->ioLoop = nullptr;
}
if(this->server_reserve_fd > 0) {
if(close(this->server_reserve_fd) < 0)
logError(LOG_FT, "Failed to close backup file descriptor ({} | {})", errno, strerror(errno));
}
this->server_reserve_fd = -1;
}
inline std::string logging_address(const sockaddr_storage& address) {
if(config::server::disable_ip_saving)
return "[0|X.X.X.X" + to_string(net::port(address)) + "|unconnected]";
return "[0|X.X.X.X" + net::to_string(address, true) + "|unconnected]";
}
#define CLOSE_CONNECTION \
if(shutdown(file_descriptor, SHUT_RDWR) < 0) { \
debugMessage(LOG_FT, "[{}] Failed to shutdown socket ({} | {}).", logging_address(remote_address), errno, strerror(errno)); \
} \
if(close(file_descriptor) < 0) { \
debugMessage(LOG_FT, "[{}] Failed to close socket ({} | {}).", logging_address(remote_address), errno, strerror(errno)); \
}
void LocalFileServer::on_client_accept(int _server_file_descriptor, short ev, void *arg) {
sockaddr_storage remote_address{};
memset(&remote_address, 0, sizeof(remote_address));
socklen_t address_length = sizeof(remote_address);
int file_descriptor = accept(_server_file_descriptor, (struct sockaddr *) &remote_address, &address_length);
if (file_descriptor < 0) {
if(errno == EAGAIN)
return;
if(errno == EMFILE || errno == ENFILE) {
if(errno == EMFILE)
logError(LOG_FT, "Server ran out file descriptors. Please increase the process file descriptor limit or decrease the instance variable 'serverinstance_filetransfer_max_connections'");
else
logError(LOG_FT, "Server ran out file descriptors. Please increase the process and system-wide file descriptor limit or decrease the instance variable 'serverinstance_filetransfer_max_connections'");
bool tmp_close_success = false;
{
lock_guard reserve_fd_lock(server_reserve_fd_lock);
if(this->server_reserve_fd > 0) {
debugMessage(LOG_FT, "Trying to accept client with the reserved file descriptor to close the incomming connection.");
auto _ = [&]{
if(close(this->server_reserve_fd) < 0) {
debugMessage(LOG_FT, "Failed to close reserved file descriptor");
tmp_close_success = false;
return;
}
this->server_reserve_fd = 0;
errno = 0;
file_descriptor = accept(_server_file_descriptor, (struct sockaddr *) &remote_address, &address_length);
if(file_descriptor < 0) {
if(errno == EMFILE || errno == ENFILE)
debugMessage(LOG_FT, "[{}] Even with freeing the reserved descriptor accept failed. Attempting to reclaim reserved file descriptor", logging_address(remote_address));
else if(errno == EAGAIN);
else {
debugMessage(LOG_FT, "[{}] Failed to accept client with reserved file descriptor. ({} | {})", logging_address(remote_address), errno, strerror(errno));
}
this->server_reserve_fd = dup(1);
if(this->server_reserve_fd < 0)
debugMessage(LOG_FT, "[{}] Failed to reclaim reserved file descriptor. Future clients cant be accepted!", logging_address(remote_address));
else
tmp_close_success = true;
return;
}
debugMessage(LOG_FT, "[{}] Successfully accepted client via reserved descriptor (fd: {}). Disconnecting client.", logging_address(remote_address), file_descriptor);
CLOSE_CONNECTION
this->server_reserve_fd = dup(1);
if(this->server_reserve_fd < 0)
debugMessage(LOG_FT, "Failed to reclaim reserved file descriptor. Future clients cant be accepted!");
else
tmp_close_success = true;
logMessage(LOG_FT, "[{}] Dropping file transfer connection attempt because of too many open file descriptors.", logging_address(remote_address));
};
_();
}
}
if(!tmp_close_success) {
debugMessage(LOG_FT, "Sleeping two seconds because we're currently having no resources for this user. (Removing the accept event)");
for(auto& binding : this->bindings)
event_del_noblock(binding->event_accept);
accept_event_deleted = system_clock::now();
return;
}
return;
}
logMessage(LOG_FT, "Got an error while accepting a new client. (errno: {}, message: {})", errno, strerror(errno));
return;
}
{
unique_lock lock(this->clientLock);
auto max_connections = serverInstance->properties()[property::SERVERINSTANCE_FILETRANSFER_MAX_CONNECTIONS].as<size_t>();
if(max_connections > 0 && max_connections <= this->connectedClients.size()) {
lock.unlock();
logMessage(LOG_FT, "[{}] Dropping new connection attempt because of too many connected clients.", logging_address(remote_address));
CLOSE_CONNECTION
return;
}
auto max_ip_connections = serverInstance->properties()[property::SERVERINSTANCE_FILETRANSFER_MAX_CONNECTIONS_PER_IP].as<size_t>();
if(max_ip_connections > 0) {
size_t connection_count = 0;
for(auto& client : this->connectedClients) {
if(net::address_equal(client->remote_address, remote_address))
connection_count++;
}
if(connection_count >= max_ip_connections) {
lock.unlock();
logMessage(LOG_FT, "[{}] Dropping new connection attempt because of too many simultaneously connected session from this ip.", logging_address(remote_address));
CLOSE_CONNECTION
return;
}
}
}
shared_ptr<FileClient> client = std::make_shared<FileClient>(this, file_descriptor);
client->_this = client;
memcpy(&client->remote_address, &remote_address, sizeof(remote_address));
this->clientLock.lock();
this->connectedClients.push_back(client);
this->clientLock.unlock();
event_add(client->readEvent, nullptr);
logMessage(LOG_FT, "[{}] Remote peer connected. Initializing session.", logging_address(remote_address));
}
void LocalFileServer::clientTickingExecutor() {
while(this->running()){
shared_ptr<FileClient> client;
{
unique_lock<mutex> lock(this->tickingLock);
this->tickingCon.wait(lock, [&](){ return !this->running() || !this->tickQueue.empty(); });
if(!this->running()) return;
client = this->tickQueue.front();
this->tickQueue.pop_front();
}
if(client->tick()) this->tickFileClient(client); //Client needs more ticking
}
}
std::deque<std::shared_ptr<FileClient>> LocalFileServer::running_file_transfers(const std::shared_ptr<ts::server::ConnectedClient> &client) {
std::deque<std::shared_ptr<FileClient>> result;
{
threads::MutexLock lock(this->clientLock);
for(const auto& c : this->connectedClients) {
if(!client || c->client == client) result.push_back(c);
}
}
return result;
}
std::deque<std::shared_ptr<file::FileTransfereKey>> LocalFileServer::pending_file_transfers(const std::shared_ptr<ts::server::ConnectedClient> &client) {
std::deque<std::shared_ptr<file::FileTransfereKey>> result;
for(const auto& key : this->pending_keys())
if(!client || key->owner.lock() == client)
result.push_back(key);
return result;
}
void LocalFileServer::tickFileClient(std::shared_ptr<FileClient> cl) {
lock_guard<mutex> lock(this->tickingLock);
this->tickQueue.push_back(cl);
this->tickingCon.notify_one();
}
struct TransfareGroup {
shared_ptr<VirtualServer> server;
deque<shared_ptr<FileClient>> clients;
enum {
upload,
download
} direction = upload;
size_t used_bandwidth = 0;
ssize_t max_bandwidth = -1;
};
void LocalFileServer::instanceTick() {
{
//tickQueue
auto client = this->connected_clients();
lock_guard<mutex> tick_lock(this->tickingLock);
this->tickQueue.insert(this->tickQueue.end(), client.begin(), client.end()); //Tick all clients :)
this->tickingCon.notify_all();
}
if(this->accept_event_deleted.time_since_epoch().count() != 0 && accept_event_deleted + seconds(5) < system_clock::now()) {
debugMessage(LOG_FT, "Readding accept event and try again if we have enough resources again.");
for(auto& binding : this->bindings)
event_add(binding->event_accept, nullptr);
accept_event_deleted = system_clock::time_point{};
}
auto now = system_clock::now();
if(timestamp_bandwidth_update + seconds(1) < now) {
timestamp_bandwidth_update = now;
for(const auto& entry : this->pending_keys()) {
if(!entry) continue;
if(entry->createTimestamp + minutes(1) < now) {
logMessage(entry->server, "[FILE] Timeout file transfer for file " + entry->targetFile + "! Dropping key");
{
threads::MutexLock lock(this->keylock);
auto index = find(this->pendingKeys.begin(), this->pendingKeys.end(), entry);
if(index != this->pendingKeys.end())
this->pendingKeys.erase(index);
}
}
}
deque<shared_ptr<TransfareGroup>> groups;
{
for(const auto& client : this->connected_clients()) {
if(!client || !client->client) continue;
lock_guard<threads::Mutex> l(client->tickLock); //Instance hangup here
if(!client->pendingKey) continue;
auto used_band = client->used_bandwidth();
if(client->pendingKey && client->state_connection != FileClient::C_DISCONNECTED && (client->pendingKey->max_bandwhidth < 0 || used_band < client->pendingKey->max_bandwhidth)) {
if(client->event_read_hold) { //Reactivate read!
event_add(client->readEvent, nullptr);
client->event_read_hold = false;
debugMessage(LOG_FT, "{} Reattaching client read event! (Current bandwidth {})", client->client_prefix(), used_band);
}
if(client->event_write_hold) { //Reactivate read!
event_add(client->writeEvent, nullptr);
client->event_write_hold = false;
debugMessage(LOG_FT, "{} Reattaching client write event! (Current bandwidth {})", client->client_prefix(), used_band);
}
}
if(!client->client->getServer()) continue;
bool found = false;
for(auto& group : groups) {
if (group->server == client->client->getServer() && (group->direction == TransfareGroup::upload) == client->pendingKey->upload) {
group->clients.push_back(client);
group->used_bandwidth += client->used_bandwidth();
found = true;
break;
}
}
if(found) continue;
auto entry = make_shared<TransfareGroup>();
entry->direction = client->pendingKey->upload ? TransfareGroup::upload : TransfareGroup::download;
entry->used_bandwidth = client->used_bandwidth();
entry->clients.push_back(client);
entry->server = client->client->getServer();
groups.push_back(entry);
}
}
{ //Instance limit upload
auto limit_upload = serverInstance->properties()[property::SERVERINSTANCE_MAX_UPLOAD_TOTAL_BANDWIDTH].as<ssize_t>();
if(limit_upload >= 0) {
deque<shared_ptr<TransfareGroup>> elements;
size_t used_bandwidth = 0;
for(const auto& entry : groups) {
if (entry->direction == TransfareGroup::upload) {
elements.push_back(entry);
used_bandwidth += entry->used_bandwidth;
}
}
if(!elements.empty() && used_bandwidth > limit_upload) {
for(auto& entry : elements) {
entry->max_bandwidth = ssize_t(ceil(double(entry->used_bandwidth) * double(limit_upload) / double(used_bandwidth))); //Adjust the total
}
}
}
}
{ //Instance limit download
auto limit_download = serverInstance->properties()[property::SERVERINSTANCE_MAX_DOWNLOAD_TOTAL_BANDWIDTH].as<ssize_t>();
if(limit_download >= 0) {
deque<shared_ptr<TransfareGroup>> elements;
size_t used_bandwidth = 0;
for(const auto& entry : groups) {
if (entry->direction == TransfareGroup::download) {
elements.push_back(entry);
used_bandwidth += entry->used_bandwidth;
}
}
if(!elements.empty() && used_bandwidth > limit_download) {
for(auto& entry : elements) {
entry->max_bandwidth = ssize_t(ceil(double(entry->used_bandwidth) * double(limit_download) / double(used_bandwidth))); //Adjust the total
}
}
}
}
map<ServerId, deque<shared_ptr<TransfareGroup>>> servers;
for(const auto& entry : groups) {
servers[entry->server->getServerId()].push_back(entry);
}
for(const auto& server : servers) {
if(server.second.empty()) continue;
{ //Adjust server upload
auto max_bandwidth = server.second.front()->server->properties()[property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH].as<ssize_t>();
if(max_bandwidth >= 0) {
deque<shared_ptr<TransfareGroup>> tranfares;
size_t bandwidth = 0;
for(const auto& trans : server.second) {
if(trans->direction == TransfareGroup::upload) {
tranfares.push_back(trans);
bandwidth += trans->max_bandwidth;
}
}
if(bandwidth > max_bandwidth) {
for(auto& trans : tranfares)
trans->max_bandwidth = ssize_t(ceil(double(trans->max_bandwidth) * double(max_bandwidth) / double(bandwidth))); //Adjust the total
}
}
}
{ //Adjust server download
auto max_bandwidth = server.second.front()->server->properties()[property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH].as<ssize_t>();
if(max_bandwidth >= 0) {
deque<shared_ptr<TransfareGroup>> tranfares;
size_t bandwidth = 0;
for(const auto& trans : server.second)
if(trans->direction == TransfareGroup::download) {
tranfares.push_back(trans);
bandwidth += trans->max_bandwidth;
}
if(bandwidth > max_bandwidth) {
for(auto& trans : tranfares)
trans->max_bandwidth = ssize_t(ceil(double(trans->max_bandwidth) * double(max_bandwidth) / double(bandwidth))); //Adjust the total
}
}
}
}
}
}

163
server/src/server/file/LocalFileServer.h
View File

@ -1,163 +0,0 @@
#pragma once
#include <vector>
#include <string>
#include <chrono>
#include <memory>
#include <ThreadPool/Thread.h>
#include <deque>
#include <ThreadPool/Mutex.h>
#include <netinet/in.h>
#include <event.h>
#include <condition_variable>
#include "Variable.h"
#include <Definitions.h>
#include <misc/net.h>
namespace ts {
namespace file {
namespace FileType {
enum FileType {
DIRECTORY,
FILE
};
}
}
}
DEFINE_VARIABLE_TRANSFORM(ts::file::FileType::FileType, VARTYPE_INT, std::to_string((uint8_t) in), static_cast<ts::file::FileType::FileType>(in.as<uint8_t>()));
namespace ts {
class BasicChannel;
namespace server {
class ConnectedClient;
}
namespace file {
struct FileEntry {
FileType::FileType type;
std::string path;
std::string name;
std::chrono::time_point<std::chrono::system_clock> lastChanged;
};
struct File : public FileEntry {
int64_t fileSize;
};
struct Directory : public FileEntry { };
struct SymLink : public FileEntry {
std::string targetPath;
std::string targetName;
};
struct FileTransfereKey {
uint16_t key_id = 0;
ServerId server;
std::weak_ptr<server::ConnectedClient> owner;
std::chrono::time_point<std::chrono::system_clock> createTimestamp;
std::string key;
std::string targetFile; //Relative
int64_t size;
size_t offset;
bool upload = false;
int64_t max_bandwhidth = -1; //Get calculated each time
int64_t max_server_bandwidth = -1;
};
}
namespace server {
class VirtualServer;
class FileClient;
//FIXME Valid path
class LocalFileServer {
friend class FileClient;
public:
struct Binding {
sockaddr_storage address{};
int file_descriptor = 0;
::event* event_accept = nullptr;
inline std::string as_string() { return net::to_string(address, true); }
};
LocalFileServer();
~LocalFileServer();
bool start(const std::deque<std::shared_ptr<Binding>>& /* bindings */, std::string& /* error */);
void stop();
ts_always_inline bool running(){ return active; }
ts_always_inline std::deque<std::shared_ptr<Binding>> list_bindings() { return this->bindings; }
std::shared_ptr<file::Directory> createDirectory(std::string name, std::shared_ptr<file::Directory> parent);
bool fileExists(std::shared_ptr<file::Directory>);
bool fileExists(std::shared_ptr<file::File>);
std::shared_ptr<file::FileEntry> findFile(std::string, std::shared_ptr<file::Directory> = nullptr);
std::vector<std::shared_ptr<file::FileEntry>> listFiles(std::shared_ptr<file::Directory> directory = nullptr);
bool deleteFile(std::shared_ptr<file::FileEntry>);
std::shared_ptr<file::FileTransfereKey> generateDownloadTransferKey(std::string &errorMessage, std::string targetFile, size_t offset, const std::shared_ptr<ConnectedClient>&);
std::shared_ptr<file::FileTransfereKey> generateUploadTransferKey(std::string &errorMessage, std::string targetFile, size_t size, size_t offset, const std::shared_ptr<ConnectedClient>&);
std::shared_ptr<file::Directory> resolveDirectory(const std::shared_ptr<VirtualServer> &, std::shared_ptr<BasicChannel>, std::string = "");
std::shared_ptr<file::Directory> iconDirectory(const std::shared_ptr<VirtualServer> &);
bool iconExists(const std::shared_ptr<VirtualServer> &, IconId);
std::shared_ptr<file::Directory> avatarDirectory(const std::shared_ptr<VirtualServer> &);
std::string server_file_base(const std::shared_ptr<VirtualServer> &);
void setupServer(const std::shared_ptr<VirtualServer> &);
void deleteServer(const std::shared_ptr<VirtualServer> &);
void tickFileClient(std::shared_ptr<FileClient>);
void instanceTick();
std::deque<std::shared_ptr<FileClient>> running_file_transfers(const std::shared_ptr<ConnectedClient> & /* client */ = nullptr);
std::deque<std::shared_ptr<file::FileTransfereKey>> pending_file_transfers(const std::shared_ptr<ConnectedClient> & /* client */ = nullptr);
private:
bool active = false;
std::deque<std::shared_ptr<Binding>> bindings;
std::string rootPath = "./files/";
//IO stuff
event_base* ioLoop = nullptr;
std::chrono::system_clock::time_point accept_event_deleted;
std::mutex server_reserve_fd_lock;
int server_reserve_fd = -1; /* -1 = unset | 0 = in use | > 0 ready to use */
threads::Mutex clientLock;
std::deque<std::shared_ptr<FileClient>> connectedClients;
inline std::deque<std::shared_ptr<FileClient>> connected_clients(){
threads::MutexLock lock(clientLock);
return connectedClients;
}
threads::Thread* ioThread = nullptr;
void on_client_accept(int fd, short ev, void *arg);
std::deque<std::shared_ptr<FileClient>> tickQueue;
std::deque<threads::Thread*> tickingThreads;
std::mutex tickingLock;
std::condition_variable tickingCon;
std::chrono::system_clock::time_point timestamp_bandwidth_update;
//file management
threads::Mutex keylock;
std::deque<std::shared_ptr<file::FileTransfereKey>> pendingKeys;
inline std::deque<std::shared_ptr<file::FileTransfereKey>> pending_keys(){
threads::MutexLock lock(keylock);
return pendingKeys;
}
void clientTickingExecutor();
};
}
}

2
shared

@ -1 +1 @@
Subproject commit b60608ff94b06145bc808426392871ebd95fe9d3
Subproject commit a4febf7b5af191d41c566292958c55155128f16f