// // Created by WolverinDEV on 04/05/2020. // #include #include #include #include #include #include "./LocalFileProvider.h" #include "./duration_utils.h" #include "LocalFileProvider.h" #if defined(TCP_CORK) && !defined(TCP_NOPUSH) #define TCP_NOPUSH TCP_CORK #endif using namespace ts::server::file; using namespace ts::server::file::transfer; inline void add_network_event(FileClient& transfer, event* ev, bool& ev_throttle_readd_flag, bool ignore_bandwidth) { timeval tv{0, 1}, *ptv{nullptr}; { auto timeout = transfer.networking.disconnect_timeout; if(timeout.time_since_epoch().count() > 0) { auto now = std::chrono::system_clock::now(); if(now < timeout) { auto duration = timeout - now; auto seconds = std::chrono::duration_cast(timeout - now); duration -= seconds; auto microseconds = std::chrono::duration_cast(timeout - now); tv.tv_sec = seconds.count(); tv.tv_usec = microseconds.count(); } ptv = &tv; } } if(!ignore_bandwidth) { if(ev_throttle_readd_flag) return; /* we're already throttled */ timeval ttv{}; if(transfer.networking.throttle.should_throttle(ttv)) { if(transfer.networking.event_throttle) event_add(transfer.networking.event_throttle, &ttv); ev_throttle_readd_flag = true; return; } } event_add(ev, ptv); } void FileClient::add_network_read_event(bool ignore_bandwidth) { std::shared_lock slock{this->state_mutex}; switch (this->state) { case STATE_DISCONNECTING: case STATE_DISCONNECTED: return; case STATE_AWAITING_KEY: case STATE_TRANSFERRING: break; default: assert(false); return; } if(this->state != STATE_AWAITING_KEY && this->state != STATE_TRANSFERRING) return; add_network_event(*this, this->networking.event_read, this->networking.add_event_read, ignore_bandwidth); } void FileClient::add_network_write_event(bool ignore_bandwidth) { std::shared_lock slock{this->state_mutex}; this->add_network_write_event_nolock(ignore_bandwidth); } void FileClient::add_network_write_event_nolock(bool ignore_bandwidth) { switch (this->state) { case STATE_DISCONNECTED: return; case STATE_DISCONNECTING: if(this->transfer->direction == Transfer::DIRECTION_UPLOAD) return; /* flush our write buffer */ break; case STATE_TRANSFERRING: break; //case STATE_PENDING: //case STATE_AWAITING_KEY: default: assert(false); break; } add_network_event(*this, this->networking.event_write, this->networking.add_event_write, ignore_bandwidth); } bool FileClient::send_file_bytes(const void *snd_buffer, size_t size) { if(this->networking.protocol == FileClient::PROTOCOL_TS_V1) { auto tbuffer = allocate_buffer(size); tbuffer->length = size; tbuffer->offset = 0; memcpy(tbuffer->data, snd_buffer, size); size_t buffer_size; { std::lock_guard block{this->buffer.mutex}; *this->buffer.buffer_tail = tbuffer; this->buffer.buffer_tail = &tbuffer->next; buffer_size = (this->buffer.bytes += size); } this->add_network_write_event(false); return buffer_size > TRANSFER_MAX_CACHED_BYTES; } else if(this->networking.protocol == FileClient::PROTOCOL_HTTPS) { this->networking.pipe_ssl.send(pipes::buffer_view{snd_buffer, size}); return this->buffer.bytes > TRANSFER_MAX_CACHED_BYTES; } else { return false; } } NetworkingStartResult LocalFileTransfer::start_networking() { assert(!this->network.active); this->network.active = true; this->network.event_base = event_base_new(); if(!this->network.event_base) return NetworkingStartResult::OUT_OF_MEMORY; bool bound{false}; for(auto& binding : this->network.bindings) { binding->file_descriptor = socket(binding->address.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0); if(!binding->file_descriptor) { logWarning(LOG_FT, "Failed to allocate socket for {}: {}/{}", binding->hostname, errno, strerror(errno)); continue; } int enable = 1, disabled = 0; if (setsockopt(binding->file_descriptor, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int)) < 0) logWarning(LOG_FT, "Failed to activate SO_REUSEADDR for binding {} ({} | {})", binding->hostname, errno, strerror(errno)); if(setsockopt(binding->file_descriptor, IPPROTO_TCP, TCP_NOPUSH, &disabled, sizeof disabled) < 0) logWarning(LOG_FT, "Failed to deactivate TCP_NOPUSH for binding {} ({} | {})", binding->hostname, errno, strerror(errno)); if(binding->address.ss_family == AF_INET6) { if(setsockopt(binding->file_descriptor, IPPROTO_IPV6, IPV6_V6ONLY, &enable, sizeof(int)) < 0) logWarning(LOG_FT, "Failed to activate IPV6_V6ONLY for IPv6 binding {} ({} | {})", binding->hostname, errno, strerror(errno)); } if(fcntl(binding->file_descriptor, F_SETFD, FD_CLOEXEC) < 0) logWarning(LOG_FT, "Failed to set flag FD_CLOEXEC for binding {} ({} | {})", binding->hostname, errno, strerror(errno)); if (bind(binding->file_descriptor, (struct sockaddr *) &binding->address, sizeof(binding->address)) < 0) { logError(LOG_FT, "Failed to bind server to {}. (Failed to bind socket: {} | {})", binding->hostname, errno, strerror(errno)); goto reset_binding; } if (listen(binding->file_descriptor, 8) < 0) { logError(LOG_FT, "Failed to bind server to {}. (Failed to listen: {} | {})", binding->hostname, errno, strerror(errno)); goto reset_binding; } binding->handle = this; binding->accept_event = event_new(this->network.event_base, binding->file_descriptor, (unsigned) EV_READ | (unsigned) EV_PERSIST, &LocalFileTransfer::callback_transfer_network_accept, &*binding); if(!binding->accept_event) goto reset_binding; event_add(binding->accept_event, nullptr); logMessage(LOG_FT, "Started to listen on {}:{}", binding->hostname, net::port(binding->address)); bound = true; continue; reset_binding: if(binding->accept_event) { event_free(binding->accept_event); binding->accept_event = nullptr; } if(binding->file_descriptor > 0) ::close(binding->file_descriptor); binding->file_descriptor = 0; binding->handle = nullptr; } if(!bound) { event_base_free(std::exchange(this->network.event_base, nullptr)); return NetworkingStartResult::NO_BINDINGS; } this->network.dispatch_thread = std::thread(&LocalFileTransfer::dispatch_loop_network, this); return NetworkingStartResult::SUCCESS; } void LocalFileTransfer::shutdown_networking() { if(!this->network.active) return; this->network.active = false; for(auto& binding : this->network.bindings) { if(binding->accept_event) { event_del_block(binding->accept_event); event_free(binding->accept_event); binding->accept_event = nullptr; } if(binding->file_descriptor > 0) ::close(binding->file_descriptor); binding->file_descriptor = 0; binding->handle = nullptr; } { std::unique_lock tlock{this->transfers_mutex}; auto transfers = this->transfers_; tlock.unlock(); for(const auto& transfer : transfers) { std::unique_lock slock{transfer->state_mutex}; this->disconnect_client(transfer, slock, false); } } auto ev_base = std::exchange(this->network.event_base, nullptr); if(ev_base) event_base_loopbreak(ev_base); if(this->network.dispatch_thread.joinable()) this->network.dispatch_thread.join(); if(ev_base) event_base_free(ev_base); } void LocalFileTransfer::dispatch_loop_network(void *provider_ptr) { auto provider = reinterpret_cast(provider_ptr); while(provider->network.active) { assert(provider->network.event_base); event_base_loop(provider->network.event_base, EVLOOP_NO_EXIT_ON_EMPTY); } } NetworkInitializeResult LocalFileTransfer::initialize_networking(const std::shared_ptr &client, int file_descriptor) { client->networking.file_descriptor = file_descriptor; client->networking.event_read = event_new(this->network.event_base, file_descriptor, EV_READ, &LocalFileTransfer::callback_transfer_network_read, &*client); client->networking.event_write = event_new(this->network.event_base, file_descriptor, EV_WRITE, &LocalFileTransfer::callback_transfer_network_write, &*client); client->networking.event_throttle = evtimer_new(this->network.event_base, &LocalFileTransfer::callback_transfer_network_throttle, &*client); if(!client->networking.event_read || !client->networking.event_write || !client->networking.event_throttle) goto oom_exit; client->add_network_read_event(true); client->timings.connected = std::chrono::system_clock::now(); client->timings.last_write = client->timings.connected; client->timings.last_read = client->timings.connected; return NetworkInitializeResult::SUCCESS; oom_exit: if(auto event{std::exchange(client->networking.event_read, nullptr)}; event) event_free(event); if(auto event{std::exchange(client->networking.event_write, nullptr)}; event) event_free(event); if(auto event{std::exchange(client->networking.event_throttle, nullptr)}; event) event_free(event); return NetworkInitializeResult::OUT_OF_MEMORY; } void LocalFileTransfer::finalize_networking(const std::shared_ptr &client, std::unique_lock& state_lock) { assert(state_lock.owns_lock()); auto ev_read = std::exchange(client->networking.event_read, nullptr); auto ev_write = std::exchange(client->networking.event_write, nullptr); auto ev_throttle = std::exchange(client->networking.event_throttle, nullptr); state_lock.unlock(); if (ev_read) { event_del_block(ev_read); event_free(ev_read); } if (ev_write) { event_del_block(ev_write); event_free(ev_write); } if (ev_throttle) { event_del_block(ev_throttle); event_free(ev_throttle); } state_lock.lock(); if (client->networking.file_descriptor > 0) { ::shutdown(client->networking.file_descriptor, SHUT_RDWR); ::close(client->networking.file_descriptor); } client->networking.file_descriptor = 0; } void LocalFileTransfer::callback_transfer_network_accept(int fd, short, void *ptr_binding) { auto binding = reinterpret_cast(ptr_binding); auto transfer = binding->handle; sockaddr_storage address{}; socklen_t address_length{sizeof(address)}; auto client_fd = ::accept4(fd, reinterpret_cast(&address), &address_length, SOCK_NONBLOCK); if(client_fd <= 0) { /* TODO: Reserve one file descriptor in case of out of file descriptors (see current implementation) */ logError(LOG_FT, "Failed to accept new client: {}/{}", errno, strerror(errno)); return; } auto client = std::make_shared(transfer); memcpy(&client->networking.address, &address, sizeof(sockaddr_storage)); logMessage(LOG_FT, "{} Connection received.", client->log_prefix()); auto ninit = transfer->initialize_networking(client, client_fd); switch(ninit) { case NetworkInitializeResult::SUCCESS: { std::lock_guard tlock{transfer->transfers_mutex}; transfer->transfers_.push_back(std::move(client)); return; } case NetworkInitializeResult::OUT_OF_MEMORY: client->state = FileClient::STATE_DISCONNECTED; /* required else the deallocate assert will fail */ logError(LOG_FT, "{} Failed to initialize transfer client because we ran out of memory. Closing connection.", client->log_prefix()); ::close(client_fd); return; default: client->state = FileClient::STATE_DISCONNECTED; /* required else the deallocate assert will fail */ logError(LOG_FT, "{} Failed to initialize transfer client. Closing connection.", client->log_prefix()); ::close(client_fd); return; } } void LocalFileTransfer::callback_transfer_network_throttle(int, short, void *ptr_transfer) { auto transfer = reinterpret_cast(ptr_transfer); if(std::exchange(transfer->networking.add_event_write, false)) transfer->add_network_write_event(true); if(std::exchange(transfer->networking.add_event_read, false)) transfer->add_network_read_event(true); } void LocalFileTransfer::callback_transfer_network_read(int fd, short events, void *ptr_transfer) { auto transfer = reinterpret_cast(ptr_transfer); if((unsigned) events & (unsigned) EV_TIMEOUT) { /* should never happen, receive timeouts are done via the client tick */ } if((unsigned) events & (unsigned) EV_READ) { constexpr size_t buffer_size{4096}; char buffer[buffer_size]; while(true) { const auto max_read_buffer = transfer->networking.throttle.bytes_left(); if(!max_read_buffer) break; /* network throttle */ auto read = ::recv(fd, buffer, std::min(buffer_size, std::min(max_read_buffer, transfer->networking.max_read_buffer_size)), MSG_NOSIGNAL | MSG_DONTWAIT); //logTrace(0, "Read {}, max {} | {}", read, std::min(buffer_size, std::min(max_read_buffer, transfer->networking.max_read_buffer_size)), max_read_buffer); if(read <= 0) { if(read == 0) { std::unique_lock slock{transfer->state_mutex}; auto original_state = transfer->state; transfer->handle->disconnect_client(transfer->shared_from_this(), slock, false); slock.unlock(); switch(original_state) { case FileClient::STATE_AWAITING_KEY: logMessage(LOG_FT, "{} Disconnected without sending any key or initializing a transfer.", transfer->log_prefix()); break; case FileClient::STATE_TRANSFERRING: { assert(transfer->transfer); /* since we're counting the bytes directly on read, a disconnect would mean that not all data has been transferred */ if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) { logMessage(LOG_FT, "{} Disconnected while receiving file. Received {} out of {} bytes", transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset); } else if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) { logMessage(LOG_FT, "{} Disconnected while sending file. Send {} out of {} bytes", transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset); } transfer->handle->report_transfer_statistics(transfer->shared_from_this()); if(auto callback{transfer->handle->callback_transfer_aborted}; callback) callback(transfer->transfer, { TransferError::UNEXPECTED_CLIENT_DISCONNECT, "" }); break; } default: break; } } else { if(errno == EAGAIN) { transfer->add_network_read_event(false); return; } std::unique_lock slock{transfer->state_mutex}; auto original_state = transfer->state; transfer->handle->disconnect_client(transfer->shared_from_this(), slock, false); slock.unlock(); switch(original_state) { case FileClient::STATE_AWAITING_KEY: logMessage(LOG_FT, "{} Received a read error for an unauthenticated client: {}/{}. Closing connection.", transfer->log_prefix(), errno, strerror(errno)); break; case FileClient::STATE_TRANSFERRING: assert(transfer->transfer); /* since we're counting the bytes directly on read, a disconnect would mean that not all data has been transferred */ if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) { logMessage(LOG_FT, "{} Received read error while receiving file. Received {} out of {} bytes: {}/{}", transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset, errno, strerror(errno)); } else if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) { logMessage(LOG_FT, "{} Received read error while sending file. Send {} out of {} bytes: {}/{}", transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset, errno, strerror(errno)); } transfer->handle->report_transfer_statistics(transfer->shared_from_this()); if(auto callback{transfer->handle->callback_transfer_aborted}; callback) callback(transfer->transfer, { TransferError::NETWORK_IO_ERROR, strerror(errno) }); break; default: break; } return; } return; } else { transfer->timings.last_read = std::chrono::system_clock::now(); transfer->statistics.network_bytes_received += read; bool throttle_read = transfer->networking.throttle.increase_bytes(read); if(transfer->state != FileClient::STATE_AWAITING_KEY && !(transfer->state == FileClient::STATE_TRANSFERRING && transfer->transfer->direction == Transfer::DIRECTION_UPLOAD)) { debugMessage(LOG_FT, "{} Received {} bytes without any need. Dropping them.", transfer->log_prefix(), read); return; } size_t bytes_buffered{0}; if(transfer->state == FileClient::STATE_AWAITING_KEY) { bytes_buffered = transfer->handle->handle_transfer_read(transfer->shared_from_this(), buffer, read); } else if(transfer->state == FileClient::STATE_TRANSFERRING) { if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) { bytes_buffered = transfer->handle->handle_transfer_read(transfer->shared_from_this(), buffer, read); } else { debugMessage(LOG_FT, "{} Received {} bytes without any need. Dropping them.", transfer->log_prefix(), read); } } if(transfer->state == FileClient::STATE_DISCONNECTING || transfer->state == FileClient::STATE_DISCONNECTED) break; if(bytes_buffered > TRANSFER_MAX_CACHED_BYTES) { transfer->buffer.buffering_stopped = true; debugMessage(LOG_FT, "{} Stopping network read, temp buffer full.", transfer->log_prefix()); return; /* no read event readd, buffer filled */ } if(throttle_read) break; } } transfer->add_network_read_event(false); /* read event is not persistent */ } } void LocalFileTransfer::callback_transfer_network_write(int fd, short events, void *ptr_transfer) { auto transfer = reinterpret_cast(ptr_transfer); if((unsigned) events & (unsigned) EV_TIMEOUT) { if(transfer->state == FileClient::STATE_DISCONNECTING) { { std::unique_lock slock{transfer->state_mutex}; transfer->handle->disconnect_client(transfer->shared_from_this(), slock, false); } logMessage(LOG_FT, "{} Flush timeout. Force closing connection.", transfer->log_prefix()); return; } } if((unsigned) events & (unsigned) EV_WRITE) { if(transfer->state != FileClient::STATE_DISCONNECTING && transfer->state != FileClient::STATE_TRANSFERRING) { debugMessage(LOG_FT, "{} Tried to write data to send only stream. Dropping buffers.", transfer->log_prefix()); std::unique_lock block{transfer->buffer.mutex}; auto head = std::exchange(transfer->buffer.buffer_head, nullptr); transfer->buffer.buffer_tail = &transfer->buffer.buffer_head; while(head) { auto next = head->next; free_buffer(head); head = next; } return; } Buffer* buffer{nullptr}; size_t buffer_left_size{0}; while(true) { { std::lock_guard block{transfer->buffer.mutex}; buffer = transfer->buffer.buffer_head; buffer_left_size = transfer->buffer.bytes; } if(!buffer) { break; } const auto max_write_bytes = transfer->networking.throttle.bytes_left(); if(!max_write_bytes) break; /* network throttle */ assert(buffer->offset < buffer->length); auto written = ::send(fd, buffer->data + buffer->offset, std::min(buffer->length - buffer->offset, max_write_bytes), MSG_DONTWAIT | MSG_NOSIGNAL); if(written <= 0) { if(written == 0) { /* EOF, how the hell is this event possible?! (Read should already catch it) */ logError(LOG_FT, "{} Client disconnected unexpectedly on write. Send {} bytes out of {}.", transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset); transfer->handle->report_transfer_statistics(transfer->shared_from_this()); if(auto callback{transfer->handle->callback_transfer_aborted}; callback) callback(transfer->transfer, { TransferError::UNEXPECTED_CLIENT_DISCONNECT, strerror(errno) }); { std::unique_lock slock{transfer->state_mutex}; transfer->handle->disconnect_client(transfer->shared_from_this(), slock, true); } } else { if(errno == EAGAIN) { transfer->add_network_write_event(false); break; } logError(LOG_FT, "{} Received network write error. Send {} bytes out of {}. Closing transfer.", transfer->log_prefix(), transfer->statistics.file_bytes_transferred, transfer->transfer->expected_file_size - transfer->transfer->file_offset); transfer->handle->report_transfer_statistics(transfer->shared_from_this()); if(auto callback{transfer->handle->callback_transfer_aborted}; callback) callback(transfer->transfer, { TransferError::NETWORK_IO_ERROR, strerror(errno) }); { std::unique_lock slock{transfer->state_mutex}; transfer->handle->disconnect_client(transfer->shared_from_this(), slock, false); } } return; } else { buffer->offset += written; assert(buffer->offset <= buffer->length); if(buffer->length == buffer->offset) { { std::lock_guard block{transfer->buffer.mutex}; transfer->buffer.buffer_head = buffer->next; if(!buffer->next) transfer->buffer.buffer_tail = &transfer->buffer.buffer_head; assert(transfer->buffer.bytes >= written); transfer->buffer.bytes -= written; buffer_left_size = transfer->buffer.bytes; } free_buffer(buffer); } else { std::lock_guard block{transfer->buffer.mutex}; assert(transfer->buffer.bytes >= written); transfer->buffer.bytes -= written; buffer_left_size = transfer->buffer.bytes; } transfer->timings.last_write = std::chrono::system_clock::now(); transfer->statistics.network_bytes_send += written; if(transfer->networking.throttle.increase_bytes(written)) break; /* we've to slow down */ } } if(buffer_left_size > 0) transfer->add_network_write_event(false); else if(transfer->state == FileClient::STATE_DISCONNECTING) { if(transfer->statistics.file_bytes_transferred + transfer->transfer->file_offset == transfer->transfer->expected_file_size) { logMessage(LOG_FT, "{} Finished file transfer within {}. Closing connection.", transfer->log_prefix(), duration_to_string(std::chrono::system_clock::now() - transfer->timings.key_received)); transfer->handle->report_transfer_statistics(transfer->shared_from_this()); if(auto callback{transfer->handle->callback_transfer_finished}; callback) callback(transfer->transfer); } else { debugMessage(LOG_FT, "{} Flushed output buffer.", transfer->log_prefix()); } std::unique_lock slock{transfer->state_mutex}; transfer->handle->disconnect_client(transfer->shared_from_this(), slock, false); return; } transfer->handle->enqueue_disk_io(transfer->shared_from_this()); } } inline std::string transfer_key_to_string(char key[TRANSFER_KEY_LENGTH]) { std::string result{}; result.resize(TRANSFER_KEY_LENGTH); for(size_t index{0}; index < TRANSFER_KEY_LENGTH; index++) result[index] = isprint(key[index]) ? key[index] : '.'; return result; } size_t LocalFileTransfer::handle_transfer_read(const std::shared_ptr &client, const char *buffer, size_t length) { if(client->state == FileClient::STATE_AWAITING_KEY) { if(client->networking.protocol == FileClient::PROTOCOL_UNKNOWN) { /* lets read the key bytes (16) and then decide */ if(client->transfer_key.provided_bytes < TRANSFER_KEY_LENGTH) { const auto bytes_write = std::min(TRANSFER_KEY_LENGTH - client->transfer_key.provided_bytes, length); memcpy(client->transfer_key.key + client->transfer_key.provided_bytes, buffer, bytes_write); length -= bytes_write; buffer += bytes_write; client->transfer_key.provided_bytes += bytes_write; } if(client->transfer_key.provided_bytes < TRANSFER_KEY_LENGTH) return 0; /* we need more data */ if(pipes::SSL::is_ssl((uint8_t*) client->transfer_key.key, client->transfer_key.provided_bytes)) { client->networking.protocol = FileClient::PROTOCOL_HTTPS; client->networking.max_read_buffer_size = (size_t) 1024 * 2; /* HTTPS-Header are sometimes a bit bigger. Dont cap max bandwidth here. */ debugMessage(LOG_FT, "{} Using protocol HTTPS for file transfer.", client->log_prefix()); char first_bytes[TRANSFER_KEY_LENGTH]; memcpy(first_bytes, client->transfer_key.key, TRANSFER_KEY_LENGTH); client->transfer_key.provided_bytes = 0; this->handle_transfer_read(client, first_bytes, TRANSFER_KEY_LENGTH); return this->handle_transfer_read(client, buffer, length); } else { client->networking.protocol = FileClient::PROTOCOL_TS_V1; debugMessage(LOG_FT, "{} Using protocol RAWv1 for file transfer.", client->log_prefix()); return this->handle_transfer_read(client, buffer, length); } } else if(client->networking.protocol == FileClient::PROTOCOL_TS_V1) { auto key_result = this->handle_transfer_key_provided(client); switch(key_result) { case TransferKeyApplyResult::SUCCESS: return length ? this->handle_transfer_read(client, buffer, length) : 0; case TransferKeyApplyResult::FILE_ERROR: logMessage(LOG_FT, "{} Disconnecting client because we failed to open the target file.", client->log_prefix()); return (size_t) -1; case TransferKeyApplyResult::UNKNOWN_KEY: logMessage(LOG_FT, "{} Disconnecting client because we don't recognise his key ({}).", client->log_prefix(), transfer_key_to_string(client->transfer_key.key)); return (size_t) -1; default: logMessage(LOG_FT, "{} Disconnecting client because of an unknown key initialize error ({}).", client->log_prefix(), (int) key_result); return (size_t) -1; } } else if(client->networking.protocol == FileClient::PROTOCOL_HTTPS) { /* TODO! */ /* 1. Await HTTP header * 2. Test for file transfer key & if its upload or download */ } else { logError(LOG_FT, "{} Protocol variable contains unknown protocol. Disconnecting client.", client->log_prefix()); std::unique_lock slock{client->state_mutex}; client->handle->disconnect_client(client->shared_from_this(), slock, true); return (size_t) -1; } } else if(client->state == FileClient::STATE_TRANSFERRING) { assert(client->transfer); if(client->transfer->direction != Transfer::DIRECTION_UPLOAD) { debugMessage(LOG_FT, "{} Read {} bytes from client even though we're only sending a file. Ignoring it.", client->log_prefix(), length); return 0; } if(client->networking.protocol == FileClient::PROTOCOL_HTTPS) { client->networking.pipe_ssl.process_incoming_data(pipes::buffer_view{buffer, length}); return client->buffer.bytes; /* a bit unexact but the best we could get away with it */ } else if(client->networking.protocol == FileClient::PROTOCOL_TS_V1) { client->statistics.file_bytes_transferred += length; bool transfer_finished{false}; auto writte_offset = client->statistics.file_bytes_transferred + client->transfer->file_offset; if(writte_offset > client->transfer->expected_file_size) { logMessage(LOG_FT, "{} Client send {} too many bytes (Transfer length was {}). Dropping them, flushing the disk IO and closing the transfer.", client->log_prefix(), writte_offset - client->transfer->expected_file_size, duration_to_string(std::chrono::system_clock::now() - client->timings.key_received)); length -= writte_offset - client->transfer->expected_file_size; transfer_finished = true; } else if(writte_offset == client->transfer->expected_file_size) { logMessage(LOG_FT, "{} File upload has been completed in {}. Flushing disk IO and closing the connection.", client->log_prefix(), duration_to_string(std::chrono::system_clock::now() - client->timings.key_received)); transfer_finished = true; } auto tbuffer = allocate_buffer(length); tbuffer->offset = 0; tbuffer->length = length; memcpy(tbuffer->data, buffer, length); size_t buffered_bytes; { std::lock_guard block{client->buffer.mutex}; *client->buffer.buffer_tail = tbuffer; client->buffer.buffer_tail = &tbuffer->next; buffered_bytes = (client->buffer.bytes += length); } this->enqueue_disk_io(client); if(transfer_finished) { this->report_transfer_statistics(client); if(auto callback{this->callback_transfer_finished}; callback) callback(client->transfer); std::unique_lock slock{client->state_mutex}; client->handle->disconnect_client(client->shared_from_this(), slock, true); } return buffered_bytes; } else { logWarning(LOG_FT, "{} Read message for client with unknown protocol. Dropping {} bytes.", client->log_prefix(), length); return 0; } } else { logWarning(LOG_FT, "{} Read message at invalid client state ({}). Dropping message.", client->log_prefix(), client->state); } return 0; } TransferKeyApplyResult LocalFileTransfer::handle_transfer_key_provided(const std::shared_ptr &client) { { std::lock_guard tlock{this->transfers_mutex}; for(auto it = this->pending_transfers.begin(); it != this->pending_transfers.end(); it++) { if(memcmp((*it)->transfer_key, client->transfer_key.key, TRANSFER_KEY_LENGTH) == 0) { client->transfer = *it; this->pending_transfers.erase(it); break; } } } if(!client->transfer) { logMessage(LOG_FT, "{} Received invalid transfer key. Disconnecting client.", client->log_prefix()); std::unique_lock slock{client->state_mutex}; client->handle->disconnect_client(client->shared_from_this(), slock, false); return TransferKeyApplyResult::UNKNOWN_KEY; } { std::string absolute_path{}; auto transfer = client->transfer; switch (transfer->target_type) { case Transfer::TARGET_TYPE_AVATAR: absolute_path = this->file_system_.absolute_avatar_path(transfer->server_id, transfer->target_file_path); logMessage(LOG_FT, "{} Initialized avatar transfer for avatar \"{}\" ({} bytes, transferring {} bytes).", client->log_prefix(), transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset); break; case Transfer::TARGET_TYPE_ICON: absolute_path = this->file_system_.absolute_icon_path(transfer->server_id, transfer->target_file_path); logMessage(LOG_FT, "{} Initialized icon transfer for icon \"{}\" ({} bytes, transferring {} bytes).", client->log_prefix(), transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset); break; case Transfer::TARGET_TYPE_CHANNEL_FILE: absolute_path = this->file_system_.absolute_channel_path(transfer->server_id, transfer->channel_id, transfer->target_file_path); logMessage(LOG_FT, "{} Initialized channel transfer for file \"{}/{}\" ({} bytes, transferring {} bytes).", client->log_prefix(), transfer->channel_id, transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset); break; default: logError(LOG_FT, "{} Tried to initialize client with unknown file target type ({}). Dropping transfer.", client->log_prefix(), (int) transfer->target_type); this->report_transfer_statistics(client); if(auto callback{this->callback_transfer_aborted}; callback) callback(client->transfer, { TransferError::UNKNOWN, "invalid transfer type" }); { std::unique_lock slock{client->state_mutex}; client->handle->disconnect_client(client->shared_from_this(), slock, false); } return TransferKeyApplyResult::FILE_ERROR; } debugMessage(LOG_FT, "{} Absolute file path: {}", client->log_prefix(), absolute_path); client->file.absolute_path = absolute_path; } if(client->transfer->max_bandwidth > 0) { debugMessage(LOG_FT, "{} Limit network bandwidth to {} bytes/second", client->log_prefix(), client->transfer->max_bandwidth); client->networking.throttle.set_max_bandwidth(client->transfer->max_bandwidth); } client->networking.max_read_buffer_size = (size_t) -1; /* limit max bandwidth via throttle */ auto io_init_result = this->initialize_file_io(client); if(io_init_result != FileInitializeResult::SUCCESS) { logMessage(LOG_FT, "{} Failed to initialize file {}: {}/{}. Disconnecting client.", client->log_prefix(), client->transfer->direction == Transfer::DIRECTION_UPLOAD ? "writer" : "reader", (int) io_init_result, kFileInitializeResultMessages[(int) io_init_result]); this->report_transfer_statistics(client); if(auto callback{this->callback_transfer_aborted}; callback) callback(client->transfer, { TransferError::DISK_INITIALIZE_ERROR, std::to_string((int) io_init_result) + "/" + std::string{kFileInitializeResultMessages[(int) io_init_result]} }); std::unique_lock slock{client->state_mutex}; client->handle->disconnect_client(client->shared_from_this(), slock, false); return TransferKeyApplyResult::FILE_ERROR; } { std::unique_lock slock{client->state_mutex}; if(client->state != FileClient::STATE_AWAITING_KEY) return TransferKeyApplyResult::SUCCESS; /* something disconnected the client */ client->state = FileClient::STATE_TRANSFERRING; } if(auto callback{this->callback_transfer_started}; callback) callback(client->transfer); client->timings.key_received = std::chrono::system_clock::now(); if(client->transfer->direction == Transfer::DIRECTION_DOWNLOAD) this->enqueue_disk_io(client); /* we've to take initiative */ return TransferKeyApplyResult::SUCCESS; }