#include "AudioSender.h" #include "VoiceConnection.h" #include "../ServerConnection.h" #include "../../logger.h" #include "AudioEventLoop.h" #include "../../audio/AudioMerger.h" using namespace std; using namespace tc; using namespace tc::audio; using namespace tc::audio::codec; using namespace tc::connection; VoiceSender::VoiceSender(tc::connection::VoiceConnection *handle) : handle(handle) {} VoiceSender::~VoiceSender() { audio::encode_event_loop->cancel(static_pointer_cast(this->_ref.lock())); this->clear_buffer(); /* buffer might be accessed within encode_raw_frame, but this could not be trigered while this will be deallocated! */ } bool VoiceSender::initialize_codec(std::string& error, connection::codec::value codec, size_t channels, size_t rate, bool reset_encoder) { auto& data = this->codec[codec]; bool new_allocated = !data; if(new_allocated) { data = make_unique(); data->packet_counter = 0; data->last_packet = chrono::system_clock::now(); } auto info = codec::get_info(codec); if(!info || !info->supported) { if(new_allocated) log_error(category::voice_connection, tr("Tried to send voice packet but we dont support the current codec ({})"), codec); return false; } if(!data->converter) { data->converter = info->new_converter(error); if(!data->converter) return false; } else if(reset_encoder) { data->converter->reset_encoder(); } if(!data->resampler || data->resampler->input_rate() != rate) data->resampler = make_shared(rate, data->converter->sample_rate(), data->converter->channels()); if(!data->resampler->valid()) { error = "resampler is invalid"; return false; } return true; } void VoiceSender::send_data(const void *data, size_t samples, size_t rate, size_t channels) { unique_lock lock(this->_execute_lock); if(!this->handle) { log_warn(category::voice_connection, tr("Dropping raw audio frame because of an invalid handle.")); return; } lock.unlock(); auto frame = make_unique(); frame->sample_rate = rate; frame->channels = channels; frame->buffer = pipes::buffer{(void*) data, samples * channels * 4}; frame->timestamp = chrono::system_clock::now(); { lock_guard buffer_lock(this->raw_audio_buffer_lock); this->raw_audio_buffers.push_back(move(frame)); } audio::encode_event_loop->schedule(static_pointer_cast(this->_ref.lock())); } void VoiceSender::send_stop() { unique_lock lock(this->_execute_lock); if(!this->handle) { log_warn(category::voice_connection, tr("Dropping audio end frame because of an invalid handle.")); return; } lock.unlock(); auto frame = make_unique(); frame->sample_rate = 0; frame->channels = 0; frame->buffer = pipes::buffer{nullptr, 0}; frame->timestamp = chrono::system_clock::now(); { lock_guard buffer_lock(this->raw_audio_buffer_lock); this->raw_audio_buffers.push_back(move(frame)); } audio::encode_event_loop->schedule(static_pointer_cast(this->_ref.lock())); } void VoiceSender::finalize() { lock_guard lock(this->_execute_lock); this->handle = nullptr; } void VoiceSender::event_execute(const std::chrono::system_clock::time_point &point) { static auto max_time = chrono::milliseconds(10); bool reschedule = false; auto now = chrono::system_clock::now(); while(true) { unique_lock buffer_lock(this->raw_audio_buffer_lock); if(this->raw_audio_buffers.empty()) break; if(chrono::system_clock::now() - now > max_time) { reschedule = true; break; } auto entry = move(this->raw_audio_buffers.front()); this->raw_audio_buffers.pop_front(); buffer_lock.unlock(); //TODO: Drop too old buffers! this->encode_raw_frame(entry); } if(reschedule) { log_warn(category::voice_connection, tr("Audio data decode will take longer than {} us. Enqueueing for later"), chrono::duration_cast(max_time).count()); audio::decode_event_loop->schedule(static_pointer_cast(this->_ref.lock())); } } void VoiceSender::encode_raw_frame(const std::unique_ptr &frame) { auto codec = this->_current_codec; auto& codec_data = this->codec[codec]; bool flag_head = true, flag_reset = true; if(codec_data) { if(codec_data->last_packet + chrono::seconds(1) < frame->timestamp) codec_data->packet_counter = 0; flag_head = codec_data->packet_counter < 5; flag_reset = codec_data->packet_counter == 0; codec_data->packet_counter++; codec_data->last_packet = frame->timestamp; } if(frame->channels == 0 || frame->sample_rate == 0 || frame->buffer.empty()) { lock_guard lock(this->_execute_lock); if(!this->handle) { log_warn(category::voice_connection, tr("Dropping audio end because of an invalid handle.")); return; } if(codec_data) codec_data->packet_counter = 0; auto server = this->handle->handle(); server->send_voice_data(this->_buffer, 0, codec, flag_head); return; } string error; if(flag_reset) { log_trace(category::voice_connection, tr("Resetting encoder for voice sender")); } if(!this->initialize_codec(error, codec, frame->channels, frame->sample_rate, flag_reset)) { log_error(category::voice_connection, tr("Failed to initialize codec: {}"), error); return; } auto merged_channel_byte_size = codec_data->converter->channels() * (frame->buffer.length() / frame->channels); auto estimated_resampled_byte_size = codec_data->resampler->estimated_output_size(merged_channel_byte_size); this->ensure_buffer(max(estimated_resampled_byte_size, merged_channel_byte_size)); auto codec_channels = codec_data->converter->channels(); if(!audio::merge::merge_channels_interleaved(this->_buffer, codec_channels, frame->buffer.data_ptr(), frame->channels, frame->buffer.length() / frame->channels / 4)) { log_warn(category::voice_connection, tr("Failed to merge channels to output stream channel count! Dropping local voice packet")); return; } auto resampled_samples = codec_data->resampler->process(this->_buffer, this->_buffer, merged_channel_byte_size / codec_channels / 4); if(resampled_samples <= 0) { log_error(category::voice_connection, tr("Resampler returned {}"), resampled_samples); return; } if(resampled_samples * codec_channels * 4 != codec_data->converter->bytes_per_frame()) { log_error(category::voice_connection, tr("Could not encode audio frame. Frame length is not equal to code frame length! Codec: {}, Packet: {}"), codec_data->converter->bytes_per_frame(), resampled_samples * codec_channels * 4 ); return; } char _packet_buffer[512]; auto encoded_bytes = codec_data->converter->encode(error, this->_buffer, _packet_buffer, 512); if(encoded_bytes <= 0) { log_error(category::voice_connection, tr("Failed to encode voice: {}"), error); return; } { lock_guard lock(this->_execute_lock); if(!this->handle) { log_warn(category::voice_connection, tr("Dropping audio frame because of an invalid handle.")); return; } auto server = this->handle->handle(); server->send_voice_data(_packet_buffer, encoded_bytes, codec, flag_head); } }