From b3d4bc89f18c22eb0590d2315631852b54bdf12d Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Mon, 19 Apr 2021 19:08:44 +0200 Subject: [PATCH] Bumped electron version and adding the possibility to dynamically change the device output sample rate --- native/CMakeLists.txt | 2 +- .../src/audio/AudioLevelMeter.cpp | 12 ++++--- .../src/audio/AudioOutput.cpp | 36 +++++++++++-------- .../serverconnection/src/audio/AudioOutput.h | 4 ++- .../src/audio/driver/AudioDriver.cpp | 10 +++--- .../src/audio/driver/AudioDriver.h | 10 ++++-- .../src/audio/driver/PortAudio.h | 2 +- .../src/audio/driver/PortAudioPlayback.cpp | 10 ++++-- .../src/audio/js/AudioProcessor.cpp | 8 ++--- .../src/audio/js/AudioRecorder.cpp | 1 + .../src/audio/processing/AudioProcessor.cpp | 1 + 11 files changed, 59 insertions(+), 37 deletions(-) diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 77c5719..7064f68 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -23,7 +23,7 @@ message("Module path: ${CMAKE_MODULE_PATH}") function(setup_nodejs) set(NodeJS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") set(NODEJS_URL "https://atom.io/download/atom-shell") - set(NODEJS_VERSION "v8.0.0") + set(NODEJS_VERSION "v8.5.5") #set(NODEJS_URL "https://nodejs.org/download/release/") #set(NODEJS_VERSION "v12.13.0") diff --git a/native/serverconnection/src/audio/AudioLevelMeter.cpp b/native/serverconnection/src/audio/AudioLevelMeter.cpp index 74735a7..ade5a72 100644 --- a/native/serverconnection/src/audio/AudioLevelMeter.cpp +++ b/native/serverconnection/src/audio/AudioLevelMeter.cpp @@ -83,11 +83,13 @@ bool InputDeviceAudioLevelMeter::running() const { } void InputDeviceAudioLevelMeter::stop() { - std::lock_guard lock{this->mutex}; - if(this->recorder_instance) { - this->recorder_instance->remove_consumer(this); - this->recorder_instance->stop_if_possible(); - this->recorder_instance = nullptr; + std::unique_lock lock{this->mutex}; + auto recorder_instance_ = std::exchange(this->recorder_instance, nullptr); + if(recorder_instance_) { + lock.unlock(); + /* The recorder might wait for us in this right moment */ + recorder_instance_->remove_consumer(this); + recorder_instance_->stop_if_possible(); } } diff --git a/native/serverconnection/src/audio/AudioOutput.cpp b/native/serverconnection/src/audio/AudioOutput.cpp index f36a863..15847aa 100644 --- a/native/serverconnection/src/audio/AudioOutput.cpp +++ b/native/serverconnection/src/audio/AudioOutput.cpp @@ -72,8 +72,22 @@ void AudioOutput::ensure_chunk_buffer_space(size_t output_samples) { } } -void AudioOutput::fill_buffer(void *output, size_t request_sample_count, size_t out_channels) { +void AudioOutput::fill_buffer(void *output, size_t request_sample_count, size_t out_sample_rate, size_t out_channels) { assert(output); + assert(this->playback_); + if(!this->resampler_ || this->current_output_sample_rate != out_sample_rate) { + log_info(category::audio, tr("Output sample rate changed from {} to {}"), this->resampler_ ? this->resampler_->output_rate() : 0, out_sample_rate); + this->current_output_sample_rate = out_sample_rate; + + this->resampler_ = std::make_unique(this->sample_rate(), out_sample_rate, this->channel_count()); + if(!this->resampler_->valid()) { + log_critical(category::audio, tr("Failed to allocate a new resampler. Audio output will be silent.")); + } + } + + if(!this->resampler_->valid()) { + return; + } if(out_channels != this->current_output_channels) { log_info(category::audio, tr("Output channel count changed from {} to {}"), this->current_output_channels, out_channels); @@ -86,6 +100,10 @@ void AudioOutput::fill_buffer(void *output, size_t request_sample_count, size_t this->chunk_buffer_sample_offset = 0; } + return this->fill_buffer_nochecks(output, request_sample_count, out_channels); +} + +void AudioOutput::fill_buffer_nochecks(void *output, size_t request_sample_count, size_t out_channels) { auto remaining_samples{request_sample_count}; auto remaining_buffer{output}; @@ -109,8 +127,7 @@ void AudioOutput::fill_buffer(void *output, size_t request_sample_count, size_t this->fill_chunk_buffer(); this->chunk_buffer_sample_offset = 0; - - return this->fill_buffer(remaining_buffer, remaining_samples, out_channels); + return this->fill_buffer_nochecks(remaining_buffer, remaining_samples, out_channels); } constexpr static auto kTempChunkBufferSize{64 * 1024}; @@ -222,7 +239,7 @@ void AudioOutput::fill_chunk_buffer() { assert(kTempChunkBufferSize >= this->current_output_channels * this->chunk_buffer_sample_length * sizeof(float)); audio::deinterleave(temp_chunk_buffer, (const float*) this->chunk_buffer, this->current_output_channels, this->chunk_buffer_sample_length); - webrtc::StreamConfig stream_config{(int) this->playback_->sample_rate(), this->current_output_channels}; + webrtc::StreamConfig stream_config{(int) this->current_output_sample_rate, this->current_output_channels}; float* channel_ptr[kMaxChannelCount]; for(size_t channel{0}; channel < this->current_output_channels; channel++) { @@ -237,7 +254,7 @@ void AudioOutput::fill_chunk_buffer() { return; zero_chunk_exit: { - this->chunk_buffer_sample_length = (this->playback_->sample_rate() * kChunkTimeMs) / 1000; + this->chunk_buffer_sample_length = (this->current_output_sample_rate * kChunkTimeMs) / 1000; this->ensure_chunk_buffer_space(this->chunk_buffer_sample_length); memset(this->chunk_buffer, 0, this->chunk_buffer_sample_length * this->current_output_channels * sizeof(float)); return; @@ -282,15 +299,6 @@ bool AudioOutput::playback(std::string& error) { return false; } - if(this->playback_->sample_rate() != this->sample_rate()) { - this->resampler_ = std::make_unique(this->sample_rate(), this->playback_->sample_rate(), this->channel_count()); - if(!this->resampler_->valid()) { - error = "failed to allocate a resampler"; - this->playback_ = nullptr; - return false; - } - } - this->ensure_chunk_buffer_space(0); this->playback_->register_source(this); return this->playback_->start(error); diff --git a/native/serverconnection/src/audio/AudioOutput.h b/native/serverconnection/src/audio/AudioOutput.h index 66d6555..9689ef0 100644 --- a/native/serverconnection/src/audio/AudioOutput.h +++ b/native/serverconnection/src/audio/AudioOutput.h @@ -153,7 +153,8 @@ namespace tc::audio { /* One audio chunk should be 10ms long */ constexpr static auto kChunkTimeMs{10}; - void fill_buffer(void *, size_t request_sample_count, size_t out_channels) override; + void fill_buffer(void *, size_t request_sample_count, size_t sample_rate, size_t out_channels) override; + void fill_buffer_nochecks(void *, size_t request_sample_count, size_t out_channels); void fill_chunk_buffer(); size_t const channel_count_; @@ -183,6 +184,7 @@ namespace tc::audio { size_t chunk_buffer_sample_length{0}; size_t chunk_buffer_sample_offset{0}; + size_t current_output_sample_rate{0}; size_t current_output_channels{0}; void ensure_chunk_buffer_space(size_t /* output samples */); diff --git a/native/serverconnection/src/audio/driver/AudioDriver.cpp b/native/serverconnection/src/audio/driver/AudioDriver.cpp index 91f1188..a0f513c 100644 --- a/native/serverconnection/src/audio/driver/AudioDriver.cpp +++ b/native/serverconnection/src/audio/driver/AudioDriver.cpp @@ -185,21 +185,21 @@ namespace tc::audio { } #define TMP_BUFFER_SIZE (4096 * 16) /* 64k */ - void AudioDevicePlayback::fill_buffer(void *buffer, size_t samples, size_t channels) { + void AudioDevicePlayback::fill_buffer(void *buffer, size_t samples, size_t sample_rate, size_t channels) { std::lock_guard lock{this->source_lock}; if(!buffer) { for(auto& source : this->_sources) { - source->fill_buffer(nullptr, samples, channels); + source->fill_buffer(nullptr, samples, sample_rate, channels); } return; } const auto size = this->_sources.size(); if(size == 1) { - this->_sources.front()->fill_buffer(buffer, samples, channels); + this->_sources.front()->fill_buffer(buffer, samples, sample_rate, channels); } else if(size > 1) { - this->_sources.front()->fill_buffer(buffer, samples, channels); + this->_sources.front()->fill_buffer(buffer, samples, sample_rate, channels); uint8_t tmp_buffer[TMP_BUFFER_SIZE]; if(sizeof(float) * samples * channels > TMP_BUFFER_SIZE) { log_warn(category::audio, tr("Dropping input source data because of too small merge buffer")); @@ -207,7 +207,7 @@ namespace tc::audio { } for(auto it = this->_sources.begin() + 1; it != this->_sources.end(); it++) { - (*it)->fill_buffer(tmp_buffer, samples, channels); + (*it)->fill_buffer(tmp_buffer, samples, sample_rate, channels); merge::merge_sources(buffer, buffer, tmp_buffer, channels, samples); } } else { diff --git a/native/serverconnection/src/audio/driver/AudioDriver.h b/native/serverconnection/src/audio/driver/AudioDriver.h index eb04709..80d56b8 100644 --- a/native/serverconnection/src/audio/driver/AudioDriver.h +++ b/native/serverconnection/src/audio/driver/AudioDriver.h @@ -43,10 +43,14 @@ namespace tc::audio { public: class Source { public: - virtual void fill_buffer(void* /* target */, size_t /* samples */, size_t /* channel count */) = 0; + virtual void fill_buffer(void* /* target */, size_t /* samples */, size_t /* sample rate */, size_t /* channel count */) = 0; }; - [[nodiscard]] virtual size_t sample_rate() const = 0; + /** + * Get the current playback sample rate. + * Note: If the playback hasn't been started it might be zero. + */ + [[nodiscard]] virtual size_t current_sample_rate() const = 0; [[nodiscard]] bool start(std::string& /* error */); void stop_if_possible(); @@ -65,7 +69,7 @@ namespace tc::audio { virtual bool impl_start(std::string& /* error */) = 0; virtual void impl_stop() = 0; - void fill_buffer(void* /* target */, size_t /* samples */, size_t /* channel count */); + void fill_buffer(void* /* target */, size_t /* samples */, size_t /* sample rate */, size_t /* channel count */); std::mutex state_lock{}; bool running{false}; diff --git a/native/serverconnection/src/audio/driver/PortAudio.h b/native/serverconnection/src/audio/driver/PortAudio.h index 573a495..4f5f11a 100644 --- a/native/serverconnection/src/audio/driver/PortAudio.h +++ b/native/serverconnection/src/audio/driver/PortAudio.h @@ -16,7 +16,7 @@ namespace tc::audio::pa { explicit PortAudioPlayback(PaDeviceIndex index, const PaDeviceInfo* info); virtual ~PortAudioPlayback(); - [[nodiscard]] size_t sample_rate() const override; + [[nodiscard]] size_t current_sample_rate() const override; protected: bool impl_start(std::string& /* error */) override; void impl_stop() override; diff --git a/native/serverconnection/src/audio/driver/PortAudioPlayback.cpp b/native/serverconnection/src/audio/driver/PortAudioPlayback.cpp index de22982..00d8722 100644 --- a/native/serverconnection/src/audio/driver/PortAudioPlayback.cpp +++ b/native/serverconnection/src/audio/driver/PortAudioPlayback.cpp @@ -18,7 +18,7 @@ PortAudioPlayback::~PortAudioPlayback() { } bool PortAudioPlayback::impl_start(std::string &error) { - //TODO: Detect correct sample rate + //TODO: Detect a supported sample rate and use that { auto device_info = Pa_GetDeviceInfo(this->index); if(this->info != device_info) { @@ -56,6 +56,7 @@ bool PortAudioPlayback::impl_start(std::string &error) { parameters.channelCount = (int) kChannelCount; this->source_channel_count = kChannelCount; } + log_debug(category::audio, "Opening playback device {} (MaxChannels: {}, Channels: {})", this->info->name, this->info->maxOutputChannels, this->source_channel_count); auto err = Pa_OpenStream( @@ -99,11 +100,14 @@ void PortAudioPlayback::impl_stop() { this->stream = nullptr; } -size_t PortAudioPlayback::sample_rate() const { +size_t PortAudioPlayback::current_sample_rate() const { + /* We currently only support one sample rate */ return (size_t) kSampleRate; } void PortAudioPlayback::write_callback(void *output, unsigned long frameCount, const PaStreamCallbackTimeInfo *timeInfo, PaStreamCallbackFlags statusFlags) { - this->fill_buffer(output, frameCount, this->source_channel_count); + (void) timeInfo; + (void) statusFlags; + this->fill_buffer(output, frameCount, kSampleRate, this->source_channel_count); } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioProcessor.cpp b/native/serverconnection/src/audio/js/AudioProcessor.cpp index 5b863df..8294c3e 100644 --- a/native/serverconnection/src/audio/js/AudioProcessor.cpp +++ b/native/serverconnection/src/audio/js/AudioProcessor.cpp @@ -211,13 +211,13 @@ inline bool load_config_value( return false; } - if(value < (double) min_value) { - Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " exceeds min value of " + std::to_string(min_value))); + if((T) value < min_value) { + Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " exceeds min value of " + std::to_string((T) min_value) + " (value: " + std::to_string((T) value) + ")")); return false; } - if(value > (double) max_value) { - Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " exceeds max value of " + std::to_string(max_value))); + if((T) value > (double) max_value) { + Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " exceeds max value of " + std::to_string((T) max_value) + " (value: " + std::to_string((T) value) + ")")); return false; } diff --git a/native/serverconnection/src/audio/js/AudioRecorder.cpp b/native/serverconnection/src/audio/js/AudioRecorder.cpp index 693e3c5..e697362 100644 --- a/native/serverconnection/src/audio/js/AudioRecorder.cpp +++ b/native/serverconnection/src/audio/js/AudioRecorder.cpp @@ -50,6 +50,7 @@ NAN_MODULE_INIT(AudioRecorderWrapper::Init) { Nan::SetPrototypeMethod(klass, "delete_consumer", AudioRecorderWrapper::_delete_consumer); Nan::SetPrototypeMethod(klass, "get_audio_processor", AudioRecorderWrapper::get_audio_processor); + Nan::SetPrototypeMethod(klass, "create_level_meter", AudioRecorderWrapper::create_level_meter); constructor().Reset(Nan::GetFunction(klass).ToLocalChecked()); } diff --git a/native/serverconnection/src/audio/processing/AudioProcessor.cpp b/native/serverconnection/src/audio/processing/AudioProcessor.cpp index 88d57b1..bfa033b 100644 --- a/native/serverconnection/src/audio/processing/AudioProcessor.cpp +++ b/native/serverconnection/src/audio/processing/AudioProcessor.cpp @@ -125,6 +125,7 @@ void AudioProcessor::apply_config_unlocked(const Config &config) { if(!this->current_config.rnnoise.enabled) { this->rnnoise_volume = absl::nullopt; } + log_trace(category::audio, tr("Applying process config:\n{}\nRNNoise: "), config.ToString(), this->current_config.rnnoise.enabled); } AudioProcessor::Stats AudioProcessor::get_statistics() const {