diff --git a/native/serverconnection/exports/exports.d.ts b/native/serverconnection/exports/exports.d.ts index 6833686..4033ebf 100644 --- a/native/serverconnection/exports/exports.d.ts +++ b/native/serverconnection/exports/exports.d.ts @@ -324,7 +324,7 @@ export namespace audio { export type DeviceSetResult = "success" | "invalid-device"; export interface AudioRecorder { get_device() : string; - set_device(device_id: string, callback: (result: DeviceSetResult) => void); /* Recorder needs to be started afterwards */ + set_device(deviceId: string, callback: (result: DeviceSetResult) => void); /* Recorder needs to be started afterwards */ start(callback: (result: boolean | string) => void); started() : boolean; @@ -338,6 +338,7 @@ export namespace audio { delete_consumer(consumer: AudioConsumer); get_audio_processor() : AudioProcessor | undefined; + create_level_meter(mode: "pre-process" | "post-process") : AudioLevelMeter; } export interface AudioLevelMeter { @@ -348,7 +349,7 @@ export namespace audio { set_callback(callback: (level: number) => void, updateInterval?: number); } - export function create_level_meter(targetDeviceId: string) : AudioLevelMeter; + export function create_device_level_meter(targetDeviceId: string) : AudioLevelMeter; export function create_recorder() : AudioRecorder; } diff --git a/native/serverconnection/src/audio/AudioInput.cpp b/native/serverconnection/src/audio/AudioInput.cpp index 86798b0..123fbc7 100644 --- a/native/serverconnection/src/audio/AudioInput.cpp +++ b/native/serverconnection/src/audio/AudioInput.cpp @@ -8,6 +8,7 @@ #include "./AudioGain.h" #include "./AudioInterleaved.h" #include "./AudioOutput.h" +#include "./AudioLevelMeter.h" #include "./processing/AudioProcessor.h" #include "../logger.h" #include "AudioEventLoop.h" @@ -192,6 +193,18 @@ std::shared_ptr AudioInput::create_consumer(size_t frame_length) return result; } +std::shared_ptr AudioInput::create_level_meter(bool preprocess) { + auto level_meter = std::make_shared(); + + { + std::lock_guard lock{this->consumers_mutex}; + auto& list = preprocess ? this->level_meter_preprocess : this->level_meter_postprocess; + list.push_back(level_meter); + } + + return level_meter; +} + void AudioInput::allocate_input_buffer_samples(size_t samples) { const auto expected_byte_size = samples * this->channel_count_ * sizeof(float); if(expected_byte_size > this->input_buffer.capacity()) { @@ -267,12 +280,12 @@ void AudioInput::process_audio() { * It's save to mutate the current memory. * If overflows occur it could lead to wired artifacts but all memory access is save. */ - this->process_audio_chunk((void*) input); + this->process_audio_chunk((float*) input); this->input_buffer.advance_read_ptr(chunk_sample_count * this->channel_count_ * sizeof(float)); } } -void AudioInput::process_audio_chunk(void *chunk) { +void AudioInput::process_audio_chunk(float *chunk) { constexpr static auto kTempSampleBufferSize{1024 * 8}; constexpr static auto kMaxChannelCount{32}; @@ -282,10 +295,11 @@ void AudioInput::process_audio_chunk(void *chunk) { assert(memset(temp_sample_buffer, 0, sizeof(float) * kTempSampleBufferSize)); assert(memset(out_sample_buffer, 0, sizeof(float) * kTempSampleBufferSize)); + this->invoke_level_meter(true, chunk, this->channel_count_, chunk_sample_count); if(auto processor{this->audio_processor_}; processor) { assert(kTempSampleBufferSize >= chunk_sample_count * this->channel_count_ * sizeof(float)); - audio::deinterleave(temp_sample_buffer, (const float*) chunk, this->channel_count_, chunk_sample_count); + audio::deinterleave(temp_sample_buffer, chunk, this->channel_count_, chunk_sample_count); webrtc::StreamConfig stream_config{(int) this->sample_rate_, this->channel_count_}; float* channel_ptr[kMaxChannelCount]; @@ -304,8 +318,8 @@ void AudioInput::process_audio_chunk(void *chunk) { chunk = out_sample_buffer; } - /* TODO: Is this even needed if we have the processor? */ audio::apply_gain(chunk, this->channel_count_, chunk_sample_count, this->volume_); + this->invoke_level_meter(false, out_sample_buffer, this->channel_count_, chunk_sample_count); auto begin = std::chrono::system_clock::now(); for(const auto& consumer : this->consumers()) { @@ -322,3 +336,32 @@ void AudioInput::process_audio_chunk(void *chunk) { void AudioInput::EventLoopCallback::event_execute(const chrono::system_clock::time_point &point) { this->input->process_audio(); } + +void AudioInput::invoke_level_meter(bool preprocess, const float *buffer, size_t channel_count, size_t sample_size) { + std::vector> level_meters{}; + level_meters.reserve(10); + + { + std::lock_guard lock{this->consumers_mutex}; + auto& list = preprocess ? this->level_meter_preprocess : this->level_meter_postprocess; + level_meters.reserve(list.size()); + + list.erase(std::remove_if(list.begin(), list.end(), [&](const std::weak_ptr& weak_meter) { + auto meter = weak_meter.lock(); + if(!meter) { + return true; + } + + level_meters.push_back(meter); + return false; + }), list.end()); + } + + for(const auto& level_meter : level_meters) { + if(!level_meter->active) { + continue; + } + + level_meter->analyze_buffer(buffer, channel_count, sample_size); + } +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioInput.h b/native/serverconnection/src/audio/AudioInput.h index 4acac1c..c235585 100644 --- a/native/serverconnection/src/audio/AudioInput.h +++ b/native/serverconnection/src/audio/AudioInput.h @@ -15,7 +15,7 @@ namespace tc::audio { class InputReframer; class AudioResampler; class AudioProcessor; - class AudioInputSource; + class AudioInputAudioLevelMeter; class AudioConsumer { friend class AudioInput; @@ -37,7 +37,6 @@ namespace tc::audio { }; class AudioInput : public AudioDeviceRecord::Consumer { - friend class AudioInputSource; public: AudioInput(size_t /* channels */, size_t /* sample rate */); virtual ~AudioInput(); @@ -52,6 +51,7 @@ namespace tc::audio { [[nodiscard]] std::vector> consumers(); [[nodiscard]] std::shared_ptr create_consumer(size_t /* frame size */); + [[nodiscard]] std::shared_ptr create_level_meter(bool /* pre process */); [[nodiscard]] inline auto audio_processor() { return this->audio_processor_; } @@ -78,13 +78,16 @@ namespace tc::audio { void consume(const void *, size_t, size_t) override; void process_audio(); - void process_audio_chunk(void *); + void process_audio_chunk(float *); size_t const channel_count_; size_t const sample_rate_; std::mutex consumers_mutex{}; std::deque> consumers_{}; + std::deque> level_meter_preprocess{}; + std::deque> level_meter_postprocess{}; + std::recursive_mutex input_source_lock{}; std::shared_ptr event_loop_entry{}; @@ -102,5 +105,7 @@ namespace tc::audio { void allocate_input_buffer_samples(size_t /* sample count */); [[nodiscard]] inline auto chunk_sample_count() const { return (kChunkSizeMs * this->sample_rate_) / 1000; } + + void invoke_level_meter(bool /* preprocess */, const float* /* buffer */, size_t /* channel count */, size_t /* sample size */); }; } \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioLevelMeter.cpp b/native/serverconnection/src/audio/AudioLevelMeter.cpp index ebc24d6..74735a7 100644 --- a/native/serverconnection/src/audio/AudioLevelMeter.cpp +++ b/native/serverconnection/src/audio/AudioLevelMeter.cpp @@ -9,18 +9,55 @@ using namespace tc::audio; -AudioLevelMeter::AudioLevelMeter(std::shared_ptr target_device) : target_device{std::move(target_device)} { - log_allocate("AudioLevelMeter", this); +AbstractAudioLevelMeter::AbstractAudioLevelMeter() { + log_allocate("AbstractAudioLevelMeter", this); +} + +AbstractAudioLevelMeter::~AbstractAudioLevelMeter() { + log_free("AbstractAudioLevelMeter", this); +} + +void AbstractAudioLevelMeter::register_observer(Observer *observer) { + std::lock_guard lock{this->mutex}; + this->registered_observer.push_back(observer); +} + +bool AbstractAudioLevelMeter::unregister_observer(Observer *observer) { + std::lock_guard lock{this->mutex}; + auto index = std::find(this->registered_observer.begin(), this->registered_observer.end(), observer); + if(index == this->registered_observer.end()) { + return false; + } + + this->registered_observer.erase(index); + return true; +} + +void AbstractAudioLevelMeter::analyze_buffer(const float *buffer, size_t channel_count, size_t sample_count) { + auto volume = audio::audio_buffer_level(buffer, channel_count, sample_count); + + std::lock_guard lock{this->mutex}; + if(volume == this->current_audio_volume) { + return; + } + this->current_audio_volume = volume; + + for(auto& observer : this->registered_observer) { + observer->input_level_changed(volume); + } +} + +/* For a target device */ +InputDeviceAudioLevelMeter::InputDeviceAudioLevelMeter(std::shared_ptr target_device) : target_device{std::move(target_device)} { assert(this->target_device); } -AudioLevelMeter::~AudioLevelMeter() { +InputDeviceAudioLevelMeter::~InputDeviceAudioLevelMeter() { this->stop(); - log_free("AudioLevelMeter", this); } -bool AudioLevelMeter::start(std::string &error) { - std::lock_guard lock{this->recorder_mutex}; +bool InputDeviceAudioLevelMeter::start(std::string &error) { + std::lock_guard lock{this->mutex}; if(this->recorder_instance) { return true; } @@ -40,13 +77,13 @@ bool AudioLevelMeter::start(std::string &error) { return true; } -bool AudioLevelMeter::running() const { - std::lock_guard lock{this->recorder_mutex}; +bool InputDeviceAudioLevelMeter::running() const { + std::lock_guard lock{this->mutex}; return this->recorder_instance != nullptr; } -void AudioLevelMeter::stop() { - std::lock_guard lock{this->recorder_mutex}; +void InputDeviceAudioLevelMeter::stop() { + std::lock_guard lock{this->mutex}; if(this->recorder_instance) { this->recorder_instance->remove_consumer(this); this->recorder_instance->stop_if_possible(); @@ -54,33 +91,23 @@ void AudioLevelMeter::stop() { } } -void AudioLevelMeter::register_observer(Observer *observer) { - std::lock_guard lock{this->recorder_mutex}; - this->registered_observer.push_back(observer); +/* Note the parameter order! */ +void InputDeviceAudioLevelMeter::consume(const void *buffer, size_t sample_count, size_t channel_count) { + this->analyze_buffer((const float*) buffer, channel_count, sample_count); } -bool AudioLevelMeter::unregister_observer(Observer *observer) { - std::lock_guard lock{this->recorder_mutex}; - auto index = std::find(this->registered_observer.begin(), this->registered_observer.end(), observer); - if(index == this->registered_observer.end()) { - return false; - } - - this->registered_observer.erase(index); +bool AudioInputAudioLevelMeter::start(std::string &) { + std::lock_guard lock{this->mutex}; + this->active = true; return true; } -/* Note the parameter order! */ -void AudioLevelMeter::consume(const void *buffer, size_t sample_count, size_t channel_count) { - auto volume = audio::audio_buffer_level((float*) buffer, channel_count, sample_count); - - std::lock_guard lock{this->recorder_mutex}; - if(volume == this->current_audio_volume) { - return; - } - this->current_audio_volume = volume; - - for(auto& observer : this->registered_observer) { - observer->input_level_changed(volume); - } +void AudioInputAudioLevelMeter::stop() { + std::lock_guard lock{this->mutex}; + this->active = false; } + +bool AudioInputAudioLevelMeter::running() const { + std::lock_guard lock{this->mutex}; + return this->active; +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioLevelMeter.h b/native/serverconnection/src/audio/AudioLevelMeter.h index db7823f..969f265 100644 --- a/native/serverconnection/src/audio/AudioLevelMeter.h +++ b/native/serverconnection/src/audio/AudioLevelMeter.h @@ -1,37 +1,75 @@ #pragma once + #include "./driver/AudioDriver.h" namespace tc::audio { + class AudioInput; + /** * Note: Within the observer callback no methods of the level meter should be called nor the level meter should be destructed. */ - class AudioLevelMeter : public AudioDeviceRecord::Consumer { + class AbstractAudioLevelMeter { public: struct Observer { public: virtual void input_level_changed(float /* new level */) = 0; }; - explicit AudioLevelMeter(std::shared_ptr /* target device */); - virtual ~AudioLevelMeter(); + explicit AbstractAudioLevelMeter(); + virtual ~AbstractAudioLevelMeter(); - [[nodiscard]] bool start(std::string& /* error */); - void stop(); - [[nodiscard]] bool running() const; + + [[nodiscard]] virtual bool start(std::string& /* error */) = 0; + virtual void stop() = 0; + [[nodiscard]] virtual bool running() const = 0; [[nodiscard]] inline float current_volume() const { return this->current_audio_volume; } void register_observer(Observer* /* observer */); bool unregister_observer(Observer* /* observer */); - private: - std::shared_ptr target_device{}; - - mutable std::mutex recorder_mutex{}; - std::shared_ptr recorder_instance{}; + protected: + mutable std::mutex mutex{}; std::vector registered_observer{}; float current_audio_volume{0.f}; + void analyze_buffer(const float* /* buffer */, size_t /* channel count */, size_t /* sample count */); + }; + + /** + * This audio level meter operates directly on the raw input device without any processing. + */ + class InputDeviceAudioLevelMeter : public AbstractAudioLevelMeter, public AudioDeviceRecord::Consumer { + public: + explicit InputDeviceAudioLevelMeter(std::shared_ptr /* target device */); + + ~InputDeviceAudioLevelMeter() override; + + bool start(std::string &string) override; + + void stop() override; + + bool running() const override; + + private: + std::shared_ptr target_device{}; + std::shared_ptr recorder_instance{}; + void consume(const void *, size_t, size_t) override; }; + + class AudioInputAudioLevelMeter : public AbstractAudioLevelMeter { + friend class AudioInput; + + public: + AudioInputAudioLevelMeter() = default; + ~AudioInputAudioLevelMeter() override = default; + + bool start(std::string &string) override; + void stop() override; + bool running() const override; + + private: + bool active{false}; + }; } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioConsumer.h b/native/serverconnection/src/audio/js/AudioConsumer.h index 261ef90..c633680 100644 --- a/native/serverconnection/src/audio/js/AudioConsumer.h +++ b/native/serverconnection/src/audio/js/AudioConsumer.h @@ -25,7 +25,6 @@ namespace tc::audio { class AudioConsumerWrapper : public Nan::ObjectWrap { friend class AudioRecorderWrapper; - constexpr static auto kMaxChannelCount{2}; public: static NAN_MODULE_INIT(Init); static NAN_METHOD(NewInstance); diff --git a/native/serverconnection/src/audio/js/AudioLevelMeter.cpp b/native/serverconnection/src/audio/js/AudioLevelMeter.cpp index 73e4f4f..e6d5666 100644 --- a/native/serverconnection/src/audio/js/AudioLevelMeter.cpp +++ b/native/serverconnection/src/audio/js/AudioLevelMeter.cpp @@ -6,7 +6,6 @@ #include #include "./AudioLevelMeter.h" #include "../AudioLevelMeter.h" -#include "../../logger.h" using namespace tc::audio; using namespace tc::audio::recorder; @@ -21,7 +20,7 @@ NAN_MODULE_INIT(AudioLevelMeterWrapper::Init) { Nan::SetPrototypeMethod(klass, "stop", AudioLevelMeterWrapper::stop); Nan::SetPrototypeMethod(klass, "set_callback", AudioLevelMeterWrapper::set_callback); - Nan::Set(target, Nan::LocalStringUTF8("create_level_meter"), Nan::GetFunction(Nan::New(AudioLevelMeterWrapper::create_level_meter)).ToLocalChecked()); + Nan::Set(target, Nan::LocalStringUTF8("create_device_level_meter"), Nan::GetFunction(Nan::New(AudioLevelMeterWrapper::create_device_level_meter)).ToLocalChecked()); constructor().Reset(Nan::GetFunction(klass).ToLocalChecked()); } @@ -32,7 +31,7 @@ NAN_METHOD(AudioLevelMeterWrapper::NewInstance) { } } -NAN_METHOD(AudioLevelMeterWrapper::create_level_meter) { +NAN_METHOD(AudioLevelMeterWrapper::create_device_level_meter) { if(info.Length() != 1 || !info[0]->IsString()) { Nan::ThrowError("invalid arguments"); return; @@ -52,13 +51,13 @@ NAN_METHOD(AudioLevelMeterWrapper::create_level_meter) { return; } - auto wrapper = new AudioLevelMeterWrapper(std::make_shared(target_device)); + auto wrapper = new AudioLevelMeterWrapper(std::make_shared(target_device)); auto js_object = Nan::NewInstance(Nan::New(AudioLevelMeterWrapper::constructor())).ToLocalChecked(); wrapper->wrap(js_object); info.GetReturnValue().Set(js_object); } -AudioLevelMeterWrapper::AudioLevelMeterWrapper(std::shared_ptr handle) : handle{std::move(handle)} { +AudioLevelMeterWrapper::AudioLevelMeterWrapper(std::shared_ptr handle) : handle{std::move(handle)} { assert(this->handle); memset(&this->update_timer, 0, sizeof(this->update_timer)); diff --git a/native/serverconnection/src/audio/js/AudioLevelMeter.h b/native/serverconnection/src/audio/js/AudioLevelMeter.h index c8f2e1b..e1d839a 100644 --- a/native/serverconnection/src/audio/js/AudioLevelMeter.h +++ b/native/serverconnection/src/audio/js/AudioLevelMeter.h @@ -4,7 +4,7 @@ #include namespace tc::audio { - class AudioLevelMeter; + class AbstractAudioLevelMeter; } namespace tc::audio::recorder { @@ -13,14 +13,14 @@ namespace tc::audio::recorder { /* Static JavaScript methods */ static NAN_MODULE_INIT(Init); static NAN_METHOD(NewInstance); + static NAN_METHOD(create_device_level_meter); + static inline Nan::Persistent & constructor() { static Nan::Persistent my_constructor; return my_constructor; } - static NAN_METHOD(create_level_meter); - - explicit AudioLevelMeterWrapper(std::shared_ptr); + explicit AudioLevelMeterWrapper(std::shared_ptr); ~AudioLevelMeterWrapper() override; /* JavaScript member methods */ @@ -35,7 +35,7 @@ namespace tc::audio::recorder { private: static void timer_callback(uv_timer_t*); - std::shared_ptr handle{}; + std::shared_ptr handle{}; /* Access only within the js event loop */ uv_timer_t update_timer{}; diff --git a/native/serverconnection/src/audio/js/AudioRecorder.cpp b/native/serverconnection/src/audio/js/AudioRecorder.cpp index 6eb4e3a..f66f66b 100644 --- a/native/serverconnection/src/audio/js/AudioRecorder.cpp +++ b/native/serverconnection/src/audio/js/AudioRecorder.cpp @@ -1,10 +1,12 @@ #include #include -#include "AudioRecorder.h" -#include "AudioConsumer.h" +#include "./AudioRecorder.h" +#include "./AudioConsumer.h" +#include "./AudioLevelMeter.h" #include "./AudioProcessor.h" #include "../AudioInput.h" +#include "../AudioLevelMeter.h" #include "../../logger.h" using namespace std; @@ -290,4 +292,29 @@ NAN_METHOD(AudioRecorderWrapper::get_audio_processor) { auto wrapper = new AudioProcessorWrapper(processor); wrapper->wrap(js_object); info.GetReturnValue().Set(js_object); +} + +NAN_METHOD(AudioRecorderWrapper::create_level_meter) { + if(info.Length() != 1 || !info[0]->IsString()) { + Nan::ThrowError("invalid argument"); + return; + } + + auto mode = *Nan::Utf8String{info[0]}; + bool preprocess; + if(mode == std::string_view{"pre-process"}) { + preprocess = true; + } else if(mode == std::string_view{"post-process"}) { + preprocess = false; + } else { + Nan::ThrowError("invalid first argument"); + return; + } + + auto handle = ObjectWrap::Unwrap(info.Holder()); + + auto wrapper = new AudioLevelMeterWrapper(handle->input_->create_level_meter(preprocess)); + auto js_object = Nan::NewInstance(Nan::New(AudioLevelMeterWrapper::constructor())).ToLocalChecked(); + wrapper->wrap(js_object); + info.GetReturnValue().Set(js_object); } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioRecorder.h b/native/serverconnection/src/audio/js/AudioRecorder.h index c6c2bbf..e46c50e 100644 --- a/native/serverconnection/src/audio/js/AudioRecorder.h +++ b/native/serverconnection/src/audio/js/AudioRecorder.h @@ -41,6 +41,7 @@ namespace tc::audio { static NAN_METHOD(_get_volume); static NAN_METHOD(get_audio_processor); + static NAN_METHOD(create_level_meter); std::shared_ptr create_consumer(); void delete_consumer(const AudioConsumerWrapper*);