Adding the possibility to create level meters for the unprocessed and processed audio input

native/serverconnection/exports/exports.d.ts

@@ -324,7 +324,7 @@ export namespace audio {
         export type DeviceSetResult = "success" | "invalid-device";
         export interface AudioRecorder {
             get_device() : string;
-            set_device(device_id: string, callback: (result: DeviceSetResult) => void); /* Recorder needs to be started afterwards */
+            set_device(deviceId: string, callback: (result: DeviceSetResult) => void); /* Recorder needs to be started afterwards */
 
             start(callback: (result: boolean | string) => void);
             started() : boolean;
@@ -338,6 +338,7 @@ export namespace audio {
             delete_consumer(consumer: AudioConsumer);
 
             get_audio_processor() : AudioProcessor | undefined;
+            create_level_meter(mode: "pre-process" | "post-process") : AudioLevelMeter;
         }
 
         export interface AudioLevelMeter {
@@ -348,7 +349,7 @@ export namespace audio {
             set_callback(callback: (level: number) => void, updateInterval?: number);
         }
 
-        export function create_level_meter(targetDeviceId: string) : AudioLevelMeter;
+        export function create_device_level_meter(targetDeviceId: string) : AudioLevelMeter;
         export function create_recorder() : AudioRecorder;
     }
 

native/serverconnection/src/audio/AudioInput.cpp

@@ -8,6 +8,7 @@
 #include "./AudioGain.h"
 #include "./AudioInterleaved.h"
 #include "./AudioOutput.h"
+#include "./AudioLevelMeter.h"
 #include "./processing/AudioProcessor.h"
 #include "../logger.h"
 #include "AudioEventLoop.h"
@@ -192,6 +193,18 @@ std::shared_ptr<AudioConsumer> AudioInput::create_consumer(size_t frame_length)
 	return result;
 }
 
+std::shared_ptr<AudioInputAudioLevelMeter> AudioInput::create_level_meter(bool preprocess) {
+    auto level_meter = std::make_shared<AudioInputAudioLevelMeter>();
+
+    {
+        std::lock_guard lock{this->consumers_mutex};
+        auto& list = preprocess ? this->level_meter_preprocess : this->level_meter_postprocess;
+        list.push_back(level_meter);
+    }
+
+    return level_meter;
+}
+
 void AudioInput::allocate_input_buffer_samples(size_t samples) {
     const auto expected_byte_size = samples * this->channel_count_ * sizeof(float);
     if(expected_byte_size > this->input_buffer.capacity()) {
@@ -267,12 +280,12 @@ void AudioInput::process_audio() {
          * It's save to mutate the current memory.
          * If overflows occur it could lead to wired artifacts but all memory access is save.
          */
-        this->process_audio_chunk((void*) input);
+        this->process_audio_chunk((float*) input);
         this->input_buffer.advance_read_ptr(chunk_sample_count * this->channel_count_ * sizeof(float));
     }
 }
 
-void AudioInput::process_audio_chunk(void *chunk) {
+void AudioInput::process_audio_chunk(float *chunk) {
     constexpr static auto kTempSampleBufferSize{1024 * 8};
     constexpr static auto kMaxChannelCount{32};
 
@@ -282,10 +295,11 @@ void AudioInput::process_audio_chunk(void *chunk) {
     assert(memset(temp_sample_buffer, 0, sizeof(float) * kTempSampleBufferSize));
     assert(memset(out_sample_buffer, 0, sizeof(float) * kTempSampleBufferSize));
 
+    this->invoke_level_meter(true, chunk, this->channel_count_, chunk_sample_count);
     if(auto processor{this->audio_processor_}; processor) {
         assert(kTempSampleBufferSize >= chunk_sample_count * this->channel_count_ * sizeof(float));
 
-        audio::deinterleave(temp_sample_buffer, (const float*) chunk, this->channel_count_, chunk_sample_count);
+        audio::deinterleave(temp_sample_buffer, chunk, this->channel_count_, chunk_sample_count);
         webrtc::StreamConfig stream_config{(int) this->sample_rate_, this->channel_count_};
 
         float* channel_ptr[kMaxChannelCount];
@@ -304,8 +318,8 @@ void AudioInput::process_audio_chunk(void *chunk) {
         chunk = out_sample_buffer;
     }
 
-    /* TODO: Is this even needed if we have the processor? */
     audio::apply_gain(chunk, this->channel_count_, chunk_sample_count, this->volume_);
+    this->invoke_level_meter(false, out_sample_buffer, this->channel_count_, chunk_sample_count);
 
     auto begin = std::chrono::system_clock::now();
     for(const auto& consumer : this->consumers()) {
@@ -322,3 +336,32 @@ void AudioInput::process_audio_chunk(void *chunk) {
 void AudioInput::EventLoopCallback::event_execute(const chrono::system_clock::time_point &point) {
     this->input->process_audio();
 }
+
+void AudioInput::invoke_level_meter(bool preprocess, const float *buffer, size_t channel_count, size_t sample_size) {
+    std::vector<std::shared_ptr<AudioInputAudioLevelMeter>> level_meters{};
+    level_meters.reserve(10);
+
+    {
+        std::lock_guard lock{this->consumers_mutex};
+        auto& list = preprocess ? this->level_meter_preprocess : this->level_meter_postprocess;
+        level_meters.reserve(list.size());
+
+        list.erase(std::remove_if(list.begin(), list.end(), [&](const std::weak_ptr<AudioInputAudioLevelMeter>& weak_meter) {
+            auto meter = weak_meter.lock();
+            if(!meter) {
+                return true;
+            }
+
+            level_meters.push_back(meter);
+            return false;
+        }), list.end());
+    }
+
+    for(const auto& level_meter : level_meters) {
+        if(!level_meter->active) {
+            continue;
+        }
+
+        level_meter->analyze_buffer(buffer, channel_count, sample_size);
+    }
+}

native/serverconnection/src/audio/AudioInput.h

@@ -15,7 +15,7 @@ namespace tc::audio {
     class InputReframer;
     class AudioResampler;
     class AudioProcessor;
-    class AudioInputSource;
+    class AudioInputAudioLevelMeter;
 
     class AudioConsumer {
             friend class AudioInput;
@@ -37,7 +37,6 @@ namespace tc::audio {
     };
 
     class AudioInput : public AudioDeviceRecord::Consumer {
-            friend class AudioInputSource;
         public:
             AudioInput(size_t /* channels */, size_t /* sample rate */);
             virtual ~AudioInput();
@@ -52,6 +51,7 @@ namespace tc::audio {
 
             [[nodiscard]] std::vector<std::shared_ptr<AudioConsumer>> consumers();
             [[nodiscard]] std::shared_ptr<AudioConsumer> create_consumer(size_t /* frame size */);
+            [[nodiscard]] std::shared_ptr<AudioInputAudioLevelMeter> create_level_meter(bool /* pre process */);
 
             [[nodiscard]] inline auto audio_processor() { return this->audio_processor_; }
 
@@ -78,13 +78,16 @@ namespace tc::audio {
 
             void consume(const void *, size_t, size_t) override;
             void process_audio();
-            void process_audio_chunk(void *);
+            void process_audio_chunk(float *);
 
             size_t const channel_count_;
             size_t const sample_rate_;
 
             std::mutex consumers_mutex{};
             std::deque<std::weak_ptr<AudioConsumer>> consumers_{};
+            std::deque<std::weak_ptr<AudioInputAudioLevelMeter>> level_meter_preprocess{};
+            std::deque<std::weak_ptr<AudioInputAudioLevelMeter>> level_meter_postprocess{};
+
             std::recursive_mutex input_source_lock{};
 
             std::shared_ptr<EventLoopCallback> event_loop_entry{};
@@ -102,5 +105,7 @@ namespace tc::audio {
 
             void allocate_input_buffer_samples(size_t /* sample count */);
             [[nodiscard]] inline auto chunk_sample_count() const { return (kChunkSizeMs * this->sample_rate_) / 1000; }
+
+            void invoke_level_meter(bool /* preprocess */, const float* /* buffer */, size_t /* channel count */, size_t /* sample size */);
     };
 }

native/serverconnection/src/audio/AudioLevelMeter.cpp

@@ -9,18 +9,55 @@
 
 using namespace tc::audio;
 
-AudioLevelMeter::AudioLevelMeter(std::shared_ptr<AudioDevice> target_device) : target_device{std::move(target_device)} {
+AbstractAudioLevelMeter::AbstractAudioLevelMeter() {
-    log_allocate("AudioLevelMeter", this);
+    log_allocate("AbstractAudioLevelMeter", this);
+}
+
+AbstractAudioLevelMeter::~AbstractAudioLevelMeter() {
+    log_free("AbstractAudioLevelMeter", this);
+}
+
+void AbstractAudioLevelMeter::register_observer(Observer *observer) {
+    std::lock_guard lock{this->mutex};
+    this->registered_observer.push_back(observer);
+}
+
+bool AbstractAudioLevelMeter::unregister_observer(Observer *observer) {
+    std::lock_guard lock{this->mutex};
+    auto index = std::find(this->registered_observer.begin(), this->registered_observer.end(), observer);
+    if(index == this->registered_observer.end()) {
+        return false;
+    }
+
+    this->registered_observer.erase(index);
+    return true;
+}
+
+void AbstractAudioLevelMeter::analyze_buffer(const float *buffer, size_t channel_count, size_t sample_count) {
+    auto volume = audio::audio_buffer_level(buffer, channel_count, sample_count);
+
+    std::lock_guard lock{this->mutex};
+    if(volume == this->current_audio_volume) {
+        return;
+    }
+    this->current_audio_volume = volume;
+
+    for(auto& observer : this->registered_observer) {
+        observer->input_level_changed(volume);
+    }
+}
+
+/* For a target device */
+InputDeviceAudioLevelMeter::InputDeviceAudioLevelMeter(std::shared_ptr<AudioDevice> target_device) : target_device{std::move(target_device)} {
     assert(this->target_device);
 }
 
-AudioLevelMeter::~AudioLevelMeter() {
+InputDeviceAudioLevelMeter::~InputDeviceAudioLevelMeter() {
     this->stop();
-    log_free("AudioLevelMeter", this);
 }
 
-bool AudioLevelMeter::start(std::string &error) {
+bool InputDeviceAudioLevelMeter::start(std::string &error) {
-    std::lock_guard lock{this->recorder_mutex};
+    std::lock_guard lock{this->mutex};
     if(this->recorder_instance) {
         return true;
     }
@@ -40,13 +77,13 @@ bool AudioLevelMeter::start(std::string &error) {
     return true;
 }
 
-bool AudioLevelMeter::running() const {
+bool InputDeviceAudioLevelMeter::running() const {
-    std::lock_guard lock{this->recorder_mutex};
+    std::lock_guard lock{this->mutex};
     return this->recorder_instance != nullptr;
 }
 
-void AudioLevelMeter::stop() {
+void InputDeviceAudioLevelMeter::stop() {
-    std::lock_guard lock{this->recorder_mutex};
+    std::lock_guard lock{this->mutex};
     if(this->recorder_instance) {
         this->recorder_instance->remove_consumer(this);
         this->recorder_instance->stop_if_possible();
@@ -54,33 +91,23 @@ void AudioLevelMeter::stop() {
     }
 }
 
-void AudioLevelMeter::register_observer(Observer *observer) {
+/* Note the parameter order! */
-    std::lock_guard lock{this->recorder_mutex};
+void InputDeviceAudioLevelMeter::consume(const void *buffer, size_t sample_count, size_t channel_count) {
-    this->registered_observer.push_back(observer);
+    this->analyze_buffer((const float*) buffer, channel_count, sample_count);
 }
 
-bool AudioLevelMeter::unregister_observer(Observer *observer) {
+bool AudioInputAudioLevelMeter::start(std::string &) {
-    std::lock_guard lock{this->recorder_mutex};
+    std::lock_guard lock{this->mutex};
-    auto index = std::find(this->registered_observer.begin(), this->registered_observer.end(), observer);
+    this->active = true;
-    if(index == this->registered_observer.end()) {
-        return false;
-    }
-
-    this->registered_observer.erase(index);
     return true;
 }
 
-/* Note the parameter order! */
+void AudioInputAudioLevelMeter::stop() {
-void AudioLevelMeter::consume(const void *buffer, size_t sample_count, size_t channel_count) {
+    std::lock_guard lock{this->mutex};
-    auto volume = audio::audio_buffer_level((float*) buffer, channel_count, sample_count);
+    this->active = false;
-
-    std::lock_guard lock{this->recorder_mutex};
-    if(volume == this->current_audio_volume) {
-        return;
-    }
-    this->current_audio_volume = volume;
-
-    for(auto& observer : this->registered_observer) {
-        observer->input_level_changed(volume);
-    }
 }
+
+bool AudioInputAudioLevelMeter::running() const {
+    std::lock_guard lock{this->mutex};
+    return this->active;
+}

native/serverconnection/src/audio/AudioLevelMeter.h

@@ -1,37 +1,75 @@
 #pragma once
+
 #include "./driver/AudioDriver.h"
 
 namespace tc::audio {
+    class AudioInput;
+
     /**
      * Note: Within the observer callback no methods of the level meter should be called nor the level meter should be destructed.
      */
-    class AudioLevelMeter : public AudioDeviceRecord::Consumer {
+    class AbstractAudioLevelMeter {
         public:
             struct Observer {
                 public:
                     virtual void input_level_changed(float /* new level */) = 0;
             };
 
-            explicit AudioLevelMeter(std::shared_ptr<AudioDevice> /* target device */);
+            explicit AbstractAudioLevelMeter();
-            virtual ~AudioLevelMeter();
+            virtual ~AbstractAudioLevelMeter();
 
-            [[nodiscard]] bool start(std::string& /* error */);
+
-            void stop();
+            [[nodiscard]] virtual bool start(std::string& /* error */) = 0;
-            [[nodiscard]] bool running() const;
+            virtual void stop() = 0;
+            [[nodiscard]] virtual bool running() const = 0;
 
             [[nodiscard]] inline float current_volume() const { return this->current_audio_volume; }
 
             void register_observer(Observer* /* observer */);
             bool unregister_observer(Observer* /* observer */);
-        private:
+        protected:
-            std::shared_ptr<AudioDevice> target_device{};
+            mutable std::mutex mutex{};
-
-            mutable std::mutex recorder_mutex{};
-            std::shared_ptr<AudioDeviceRecord> recorder_instance{};
             std::vector<Observer*> registered_observer{};
 
             float current_audio_volume{0.f};
 
+            void analyze_buffer(const float* /* buffer */, size_t /* channel count */, size_t /* sample count */);
+    };
+
+    /**
+     * This audio level meter operates directly on the raw input device without any processing.
+     */
+    class InputDeviceAudioLevelMeter : public AbstractAudioLevelMeter, public AudioDeviceRecord::Consumer {
+        public:
+            explicit InputDeviceAudioLevelMeter(std::shared_ptr<AudioDevice> /* target device */);
+
+            ~InputDeviceAudioLevelMeter() override;
+
+            bool start(std::string &string) override;
+
+            void stop() override;
+
+            bool running() const override;
+
+        private:
+            std::shared_ptr<AudioDevice> target_device{};
+            std::shared_ptr<AudioDeviceRecord> recorder_instance{};
+
             void consume(const void *, size_t, size_t) override;
     };
+
+    class AudioInputAudioLevelMeter : public AbstractAudioLevelMeter {
+        friend class AudioInput;
+
+        public:
+            AudioInputAudioLevelMeter() = default;
+            ~AudioInputAudioLevelMeter() override = default;
+
+            bool start(std::string &string) override;
+            void stop() override;
+            bool running() const override;
+
+        private:
+            bool active{false};
+    };
 }

native/serverconnection/src/audio/js/AudioConsumer.h

@@ -25,7 +25,6 @@ namespace tc::audio {
 
         class AudioConsumerWrapper : public Nan::ObjectWrap {
                 friend class AudioRecorderWrapper;
-                constexpr static auto kMaxChannelCount{2};
             public:
                 static NAN_MODULE_INIT(Init);
                 static NAN_METHOD(NewInstance);

native/serverconnection/src/audio/js/AudioLevelMeter.cpp

@@ -6,7 +6,6 @@
 #include <include/NanStrings.h>
 #include "./AudioLevelMeter.h"
 #include "../AudioLevelMeter.h"
-#include "../../logger.h"
 
 using namespace tc::audio;
 using namespace tc::audio::recorder;
@@ -21,7 +20,7 @@ NAN_MODULE_INIT(AudioLevelMeterWrapper::Init) {
     Nan::SetPrototypeMethod(klass, "stop", AudioLevelMeterWrapper::stop);
     Nan::SetPrototypeMethod(klass, "set_callback", AudioLevelMeterWrapper::set_callback);
 
-    Nan::Set(target, Nan::LocalStringUTF8("create_level_meter"), Nan::GetFunction(Nan::New<v8::FunctionTemplate>(AudioLevelMeterWrapper::create_level_meter)).ToLocalChecked());
+    Nan::Set(target, Nan::LocalStringUTF8("create_device_level_meter"), Nan::GetFunction(Nan::New<v8::FunctionTemplate>(AudioLevelMeterWrapper::create_device_level_meter)).ToLocalChecked());
 
     constructor().Reset(Nan::GetFunction(klass).ToLocalChecked());
 }
@@ -32,7 +31,7 @@ NAN_METHOD(AudioLevelMeterWrapper::NewInstance) {
     }
 }
 
-NAN_METHOD(AudioLevelMeterWrapper::create_level_meter) {
+NAN_METHOD(AudioLevelMeterWrapper::create_device_level_meter) {
     if(info.Length() != 1 || !info[0]->IsString()) {
         Nan::ThrowError("invalid arguments");
         return;
@@ -52,13 +51,13 @@ NAN_METHOD(AudioLevelMeterWrapper::create_level_meter) {
         return;
     }
 
-    auto wrapper = new AudioLevelMeterWrapper(std::make_shared<AudioLevelMeter>(target_device));
+    auto wrapper = new AudioLevelMeterWrapper(std::make_shared<InputDeviceAudioLevelMeter>(target_device));
     auto js_object = Nan::NewInstance(Nan::New(AudioLevelMeterWrapper::constructor())).ToLocalChecked();
     wrapper->wrap(js_object);
     info.GetReturnValue().Set(js_object);
 }
 
-AudioLevelMeterWrapper::AudioLevelMeterWrapper(std::shared_ptr<AudioLevelMeter> handle) : handle{std::move(handle)} {
+AudioLevelMeterWrapper::AudioLevelMeterWrapper(std::shared_ptr<AbstractAudioLevelMeter> handle) : handle{std::move(handle)} {
     assert(this->handle);
 
     memset(&this->update_timer, 0, sizeof(this->update_timer));

native/serverconnection/src/audio/js/AudioLevelMeter.h

@@ -4,7 +4,7 @@
 #include <atomic>
 
 namespace tc::audio {
-    class AudioLevelMeter;
+    class AbstractAudioLevelMeter;
 }
 
 namespace tc::audio::recorder {
@@ -13,14 +13,14 @@ namespace tc::audio::recorder {
             /* Static JavaScript methods */
             static NAN_MODULE_INIT(Init);
             static NAN_METHOD(NewInstance);
+            static NAN_METHOD(create_device_level_meter);
+
             static inline Nan::Persistent<v8::Function> & constructor() {
                 static Nan::Persistent<v8::Function> my_constructor;
                 return my_constructor;
             }
 
-            static NAN_METHOD(create_level_meter);
+            explicit AudioLevelMeterWrapper(std::shared_ptr<AbstractAudioLevelMeter>);
-
-            explicit AudioLevelMeterWrapper(std::shared_ptr<AudioLevelMeter>);
             ~AudioLevelMeterWrapper() override;
 
             /* JavaScript member methods */
@@ -35,7 +35,7 @@ namespace tc::audio::recorder {
         private:
             static void timer_callback(uv_timer_t*);
 
-            std::shared_ptr<AudioLevelMeter> handle{};
+            std::shared_ptr<AbstractAudioLevelMeter> handle{};
 
             /* Access only within the js event loop */
             uv_timer_t update_timer{};

native/serverconnection/src/audio/js/AudioRecorder.cpp

@@ -1,10 +1,12 @@
 #include <utility>
 #include <NanStrings.h>
 
-#include "AudioRecorder.h"
+#include "./AudioRecorder.h"
-#include "AudioConsumer.h"
+#include "./AudioConsumer.h"
+#include "./AudioLevelMeter.h"
 #include "./AudioProcessor.h"
 #include "../AudioInput.h"
+#include "../AudioLevelMeter.h"
 #include "../../logger.h"
 
 using namespace std;
@@ -290,4 +292,29 @@ NAN_METHOD(AudioRecorderWrapper::get_audio_processor) {
     auto wrapper = new AudioProcessorWrapper(processor);
     wrapper->wrap(js_object);
     info.GetReturnValue().Set(js_object);
+}
+
+NAN_METHOD(AudioRecorderWrapper::create_level_meter) {
+    if(info.Length() != 1 || !info[0]->IsString()) {
+        Nan::ThrowError("invalid argument");
+        return;
+    }
+
+    auto mode = *Nan::Utf8String{info[0]};
+    bool preprocess;
+    if(mode == std::string_view{"pre-process"}) {
+        preprocess = true;
+    } else if(mode == std::string_view{"post-process"}) {
+        preprocess = false;
+    } else {
+        Nan::ThrowError("invalid first argument");
+        return;
+    }
+
+    auto handle = ObjectWrap::Unwrap<AudioRecorderWrapper>(info.Holder());
+
+    auto wrapper = new AudioLevelMeterWrapper(handle->input_->create_level_meter(preprocess));
+    auto js_object = Nan::NewInstance(Nan::New(AudioLevelMeterWrapper::constructor())).ToLocalChecked();
+    wrapper->wrap(js_object);
+    info.GetReturnValue().Set(js_object);
 }

native/serverconnection/src/audio/js/AudioRecorder.h

@@ -41,6 +41,7 @@ namespace tc::audio {
                 static NAN_METHOD(_get_volume);
 
                 static NAN_METHOD(get_audio_processor);
+                static NAN_METHOD(create_level_meter);
 
                 std::shared_ptr<AudioConsumerWrapper> create_consumer();
                 void delete_consumer(const AudioConsumerWrapper*);