From c956b213e9a63c4725e63321d6ce75fab6fc3964 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Sun, 28 Mar 2021 21:24:07 +0200 Subject: [PATCH] Adding a custom audio level meter API --- native/serverconnection/CMakeLists.txt | 3 + native/serverconnection/exports/exports.d.ts | 9 + .../src/audio/AudioLevelMeter.cpp | 86 +++++++++ .../src/audio/AudioLevelMeter.h | 37 ++++ .../src/audio/driver/PortAudio.cpp | 3 +- .../src/audio/js/AudioLevelMeter.cpp | 174 ++++++++++++++++++ .../src/audio/js/AudioLevelMeter.h | 48 +++++ .../src/audio/js/AudioPlayer.h | 12 -- .../src/audio/processing/AudioVolume.cpp | 1 + native/serverconnection/src/bindings.cpp | 2 + native/serverconnection/test/js/audio.ts | 37 +++- 11 files changed, 394 insertions(+), 18 deletions(-) create mode 100644 native/serverconnection/src/audio/AudioLevelMeter.cpp create mode 100644 native/serverconnection/src/audio/AudioLevelMeter.h create mode 100644 native/serverconnection/src/audio/js/AudioLevelMeter.cpp create mode 100644 native/serverconnection/src/audio/js/AudioLevelMeter.h diff --git a/native/serverconnection/CMakeLists.txt b/native/serverconnection/CMakeLists.txt index a2e9236..8efab94 100644 --- a/native/serverconnection/CMakeLists.txt +++ b/native/serverconnection/CMakeLists.txt @@ -20,6 +20,7 @@ set(SOURCE_FILES src/audio/AudioReframer.cpp src/audio/AudioEventLoop.cpp src/audio/AudioInterleaved.cpp + src/audio/AudioLevelMeter.cpp src/audio/filter/FilterVad.cpp src/audio/filter/FilterThreshold.cpp @@ -29,6 +30,7 @@ set(SOURCE_FILES src/audio/codec/OpusConverter.cpp src/audio/processing/AudioProcessor.cpp + src/audio/processing/AudioVolume.cpp src/audio/driver/AudioDriver.cpp src/audio/sounds/SoundPlayer.cpp @@ -56,6 +58,7 @@ set(NODEJS_SOURCE_FILES src/audio/js/AudioRecorder.cpp src/audio/js/AudioConsumer.cpp src/audio/js/AudioFilter.cpp + src/audio/js/AudioLevelMeter.cpp ) if (SOUNDIO_BACKED) diff --git a/native/serverconnection/exports/exports.d.ts b/native/serverconnection/exports/exports.d.ts index 1279011..6833686 100644 --- a/native/serverconnection/exports/exports.d.ts +++ b/native/serverconnection/exports/exports.d.ts @@ -340,6 +340,15 @@ export namespace audio { get_audio_processor() : AudioProcessor | undefined; } + export interface AudioLevelMeter { + start(callback: (error?: string) => void); + running() : boolean; + stop(); + + set_callback(callback: (level: number) => void, updateInterval?: number); + } + + export function create_level_meter(targetDeviceId: string) : AudioLevelMeter; export function create_recorder() : AudioRecorder; } diff --git a/native/serverconnection/src/audio/AudioLevelMeter.cpp b/native/serverconnection/src/audio/AudioLevelMeter.cpp new file mode 100644 index 0000000..ebc24d6 --- /dev/null +++ b/native/serverconnection/src/audio/AudioLevelMeter.cpp @@ -0,0 +1,86 @@ +// +// Created by WolverinDEV on 28/03/2021. +// + +#include +#include "./AudioLevelMeter.h" +#include "./processing/AudioVolume.h" +#include "../logger.h" + +using namespace tc::audio; + +AudioLevelMeter::AudioLevelMeter(std::shared_ptr target_device) : target_device{std::move(target_device)} { + log_allocate("AudioLevelMeter", this); + assert(this->target_device); +} + +AudioLevelMeter::~AudioLevelMeter() { + this->stop(); + log_free("AudioLevelMeter", this); +} + +bool AudioLevelMeter::start(std::string &error) { + std::lock_guard lock{this->recorder_mutex}; + if(this->recorder_instance) { + return true; + } + + this->recorder_instance = this->target_device->record(); + if(!this->recorder_instance) { + error = tr("failed to create device recorder"); + return false; + } + + if(!this->recorder_instance->start(error)) { + this->recorder_instance = nullptr; + return false; + } + + this->recorder_instance->register_consumer(this); + return true; +} + +bool AudioLevelMeter::running() const { + std::lock_guard lock{this->recorder_mutex}; + return this->recorder_instance != nullptr; +} + +void AudioLevelMeter::stop() { + std::lock_guard lock{this->recorder_mutex}; + if(this->recorder_instance) { + this->recorder_instance->remove_consumer(this); + this->recorder_instance->stop_if_possible(); + this->recorder_instance = nullptr; + } +} + +void AudioLevelMeter::register_observer(Observer *observer) { + std::lock_guard lock{this->recorder_mutex}; + this->registered_observer.push_back(observer); +} + +bool AudioLevelMeter::unregister_observer(Observer *observer) { + std::lock_guard lock{this->recorder_mutex}; + auto index = std::find(this->registered_observer.begin(), this->registered_observer.end(), observer); + if(index == this->registered_observer.end()) { + return false; + } + + this->registered_observer.erase(index); + return true; +} + +/* Note the parameter order! */ +void AudioLevelMeter::consume(const void *buffer, size_t sample_count, size_t channel_count) { + auto volume = audio::audio_buffer_level((float*) buffer, channel_count, sample_count); + + std::lock_guard lock{this->recorder_mutex}; + if(volume == this->current_audio_volume) { + return; + } + this->current_audio_volume = volume; + + for(auto& observer : this->registered_observer) { + observer->input_level_changed(volume); + } +} diff --git a/native/serverconnection/src/audio/AudioLevelMeter.h b/native/serverconnection/src/audio/AudioLevelMeter.h new file mode 100644 index 0000000..db7823f --- /dev/null +++ b/native/serverconnection/src/audio/AudioLevelMeter.h @@ -0,0 +1,37 @@ +#pragma once +#include "./driver/AudioDriver.h" + +namespace tc::audio { + /** + * Note: Within the observer callback no methods of the level meter should be called nor the level meter should be destructed. + */ + class AudioLevelMeter : public AudioDeviceRecord::Consumer { + public: + struct Observer { + public: + virtual void input_level_changed(float /* new level */) = 0; + }; + + explicit AudioLevelMeter(std::shared_ptr /* target device */); + virtual ~AudioLevelMeter(); + + [[nodiscard]] bool start(std::string& /* error */); + void stop(); + [[nodiscard]] bool running() const; + + [[nodiscard]] inline float current_volume() const { return this->current_audio_volume; } + + void register_observer(Observer* /* observer */); + bool unregister_observer(Observer* /* observer */); + private: + std::shared_ptr target_device{}; + + mutable std::mutex recorder_mutex{}; + std::shared_ptr recorder_instance{}; + std::vector registered_observer{}; + + float current_audio_volume{0.f}; + + void consume(const void *, size_t, size_t) override; + }; +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/driver/PortAudio.cpp b/native/serverconnection/src/audio/driver/PortAudio.cpp index 30bbda7..4f97807 100644 --- a/native/serverconnection/src/audio/driver/PortAudio.cpp +++ b/native/serverconnection/src/audio/driver/PortAudio.cpp @@ -105,8 +105,9 @@ namespace tc::audio::pa { } std::lock_guard lock{this->io_lock}; - if(!this->_record) + if(!this->_record) { this->_record = std::make_shared(this->_index, this->_info); + } return this->_record; } } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioLevelMeter.cpp b/native/serverconnection/src/audio/js/AudioLevelMeter.cpp new file mode 100644 index 0000000..3e319d7 --- /dev/null +++ b/native/serverconnection/src/audio/js/AudioLevelMeter.cpp @@ -0,0 +1,174 @@ +// +// Created by WolverinDEV on 28/03/2021. +// + +#include +#include +#include "./AudioLevelMeter.h" +#include "../AudioLevelMeter.h" +#include "../../logger.h" + +using namespace tc::audio; +using namespace tc::audio::recorder; + +NAN_MODULE_INIT(AudioLevelMeterWrapper::Init) { + auto klass = Nan::New(AudioLevelMeterWrapper::NewInstance); + klass->SetClassName(Nan::New("AudioLevelMeter").ToLocalChecked()); + klass->InstanceTemplate()->SetInternalFieldCount(1); + + Nan::SetPrototypeMethod(klass, "start", AudioLevelMeterWrapper::start); + Nan::SetPrototypeMethod(klass, "running", AudioLevelMeterWrapper::running); + Nan::SetPrototypeMethod(klass, "stop", AudioLevelMeterWrapper::stop); + Nan::SetPrototypeMethod(klass, "set_callback", AudioLevelMeterWrapper::set_callback); + + Nan::Set(target, Nan::LocalStringUTF8("create_level_meter"), Nan::GetFunction(Nan::New(AudioLevelMeterWrapper::create_level_meter)).ToLocalChecked()); + + constructor().Reset(Nan::GetFunction(klass).ToLocalChecked()); +} + +NAN_METHOD(AudioLevelMeterWrapper::NewInstance) { + if(!info.IsConstructCall()) { + Nan::ThrowError("invalid invoke!"); + } +} + +NAN_METHOD(AudioLevelMeterWrapper::create_level_meter) { + if(info.Length() != 1 || !info[0]->IsString()) { + Nan::ThrowError("invalid arguments"); + return; + } + + auto target_device_id = *Nan::Utf8String(info[0]); + std::shared_ptr target_device{}; + for(const auto& device : audio::devices()) { + if(device->id() == target_device_id) { + target_device = device; + break; + } + } + + if(!target_device || !target_device->is_input_supported()) { + Nan::ThrowError("invalid target device"); + return; + } + + auto wrapper = new AudioLevelMeterWrapper(std::make_shared(target_device)); + auto js_object = Nan::NewInstance(Nan::New(AudioLevelMeterWrapper::constructor())).ToLocalChecked(); + wrapper->wrap(js_object); + info.GetReturnValue().Set(js_object); +} + +AudioLevelMeterWrapper::AudioLevelMeterWrapper(std::shared_ptr handle) : handle{std::move(handle)} { + log_allocate("AudioLevelMeterWrapper", this); + assert(this->handle); + + memset(&this->update_timer, 0, sizeof(this->update_timer)); + this->update_timer.data = this; + + uv_timer_init(Nan::GetCurrentEventLoop(), &this->update_timer); +} + +AudioLevelMeterWrapper::~AudioLevelMeterWrapper() noexcept { + log_free("AudioLevelMeterWrapper", this); + uv_timer_stop(&this->update_timer); + this->update_timer.data = nullptr; + this->callback.Reset(); +} + +NAN_METHOD(AudioLevelMeterWrapper::start) { + auto handle = ObjectWrap::Unwrap(info.Holder()); + + if(info.Length() != 1 || !info[0]->IsFunction()) { + Nan::ThrowError("invalid arguments"); + return; + } + + auto js_queue = std::make_unique(); + auto js_callback = std::make_unique>(info[0].As()); + + handle->Ref(); + std::thread{[handle, js_queue = std::move(js_queue), js_callback = std::move(js_callback)]() mutable { + std::string error{}; + auto result = handle->handle->start(error); + + js_queue->enqueue([handle, result, error = std::move(error), js_callback = std::move(js_callback)]{ + auto isolate = Nan::GetCurrentContext()->GetIsolate(); + auto start_callback = js_callback->Get(isolate); + if(!result) { + auto js_error = Nan::LocalStringUTF8(error); + (void) start_callback->Call(isolate->GetCurrentContext(), Nan::Undefined(), 1, (v8::Local*) &js_error); + } else { + (void) start_callback->Call(isolate->GetCurrentContext(), Nan::Undefined(), 0, nullptr); + } + + handle->test_timer(); + js_callback->Reset(); + handle->Unref(); + }); + }}.detach(); +} + +NAN_METHOD(AudioLevelMeterWrapper::running) { + auto handle = ObjectWrap::Unwrap(info.Holder())->handle; + info.GetReturnValue().Set(handle->running()); +} + +NAN_METHOD(AudioLevelMeterWrapper::stop) { + auto handle = ObjectWrap::Unwrap(info.Holder()); + handle->handle->stop(); + handle->test_timer(); +} + +NAN_METHOD(AudioLevelMeterWrapper::set_callback) { + auto handle = ObjectWrap::Unwrap(info.Holder()); + + if(info.Length() < 1) { + Nan::ThrowError("invalid arguments"); + return; + } + + if(info[0]->IsFunction()) { + handle->update_timer_interval = 50; + if(info.Length() >= 2 && info[1]->IsNumber()) { + auto value = info[1]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0); + if(value < 1) { + Nan::ThrowError("invalid update interval"); + return; + } + + handle->update_timer_interval = (size_t) value; + } + + handle->callback.Reset(info[0].As()); + } else if(info[0]->IsNullOrUndefined()) { + handle->callback.Reset(); + } else { + Nan::ThrowError("invalid arguments"); + return; + } + handle->test_timer(); +} + +void AudioLevelMeterWrapper::test_timer() { + if(this->handle->running() && !this->callback.IsEmpty()) { + uv_timer_start(&this->update_timer, AudioLevelMeterWrapper::timer_callback, 0, this->update_timer_interval); + } else { + uv_timer_stop(&this->update_timer); + } +} + +void AudioLevelMeterWrapper::timer_callback(uv_timer_t *callback) { + Nan::HandleScope scope{}; + + auto level_meter = (AudioLevelMeterWrapper*) callback->data; + if(level_meter->callback.IsEmpty()) { + return; + } + + auto isolate = Nan::GetCurrentContext()->GetIsolate(); + assert(isolate); + auto timer_callback = level_meter->callback.Get(isolate); + + auto level = Nan::New(level_meter->handle->current_volume()); + (void) timer_callback->Call(Nan::GetCurrentContext(), Nan::Undefined(), 1, (v8::Local*) &level); +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioLevelMeter.h b/native/serverconnection/src/audio/js/AudioLevelMeter.h new file mode 100644 index 0000000..c8f2e1b --- /dev/null +++ b/native/serverconnection/src/audio/js/AudioLevelMeter.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include + +namespace tc::audio { + class AudioLevelMeter; +} + +namespace tc::audio::recorder { + class AudioLevelMeterWrapper : public Nan::ObjectWrap { + public: + /* Static JavaScript methods */ + static NAN_MODULE_INIT(Init); + static NAN_METHOD(NewInstance); + static inline Nan::Persistent & constructor() { + static Nan::Persistent my_constructor; + return my_constructor; + } + + static NAN_METHOD(create_level_meter); + + explicit AudioLevelMeterWrapper(std::shared_ptr); + ~AudioLevelMeterWrapper() override; + + /* JavaScript member methods */ + static NAN_METHOD(start); + static NAN_METHOD(running); + static NAN_METHOD(stop); + static NAN_METHOD(set_callback); + + inline void wrap(v8::Local object) { + Nan::ObjectWrap::Wrap(object); + } + private: + static void timer_callback(uv_timer_t*); + + std::shared_ptr handle{}; + + /* Access only within the js event loop */ + uv_timer_t update_timer{}; + size_t update_timer_interval{50}; + + Nan::Persistent callback{}; + + void test_timer(); + }; +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioPlayer.h b/native/serverconnection/src/audio/js/AudioPlayer.h index 34f55ee..7f43563 100644 --- a/native/serverconnection/src/audio/js/AudioPlayer.h +++ b/native/serverconnection/src/audio/js/AudioPlayer.h @@ -17,17 +17,5 @@ namespace tc::audio { extern NAN_METHOD(get_master_volume); extern NAN_METHOD(set_master_volume); - /* - export function get_master_volume() : number; - export function set_master_volume(volume: number); - - export function set_device(device: AudioDevice) : Promise; - export function current_device() : AudioDevice; - - export function available_devices() : AudioDevice[]; - - export function create_stream() : OwnedAudioOutputStream; - export function delete_stream(stream: OwnedAudioOutputStream) : number; - */ } } \ No newline at end of file diff --git a/native/serverconnection/src/audio/processing/AudioVolume.cpp b/native/serverconnection/src/audio/processing/AudioVolume.cpp index 5413781..eaf6319 100644 --- a/native/serverconnection/src/audio/processing/AudioVolume.cpp +++ b/native/serverconnection/src/audio/processing/AudioVolume.cpp @@ -42,6 +42,7 @@ float audio::audio_buffer_level(const float *buffer, size_t channel_count, size_ } long double square_buffer[kMaxChannelCount]; + memset(square_buffer, 0, sizeof(long double) * kMaxChannelCount); auto buffer_ptr = buffer; for(size_t sample{0}; sample < sample_count; sample++) { diff --git a/native/serverconnection/src/bindings.cpp b/native/serverconnection/src/bindings.cpp index 4dee19d..64ac0e8 100644 --- a/native/serverconnection/src/bindings.cpp +++ b/native/serverconnection/src/bindings.cpp @@ -19,6 +19,7 @@ #include "audio/js/AudioConsumer.h" #include "audio/js/AudioFilter.h" #include "audio/js/AudioProcessor.h" +#include "audio/js/AudioLevelMeter.h" #include "audio/AudioEventLoop.h" #include "audio/sounds/SoundPlayer.h" @@ -186,6 +187,7 @@ NAN_MODULE_INIT(init) { audio::recorder::AudioRecorderWrapper::Init(namespace_record); audio::recorder::AudioConsumerWrapper::Init(namespace_record); audio::recorder::AudioFilterWrapper::Init(namespace_record); + audio::recorder::AudioLevelMeterWrapper::Init(namespace_record); audio::AudioProcessorWrapper::Init(namespace_record); { diff --git a/native/serverconnection/test/js/audio.ts b/native/serverconnection/test/js/audio.ts index 40969e1..ac4d7b8 100644 --- a/native/serverconnection/test/js/audio.ts +++ b/native/serverconnection/test/js/audio.ts @@ -26,19 +26,45 @@ function printDevices() { } } +async function levelMeterDevice(deviceId: string) { + const levelMeter = record.create_level_meter(deviceId); + await new Promise((resolve, reject) => { + levelMeter.start(error => { + if(typeof error !== "undefined") { + reject(error); + } else { + resolve(error); + } + }); + }); + + console.info("Started"); + levelMeter.set_callback(level => console.info("New level: %o", level)); + await new Promise(resolve => setTimeout(resolve, 10 * 1000)); + + console.info("Stop"); + levelMeter.stop(); + + await new Promise(resolve => setTimeout(resolve, 1000)); + console.info("Done"); +} + async function main() { await new Promise(resolve => audio.initialize(resolve)); console.info("Audio initialized"); //printDevices(); + const defaultInput = audio.available_devices().find(device => device.input_default); + if(!defaultInput) { + throw "missing default input device"; + } + + await levelMeterDevice(defaultInput.device_id); + return; + const recorder = record.create_recorder(); await new Promise((resolve, reject) => { - const defaultInput = audio.available_devices().find(device => device.input_default); - if(!defaultInput) { - reject("missing default input device"); - return; - } recorder.set_device(defaultInput.device_id, result => { if(result === "success") { @@ -86,6 +112,7 @@ async function main() { } main().catch(error => { + console.error("An error occurred:"); console.error(error); process.exit(1); });