From 22d9059ad550bead624b1ffe1be5ad64550d67bd Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Sun, 28 Mar 2021 18:37:36 +0200 Subject: [PATCH] Implementing an audio processor which now also takes care of the RNNoise part --- native/serverconnection/CMakeLists.txt | 5 +- native/serverconnection/exports/exports.d.ts | 588 ++++++++++-------- native/serverconnection/src/EventLoop.h | 112 ++-- .../serverconnection/src/audio/AudioInput.cpp | 279 ++++++--- .../serverconnection/src/audio/AudioInput.h | 82 ++- .../src/audio/AudioInterleaved.cpp | 10 + .../src/audio/AudioInterleaved.h | 7 + .../src/audio/AudioReframer.cpp | 25 +- .../src/audio/filter/FilterVad.cpp | 1 - .../src/audio/js/AudioConsumer.cpp | 106 +--- .../src/audio/js/AudioConsumer.h | 10 +- .../src/audio/js/AudioOutputStream.cpp | 5 +- .../src/audio/js/AudioOutputStream.h | 72 ++- .../src/audio/js/AudioProcessor.cpp | 396 ++++++++++++ .../src/audio/js/AudioProcessor.h | 41 ++ .../src/audio/js/AudioRecorder.cpp | 31 +- .../src/audio/js/AudioRecorder.h | 4 +- .../src/audio/processing/AudioProcessor.cpp | 228 ++++++- .../src/audio/processing/AudioProcessor.h | 55 +- .../src/audio/sounds/SoundPlayer.cpp | 11 +- .../src/audio/sounds/SoundPlayer.h | 5 + native/serverconnection/src/bindings.cpp | 13 +- native/serverconnection/test/audio/main.cpp | 43 +- native/serverconnection/test/js/audio.ts | 103 ++- native/serverconnection/test/js/tsconfig.json | 18 + 25 files changed, 1614 insertions(+), 636 deletions(-) create mode 100644 native/serverconnection/src/audio/js/AudioProcessor.cpp create mode 100644 native/serverconnection/src/audio/js/AudioProcessor.h create mode 100644 native/serverconnection/test/js/tsconfig.json diff --git a/native/serverconnection/CMakeLists.txt b/native/serverconnection/CMakeLists.txt index 2cf0546..a2e9236 100644 --- a/native/serverconnection/CMakeLists.txt +++ b/native/serverconnection/CMakeLists.txt @@ -52,7 +52,7 @@ set(NODEJS_SOURCE_FILES src/audio/js/AudioPlayer.cpp src/audio/js/AudioOutputStream.cpp - + src/audio/js/AudioProcessor.cpp src/audio/js/AudioRecorder.cpp src/audio/js/AudioConsumer.cpp src/audio/js/AudioFilter.cpp @@ -88,7 +88,7 @@ endif() add_nodejs_module(${MODULE_NAME} ${SOURCE_FILES} ${NODEJS_SOURCE_FILES}) target_link_libraries(${MODULE_NAME} ${NODEJS_LIBRARIES}) -#target_compile_options(${MODULE_NAME} PUBLIC "-fPIC") +target_compile_definitions(${MODULE_NAME} PRIVATE "NOMINMAX") find_package(PortAudio REQUIRED) include_directories(${PortAudio_INCLUDE_DIR}) @@ -173,6 +173,7 @@ else() ) endif() + add_definitions(-DNO_OPEN_SSL) target_link_libraries(${MODULE_NAME} ${REQUIRED_LIBRARIES}) target_compile_definitions(${MODULE_NAME} PUBLIC -DNODEJS_API) diff --git a/native/serverconnection/exports/exports.d.ts b/native/serverconnection/exports/exports.d.ts index 74efbf2..1279011 100644 --- a/native/serverconnection/exports/exports.d.ts +++ b/native/serverconnection/exports/exports.d.ts @@ -1,291 +1,367 @@ -declare module "tc-native/connection" { - export enum ServerType { - UNKNOWN, - TEASPEAK, - TEAMSPEAK +export enum ServerType { + UNKNOWN, + TEASPEAK, + TEAMSPEAK +} + +export enum PlayerState { + BUFFERING, + PLAYING, + STOPPING, + STOPPED +} + +export interface NativeVoiceClient { + client_id: number; + + callback_playback: () => any; + callback_stopped: () => any; + + callback_state_changed: (new_state: PlayerState) => any; + + get_state() : PlayerState; + + get_volume() : number; + set_volume(volume: number) : void; + + abort_replay(); + get_stream() : audio.playback.AudioOutputStream; +} + +export interface NativeVoiceConnection { + register_client(client_id: number) : NativeVoiceClient; + available_clients() : NativeVoiceClient[]; + unregister_client(client_id: number); + + audio_source() : audio.record.AudioConsumer; + set_audio_source(consumer: audio.record.AudioConsumer | undefined); + + decoding_supported(codec: number) : boolean; + encoding_supported(codec: number) : boolean; + + get_encoder_codec() : number; + set_encoder_codec(codec: number); + + /* could may throw an exception when the underlying voice sender has been deallocated */ + enable_voice_send(flag: boolean); +} + +export interface NativeServerConnection { + callback_voice_data: (buffer: Uint8Array, client_id: number, codec_id: number, flag_head: boolean, packet_id: number) => any; + callback_command: (command: string, arguments: any, switches: string[]) => any; + callback_disconnect: (reason?: string) => any; + _voice_connection: NativeVoiceConnection; + server_type: ServerType; + + connected(): boolean; + + connect(properties: { + remote_host: string, + remote_port: number, + + timeout: number, + + callback: (error: number) => any, + + identity_key: string | undefined, /* if identity_key empty, then an ephemeral license will be created */ + teamspeak: boolean + }); + + disconnect(reason: string | undefined, callback: (error: number) => any) : boolean; + + error_message(id: number) : string; + + send_command(command: string, arguments: any[], switches: string[]); + send_voice_data(buffer: Uint8Array, codec_id: number, header: boolean); + send_voice_data_raw(buffer: Float32Array, channels: number, sample_rate: number, header: boolean); + + /* ping in microseconds */ + current_ping() : number; + + statistics() : { voice_bytes_received: number, voice_bytes_send: number, control_bytes_received: number, control_bytes_send } | undefined +} + +export function spawn_server_connection() : NativeServerConnection; +export function destroy_server_connection(connection: NativeServerConnection); + +export namespace ft { + export interface TransferObject { + name: string; + direction: "upload" | "download"; } - export enum PlayerState { - BUFFERING, - PLAYING, - STOPPING, - STOPPED + export interface FileTransferSource extends TransferObject { + total_size: number; } - export interface NativeVoiceClient { - client_id: number; - - callback_playback: () => any; - callback_stopped: () => any; - - callback_state_changed: (new_state: PlayerState) => any; - - get_state() : PlayerState; - - get_volume() : number; - set_volume(volume: number) : void; - - abort_replay(); - get_stream() : audio.playback.AudioOutputStream; + export interface FileTransferTarget extends TransferObject { + expected_size: number; } - export interface NativeVoiceConnection { - register_client(client_id: number) : NativeVoiceClient; - available_clients() : NativeVoiceClient[]; - unregister_client(client_id: number); + export interface NativeFileTransfer { + handle: TransferObject; - audio_source() : audio.record.AudioConsumer; - set_audio_source(consumer: audio.record.AudioConsumer | undefined); + callback_finished: (aborted?: boolean) => any; + callback_start: () => any; + callback_progress: (current: number, max: number) => any; + callback_failed: (message: string) => any; - decoding_supported(codec: number) : boolean; - encoding_supported(codec: number) : boolean; - - get_encoder_codec() : number; - set_encoder_codec(codec: number); - - /* could may throw an exception when the underlying voice sender has been deallocated */ - enable_voice_send(flag: boolean); + /** + * @return true if the connect method has been executed successfully + * false if the connect fails, callback_failed will be called with the exact reason + */ + start() : boolean; } - export interface NativeServerConnection { - callback_voice_data: (buffer: Uint8Array, client_id: number, codec_id: number, flag_head: boolean, packet_id: number) => any; - callback_command: (command: string, arguments: any, switches: string[]) => any; - callback_disconnect: (reason?: string) => any; - _voice_connection: NativeVoiceConnection; - server_type: ServerType; + export interface TransferOptions { + remote_address: string; + remote_port: number; - connected(): boolean; + transfer_key: string; + client_transfer_id: number; + server_transfer_id: number; - connect(properties: { - remote_host: string, - remote_port: number, - - timeout: number, - - callback: (error: number) => any, - - identity_key: string | undefined, /* if identity_key empty, then an ephemeral license will be created */ - teamspeak: boolean - }); - - disconnect(reason: string | undefined, callback: (error: number) => any) : boolean; - - error_message(id: number) : string; - - send_command(command: string, arguments: any[], switches: string[]); - send_voice_data(buffer: Uint8Array, codec_id: number, header: boolean); - send_voice_data_raw(buffer: Float32Array, channels: number, sample_rate: number, header: boolean); - - /* ping in microseconds */ - current_ping() : number; - - statistics() : { voice_bytes_received: number, voice_bytes_send: number, control_bytes_received: number, control_bytes_send } | undefined + object: TransferObject; } - export function spawn_server_connection() : NativeServerConnection; - export function destroy_server_connection(connection: NativeServerConnection); + export function upload_transfer_object_from_file(path: string, name: string) : FileTransferSource; + export function upload_transfer_object_from_buffer(buffer: ArrayBuffer) : FileTransferSource; - export namespace ft { - export interface TransferObject { - name: string; - direction: "upload" | "download"; - } + export function download_transfer_object_from_buffer(target_buffer: ArrayBuffer) : FileTransferTarget; + export function download_transfer_object_from_file(path: string, name: string, expectedSize: number) : FileTransferTarget; - export interface FileTransferSource extends TransferObject { - total_size: number; - } + export function destroy_connection(connection: NativeFileTransfer); + export function spawn_connection(transfer: TransferOptions) : NativeFileTransfer; +} - export interface FileTransferTarget extends TransferObject { - expected_size: number; - } +export namespace audio { + export interface AudioDevice { + name: string; + driver: string; - export interface NativeFileTransfer { - handle: TransferObject; + device_id: string; - callback_finished: (aborted?: boolean) => any; - callback_start: () => any; - callback_progress: (current: number, max: number) => any; - callback_failed: (message: string) => any; + input_supported: boolean; + output_supported: boolean; - /** - * @return true if the connect method has been executed successfully - * false if the connect fails, callback_failed will be called with the exact reason - */ - start() : boolean; - } - - export interface TransferOptions { - remote_address: string; - remote_port: number; - - transfer_key: string; - client_transfer_id: number; - server_transfer_id: number; - - object: TransferObject; - } - - export function upload_transfer_object_from_file(path: string, name: string) : FileTransferSource; - export function upload_transfer_object_from_buffer(buffer: ArrayBuffer) : FileTransferSource; - - export function download_transfer_object_from_buffer(target_buffer: ArrayBuffer) : FileTransferTarget; - export function download_transfer_object_from_file(path: string, name: string, expectedSize: number) : FileTransferTarget; - - export function destroy_connection(connection: NativeFileTransfer); - export function spawn_connection(transfer: TransferOptions) : NativeFileTransfer; + input_default: boolean; + output_default: boolean; } - export namespace audio { - export interface AudioDevice { - name: string; - driver: string; + export namespace playback { + export interface AudioOutputStream { + sample_rate: number; + channels: number; - device_id: string; + get_buffer_latency() : number; + set_buffer_latency(value: number); - input_supported: boolean; - output_supported: boolean; + get_buffer_max_latency() : number; + set_buffer_max_latency(value: number); - input_default: boolean; - output_default: boolean; + flush_buffer(); } - export namespace playback { - export interface AudioOutputStream { - sample_rate: number; - channels: number; + export interface OwnedAudioOutputStream extends AudioOutputStream { + callback_underflow: () => any; + callback_overflow: () => any; - get_buffer_latency() : number; - set_buffer_latency(value: number); + clear(); + write_data(buffer: ArrayBuffer, interleaved: boolean); + write_data_rated(buffer: ArrayBuffer, interleaved: boolean, sample_rate: number); - get_buffer_max_latency() : number; - set_buffer_max_latency(value: number); - - flush_buffer(); - } - - export interface OwnedAudioOutputStream extends AudioOutputStream { - callback_underflow: () => any; - callback_overflow: () => any; - - clear(); - write_data(buffer: ArrayBuffer, interleaved: boolean); - write_data_rated(buffer: ArrayBuffer, interleaved: boolean, sample_rate: number); - - deleted() : boolean; - delete(); - } - - export function set_device(device_id: string); - export function current_device() : string; - - export function create_stream() : OwnedAudioOutputStream; - - export function get_master_volume() : number; - export function set_master_volume(volume: number); + deleted() : boolean; + delete(); } - export namespace record { - enum FilterMode { - Bypass, - Filter, - Block - } + export function set_device(device_id: string); + export function current_device() : string; - export interface ConsumeFilter { - get_name() : string; - } + export function create_stream() : OwnedAudioOutputStream; - export interface MarginedFilter { - /* in seconds */ - get_margin_time() : number; - set_margin_time(value: number); - } - - export interface VADConsumeFilter extends ConsumeFilter, MarginedFilter { - get_level() : number; - } - - export interface ThresholdConsumeFilter extends ConsumeFilter, MarginedFilter { - get_threshold() : number; - set_threshold(value: number); - - get_attack_smooth() : number; - set_attack_smooth(value: number); - - get_release_smooth() : number; - set_release_smooth(value: number); - - set_analyze_filter(callback: (value: number) => any); - } - - export interface StateConsumeFilter extends ConsumeFilter { - is_consuming() : boolean; - set_consuming(flag: boolean); - } - - export interface AudioConsumer { - readonly sampleRate: number; - readonly channelCount: number; - readonly frameSize: number; - - get_filters() : ConsumeFilter[]; - unregister_filter(filter: ConsumeFilter); - - create_filter_vad(level: number) : VADConsumeFilter; - create_filter_threshold(threshold: number) : ThresholdConsumeFilter; - create_filter_state() : StateConsumeFilter; - - set_filter_mode(mode: FilterMode); - get_filter_mode() : FilterMode; - - toggle_rnnoise(enabled: boolean); - rnnoise_enabled() : boolean; - - callback_data: (buffer: Float32Array) => any; - callback_ended: () => any; - callback_started: () => any; - } - - export type DeviceSetResult = "success" | "invalid-device"; - export interface AudioRecorder { - get_device() : string; - set_device(device_id: string, callback: (result: DeviceSetResult) => void); /* Recorder needs to be started afterwards */ - - start(callback: (result: boolean | string) => void); - started() : boolean; - stop(); - - get_volume() : number; - set_volume(volume: number); - - create_consumer() : AudioConsumer; - get_consumers() : AudioConsumer[]; - delete_consumer(consumer: AudioConsumer); - } - - export function create_recorder() : AudioRecorder; - } - - export namespace sounds { - export enum PlaybackResult { - SUCCEEDED, - CANCELED, - SOUND_NOT_INITIALIZED, - FILE_OPEN_ERROR, - PLAYBACK_ERROR - } - - export interface PlaybackSettings { - file: string; - volume?: number; - callback?: (result: PlaybackResult, message: string) => void; - } - export function playback_sound(settings: PlaybackSettings) : number; - export function cancel_playback(playback: number); - } - - export function initialize(callback: () => any); - export function initialized() : boolean; - export function available_devices() : AudioDevice[]; + export function get_master_volume() : number; + export function set_master_volume(volume: number); } + + export namespace record { + enum FilterMode { + Bypass, + Filter, + Block + } + + export interface ConsumeFilter { + get_name() : string; + } + + export interface MarginedFilter { + /* in seconds */ + get_margin_time() : number; + set_margin_time(value: number); + } + + export interface VADConsumeFilter extends ConsumeFilter, MarginedFilter { + get_level() : number; + } + + export interface ThresholdConsumeFilter extends ConsumeFilter, MarginedFilter { + get_threshold() : number; + set_threshold(value: number); + + get_attack_smooth() : number; + set_attack_smooth(value: number); + + get_release_smooth() : number; + set_release_smooth(value: number); + + set_analyze_filter(callback: (value: number) => any); + } + + export interface StateConsumeFilter extends ConsumeFilter { + is_consuming() : boolean; + set_consuming(flag: boolean); + } + + export interface AudioConsumer { + readonly sampleRate: number; + readonly channelCount: number; + readonly frameSize: number; + + get_filters() : ConsumeFilter[]; + unregister_filter(filter: ConsumeFilter); + + create_filter_vad(level: number) : VADConsumeFilter; + create_filter_threshold(threshold: number) : ThresholdConsumeFilter; + create_filter_state() : StateConsumeFilter; + + set_filter_mode(mode: FilterMode); + get_filter_mode() : FilterMode; + + callback_data: (buffer: Float32Array) => any; + callback_ended: () => any; + callback_started: () => any; + } + + export interface AudioProcessorConfig { + "pipeline.maximum_internal_processing_rate": number, + "pipeline.multi_channel_render": boolean, + "pipeline.multi_channel_capture": boolean, + + "pre_amplifier.enabled": boolean, + "pre_amplifier.fixed_gain_factor": number, + + "high_pass_filter.enabled": boolean, + "high_pass_filter.apply_in_full_band": boolean, + + "echo_canceller.enabled": boolean, + "echo_canceller.mobile_mode": boolean, + "echo_canceller.export_linear_aec_output": boolean, + "echo_canceller.enforce_high_pass_filtering": boolean, + + "noise_suppression.enabled": boolean, + "noise_suppression.level": "low" | "moderate" | "high" | "very-high", + "noise_suppression.analyze_linear_aec_output_when_available": boolean, + + "transient_suppression.enabled": boolean, + + "voice_detection.enabled": boolean, + + "gain_controller1.enabled": boolean, + "gain_controller1.mode": "adaptive-analog" | "adaptive-digital" | "fixed-digital", + "gain_controller1.target_level_dbfs": number, + "gain_controller1.compression_gain_db": number, + "gain_controller1.enable_limiter": boolean, + "gain_controller1.analog_level_minimum": number, + "gain_controller1.analog_level_maximum": number, + + "gain_controller1.analog_gain_controller.enabled": boolean, + "gain_controller1.analog_gain_controller.startup_min_volume": number, + "gain_controller1.analog_gain_controller.clipped_level_min": number, + "gain_controller1.analog_gain_controller.enable_agc2_level_estimator": boolean, + "gain_controller1.analog_gain_controller.enable_digital_adaptive": boolean, + + "gain_controller2.enabled": boolean, + + "gain_controller2.fixed_digital.gain_db": number, + + "gain_controller2.adaptive_digital.enabled": boolean, + "gain_controller2.adaptive_digital.vad_probability_attack": number, + "gain_controller2.adaptive_digital.level_estimator": "rms" | "peak", + "gain_controller2.adaptive_digital.level_estimator_adjacent_speech_frames_threshold": number, + "gain_controller2.adaptive_digital.use_saturation_protector": boolean, + "gain_controller2.adaptive_digital.initial_saturation_margin_db": number, + "gain_controller2.adaptive_digital.extra_saturation_margin_db": number, + "gain_controller2.adaptive_digital.gain_applier_adjacent_speech_frames_threshold": number, + "gain_controller2.adaptive_digital.max_gain_change_db_per_second": number, + "gain_controller2.adaptive_digital.max_output_noise_level_dbfs": number, + + "residual_echo_detector.enabled": boolean, + "level_estimation.enabled": boolean, + "rnnoise.enabled": boolean + } + + export interface AudioProcessorStatistics { + output_rms_dbfs: number | undefined, + voice_detected: number | undefined, + echo_return_loss: number | undefined, + echo_return_loss_enhancement: number | undefined, + divergent_filter_fraction: number | undefined, + delay_median_ms: number | undefined, + delay_standard_deviation_ms: number | undefined, + residual_echo_likelihood: number | undefined, + residual_echo_likelihood_recent_max: number | undefined, + delay_ms: number | undefined, + rnnoise_volume: number | undefined + } + + export interface AudioProcessor { + get_config() : AudioProcessorConfig; + apply_config(config: Partial); + + get_statistics() : AudioProcessorStatistics; + } + + export type DeviceSetResult = "success" | "invalid-device"; + export interface AudioRecorder { + get_device() : string; + set_device(device_id: string, callback: (result: DeviceSetResult) => void); /* Recorder needs to be started afterwards */ + + start(callback: (result: boolean | string) => void); + started() : boolean; + stop(); + + get_volume() : number; + set_volume(volume: number); + + create_consumer() : AudioConsumer; + get_consumers() : AudioConsumer[]; + delete_consumer(consumer: AudioConsumer); + + get_audio_processor() : AudioProcessor | undefined; + } + + export function create_recorder() : AudioRecorder; + } + + export namespace sounds { + export enum PlaybackResult { + SUCCEEDED, + CANCELED, + SOUND_NOT_INITIALIZED, + FILE_OPEN_ERROR, + PLAYBACK_ERROR + } + + export interface PlaybackSettings { + file: string; + volume?: number; + callback?: (result: PlaybackResult, message: string) => void; + } + export function playback_sound(settings: PlaybackSettings) : number; + export function cancel_playback(playback: number); + } + + export function initialize(callback: () => any); + export function initialized() : boolean; + export function available_devices() : AudioDevice[]; } \ No newline at end of file diff --git a/native/serverconnection/src/EventLoop.h b/native/serverconnection/src/EventLoop.h index 807c198..a7864e0 100644 --- a/native/serverconnection/src/EventLoop.h +++ b/native/serverconnection/src/EventLoop.h @@ -7,71 +7,69 @@ #include #include -namespace tc { - namespace event { - class EventExecutor; - class EventEntry { - friend class EventExecutor; - public: - virtual void event_execute(const std::chrono::system_clock::time_point& /* scheduled timestamp */) = 0; - virtual void event_execute_dropped(const std::chrono::system_clock::time_point& /* scheduled timestamp */) {} +namespace tc::event { + class EventExecutor; + class EventEntry { + friend class EventExecutor; + public: + virtual void event_execute(const std::chrono::system_clock::time_point& /* scheduled timestamp */) = 0; + virtual void event_execute_dropped(const std::chrono::system_clock::time_point& /* scheduled timestamp */) {} - std::unique_lock execute_lock(bool force) { - if(force) { - return std::unique_lock(this->_execute_mutex); - } else { - auto lock = std::unique_lock(this->_execute_mutex, std::defer_lock); - if(this->execute_lock_timeout.count() > 0) { - (void) lock.try_lock_for(this->execute_lock_timeout); - } else { - (void) lock.try_lock(); - } - return lock; - } - } + std::unique_lock execute_lock(bool force) { + if(force) { + return std::unique_lock(this->_execute_mutex); + } else { + auto lock = std::unique_lock(this->_execute_mutex, std::defer_lock); + if(this->execute_lock_timeout.count() > 0) { + (void) lock.try_lock_for(this->execute_lock_timeout); + } else { + (void) lock.try_lock(); + } + return lock; + } + } - inline bool single_thread_executed() const { return this->_single_thread; } - inline void single_thread_executed(bool value) { this->_single_thread = value; } + [[nodiscard]] inline bool single_thread_executed() const { return this->_single_thread; } + inline void single_thread_executed(bool value) { this->_single_thread = value; } - protected: - std::chrono::nanoseconds execute_lock_timeout{0}; - private: - void* _event_ptr = nullptr; - bool _single_thread = true; /* if its set to true there might are some dropped executes! */ - std::timed_mutex _execute_mutex; - }; + protected: + std::chrono::nanoseconds execute_lock_timeout{0}; + private: + void* _event_ptr = nullptr; + bool _single_thread = true; /* if its set to true there might are some dropped executes! */ + std::timed_mutex _execute_mutex; + }; - class EventExecutor { - public: - explicit EventExecutor(const std::string& /* thread prefix */); - virtual ~EventExecutor(); + class EventExecutor { + public: + explicit EventExecutor(const std::string& /* thread prefix */); + virtual ~EventExecutor(); - bool initialize(int /* num threads */); - bool schedule(const std::shared_ptr& /* entry */); - bool cancel(const std::shared_ptr& /* entry */); /* Note: Will not cancel already running executes */ - void shutdown(); - private: - struct LinkedEntry { - LinkedEntry* previous; - LinkedEntry* next; + bool initialize(int /* num threads */); + bool schedule(const std::shared_ptr& /* entry */); + bool cancel(const std::shared_ptr& /* entry */); /* Note: Will not cancel already running executes */ + void shutdown(); + private: + struct LinkedEntry { + LinkedEntry* previous; + LinkedEntry* next; - std::chrono::system_clock::time_point scheduled; - std::weak_ptr entry; - }; - static void _executor(EventExecutor*); - void _shutdown(std::unique_lock&); - void _reset_events(std::unique_lock&); + std::chrono::system_clock::time_point scheduled; + std::weak_ptr entry; + }; + static void _executor(EventExecutor*); + void _shutdown(std::unique_lock&); + void _reset_events(std::unique_lock&); - bool should_shutdown = true; + bool should_shutdown = true; - std::vector threads; - std::mutex lock; - std::condition_variable condition; + std::vector threads; + std::mutex lock; + std::condition_variable condition; - LinkedEntry* head = nullptr; - LinkedEntry* tail = nullptr; + LinkedEntry* head = nullptr; + LinkedEntry* tail = nullptr; - std::string thread_prefix; - }; - } + std::string thread_prefix; + }; } \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioInput.cpp b/native/serverconnection/src/audio/AudioInput.cpp index dfe1edf..86798b0 100644 --- a/native/serverconnection/src/audio/AudioInput.cpp +++ b/native/serverconnection/src/audio/AudioInput.cpp @@ -1,21 +1,25 @@ #include +#include #include #include "./AudioInput.h" #include "./AudioReframer.h" #include "./AudioResampler.h" #include "./AudioMerger.h" +#include "./AudioGain.h" +#include "./AudioInterleaved.h" +#include "./AudioOutput.h" +#include "./processing/AudioProcessor.h" #include "../logger.h" -#include "AudioGain.h" +#include "AudioEventLoop.h" using namespace std; using namespace tc; using namespace tc::audio; -AudioConsumer::AudioConsumer(tc::audio::AudioInput *handle, size_t channel_count, size_t sample_rate, size_t frame_size) : - handle(handle), - channel_count(channel_count), - sample_rate(sample_rate) , - frame_size(frame_size) { +AudioConsumer::AudioConsumer(size_t channel_count, size_t sample_rate, size_t frame_size) : + channel_count{channel_count}, + sample_rate{sample_rate}, + frame_size{frame_size} { if(this->frame_size > 0) { this->reframer = std::make_unique(channel_count, frame_size); this->reframer->on_frame = [&](const void* buffer) { this->handle_framed_data(buffer, this->frame_size); }; @@ -40,15 +44,60 @@ void AudioConsumer::process_data(const void *buffer, size_t samples) { } } -AudioInput::AudioInput(size_t channels, size_t rate) : _channel_count(channels), _sample_rate(rate) {} -AudioInput::~AudioInput() { - this->close_device(); +extern tc::audio::AudioOutput* global_audio_output; +AudioInput::AudioInput(size_t channels, size_t sample_rate) : + channel_count_{channels}, + sample_rate_{sample_rate}, + input_buffer{(sample_rate * channels * sizeof(float) * kInputBufferCapacityMs) / 1000} +{ + this->event_loop_entry = std::make_shared(this); + { - std::lock_guard lock(this->consumers_lock); - for(const auto& consumer : this->_consumers) - consumer->handle = nullptr; + this->initialize_hook_handle = std::make_shared(); + this->initialize_hook_handle->input = this; + + std::weak_ptr weak_handle{this->initialize_hook_handle}; + audio::initialize([weak_handle] { + auto handle = weak_handle.lock(); + if(!handle) { + return; + } + + std::lock_guard lock{handle->mutex}; + if(!handle->input) { + return; + } + + auto processor = std::make_shared(); + if(!processor->initialize()) { + log_error(category::audio, tr("Failed to initialize audio processor.")); + return; + } + + global_audio_output->register_audio_processor(processor); + handle->input->audio_processor_ = processor; + }); } - free(this->resample_buffer); +} + +AudioInput::~AudioInput() { + { + std::lock_guard lock{this->initialize_hook_handle->mutex}; + this->initialize_hook_handle->input = nullptr; + } + + { + audio::encode_event_loop->cancel(this->event_loop_entry); + this->event_loop_entry->execute_lock(true); + } + + if(this->audio_processor_) { + assert(global_audio_output); + global_audio_output->unregister_audio_processor(this->audio_processor_); + this->audio_processor_ = nullptr; + } + + this->close_device(); } void AudioInput::set_device(const std::shared_ptr &device) { @@ -68,7 +117,7 @@ void AudioInput::close_device() { this->input_recorder->stop_if_possible(); this->input_recorder.reset(); } - this->_resampler = nullptr; + this->resampler_ = nullptr; this->input_device = nullptr; } @@ -90,7 +139,7 @@ bool AudioInput::record(std::string& error) { } if(this->input_recorder->sample_rate() != this->sample_rate()) { - this->_resampler = std::make_unique(this->input_recorder->sample_rate(), this->sample_rate(), this->_channel_count); + this->resampler_ = std::make_unique(this->input_recorder->sample_rate(), this->sample_rate(), this->channel_count_); } this->input_recorder->register_consumer(this); @@ -103,7 +152,7 @@ bool AudioInput::record(std::string& error) { } bool AudioInput::recording() { - return !!this->input_recorder; + return this->input_recorder != nullptr; } void AudioInput::stop() { @@ -114,80 +163,162 @@ void AudioInput::stop() { this->input_recorder.reset(); } +std::vector> AudioInput::consumers() { + std::vector> result{}; + result.reserve(10); + + std::lock_guard consumer_lock{this->consumers_mutex}; + result.reserve(this->consumers_.size()); + + this->consumers_.erase(std::remove_if(this->consumers_.begin(), this->consumers_.end(), [&](const std::weak_ptr& weak_consumer) { + auto consumer = weak_consumer.lock(); + if(!consumer) { + return true; + } + + result.push_back(consumer); + return false; + }), this->consumers_.end()); + + return result; +} + std::shared_ptr AudioInput::create_consumer(size_t frame_length) { - auto result = std::shared_ptr(new AudioConsumer(this, this->_channel_count, this->_sample_rate, frame_length)); + auto result = std::shared_ptr(new AudioConsumer(this->channel_count_, this->sample_rate_, frame_length)); { - std::lock_guard lock(this->consumers_lock); - this->_consumers.push_back(result); + std::lock_guard lock(this->consumers_mutex); + this->consumers_.push_back(result); } return result; } -void AudioInput::delete_consumer(const std::shared_ptr &source) { - { - std::lock_guard lock(this->consumers_lock); - auto it = find(this->_consumers.begin(), this->_consumers.end(), source); - if(it != this->_consumers.end()) - this->_consumers.erase(it); - } - - source->handle = nullptr; -} - -void AudioInput::consume(const void *input, size_t frameCount, size_t channels) { - if(channels != this->_channel_count) { - if(channels < 1 || channels > 2) { - log_critical(category::audio, tr("Channel count miss match (Received: {})!"), channels); - } - - this->ensure_resample_buffer_capacity(frameCount * this->_channel_count * sizeof(float)); - audio::merge::merge_channels_interleaved(this->resample_buffer, this->_channel_count, input, channels, frameCount); - input = this->resample_buffer; +void AudioInput::allocate_input_buffer_samples(size_t samples) { + const auto expected_byte_size = samples * this->channel_count_ * sizeof(float); + if(expected_byte_size > this->input_buffer.capacity()) { + log_critical(category::audio, tr("Resampled audio input data would be larger than our input buffer capacity.")); + return; } - if(this->_resampler) { - const auto expected_size = this->_resampler->estimated_output_size(frameCount); - const auto expected_byte_size = expected_size * this->_channel_count * sizeof(float); - this->ensure_resample_buffer_capacity(expected_byte_size); + if(this->input_buffer.free_count() < expected_byte_size) { + log_warn(category::audio, tr("Audio input buffer overflow.")); - size_t sample_count{expected_size}; - if(!this->_resampler->process(this->resample_buffer, input, frameCount, sample_count)) { + const auto free_samples = this->input_buffer.free_count() / this->channel_count_ / sizeof(float); + assert(samples >= free_samples); + + const auto missing_samples = samples - free_samples; + this->input_buffer.advance_read_ptr(missing_samples * this->channel_count_ * sizeof(float)); + } +} + +void AudioInput::consume(const void *input, size_t sample_count, size_t channels) { + constexpr static auto kTempBufferMaxSampleCount{1024 * 8}; + float temp_buffer[kTempBufferMaxSampleCount]; + + /* TODO: Short circuit for silence here */ + + if(channels != this->channel_count_) { + if(channels < 1 || channels > 2) { + log_critical(category::audio, tr("AudioInput received audio data with an unsupported channel count of {}."), channels); + return; + } + + if(sample_count * this->channel_count_ > kTempBufferMaxSampleCount) { + log_critical(category::audio, tr("Received audio chunk bigger than our temp stack buffer. Received {} samples but can hold only {}."), sample_count, kTempBufferMaxSampleCount / this->channel_count_); + return; + } + + audio::merge::merge_channels_interleaved(temp_buffer, this->channel_count_, input, channels, sample_count); + input = temp_buffer; + } + + if(this->resampler_) { + const auto expected_size = this->resampler_->estimated_output_size(sample_count); + this->allocate_input_buffer_samples(expected_size); + + size_t resampled_sample_count{expected_size}; + if(!this->resampler_->process(this->input_buffer.write_ptr(), input, sample_count, resampled_sample_count)) { log_error(category::audio, tr("Failed to resample input audio.")); return; } - frameCount = sample_count; - input = this->resample_buffer; + this->input_buffer.advance_write_ptr(resampled_sample_count * this->channel_count_ * sizeof(float)); + } else { + this->allocate_input_buffer_samples(sample_count); - audio::apply_gain(this->resample_buffer, this->_channel_count, frameCount, this->_volume); - } else if(this->_volume != 1) { - const auto byte_size = frameCount * this->_channel_count * sizeof(float); - this->ensure_resample_buffer_capacity(byte_size); - - if(this->resample_buffer != input) { - memcpy(this->resample_buffer, input, byte_size); - input = this->resample_buffer; - } - - audio::apply_gain(this->resample_buffer, this->_channel_count, frameCount, this->_volume); + const auto sample_byte_size = sample_count * this->channel_count_ * sizeof(float); + memcpy(this->input_buffer.write_ptr(), input, sample_byte_size); + this->input_buffer.advance_write_ptr(sample_byte_size); } - auto begin = std::chrono::system_clock::now(); - for(const auto& consumer : this->consumers()) { - consumer->process_data(input, frameCount); - } - - auto end = std::chrono::system_clock::now(); - auto ms = std::chrono::duration_cast(end - begin).count(); - if(ms > 5) { - log_warn(category::audio, tr("Processing of audio input needed {}ms. This could be an issue!"), std::chrono::duration_cast(end - begin).count()); - } + audio::encode_event_loop->schedule(this->event_loop_entry); } -void AudioInput::ensure_resample_buffer_capacity(size_t size) { - if(this->resample_buffer_size < size) { - free(this->resample_buffer); - this->resample_buffer = malloc(size); - this->resample_buffer_size = size; + +void AudioInput::process_audio() { + const auto chunk_sample_count = (kChunkSizeMs * this->sample_rate_) / 1000; + while(true) { + auto available_sample_count = this->input_buffer.fill_count() / this->channel_count_ / sizeof(float); + if(available_sample_count < chunk_sample_count) { + break; + } + + auto input = this->input_buffer.read_ptr(); + /* + * It's save to mutate the current memory. + * If overflows occur it could lead to wired artifacts but all memory access is save. + */ + this->process_audio_chunk((void*) input); + this->input_buffer.advance_read_ptr(chunk_sample_count * this->channel_count_ * sizeof(float)); } -} \ No newline at end of file +} + +void AudioInput::process_audio_chunk(void *chunk) { + constexpr static auto kTempSampleBufferSize{1024 * 8}; + constexpr static auto kMaxChannelCount{32}; + + const auto chunk_sample_count = this->chunk_sample_count(); + float temp_sample_buffer[kTempSampleBufferSize]; + float out_sample_buffer[kTempSampleBufferSize]; + assert(memset(temp_sample_buffer, 0, sizeof(float) * kTempSampleBufferSize)); + assert(memset(out_sample_buffer, 0, sizeof(float) * kTempSampleBufferSize)); + + if(auto processor{this->audio_processor_}; processor) { + assert(kTempSampleBufferSize >= chunk_sample_count * this->channel_count_ * sizeof(float)); + + audio::deinterleave(temp_sample_buffer, (const float*) chunk, this->channel_count_, chunk_sample_count); + webrtc::StreamConfig stream_config{(int) this->sample_rate_, this->channel_count_}; + + float* channel_ptr[kMaxChannelCount]; + assert(memset(channel_ptr, 0, sizeof(float*) * kMaxChannelCount)); + for(size_t channel{0}; channel < this->channel_count_; channel++) { + channel_ptr[channel] = temp_sample_buffer + (channel * chunk_sample_count); + } + + auto process_statistics = processor->process_stream(stream_config, channel_ptr); + if(!process_statistics.has_value()) { + /* TODO: Some kind of error message? */ + return; + } + + audio::interleave_vec(out_sample_buffer, channel_ptr, this->channel_count_, chunk_sample_count); + chunk = out_sample_buffer; + } + + /* TODO: Is this even needed if we have the processor? */ + audio::apply_gain(chunk, this->channel_count_, chunk_sample_count, this->volume_); + + auto begin = std::chrono::system_clock::now(); + for(const auto& consumer : this->consumers()) { + consumer->process_data(out_sample_buffer, chunk_sample_count); + } + + auto end = std::chrono::system_clock::now(); + auto ms = std::chrono::duration_cast(end - begin).count(); + if(ms > 5) { + log_warn(category::audio, tr("Processing of audio input needed {}ms. This could be an issue!"), std::chrono::duration_cast(end - begin).count()); + } +} + +void AudioInput::EventLoopCallback::event_execute(const chrono::system_clock::time_point &point) { + this->input->process_audio(); +} diff --git a/native/serverconnection/src/audio/AudioInput.h b/native/serverconnection/src/audio/AudioInput.h index f820052..4acac1c 100644 --- a/native/serverconnection/src/audio/AudioInput.h +++ b/native/serverconnection/src/audio/AudioInput.h @@ -5,29 +5,30 @@ #include #include #include -#include "AudioSamples.h" -#include "driver/AudioDriver.h" +#include "./AudioSamples.h" +#include "./driver/AudioDriver.h" +#include "../ring_buffer.h" +#include "../EventLoop.h" -class AudioInputSource; namespace tc::audio { class AudioInput; class InputReframer; class AudioResampler; + class AudioProcessor; + class AudioInputSource; class AudioConsumer { friend class AudioInput; public: - AudioInput* handle; + size_t const channel_count; + size_t const sample_rate; - size_t const channel_count = 0; - size_t const sample_rate = 0; - - size_t const frame_size = 0; + size_t const frame_size; std::mutex on_read_lock{}; /* locked to access the function */ std::function on_read; private: - AudioConsumer(AudioInput* handle, size_t channel_count, size_t sample_rate, size_t frame_size); + AudioConsumer(size_t channel_count, size_t sample_rate, size_t frame_size); std::unique_ptr reframer; @@ -36,7 +37,7 @@ namespace tc::audio { }; class AudioInput : public AudioDeviceRecord::Consumer { - friend class ::AudioInputSource; + friend class AudioInputSource; public: AudioInput(size_t /* channels */, size_t /* sample rate */); virtual ~AudioInput(); @@ -49,38 +50,57 @@ namespace tc::audio { [[nodiscard]] bool recording(); void stop(); - std::deque> consumers() { - std::lock_guard lock(this->consumers_lock); - return this->_consumers; - } + [[nodiscard]] std::vector> consumers(); + [[nodiscard]] std::shared_ptr create_consumer(size_t /* frame size */); - std::shared_ptr create_consumer(size_t /* frame size */); - void delete_consumer(const std::shared_ptr& /* source */); + [[nodiscard]] inline auto audio_processor() { return this->audio_processor_; } - [[nodiscard]] inline size_t channel_count() const { return this->_channel_count; } - [[nodiscard]] inline size_t sample_rate() const { return this->_sample_rate; } + [[nodiscard]] inline size_t channel_count() const { return this->channel_count_; } + [[nodiscard]] inline size_t sample_rate() const { return this->sample_rate_; } - [[nodiscard]] inline float volume() const { return this->_volume; } - inline void set_volume(float value) { this->_volume = value; } + [[nodiscard]] inline float volume() const { return this->volume_; } + inline void set_volume(float value) { this->volume_ = value; } private: + constexpr static auto kInputBufferCapacityMs{750}; + constexpr static auto kChunkSizeMs{10}; /* Must be 10ms else the audio processor will fuck up */ + + struct EventLoopCallback : public event::EventEntry { + AudioInput* const input; + + explicit EventLoopCallback(AudioInput* input) : input{input} {} + void event_execute(const std::chrono::system_clock::time_point &point) override; + }; + + struct AudioInitializeHook { + std::mutex mutex{}; + AudioInput* input{nullptr}; + }; + void consume(const void *, size_t, size_t) override; - void ensure_resample_buffer_capacity(size_t); + void process_audio(); + void process_audio_chunk(void *); - size_t const _channel_count; - size_t const _sample_rate; + size_t const channel_count_; + size_t const sample_rate_; - std::mutex consumers_lock; - std::deque> _consumers; - std::recursive_mutex input_source_lock; + std::mutex consumers_mutex{}; + std::deque> consumers_{}; + std::recursive_mutex input_source_lock{}; - std::unique_ptr _resampler{nullptr}; + std::shared_ptr event_loop_entry{}; + ring_buffer input_buffer; + + std::unique_ptr resampler_{nullptr}; std::shared_ptr input_device{}; - void* resample_buffer{nullptr}; - size_t resample_buffer_size{0}; - - float _volume{1.f}; + float volume_{1.f}; std::shared_ptr input_recorder{}; + std::shared_ptr audio_processor_{}; + + std::shared_ptr initialize_hook_handle{}; + + void allocate_input_buffer_samples(size_t /* sample count */); + [[nodiscard]] inline auto chunk_sample_count() const { return (kChunkSizeMs * this->sample_rate_) / 1000; } }; } \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioInterleaved.cpp b/native/serverconnection/src/audio/AudioInterleaved.cpp index e7341e4..93b7509 100644 --- a/native/serverconnection/src/audio/AudioInterleaved.cpp +++ b/native/serverconnection/src/audio/AudioInterleaved.cpp @@ -3,6 +3,7 @@ // #include +#include #include "AudioInterleaved.h" using namespace tc; @@ -33,6 +34,15 @@ void audio::interleave(float *target, const float *source, size_t channel_count, source_ptr[channel] = source + (channel * sample_count); } + audio::interleave_vec(target, source_ptr, channel_count, sample_count); +} + +void audio::interleave_vec(float *target, const float *const *source, size_t channel_count, size_t sample_count) { + assert(channel_count <= kMaxChannelCount); + + const float* source_ptr[kMaxChannelCount]; + memcpy(source_ptr, source, channel_count * sizeof(float*)); + for(size_t sample{0}; sample < sample_count; sample++) { for(size_t channel{0}; channel < channel_count; channel++) { *target++ = *source_ptr[channel]++; diff --git a/native/serverconnection/src/audio/AudioInterleaved.h b/native/serverconnection/src/audio/AudioInterleaved.h index 0bce7c8..91568a5 100644 --- a/native/serverconnection/src/audio/AudioInterleaved.h +++ b/native/serverconnection/src/audio/AudioInterleaved.h @@ -14,4 +14,11 @@ namespace tc::audio { size_t /* channel count */, size_t /* sample count */ ); + + extern void interleave_vec( + float* /* dest */, + const float * const * /* sources */, + size_t /* channel count */, + size_t /* sample count */ + ); } \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioReframer.cpp b/native/serverconnection/src/audio/AudioReframer.cpp index b6da498..85823eb 100644 --- a/native/serverconnection/src/audio/AudioReframer.cpp +++ b/native/serverconnection/src/audio/AudioReframer.cpp @@ -10,8 +10,9 @@ InputReframer::InputReframer(size_t channels, size_t frame_size) : _frame_size(f } InputReframer::~InputReframer() { - if(this->buffer) - free(this->buffer); + if(this->buffer) { + free(this->buffer); + } } void InputReframer::process(const void *source, size_t samples) { @@ -23,15 +24,15 @@ void InputReframer::process(const void *source, size_t samples) { if(this->_buffer_index > 0) { if(this->_buffer_index + samples > this->_frame_size) { auto required = this->_frame_size - this->_buffer_index; - auto length = required * this->_channels * 4; + auto length = required * this->_channels * sizeof(float); - memcpy((char*) this->buffer + this->_buffer_index * 4 * this->_channels, source, length); + memcpy((char*) this->buffer + this->_buffer_index * sizeof(float) * this->_channels, source, length); samples -= required; source = (char*) source + length; this->on_frame(this->buffer); } else { - memcpy((char*) this->buffer + this->_buffer_index * 4 * this->_channels, source, samples * this->_channels * 4); + memcpy((char*) this->buffer + this->_buffer_index * sizeof(float) * this->_channels, source, samples * this->_channels * sizeof(float)); this->_buffer_index += samples; return; } @@ -39,12 +40,16 @@ void InputReframer::process(const void *source, size_t samples) { auto _on_frame = this->on_frame; while(samples > this->_frame_size) { - if(_on_frame) - _on_frame(source); + if(_on_frame) { + _on_frame(source); + } + samples -= this->_frame_size; - source = (char*) source + this->_frame_size * this->_channels * 4; + source = (char*) source + this->_frame_size * this->_channels * sizeof(float); } - if(samples > 0) - memcpy((char*) this->buffer, source, samples * this->_channels * 4); + if(samples > 0) { + memcpy((char*) this->buffer, source, samples * this->_channels * sizeof(float)); + } + this->_buffer_index = samples; } \ No newline at end of file diff --git a/native/serverconnection/src/audio/filter/FilterVad.cpp b/native/serverconnection/src/audio/filter/FilterVad.cpp index 6f0f493..5211b65 100644 --- a/native/serverconnection/src/audio/filter/FilterVad.cpp +++ b/native/serverconnection/src/audio/filter/FilterVad.cpp @@ -1,4 +1,3 @@ -#include #include "FilterVad.h" #include "../AudioMerger.h" #include "../../logger.h" diff --git a/native/serverconnection/src/audio/js/AudioConsumer.cpp b/native/serverconnection/src/audio/js/AudioConsumer.cpp index 7f70c04..325e170 100644 --- a/native/serverconnection/src/audio/js/AudioConsumer.cpp +++ b/native/serverconnection/src/audio/js/AudioConsumer.cpp @@ -32,9 +32,6 @@ NAN_MODULE_INIT(AudioConsumerWrapper::Init) { Nan::SetPrototypeMethod(klass, "get_filter_mode", AudioConsumerWrapper::_get_filter_mode); Nan::SetPrototypeMethod(klass, "set_filter_mode", AudioConsumerWrapper::_set_filter_mode); - Nan::SetPrototypeMethod(klass, "rnnoise_enabled", AudioConsumerWrapper::rnnoise_enabled); - Nan::SetPrototypeMethod(klass, "toggle_rnnoise", AudioConsumerWrapper::toggle_rnnoise); - constructor_template().Reset(klass); constructor().Reset(Nan::GetFunction(klass).ToLocalChecked()); } @@ -61,17 +58,6 @@ AudioConsumerWrapper::~AudioConsumerWrapper() { lock_guard lock{this->execute_mutex}; this->unbind(); - if(this->_handle->handle) { - this->_handle->handle->delete_consumer(this->_handle); - this->_handle = nullptr; - } - - for(auto& instance : this->rnnoise_processor) { - if(!instance) { continue; } - - rnnoise_destroy((DenoiseState*) instance); - instance = nullptr; - } for(auto index{0}; index < kInternalFrameBufferCount; index++) { if(!this->internal_frame_buffer[index]) { continue; } @@ -104,9 +90,11 @@ void AudioConsumerWrapper::do_wrap(const v8::Local &obj) { std::unique_ptr buffer; while(true) { { - lock_guard lock(this->_data_lock); - if(this->_data_entries.empty()) - break; + lock_guard lock{this->_data_lock}; + if(this->_data_entries.empty()) { + break; + } + buffer = move(this->_data_entries.front()); this->_data_entries.pop_front(); } @@ -155,7 +143,6 @@ void AudioConsumerWrapper::unbind() { } } -static const float kRnNoiseScale = -INT16_MIN; void AudioConsumerWrapper::process_data(const void *buffer, size_t samples) { if(samples != 960) { logger::error(logger::category::audio, tr("Received audio frame with invalid sample count (Expected 960, Received {})"), samples); @@ -165,67 +152,6 @@ void AudioConsumerWrapper::process_data(const void *buffer, size_t samples) { lock_guard lock{this->execute_mutex}; if(this->filter_mode_ == FilterMode::BLOCK) { return; } - /* apply input modifiers */ - if(this->rnnoise) { - /* TODO: don't call reserve_internal_buffer every time and assume the buffers are initialized */ - /* TODO: Maybe find out if the microphone is some kind of pseudo stero so we can handle it as mono? */ - - if(this->_handle->channel_count > 1) { - auto channel_count = this->_handle->channel_count; - this->reserve_internal_buffer(0, samples * channel_count * sizeof(float)); - this->reserve_internal_buffer(1, samples * channel_count * sizeof(float)); - - for(size_t channel{0}; channel < channel_count; channel++) { - auto target_buffer = (float*) this->internal_frame_buffer[1]; - auto source_buffer = (const float*) buffer + channel; - - for(size_t index{0}; index < samples; index++) { - *target_buffer = *source_buffer * kRnNoiseScale; - source_buffer += channel_count; - target_buffer++; - } - - /* rnnoise uses a frame size of 480 */ - this->initialize_rnnoise(channel); - rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[channel], (float*) this->internal_frame_buffer[0] + channel * samples, (const float*) this->internal_frame_buffer[1]); - rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[channel], (float*) this->internal_frame_buffer[0] + channel * samples + 480, (const float*) this->internal_frame_buffer[1] + 480); - } - - const float* channel_buffer_ptr[kMaxChannelCount]; - for(size_t channel{0}; channel < channel_count; channel++) { - channel_buffer_ptr[channel] = (const float*) this->internal_frame_buffer[0] + channel * samples; - } - - /* now back again to interlanced */ - auto target_buffer = (float*) this->internal_frame_buffer[1]; - for(size_t index{0}; index < samples; index++) { - for(size_t channel{0}; channel < channel_count; channel++) { - *target_buffer = *(channel_buffer_ptr[channel]++) / kRnNoiseScale; - target_buffer++; - } - } - - buffer = this->internal_frame_buffer[1]; - } else { - /* rnnoise uses a frame size of 480 */ - this->reserve_internal_buffer(0, samples * sizeof(float)); - - auto target_buffer = (float*) this->internal_frame_buffer[0]; - for(size_t index{0}; index < samples; index++) { - target_buffer[index] = ((float*) buffer)[index] * kRnNoiseScale; - } - - this->initialize_rnnoise(0); - rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[0], target_buffer, target_buffer); - rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[0], &target_buffer[480], &target_buffer[480]); - - buffer = target_buffer; - for(size_t index{0}; index < samples; index++) { - target_buffer[index] /= kRnNoiseScale; - } - } - } - bool should_process{true}; if(this->filter_mode_ == FilterMode::FILTER) { auto filters = this->filters(); @@ -350,12 +276,6 @@ void AudioConsumerWrapper::reserve_internal_buffer(int index, size_t target) { } } -void AudioConsumerWrapper::initialize_rnnoise(int channel) { - if(!this->rnnoise_processor[channel]) { - this->rnnoise_processor[channel] = rnnoise_create(nullptr); - } -} - NAN_METHOD(AudioConsumerWrapper::_get_filters) { auto handle = ObjectWrap::Unwrap(info.Holder()); auto filters = handle->filters(); @@ -459,20 +379,4 @@ NAN_METHOD(AudioConsumerWrapper::_set_filter_mode) { auto value = info[0].As()->Int32Value(info.GetIsolate()->GetCurrentContext()).FromMaybe(0); handle->filter_mode_ = (FilterMode) value; -} - -NAN_METHOD(AudioConsumerWrapper::rnnoise_enabled) { - auto handle = ObjectWrap::Unwrap(info.Holder()); - info.GetReturnValue().Set(handle->rnnoise); -} - -NAN_METHOD(AudioConsumerWrapper::toggle_rnnoise) { - auto handle = ObjectWrap::Unwrap(info.Holder()); - - if(info.Length() != 1 || !info[0]->IsBoolean()) { - Nan::ThrowError("invalid argument"); - return; - } - - handle->rnnoise = info[0]->BooleanValue(info.GetIsolate()); } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioConsumer.h b/native/serverconnection/src/audio/js/AudioConsumer.h index 272ccfe..261ef90 100644 --- a/native/serverconnection/src/audio/js/AudioConsumer.h +++ b/native/serverconnection/src/audio/js/AudioConsumer.h @@ -52,9 +52,6 @@ namespace tc::audio { static NAN_METHOD(_get_filter_mode); static NAN_METHOD(_set_filter_mode); - static NAN_METHOD(toggle_rnnoise); - static NAN_METHOD(rnnoise_enabled); - std::shared_ptr create_filter(const std::string& /* name */, const std::shared_ptr& /* filter impl */); void delete_filter(const AudioFilterWrapper*); @@ -63,7 +60,7 @@ namespace tc::audio { return this->filter_; } - inline FilterMode filter_mode() const { return this->filter_mode_; } + [[nodiscard]] inline FilterMode filter_mode() const { return this->filter_mode_; } inline std::shared_ptr native_consumer() { return this->_handle; } std::mutex native_read_callback_lock; @@ -71,10 +68,6 @@ namespace tc::audio { private: AudioRecorderWrapper* _recorder; - /* preprocessors */ - bool rnnoise{false}; - std::array rnnoise_processor{nullptr}; - std::mutex execute_mutex; std::shared_ptr _handle; @@ -93,7 +86,6 @@ namespace tc::audio { void process_data(const void* /* buffer */, size_t /* samples */); void reserve_internal_buffer(int /* buffer */, size_t /* bytes */); - void initialize_rnnoise(int /* channel */); struct DataEntry { void* buffer = nullptr; diff --git a/native/serverconnection/src/audio/js/AudioOutputStream.cpp b/native/serverconnection/src/audio/js/AudioOutputStream.cpp index 492343e..0c0f7ad 100644 --- a/native/serverconnection/src/audio/js/AudioOutputStream.cpp +++ b/native/serverconnection/src/audio/js/AudioOutputStream.cpp @@ -32,8 +32,9 @@ NAN_MODULE_INIT(AudioOutputStreamWrapper::Init) { } NAN_METHOD(AudioOutputStreamWrapper::NewInstance) { - if(!info.IsConstructCall()) - Nan::ThrowError("invalid invoke!"); + if(!info.IsConstructCall()) { + Nan::ThrowError("invalid invoke!"); + } } diff --git a/native/serverconnection/src/audio/js/AudioOutputStream.h b/native/serverconnection/src/audio/js/AudioOutputStream.h index dfae54b..805e254 100644 --- a/native/serverconnection/src/audio/js/AudioOutputStream.h +++ b/native/serverconnection/src/audio/js/AudioOutputStream.h @@ -3,50 +3,48 @@ #include #include -namespace tc { - namespace audio { - class AudioResampler; - class AudioOutputSource; +namespace tc::audio { + class AudioResampler; + class AudioOutputSource; - class AudioOutputStreamWrapper : public Nan::ObjectWrap { - public: - static NAN_MODULE_INIT(Init); - static NAN_METHOD(NewInstance); - static inline Nan::Persistent & constructor() { - static Nan::Persistent my_constructor; - return my_constructor; - } + class AudioOutputStreamWrapper : public Nan::ObjectWrap { + public: + static NAN_MODULE_INIT(Init); + static NAN_METHOD(NewInstance); + static inline Nan::Persistent & constructor() { + static Nan::Persistent my_constructor; + return my_constructor; + } - AudioOutputStreamWrapper(const std::shared_ptr& /* stream */, bool /* own */); - virtual ~AudioOutputStreamWrapper(); + AudioOutputStreamWrapper(const std::shared_ptr& /* stream */, bool /* own */); + ~AudioOutputStreamWrapper() override; - void do_wrap(const v8::Local&); - void drop_stream(); - private: - static ssize_t write_data(const std::shared_ptr&, void* source, size_t samples, bool interleaved); + void do_wrap(const v8::Local&); + void drop_stream(); + private: + static ssize_t write_data(const std::shared_ptr&, void* source, size_t samples, bool interleaved); - /* general methods */ - static NAN_METHOD(_get_buffer_latency); - static NAN_METHOD(_set_buffer_latency); - static NAN_METHOD(_get_buffer_max_latency); - static NAN_METHOD(_set_buffer_max_latency); + /* general methods */ + static NAN_METHOD(_get_buffer_latency); + static NAN_METHOD(_set_buffer_latency); + static NAN_METHOD(_get_buffer_max_latency); + static NAN_METHOD(_set_buffer_max_latency); - static NAN_METHOD(_flush_buffer); + static NAN_METHOD(_flush_buffer); - /* methods for owned streams only */ - static NAN_METHOD(_write_data); - static NAN_METHOD(_write_data_rated); + /* methods for owned streams only */ + static NAN_METHOD(_write_data); + static NAN_METHOD(_write_data_rated); - static NAN_METHOD(_clear); - static NAN_METHOD(_deleted); - static NAN_METHOD(_delete); + static NAN_METHOD(_clear); + static NAN_METHOD(_deleted); + static NAN_METHOD(_delete); - std::unique_ptr _resampler; - std::shared_ptr _own_handle; - std::weak_ptr _handle; + std::unique_ptr _resampler; + std::shared_ptr _own_handle; + std::weak_ptr _handle; - Nan::callback_t<> call_underflow; - Nan::callback_t<> call_overflow; - }; - } + Nan::callback_t<> call_underflow; + Nan::callback_t<> call_overflow; + }; } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioProcessor.cpp b/native/serverconnection/src/audio/js/AudioProcessor.cpp new file mode 100644 index 0000000..f0cb504 --- /dev/null +++ b/native/serverconnection/src/audio/js/AudioProcessor.cpp @@ -0,0 +1,396 @@ +// +// Created by WolverinDEV on 28/03/2021. +// + +#include "./AudioProcessor.h" +#include "../../logger.h" +#include + +using namespace tc::audio; + +NAN_MODULE_INIT(AudioProcessorWrapper::Init) { + auto klass = Nan::New(AudioProcessorWrapper::NewInstance); + klass->SetClassName(Nan::New("AudioProcessor").ToLocalChecked()); + klass->InstanceTemplate()->SetInternalFieldCount(1); + + Nan::SetPrototypeMethod(klass, "get_config", AudioProcessorWrapper::get_config); + Nan::SetPrototypeMethod(klass, "apply_config", AudioProcessorWrapper::apply_config); + + Nan::SetPrototypeMethod(klass, "get_statistics", AudioProcessorWrapper::get_statistics); + + constructor().Reset(Nan::GetFunction(klass).ToLocalChecked()); +} + +NAN_METHOD(AudioProcessorWrapper::NewInstance) { + if(!info.IsConstructCall()) { + Nan::ThrowError("invalid invoke!"); + } +} + +AudioProcessorWrapper::AudioProcessorWrapper(const std::shared_ptr &processor) { + log_allocate("AudioProcessorWrapper", this); + + this->registered_observer = new Observer{this}; + this->weak_processor = processor; + processor->register_process_observer(this->registered_observer); +} + +AudioProcessorWrapper::~AudioProcessorWrapper() noexcept { + log_allocate("AudioProcessorWrapper", this); + + if(auto processor{this->weak_processor.lock()}; processor) { + processor->unregister_process_observer(this->registered_observer); + } + + delete this->registered_observer; +} + +#define PUT_VALUE(key, value) \ + result->Set(context, Nan::LocalStringUTF8(#key), value).Check() + +#define PUT_CONFIG(path) \ + PUT_VALUE(path, Nan::New(config.path)) + +#define LOAD_CONFIG(path, ...) \ +do { \ + if(!load_config_value(context, js_config, #path, config.path, __VA_ARGS__)) { \ + return; \ + } \ +} while(0) + +NAN_METHOD(AudioProcessorWrapper::get_config) { + auto handle = Nan::ObjectWrap::Unwrap(info.Holder()); + auto processor = handle->weak_processor.lock(); + if(!processor) { + Nan::ThrowError("processor passed away"); + return; + } + + auto config = processor->get_config(); + auto result = Nan::New(); + auto context = info.GetIsolate()->GetCurrentContext(); + + PUT_CONFIG(pipeline.maximum_internal_processing_rate); + PUT_CONFIG(pipeline.multi_channel_render); + PUT_CONFIG(pipeline.multi_channel_capture); + + PUT_CONFIG(pre_amplifier.enabled); + PUT_CONFIG(pre_amplifier.fixed_gain_factor); + + PUT_CONFIG(high_pass_filter.enabled); + PUT_CONFIG(high_pass_filter.apply_in_full_band); + + PUT_CONFIG(echo_canceller.enabled); + PUT_CONFIG(echo_canceller.mobile_mode); + PUT_CONFIG(echo_canceller.export_linear_aec_output); /* TODO: Consider removing? */ + PUT_CONFIG(echo_canceller.enforce_high_pass_filtering); + + PUT_CONFIG(noise_suppression.enabled); + switch (config.noise_suppression.level) { + using Level = webrtc::AudioProcessing::Config::NoiseSuppression::Level; + case Level::kLow: + PUT_VALUE(noise_suppression.level, Nan::LocalStringUTF8("low")); + break; + + case Level::kModerate: + PUT_VALUE(noise_suppression.level, Nan::LocalStringUTF8("moderate")); + break; + + case Level::kHigh: + PUT_VALUE(noise_suppression.level, Nan::LocalStringUTF8("high")); + break; + + case Level::kVeryHigh: + PUT_VALUE(noise_suppression.level, Nan::LocalStringUTF8("very-high")); + break; + + default: + PUT_VALUE(noise_suppression.level, Nan::LocalStringUTF8("unknown")); + break; + } + PUT_CONFIG(noise_suppression.analyze_linear_aec_output_when_available); + + PUT_CONFIG(transient_suppression.enabled); + + PUT_CONFIG(voice_detection.enabled); + + PUT_CONFIG(gain_controller1.enabled); + switch (config.gain_controller1.mode) { + using Mode = webrtc::AudioProcessing::Config::GainController1::Mode; + case Mode::kAdaptiveAnalog: + PUT_VALUE(gain_controller1.mode, Nan::LocalStringUTF8("adaptive-analog")); + break; + + case Mode::kAdaptiveDigital: + PUT_VALUE(gain_controller1.mode, Nan::LocalStringUTF8("adaptive-digital")); + break; + + case Mode::kFixedDigital: + PUT_VALUE(gain_controller1.mode, Nan::LocalStringUTF8("fixed-digital")); + break; + + default: + PUT_VALUE(gain_controller1.mode, Nan::LocalStringUTF8("unknown")); + break; + } + PUT_CONFIG(gain_controller1.target_level_dbfs); + PUT_CONFIG(gain_controller1.compression_gain_db); + PUT_CONFIG(gain_controller1.enable_limiter); + PUT_CONFIG(gain_controller1.analog_level_minimum); + PUT_CONFIG(gain_controller1.analog_level_maximum); + + PUT_CONFIG(gain_controller1.analog_gain_controller.enabled); + PUT_CONFIG(gain_controller1.analog_gain_controller.startup_min_volume); + PUT_CONFIG(gain_controller1.analog_gain_controller.clipped_level_min); + PUT_CONFIG(gain_controller1.analog_gain_controller.enable_agc2_level_estimator); + PUT_CONFIG(gain_controller1.analog_gain_controller.enable_digital_adaptive); + + PUT_CONFIG(gain_controller2.enabled); + + PUT_CONFIG(gain_controller2.fixed_digital.gain_db); + + PUT_CONFIG(gain_controller2.adaptive_digital.enabled); + switch(config.gain_controller2.adaptive_digital.level_estimator) { + using LevelEstimator = webrtc::AudioProcessing::Config::GainController2::LevelEstimator; + + case LevelEstimator::kPeak: + PUT_VALUE(gain_controller2.adaptive_digital.level_estimator, Nan::LocalStringUTF8("peak")); + break; + + case LevelEstimator::kRms: + PUT_VALUE(gain_controller2.adaptive_digital.level_estimator, Nan::LocalStringUTF8("rms")); + break; + + default: + PUT_VALUE(gain_controller2.adaptive_digital.level_estimator, Nan::LocalStringUTF8("unknown")); + break; + } + PUT_CONFIG(gain_controller2.adaptive_digital.vad_probability_attack); + PUT_CONFIG(gain_controller2.adaptive_digital.level_estimator_adjacent_speech_frames_threshold); + PUT_CONFIG(gain_controller2.adaptive_digital.use_saturation_protector); + PUT_CONFIG(gain_controller2.adaptive_digital.initial_saturation_margin_db); + PUT_CONFIG(gain_controller2.adaptive_digital.extra_saturation_margin_db); + PUT_CONFIG(gain_controller2.adaptive_digital.gain_applier_adjacent_speech_frames_threshold); + PUT_CONFIG(gain_controller2.adaptive_digital.max_gain_change_db_per_second); + PUT_CONFIG(gain_controller2.adaptive_digital.max_output_noise_level_dbfs); + + PUT_CONFIG(residual_echo_detector.enabled); + PUT_CONFIG(level_estimation.enabled); + PUT_CONFIG(rnnoise.enabled); + + info.GetReturnValue().Set(result); +} + +template ::value, T>::type> +inline bool load_config_value( + const v8::Local& context, + const v8::Local& js_config, + const std::string_view& key, + T& value_ref, + T min_value = std::numeric_limits::min(), + T max_value = std::numeric_limits::max() +) { + auto maybe_value = js_config->Get(context, Nan::LocalStringUTF8(key)); + if(maybe_value.IsEmpty() || maybe_value.ToLocalChecked()->IsNullOrUndefined()) { + return true; + } + + double value; + + if(maybe_value.ToLocalChecked()->IsNumber()) { + value = maybe_value.ToLocalChecked()->NumberValue(context).ToChecked(); + } else if(maybe_value.ToLocalChecked()->IsBoolean()) { + value = maybe_value.ToLocalChecked()->BooleanValue(v8::Isolate::GetCurrent()); + } else { + Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " isn't a number or boolean")); + return false; + } + + if(std::numeric_limits::is_integer && (double) (T) value != value) { + Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " isn't an integer")); + return false; + } + + if(value < (double) min_value) { + Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " exceeds min value of " + std::to_string(min_value))); + return false; + } + + if(value > (double) max_value) { + Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " exceeds max value of " + std::to_string(max_value))); + return false; + } + + value_ref = value; + return true; +} + +template +inline bool load_config_enum( + const v8::Local& context, + const v8::Local& js_config, + const std::string_view& key, + T& value_ref, + const std::array, kValueSize>& values +) { + auto maybe_value = js_config->Get(context, Nan::LocalStringUTF8(key)); + if(maybe_value.IsEmpty() || maybe_value.ToLocalChecked()->IsNullOrUndefined()) { + return true; + } else if(!maybe_value.ToLocalChecked()->IsString()) { + Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " isn't a string")); + return false; + } + + auto str_value = maybe_value.ToLocalChecked()->ToString(context).ToLocalChecked(); + auto value = *Nan::Utf8String(str_value); + for(const auto& [ key, key_value ] : values) { + if(key != value) { + continue; + } + + value_ref = key_value; + return true; + } + + Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " contains an invalid enum value (" + value + ")")); + return false; +} + +#define LOAD_ENUM(path, arg_count, ...) \ +do { \ + if(!load_config_enum(context, js_config, #path, config.path, {{ __VA_ARGS__ }})) { \ + return; \ + } \ +} while(0) + +NAN_METHOD(AudioProcessorWrapper::apply_config) { + auto handle = Nan::ObjectWrap::Unwrap(info.Holder()); + auto processor = handle->weak_processor.lock(); + if (!processor) { + Nan::ThrowError("processor passed away"); + return; + } + + if(info.Length() != 1 || !info[0]->IsObject()) { + Nan::ThrowError("Invalid arguments"); + return; + } + + auto config = processor->get_config(); + auto context = info.GetIsolate()->GetCurrentContext(); + auto js_config = info[0]->ToObject(info.GetIsolate()->GetCurrentContext()).ToLocalChecked(); + + using GainControllerMode = webrtc::AudioProcessing::Config::GainController1::Mode; + using GainControllerLevelEstimator = webrtc::AudioProcessing::Config::GainController2::LevelEstimator; + using NoiseSuppressionLevel = webrtc::AudioProcessing::Config::NoiseSuppression::Level; + + LOAD_CONFIG(pipeline.maximum_internal_processing_rate); + LOAD_CONFIG(pipeline.multi_channel_render); + LOAD_CONFIG(pipeline.multi_channel_capture); + + LOAD_CONFIG(pre_amplifier.enabled); + LOAD_CONFIG(pre_amplifier.fixed_gain_factor); + + LOAD_CONFIG(high_pass_filter.enabled); + LOAD_CONFIG(high_pass_filter.apply_in_full_band); + + LOAD_CONFIG(echo_canceller.enabled); + LOAD_CONFIG(echo_canceller.mobile_mode); + LOAD_CONFIG(echo_canceller.export_linear_aec_output); /* TODO: Consider removing? */ + LOAD_CONFIG(echo_canceller.enforce_high_pass_filtering); + + LOAD_CONFIG(noise_suppression.enabled); + LOAD_ENUM(noise_suppression.level, 4, + { "low", NoiseSuppressionLevel::kLow }, + { "moderate", NoiseSuppressionLevel::kModerate }, + { "high", NoiseSuppressionLevel::kHigh }, + { "very-high", NoiseSuppressionLevel::kVeryHigh } + ); + LOAD_CONFIG(noise_suppression.analyze_linear_aec_output_when_available); + + LOAD_CONFIG(transient_suppression.enabled); + + LOAD_CONFIG(voice_detection.enabled); + + LOAD_CONFIG(gain_controller1.enabled); + LOAD_ENUM(gain_controller1.mode, 3, + { "adaptive-analog", GainControllerMode::kAdaptiveAnalog }, + { "adaptive-digital", GainControllerMode::kAdaptiveDigital }, + { "fixed-digital", GainControllerMode::kFixedDigital } + ); + LOAD_CONFIG(gain_controller1.target_level_dbfs); + LOAD_CONFIG(gain_controller1.compression_gain_db); + LOAD_CONFIG(gain_controller1.enable_limiter); + LOAD_CONFIG(gain_controller1.analog_level_minimum); + LOAD_CONFIG(gain_controller1.analog_level_maximum); + + LOAD_CONFIG(gain_controller1.analog_gain_controller.enabled); + LOAD_CONFIG(gain_controller1.analog_gain_controller.startup_min_volume); + LOAD_CONFIG(gain_controller1.analog_gain_controller.clipped_level_min); + LOAD_CONFIG(gain_controller1.analog_gain_controller.enable_agc2_level_estimator); + LOAD_CONFIG(gain_controller1.analog_gain_controller.enable_digital_adaptive); + + LOAD_CONFIG(gain_controller2.enabled); + + LOAD_CONFIG(gain_controller2.fixed_digital.gain_db); + + LOAD_CONFIG(gain_controller2.adaptive_digital.enabled); + LOAD_ENUM(gain_controller2.adaptive_digital.level_estimator, 2, + { "peak", GainControllerLevelEstimator::kPeak }, + { "rms", GainControllerLevelEstimator::kRms } + ); + LOAD_CONFIG(gain_controller2.adaptive_digital.vad_probability_attack); + LOAD_CONFIG(gain_controller2.adaptive_digital.level_estimator_adjacent_speech_frames_threshold); + LOAD_CONFIG(gain_controller2.adaptive_digital.use_saturation_protector); + LOAD_CONFIG(gain_controller2.adaptive_digital.initial_saturation_margin_db); + LOAD_CONFIG(gain_controller2.adaptive_digital.extra_saturation_margin_db); + LOAD_CONFIG(gain_controller2.adaptive_digital.gain_applier_adjacent_speech_frames_threshold); + LOAD_CONFIG(gain_controller2.adaptive_digital.max_gain_change_db_per_second); + LOAD_CONFIG(gain_controller2.adaptive_digital.max_output_noise_level_dbfs); + + LOAD_CONFIG(residual_echo_detector.enabled); + LOAD_CONFIG(level_estimation.enabled); + LOAD_CONFIG(rnnoise.enabled); + + processor->apply_config(config); +} + +#define PUT_STATISTIC(path) \ +do { \ + if(config.path.has_value()) { \ + PUT_VALUE(path, Nan::New(*config.path)); \ + } else { \ + PUT_VALUE(path, Nan::Undefined()); \ + } \ +} while(0) + +NAN_METHOD(AudioProcessorWrapper::get_statistics) { + auto handle = Nan::ObjectWrap::Unwrap(info.Holder()); + auto processor = handle->weak_processor.lock(); + if(!processor) { + Nan::ThrowError("processor passed away"); + return; + } + + auto config = processor->get_statistics(); + auto result = Nan::New(); + auto context = info.GetIsolate()->GetCurrentContext(); + + PUT_STATISTIC(output_rms_dbfs); + PUT_STATISTIC(voice_detected); + PUT_STATISTIC(echo_return_loss); + PUT_STATISTIC(echo_return_loss_enhancement); + PUT_STATISTIC(divergent_filter_fraction); + PUT_STATISTIC(delay_median_ms); + PUT_STATISTIC(delay_standard_deviation_ms); + PUT_STATISTIC(residual_echo_likelihood); + PUT_STATISTIC(residual_echo_likelihood_recent_max); + PUT_STATISTIC(delay_ms); + PUT_STATISTIC(rnnoise_volume); + + info.GetReturnValue().Set(result); +} + +void AudioProcessorWrapper::Observer::stream_processed(const AudioProcessor::Stats &stats) { + /* TODO! */ +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioProcessor.h b/native/serverconnection/src/audio/js/AudioProcessor.h new file mode 100644 index 0000000..e10cdda --- /dev/null +++ b/native/serverconnection/src/audio/js/AudioProcessor.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include "../processing/AudioProcessor.h" + +namespace tc::audio { + class AudioProcessorWrapper : public Nan::ObjectWrap { + public: + static NAN_MODULE_INIT(Init); + static NAN_METHOD(NewInstance); + static inline Nan::Persistent & constructor() { + static Nan::Persistent my_constructor; + return my_constructor; + } + + explicit AudioProcessorWrapper(const std::shared_ptr& /* processor */); + ~AudioProcessorWrapper() override; + + inline void wrap(v8::Local object) { + Nan::ObjectWrap::Wrap(object); + } + + static NAN_METHOD(get_config); + static NAN_METHOD(apply_config); + + static NAN_METHOD(get_statistics); + private: + struct Observer : public AudioProcessor::ProcessObserver { + public: + explicit Observer(AudioProcessorWrapper* wrapper) : wrapper{wrapper} {} + + private: + AudioProcessorWrapper* wrapper; + + void stream_processed(const AudioProcessor::Stats &stats) override; + }; + + std::weak_ptr weak_processor{}; + Observer* registered_observer{nullptr}; + }; +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioRecorder.cpp b/native/serverconnection/src/audio/js/AudioRecorder.cpp index b4703b7..6eb4e3a 100644 --- a/native/serverconnection/src/audio/js/AudioRecorder.cpp +++ b/native/serverconnection/src/audio/js/AudioRecorder.cpp @@ -3,6 +3,7 @@ #include "AudioRecorder.h" #include "AudioConsumer.h" +#include "./AudioProcessor.h" #include "../AudioInput.h" #include "../../logger.h" @@ -46,6 +47,8 @@ NAN_MODULE_INIT(AudioRecorderWrapper::Init) { Nan::SetPrototypeMethod(klass, "create_consumer", AudioRecorderWrapper::_create_consumer); Nan::SetPrototypeMethod(klass, "delete_consumer", AudioRecorderWrapper::_delete_consumer); + Nan::SetPrototypeMethod(klass, "get_audio_processor", AudioRecorderWrapper::get_audio_processor); + constructor().Reset(Nan::GetFunction(klass).ToLocalChecked()); } @@ -93,7 +96,7 @@ std::shared_ptr AudioRecorderWrapper::create_consumer() { } void AudioRecorderWrapper::delete_consumer(const AudioConsumerWrapper* consumer) { - shared_ptr handle; /* need to keep the handle 'till everything has been finished */ + std::shared_ptr handle; /* need to keep the handle 'till everything has been finished */ { lock_guard lock(this->consumer_mutex); for(auto& c : this->consumer_) { @@ -102,20 +105,22 @@ void AudioRecorderWrapper::delete_consumer(const AudioConsumerWrapper* consumer) break; } } - if(!handle) - return; + + if(!handle) { + return; + } { auto it = find(this->consumer_.begin(), this->consumer_.end(), handle); - if(it != this->consumer_.end()) - this->consumer_.erase(it); + if(it != this->consumer_.end()) { + this->consumer_.erase(it); + } } } { lock_guard lock(handle->execute_mutex); /* if we delete the consumer while executing strange stuff could happen */ handle->unbind(); - this->input_->delete_consumer(handle->_handle); } } @@ -271,4 +276,18 @@ NAN_METHOD(AudioRecorderWrapper::_set_volume) { NAN_METHOD(AudioRecorderWrapper::_get_volume) { auto handle = ObjectWrap::Unwrap(info.Holder()); info.GetReturnValue().Set(handle->input_->volume()); +} + +NAN_METHOD(AudioRecorderWrapper::get_audio_processor) { + auto handle = ObjectWrap::Unwrap(info.Holder()); + + auto processor = handle->input_->audio_processor(); + if(!processor) { + return; + } + + auto js_object = Nan::NewInstance(Nan::New(AudioProcessorWrapper::constructor()), 0, nullptr).ToLocalChecked(); + auto wrapper = new AudioProcessorWrapper(processor); + wrapper->wrap(js_object); + info.GetReturnValue().Set(js_object); } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioRecorder.h b/native/serverconnection/src/audio/js/AudioRecorder.h index 25e7844..c6c2bbf 100644 --- a/native/serverconnection/src/audio/js/AudioRecorder.h +++ b/native/serverconnection/src/audio/js/AudioRecorder.h @@ -23,7 +23,7 @@ namespace tc::audio { return my_constructor; } - explicit AudioRecorderWrapper(std::shared_ptr /* input */); + explicit AudioRecorderWrapper(std::shared_ptr /* input */); ~AudioRecorderWrapper() override; static NAN_METHOD(_get_device); @@ -40,6 +40,8 @@ namespace tc::audio { static NAN_METHOD(_set_volume); static NAN_METHOD(_get_volume); + static NAN_METHOD(get_audio_processor); + std::shared_ptr create_consumer(); void delete_consumer(const AudioConsumerWrapper*); diff --git a/native/serverconnection/src/audio/processing/AudioProcessor.cpp b/native/serverconnection/src/audio/processing/AudioProcessor.cpp index 7aa2244..88d57b1 100644 --- a/native/serverconnection/src/audio/processing/AudioProcessor.cpp +++ b/native/serverconnection/src/audio/processing/AudioProcessor.cpp @@ -2,10 +2,234 @@ // Created by WolverinDEV on 27/03/2021. // -#include "AudioProcessor.h" +#include "./AudioProcessor.h" +#include "../../logger.h" +#include +#include using namespace tc::audio; -void AudioProcessor::analyze_reverse_stream(const float *const *data, const webrtc::StreamConfig &reverse_config) { +AudioProcessor::AudioProcessor() { + this->current_config.echo_canceller.enabled = true; + this->current_config.echo_canceller.mobile_mode = false; + this->current_config.gain_controller1.enabled = true; + this->current_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + this->current_config.gain_controller1.analog_level_minimum = 0; + this->current_config.gain_controller1.analog_level_maximum = 255; + + this->current_config.gain_controller2.enabled = true; + + this->current_config.high_pass_filter.enabled = true; + + this->current_config.voice_detection.enabled = true; +} + +AudioProcessor::~AudioProcessor() { + std::lock_guard processor_lock{this->processor_mutex}; + delete this->processor; + + for(auto& entry : this->rnnoise_processor) { + if(!entry) { continue; } + + rnnoise_destroy((DenoiseState*) entry); + } +} + +constexpr static inline auto process_error_to_string(int error) { + switch (error) { + case 0: return "kNoError"; + case -1: return "kUnspecifiedError"; + case -2: return "kCreationFailedError"; + case -3: return "kUnsupportedComponentError"; + case -4: return "kUnsupportedFunctionError"; + case -5: return "kNullPointerError"; + case -6: return "kBadParameterError"; + case -7: return "kBadSampleRateError"; + case -8: return "kBadDataLengthError"; + case -9: return "kBadNumberChannelsError"; + case -10: return "kFileError"; + case -11: return "kStreamParameterNotSetError"; + case -12: return "kNotEnabledError"; + case -13: return "kBadStreamParameterWarning"; + default: return "unkown error code"; + } +} + +template +inline std::ostream& operator<<(std::ostream &ss, const absl::optional& optional) { + if(optional.has_value()) { + ss << "optional{" << *optional << "}"; + } else { + ss << "nullopt"; + } + return ss; +} + +inline std::string statistics_to_string(const webrtc::AudioProcessingStats& stats) { + std::stringstream ss{}; + + ss << "AudioProcessingStats{"; + ss << "output_rms_dbfs: " << stats.output_rms_dbfs << ", "; + ss << "voice_detected: " << stats.voice_detected << ", "; + ss << "echo_return_loss: " << stats.echo_return_loss << ", "; + ss << "echo_return_loss_enhancement: " << stats.echo_return_loss_enhancement << ", "; + ss << "divergent_filter_fraction: " << stats.divergent_filter_fraction << ", "; + ss << "delay_median_ms: " << stats.delay_median_ms << ", "; + ss << "delay_standard_deviation_ms: " << stats.delay_standard_deviation_ms << ", "; + ss << "residual_echo_likelihood: " << stats.residual_echo_likelihood << ", "; + ss << "residual_echo_likelihood_recent_max: " << stats.residual_echo_likelihood_recent_max << ", "; + ss << "delay_ms: " << stats.delay_ms; + ss << "}"; + + return ss.str(); +} + +bool AudioProcessor::initialize() { + std::lock_guard processor_lock{this->processor_mutex}; + if(this->processor) { + /* double initialize */ + return false; + } + + using namespace webrtc; + + AudioProcessingBuilder builder{}; + this->processor = builder.Create(); + if(!this->processor) { + return false; + } + + this->apply_config_unlocked(this->current_config); + this->processor->Initialize(); + + return true; +} + +AudioProcessor::Config AudioProcessor::get_config() const { + std::shared_lock processor_lock{this->processor_mutex}; + return this->current_config; +} + +void AudioProcessor::apply_config(const AudioProcessor::Config &config) { + std::lock_guard processor_lock{this->processor_mutex}; + this->apply_config_unlocked(config); +} + +void AudioProcessor::apply_config_unlocked(const Config &config) { + this->current_config = config; + if(this->processor) { + this->processor->ApplyConfig(config); + } + + if(!this->current_config.rnnoise.enabled) { + this->rnnoise_volume = absl::nullopt; + } +} + +AudioProcessor::Stats AudioProcessor::get_statistics() const { + std::shared_lock processor_lock{this->processor_mutex}; + return this->get_statistics_unlocked(); +} + +AudioProcessor::Stats AudioProcessor::get_statistics_unlocked() const { + if(!this->processor) { + return AudioProcessor::Stats{}; + } + + AudioProcessor::Stats result{this->processor->GetStatistics()}; + result.rnnoise_volume = this->rnnoise_volume; + return result; +} + +void AudioProcessor::register_process_observer(ProcessObserver *observer) { + std::lock_guard processor_lock{this->processor_mutex}; + this->process_observer.push_back(observer); +} + +bool AudioProcessor::unregister_process_observer(ProcessObserver *observer) { + std::lock_guard processor_lock{this->processor_mutex}; + auto index = std::find(this->process_observer.begin(), this->process_observer.end(), observer); + if(index == this->process_observer.end()) { + return false; + } + + this->process_observer.erase(index); + return true; +} + +void AudioProcessor::analyze_reverse_stream(const float *const *data, const webrtc::StreamConfig &reverse_config) { + std::shared_lock processor_lock{this->processor_mutex}; + if(!this->processor) { + return; + } + + auto result = this->processor->AnalyzeReverseStream(data, reverse_config); + if(result != webrtc::AudioProcessing::kNoError) { + log_error(category::audio, tr("Failed to process reverse stream: {}"), process_error_to_string(result)); + return; + } +} + +std::optional AudioProcessor::process_stream(const webrtc::StreamConfig &config, float *const *buffer) { + + std::shared_lock processor_lock{this->processor_mutex}; + if(!this->processor) { + return std::nullopt; + } else if(config.num_channels() > kMaxChannelCount) { + log_error(category::audio, tr("AudioProcessor received input buffer with too many channels ({} channels but supported is only {})"), config.num_channels(), kMaxChannelCount); + return std::nullopt; + } + + if(this->current_config.rnnoise.enabled) { + if(config.sample_rate_hz() != 48000) { + log_warn(category::audio, tr("Don't apply RNNoise. Source sample rate isn't 480kHz ({}kHz)"), config.sample_rate_hz() / 1000); + this->rnnoise_volume.reset(); + } else { + static const float kRnNoiseScale = -INT16_MIN; + + double volume_sum{0}; + for(size_t channel{0}; channel < config.num_channels(); channel++) { + if(!this->rnnoise_processor[channel]) { + this->rnnoise_processor[channel] = (void*) rnnoise_create(nullptr); + } + + { + /* RNNoise uses a frame size of 10ms for 48kHz aka 480 samples */ + auto buffer_ptr = buffer[channel]; + for(size_t sample{0}; sample < 480; sample++) { + *buffer_ptr++ *= kRnNoiseScale; + } + } + + volume_sum += rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[channel], buffer[channel], buffer[channel]); + + { + auto buffer_ptr = buffer[channel]; + for(size_t sample{0}; sample < 480; sample++) { + *buffer_ptr++ /= kRnNoiseScale; + } + } + } + + this->rnnoise_volume = absl::make_optional(volume_sum / config.num_channels()); + } + } + + this->processor->set_stream_delay_ms(2); /* TODO: Measure it and not just guess it! */ + this->processor->set_stream_analog_level(0); + + auto result = this->processor->ProcessStream(buffer, config, config, buffer); + if(result != webrtc::AudioProcessing::kNoError) { + log_error(category::audio, tr("Failed to process stream: {}"), process_error_to_string(result)); + return std::nullopt; + } + + auto statistics = this->get_statistics_unlocked(); + for(const auto& observer : this->process_observer) { + observer->stream_processed(statistics); + } + + //log_trace(category::audio, tr("Processing stats: {}"), statistics_to_string(statistics)); + return std::make_optional(std::move(statistics)); } \ No newline at end of file diff --git a/native/serverconnection/src/audio/processing/AudioProcessor.h b/native/serverconnection/src/audio/processing/AudioProcessor.h index 52fcb7e..d0dcb5d 100644 --- a/native/serverconnection/src/audio/processing/AudioProcessor.h +++ b/native/serverconnection/src/audio/processing/AudioProcessor.h @@ -1,11 +1,50 @@ #pragma once #include +#include +#include #include namespace tc::audio { - class AudioProcessor : public std::enable_shared_from_this { + class AudioProcessor { public: + struct Config : public webrtc::AudioProcessing::Config { + struct { + bool enabled{false}; + } rnnoise; + }; + + struct Stats : public webrtc::AudioProcessingStats { + // The RNNoise returned sample volume + absl::optional rnnoise_volume; + }; + + struct ProcessObserver { + public: + virtual void stream_processed(const AudioProcessor::Stats&) = 0; + }; + + AudioProcessor(); + virtual ~AudioProcessor(); + + [[nodiscard]] bool initialize(); + + [[nodiscard]] Config get_config() const; + void apply_config(const Config &/* config */); + + [[nodiscard]] AudioProcessor::Stats get_statistics() const; + + void register_process_observer(ProcessObserver* /* observer */); + /** + * Unregister a process observer. + * Note: Never call this within the observer callback! + * This will cause a deadlock. + * @return + */ + bool unregister_process_observer(ProcessObserver* /* observer */); + + /* 10ms audio chunk */ + [[nodiscard]] std::optional process_stream( const webrtc::StreamConfig& /* config */, float* const* /* buffer */); /** * Accepts deinterleaved float audio with the range [-1, 1]. Each element @@ -14,6 +53,20 @@ namespace tc::audio { */ void analyze_reverse_stream(const float* const* data, const webrtc::StreamConfig& reverse_config); + private: + constexpr static auto kMaxChannelCount{2}; + + mutable std::shared_mutex processor_mutex{}; + + Config current_config{}; + std::vector process_observer{}; + webrtc::AudioProcessing* processor{nullptr}; + + absl::optional rnnoise_volume{}; + std::array rnnoise_processor{nullptr}; + + [[nodiscard]] AudioProcessor::Stats get_statistics_unlocked() const; + void apply_config_unlocked(const Config &/* config */); }; } \ No newline at end of file diff --git a/native/serverconnection/src/audio/sounds/SoundPlayer.cpp b/native/serverconnection/src/audio/sounds/SoundPlayer.cpp index a22f8f2..76f514d 100644 --- a/native/serverconnection/src/audio/sounds/SoundPlayer.cpp +++ b/native/serverconnection/src/audio/sounds/SoundPlayer.cpp @@ -3,8 +3,6 @@ // #include -#include -#include #include "./SoundPlayer.h" #include "../AudioOutput.h" #include "../file/wav.h" @@ -15,6 +13,11 @@ #include "../AudioMerger.h" #include "../AudioGain.h" +#ifdef NODEJS_API +#include +#include +#endif + #ifdef max #undef max #endif @@ -275,6 +278,7 @@ namespace tc::audio::sounds { } } +#ifdef NODEJS_API NAN_METHOD(tc::audio::sounds::playback_sound_js) { if(info.Length() != 1 || !info[0]->IsObject()) { Nan::ThrowError("invalid arguments"); @@ -330,4 +334,5 @@ NAN_METHOD(tc::audio::sounds::cancel_playback_js) { } cancel_playback((sound_playback_id) info[0].As()->Value()); -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/native/serverconnection/src/audio/sounds/SoundPlayer.h b/native/serverconnection/src/audio/sounds/SoundPlayer.h index 41de329..add713a 100644 --- a/native/serverconnection/src/audio/sounds/SoundPlayer.h +++ b/native/serverconnection/src/audio/sounds/SoundPlayer.h @@ -2,7 +2,10 @@ #include #include + +#ifdef NODEJS_API #include +#endif namespace tc::audio::sounds { typedef uintptr_t sound_playback_id; @@ -26,6 +29,8 @@ namespace tc::audio::sounds { extern sound_playback_id playback_sound(const PlaybackSettings& /* settings */); extern void cancel_playback(const sound_playback_id&); +#ifdef NODEJS_API extern NAN_METHOD(playback_sound_js); extern NAN_METHOD(cancel_playback_js); +#endif } \ No newline at end of file diff --git a/native/serverconnection/src/bindings.cpp b/native/serverconnection/src/bindings.cpp index f513edf..4dee19d 100644 --- a/native/serverconnection/src/bindings.cpp +++ b/native/serverconnection/src/bindings.cpp @@ -18,12 +18,10 @@ #include "audio/js/AudioRecorder.h" #include "audio/js/AudioConsumer.h" #include "audio/js/AudioFilter.h" +#include "audio/js/AudioProcessor.h" #include "audio/AudioEventLoop.h" #include "audio/sounds/SoundPlayer.h" -//#include -#include - #ifndef WIN32 #include #endif @@ -40,14 +38,6 @@ using namespace tc; using namespace tc::connection; using namespace tc::ft; -void processor() { - webrtc::AudioProcessingBuilder builder{}; - webrtc::AudioProcessing::Config config{}; - - auto processor = builder.Create(); - //processor->AnalyzeReverseStream() -} - void testTomMath(){ mp_int x{}; mp_init(&x); @@ -196,6 +186,7 @@ NAN_MODULE_INIT(init) { audio::recorder::AudioRecorderWrapper::Init(namespace_record); audio::recorder::AudioConsumerWrapper::Init(namespace_record); audio::recorder::AudioFilterWrapper::Init(namespace_record); + audio::AudioProcessorWrapper::Init(namespace_record); { auto enum_object = Nan::New(); diff --git a/native/serverconnection/test/audio/main.cpp b/native/serverconnection/test/audio/main.cpp index b8afae0..e83cff1 100644 --- a/native/serverconnection/test/audio/main.cpp +++ b/native/serverconnection/test/audio/main.cpp @@ -6,56 +6,55 @@ #include "../../src/audio/AudioOutput.h" #include "../../src/audio/AudioInput.h" +#include "../../src/audio/AudioEventLoop.h" #include "../../src/audio/filter/FilterVad.h" #include "../../src/audio/filter/FilterThreshold.h" #include "../../src/logger.h" -#ifdef WIN32 - #include -#endif - using namespace std; using namespace tc; +tc::audio::AudioOutput* global_audio_output{nullptr}; int main() { std::string error{}; - Pa_Initialize(); - logger::initialize_raw(); + tc::audio::init_event_loops(); tc::audio::initialize(); tc::audio::await_initialized(); std::shared_ptr default_playback{nullptr}, default_record{nullptr}; for(auto& device : tc::audio::devices()) { - if(device->is_output_default()) + if(device->is_output_default()) { default_playback = device; - if(device->is_input_default()) + } + + if(device->is_input_default()) { default_record = device; + } } assert(default_record); assert(default_playback); for(auto& dev : tc::audio::devices()) { - if(!dev->is_input_supported()) continue; + if(!dev->is_input_supported()) { + continue; + } auto playback_manager = std::make_unique(2, 48000); - if(!playback_manager->set_device(error, default_playback)) { - cerr << "Failed to open output device (" << error << ")" << endl; - return 1; - } - if(!playback_manager->playback()) { - cerr << "failed to start playback" << endl; + global_audio_output = &*playback_manager; + + playback_manager->set_device(default_playback); + if(!playback_manager->playback(error)) { + cerr << "failed to start playback: " << error << endl; return 1; } auto input = std::make_unique(2, 48000); - if(!input->set_device(error, dev)) { - cerr << "Failed to open input device (" << error << "): " << dev->id() << " (" << dev->name() << ")" << endl; - continue; - } - if(!input->record()) { - cerr << "failed to start record for " << dev->id() << " (" << dev->name() << ")" << endl; + input->set_device(default_record); + + if(!input->record(error)) { + cerr << "failed to start record for " << dev->id() << " (" << dev->name() << "): " << error << endl; continue; } @@ -94,6 +93,7 @@ int main() { } playback_manager.release(); //FIXME: Memory leak! + break; } this_thread::sleep_for(chrono::seconds(360)); @@ -103,6 +103,5 @@ int main() { this_thread::sleep_for(chrono::seconds(1000)); } */ - Pa_Terminate(); return 1; } \ No newline at end of file diff --git a/native/serverconnection/test/js/audio.ts b/native/serverconnection/test/js/audio.ts index 8f9b0b0..40969e1 100644 --- a/native/serverconnection/test/js/audio.ts +++ b/native/serverconnection/test/js/audio.ts @@ -1,14 +1,96 @@ -/// -console.log("Starting app"); -module.paths.push("../../build/linux_x64"); -module.paths.push("../../build/win32_x64"); +import * as path from "path"; +module.paths.push(path.join(__dirname, "..", "..", "..", "build", "win32_x64")); +module.paths.push(path.join(__dirname, "..", "..", "..", "build", "linux_x64")); -const original_require = require; -require = (module => original_require(__dirname + "/../../../build/win32_x64/" + module + ".node")) as any; -import * as handle from "teaclient_connection"; -require = original_require; +import {audio} from "teaclient_connection.node"; +import record = audio.record; +import playback = audio.playback; -console.dir(handle.audio); +function printDevices() { + console.info("Available input devices:"); + for(const device of audio.available_devices()) { + if(!device.input_supported) { + continue; + } + + console.info(" - " + device.driver + " - " + device.device_id + " (" + device.name + ")" + (device.input_default ? " (default)" : "")); + } + + console.info("Available output devices:"); + for(const device of audio.available_devices()) { + if(!device.output_supported) { + continue; + } + + console.info(" - " + device.driver + " - " + device.device_id + " (" + device.name + ")" + (device.output_default ? " (default)" : "")); + } +} + +async function main() { + await new Promise(resolve => audio.initialize(resolve)); + + console.info("Audio initialized"); + //printDevices(); + + const recorder = record.create_recorder(); + await new Promise((resolve, reject) => { + const defaultInput = audio.available_devices().find(device => device.input_default); + if(!defaultInput) { + reject("missing default input device"); + return; + } + + recorder.set_device(defaultInput.device_id, result => { + if(result === "success") { + resolve(); + } else { + reject(result); + } + }); + }); + + await new Promise((resolve, reject) => { + recorder.start(result => { + if(typeof result === "boolean" && result) { + resolve(); + } else { + reject(result); + } + }); + }); + + const output = playback.create_stream(); + const recorderConsumer = recorder.create_consumer(); + + if(output.channels !== recorderConsumer.channelCount) { + throw "miss matching channel count"; + } + + if(output.sample_rate !== recorderConsumer.sampleRate) { + throw "miss matching sample rate"; + } + + recorderConsumer.callback_data = buffer => output.write_data(buffer.buffer, true); + + setInterval(() => { + const processor = recorder.get_audio_processor(); + if(!processor) { return; } + + console.error("Config:\n%o", processor.get_config()); + console.error("Statistics:\n%o", processor.get_statistics()); + processor.apply_config({ + "echo_canceller.enabled": false, + "rnnoise.enabled": true + }); + }, 2500); +} + +main().catch(error => { + console.error(error); + process.exit(1); +}); + +/* handle.audio.initialize(() => { console.log("Audio initialized"); @@ -60,5 +142,6 @@ handle.audio.initialize(() => { }); }); }, 1000); - */ + }); +*/ \ No newline at end of file diff --git a/native/serverconnection/test/js/tsconfig.json b/native/serverconnection/test/js/tsconfig.json new file mode 100644 index 0000000..80bbb59 --- /dev/null +++ b/native/serverconnection/test/js/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "module": "CommonJS", + "noImplicitAny": true, + "removeComments": true, + "preserveConstEnums": true, + "sourceMap": true, + "baseUrl": ".", + "paths": { + "teaclient_connection.node": ["../../exports/exports.d.ts"] + }, + "lib": [ + "dom", + "es6", + "scripthost", + ] + } +} \ No newline at end of file