TeaSpeak/TeaSpeak-Client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implementing an audio processor which now also takes care of the RNNoise part

Browse Source
This commit is contained in:
WolverinDEV 2021-03-28 18:37:36 +02:00
parent 1aab9e630a
commit 22d9059ad5
25 changed files with 1614 additions and 636 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
native/serverconnection/CMakeLists.txt
View File

@ -52,7 +52,7 @@ set(NODEJS_SOURCE_FILES
src/audio/js/AudioPlayer.cpp
src/audio/js/AudioOutputStream.cpp
src/audio/js/AudioProcessor.cpp
src/audio/js/AudioRecorder.cpp
src/audio/js/AudioConsumer.cpp
src/audio/js/AudioFilter.cpp
@ -88,7 +88,7 @@ endif()
add_nodejs_module(${MODULE_NAME} ${SOURCE_FILES} ${NODEJS_SOURCE_FILES})
target_link_libraries(${MODULE_NAME} ${NODEJS_LIBRARIES})
#target_compile_options(${MODULE_NAME} PUBLIC "-fPIC")
target_compile_definitions(${MODULE_NAME} PRIVATE "NOMINMAX")
find_package(PortAudio REQUIRED)
include_directories(${PortAudio_INCLUDE_DIR})
@ -173,6 +173,7 @@ else()
)
endif()
add_definitions(-DNO_OPEN_SSL)
target_link_libraries(${MODULE_NAME} ${REQUIRED_LIBRARIES})
target_compile_definitions(${MODULE_NAME} PUBLIC -DNODEJS_API)

588
native/serverconnection/exports/exports.d.ts vendored
View File

@ -1,291 +1,367 @@
declare module "tc-native/connection" {
export enum ServerType {
UNKNOWN,
TEASPEAK,
TEAMSPEAK
export enum ServerType {
UNKNOWN,
TEASPEAK,
TEAMSPEAK
}
export enum PlayerState {
BUFFERING,
PLAYING,
STOPPING,
STOPPED
}
export interface NativeVoiceClient {
client_id: number;
callback_playback: () => any;
callback_stopped: () => any;
callback_state_changed: (new_state: PlayerState) => any;
get_state() : PlayerState;
get_volume() : number;
set_volume(volume: number) : void;
abort_replay();
get_stream() : audio.playback.AudioOutputStream;
}
export interface NativeVoiceConnection {
register_client(client_id: number) : NativeVoiceClient;
available_clients() : NativeVoiceClient[];
unregister_client(client_id: number);
audio_source() : audio.record.AudioConsumer;
set_audio_source(consumer: audio.record.AudioConsumer | undefined);
decoding_supported(codec: number) : boolean;
encoding_supported(codec: number) : boolean;
get_encoder_codec() : number;
set_encoder_codec(codec: number);
/* could may throw an exception when the underlying voice sender has been deallocated */
enable_voice_send(flag: boolean);
}
export interface NativeServerConnection {
callback_voice_data: (buffer: Uint8Array, client_id: number, codec_id: number, flag_head: boolean, packet_id: number) => any;
callback_command: (command: string, arguments: any, switches: string[]) => any;
callback_disconnect: (reason?: string) => any;
_voice_connection: NativeVoiceConnection;
server_type: ServerType;
connected(): boolean;
connect(properties: {
remote_host: string,
remote_port: number,
timeout: number,
callback: (error: number) => any,
identity_key: string | undefined, /* if identity_key empty, then an ephemeral license will be created */
teamspeak: boolean
});
disconnect(reason: string | undefined, callback: (error: number) => any) : boolean;
error_message(id: number) : string;
send_command(command: string, arguments: any[], switches: string[]);
send_voice_data(buffer: Uint8Array, codec_id: number, header: boolean);
send_voice_data_raw(buffer: Float32Array, channels: number, sample_rate: number, header: boolean);
/* ping in microseconds */
current_ping() : number;
statistics() : { voice_bytes_received: number, voice_bytes_send: number, control_bytes_received: number, control_bytes_send } | undefined
}
export function spawn_server_connection() : NativeServerConnection;
export function destroy_server_connection(connection: NativeServerConnection);
export namespace ft {
export interface TransferObject {
name: string;
direction: "upload" | "download";
}
export enum PlayerState {
BUFFERING,
PLAYING,
STOPPING,
STOPPED
export interface FileTransferSource extends TransferObject {
total_size: number;
}
export interface NativeVoiceClient {
client_id: number;
callback_playback: () => any;
callback_stopped: () => any;
callback_state_changed: (new_state: PlayerState) => any;
get_state() : PlayerState;
get_volume() : number;
set_volume(volume: number) : void;
abort_replay();
get_stream() : audio.playback.AudioOutputStream;
export interface FileTransferTarget extends TransferObject {
expected_size: number;
}
export interface NativeVoiceConnection {
register_client(client_id: number) : NativeVoiceClient;
available_clients() : NativeVoiceClient[];
unregister_client(client_id: number);
export interface NativeFileTransfer {
handle: TransferObject;
audio_source() : audio.record.AudioConsumer;
set_audio_source(consumer: audio.record.AudioConsumer | undefined);
callback_finished: (aborted?: boolean) => any;
callback_start: () => any;
callback_progress: (current: number, max: number) => any;
callback_failed: (message: string) => any;
decoding_supported(codec: number) : boolean;
encoding_supported(codec: number) : boolean;
get_encoder_codec() : number;
set_encoder_codec(codec: number);
/* could may throw an exception when the underlying voice sender has been deallocated */
enable_voice_send(flag: boolean);
/**
* @return true if the connect method has been executed successfully
* false if the connect fails, callback_failed will be called with the exact reason
*/
start() : boolean;
}
export interface NativeServerConnection {
callback_voice_data: (buffer: Uint8Array, client_id: number, codec_id: number, flag_head: boolean, packet_id: number) => any;
callback_command: (command: string, arguments: any, switches: string[]) => any;
callback_disconnect: (reason?: string) => any;
_voice_connection: NativeVoiceConnection;
server_type: ServerType;
export interface TransferOptions {
remote_address: string;
remote_port: number;
connected(): boolean;
transfer_key: string;
client_transfer_id: number;
server_transfer_id: number;
connect(properties: {
remote_host: string,
remote_port: number,
timeout: number,
callback: (error: number) => any,
identity_key: string | undefined, /* if identity_key empty, then an ephemeral license will be created */
teamspeak: boolean
});
disconnect(reason: string | undefined, callback: (error: number) => any) : boolean;
error_message(id: number) : string;
send_command(command: string, arguments: any[], switches: string[]);
send_voice_data(buffer: Uint8Array, codec_id: number, header: boolean);
send_voice_data_raw(buffer: Float32Array, channels: number, sample_rate: number, header: boolean);
/* ping in microseconds */
current_ping() : number;
statistics() : { voice_bytes_received: number, voice_bytes_send: number, control_bytes_received: number, control_bytes_send } | undefined
object: TransferObject;
}
export function spawn_server_connection() : NativeServerConnection;
export function destroy_server_connection(connection: NativeServerConnection);
export function upload_transfer_object_from_file(path: string, name: string) : FileTransferSource;
export function upload_transfer_object_from_buffer(buffer: ArrayBuffer) : FileTransferSource;
export namespace ft {
export interface TransferObject {
name: string;
direction: "upload" | "download";
}
export function download_transfer_object_from_buffer(target_buffer: ArrayBuffer) : FileTransferTarget;
export function download_transfer_object_from_file(path: string, name: string, expectedSize: number) : FileTransferTarget;
export interface FileTransferSource extends TransferObject {
total_size: number;
}
export function destroy_connection(connection: NativeFileTransfer);
export function spawn_connection(transfer: TransferOptions) : NativeFileTransfer;
}
export interface FileTransferTarget extends TransferObject {
expected_size: number;
}
export namespace audio {
export interface AudioDevice {
name: string;
driver: string;
export interface NativeFileTransfer {
handle: TransferObject;
device_id: string;
callback_finished: (aborted?: boolean) => any;
callback_start: () => any;
callback_progress: (current: number, max: number) => any;
callback_failed: (message: string) => any;
input_supported: boolean;
output_supported: boolean;
/**
* @return true if the connect method has been executed successfully
* false if the connect fails, callback_failed will be called with the exact reason
*/
start() : boolean;
}
export interface TransferOptions {
remote_address: string;
remote_port: number;
transfer_key: string;
client_transfer_id: number;
server_transfer_id: number;
object: TransferObject;
}
export function upload_transfer_object_from_file(path: string, name: string) : FileTransferSource;
export function upload_transfer_object_from_buffer(buffer: ArrayBuffer) : FileTransferSource;
export function download_transfer_object_from_buffer(target_buffer: ArrayBuffer) : FileTransferTarget;
export function download_transfer_object_from_file(path: string, name: string, expectedSize: number) : FileTransferTarget;
export function destroy_connection(connection: NativeFileTransfer);
export function spawn_connection(transfer: TransferOptions) : NativeFileTransfer;
input_default: boolean;
output_default: boolean;
}
export namespace audio {
export interface AudioDevice {
name: string;
driver: string;
export namespace playback {
export interface AudioOutputStream {
sample_rate: number;
channels: number;
device_id: string;
get_buffer_latency() : number;
set_buffer_latency(value: number);
input_supported: boolean;
output_supported: boolean;
get_buffer_max_latency() : number;
set_buffer_max_latency(value: number);
input_default: boolean;
output_default: boolean;
flush_buffer();
}
export namespace playback {
export interface AudioOutputStream {
sample_rate: number;
channels: number;
export interface OwnedAudioOutputStream extends AudioOutputStream {
callback_underflow: () => any;
callback_overflow: () => any;
get_buffer_latency() : number;
set_buffer_latency(value: number);
clear();
write_data(buffer: ArrayBuffer, interleaved: boolean);
write_data_rated(buffer: ArrayBuffer, interleaved: boolean, sample_rate: number);
get_buffer_max_latency() : number;
set_buffer_max_latency(value: number);
flush_buffer();
}
export interface OwnedAudioOutputStream extends AudioOutputStream {
callback_underflow: () => any;
callback_overflow: () => any;
clear();
write_data(buffer: ArrayBuffer, interleaved: boolean);
write_data_rated(buffer: ArrayBuffer, interleaved: boolean, sample_rate: number);
deleted() : boolean;
delete();
}
export function set_device(device_id: string);
export function current_device() : string;
export function create_stream() : OwnedAudioOutputStream;
export function get_master_volume() : number;
export function set_master_volume(volume: number);
deleted() : boolean;
delete();
}
export namespace record {
enum FilterMode {
Bypass,
Filter,
Block
}
export function set_device(device_id: string);
export function current_device() : string;
export interface ConsumeFilter {
get_name() : string;
}
export function create_stream() : OwnedAudioOutputStream;
export interface MarginedFilter {
/* in seconds */
get_margin_time() : number;
set_margin_time(value: number);
}
export interface VADConsumeFilter extends ConsumeFilter, MarginedFilter {
get_level() : number;
}
export interface ThresholdConsumeFilter extends ConsumeFilter, MarginedFilter {
get_threshold() : number;
set_threshold(value: number);
get_attack_smooth() : number;
set_attack_smooth(value: number);
get_release_smooth() : number;
set_release_smooth(value: number);
set_analyze_filter(callback: (value: number) => any);
}
export interface StateConsumeFilter extends ConsumeFilter {
is_consuming() : boolean;
set_consuming(flag: boolean);
}
export interface AudioConsumer {
readonly sampleRate: number;
readonly channelCount: number;
readonly frameSize: number;
get_filters() : ConsumeFilter[];
unregister_filter(filter: ConsumeFilter);
create_filter_vad(level: number) : VADConsumeFilter;
create_filter_threshold(threshold: number) : ThresholdConsumeFilter;
create_filter_state() : StateConsumeFilter;
set_filter_mode(mode: FilterMode);
get_filter_mode() : FilterMode;
toggle_rnnoise(enabled: boolean);
rnnoise_enabled() : boolean;
callback_data: (buffer: Float32Array) => any;
callback_ended: () => any;
callback_started: () => any;
}
export type DeviceSetResult = "success" | "invalid-device";
export interface AudioRecorder {
get_device() : string;
set_device(device_id: string, callback: (result: DeviceSetResult) => void); /* Recorder needs to be started afterwards */
start(callback: (result: boolean | string) => void);
started() : boolean;
stop();
get_volume() : number;
set_volume(volume: number);
create_consumer() : AudioConsumer;
get_consumers() : AudioConsumer[];
delete_consumer(consumer: AudioConsumer);
}
export function create_recorder() : AudioRecorder;
}
export namespace sounds {
export enum PlaybackResult {
SUCCEEDED,
CANCELED,
SOUND_NOT_INITIALIZED,
FILE_OPEN_ERROR,
PLAYBACK_ERROR
}
export interface PlaybackSettings {
file: string;
volume?: number;
callback?: (result: PlaybackResult, message: string) => void;
}
export function playback_sound(settings: PlaybackSettings) : number;
export function cancel_playback(playback: number);
}
export function initialize(callback: () => any);
export function initialized() : boolean;
export function available_devices() : AudioDevice[];
export function get_master_volume() : number;
export function set_master_volume(volume: number);
}
export namespace record {
enum FilterMode {
Bypass,
Filter,
Block
}
export interface ConsumeFilter {
get_name() : string;
}
export interface MarginedFilter {
/* in seconds */
get_margin_time() : number;
set_margin_time(value: number);
}
export interface VADConsumeFilter extends ConsumeFilter, MarginedFilter {
get_level() : number;
}
export interface ThresholdConsumeFilter extends ConsumeFilter, MarginedFilter {
get_threshold() : number;
set_threshold(value: number);
get_attack_smooth() : number;
set_attack_smooth(value: number);
get_release_smooth() : number;
set_release_smooth(value: number);
set_analyze_filter(callback: (value: number) => any);
}
export interface StateConsumeFilter extends ConsumeFilter {
is_consuming() : boolean;
set_consuming(flag: boolean);
}
export interface AudioConsumer {
readonly sampleRate: number;
readonly channelCount: number;
readonly frameSize: number;
get_filters() : ConsumeFilter[];
unregister_filter(filter: ConsumeFilter);
create_filter_vad(level: number) : VADConsumeFilter;
create_filter_threshold(threshold: number) : ThresholdConsumeFilter;
create_filter_state() : StateConsumeFilter;
set_filter_mode(mode: FilterMode);
get_filter_mode() : FilterMode;
callback_data: (buffer: Float32Array) => any;
callback_ended: () => any;
callback_started: () => any;
}
export interface AudioProcessorConfig {
"pipeline.maximum_internal_processing_rate": number,
"pipeline.multi_channel_render": boolean,
"pipeline.multi_channel_capture": boolean,
"pre_amplifier.enabled": boolean,
"pre_amplifier.fixed_gain_factor": number,
"high_pass_filter.enabled": boolean,
"high_pass_filter.apply_in_full_band": boolean,
"echo_canceller.enabled": boolean,
"echo_canceller.mobile_mode": boolean,
"echo_canceller.export_linear_aec_output": boolean,
"echo_canceller.enforce_high_pass_filtering": boolean,
"noise_suppression.enabled": boolean,
"noise_suppression.level": "low" | "moderate" | "high" | "very-high",
"noise_suppression.analyze_linear_aec_output_when_available": boolean,
"transient_suppression.enabled": boolean,
"voice_detection.enabled": boolean,
"gain_controller1.enabled": boolean,
"gain_controller1.mode": "adaptive-analog" | "adaptive-digital" | "fixed-digital",
"gain_controller1.target_level_dbfs": number,
"gain_controller1.compression_gain_db": number,
"gain_controller1.enable_limiter": boolean,
"gain_controller1.analog_level_minimum": number,
"gain_controller1.analog_level_maximum": number,
"gain_controller1.analog_gain_controller.enabled": boolean,
"gain_controller1.analog_gain_controller.startup_min_volume": number,
"gain_controller1.analog_gain_controller.clipped_level_min": number,
"gain_controller1.analog_gain_controller.enable_agc2_level_estimator": boolean,
"gain_controller1.analog_gain_controller.enable_digital_adaptive": boolean,
"gain_controller2.enabled": boolean,
"gain_controller2.fixed_digital.gain_db": number,
"gain_controller2.adaptive_digital.enabled": boolean,
"gain_controller2.adaptive_digital.vad_probability_attack": number,
"gain_controller2.adaptive_digital.level_estimator": "rms" | "peak",
"gain_controller2.adaptive_digital.level_estimator_adjacent_speech_frames_threshold": number,
"gain_controller2.adaptive_digital.use_saturation_protector": boolean,
"gain_controller2.adaptive_digital.initial_saturation_margin_db": number,
"gain_controller2.adaptive_digital.extra_saturation_margin_db": number,
"gain_controller2.adaptive_digital.gain_applier_adjacent_speech_frames_threshold": number,
"gain_controller2.adaptive_digital.max_gain_change_db_per_second": number,
"gain_controller2.adaptive_digital.max_output_noise_level_dbfs": number,
"residual_echo_detector.enabled": boolean,
"level_estimation.enabled": boolean,
"rnnoise.enabled": boolean
}
export interface AudioProcessorStatistics {
output_rms_dbfs: number | undefined,
voice_detected: number | undefined,
echo_return_loss: number | undefined,
echo_return_loss_enhancement: number | undefined,
divergent_filter_fraction: number | undefined,
delay_median_ms: number | undefined,
delay_standard_deviation_ms: number | undefined,
residual_echo_likelihood: number | undefined,
residual_echo_likelihood_recent_max: number | undefined,
delay_ms: number | undefined,
rnnoise_volume: number | undefined
}
export interface AudioProcessor {
get_config() : AudioProcessorConfig;
apply_config(config: Partial<AudioProcessorConfig>);
get_statistics() : AudioProcessorStatistics;
}
export type DeviceSetResult = "success" | "invalid-device";
export interface AudioRecorder {
get_device() : string;
set_device(device_id: string, callback: (result: DeviceSetResult) => void); /* Recorder needs to be started afterwards */
start(callback: (result: boolean | string) => void);
started() : boolean;
stop();
get_volume() : number;
set_volume(volume: number);
create_consumer() : AudioConsumer;
get_consumers() : AudioConsumer[];
delete_consumer(consumer: AudioConsumer);
get_audio_processor() : AudioProcessor | undefined;
}
export function create_recorder() : AudioRecorder;
}
export namespace sounds {
export enum PlaybackResult {
SUCCEEDED,
CANCELED,
SOUND_NOT_INITIALIZED,
FILE_OPEN_ERROR,
PLAYBACK_ERROR
}
export interface PlaybackSettings {
file: string;
volume?: number;
callback?: (result: PlaybackResult, message: string) => void;
}
export function playback_sound(settings: PlaybackSettings) : number;
export function cancel_playback(playback: number);
}
export function initialize(callback: () => any);
export function initialized() : boolean;
export function available_devices() : AudioDevice[];
}

112
native/serverconnection/src/EventLoop.h
View File

@ -7,71 +7,69 @@
#include <thread>
#include <condition_variable>
namespace tc {
namespace event {
class EventExecutor;
class EventEntry {
friend class EventExecutor;
public:
virtual void event_execute(const std::chrono::system_clock::time_point& /* scheduled timestamp */) = 0;
virtual void event_execute_dropped(const std::chrono::system_clock::time_point& /* scheduled timestamp */) {}
namespace tc::event {
class EventExecutor;
class EventEntry {
friend class EventExecutor;
public:
virtual void event_execute(const std::chrono::system_clock::time_point& /* scheduled timestamp */) = 0;
virtual void event_execute_dropped(const std::chrono::system_clock::time_point& /* scheduled timestamp */) {}
std::unique_lock<std::timed_mutex> execute_lock(bool force) {
if(force) {
return std::unique_lock<std::timed_mutex>(this->_execute_mutex);
} else {
auto lock = std::unique_lock<std::timed_mutex>(this->_execute_mutex, std::defer_lock);
if(this->execute_lock_timeout.count() > 0) {
(void) lock.try_lock_for(this->execute_lock_timeout);
} else {
(void) lock.try_lock();
}
return lock;
}
}
std::unique_lock<std::timed_mutex> execute_lock(bool force) {
if(force) {
return std::unique_lock<std::timed_mutex>(this->_execute_mutex);
} else {
auto lock = std::unique_lock<std::timed_mutex>(this->_execute_mutex, std::defer_lock);
if(this->execute_lock_timeout.count() > 0) {
(void) lock.try_lock_for(this->execute_lock_timeout);
} else {
(void) lock.try_lock();
}
return lock;
}
}
inline bool single_thread_executed() const { return this->_single_thread; }
inline void single_thread_executed(bool value) { this->_single_thread = value; }
[[nodiscard]] inline bool single_thread_executed() const { return this->_single_thread; }
inline void single_thread_executed(bool value) { this->_single_thread = value; }
protected:
std::chrono::nanoseconds execute_lock_timeout{0};
private:
void* _event_ptr = nullptr;
bool _single_thread = true; /* if its set to true there might are some dropped executes! */
std::timed_mutex _execute_mutex;
};
protected:
std::chrono::nanoseconds execute_lock_timeout{0};
private:
void* _event_ptr = nullptr;
bool _single_thread = true; /* if its set to true there might are some dropped executes! */
std::timed_mutex _execute_mutex;
};
class EventExecutor {
public:
explicit EventExecutor(const std::string& /* thread prefix */);
virtual ~EventExecutor();
class EventExecutor {
public:
explicit EventExecutor(const std::string& /* thread prefix */);
virtual ~EventExecutor();
bool initialize(int /* num threads */);
bool schedule(const std::shared_ptr<EventEntry>& /* entry */);
bool cancel(const std::shared_ptr<EventEntry>& /* entry */); /* Note: Will not cancel already running executes */
void shutdown();
private:
struct LinkedEntry {
LinkedEntry* previous;
LinkedEntry* next;
bool initialize(int /* num threads */);
bool schedule(const std::shared_ptr<EventEntry>& /* entry */);
bool cancel(const std::shared_ptr<EventEntry>& /* entry */); /* Note: Will not cancel already running executes */
void shutdown();
private:
struct LinkedEntry {
LinkedEntry* previous;
LinkedEntry* next;
std::chrono::system_clock::time_point scheduled;
std::weak_ptr<EventEntry> entry;
};
static void _executor(EventExecutor*);
void _shutdown(std::unique_lock<std::mutex>&);
void _reset_events(std::unique_lock<std::mutex>&);
std::chrono::system_clock::time_point scheduled;
std::weak_ptr<EventEntry> entry;
};
static void _executor(EventExecutor*);
void _shutdown(std::unique_lock<std::mutex>&);
void _reset_events(std::unique_lock<std::mutex>&);
bool should_shutdown = true;
bool should_shutdown = true;
std::vector<std::thread> threads;
std::mutex lock;
std::condition_variable condition;
std::vector<std::thread> threads;
std::mutex lock;
std::condition_variable condition;
LinkedEntry* head = nullptr;
LinkedEntry* tail = nullptr;
LinkedEntry* head = nullptr;
LinkedEntry* tail = nullptr;
std::string thread_prefix;
};
}
std::string thread_prefix;
};
}

279
native/serverconnection/src/audio/AudioInput.cpp
View File

@ -1,21 +1,25 @@
#include <cstring>
#include <memory>
#include <string>
#include "./AudioInput.h"
#include "./AudioReframer.h"
#include "./AudioResampler.h"
#include "./AudioMerger.h"
#include "./AudioGain.h"
#include "./AudioInterleaved.h"
#include "./AudioOutput.h"
#include "./processing/AudioProcessor.h"
#include "../logger.h"
#include "AudioGain.h"
#include "AudioEventLoop.h"
using namespace std;
using namespace tc;
using namespace tc::audio;
AudioConsumer::AudioConsumer(tc::audio::AudioInput *handle, size_t channel_count, size_t sample_rate, size_t frame_size) :
handle(handle),
channel_count(channel_count),
sample_rate(sample_rate) ,
frame_size(frame_size) {
AudioConsumer::AudioConsumer(size_t channel_count, size_t sample_rate, size_t frame_size) :
channel_count{channel_count},
sample_rate{sample_rate},
frame_size{frame_size} {
if(this->frame_size > 0) {
this->reframer = std::make_unique<InputReframer>(channel_count, frame_size);
this->reframer->on_frame = [&](const void* buffer) { this->handle_framed_data(buffer, this->frame_size); };
@ -40,15 +44,60 @@ void AudioConsumer::process_data(const void *buffer, size_t samples) {
}
}
AudioInput::AudioInput(size_t channels, size_t rate) : _channel_count(channels), _sample_rate(rate) {}
AudioInput::~AudioInput() {
this->close_device();
extern tc::audio::AudioOutput* global_audio_output;
AudioInput::AudioInput(size_t channels, size_t sample_rate) :
channel_count_{channels},
sample_rate_{sample_rate},
input_buffer{(sample_rate * channels * sizeof(float) * kInputBufferCapacityMs) / 1000}
{
this->event_loop_entry = std::make_shared<EventLoopCallback>(this);
{
std::lock_guard lock(this->consumers_lock);
for(const auto& consumer : this->_consumers)
consumer->handle = nullptr;
this->initialize_hook_handle = std::make_shared<AudioInitializeHook>();
this->initialize_hook_handle->input = this;
std::weak_ptr weak_handle{this->initialize_hook_handle};
audio::initialize([weak_handle] {
auto handle = weak_handle.lock();
if(!handle) {
return;
}
std::lock_guard lock{handle->mutex};
if(!handle->input) {
return;
}
auto processor = std::make_shared<AudioProcessor>();
if(!processor->initialize()) {
log_error(category::audio, tr("Failed to initialize audio processor."));
return;
}
global_audio_output->register_audio_processor(processor);
handle->input->audio_processor_ = processor;
});
}
free(this->resample_buffer);
}
AudioInput::~AudioInput() {
{
std::lock_guard lock{this->initialize_hook_handle->mutex};
this->initialize_hook_handle->input = nullptr;
}
{
audio::encode_event_loop->cancel(this->event_loop_entry);
this->event_loop_entry->execute_lock(true);
}
if(this->audio_processor_) {
assert(global_audio_output);
global_audio_output->unregister_audio_processor(this->audio_processor_);
this->audio_processor_ = nullptr;
}
this->close_device();
}
void AudioInput::set_device(const std::shared_ptr<AudioDevice> &device) {
@ -68,7 +117,7 @@ void AudioInput::close_device() {
this->input_recorder->stop_if_possible();
this->input_recorder.reset();
}
this->_resampler = nullptr;
this->resampler_ = nullptr;
this->input_device = nullptr;
}
@ -90,7 +139,7 @@ bool AudioInput::record(std::string& error) {
}
if(this->input_recorder->sample_rate() != this->sample_rate()) {
this->_resampler = std::make_unique<AudioResampler>(this->input_recorder->sample_rate(), this->sample_rate(), this->_channel_count);
this->resampler_ = std::make_unique<AudioResampler>(this->input_recorder->sample_rate(), this->sample_rate(), this->channel_count_);
}
this->input_recorder->register_consumer(this);
@ -103,7 +152,7 @@ bool AudioInput::record(std::string& error) {
}
bool AudioInput::recording() {
return !!this->input_recorder;
return this->input_recorder != nullptr;
}
void AudioInput::stop() {
@ -114,80 +163,162 @@ void AudioInput::stop() {
this->input_recorder.reset();
}
std::vector<std::shared_ptr<AudioConsumer>> AudioInput::consumers() {
std::vector<std::shared_ptr<AudioConsumer>> result{};
result.reserve(10);
std::lock_guard consumer_lock{this->consumers_mutex};
result.reserve(this->consumers_.size());
this->consumers_.erase(std::remove_if(this->consumers_.begin(), this->consumers_.end(), [&](const std::weak_ptr<AudioConsumer>& weak_consumer) {
auto consumer = weak_consumer.lock();
if(!consumer) {
return true;
}
result.push_back(consumer);
return false;
}), this->consumers_.end());
return result;
}
std::shared_ptr<AudioConsumer> AudioInput::create_consumer(size_t frame_length) {
auto result = std::shared_ptr<AudioConsumer>(new AudioConsumer(this, this->_channel_count, this->_sample_rate, frame_length));
auto result = std::shared_ptr<AudioConsumer>(new AudioConsumer(this->channel_count_, this->sample_rate_, frame_length));
{
std::lock_guard lock(this->consumers_lock);
this->_consumers.push_back(result);
std::lock_guard lock(this->consumers_mutex);
this->consumers_.push_back(result);
}
return result;
}
void AudioInput::delete_consumer(const std::shared_ptr<AudioConsumer> &source) {
{
std::lock_guard lock(this->consumers_lock);
auto it = find(this->_consumers.begin(), this->_consumers.end(), source);
if(it != this->_consumers.end())
this->_consumers.erase(it);
}
source->handle = nullptr;
}
void AudioInput::consume(const void *input, size_t frameCount, size_t channels) {
if(channels != this->_channel_count) {
if(channels < 1 || channels > 2) {
log_critical(category::audio, tr("Channel count miss match (Received: {})!"), channels);
}
this->ensure_resample_buffer_capacity(frameCount * this->_channel_count * sizeof(float));
audio::merge::merge_channels_interleaved(this->resample_buffer, this->_channel_count, input, channels, frameCount);
input = this->resample_buffer;
void AudioInput::allocate_input_buffer_samples(size_t samples) {
const auto expected_byte_size = samples * this->channel_count_ * sizeof(float);
if(expected_byte_size > this->input_buffer.capacity()) {
log_critical(category::audio, tr("Resampled audio input data would be larger than our input buffer capacity."));
return;
}
if(this->_resampler) {
const auto expected_size = this->_resampler->estimated_output_size(frameCount);
const auto expected_byte_size = expected_size * this->_channel_count * sizeof(float);
this->ensure_resample_buffer_capacity(expected_byte_size);
if(this->input_buffer.free_count() < expected_byte_size) {
log_warn(category::audio, tr("Audio input buffer overflow."));
size_t sample_count{expected_size};
if(!this->_resampler->process(this->resample_buffer, input, frameCount, sample_count)) {
const auto free_samples = this->input_buffer.free_count() / this->channel_count_ / sizeof(float);
assert(samples >= free_samples);
const auto missing_samples = samples - free_samples;
this->input_buffer.advance_read_ptr(missing_samples * this->channel_count_ * sizeof(float));
}
}
void AudioInput::consume(const void *input, size_t sample_count, size_t channels) {
constexpr static auto kTempBufferMaxSampleCount{1024 * 8};
float temp_buffer[kTempBufferMaxSampleCount];
/* TODO: Short circuit for silence here */
if(channels != this->channel_count_) {
if(channels < 1 || channels > 2) {
log_critical(category::audio, tr("AudioInput received audio data with an unsupported channel count of {}."), channels);
return;
}
if(sample_count * this->channel_count_ > kTempBufferMaxSampleCount) {
log_critical(category::audio, tr("Received audio chunk bigger than our temp stack buffer. Received {} samples but can hold only {}."), sample_count, kTempBufferMaxSampleCount / this->channel_count_);
return;
}
audio::merge::merge_channels_interleaved(temp_buffer, this->channel_count_, input, channels, sample_count);
input = temp_buffer;
}
if(this->resampler_) {
const auto expected_size = this->resampler_->estimated_output_size(sample_count);
this->allocate_input_buffer_samples(expected_size);
size_t resampled_sample_count{expected_size};
if(!this->resampler_->process(this->input_buffer.write_ptr(), input, sample_count, resampled_sample_count)) {
log_error(category::audio, tr("Failed to resample input audio."));
return;
}
frameCount = sample_count;
input = this->resample_buffer;
this->input_buffer.advance_write_ptr(resampled_sample_count * this->channel_count_ * sizeof(float));
} else {
this->allocate_input_buffer_samples(sample_count);
audio::apply_gain(this->resample_buffer, this->_channel_count, frameCount, this->_volume);
} else if(this->_volume != 1) {
const auto byte_size = frameCount * this->_channel_count * sizeof(float);
this->ensure_resample_buffer_capacity(byte_size);
if(this->resample_buffer != input) {
memcpy(this->resample_buffer, input, byte_size);
input = this->resample_buffer;
}
audio::apply_gain(this->resample_buffer, this->_channel_count, frameCount, this->_volume);
const auto sample_byte_size = sample_count * this->channel_count_ * sizeof(float);
memcpy(this->input_buffer.write_ptr(), input, sample_byte_size);
this->input_buffer.advance_write_ptr(sample_byte_size);
}
auto begin = std::chrono::system_clock::now();
for(const auto& consumer : this->consumers()) {
consumer->process_data(input, frameCount);
}
auto end = std::chrono::system_clock::now();
auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - begin).count();
if(ms > 5) {
log_warn(category::audio, tr("Processing of audio input needed {}ms. This could be an issue!"), std::chrono::duration_cast<chrono::milliseconds>(end - begin).count());
}
audio::encode_event_loop->schedule(this->event_loop_entry);
}
void AudioInput::ensure_resample_buffer_capacity(size_t size) {
if(this->resample_buffer_size < size) {
free(this->resample_buffer);
this->resample_buffer = malloc(size);
this->resample_buffer_size = size;
void AudioInput::process_audio() {
const auto chunk_sample_count = (kChunkSizeMs * this->sample_rate_) / 1000;
while(true) {
auto available_sample_count = this->input_buffer.fill_count() / this->channel_count_ / sizeof(float);
if(available_sample_count < chunk_sample_count) {
break;
}
auto input = this->input_buffer.read_ptr();
/*
* It's save to mutate the current memory.
* If overflows occur it could lead to wired artifacts but all memory access is save.
*/
this->process_audio_chunk((void*) input);
this->input_buffer.advance_read_ptr(chunk_sample_count * this->channel_count_ * sizeof(float));
}
}
}
void AudioInput::process_audio_chunk(void *chunk) {
constexpr static auto kTempSampleBufferSize{1024 * 8};
constexpr static auto kMaxChannelCount{32};
const auto chunk_sample_count = this->chunk_sample_count();
float temp_sample_buffer[kTempSampleBufferSize];
float out_sample_buffer[kTempSampleBufferSize];
assert(memset(temp_sample_buffer, 0, sizeof(float) * kTempSampleBufferSize));
assert(memset(out_sample_buffer, 0, sizeof(float) * kTempSampleBufferSize));
if(auto processor{this->audio_processor_}; processor) {
assert(kTempSampleBufferSize >= chunk_sample_count * this->channel_count_ * sizeof(float));
audio::deinterleave(temp_sample_buffer, (const float*) chunk, this->channel_count_, chunk_sample_count);
webrtc::StreamConfig stream_config{(int) this->sample_rate_, this->channel_count_};
float* channel_ptr[kMaxChannelCount];
assert(memset(channel_ptr, 0, sizeof(float*) * kMaxChannelCount));
for(size_t channel{0}; channel < this->channel_count_; channel++) {
channel_ptr[channel] = temp_sample_buffer + (channel * chunk_sample_count);
}
auto process_statistics = processor->process_stream(stream_config, channel_ptr);
if(!process_statistics.has_value()) {
/* TODO: Some kind of error message? */
return;
}
audio::interleave_vec(out_sample_buffer, channel_ptr, this->channel_count_, chunk_sample_count);
chunk = out_sample_buffer;
}
/* TODO: Is this even needed if we have the processor? */
audio::apply_gain(chunk, this->channel_count_, chunk_sample_count, this->volume_);
auto begin = std::chrono::system_clock::now();
for(const auto& consumer : this->consumers()) {
consumer->process_data(out_sample_buffer, chunk_sample_count);
}
auto end = std::chrono::system_clock::now();
auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - begin).count();
if(ms > 5) {
log_warn(category::audio, tr("Processing of audio input needed {}ms. This could be an issue!"), std::chrono::duration_cast<chrono::milliseconds>(end - begin).count());
}
}
void AudioInput::EventLoopCallback::event_execute(const chrono::system_clock::time_point &point) {
this->input->process_audio();
}

82
native/serverconnection/src/audio/AudioInput.h
View File

@ -5,29 +5,30 @@
#include <memory>
#include <iostream>
#include <functional>
#include "AudioSamples.h"
#include "driver/AudioDriver.h"
#include "./AudioSamples.h"
#include "./driver/AudioDriver.h"
#include "../ring_buffer.h"
#include "../EventLoop.h"
class AudioInputSource;
namespace tc::audio {
class AudioInput;
class InputReframer;
class AudioResampler;
class AudioProcessor;
class AudioInputSource;
class AudioConsumer {
friend class AudioInput;
public:
AudioInput* handle;
size_t const channel_count;
size_t const sample_rate;
size_t const channel_count = 0;
size_t const sample_rate = 0;
size_t const frame_size = 0;
size_t const frame_size;
std::mutex on_read_lock{}; /* locked to access the function */
std::function<void(const void* /* buffer */, size_t /* samples */)> on_read;
private:
AudioConsumer(AudioInput* handle, size_t channel_count, size_t sample_rate, size_t frame_size);
AudioConsumer(size_t channel_count, size_t sample_rate, size_t frame_size);
std::unique_ptr<InputReframer> reframer;
@ -36,7 +37,7 @@ namespace tc::audio {
};
class AudioInput : public AudioDeviceRecord::Consumer {
friend class ::AudioInputSource;
friend class AudioInputSource;
public:
AudioInput(size_t /* channels */, size_t /* sample rate */);
virtual ~AudioInput();
@ -49,38 +50,57 @@ namespace tc::audio {
[[nodiscard]] bool recording();
void stop();
std::deque<std::shared_ptr<AudioConsumer>> consumers() {
std::lock_guard lock(this->consumers_lock);
return this->_consumers;
}
[[nodiscard]] std::vector<std::shared_ptr<AudioConsumer>> consumers();
[[nodiscard]] std::shared_ptr<AudioConsumer> create_consumer(size_t /* frame size */);
std::shared_ptr<AudioConsumer> create_consumer(size_t /* frame size */);
void delete_consumer(const std::shared_ptr<AudioConsumer>& /* source */);
[[nodiscard]] inline auto audio_processor() { return this->audio_processor_; }
[[nodiscard]] inline size_t channel_count() const { return this->_channel_count; }
[[nodiscard]] inline size_t sample_rate() const { return this->_sample_rate; }
[[nodiscard]] inline size_t channel_count() const { return this->channel_count_; }
[[nodiscard]] inline size_t sample_rate() const { return this->sample_rate_; }
[[nodiscard]] inline float volume() const { return this->_volume; }
inline void set_volume(float value) { this->_volume = value; }
[[nodiscard]] inline float volume() const { return this->volume_; }
inline void set_volume(float value) { this->volume_ = value; }
private:
constexpr static auto kInputBufferCapacityMs{750};
constexpr static auto kChunkSizeMs{10}; /* Must be 10ms else the audio processor will fuck up */
struct EventLoopCallback : public event::EventEntry {
AudioInput* const input;
explicit EventLoopCallback(AudioInput* input) : input{input} {}
void event_execute(const std::chrono::system_clock::time_point &point) override;
};
struct AudioInitializeHook {
std::mutex mutex{};
AudioInput* input{nullptr};
};
void consume(const void *, size_t, size_t) override;
void ensure_resample_buffer_capacity(size_t);
void process_audio();
void process_audio_chunk(void *);
size_t const _channel_count;
size_t const _sample_rate;
size_t const channel_count_;
size_t const sample_rate_;
std::mutex consumers_lock;
std::deque<std::shared_ptr<AudioConsumer>> _consumers;
std::recursive_mutex input_source_lock;
std::mutex consumers_mutex{};
std::deque<std::weak_ptr<AudioConsumer>> consumers_{};
std::recursive_mutex input_source_lock{};
std::unique_ptr<AudioResampler> _resampler{nullptr};
std::shared_ptr<EventLoopCallback> event_loop_entry{};
ring_buffer input_buffer;
std::unique_ptr<AudioResampler> resampler_{nullptr};
std::shared_ptr<AudioDevice> input_device{};
void* resample_buffer{nullptr};
size_t resample_buffer_size{0};
float _volume{1.f};
float volume_{1.f};
std::shared_ptr<AudioDeviceRecord> input_recorder{};
std::shared_ptr<AudioProcessor> audio_processor_{};
std::shared_ptr<AudioInitializeHook> initialize_hook_handle{};
void allocate_input_buffer_samples(size_t /* sample count */);
[[nodiscard]] inline auto chunk_sample_count() const { return (kChunkSizeMs * this->sample_rate_) / 1000; }
};
}

10
native/serverconnection/src/audio/AudioInterleaved.cpp
View File

@ -3,6 +3,7 @@
//
#include <cassert>
#include <cstring>
#include "AudioInterleaved.h"
using namespace tc;
@ -33,6 +34,15 @@ void audio::interleave(float *target, const float *source, size_t channel_count,
source_ptr[channel] = source + (channel * sample_count);
}
audio::interleave_vec(target, source_ptr, channel_count, sample_count);
}
void audio::interleave_vec(float *target, const float *const *source, size_t channel_count, size_t sample_count) {
assert(channel_count <= kMaxChannelCount);
const float* source_ptr[kMaxChannelCount];
memcpy(source_ptr, source, channel_count * sizeof(float*));
for(size_t sample{0}; sample < sample_count; sample++) {
for(size_t channel{0}; channel < channel_count; channel++) {
*target++ = *source_ptr[channel]++;

7
native/serverconnection/src/audio/AudioInterleaved.h
View File

@ -14,4 +14,11 @@ namespace tc::audio {
size_t /* channel count */,
size_t /* sample count */
);
extern void interleave_vec(
float* /* dest */,
const float * const * /* sources */,
size_t /* channel count */,
size_t /* sample count */
);
}

25
native/serverconnection/src/audio/AudioReframer.cpp
View File

@ -10,8 +10,9 @@ InputReframer::InputReframer(size_t channels, size_t frame_size) : _frame_size(f
}
InputReframer::~InputReframer() {
if(this->buffer)
free(this->buffer);
if(this->buffer) {
free(this->buffer);
}
}
void InputReframer::process(const void *source, size_t samples) {
@ -23,15 +24,15 @@ void InputReframer::process(const void *source, size_t samples) {
if(this->_buffer_index > 0) {
if(this->_buffer_index + samples > this->_frame_size) {
auto required = this->_frame_size - this->_buffer_index;
auto length = required * this->_channels * 4;
auto length = required * this->_channels * sizeof(float);
memcpy((char*) this->buffer + this->_buffer_index * 4 * this->_channels, source, length);
memcpy((char*) this->buffer + this->_buffer_index * sizeof(float) * this->_channels, source, length);
samples -= required;
source = (char*) source + length;
this->on_frame(this->buffer);
} else {
memcpy((char*) this->buffer + this->_buffer_index * 4 * this->_channels, source, samples * this->_channels * 4);
memcpy((char*) this->buffer + this->_buffer_index * sizeof(float) * this->_channels, source, samples * this->_channels * sizeof(float));
this->_buffer_index += samples;
return;
}
@ -39,12 +40,16 @@ void InputReframer::process(const void *source, size_t samples) {
auto _on_frame = this->on_frame;
while(samples > this->_frame_size) {
if(_on_frame)
_on_frame(source);
if(_on_frame) {
_on_frame(source);
}
samples -= this->_frame_size;
source = (char*) source + this->_frame_size * this->_channels * 4;
source = (char*) source + this->_frame_size * this->_channels * sizeof(float);
}
if(samples > 0)
memcpy((char*) this->buffer, source, samples * this->_channels * 4);
if(samples > 0) {
memcpy((char*) this->buffer, source, samples * this->_channels * sizeof(float));
}
this->_buffer_index = samples;
}

1
native/serverconnection/src/audio/filter/FilterVad.cpp
View File

@ -1,4 +1,3 @@
#include <iostream>
#include "FilterVad.h"
#include "../AudioMerger.h"
#include "../../logger.h"

106
native/serverconnection/src/audio/js/AudioConsumer.cpp
View File

@ -32,9 +32,6 @@ NAN_MODULE_INIT(AudioConsumerWrapper::Init) {
Nan::SetPrototypeMethod(klass, "get_filter_mode", AudioConsumerWrapper::_get_filter_mode);
Nan::SetPrototypeMethod(klass, "set_filter_mode", AudioConsumerWrapper::_set_filter_mode);
Nan::SetPrototypeMethod(klass, "rnnoise_enabled", AudioConsumerWrapper::rnnoise_enabled);
Nan::SetPrototypeMethod(klass, "toggle_rnnoise", AudioConsumerWrapper::toggle_rnnoise);
constructor_template().Reset(klass);
constructor().Reset(Nan::GetFunction(klass).ToLocalChecked());
}
@ -61,17 +58,6 @@ AudioConsumerWrapper::~AudioConsumerWrapper() {
lock_guard lock{this->execute_mutex};
this->unbind();
if(this->_handle->handle) {
this->_handle->handle->delete_consumer(this->_handle);
this->_handle = nullptr;
}
for(auto& instance : this->rnnoise_processor) {
if(!instance) { continue; }
rnnoise_destroy((DenoiseState*) instance);
instance = nullptr;
}
for(auto index{0}; index < kInternalFrameBufferCount; index++) {
if(!this->internal_frame_buffer[index]) { continue; }
@ -104,9 +90,11 @@ void AudioConsumerWrapper::do_wrap(const v8::Local<v8::Object> &obj) {
std::unique_ptr<DataEntry> buffer;
while(true) {
{
lock_guard lock(this->_data_lock);
if(this->_data_entries.empty())
break;
lock_guard lock{this->_data_lock};
if(this->_data_entries.empty()) {
break;
}
buffer = move(this->_data_entries.front());
this->_data_entries.pop_front();
}
@ -155,7 +143,6 @@ void AudioConsumerWrapper::unbind() {
}
}
static const float kRnNoiseScale = -INT16_MIN;
void AudioConsumerWrapper::process_data(const void *buffer, size_t samples) {
if(samples != 960) {
logger::error(logger::category::audio, tr("Received audio frame with invalid sample count (Expected 960, Received {})"), samples);
@ -165,67 +152,6 @@ void AudioConsumerWrapper::process_data(const void *buffer, size_t samples) {
lock_guard lock{this->execute_mutex};
if(this->filter_mode_ == FilterMode::BLOCK) { return; }
/* apply input modifiers */
if(this->rnnoise) {
/* TODO: don't call reserve_internal_buffer every time and assume the buffers are initialized */
/* TODO: Maybe find out if the microphone is some kind of pseudo stero so we can handle it as mono? */
if(this->_handle->channel_count > 1) {
auto channel_count = this->_handle->channel_count;
this->reserve_internal_buffer(0, samples * channel_count * sizeof(float));
this->reserve_internal_buffer(1, samples * channel_count * sizeof(float));
for(size_t channel{0}; channel < channel_count; channel++) {
auto target_buffer = (float*) this->internal_frame_buffer[1];
auto source_buffer = (const float*) buffer + channel;
for(size_t index{0}; index < samples; index++) {
*target_buffer = *source_buffer * kRnNoiseScale;
source_buffer += channel_count;
target_buffer++;
}
/* rnnoise uses a frame size of 480 */
this->initialize_rnnoise(channel);
rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[channel], (float*) this->internal_frame_buffer[0] + channel * samples, (const float*) this->internal_frame_buffer[1]);
rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[channel], (float*) this->internal_frame_buffer[0] + channel * samples + 480, (const float*) this->internal_frame_buffer[1] + 480);
}
const float* channel_buffer_ptr[kMaxChannelCount];
for(size_t channel{0}; channel < channel_count; channel++) {
channel_buffer_ptr[channel] = (const float*) this->internal_frame_buffer[0] + channel * samples;
}
/* now back again to interlanced */
auto target_buffer = (float*) this->internal_frame_buffer[1];
for(size_t index{0}; index < samples; index++) {
for(size_t channel{0}; channel < channel_count; channel++) {
*target_buffer = *(channel_buffer_ptr[channel]++) / kRnNoiseScale;
target_buffer++;
}
}
buffer = this->internal_frame_buffer[1];
} else {
/* rnnoise uses a frame size of 480 */
this->reserve_internal_buffer(0, samples * sizeof(float));
auto target_buffer = (float*) this->internal_frame_buffer[0];
for(size_t index{0}; index < samples; index++) {
target_buffer[index] = ((float*) buffer)[index] * kRnNoiseScale;
}
this->initialize_rnnoise(0);
rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[0], target_buffer, target_buffer);
rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[0], &target_buffer[480], &target_buffer[480]);
buffer = target_buffer;
for(size_t index{0}; index < samples; index++) {
target_buffer[index] /= kRnNoiseScale;
}
}
}
bool should_process{true};
if(this->filter_mode_ == FilterMode::FILTER) {
auto filters = this->filters();
@ -350,12 +276,6 @@ void AudioConsumerWrapper::reserve_internal_buffer(int index, size_t target) {
}
}
void AudioConsumerWrapper::initialize_rnnoise(int channel) {
if(!this->rnnoise_processor[channel]) {
this->rnnoise_processor[channel] = rnnoise_create(nullptr);
}
}
NAN_METHOD(AudioConsumerWrapper::_get_filters) {
auto handle = ObjectWrap::Unwrap<AudioConsumerWrapper>(info.Holder());
auto filters = handle->filters();
@ -459,20 +379,4 @@ NAN_METHOD(AudioConsumerWrapper::_set_filter_mode) {
auto value = info[0].As<v8::Number>()->Int32Value(info.GetIsolate()->GetCurrentContext()).FromMaybe(0);
handle->filter_mode_ = (FilterMode) value;
}
NAN_METHOD(AudioConsumerWrapper::rnnoise_enabled) {
auto handle = ObjectWrap::Unwrap<AudioConsumerWrapper>(info.Holder());
info.GetReturnValue().Set(handle->rnnoise);
}
NAN_METHOD(AudioConsumerWrapper::toggle_rnnoise) {
auto handle = ObjectWrap::Unwrap<AudioConsumerWrapper>(info.Holder());
if(info.Length() != 1 || !info[0]->IsBoolean()) {
Nan::ThrowError("invalid argument");
return;
}
handle->rnnoise = info[0]->BooleanValue(info.GetIsolate());
}

10
native/serverconnection/src/audio/js/AudioConsumer.h
View File

@ -52,9 +52,6 @@ namespace tc::audio {
static NAN_METHOD(_get_filter_mode);
static NAN_METHOD(_set_filter_mode);
static NAN_METHOD(toggle_rnnoise);
static NAN_METHOD(rnnoise_enabled);
std::shared_ptr<AudioFilterWrapper> create_filter(const std::string& /* name */, const std::shared_ptr<filter::Filter>& /* filter impl */);
void delete_filter(const AudioFilterWrapper*);
@ -63,7 +60,7 @@ namespace tc::audio {
return this->filter_;
}
inline FilterMode filter_mode() const { return this->filter_mode_; }
[[nodiscard]] inline FilterMode filter_mode() const { return this->filter_mode_; }
inline std::shared_ptr<AudioConsumer> native_consumer() { return this->_handle; }
std::mutex native_read_callback_lock;
@ -71,10 +68,6 @@ namespace tc::audio {
private:
AudioRecorderWrapper* _recorder;
/* preprocessors */
bool rnnoise{false};
std::array<void*, kMaxChannelCount> rnnoise_processor{nullptr};
std::mutex execute_mutex;
std::shared_ptr<AudioConsumer> _handle;
@ -93,7 +86,6 @@ namespace tc::audio {
void process_data(const void* /* buffer */, size_t /* samples */);
void reserve_internal_buffer(int /* buffer */, size_t /* bytes */);
void initialize_rnnoise(int /* channel */);
struct DataEntry {
void* buffer = nullptr;

5
native/serverconnection/src/audio/js/AudioOutputStream.cpp
View File

@ -32,8 +32,9 @@ NAN_MODULE_INIT(AudioOutputStreamWrapper::Init) {
}
NAN_METHOD(AudioOutputStreamWrapper::NewInstance) {
if(!info.IsConstructCall())
Nan::ThrowError("invalid invoke!");
if(!info.IsConstructCall()) {
Nan::ThrowError("invalid invoke!");
}
}

72
native/serverconnection/src/audio/js/AudioOutputStream.h
View File

@ -3,50 +3,48 @@
#include <nan.h>
#include <include/NanEventCallback.h>
namespace tc {
namespace audio {
class AudioResampler;
class AudioOutputSource;
namespace tc::audio {
class AudioResampler;
class AudioOutputSource;
class AudioOutputStreamWrapper : public Nan::ObjectWrap {
public:
static NAN_MODULE_INIT(Init);
static NAN_METHOD(NewInstance);
static inline Nan::Persistent<v8::Function> & constructor() {
static Nan::Persistent<v8::Function> my_constructor;
return my_constructor;
}
class AudioOutputStreamWrapper : public Nan::ObjectWrap {
public:
static NAN_MODULE_INIT(Init);
static NAN_METHOD(NewInstance);
static inline Nan::Persistent<v8::Function> & constructor() {
static Nan::Persistent<v8::Function> my_constructor;
return my_constructor;
}
AudioOutputStreamWrapper(const std::shared_ptr<AudioOutputSource>& /* stream */, bool /* own */);
virtual ~AudioOutputStreamWrapper();
AudioOutputStreamWrapper(const std::shared_ptr<AudioOutputSource>& /* stream */, bool /* own */);
~AudioOutputStreamWrapper() override;
void do_wrap(const v8::Local<v8::Object>&);
void drop_stream();
private:
static ssize_t write_data(const std::shared_ptr<AudioOutputSource>&, void* source, size_t samples, bool interleaved);
void do_wrap(const v8::Local<v8::Object>&);
void drop_stream();
private:
static ssize_t write_data(const std::shared_ptr<AudioOutputSource>&, void* source, size_t samples, bool interleaved);
/* general methods */
static NAN_METHOD(_get_buffer_latency);
static NAN_METHOD(_set_buffer_latency);
static NAN_METHOD(_get_buffer_max_latency);
static NAN_METHOD(_set_buffer_max_latency);
/* general methods */
static NAN_METHOD(_get_buffer_latency);
static NAN_METHOD(_set_buffer_latency);
static NAN_METHOD(_get_buffer_max_latency);
static NAN_METHOD(_set_buffer_max_latency);
static NAN_METHOD(_flush_buffer);
static NAN_METHOD(_flush_buffer);
/* methods for owned streams only */
static NAN_METHOD(_write_data);
static NAN_METHOD(_write_data_rated);
/* methods for owned streams only */
static NAN_METHOD(_write_data);
static NAN_METHOD(_write_data_rated);
static NAN_METHOD(_clear);
static NAN_METHOD(_deleted);
static NAN_METHOD(_delete);
static NAN_METHOD(_clear);
static NAN_METHOD(_deleted);
static NAN_METHOD(_delete);
std::unique_ptr<AudioResampler> _resampler;
std::shared_ptr<AudioOutputSource> _own_handle;
std::weak_ptr<AudioOutputSource> _handle;
std::unique_ptr<AudioResampler> _resampler;
std::shared_ptr<AudioOutputSource> _own_handle;
std::weak_ptr<AudioOutputSource> _handle;
Nan::callback_t<> call_underflow;
Nan::callback_t<> call_overflow;
};
}
Nan::callback_t<> call_underflow;
Nan::callback_t<> call_overflow;
};
}

396
native/serverconnection/src/audio/js/AudioProcessor.cpp Normal file
View File

@ -0,0 +1,396 @@
//
// Created by WolverinDEV on 28/03/2021.
//
#include "./AudioProcessor.h"
#include "../../logger.h"
#include <NanStrings.h>
using namespace tc::audio;
NAN_MODULE_INIT(AudioProcessorWrapper::Init) {
auto klass = Nan::New<v8::FunctionTemplate>(AudioProcessorWrapper::NewInstance);
klass->SetClassName(Nan::New("AudioProcessor").ToLocalChecked());
klass->InstanceTemplate()->SetInternalFieldCount(1);
Nan::SetPrototypeMethod(klass, "get_config", AudioProcessorWrapper::get_config);
Nan::SetPrototypeMethod(klass, "apply_config", AudioProcessorWrapper::apply_config);
Nan::SetPrototypeMethod(klass, "get_statistics", AudioProcessorWrapper::get_statistics);
constructor().Reset(Nan::GetFunction(klass).ToLocalChecked());
}
NAN_METHOD(AudioProcessorWrapper::NewInstance) {
if(!info.IsConstructCall()) {
Nan::ThrowError("invalid invoke!");
}
}
AudioProcessorWrapper::AudioProcessorWrapper(const std::shared_ptr<AudioProcessor> &processor) {
log_allocate("AudioProcessorWrapper", this);
this->registered_observer = new Observer{this};
this->weak_processor = processor;
processor->register_process_observer(this->registered_observer);
}
AudioProcessorWrapper::~AudioProcessorWrapper() noexcept {
log_allocate("AudioProcessorWrapper", this);
if(auto processor{this->weak_processor.lock()}; processor) {
processor->unregister_process_observer(this->registered_observer);
}
delete this->registered_observer;
}
#define PUT_VALUE(key, value) \
result->Set(context, Nan::LocalStringUTF8(#key), value).Check()
#define PUT_CONFIG(path) \
PUT_VALUE(path, Nan::New(config.path))
#define LOAD_CONFIG(path, ...) \
do { \
if(!load_config_value(context, js_config, #path, config.path, __VA_ARGS__)) { \
return; \
} \
} while(0)
NAN_METHOD(AudioProcessorWrapper::get_config) {
auto handle = Nan::ObjectWrap::Unwrap<AudioProcessorWrapper>(info.Holder());
auto processor = handle->weak_processor.lock();
if(!processor) {
Nan::ThrowError("processor passed away");
return;
}
auto config = processor->get_config();
auto result = Nan::New<v8::Object>();
auto context = info.GetIsolate()->GetCurrentContext();
PUT_CONFIG(pipeline.maximum_internal_processing_rate);
PUT_CONFIG(pipeline.multi_channel_render);
PUT_CONFIG(pipeline.multi_channel_capture);
PUT_CONFIG(pre_amplifier.enabled);
PUT_CONFIG(pre_amplifier.fixed_gain_factor);
PUT_CONFIG(high_pass_filter.enabled);
PUT_CONFIG(high_pass_filter.apply_in_full_band);
PUT_CONFIG(echo_canceller.enabled);
PUT_CONFIG(echo_canceller.mobile_mode);
PUT_CONFIG(echo_canceller.export_linear_aec_output); /* TODO: Consider removing? */
PUT_CONFIG(echo_canceller.enforce_high_pass_filtering);
PUT_CONFIG(noise_suppression.enabled);
switch (config.noise_suppression.level) {
using Level = webrtc::AudioProcessing::Config::NoiseSuppression::Level;
case Level::kLow:
PUT_VALUE(noise_suppression.level, Nan::LocalStringUTF8("low"));
break;
case Level::kModerate:
PUT_VALUE(noise_suppression.level, Nan::LocalStringUTF8("moderate"));
break;
case Level::kHigh:
PUT_VALUE(noise_suppression.level, Nan::LocalStringUTF8("high"));
break;
case Level::kVeryHigh:
PUT_VALUE(noise_suppression.level, Nan::LocalStringUTF8("very-high"));
break;
default:
PUT_VALUE(noise_suppression.level, Nan::LocalStringUTF8("unknown"));
break;
}
PUT_CONFIG(noise_suppression.analyze_linear_aec_output_when_available);
PUT_CONFIG(transient_suppression.enabled);
PUT_CONFIG(voice_detection.enabled);
PUT_CONFIG(gain_controller1.enabled);
switch (config.gain_controller1.mode) {
using Mode = webrtc::AudioProcessing::Config::GainController1::Mode;
case Mode::kAdaptiveAnalog:
PUT_VALUE(gain_controller1.mode, Nan::LocalStringUTF8("adaptive-analog"));
break;
case Mode::kAdaptiveDigital:
PUT_VALUE(gain_controller1.mode, Nan::LocalStringUTF8("adaptive-digital"));
break;
case Mode::kFixedDigital:
PUT_VALUE(gain_controller1.mode, Nan::LocalStringUTF8("fixed-digital"));
break;
default:
PUT_VALUE(gain_controller1.mode, Nan::LocalStringUTF8("unknown"));
break;
}
PUT_CONFIG(gain_controller1.target_level_dbfs);
PUT_CONFIG(gain_controller1.compression_gain_db);
PUT_CONFIG(gain_controller1.enable_limiter);
PUT_CONFIG(gain_controller1.analog_level_minimum);
PUT_CONFIG(gain_controller1.analog_level_maximum);
PUT_CONFIG(gain_controller1.analog_gain_controller.enabled);
PUT_CONFIG(gain_controller1.analog_gain_controller.startup_min_volume);
PUT_CONFIG(gain_controller1.analog_gain_controller.clipped_level_min);
PUT_CONFIG(gain_controller1.analog_gain_controller.enable_agc2_level_estimator);
PUT_CONFIG(gain_controller1.analog_gain_controller.enable_digital_adaptive);
PUT_CONFIG(gain_controller2.enabled);
PUT_CONFIG(gain_controller2.fixed_digital.gain_db);
PUT_CONFIG(gain_controller2.adaptive_digital.enabled);
switch(config.gain_controller2.adaptive_digital.level_estimator) {
using LevelEstimator = webrtc::AudioProcessing::Config::GainController2::LevelEstimator;
case LevelEstimator::kPeak:
PUT_VALUE(gain_controller2.adaptive_digital.level_estimator, Nan::LocalStringUTF8("peak"));
break;
case LevelEstimator::kRms:
PUT_VALUE(gain_controller2.adaptive_digital.level_estimator, Nan::LocalStringUTF8("rms"));
break;
default:
PUT_VALUE(gain_controller2.adaptive_digital.level_estimator, Nan::LocalStringUTF8("unknown"));
break;
}
PUT_CONFIG(gain_controller2.adaptive_digital.vad_probability_attack);
PUT_CONFIG(gain_controller2.adaptive_digital.level_estimator_adjacent_speech_frames_threshold);
PUT_CONFIG(gain_controller2.adaptive_digital.use_saturation_protector);
PUT_CONFIG(gain_controller2.adaptive_digital.initial_saturation_margin_db);
PUT_CONFIG(gain_controller2.adaptive_digital.extra_saturation_margin_db);
PUT_CONFIG(gain_controller2.adaptive_digital.gain_applier_adjacent_speech_frames_threshold);
PUT_CONFIG(gain_controller2.adaptive_digital.max_gain_change_db_per_second);
PUT_CONFIG(gain_controller2.adaptive_digital.max_output_noise_level_dbfs);
PUT_CONFIG(residual_echo_detector.enabled);
PUT_CONFIG(level_estimation.enabled);
PUT_CONFIG(rnnoise.enabled);
info.GetReturnValue().Set(result);
}
template <typename T, typename = typename std::enable_if<std::is_arithmetic<T>::value, T>::type>
inline bool load_config_value(
const v8::Local<v8::Context>& context,
const v8::Local<v8::Object>& js_config,
const std::string_view& key,
T& value_ref,
T min_value = std::numeric_limits<T>::min(),
T max_value = std::numeric_limits<T>::max()
) {
auto maybe_value = js_config->Get(context, Nan::LocalStringUTF8(key));
if(maybe_value.IsEmpty() || maybe_value.ToLocalChecked()->IsNullOrUndefined()) {
return true;
}
double value;
if(maybe_value.ToLocalChecked()->IsNumber()) {
value = maybe_value.ToLocalChecked()->NumberValue(context).ToChecked();
} else if(maybe_value.ToLocalChecked()->IsBoolean()) {
value = maybe_value.ToLocalChecked()->BooleanValue(v8::Isolate::GetCurrent());
} else {
Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " isn't a number or boolean"));
return false;
}
if(std::numeric_limits<T>::is_integer && (double) (T) value != value) {
Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " isn't an integer"));
return false;
}
if(value < (double) min_value) {
Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " exceeds min value of " + std::to_string(min_value)));
return false;
}
if(value > (double) max_value) {
Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " exceeds max value of " + std::to_string(max_value)));
return false;
}
value_ref = value;
return true;
}
template <size_t kValueSize, typename T>
inline bool load_config_enum(
const v8::Local<v8::Context>& context,
const v8::Local<v8::Object>& js_config,
const std::string_view& key,
T& value_ref,
const std::array<std::pair<std::string_view, T>, kValueSize>& values
) {
auto maybe_value = js_config->Get(context, Nan::LocalStringUTF8(key));
if(maybe_value.IsEmpty() || maybe_value.ToLocalChecked()->IsNullOrUndefined()) {
return true;
} else if(!maybe_value.ToLocalChecked()->IsString()) {
Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " isn't a string"));
return false;
}
auto str_value = maybe_value.ToLocalChecked()->ToString(context).ToLocalChecked();
auto value = *Nan::Utf8String(str_value);
for(const auto& [ key, key_value ] : values) {
if(key != value) {
continue;
}
value_ref = key_value;
return true;
}
Nan::ThrowError(Nan::LocalStringUTF8("property " + std::string{key} + " contains an invalid enum value (" + value + ")"));
return false;
}
#define LOAD_ENUM(path, arg_count, ...) \
do { \
if(!load_config_enum<arg_count>(context, js_config, #path, config.path, {{ __VA_ARGS__ }})) { \
return; \
} \
} while(0)
NAN_METHOD(AudioProcessorWrapper::apply_config) {
auto handle = Nan::ObjectWrap::Unwrap<AudioProcessorWrapper>(info.Holder());
auto processor = handle->weak_processor.lock();
if (!processor) {
Nan::ThrowError("processor passed away");
return;
}
if(info.Length() != 1 || !info[0]->IsObject()) {
Nan::ThrowError("Invalid arguments");
return;
}
auto config = processor->get_config();
auto context = info.GetIsolate()->GetCurrentContext();
auto js_config = info[0]->ToObject(info.GetIsolate()->GetCurrentContext()).ToLocalChecked();
using GainControllerMode = webrtc::AudioProcessing::Config::GainController1::Mode;
using GainControllerLevelEstimator = webrtc::AudioProcessing::Config::GainController2::LevelEstimator;
using NoiseSuppressionLevel = webrtc::AudioProcessing::Config::NoiseSuppression::Level;
LOAD_CONFIG(pipeline.maximum_internal_processing_rate);
LOAD_CONFIG(pipeline.multi_channel_render);
LOAD_CONFIG(pipeline.multi_channel_capture);
LOAD_CONFIG(pre_amplifier.enabled);
LOAD_CONFIG(pre_amplifier.fixed_gain_factor);
LOAD_CONFIG(high_pass_filter.enabled);
LOAD_CONFIG(high_pass_filter.apply_in_full_band);
LOAD_CONFIG(echo_canceller.enabled);
LOAD_CONFIG(echo_canceller.mobile_mode);
LOAD_CONFIG(echo_canceller.export_linear_aec_output); /* TODO: Consider removing? */
LOAD_CONFIG(echo_canceller.enforce_high_pass_filtering);
LOAD_CONFIG(noise_suppression.enabled);
LOAD_ENUM(noise_suppression.level, 4,
{ "low", NoiseSuppressionLevel::kLow },
{ "moderate", NoiseSuppressionLevel::kModerate },
{ "high", NoiseSuppressionLevel::kHigh },
{ "very-high", NoiseSuppressionLevel::kVeryHigh }
);
LOAD_CONFIG(noise_suppression.analyze_linear_aec_output_when_available);
LOAD_CONFIG(transient_suppression.enabled);
LOAD_CONFIG(voice_detection.enabled);
LOAD_CONFIG(gain_controller1.enabled);
LOAD_ENUM(gain_controller1.mode, 3,
{ "adaptive-analog", GainControllerMode::kAdaptiveAnalog },
{ "adaptive-digital", GainControllerMode::kAdaptiveDigital },
{ "fixed-digital", GainControllerMode::kFixedDigital }
);
LOAD_CONFIG(gain_controller1.target_level_dbfs);
LOAD_CONFIG(gain_controller1.compression_gain_db);
LOAD_CONFIG(gain_controller1.enable_limiter);
LOAD_CONFIG(gain_controller1.analog_level_minimum);
LOAD_CONFIG(gain_controller1.analog_level_maximum);
LOAD_CONFIG(gain_controller1.analog_gain_controller.enabled);
LOAD_CONFIG(gain_controller1.analog_gain_controller.startup_min_volume);
LOAD_CONFIG(gain_controller1.analog_gain_controller.clipped_level_min);
LOAD_CONFIG(gain_controller1.analog_gain_controller.enable_agc2_level_estimator);
LOAD_CONFIG(gain_controller1.analog_gain_controller.enable_digital_adaptive);
LOAD_CONFIG(gain_controller2.enabled);
LOAD_CONFIG(gain_controller2.fixed_digital.gain_db);
LOAD_CONFIG(gain_controller2.adaptive_digital.enabled);
LOAD_ENUM(gain_controller2.adaptive_digital.level_estimator, 2,
{ "peak", GainControllerLevelEstimator::kPeak },
{ "rms", GainControllerLevelEstimator::kRms }
);
LOAD_CONFIG(gain_controller2.adaptive_digital.vad_probability_attack);
LOAD_CONFIG(gain_controller2.adaptive_digital.level_estimator_adjacent_speech_frames_threshold);
LOAD_CONFIG(gain_controller2.adaptive_digital.use_saturation_protector);
LOAD_CONFIG(gain_controller2.adaptive_digital.initial_saturation_margin_db);
LOAD_CONFIG(gain_controller2.adaptive_digital.extra_saturation_margin_db);
LOAD_CONFIG(gain_controller2.adaptive_digital.gain_applier_adjacent_speech_frames_threshold);
LOAD_CONFIG(gain_controller2.adaptive_digital.max_gain_change_db_per_second);
LOAD_CONFIG(gain_controller2.adaptive_digital.max_output_noise_level_dbfs);
LOAD_CONFIG(residual_echo_detector.enabled);
LOAD_CONFIG(level_estimation.enabled);
LOAD_CONFIG(rnnoise.enabled);
processor->apply_config(config);
}
#define PUT_STATISTIC(path) \
do { \
if(config.path.has_value()) { \
PUT_VALUE(path, Nan::New(*config.path)); \
} else { \
PUT_VALUE(path, Nan::Undefined()); \
} \
} while(0)
NAN_METHOD(AudioProcessorWrapper::get_statistics) {
auto handle = Nan::ObjectWrap::Unwrap<AudioProcessorWrapper>(info.Holder());
auto processor = handle->weak_processor.lock();
if(!processor) {
Nan::ThrowError("processor passed away");
return;
}
auto config = processor->get_statistics();
auto result = Nan::New<v8::Object>();
auto context = info.GetIsolate()->GetCurrentContext();
PUT_STATISTIC(output_rms_dbfs);
PUT_STATISTIC(voice_detected);
PUT_STATISTIC(echo_return_loss);
PUT_STATISTIC(echo_return_loss_enhancement);
PUT_STATISTIC(divergent_filter_fraction);
PUT_STATISTIC(delay_median_ms);
PUT_STATISTIC(delay_standard_deviation_ms);
PUT_STATISTIC(residual_echo_likelihood);
PUT_STATISTIC(residual_echo_likelihood_recent_max);
PUT_STATISTIC(delay_ms);
PUT_STATISTIC(rnnoise_volume);
info.GetReturnValue().Set(result);
}
void AudioProcessorWrapper::Observer::stream_processed(const AudioProcessor::Stats &stats) {
/* TODO! */
}

41
native/serverconnection/src/audio/js/AudioProcessor.h Normal file
View File

@ -0,0 +1,41 @@
#pragma once
#include <nan.h>
#include "../processing/AudioProcessor.h"
namespace tc::audio {
class AudioProcessorWrapper : public Nan::ObjectWrap {
public:
static NAN_MODULE_INIT(Init);
static NAN_METHOD(NewInstance);
static inline Nan::Persistent<v8::Function> & constructor() {
static Nan::Persistent<v8::Function> my_constructor;
return my_constructor;
}
explicit AudioProcessorWrapper(const std::shared_ptr<AudioProcessor>& /* processor */);
~AudioProcessorWrapper() override;
inline void wrap(v8::Local<v8::Object> object) {
Nan::ObjectWrap::Wrap(object);
}
static NAN_METHOD(get_config);
static NAN_METHOD(apply_config);
static NAN_METHOD(get_statistics);
private:
struct Observer : public AudioProcessor::ProcessObserver {
public:
explicit Observer(AudioProcessorWrapper* wrapper) : wrapper{wrapper} {}
private:
AudioProcessorWrapper* wrapper;
void stream_processed(const AudioProcessor::Stats &stats) override;
};
std::weak_ptr<AudioProcessor> weak_processor{};
Observer* registered_observer{nullptr};
};
}

31
native/serverconnection/src/audio/js/AudioRecorder.cpp
View File

@ -3,6 +3,7 @@
#include "AudioRecorder.h"
#include "AudioConsumer.h"
#include "./AudioProcessor.h"
#include "../AudioInput.h"
#include "../../logger.h"
@ -46,6 +47,8 @@ NAN_MODULE_INIT(AudioRecorderWrapper::Init) {
Nan::SetPrototypeMethod(klass, "create_consumer", AudioRecorderWrapper::_create_consumer);
Nan::SetPrototypeMethod(klass, "delete_consumer", AudioRecorderWrapper::_delete_consumer);
Nan::SetPrototypeMethod(klass, "get_audio_processor", AudioRecorderWrapper::get_audio_processor);
constructor().Reset(Nan::GetFunction(klass).ToLocalChecked());
}
@ -93,7 +96,7 @@ std::shared_ptr<AudioConsumerWrapper> AudioRecorderWrapper::create_consumer() {
}
void AudioRecorderWrapper::delete_consumer(const AudioConsumerWrapper* consumer) {
shared_ptr<AudioConsumerWrapper> handle; /* need to keep the handle 'till everything has been finished */
std::shared_ptr<AudioConsumerWrapper> handle; /* need to keep the handle 'till everything has been finished */
{
lock_guard lock(this->consumer_mutex);
for(auto& c : this->consumer_) {
@ -102,20 +105,22 @@ void AudioRecorderWrapper::delete_consumer(const AudioConsumerWrapper* consumer)
break;
}
}
if(!handle)
return;
if(!handle) {
return;
}
{
auto it = find(this->consumer_.begin(), this->consumer_.end(), handle);
if(it != this->consumer_.end())
this->consumer_.erase(it);
if(it != this->consumer_.end()) {
this->consumer_.erase(it);
}
}
}
{
lock_guard lock(handle->execute_mutex); /* if we delete the consumer while executing strange stuff could happen */
handle->unbind();
this->input_->delete_consumer(handle->_handle);
}
}
@ -271,4 +276,18 @@ NAN_METHOD(AudioRecorderWrapper::_set_volume) {
NAN_METHOD(AudioRecorderWrapper::_get_volume) {
auto handle = ObjectWrap::Unwrap<AudioRecorderWrapper>(info.Holder());
info.GetReturnValue().Set(handle->input_->volume());
}
NAN_METHOD(AudioRecorderWrapper::get_audio_processor) {
auto handle = ObjectWrap::Unwrap<AudioRecorderWrapper>(info.Holder());
auto processor = handle->input_->audio_processor();
if(!processor) {
return;
}
auto js_object = Nan::NewInstance(Nan::New(AudioProcessorWrapper::constructor()), 0, nullptr).ToLocalChecked();
auto wrapper = new AudioProcessorWrapper(processor);
wrapper->wrap(js_object);
info.GetReturnValue().Set(js_object);
}

4
native/serverconnection/src/audio/js/AudioRecorder.h
View File

@ -23,7 +23,7 @@ namespace tc::audio {
return my_constructor;
}
explicit AudioRecorderWrapper(std::shared_ptr<AudioInput> /* input */);
explicit AudioRecorderWrapper(std::shared_ptr<AudioInput> /* input */);
~AudioRecorderWrapper() override;
static NAN_METHOD(_get_device);
@ -40,6 +40,8 @@ namespace tc::audio {
static NAN_METHOD(_set_volume);
static NAN_METHOD(_get_volume);
static NAN_METHOD(get_audio_processor);
std::shared_ptr<AudioConsumerWrapper> create_consumer();
void delete_consumer(const AudioConsumerWrapper*);

228
native/serverconnection/src/audio/processing/AudioProcessor.cpp
View File

@ -2,10 +2,234 @@
// Created by WolverinDEV on 27/03/2021.
//
#include "AudioProcessor.h"
#include "./AudioProcessor.h"
#include "../../logger.h"
#include <rnnoise.h>
#include <sstream>
using namespace tc::audio;
void AudioProcessor::analyze_reverse_stream(const float *const *data, const webrtc::StreamConfig &reverse_config) {
AudioProcessor::AudioProcessor() {
this->current_config.echo_canceller.enabled = true;
this->current_config.echo_canceller.mobile_mode = false;
this->current_config.gain_controller1.enabled = true;
this->current_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog;
this->current_config.gain_controller1.analog_level_minimum = 0;
this->current_config.gain_controller1.analog_level_maximum = 255;
this->current_config.gain_controller2.enabled = true;
this->current_config.high_pass_filter.enabled = true;
this->current_config.voice_detection.enabled = true;
}
AudioProcessor::~AudioProcessor() {
std::lock_guard processor_lock{this->processor_mutex};
delete this->processor;
for(auto& entry : this->rnnoise_processor) {
if(!entry) { continue; }
rnnoise_destroy((DenoiseState*) entry);
}
}
constexpr static inline auto process_error_to_string(int error) {
switch (error) {
case 0: return "kNoError";
case -1: return "kUnspecifiedError";
case -2: return "kCreationFailedError";
case -3: return "kUnsupportedComponentError";
case -4: return "kUnsupportedFunctionError";
case -5: return "kNullPointerError";
case -6: return "kBadParameterError";
case -7: return "kBadSampleRateError";
case -8: return "kBadDataLengthError";
case -9: return "kBadNumberChannelsError";
case -10: return "kFileError";
case -11: return "kStreamParameterNotSetError";
case -12: return "kNotEnabledError";
case -13: return "kBadStreamParameterWarning";
default: return "unkown error code";
}
}
template <typename T>
inline std::ostream& operator<<(std::ostream &ss, const absl::optional<T>& optional) {
if(optional.has_value()) {
ss << "optional{" << *optional << "}";
} else {
ss << "nullopt";
}
return ss;
}
inline std::string statistics_to_string(const webrtc::AudioProcessingStats& stats) {
std::stringstream ss{};
ss << "AudioProcessingStats{";
ss << "output_rms_dbfs: " << stats.output_rms_dbfs << ", ";
ss << "voice_detected: " << stats.voice_detected << ", ";
ss << "echo_return_loss: " << stats.echo_return_loss << ", ";
ss << "echo_return_loss_enhancement: " << stats.echo_return_loss_enhancement << ", ";
ss << "divergent_filter_fraction: " << stats.divergent_filter_fraction << ", ";
ss << "delay_median_ms: " << stats.delay_median_ms << ", ";
ss << "delay_standard_deviation_ms: " << stats.delay_standard_deviation_ms << ", ";
ss << "residual_echo_likelihood: " << stats.residual_echo_likelihood << ", ";
ss << "residual_echo_likelihood_recent_max: " << stats.residual_echo_likelihood_recent_max << ", ";
ss << "delay_ms: " << stats.delay_ms;
ss << "}";
return ss.str();
}
bool AudioProcessor::initialize() {
std::lock_guard processor_lock{this->processor_mutex};
if(this->processor) {
/* double initialize */
return false;
}
using namespace webrtc;
AudioProcessingBuilder builder{};
this->processor = builder.Create();
if(!this->processor) {
return false;
}
this->apply_config_unlocked(this->current_config);
this->processor->Initialize();
return true;
}
AudioProcessor::Config AudioProcessor::get_config() const {
std::shared_lock processor_lock{this->processor_mutex};
return this->current_config;
}
void AudioProcessor::apply_config(const AudioProcessor::Config &config) {
std::lock_guard processor_lock{this->processor_mutex};
this->apply_config_unlocked(config);
}
void AudioProcessor::apply_config_unlocked(const Config &config) {
this->current_config = config;
if(this->processor) {
this->processor->ApplyConfig(config);
}
if(!this->current_config.rnnoise.enabled) {
this->rnnoise_volume = absl::nullopt;
}
}
AudioProcessor::Stats AudioProcessor::get_statistics() const {
std::shared_lock processor_lock{this->processor_mutex};
return this->get_statistics_unlocked();
}
AudioProcessor::Stats AudioProcessor::get_statistics_unlocked() const {
if(!this->processor) {
return AudioProcessor::Stats{};
}
AudioProcessor::Stats result{this->processor->GetStatistics()};
result.rnnoise_volume = this->rnnoise_volume;
return result;
}
void AudioProcessor::register_process_observer(ProcessObserver *observer) {
std::lock_guard processor_lock{this->processor_mutex};
this->process_observer.push_back(observer);
}
bool AudioProcessor::unregister_process_observer(ProcessObserver *observer) {
std::lock_guard processor_lock{this->processor_mutex};
auto index = std::find(this->process_observer.begin(), this->process_observer.end(), observer);
if(index == this->process_observer.end()) {
return false;
}
this->process_observer.erase(index);
return true;
}
void AudioProcessor::analyze_reverse_stream(const float *const *data, const webrtc::StreamConfig &reverse_config) {
std::shared_lock processor_lock{this->processor_mutex};
if(!this->processor) {
return;
}
auto result = this->processor->AnalyzeReverseStream(data, reverse_config);
if(result != webrtc::AudioProcessing::kNoError) {
log_error(category::audio, tr("Failed to process reverse stream: {}"), process_error_to_string(result));
return;
}
}
std::optional<AudioProcessor::Stats> AudioProcessor::process_stream(const webrtc::StreamConfig &config, float *const *buffer) {
std::shared_lock processor_lock{this->processor_mutex};
if(!this->processor) {
return std::nullopt;
} else if(config.num_channels() > kMaxChannelCount) {
log_error(category::audio, tr("AudioProcessor received input buffer with too many channels ({} channels but supported is only {})"), config.num_channels(), kMaxChannelCount);
return std::nullopt;
}
if(this->current_config.rnnoise.enabled) {
if(config.sample_rate_hz() != 48000) {
log_warn(category::audio, tr("Don't apply RNNoise. Source sample rate isn't 480kHz ({}kHz)"), config.sample_rate_hz() / 1000);
this->rnnoise_volume.reset();
} else {
static const float kRnNoiseScale = -INT16_MIN;
double volume_sum{0};
for(size_t channel{0}; channel < config.num_channels(); channel++) {
if(!this->rnnoise_processor[channel]) {
this->rnnoise_processor[channel] = (void*) rnnoise_create(nullptr);
}
{
/* RNNoise uses a frame size of 10ms for 48kHz aka 480 samples */
auto buffer_ptr = buffer[channel];
for(size_t sample{0}; sample < 480; sample++) {
*buffer_ptr++ *= kRnNoiseScale;
}
}
volume_sum += rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[channel], buffer[channel], buffer[channel]);
{
auto buffer_ptr = buffer[channel];
for(size_t sample{0}; sample < 480; sample++) {
*buffer_ptr++ /= kRnNoiseScale;
}
}
}
this->rnnoise_volume = absl::make_optional(volume_sum / config.num_channels());
}
}
this->processor->set_stream_delay_ms(2); /* TODO: Measure it and not just guess it! */
this->processor->set_stream_analog_level(0);
auto result = this->processor->ProcessStream(buffer, config, config, buffer);
if(result != webrtc::AudioProcessing::kNoError) {
log_error(category::audio, tr("Failed to process stream: {}"), process_error_to_string(result));
return std::nullopt;
}
auto statistics = this->get_statistics_unlocked();
for(const auto& observer : this->process_observer) {
observer->stream_processed(statistics);
}
//log_trace(category::audio, tr("Processing stats: {}"), statistics_to_string(statistics));
return std::make_optional(std::move(statistics));
}

55
native/serverconnection/src/audio/processing/AudioProcessor.h
View File

@ -1,11 +1,50 @@
#pragma once
#include <memory>
#include <shared_mutex>
#include <optional>
#include <modules/audio_processing/include/audio_processing.h>
namespace tc::audio {
class AudioProcessor : public std::enable_shared_from_this<AudioProcessor> {
class AudioProcessor {
public:
struct Config : public webrtc::AudioProcessing::Config {
struct {
bool enabled{false};
} rnnoise;
};
struct Stats : public webrtc::AudioProcessingStats {
// The RNNoise returned sample volume
absl::optional<float> rnnoise_volume;
};
struct ProcessObserver {
public:
virtual void stream_processed(const AudioProcessor::Stats&) = 0;
};
AudioProcessor();
virtual ~AudioProcessor();
[[nodiscard]] bool initialize();
[[nodiscard]] Config get_config() const;
void apply_config(const Config &/* config */);
[[nodiscard]] AudioProcessor::Stats get_statistics() const;
void register_process_observer(ProcessObserver* /* observer */);
/**
* Unregister a process observer.
* Note: Never call this within the observer callback!
* This will cause a deadlock.
* @return
*/
bool unregister_process_observer(ProcessObserver* /* observer */);
/* 10ms audio chunk */
[[nodiscard]] std::optional<AudioProcessor::Stats> process_stream( const webrtc::StreamConfig& /* config */, float* const* /* buffer */);
/**
* Accepts deinterleaved float audio with the range [-1, 1]. Each element
@ -14,6 +53,20 @@ namespace tc::audio {
*/
void analyze_reverse_stream(const float* const* data,
const webrtc::StreamConfig& reverse_config);
private:
constexpr static auto kMaxChannelCount{2};
mutable std::shared_mutex processor_mutex{};
Config current_config{};
std::vector<ProcessObserver*> process_observer{};
webrtc::AudioProcessing* processor{nullptr};
absl::optional<float> rnnoise_volume{};
std::array<void*, kMaxChannelCount> rnnoise_processor{nullptr};
[[nodiscard]] AudioProcessor::Stats get_statistics_unlocked() const;
void apply_config_unlocked(const Config &/* config */);
};
}

11
native/serverconnection/src/audio/sounds/SoundPlayer.cpp
View File

@ -3,8 +3,6 @@
//
#include <cassert>
#include <NanGet.h>
#include <NanEventCallback.h>
#include "./SoundPlayer.h"
#include "../AudioOutput.h"
#include "../file/wav.h"
@ -15,6 +13,11 @@
#include "../AudioMerger.h"
#include "../AudioGain.h"
#ifdef NODEJS_API
#include <NanGet.h>
#include <NanEventCallback.h>
#endif
#ifdef max
#undef max
#endif
@ -275,6 +278,7 @@ namespace tc::audio::sounds {
}
}
#ifdef NODEJS_API
NAN_METHOD(tc::audio::sounds::playback_sound_js) {
if(info.Length() != 1 || !info[0]->IsObject()) {
Nan::ThrowError("invalid arguments");
@ -330,4 +334,5 @@ NAN_METHOD(tc::audio::sounds::cancel_playback_js) {
}
cancel_playback((sound_playback_id) info[0].As<v8::Number>()->Value());
}
}
#endif

5
native/serverconnection/src/audio/sounds/SoundPlayer.h
View File

@ -2,7 +2,10 @@
#include <functional>
#include <string_view>
#ifdef NODEJS_API
#include <nan.h>
#endif
namespace tc::audio::sounds {
typedef uintptr_t sound_playback_id;
@ -26,6 +29,8 @@ namespace tc::audio::sounds {
extern sound_playback_id playback_sound(const PlaybackSettings& /* settings */);
extern void cancel_playback(const sound_playback_id&);
#ifdef NODEJS_API
extern NAN_METHOD(playback_sound_js);
extern NAN_METHOD(cancel_playback_js);
#endif
}

13
native/serverconnection/src/bindings.cpp
View File

@ -18,12 +18,10 @@
#include "audio/js/AudioRecorder.h"
#include "audio/js/AudioConsumer.h"
#include "audio/js/AudioFilter.h"
#include "audio/js/AudioProcessor.h"
#include "audio/AudioEventLoop.h"
#include "audio/sounds/SoundPlayer.h"
//#include <webrtc-audio-processing-1/modules/audio_processing/include/audio_processing.h>
#include <modules/audio_processing/include/audio_processing.h>
#ifndef WIN32
#include <unistd.h>
#endif
@ -40,14 +38,6 @@ using namespace tc;
using namespace tc::connection;
using namespace tc::ft;
void processor() {
webrtc::AudioProcessingBuilder builder{};
webrtc::AudioProcessing::Config config{};
auto processor = builder.Create();
//processor->AnalyzeReverseStream()
}
void testTomMath(){
mp_int x{};
mp_init(&x);
@ -196,6 +186,7 @@ NAN_MODULE_INIT(init) {
audio::recorder::AudioRecorderWrapper::Init(namespace_record);
audio::recorder::AudioConsumerWrapper::Init(namespace_record);
audio::recorder::AudioFilterWrapper::Init(namespace_record);
audio::AudioProcessorWrapper::Init(namespace_record);
{
auto enum_object = Nan::New<v8::Object>();

43
native/serverconnection/test/audio/main.cpp
View File

@ -6,56 +6,55 @@
#include "../../src/audio/AudioOutput.h"
#include "../../src/audio/AudioInput.h"
#include "../../src/audio/AudioEventLoop.h"
#include "../../src/audio/filter/FilterVad.h"
#include "../../src/audio/filter/FilterThreshold.h"
#include "../../src/logger.h"
#ifdef WIN32
#include <windows.h>
#endif
using namespace std;
using namespace tc;
tc::audio::AudioOutput* global_audio_output{nullptr};
int main() {
std::string error{};
Pa_Initialize();
logger::initialize_raw();
tc::audio::init_event_loops();
tc::audio::initialize();
tc::audio::await_initialized();
std::shared_ptr<tc::audio::AudioDevice> default_playback{nullptr}, default_record{nullptr};
for(auto& device : tc::audio::devices()) {
if(device->is_output_default())
if(device->is_output_default()) {
default_playback = device;
if(device->is_input_default())
}
if(device->is_input_default()) {
default_record = device;
}
}
assert(default_record);
assert(default_playback);
for(auto& dev : tc::audio::devices()) {
if(!dev->is_input_supported()) continue;
if(!dev->is_input_supported()) {
continue;
}
auto playback_manager = std::make_unique<audio::AudioOutput>(2, 48000);
if(!playback_manager->set_device(error, default_playback)) {
cerr << "Failed to open output device (" << error << ")" << endl;
return 1;
}
if(!playback_manager->playback()) {
cerr << "failed to start playback" << endl;
global_audio_output = &*playback_manager;
playback_manager->set_device(default_playback);
if(!playback_manager->playback(error)) {
cerr << "failed to start playback: " << error << endl;
return 1;
}
auto input = std::make_unique<audio::AudioInput>(2, 48000);
if(!input->set_device(error, dev)) {
cerr << "Failed to open input device (" << error << "): " << dev->id() << " (" << dev->name() << ")" << endl;
continue;
}
if(!input->record()) {
cerr << "failed to start record for " << dev->id() << " (" << dev->name() << ")" << endl;
input->set_device(default_record);
if(!input->record(error)) {
cerr << "failed to start record for " << dev->id() << " (" << dev->name() << "): " << error << endl;
continue;
}
@ -94,6 +93,7 @@ int main() {
}
playback_manager.release(); //FIXME: Memory leak!
break;
}
this_thread::sleep_for(chrono::seconds(360));
@ -103,6 +103,5 @@ int main() {
this_thread::sleep_for(chrono::seconds(1000));
}
*/
Pa_Terminate();
return 1;
}

103
native/serverconnection/test/js/audio.ts
View File

@ -1,14 +1,96 @@
/// <reference path="../../exports/exports.d.ts" />
console.log("Starting app");
module.paths.push("../../build/linux_x64");
module.paths.push("../../build/win32_x64");
import * as path from "path";
module.paths.push(path.join(__dirname, "..", "..", "..", "build", "win32_x64"));
module.paths.push(path.join(__dirname, "..", "..", "..", "build", "linux_x64"));
const original_require = require;
require = (module => original_require(__dirname + "/../../../build/win32_x64/" + module + ".node")) as any;
import * as handle from "teaclient_connection";
require = original_require;
import {audio} from "teaclient_connection.node";
import record = audio.record;
import playback = audio.playback;
console.dir(handle.audio);
function printDevices() {
console.info("Available input devices:");
for(const device of audio.available_devices()) {
if(!device.input_supported) {
continue;
}
console.info(" - " + device.driver + " - " + device.device_id + " (" + device.name + ")" + (device.input_default ? " (default)" : ""));
}
console.info("Available output devices:");
for(const device of audio.available_devices()) {
if(!device.output_supported) {
continue;
}
console.info(" - " + device.driver + " - " + device.device_id + " (" + device.name + ")" + (device.output_default ? " (default)" : ""));
}
}
async function main() {
await new Promise(resolve => audio.initialize(resolve));
console.info("Audio initialized");
//printDevices();
const recorder = record.create_recorder();
await new Promise((resolve, reject) => {
const defaultInput = audio.available_devices().find(device => device.input_default);
if(!defaultInput) {
reject("missing default input device");
return;
}
recorder.set_device(defaultInput.device_id, result => {
if(result === "success") {
resolve();
} else {
reject(result);
}
});
});
await new Promise((resolve, reject) => {
recorder.start(result => {
if(typeof result === "boolean" && result) {
resolve();
} else {
reject(result);
}
});
});
const output = playback.create_stream();
const recorderConsumer = recorder.create_consumer();
if(output.channels !== recorderConsumer.channelCount) {
throw "miss matching channel count";
}
if(output.sample_rate !== recorderConsumer.sampleRate) {
throw "miss matching sample rate";
}
recorderConsumer.callback_data = buffer => output.write_data(buffer.buffer, true);
setInterval(() => {
const processor = recorder.get_audio_processor();
if(!processor) { return; }
console.error("Config:\n%o", processor.get_config());
console.error("Statistics:\n%o", processor.get_statistics());
processor.apply_config({
"echo_canceller.enabled": false,
"rnnoise.enabled": true
});
}, 2500);
}
main().catch(error => {
console.error(error);
process.exit(1);
});
/*
handle.audio.initialize(() => {
console.log("Audio initialized");
@ -60,5 +142,6 @@ handle.audio.initialize(() => {
});
});
}, 1000);
*/
});
*/

18
native/serverconnection/test/js/tsconfig.json Normal file
View File

@ -0,0 +1,18 @@
{
"compilerOptions": {
"module": "CommonJS",
"noImplicitAny": true,
"removeComments": true,
"preserveConstEnums": true,
"sourceMap": true,
"baseUrl": ".",
"paths": {
"teaclient_connection.node": ["../../exports/exports.d.ts"]
},
"lib": [
"dom",
"es6",
"scripthost",
]
}
}