TeaSpeak/TeaSpeak-Client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Adding the possibility to create level meters for the unprocessed and processed audio input

Browse Source
This commit is contained in:
WolverinDEV 2021-03-28 22:27:48 +02:00
parent 4c264a7225
commit 105f675590
10 changed files with 207 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
native/serverconnection/exports/exports.d.ts vendored
View File

@ -324,7 +324,7 @@ export namespace audio {
export type DeviceSetResult = "success" | "invalid-device";
export interface AudioRecorder {
get_device() : string;
set_device(device_id: string, callback: (result: DeviceSetResult) => void); /* Recorder needs to be started afterwards */
set_device(deviceId: string, callback: (result: DeviceSetResult) => void); /* Recorder needs to be started afterwards */
start(callback: (result: boolean | string) => void);
started() : boolean;
@ -338,6 +338,7 @@ export namespace audio {
delete_consumer(consumer: AudioConsumer);
get_audio_processor() : AudioProcessor | undefined;
create_level_meter(mode: "pre-process" | "post-process") : AudioLevelMeter;
}
export interface AudioLevelMeter {
@ -348,7 +349,7 @@ export namespace audio {
set_callback(callback: (level: number) => void, updateInterval?: number);
}
export function create_level_meter(targetDeviceId: string) : AudioLevelMeter;
export function create_device_level_meter(targetDeviceId: string) : AudioLevelMeter;
export function create_recorder() : AudioRecorder;
}

51
native/serverconnection/src/audio/AudioInput.cpp
View File

@ -8,6 +8,7 @@
#include "./AudioGain.h"
#include "./AudioInterleaved.h"
#include "./AudioOutput.h"
#include "./AudioLevelMeter.h"
#include "./processing/AudioProcessor.h"
#include "../logger.h"
#include "AudioEventLoop.h"
@ -192,6 +193,18 @@ std::shared_ptr<AudioConsumer> AudioInput::create_consumer(size_t frame_length)
return result;
}
std::shared_ptr<AudioInputAudioLevelMeter> AudioInput::create_level_meter(bool preprocess) {
auto level_meter = std::make_shared<AudioInputAudioLevelMeter>();
{
std::lock_guard lock{this->consumers_mutex};
auto& list = preprocess ? this->level_meter_preprocess : this->level_meter_postprocess;
list.push_back(level_meter);
}
return level_meter;
}
void AudioInput::allocate_input_buffer_samples(size_t samples) {
const auto expected_byte_size = samples * this->channel_count_ * sizeof(float);
if(expected_byte_size > this->input_buffer.capacity()) {
@ -267,12 +280,12 @@ void AudioInput::process_audio() {
* It's save to mutate the current memory.
* If overflows occur it could lead to wired artifacts but all memory access is save.
*/
this->process_audio_chunk((void*) input);
this->process_audio_chunk((float*) input);
this->input_buffer.advance_read_ptr(chunk_sample_count * this->channel_count_ * sizeof(float));
}
}
void AudioInput::process_audio_chunk(void *chunk) {
void AudioInput::process_audio_chunk(float *chunk) {
constexpr static auto kTempSampleBufferSize{1024 * 8};
constexpr static auto kMaxChannelCount{32};
@ -282,10 +295,11 @@ void AudioInput::process_audio_chunk(void *chunk) {
assert(memset(temp_sample_buffer, 0, sizeof(float) * kTempSampleBufferSize));
assert(memset(out_sample_buffer, 0, sizeof(float) * kTempSampleBufferSize));
this->invoke_level_meter(true, chunk, this->channel_count_, chunk_sample_count);
if(auto processor{this->audio_processor_}; processor) {
assert(kTempSampleBufferSize >= chunk_sample_count * this->channel_count_ * sizeof(float));
audio::deinterleave(temp_sample_buffer, (const float*) chunk, this->channel_count_, chunk_sample_count);
audio::deinterleave(temp_sample_buffer, chunk, this->channel_count_, chunk_sample_count);
webrtc::StreamConfig stream_config{(int) this->sample_rate_, this->channel_count_};
float* channel_ptr[kMaxChannelCount];
@ -304,8 +318,8 @@ void AudioInput::process_audio_chunk(void *chunk) {
chunk = out_sample_buffer;
}
/* TODO: Is this even needed if we have the processor? */
audio::apply_gain(chunk, this->channel_count_, chunk_sample_count, this->volume_);
this->invoke_level_meter(false, out_sample_buffer, this->channel_count_, chunk_sample_count);
auto begin = std::chrono::system_clock::now();
for(const auto& consumer : this->consumers()) {
@ -322,3 +336,32 @@ void AudioInput::process_audio_chunk(void *chunk) {
void AudioInput::EventLoopCallback::event_execute(const chrono::system_clock::time_point &point) {
this->input->process_audio();
}
void AudioInput::invoke_level_meter(bool preprocess, const float *buffer, size_t channel_count, size_t sample_size) {
std::vector<std::shared_ptr<AudioInputAudioLevelMeter>> level_meters{};
level_meters.reserve(10);
{
std::lock_guard lock{this->consumers_mutex};
auto& list = preprocess ? this->level_meter_preprocess : this->level_meter_postprocess;
level_meters.reserve(list.size());
list.erase(std::remove_if(list.begin(), list.end(), [&](const std::weak_ptr<AudioInputAudioLevelMeter>& weak_meter) {
auto meter = weak_meter.lock();
if(!meter) {
return true;
}
level_meters.push_back(meter);
return false;
}), list.end());
}
for(const auto& level_meter : level_meters) {
if(!level_meter->active) {
continue;
}
level_meter->analyze_buffer(buffer, channel_count, sample_size);
}
}

11
native/serverconnection/src/audio/AudioInput.h
View File

@ -15,7 +15,7 @@ namespace tc::audio {
class InputReframer;
class AudioResampler;
class AudioProcessor;
class AudioInputSource;
class AudioInputAudioLevelMeter;
class AudioConsumer {
friend class AudioInput;
@ -37,7 +37,6 @@ namespace tc::audio {
};
class AudioInput : public AudioDeviceRecord::Consumer {
friend class AudioInputSource;
public:
AudioInput(size_t /* channels */, size_t /* sample rate */);
virtual ~AudioInput();
@ -52,6 +51,7 @@ namespace tc::audio {
[[nodiscard]] std::vector<std::shared_ptr<AudioConsumer>> consumers();
[[nodiscard]] std::shared_ptr<AudioConsumer> create_consumer(size_t /* frame size */);
[[nodiscard]] std::shared_ptr<AudioInputAudioLevelMeter> create_level_meter(bool /* pre process */);
[[nodiscard]] inline auto audio_processor() { return this->audio_processor_; }
@ -78,13 +78,16 @@ namespace tc::audio {
void consume(const void *, size_t, size_t) override;
void process_audio();
void process_audio_chunk(void *);
void process_audio_chunk(float *);
size_t const channel_count_;
size_t const sample_rate_;
std::mutex consumers_mutex{};
std::deque<std::weak_ptr<AudioConsumer>> consumers_{};
std::deque<std::weak_ptr<AudioInputAudioLevelMeter>> level_meter_preprocess{};
std::deque<std::weak_ptr<AudioInputAudioLevelMeter>> level_meter_postprocess{};
std::recursive_mutex input_source_lock{};
std::shared_ptr<EventLoopCallback> event_loop_entry{};
@ -102,5 +105,7 @@ namespace tc::audio {
void allocate_input_buffer_samples(size_t /* sample count */);
[[nodiscard]] inline auto chunk_sample_count() const { return (kChunkSizeMs * this->sample_rate_) / 1000; }
void invoke_level_meter(bool /* preprocess */, const float* /* buffer */, size_t /* channel count */, size_t /* sample size */);
};
}

95
native/serverconnection/src/audio/AudioLevelMeter.cpp
View File

@ -9,18 +9,55 @@
using namespace tc::audio;
AudioLevelMeter::AudioLevelMeter(std::shared_ptr<AudioDevice> target_device) : target_device{std::move(target_device)} {
log_allocate("AudioLevelMeter", this);
AbstractAudioLevelMeter::AbstractAudioLevelMeter() {
log_allocate("AbstractAudioLevelMeter", this);
}
AbstractAudioLevelMeter::~AbstractAudioLevelMeter() {
log_free("AbstractAudioLevelMeter", this);
}
void AbstractAudioLevelMeter::register_observer(Observer *observer) {
std::lock_guard lock{this->mutex};
this->registered_observer.push_back(observer);
}
bool AbstractAudioLevelMeter::unregister_observer(Observer *observer) {
std::lock_guard lock{this->mutex};
auto index = std::find(this->registered_observer.begin(), this->registered_observer.end(), observer);
if(index == this->registered_observer.end()) {
return false;
}
this->registered_observer.erase(index);
return true;
}
void AbstractAudioLevelMeter::analyze_buffer(const float *buffer, size_t channel_count, size_t sample_count) {
auto volume = audio::audio_buffer_level(buffer, channel_count, sample_count);
std::lock_guard lock{this->mutex};
if(volume == this->current_audio_volume) {
return;
}
this->current_audio_volume = volume;
for(auto& observer : this->registered_observer) {
observer->input_level_changed(volume);
}
}
/* For a target device */
InputDeviceAudioLevelMeter::InputDeviceAudioLevelMeter(std::shared_ptr<AudioDevice> target_device) : target_device{std::move(target_device)} {
assert(this->target_device);
}
AudioLevelMeter::~AudioLevelMeter() {
InputDeviceAudioLevelMeter::~InputDeviceAudioLevelMeter() {
this->stop();
log_free("AudioLevelMeter", this);
}
bool AudioLevelMeter::start(std::string &error) {
std::lock_guard lock{this->recorder_mutex};
bool InputDeviceAudioLevelMeter::start(std::string &error) {
std::lock_guard lock{this->mutex};
if(this->recorder_instance) {
return true;
}
@ -40,13 +77,13 @@ bool AudioLevelMeter::start(std::string &error) {
return true;
}
bool AudioLevelMeter::running() const {
std::lock_guard lock{this->recorder_mutex};
bool InputDeviceAudioLevelMeter::running() const {
std::lock_guard lock{this->mutex};
return this->recorder_instance != nullptr;
}
void AudioLevelMeter::stop() {
std::lock_guard lock{this->recorder_mutex};
void InputDeviceAudioLevelMeter::stop() {
std::lock_guard lock{this->mutex};
if(this->recorder_instance) {
this->recorder_instance->remove_consumer(this);
this->recorder_instance->stop_if_possible();
@ -54,33 +91,23 @@ void AudioLevelMeter::stop() {
}
}
void AudioLevelMeter::register_observer(Observer *observer) {
std::lock_guard lock{this->recorder_mutex};
this->registered_observer.push_back(observer);
/* Note the parameter order! */
void InputDeviceAudioLevelMeter::consume(const void *buffer, size_t sample_count, size_t channel_count) {
this->analyze_buffer((const float*) buffer, channel_count, sample_count);
}
bool AudioLevelMeter::unregister_observer(Observer *observer) {
std::lock_guard lock{this->recorder_mutex};
auto index = std::find(this->registered_observer.begin(), this->registered_observer.end(), observer);
if(index == this->registered_observer.end()) {
return false;
}
this->registered_observer.erase(index);
bool AudioInputAudioLevelMeter::start(std::string &) {
std::lock_guard lock{this->mutex};
this->active = true;
return true;
}
/* Note the parameter order! */
void AudioLevelMeter::consume(const void *buffer, size_t sample_count, size_t channel_count) {
auto volume = audio::audio_buffer_level((float*) buffer, channel_count, sample_count);
std::lock_guard lock{this->recorder_mutex};
if(volume == this->current_audio_volume) {
return;
}
this->current_audio_volume = volume;
for(auto& observer : this->registered_observer) {
observer->input_level_changed(volume);
}
void AudioInputAudioLevelMeter::stop() {
std::lock_guard lock{this->mutex};
this->active = false;
}
bool AudioInputAudioLevelMeter::running() const {
std::lock_guard lock{this->mutex};
return this->active;
}

60
native/serverconnection/src/audio/AudioLevelMeter.h
View File

@ -1,37 +1,75 @@
#pragma once
#include "./driver/AudioDriver.h"
namespace tc::audio {
class AudioInput;
/**
* Note: Within the observer callback no methods of the level meter should be called nor the level meter should be destructed.
*/
class AudioLevelMeter : public AudioDeviceRecord::Consumer {
class AbstractAudioLevelMeter {
public:
struct Observer {
public:
virtual void input_level_changed(float /* new level */) = 0;
};
explicit AudioLevelMeter(std::shared_ptr<AudioDevice> /* target device */);
virtual ~AudioLevelMeter();
explicit AbstractAudioLevelMeter();
virtual ~AbstractAudioLevelMeter();
[[nodiscard]] bool start(std::string& /* error */);
void stop();
[[nodiscard]] bool running() const;
[[nodiscard]] virtual bool start(std::string& /* error */) = 0;
virtual void stop() = 0;
[[nodiscard]] virtual bool running() const = 0;
[[nodiscard]] inline float current_volume() const { return this->current_audio_volume; }
void register_observer(Observer* /* observer */);
bool unregister_observer(Observer* /* observer */);
private:
std::shared_ptr<AudioDevice> target_device{};
mutable std::mutex recorder_mutex{};
std::shared_ptr<AudioDeviceRecord> recorder_instance{};
protected:
mutable std::mutex mutex{};
std::vector<Observer*> registered_observer{};
float current_audio_volume{0.f};
void analyze_buffer(const float* /* buffer */, size_t /* channel count */, size_t /* sample count */);
};
/**
* This audio level meter operates directly on the raw input device without any processing.
*/
class InputDeviceAudioLevelMeter : public AbstractAudioLevelMeter, public AudioDeviceRecord::Consumer {
public:
explicit InputDeviceAudioLevelMeter(std::shared_ptr<AudioDevice> /* target device */);
~InputDeviceAudioLevelMeter() override;
bool start(std::string &string) override;
void stop() override;
bool running() const override;
private:
std::shared_ptr<AudioDevice> target_device{};
std::shared_ptr<AudioDeviceRecord> recorder_instance{};
void consume(const void *, size_t, size_t) override;
};
class AudioInputAudioLevelMeter : public AbstractAudioLevelMeter {
friend class AudioInput;
public:
AudioInputAudioLevelMeter() = default;
~AudioInputAudioLevelMeter() override = default;
bool start(std::string &string) override;
void stop() override;
bool running() const override;
private:
bool active{false};
};
}

1
native/serverconnection/src/audio/js/AudioConsumer.h
View File

@ -25,7 +25,6 @@ namespace tc::audio {
class AudioConsumerWrapper : public Nan::ObjectWrap {
friend class AudioRecorderWrapper;
constexpr static auto kMaxChannelCount{2};
public:
static NAN_MODULE_INIT(Init);
static NAN_METHOD(NewInstance);

9
native/serverconnection/src/audio/js/AudioLevelMeter.cpp
View File

@ -6,7 +6,6 @@
#include <include/NanStrings.h>
#include "./AudioLevelMeter.h"
#include "../AudioLevelMeter.h"
#include "../../logger.h"
using namespace tc::audio;
using namespace tc::audio::recorder;
@ -21,7 +20,7 @@ NAN_MODULE_INIT(AudioLevelMeterWrapper::Init) {
Nan::SetPrototypeMethod(klass, "stop", AudioLevelMeterWrapper::stop);
Nan::SetPrototypeMethod(klass, "set_callback", AudioLevelMeterWrapper::set_callback);
Nan::Set(target, Nan::LocalStringUTF8("create_level_meter"), Nan::GetFunction(Nan::New<v8::FunctionTemplate>(AudioLevelMeterWrapper::create_level_meter)).ToLocalChecked());
Nan::Set(target, Nan::LocalStringUTF8("create_device_level_meter"), Nan::GetFunction(Nan::New<v8::FunctionTemplate>(AudioLevelMeterWrapper::create_device_level_meter)).ToLocalChecked());
constructor().Reset(Nan::GetFunction(klass).ToLocalChecked());
}
@ -32,7 +31,7 @@ NAN_METHOD(AudioLevelMeterWrapper::NewInstance) {
}
}
NAN_METHOD(AudioLevelMeterWrapper::create_level_meter) {
NAN_METHOD(AudioLevelMeterWrapper::create_device_level_meter) {
if(info.Length() != 1 || !info[0]->IsString()) {
Nan::ThrowError("invalid arguments");
return;
@ -52,13 +51,13 @@ NAN_METHOD(AudioLevelMeterWrapper::create_level_meter) {
return;
}
auto wrapper = new AudioLevelMeterWrapper(std::make_shared<AudioLevelMeter>(target_device));
auto wrapper = new AudioLevelMeterWrapper(std::make_shared<InputDeviceAudioLevelMeter>(target_device));
auto js_object = Nan::NewInstance(Nan::New(AudioLevelMeterWrapper::constructor())).ToLocalChecked();
wrapper->wrap(js_object);
info.GetReturnValue().Set(js_object);
}
AudioLevelMeterWrapper::AudioLevelMeterWrapper(std::shared_ptr<AudioLevelMeter> handle) : handle{std::move(handle)} {
AudioLevelMeterWrapper::AudioLevelMeterWrapper(std::shared_ptr<AbstractAudioLevelMeter> handle) : handle{std::move(handle)} {
assert(this->handle);
memset(&this->update_timer, 0, sizeof(this->update_timer));

10
native/serverconnection/src/audio/js/AudioLevelMeter.h
View File

@ -4,7 +4,7 @@
#include <atomic>
namespace tc::audio {
class AudioLevelMeter;
class AbstractAudioLevelMeter;
}
namespace tc::audio::recorder {
@ -13,14 +13,14 @@ namespace tc::audio::recorder {
/* Static JavaScript methods */
static NAN_MODULE_INIT(Init);
static NAN_METHOD(NewInstance);
static NAN_METHOD(create_device_level_meter);
static inline Nan::Persistent<v8::Function> & constructor() {
static Nan::Persistent<v8::Function> my_constructor;
return my_constructor;
}
static NAN_METHOD(create_level_meter);
explicit AudioLevelMeterWrapper(std::shared_ptr<AudioLevelMeter>);
explicit AudioLevelMeterWrapper(std::shared_ptr<AbstractAudioLevelMeter>);
~AudioLevelMeterWrapper() override;
/* JavaScript member methods */
@ -35,7 +35,7 @@ namespace tc::audio::recorder {
private:
static void timer_callback(uv_timer_t*);
std::shared_ptr<AudioLevelMeter> handle{};
std::shared_ptr<AbstractAudioLevelMeter> handle{};
/* Access only within the js event loop */
uv_timer_t update_timer{};

31
native/serverconnection/src/audio/js/AudioRecorder.cpp
View File

@ -1,10 +1,12 @@
#include <utility>
#include <NanStrings.h>
#include "AudioRecorder.h"
#include "AudioConsumer.h"
#include "./AudioRecorder.h"
#include "./AudioConsumer.h"
#include "./AudioLevelMeter.h"
#include "./AudioProcessor.h"
#include "../AudioInput.h"
#include "../AudioLevelMeter.h"
#include "../../logger.h"
using namespace std;
@ -290,4 +292,29 @@ NAN_METHOD(AudioRecorderWrapper::get_audio_processor) {
auto wrapper = new AudioProcessorWrapper(processor);
wrapper->wrap(js_object);
info.GetReturnValue().Set(js_object);
}
NAN_METHOD(AudioRecorderWrapper::create_level_meter) {
if(info.Length() != 1 || !info[0]->IsString()) {
Nan::ThrowError("invalid argument");
return;
}
auto mode = *Nan::Utf8String{info[0]};
bool preprocess;
if(mode == std::string_view{"pre-process"}) {
preprocess = true;
} else if(mode == std::string_view{"post-process"}) {
preprocess = false;
} else {
Nan::ThrowError("invalid first argument");
return;
}
auto handle = ObjectWrap::Unwrap<AudioRecorderWrapper>(info.Holder());
auto wrapper = new AudioLevelMeterWrapper(handle->input_->create_level_meter(preprocess));
auto js_object = Nan::NewInstance(Nan::New(AudioLevelMeterWrapper::constructor())).ToLocalChecked();
wrapper->wrap(js_object);
info.GetReturnValue().Set(js_object);
}

1
native/serverconnection/src/audio/js/AudioRecorder.h
View File

@ -41,6 +41,7 @@ namespace tc::audio {
static NAN_METHOD(_get_volume);
static NAN_METHOD(get_audio_processor);
static NAN_METHOD(create_level_meter);
std::shared_ptr<AudioConsumerWrapper> create_consumer();
void delete_consumer(const AudioConsumerWrapper*);