TeaSpeak
/
TeaSpeak-Client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added native audio sounds

This commit is contained in:
WolverinDEV 2020-03-18 23:32:57 +01:00
parent 6f9544e7f6
commit d918a822e2
26 changed files with 725 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
modules/renderer/audio/sounds.ts Normal file
View File

@ -0,0 +1,31 @@
window["require_setup"](module);
// <reference types="../imports/import_shared.d.ts" />
/// <reference types="../../modules/renderer/imports/imports_shared.d.ts" />
import {audio as naudio} from "teaclient_connection";
import * as paths from "path";
namespace audio.sounds {
export async function play_sound(file: sound.SoundFile) : Promise<void> {
await new Promise((resolve, reject) => {
let pathname = paths.dirname(location.pathname);
if(pathname[0] === '/' && pathname[2] === ':') //e.g.: /C:/test...
pathname = pathname.substr(1);
const path = paths.join(pathname, file.path);
console.log(path);
naudio.sounds.playback_sound({
callback: (result, message) => {
if(result == naudio.sounds.PlaybackResult.SUCCEEDED)
resolve();
else
reject(naudio.sounds.PlaybackResult[result].toLowerCase() + ": " + message);
},
file: path,
volume: file.volume
});
});
}
}
Object.assign(window["audio"] || (window["audio"] = {} as any), audio);

7
modules/renderer/index.ts
View File

@ -189,11 +189,11 @@ const module_loader_setup = async () => {
};
const load_basic_modules = async () => {
console.dir(require("./audio/AudioPlayer")); /* setup audio */
console.dir(require("./audio/AudioRecorder")); /* setup audio */
require("./logger");
require("./audio/AudioPlayer"); /* setup audio */
require("./audio/AudioRecorder"); /* setup audio */
require("./audio/sounds"); /* setup audio */
};
const load_modules = async () => {
@ -216,6 +216,7 @@ const load_modules = async () => {
console.dir(error);
throw error;
}
try {
const helper = require("./icon-helper");
await helper.initialize();

8
native/CMakeLists.txt
View File

@ -21,11 +21,11 @@ message("Module path: ${CMAKE_MODULE_PATH}")
#Setup NodeJS
function(setup_nodejs)
set(NodeJS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
#set(NODEJS_URL "https://atom.io/download/atom-shell")
#set(NODEJS_VERSION "v8.0.0")
set(NODEJS_URL "https://atom.io/download/atom-shell")
set(NODEJS_VERSION "v8.0.0")
set(NODEJS_URL "https://nodejs.org/download/release/")
set(NODEJS_VERSION "v12.13.0")
#set(NODEJS_URL "https://nodejs.org/download/release/")
#set(NODEJS_VERSION "v12.13.0")
find_package(NodeJS REQUIRED)

2
native/dist/ext_nan/include/NanEventCallback.h vendored
View File

@ -127,7 +127,7 @@ namespace Nan {
* @returns true if the callback will be called even
* if the handle has been destroyed
*/
bool option_destroyed_execute() const {
[[nodiscard]] bool option_destroyed_execute() const {
if(!this->handle)
std::__throw_logic_error("missing handle");
return this->handle->handle->option_destroy_run;

4
native/dist/ext_nan/include/NanGet.h vendored
View File

@ -27,8 +27,8 @@ namespace Nan {
}
template <class T, typename K>
inline v8::Local<T> GetLocal(v8::Local<v8::Object> object, K key) {
return Nan::Get<T, K>(object, key).FromMaybe(v8::Local<T>{});
inline v8::Local<T> GetLocal(v8::Local<v8::Object> object, K key, const v8::Local<T>& default_value = v8::Local<T>{}) {
return Nan::Get<T, K>(object, key).FromMaybe(default_value);
}
template <typename T>

8
native/serverconnection/CMakeLists.txt
View File

@ -16,6 +16,7 @@ set(SOURCE_FILES
src/audio/AudioInput.cpp
src/audio/AudioResampler.cpp
src/audio/AudioReframer.cpp
src/audio/AudioEventLoop.cpp
src/audio/filter/FilterVad.cpp
src/audio/filter/FilterThreshold.cpp
@ -25,7 +26,9 @@ set(SOURCE_FILES
src/audio/codec/OpusConverter.cpp
src/audio/driver/AudioDriver.cpp
)
src/audio/sounds/SoundPlayer.cpp
src/audio/file/wav.cpp
)
set(NODEJS_SOURCE_FILES
src/bindings.cpp
@ -38,7 +41,6 @@ set(NODEJS_SOURCE_FILES
src/connection/ProtocolHandlerPackets.cpp
src/connection/ProtocolHandlerCommands.cpp
src/connection/audio/AudioSender.cpp
src/connection/audio/AudioEventLoop.cpp
src/connection/audio/VoiceConnection.cpp
src/connection/audio/VoiceClient.cpp
@ -180,4 +182,4 @@ target_link_libraries(Audio-Test-2 ${REQUIRED_LIBRARIES})
add_executable(HW-UID-Test src/hwuid.cpp)
target_link_libraries(HW-UID-Test
${REQUIRED_LIBRARIES}
)
)

18
native/serverconnection/exports/exports.d.ts vendored
View File

@ -250,6 +250,24 @@ declare module "teaclient_connection" {
export function create_recorder() : AudioRecorder;
}
export namespace sounds {
export enum PlaybackResult {
SUCCEEDED,
CANCELED,
SOUND_NOT_INITIALIZED,
FILE_OPEN_ERROR,
PLAYBACK_ERROR
}
export interface PlaybackSettings {
file: string;
volume?: number;
callback?: (result: PlaybackResult, message: string) => void;
}
export function playback_sound(settings: PlaybackSettings) : number;
export function cancel_playback(playback: number);
}
export function initialize(callback: () => any);
export function initialized() : boolean;
export function available_devices() : AudioDevice[];

1
native/serverconnection/src/connection/audio/AudioEventLoop.cpp → native/serverconnection/src/audio/AudioEventLoop.cpp
View File

@ -23,5 +23,4 @@ void audio::shutdown_event_loops() {
audio::decode_event_loop = nullptr;
}
audio::encode_event_loop = nullptr;
}

11
native/serverconnection/src/audio/AudioEventLoop.h Normal file
View File

@ -0,0 +1,11 @@
#pragma once
#include "../EventLoop.h"
namespace tc::audio {
extern event::EventExecutor* encode_event_loop;
extern event::EventExecutor* decode_event_loop;
extern void init_event_loops();
extern void shutdown_event_loops();
}

9
native/serverconnection/src/audio/AudioMerger.cpp
View File

@ -18,7 +18,14 @@ inline constexpr float merge_ab(float a, float b) {
a += 1;
b += 1;
return (2 * (a + b) - a * b - 2) - 1;
auto result = (2 * (a + b) - a * b - 2) - 1;
if(result > 1) {
result = 1;
} else if(result < -1) {
result = -1;
}
return result;
}
static_assert(merge_ab(1, 0) == 1);

18
native/serverconnection/src/audio/AudioMerger.h
View File

@ -2,17 +2,13 @@
#include <cstring>
namespace tc {
namespace audio {
namespace merge {
/* the result buffer could be equal to one of the source buffers to prevent unnecessary allocations
* Note: The sample order is irrelevant
*/
extern bool merge_sources(void* /* result */, void* /* source a */, void* /* source b */, size_t /* channels */, size_t /* samples */);
namespace tc::audio::merge {
/* the result buffer could be equal to one of the source buffers to prevent unnecessary allocations
* Note: The sample order is irrelevant
*/
extern bool merge_sources(void* /* result */, void* /* source a */, void* /* source b */, size_t /* channels */, size_t /* samples */);
extern bool merge_n_sources(void* /* result */, void** /* sources */, size_t /* size_t sources count */, size_t /* channels */, size_t /* samples */);
extern bool merge_n_sources(void* /* result */, void** /* sources */, size_t /* size_t sources count */, size_t /* channels */, size_t /* samples */);
extern bool merge_channels_interleaved(void* /* result */, size_t /* result channels */, const void* /* source */, size_t /* source channels */, size_t /* samples */);
}
}
extern bool merge_channels_interleaved(void* /* result */, size_t /* result channels */, const void* /* source */, size_t /* source channels */, size_t /* samples */);
}

15
native/serverconnection/src/audio/AudioOutput.cpp
View File

@ -31,6 +31,9 @@ ssize_t AudioOutputSource::pop_samples(void *buffer, size_t samples) {
const auto byte_length = (samples - written) * sizeof(float) * this->channel_count;
if(buffer)memcpy((char*) buffer + written_bytes, this->buffer.read_ptr(), byte_length);
this->buffer.advance_read_ptr(byte_length);
if(this->on_read)
this->on_read();
return samples;
} else {
const auto byte_length = available_samples * sizeof(float) * this->channel_count;
@ -47,9 +50,9 @@ ssize_t AudioOutputSource::pop_samples(void *buffer, size_t samples) {
if(buffer)memset(buffer, 0, (samples - written) * sizeof(float) * this->channel_count);
this->buffering = true;
if(this->on_read)
this->on_read();
return written; /* return the written samples */
}
@ -140,8 +143,8 @@ AudioOutput::~AudioOutput() {
this->cleanup_buffers();
}
std::shared_ptr<AudioOutputSource> AudioOutput::create_source() {
auto result = shared_ptr<AudioOutputSource>(new AudioOutputSource(this, this->_channel_count, this->_sample_rate));
std::shared_ptr<AudioOutputSource> AudioOutput::create_source(ssize_t buf) {
auto result = shared_ptr<AudioOutputSource>(new AudioOutputSource(this, this->_channel_count, this->_sample_rate, buf));
{
lock_guard lock(this->sources_lock);
this->_sources.push_back(result);
@ -272,8 +275,12 @@ void AudioOutput::fill_buffer(void *output, size_t out_frame_count, size_t out_c
/* this->source_buffer could hold the amount of resampled data (checked above) */
auto resampled_samples = this->_resampler->process(this->source_buffer, this->source_buffer, local_frame_count);
if(resampled_samples <= 0) {
log_warn(category::audio, tr("Failed to resample audio data for client ({})"));
goto clear_buffer_exit;
}
if(resampled_samples != out_frame_count) {
if(resampled_samples > out_frame_count) {
if((size_t) resampled_samples > out_frame_count) {
const auto diff_length = resampled_samples - out_frame_count;
log_warn(category::audio, tr("enqueuing {} samples"), diff_length);
const auto overhead_buffer_offset = this->resample_overhead_samples * sizeof(float) * this->_channel_count;

12
native/serverconnection/src/audio/AudioOutput.h
View File

@ -44,6 +44,11 @@ namespace tc::audio {
return max_samples;
}
/* samples which needs to be played*/
[[nodiscard]] inline size_t current_latency() const {
return this->buffer.fill_count() / this->channel_count / sizeof(float);
}
bool buffering{true};
size_t min_buffered_samples{0};
size_t max_buffered_samples{0};
@ -60,8 +65,9 @@ namespace tc::audio {
ssize_t enqueue_samples(const void * /* input buffer */, size_t /* sample count */);
ssize_t enqueue_samples_no_interleave(const void * /* input buffer */, size_t /* sample count */);
private:
AudioOutputSource(AudioOutput* handle, size_t channel_count, size_t sample_rate) :
handle(handle), channel_count(channel_count), sample_rate(sample_rate), buffer{channel_count * sample_rate * sizeof(float)} {
AudioOutputSource(AudioOutput* handle, size_t channel_count, size_t sample_rate, ssize_t max_buffer_sample_count = -1) :
handle(handle), channel_count(channel_count), sample_rate(sample_rate),
buffer{max_buffer_sample_count == -1 ? channel_count * sample_rate * sizeof(float) : max_buffer_sample_count * channel_count * sizeof(float)} {
}
tc::ring_buffer buffer;
@ -82,7 +88,7 @@ namespace tc::audio {
return this->_sources;
}
std::shared_ptr<AudioOutputSource> create_source();
std::shared_ptr<AudioOutputSource> create_source(ssize_t /* buffer sample size */ = -1);
void delete_source(const std::shared_ptr<AudioOutputSource>& /* source */);
inline size_t channel_count() { return this->_channel_count; }

3
native/serverconnection/src/audio/AudioResampler.cpp
View File

@ -1,4 +1,3 @@
#include <iostream>
#include <cstring>
#include "AudioResampler.h"
#include "../logger.h"
@ -6,7 +5,7 @@
using namespace std;
using namespace tc::audio;
AudioResampler::AudioResampler(size_t irate, size_t orate, size_t channels) : _input_rate(irate), _output_rate(orate), _channels(channels) {
AudioResampler::AudioResampler(size_t irate, size_t orate, size_t channels) : _input_rate{irate}, _output_rate{orate}, _channels{channels} {
if(this->input_rate() != this->output_rate()) {
soxr_error_t error;
this->soxr_handle = soxr_create((double) this->_input_rate, (double) this->_output_rate, (unsigned) this->_channels, &error, nullptr, nullptr, nullptr);

180
native/serverconnection/src/audio/file/wav.cpp Normal file
View File

@ -0,0 +1,180 @@
//
// Created by WolverinDEV on 18/03/2020.
//
#include "./wav.h"
#include "../../logger.h"
using namespace tc::audio::file;
struct WAVFileHeader {
/* RIFF Chunk Descriptor */
uint8_t RIFF[4]; // RIFF Header Magic header
uint32_t ChunkSize; // RIFF Chunk Size
uint8_t WAVE[4]; // WAVE Header
/* "fmt" sub-chunk */
uint8_t fmt[4]; // FMT header
uint32_t Subchunk1Size; // Size of the fmt chunk
uint16_t AudioFormat; // Audio format 1=PCM,6=mulaw,7=alaw, 257=IBM Mu-Law, 258=IBM A-Law, 259=ADPCM
uint16_t NumOfChan; // Number of channels 1=Mono 2=Sterio
uint32_t SamplesPerSec; // Sampling Frequency in Hz
uint32_t bytesPerSec; // bytes per second
uint16_t blockAlign; // 2=16-bit mono, 4=16-bit stereo
uint16_t bitsPerSample; // Number of bits per sample
};
static_assert(sizeof(WAVFileHeader) == 0x24);
struct WAFFileChunk {
uint8_t id[4];
uint32_t size;
};
static_assert(sizeof(WAFFileChunk) == 0x08);
WAVReader::WAVReader(std::string file) : file_path_{std::move(file)} {}
WAVReader::~WAVReader() {
this->close_file();
}
FileOpenResult WAVReader::open_file(std::string& error) {
this->is_.open(this->file_path_, std::ifstream::in | std::ifstream::binary);
if(!this->is_) {
error = tr("failed to open file");
return FileOpenResult::OPEN_RESULT_ERROR;
}
WAVFileHeader header{};
if(!this->is_.read((char*) &header, sizeof(header))) {
error = tr("failed to read wav header");
return FileOpenResult::OPEN_RESULT_ERROR;
}
if(memcmp(header.RIFF, "RIFF", 4) != 0) {
error = tr("invalid RIFF header");
return FileOpenResult::OPEN_RESULT_ERROR;
}
if(memcmp(header.WAVE, "WAVE", 4) != 0) {
error = tr("invalid WAVE header");
return FileOpenResult::OPEN_RESULT_ERROR;
}
if(memcmp(header.fmt, "fmt ", 4) != 0) {
error = tr("invalid WAVE header");
return FileOpenResult::OPEN_RESULT_ERROR;
}
if(header.AudioFormat != 1) {
error = tr("Only PCM has been supported. WAV file does not contains PCM data.");
return FileOpenResult::OPEN_RESULT_FORMAT_UNSUPPORTED;
}
if(header.bytesPerSec != (header.NumOfChan * header.SamplesPerSec * header.bitsPerSample) / 8) {
error = tr("inconsistent WAV header");
return FileOpenResult::OPEN_RESULT_INVALID_FORMAT;
}
if(header.bitsPerSample != 8 && header.bitsPerSample != 16 && header.bitsPerSample != 24) {
error = tr("unsupported bitrate");
return FileOpenResult::OPEN_RESULT_FORMAT_UNSUPPORTED;
}
if(header.NumOfChan != 2 && header.NumOfChan != 1) {
error = tr("unsupported channel count");
return FileOpenResult::OPEN_RESULT_FORMAT_UNSUPPORTED;
}
WAFFileChunk chunk{};
while(true) {
if(!this->is_.read((char*) &chunk, sizeof(chunk))) {
error = tr("failed to read chunks until data chunk");
return FileOpenResult::OPEN_RESULT_ERROR;
}
if(memcmp(chunk.id, "data ", 4) == 0)
break;
this->is_.seekg(chunk.size, std::ifstream::cur);
}
this->current_sample_offset_ = 0;
this->bytes_per_sample = header.bitsPerSample / 8;
this->total_samples_ = chunk.size / this->bytes_per_sample / header.NumOfChan;
this->sample_rate_ = header.SamplesPerSec;
this->channels_ = header.NumOfChan;
return FileOpenResult::OPEN_RESULT_SUCCESS;
}
void WAVReader::close_file() {
this->total_samples_ = 0;
this->bytes_per_sample = 0;
this->sample_rate_ = 0;
this->channels_ = 0;
this->is_.close();
}
float _8bit_float_convert(const uint8_t* buffer) {
int16_t value = buffer[0] & 0xFFU;
return (float) (value - 127) * (1.0f / 127.0f);
}
float _16bit_float_convert(const uint8_t* buffer) {
int16_t value = *reinterpret_cast<const int16_t*>(buffer);
return (float) value / 32767.f;
}
float _24bit_float_convert(const uint8_t* buffer) {
#if 0
int32_t value = (*reinterpret_cast<const uint32_t*>(buffer) & 0xFFFFFFU) << 8;
return (float) (value - 1073741824) / 1073741824.f; //2147483648 / 2
#endif
#if 1
int32_t value = ((uint32_t) buffer[2] << 16U) | ((uint32_t) buffer[1] << 8U) | ((uint32_t) buffer[0] << 0U);
if (value & 0x800000) // if the 24th bit is set, this is a negative number in 24-bit world
value = value | ~0xFFFFFF; // so make sure sign is extended to the 32 bit float
auto result = (float) value / (float) 8388608.f; //8388608
return result;
#endif
}
static std::array<float(*)(const uint8_t*), 3> pcm_to_float_converters{
_8bit_float_convert,
_16bit_float_convert,
_24bit_float_convert
};
ReadResult WAVReader::read(void *buffer, size_t* samples) {
auto fbuffer = (float*) buffer;
const auto max_sample_count = this->total_samples_ - this->current_sample_offset_;
const auto max_samples = std::min(*samples, max_sample_count);
if(max_samples == 0) {
if(max_sample_count == 0) return ReadResult::READ_RESULT_EOF;
return ReadResult::READ_RESULT_SUCCESS;
}
constexpr size_t sbuffer_size{1536}; /* must be dividable by 24, 165 and 8 bit! As well by two channels so 6, 4 and 2 byte to avoid to mess up one frame */
uint8_t sbuffer[sbuffer_size];
size_t samples_read{0};
auto fconverter = pcm_to_float_converters[(this->bytes_per_sample - 1) & 0x3U];
while(samples_read < max_samples) {
const auto block_byte_length{std::min(sbuffer_size, (max_samples - samples_read) * this->bytes_per_sample * this->channels_)};
if(!this->is_.read((char*) sbuffer, block_byte_length))
return ReadResult::READ_RESULT_UNRECOVERABLE_ERROR;
uint8_t* sbufferptr = sbuffer;
uint8_t* sbuferendptr = sbuffer + block_byte_length;
while(sbufferptr != sbuferendptr) {
*fbuffer++ = fconverter(sbufferptr);
sbufferptr += this->bytes_per_sample;
}
samples_read += block_byte_length / (this->bytes_per_sample * this->channels_);
}
*samples = samples_read;
this->current_sample_offset_ += samples_read;
return ReadResult::READ_RESULT_SUCCESS;
}

51
native/serverconnection/src/audio/file/wav.h Normal file
View File

@ -0,0 +1,51 @@
#pragma once
#include <string>
#include <fstream>
namespace tc::audio::file {
enum FileOpenResult {
OPEN_RESULT_SUCCESS,
OPEN_RESULT_INVALID_FORMAT,
OPEN_RESULT_FORMAT_UNSUPPORTED,
OPEN_RESULT_ERROR
};
enum ReadResult {
READ_RESULT_SUCCESS,
READ_RESULT_EOF,
READ_RESULT_UNRECOVERABLE_ERROR
};
class WAVReader {
public:
explicit WAVReader(std::string /* path */);
~WAVReader();
[[nodiscard]] inline const std::string& file_path() const { return this->file_path_; }
[[nodiscard]] FileOpenResult open_file(std::string& /* error */);
void close_file();
[[nodiscard]] inline size_t channels() const { return this->channels_; }
[[nodiscard]] inline size_t sample_rate() const { return this->sample_rate_; }
[[nodiscard]] inline size_t total_samples() const { return this->total_samples_; }
[[nodiscard]] inline size_t current_sample_offset() const { return this->current_sample_offset_; }
/**
* Audio data in interleaved floats reachting from [-1;1].
* Must contains at least channels * sizeof(float) * sample_count bytes
*/
[[nodiscard]] ReadResult read(void* /* target buffer */, size_t* /* sample count */);
private:
const std::string file_path_;
std::ifstream is_{};
size_t channels_{0};
size_t sample_rate_{0};
size_t total_samples_{0};
size_t current_sample_offset_{0};
uint8_t bytes_per_sample{0}; /* 1, 2 or 3 (8, 16, 24 bit) */
};
}

309
native/serverconnection/src/audio/sounds/SoundPlayer.cpp Normal file
View File

@ -0,0 +1,309 @@
//
// Created by WolverinDEV on 18/03/2020.
//
#include <cassert>
#include <NanGet.h>
#include <NanEventCallback.h>
#include "./SoundPlayer.h"
#include "../AudioOutput.h"
#include "../file/wav.h"
#include "../../EventLoop.h"
#include "../../logger.h"
#include "../AudioEventLoop.h"
#include "../AudioResampler.h"
#include "../AudioMerger.h"
#ifdef max
#undef max
#endif
using namespace tc::audio;
extern tc::audio::AudioOutput* global_audio_output;
namespace tc::audio::sounds {
class FilePlayer : public event::EventEntry, public std::enable_shared_from_this<FilePlayer> {
public:
explicit FilePlayer(PlaybackSettings settings) : settings_{std::move(settings)} {
log_trace(category::memory, tr("Allocated FilePlayer instance at {}"), (void*) this);
}
~FilePlayer() {
this->finalize(true);
log_trace(category::memory, tr("Deleted FilePlayer instance at {}"), (void*) this);
}
[[nodiscard]] inline const PlaybackSettings& settings() const { return this->settings_; }
[[nodiscard]] inline bool is_finished() const { return this->state_ == PLAYER_STATE_UNSET; }
/* should not be blocking! */
bool play() {
if(this->state_ != PLAYER_STATE_UNSET) return false;
this->state_ = PLAYER_STATE_INITIALIZE;
audio::decode_event_loop->schedule(this->shared_from_this());
return true;
}
/* should not be blocking! */
void cancel() {
this->state_ = PLAYER_STATE_CANCELED;
audio::decode_event_loop->schedule(this->shared_from_this());
}
private:
constexpr static auto kBufferChunkTimespan{0.2};
const PlaybackSettings settings_;
std::unique_ptr<file::WAVReader> file_handle{nullptr};
std::unique_ptr<AudioResampler> resampler{nullptr};
std::shared_ptr<audio::AudioOutputSource> output_source;
void* cache_buffer{nullptr};
enum {
PLAYER_STATE_INITIALIZE,
PLAYER_STATE_PLAYING,
PLAYER_STATE_AWAIT_FINISH,
PLAYER_STATE_FINISHED,
PLAYER_STATE_CANCELED,
PLAYER_STATE_UNSET
} state_{PLAYER_STATE_UNSET};
void finalize(bool is_destructor_call) {
if(this->output_source && global_audio_output)
global_audio_output->delete_source(this->output_source);
if(auto buffer{std::exchange(this->cache_buffer, nullptr)}; buffer)
::free(buffer);
if(!is_destructor_call)
audio::decode_event_loop->cancel(this->shared_from_this());
this->state_ = PLAYER_STATE_UNSET;
}
void event_execute(const std::chrono::system_clock::time_point &) override {
if(this->state_ == PLAYER_STATE_INITIALIZE) {
this->file_handle = std::make_unique<file::WAVReader>(this->settings_.file);
std::string error{};
if(auto err{this->file_handle->open_file(error)}; err != file::OPEN_RESULT_SUCCESS) {
if(auto callback{this->settings_.callback}; callback)
callback(PlaybackResult::FILE_OPEN_ERROR, error);
this->finalize(false);
return;
}
if(!global_audio_output) {
if(auto callback{this->settings_.callback}; callback)
callback(PlaybackResult::SOUND_NOT_INITIALIZED, "");
this->finalize(false);
return;
}
this->initialize_playback();
auto max_samples = (size_t)
std::max(this->output_source->sample_rate, this->file_handle->sample_rate()) * kBufferChunkTimespan * 8 *
std::max(this->file_handle->channels(), this->output_source->channel_count);
this->cache_buffer = ::malloc((size_t) (max_samples * sizeof(float)));
if(!this->cache_buffer) {
if(auto callback{this->settings_.callback}; callback)
callback(PlaybackResult::PLAYBACK_ERROR, "failed to allocate cached buffer");
this->finalize(false);
return;
}
this->state_ = PLAYER_STATE_PLAYING;
}
if(this->state_ == PLAYER_STATE_PLAYING) {
if(!this->could_enqueue_next_buffer()) return;
auto samples_to_read = (size_t) (this->file_handle->sample_rate() * kBufferChunkTimespan);
auto errc = this->file_handle->read(this->cache_buffer, &samples_to_read);
switch (errc) {
case file::READ_RESULT_SUCCESS:
break;
case file::READ_RESULT_EOF:
this->state_ = PLAYER_STATE_AWAIT_FINISH;
return;
case file::READ_RESULT_UNRECOVERABLE_ERROR:
if(auto callback{this->settings_.callback}; callback)
callback(PlaybackResult::PLAYBACK_ERROR, "read resulted in an unrecoverable error");
this->finalize(false);
return;
}
if(!merge::merge_channels_interleaved(this->cache_buffer, this->output_source->channel_count, this->cache_buffer, this->file_handle->channels(), samples_to_read)) {
log_warn(category::audio, tr("failed to merge channels for replaying a sound"));
return;
}
auto resampled_samples = this->resampler->process(this->cache_buffer, this->cache_buffer, samples_to_read);
if(resampled_samples <= 0) {
log_warn(category::audio, tr("failed to resample file audio buffer ({})"), resampled_samples);
return;
}
this->output_source->enqueue_samples(this->cache_buffer, resampled_samples);
if(this->could_enqueue_next_buffer())
audio::decode_event_loop->schedule(this->shared_from_this());
} else if(this->state_ == PLAYER_STATE_FINISHED || this->state_ == PLAYER_STATE_CANCELED) {
this->finalize(false);
if(auto callback{this->settings_.callback}; callback)
callback(this->state_ == PLAYER_STATE_CANCELED ? PlaybackResult::CANCELED : PlaybackResult::SUCCEEDED, "");
this->state_ = PLAYER_STATE_UNSET;
return;
}
auto filled_samples = this->output_source->current_latency();
}
void initialize_playback() {
assert(this->file_handle);
assert(global_audio_output);
const auto max_buffer = (size_t) ceil(global_audio_output->sample_rate() * kBufferChunkTimespan * 3);
this->output_source = global_audio_output->create_source(max_buffer);
this->output_source->overflow_strategy = audio::overflow_strategy::ignore;
this->output_source->max_buffered_samples = max_buffer;
this->output_source->min_buffered_samples = (size_t) floor(this->output_source->sample_rate * 0.04);
auto weak_this = this->weak_from_this();
this->output_source->on_underflow = [weak_this](size_t sample_count){
auto self = weak_this.lock();
if(!self) return false;
if(self->state_ == PLAYER_STATE_PLAYING)
log_warn(category::audio, tr("Having an audio underflow while playing a sound."));
else if(self->state_ == PLAYER_STATE_AWAIT_FINISH)
self->state_ = PLAYER_STATE_FINISHED;
audio::decode_event_loop->schedule(self);
return false;
};
this->output_source->on_read = [weak_this] {
auto self = weak_this.lock();
if(!self) return;
if(self->could_enqueue_next_buffer() && self->state_ == PLAYER_STATE_PLAYING)
audio::decode_event_loop->schedule(self);
};
this->output_source->on_overflow = [weak_this](size_t count) {
log_warn(category::audio, tr("Having an audio overflow while playing a sound."));
};
this->resampler = std::make_unique<AudioResampler>(this->file_handle->sample_rate(), this->output_source->sample_rate, this->output_source->channel_count);
}
[[nodiscard]] inline size_t cache_buffer_sample_size() const {
return (size_t) (this->output_source->sample_rate * kBufferChunkTimespan);
}
[[nodiscard]] inline bool could_enqueue_next_buffer() const {
if(!this->output_source) return false;
const auto current_size = this->output_source->current_latency();
const auto max_size = this->output_source->max_buffered_samples;
if(current_size > max_size) return false;
const auto size_left = max_size - current_size;
return size_left >= this->cache_buffer_sample_size();
}
};
std::mutex file_player_mutex{};
std::deque<std::shared_ptr<FilePlayer>> file_players{};
sound_playback_id playback_sound(const PlaybackSettings& settings) {
if(!audio::initialized()) {
settings.callback(PlaybackResult::SOUND_NOT_INITIALIZED, "");
return 0;
}
std::unique_lock fplock{file_player_mutex};
file_players.erase(std::remove_if(file_players.begin(), file_players.end(), [](const auto& player) {
return player->is_finished();
}), file_players.end());
auto player = std::make_shared<FilePlayer>(settings);
file_players.push_back(player);
if(!player->play()) {
if(auto callback{settings.callback}; callback)
callback(PlaybackResult::PLAYBACK_ERROR, "failed to start playback.");
return 0;
}
fplock.unlock();
return (sound_playback_id) &*player;
}
void cancel_playback(const sound_playback_id& id) {
std::unique_lock fplock{file_player_mutex};
auto player_it = std::find_if(file_players.begin(), file_players.end(), [&](const auto& player) { return (sound_playback_id) &*player == id; });
if(player_it == file_players.end()) return;
auto player = *player_it;
file_players.erase(player_it);
fplock.unlock();
player->cancel();
}
}
NAN_METHOD(tc::audio::sounds::playback_sound_js) {
if(info.Length() != 1 || !info[0]->IsObject()) {
Nan::ThrowError("invalid arguments");
return;
}
auto data = info[0].As<v8::Object>();
auto file = Nan::GetLocal<v8::String>(data, "file");
auto volume = Nan::GetLocal<v8::Number>(data, "volume", Nan::New<v8::Number>(1.f));
v8::Local<v8::Function> callback = Nan::GetLocal<v8::Function>(data, "callback");
if(file.IsEmpty() || !file->IsString()) {
Nan::ThrowError("missing file path");
return;
}
if(volume.IsEmpty() || !volume->IsNumber()) {
Nan::ThrowError("invalid volume");
return;
}
PlaybackSettings settings{};
settings.file = *Nan::Utf8String(file);
settings.volume = volume->Value();
if(!callback.IsEmpty()) {
if(!callback->IsFunction()) {
Nan::ThrowError("invalid callback function");
return;
}
Nan::Global<v8::Function> cb{callback};
auto async_callback = Nan::async_callback([cb = std::move(cb)](PlaybackResult result, std::string error) mutable {
Nan::HandleScope scope{};
auto callback = cb.Get(Nan::GetCurrentContext()->GetIsolate()).As<v8::Function>();
cb.Reset();
v8::Local<v8::Value> arguments[2];
arguments[0] = Nan::New<v8::Number>((int) result);
arguments[1] = Nan::LocalStringUTF8(error);
(void) callback->Call(Nan::GetCurrentContext(), Nan::Undefined(), 2, arguments);
}).option_destroyed_execute(true);
settings.callback = [async_callback](PlaybackResult result, const std::string& error) mutable {
async_callback.call_cpy(result, error);
};
}
info.GetReturnValue().Set((uint32_t) playback_sound(settings));
}
NAN_METHOD(tc::audio::sounds::cancel_playback_js) {
if(info.Length() != 1 || !info[0]->IsNumber()) {
Nan::ThrowError("invalid arguments");
return;
}
cancel_playback(info[0].As<v8::Number>()->Value());
}

31
native/serverconnection/src/audio/sounds/SoundPlayer.h Normal file
View File

@ -0,0 +1,31 @@
#pragma once
#include <functional>
#include <string_view>
#include <nan.h>
namespace tc::audio::sounds {
typedef uintptr_t sound_playback_id;
enum struct PlaybackResult {
SUCCEEDED,
CANCELED,
SOUND_NOT_INITIALIZED,
FILE_OPEN_ERROR,
PLAYBACK_ERROR /* has the extra error set */
};
typedef std::function<void(PlaybackResult, const std::string&)> callback_status_t;
struct PlaybackSettings {
std::string file{};
double volume{1.f};
/* ATTENTION: This callback may be called within the audio loop! */
callback_status_t callback{};
};
extern sound_playback_id playback_sound(const PlaybackSettings& /* settings */);
extern void cancel_playback(const sound_playback_id&);
extern NAN_METHOD(playback_sound_js);
extern NAN_METHOD(cancel_playback_js);
}

25
native/serverconnection/src/bindings.cpp
View File

@ -22,7 +22,8 @@
#include "audio/js/AudioRecorder.h"
#include "audio/js/AudioConsumer.h"
#include "audio/js/AudioFilter.h"
#include "connection/audio/AudioEventLoop.h"
#include "audio/AudioEventLoop.h"
#include "audio/sounds/SoundPlayer.h"
#ifndef WIN32
#include <unistd.h>
@ -69,8 +70,8 @@ void testTomMath(){
tc::audio::AudioOutput* global_audio_output;
#define ENUM_SET(object, key, value) \
Nan::DefineOwnProperty(object, Nan::New<v8::String>(key).ToLocalChecked(), Nan::New<v8::Number>(value), v8::DontDelete); \
Nan::Set(object, value, Nan::New<v8::String>(key).ToLocalChecked());
Nan::DefineOwnProperty(object, Nan::New<v8::String>(key).ToLocalChecked(), Nan::New<v8::Number>((uint32_t) value), v8::DontDelete); \
Nan::Set(object, (uint32_t) value, Nan::New<v8::String>(key).ToLocalChecked());
NAN_MODULE_INIT(init) {
logger::initialize_node();
@ -177,6 +178,24 @@ NAN_MODULE_INIT(init) {
audio::recorder::AudioFilterWrapper::Init(namespace_record);
Nan::Set(namespace_audio, Nan::New<v8::String>("record").ToLocalChecked(), namespace_record);
}
{
auto namespace_sounds = Nan::New<v8::Object>();
Nan::Set(namespace_sounds, Nan::LocalString("playback_sound"), Nan::GetFunction(Nan::New<v8::FunctionTemplate>(audio::sounds::playback_sound_js)).ToLocalChecked());
Nan::Set(namespace_sounds, Nan::LocalString("cancel_playback"), Nan::GetFunction(Nan::New<v8::FunctionTemplate>(audio::sounds::cancel_playback_js)).ToLocalChecked());
{
auto enum_object = Nan::New<v8::Object>();
ENUM_SET(enum_object, "SUCCEEDED", audio::sounds::PlaybackResult::SUCCEEDED);
ENUM_SET(enum_object, "CANCELED", audio::sounds::PlaybackResult::CANCELED);
ENUM_SET(enum_object, "SOUND_NOT_INITIALIZED", audio::sounds::PlaybackResult::SOUND_NOT_INITIALIZED);
ENUM_SET(enum_object, "FILE_OPEN_ERROR", audio::sounds::PlaybackResult::FILE_OPEN_ERROR);
ENUM_SET(enum_object, "PLAYBACK_ERROR", audio::sounds::PlaybackResult::PLAYBACK_ERROR);
Nan::DefineOwnProperty(namespace_sounds, Nan::New<v8::String>("PlaybackResult").ToLocalChecked(), enum_object, v8::DontDelete);
}
Nan::Set(namespace_audio, Nan::New<v8::String>("sounds").ToLocalChecked(), namespace_sounds);
}
Nan::Set(target, Nan::New<v8::String>("audio").ToLocalChecked(), namespace_audio);
}

13
native/serverconnection/src/connection/audio/AudioEventLoop.h
View File

@ -1,13 +0,0 @@
#pragma once
#include "../../EventLoop.h"
namespace tc {
namespace audio {
extern event::EventExecutor* encode_event_loop;
extern event::EventExecutor* decode_event_loop;
extern void init_event_loops();
extern void shutdown_event_loops();
}
}

2
native/serverconnection/src/connection/audio/AudioSender.cpp
View File

@ -2,7 +2,7 @@
#include "VoiceConnection.h"
#include "../ServerConnection.h"
#include "../../logger.h"
#include "AudioEventLoop.h"
#include "../../audio/AudioEventLoop.h"
#include "../../audio/AudioMerger.h"
using namespace std;

2
native/serverconnection/src/connection/audio/VoiceClient.cpp
View File

@ -4,7 +4,7 @@
#include "../../audio/codec/OpusConverter.h"
#include "../../audio/AudioMerger.h"
#include "../../audio/js/AudioOutputStream.h"
#include "AudioEventLoop.h"
#include "../../audio/AudioEventLoop.h"
#include "../../logger.h"
using namespace std;

1
native/serverconnection/src/connection/audio/VoiceClient.h
View File

@ -56,7 +56,6 @@ namespace tc {
template<typename _Tp, typename _Up>
friend inline std::shared_ptr<_Tp> std::static_pointer_cast(const std::shared_ptr<_Up>& __r) noexcept;
#endif
//_NODISCARD shared_ptr<_Ty1> static_pointer_cast(shared_ptr<_Ty2>&& _Other) noexcept
public:
struct state {
enum value {

2
native/serverconnection/src/connection/ft/FileTransferObject.cpp
View File

@ -163,7 +163,7 @@ void TransferObjectWrap::do_wrap(v8::Local<v8::Object> object) {
);
Nan::Set(object,
Nan::New<v8::String>("name").ToLocalChecked(),
Nan::New<v8::String>(this->target()->name().c_str()).ToLocalChecked()
Nan::New<v8::String>(this->target()->name().c_str()).ToLocalChecked()
);
if(source) {

2
native/serverconnection/src/logger.h
View File

@ -4,9 +4,9 @@
#include <memory>
#include <string>
#include <string_view>
#include <vector>
#include <functional>
#include <string_view>
namespace tc_logger {
namespace level = spdlog::level;

16
native/serverconnection/test/js/audio.ts
View File

@ -18,6 +18,8 @@ handle.audio.initialize(() => {
const stream = handle.audio.playback.create_stream();
console.log("Own stream: %o", stream);
stream.set_buffer_latency(0.02);
stream.set_buffer_max_latency(0.2);
const recorder = handle.audio.record.create_recorder();
const default_input = handle.audio.available_devices().find(e => e.input_default);
@ -34,6 +36,19 @@ handle.audio.initialize(() => {
});
});
setInterval(() => {
handle.audio.sounds.playback_sound({
file: "D:\\TeaSpeak\\web\\shared\\audio\\speech\\connection.refused.wav",
//file: "D:\\Users\\WolverinDEV\\Downloads\\LRMonoPhase4.wav",
volume: 1,
callback: (result, message) => {
console.log("Result %s: %s", handle.audio.sounds.PlaybackResult[result], message);
}
});
}, 500);
/*
setInterval(() => {
const elements = handle.audio.available_devices().filter(e => e.input_supported);
const dev = elements[Math.floor(Math.random() * elements.length)];
@ -45,4 +60,5 @@ handle.audio.initialize(() => {
});
});
}, 1000);
*/
});