diff --git a/modules/renderer/audio/sounds.ts b/modules/renderer/audio/sounds.ts new file mode 100644 index 0000000..0d2b3e0 --- /dev/null +++ b/modules/renderer/audio/sounds.ts @@ -0,0 +1,31 @@ +window["require_setup"](module); +// +/// + +import {audio as naudio} from "teaclient_connection"; +import * as paths from "path"; + +namespace audio.sounds { + export async function play_sound(file: sound.SoundFile) : Promise { + await new Promise((resolve, reject) => { + let pathname = paths.dirname(location.pathname); + if(pathname[0] === '/' && pathname[2] === ':') //e.g.: /C:/test... + pathname = pathname.substr(1); + const path = paths.join(pathname, file.path); + + console.log(path); + naudio.sounds.playback_sound({ + callback: (result, message) => { + if(result == naudio.sounds.PlaybackResult.SUCCEEDED) + resolve(); + else + reject(naudio.sounds.PlaybackResult[result].toLowerCase() + ": " + message); + }, + file: path, + volume: file.volume + }); + }); + } +} + +Object.assign(window["audio"] || (window["audio"] = {} as any), audio); \ No newline at end of file diff --git a/modules/renderer/index.ts b/modules/renderer/index.ts index 70b2234..4c4b3e5 100644 --- a/modules/renderer/index.ts +++ b/modules/renderer/index.ts @@ -189,11 +189,11 @@ const module_loader_setup = async () => { }; const load_basic_modules = async () => { - console.dir(require("./audio/AudioPlayer")); /* setup audio */ - console.dir(require("./audio/AudioRecorder")); /* setup audio */ require("./logger"); - + require("./audio/AudioPlayer"); /* setup audio */ + require("./audio/AudioRecorder"); /* setup audio */ + require("./audio/sounds"); /* setup audio */ }; const load_modules = async () => { @@ -216,6 +216,7 @@ const load_modules = async () => { console.dir(error); throw error; } + try { const helper = require("./icon-helper"); await helper.initialize(); diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 36a2fbe..cc20be1 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -21,11 +21,11 @@ message("Module path: ${CMAKE_MODULE_PATH}") #Setup NodeJS function(setup_nodejs) set(NodeJS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") - #set(NODEJS_URL "https://atom.io/download/atom-shell") - #set(NODEJS_VERSION "v8.0.0") + set(NODEJS_URL "https://atom.io/download/atom-shell") + set(NODEJS_VERSION "v8.0.0") - set(NODEJS_URL "https://nodejs.org/download/release/") - set(NODEJS_VERSION "v12.13.0") + #set(NODEJS_URL "https://nodejs.org/download/release/") + #set(NODEJS_VERSION "v12.13.0") find_package(NodeJS REQUIRED) diff --git a/native/dist/ext_nan/include/NanEventCallback.h b/native/dist/ext_nan/include/NanEventCallback.h index e9b1dae..655ce22 100644 --- a/native/dist/ext_nan/include/NanEventCallback.h +++ b/native/dist/ext_nan/include/NanEventCallback.h @@ -127,7 +127,7 @@ namespace Nan { * @returns true if the callback will be called even * if the handle has been destroyed */ - bool option_destroyed_execute() const { + [[nodiscard]] bool option_destroyed_execute() const { if(!this->handle) std::__throw_logic_error("missing handle"); return this->handle->handle->option_destroy_run; diff --git a/native/dist/ext_nan/include/NanGet.h b/native/dist/ext_nan/include/NanGet.h index 12f671c..beb3e49 100644 --- a/native/dist/ext_nan/include/NanGet.h +++ b/native/dist/ext_nan/include/NanGet.h @@ -27,8 +27,8 @@ namespace Nan { } template - inline v8::Local GetLocal(v8::Local object, K key) { - return Nan::Get(object, key).FromMaybe(v8::Local{}); + inline v8::Local GetLocal(v8::Local object, K key, const v8::Local& default_value = v8::Local{}) { + return Nan::Get(object, key).FromMaybe(default_value); } template diff --git a/native/serverconnection/CMakeLists.txt b/native/serverconnection/CMakeLists.txt index 39d326d..b96a107 100644 --- a/native/serverconnection/CMakeLists.txt +++ b/native/serverconnection/CMakeLists.txt @@ -16,6 +16,7 @@ set(SOURCE_FILES src/audio/AudioInput.cpp src/audio/AudioResampler.cpp src/audio/AudioReframer.cpp + src/audio/AudioEventLoop.cpp src/audio/filter/FilterVad.cpp src/audio/filter/FilterThreshold.cpp @@ -25,7 +26,9 @@ set(SOURCE_FILES src/audio/codec/OpusConverter.cpp src/audio/driver/AudioDriver.cpp - ) + src/audio/sounds/SoundPlayer.cpp + src/audio/file/wav.cpp +) set(NODEJS_SOURCE_FILES src/bindings.cpp @@ -38,7 +41,6 @@ set(NODEJS_SOURCE_FILES src/connection/ProtocolHandlerPackets.cpp src/connection/ProtocolHandlerCommands.cpp src/connection/audio/AudioSender.cpp - src/connection/audio/AudioEventLoop.cpp src/connection/audio/VoiceConnection.cpp src/connection/audio/VoiceClient.cpp @@ -180,4 +182,4 @@ target_link_libraries(Audio-Test-2 ${REQUIRED_LIBRARIES}) add_executable(HW-UID-Test src/hwuid.cpp) target_link_libraries(HW-UID-Test ${REQUIRED_LIBRARIES} - ) \ No newline at end of file +) \ No newline at end of file diff --git a/native/serverconnection/exports/exports.d.ts b/native/serverconnection/exports/exports.d.ts index 9e0e0a6..3f270e6 100644 --- a/native/serverconnection/exports/exports.d.ts +++ b/native/serverconnection/exports/exports.d.ts @@ -250,6 +250,24 @@ declare module "teaclient_connection" { export function create_recorder() : AudioRecorder; } + export namespace sounds { + export enum PlaybackResult { + SUCCEEDED, + CANCELED, + SOUND_NOT_INITIALIZED, + FILE_OPEN_ERROR, + PLAYBACK_ERROR + } + + export interface PlaybackSettings { + file: string; + volume?: number; + callback?: (result: PlaybackResult, message: string) => void; + } + export function playback_sound(settings: PlaybackSettings) : number; + export function cancel_playback(playback: number); + } + export function initialize(callback: () => any); export function initialized() : boolean; export function available_devices() : AudioDevice[]; diff --git a/native/serverconnection/src/connection/audio/AudioEventLoop.cpp b/native/serverconnection/src/audio/AudioEventLoop.cpp similarity index 99% rename from native/serverconnection/src/connection/audio/AudioEventLoop.cpp rename to native/serverconnection/src/audio/AudioEventLoop.cpp index 86c1666..3e3caee 100644 --- a/native/serverconnection/src/connection/audio/AudioEventLoop.cpp +++ b/native/serverconnection/src/audio/AudioEventLoop.cpp @@ -23,5 +23,4 @@ void audio::shutdown_event_loops() { audio::decode_event_loop = nullptr; } audio::encode_event_loop = nullptr; - } \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioEventLoop.h b/native/serverconnection/src/audio/AudioEventLoop.h new file mode 100644 index 0000000..5869665 --- /dev/null +++ b/native/serverconnection/src/audio/AudioEventLoop.h @@ -0,0 +1,11 @@ +#pragma once + +#include "../EventLoop.h" + +namespace tc::audio { + extern event::EventExecutor* encode_event_loop; + extern event::EventExecutor* decode_event_loop; + + extern void init_event_loops(); + extern void shutdown_event_loops(); +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioMerger.cpp b/native/serverconnection/src/audio/AudioMerger.cpp index c018580..a85c46a 100644 --- a/native/serverconnection/src/audio/AudioMerger.cpp +++ b/native/serverconnection/src/audio/AudioMerger.cpp @@ -18,7 +18,14 @@ inline constexpr float merge_ab(float a, float b) { a += 1; b += 1; - return (2 * (a + b) - a * b - 2) - 1; + auto result = (2 * (a + b) - a * b - 2) - 1; + if(result > 1) { + result = 1; + } else if(result < -1) { + result = -1; + } + + return result; } static_assert(merge_ab(1, 0) == 1); diff --git a/native/serverconnection/src/audio/AudioMerger.h b/native/serverconnection/src/audio/AudioMerger.h index bf67767..e473d7f 100644 --- a/native/serverconnection/src/audio/AudioMerger.h +++ b/native/serverconnection/src/audio/AudioMerger.h @@ -2,17 +2,13 @@ #include -namespace tc { - namespace audio { - namespace merge { - /* the result buffer could be equal to one of the source buffers to prevent unnecessary allocations - * Note: The sample order is irrelevant - */ - extern bool merge_sources(void* /* result */, void* /* source a */, void* /* source b */, size_t /* channels */, size_t /* samples */); +namespace tc::audio::merge { + /* the result buffer could be equal to one of the source buffers to prevent unnecessary allocations + * Note: The sample order is irrelevant + */ + extern bool merge_sources(void* /* result */, void* /* source a */, void* /* source b */, size_t /* channels */, size_t /* samples */); - extern bool merge_n_sources(void* /* result */, void** /* sources */, size_t /* size_t sources count */, size_t /* channels */, size_t /* samples */); + extern bool merge_n_sources(void* /* result */, void** /* sources */, size_t /* size_t sources count */, size_t /* channels */, size_t /* samples */); - extern bool merge_channels_interleaved(void* /* result */, size_t /* result channels */, const void* /* source */, size_t /* source channels */, size_t /* samples */); - } - } + extern bool merge_channels_interleaved(void* /* result */, size_t /* result channels */, const void* /* source */, size_t /* source channels */, size_t /* samples */); } \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioOutput.cpp b/native/serverconnection/src/audio/AudioOutput.cpp index b2f7f99..c80dcc5 100644 --- a/native/serverconnection/src/audio/AudioOutput.cpp +++ b/native/serverconnection/src/audio/AudioOutput.cpp @@ -31,6 +31,9 @@ ssize_t AudioOutputSource::pop_samples(void *buffer, size_t samples) { const auto byte_length = (samples - written) * sizeof(float) * this->channel_count; if(buffer)memcpy((char*) buffer + written_bytes, this->buffer.read_ptr(), byte_length); this->buffer.advance_read_ptr(byte_length); + + if(this->on_read) + this->on_read(); return samples; } else { const auto byte_length = available_samples * sizeof(float) * this->channel_count; @@ -47,9 +50,9 @@ ssize_t AudioOutputSource::pop_samples(void *buffer, size_t samples) { if(buffer)memset(buffer, 0, (samples - written) * sizeof(float) * this->channel_count); this->buffering = true; + if(this->on_read) this->on_read(); - return written; /* return the written samples */ } @@ -140,8 +143,8 @@ AudioOutput::~AudioOutput() { this->cleanup_buffers(); } -std::shared_ptr AudioOutput::create_source() { - auto result = shared_ptr(new AudioOutputSource(this, this->_channel_count, this->_sample_rate)); +std::shared_ptr AudioOutput::create_source(ssize_t buf) { + auto result = shared_ptr(new AudioOutputSource(this, this->_channel_count, this->_sample_rate, buf)); { lock_guard lock(this->sources_lock); this->_sources.push_back(result); @@ -272,8 +275,12 @@ void AudioOutput::fill_buffer(void *output, size_t out_frame_count, size_t out_c /* this->source_buffer could hold the amount of resampled data (checked above) */ auto resampled_samples = this->_resampler->process(this->source_buffer, this->source_buffer, local_frame_count); + if(resampled_samples <= 0) { + log_warn(category::audio, tr("Failed to resample audio data for client ({})")); + goto clear_buffer_exit; + } if(resampled_samples != out_frame_count) { - if(resampled_samples > out_frame_count) { + if((size_t) resampled_samples > out_frame_count) { const auto diff_length = resampled_samples - out_frame_count; log_warn(category::audio, tr("enqueuing {} samples"), diff_length); const auto overhead_buffer_offset = this->resample_overhead_samples * sizeof(float) * this->_channel_count; diff --git a/native/serverconnection/src/audio/AudioOutput.h b/native/serverconnection/src/audio/AudioOutput.h index 385508c..02cd03a 100644 --- a/native/serverconnection/src/audio/AudioOutput.h +++ b/native/serverconnection/src/audio/AudioOutput.h @@ -44,6 +44,11 @@ namespace tc::audio { return max_samples; } + /* samples which needs to be played*/ + [[nodiscard]] inline size_t current_latency() const { + return this->buffer.fill_count() / this->channel_count / sizeof(float); + } + bool buffering{true}; size_t min_buffered_samples{0}; size_t max_buffered_samples{0}; @@ -60,8 +65,9 @@ namespace tc::audio { ssize_t enqueue_samples(const void * /* input buffer */, size_t /* sample count */); ssize_t enqueue_samples_no_interleave(const void * /* input buffer */, size_t /* sample count */); private: - AudioOutputSource(AudioOutput* handle, size_t channel_count, size_t sample_rate) : - handle(handle), channel_count(channel_count), sample_rate(sample_rate), buffer{channel_count * sample_rate * sizeof(float)} { + AudioOutputSource(AudioOutput* handle, size_t channel_count, size_t sample_rate, ssize_t max_buffer_sample_count = -1) : + handle(handle), channel_count(channel_count), sample_rate(sample_rate), + buffer{max_buffer_sample_count == -1 ? channel_count * sample_rate * sizeof(float) : max_buffer_sample_count * channel_count * sizeof(float)} { } tc::ring_buffer buffer; @@ -82,7 +88,7 @@ namespace tc::audio { return this->_sources; } - std::shared_ptr create_source(); + std::shared_ptr create_source(ssize_t /* buffer sample size */ = -1); void delete_source(const std::shared_ptr& /* source */); inline size_t channel_count() { return this->_channel_count; } diff --git a/native/serverconnection/src/audio/AudioResampler.cpp b/native/serverconnection/src/audio/AudioResampler.cpp index 43cae70..dde3af0 100644 --- a/native/serverconnection/src/audio/AudioResampler.cpp +++ b/native/serverconnection/src/audio/AudioResampler.cpp @@ -1,4 +1,3 @@ -#include #include #include "AudioResampler.h" #include "../logger.h" @@ -6,7 +5,7 @@ using namespace std; using namespace tc::audio; -AudioResampler::AudioResampler(size_t irate, size_t orate, size_t channels) : _input_rate(irate), _output_rate(orate), _channels(channels) { +AudioResampler::AudioResampler(size_t irate, size_t orate, size_t channels) : _input_rate{irate}, _output_rate{orate}, _channels{channels} { if(this->input_rate() != this->output_rate()) { soxr_error_t error; this->soxr_handle = soxr_create((double) this->_input_rate, (double) this->_output_rate, (unsigned) this->_channels, &error, nullptr, nullptr, nullptr); diff --git a/native/serverconnection/src/audio/file/wav.cpp b/native/serverconnection/src/audio/file/wav.cpp new file mode 100644 index 0000000..4c5b5db --- /dev/null +++ b/native/serverconnection/src/audio/file/wav.cpp @@ -0,0 +1,180 @@ +// +// Created by WolverinDEV on 18/03/2020. +// + +#include "./wav.h" +#include "../../logger.h" + +using namespace tc::audio::file; + +struct WAVFileHeader { + /* RIFF Chunk Descriptor */ + uint8_t RIFF[4]; // RIFF Header Magic header + uint32_t ChunkSize; // RIFF Chunk Size + uint8_t WAVE[4]; // WAVE Header + + /* "fmt" sub-chunk */ + uint8_t fmt[4]; // FMT header + uint32_t Subchunk1Size; // Size of the fmt chunk + uint16_t AudioFormat; // Audio format 1=PCM,6=mulaw,7=alaw, 257=IBM Mu-Law, 258=IBM A-Law, 259=ADPCM + uint16_t NumOfChan; // Number of channels 1=Mono 2=Sterio + uint32_t SamplesPerSec; // Sampling Frequency in Hz + uint32_t bytesPerSec; // bytes per second + uint16_t blockAlign; // 2=16-bit mono, 4=16-bit stereo + uint16_t bitsPerSample; // Number of bits per sample +}; +static_assert(sizeof(WAVFileHeader) == 0x24); + +struct WAFFileChunk { + uint8_t id[4]; + uint32_t size; +}; +static_assert(sizeof(WAFFileChunk) == 0x08); + +WAVReader::WAVReader(std::string file) : file_path_{std::move(file)} {} +WAVReader::~WAVReader() { + this->close_file(); +} + +FileOpenResult WAVReader::open_file(std::string& error) { + this->is_.open(this->file_path_, std::ifstream::in | std::ifstream::binary); + if(!this->is_) { + error = tr("failed to open file"); + return FileOpenResult::OPEN_RESULT_ERROR; + } + + WAVFileHeader header{}; + if(!this->is_.read((char*) &header, sizeof(header))) { + error = tr("failed to read wav header"); + return FileOpenResult::OPEN_RESULT_ERROR; + } + + if(memcmp(header.RIFF, "RIFF", 4) != 0) { + error = tr("invalid RIFF header"); + return FileOpenResult::OPEN_RESULT_ERROR; + } + + if(memcmp(header.WAVE, "WAVE", 4) != 0) { + error = tr("invalid WAVE header"); + return FileOpenResult::OPEN_RESULT_ERROR; + } + + if(memcmp(header.fmt, "fmt ", 4) != 0) { + error = tr("invalid WAVE header"); + return FileOpenResult::OPEN_RESULT_ERROR; + } + + if(header.AudioFormat != 1) { + error = tr("Only PCM has been supported. WAV file does not contains PCM data."); + return FileOpenResult::OPEN_RESULT_FORMAT_UNSUPPORTED; + } + + if(header.bytesPerSec != (header.NumOfChan * header.SamplesPerSec * header.bitsPerSample) / 8) { + error = tr("inconsistent WAV header"); + return FileOpenResult::OPEN_RESULT_INVALID_FORMAT; + } + + if(header.bitsPerSample != 8 && header.bitsPerSample != 16 && header.bitsPerSample != 24) { + error = tr("unsupported bitrate"); + return FileOpenResult::OPEN_RESULT_FORMAT_UNSUPPORTED; + } + + if(header.NumOfChan != 2 && header.NumOfChan != 1) { + error = tr("unsupported channel count"); + return FileOpenResult::OPEN_RESULT_FORMAT_UNSUPPORTED; + } + + WAFFileChunk chunk{}; + while(true) { + if(!this->is_.read((char*) &chunk, sizeof(chunk))) { + error = tr("failed to read chunks until data chunk"); + return FileOpenResult::OPEN_RESULT_ERROR; + } + + if(memcmp(chunk.id, "data ", 4) == 0) + break; + + this->is_.seekg(chunk.size, std::ifstream::cur); + } + + this->current_sample_offset_ = 0; + this->bytes_per_sample = header.bitsPerSample / 8; + this->total_samples_ = chunk.size / this->bytes_per_sample / header.NumOfChan; + this->sample_rate_ = header.SamplesPerSec; + this->channels_ = header.NumOfChan; + return FileOpenResult::OPEN_RESULT_SUCCESS; +} + +void WAVReader::close_file() { + this->total_samples_ = 0; + this->bytes_per_sample = 0; + this->sample_rate_ = 0; + this->channels_ = 0; + this->is_.close(); +} + +float _8bit_float_convert(const uint8_t* buffer) { + int16_t value = buffer[0] & 0xFFU; + return (float) (value - 127) * (1.0f / 127.0f); +} + +float _16bit_float_convert(const uint8_t* buffer) { + int16_t value = *reinterpret_cast(buffer); + return (float) value / 32767.f; +} + +float _24bit_float_convert(const uint8_t* buffer) { +#if 0 + int32_t value = (*reinterpret_cast(buffer) & 0xFFFFFFU) << 8; + return (float) (value - 1073741824) / 1073741824.f; //2147483648 / 2 +#endif +#if 1 + int32_t value = ((uint32_t) buffer[2] << 16U) | ((uint32_t) buffer[1] << 8U) | ((uint32_t) buffer[0] << 0U); + if (value & 0x800000) // if the 24th bit is set, this is a negative number in 24-bit world + value = value | ~0xFFFFFF; // so make sure sign is extended to the 32 bit float + auto result = (float) value / (float) 8388608.f; //8388608 + return result; +#endif +} + +static std::array pcm_to_float_converters{ + _8bit_float_convert, + _16bit_float_convert, + _24bit_float_convert +}; + +ReadResult WAVReader::read(void *buffer, size_t* samples) { + auto fbuffer = (float*) buffer; + + const auto max_sample_count = this->total_samples_ - this->current_sample_offset_; + const auto max_samples = std::min(*samples, max_sample_count); + if(max_samples == 0) { + if(max_sample_count == 0) return ReadResult::READ_RESULT_EOF; + + return ReadResult::READ_RESULT_SUCCESS; + } + + constexpr size_t sbuffer_size{1536}; /* must be dividable by 24, 165 and 8 bit! As well by two channels so 6, 4 and 2 byte to avoid to mess up one frame */ + uint8_t sbuffer[sbuffer_size]; + + size_t samples_read{0}; + auto fconverter = pcm_to_float_converters[(this->bytes_per_sample - 1) & 0x3U]; + while(samples_read < max_samples) { + const auto block_byte_length{std::min(sbuffer_size, (max_samples - samples_read) * this->bytes_per_sample * this->channels_)}; + if(!this->is_.read((char*) sbuffer, block_byte_length)) + return ReadResult::READ_RESULT_UNRECOVERABLE_ERROR; + + uint8_t* sbufferptr = sbuffer; + uint8_t* sbuferendptr = sbuffer + block_byte_length; + while(sbufferptr != sbuferendptr) { + *fbuffer++ = fconverter(sbufferptr); + sbufferptr += this->bytes_per_sample; + } + + samples_read += block_byte_length / (this->bytes_per_sample * this->channels_); + } + + *samples = samples_read; + this->current_sample_offset_ += samples_read; + return ReadResult::READ_RESULT_SUCCESS; +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/file/wav.h b/native/serverconnection/src/audio/file/wav.h new file mode 100644 index 0000000..88543d6 --- /dev/null +++ b/native/serverconnection/src/audio/file/wav.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include + +namespace tc::audio::file { + enum FileOpenResult { + OPEN_RESULT_SUCCESS, + OPEN_RESULT_INVALID_FORMAT, + OPEN_RESULT_FORMAT_UNSUPPORTED, + OPEN_RESULT_ERROR + }; + + enum ReadResult { + READ_RESULT_SUCCESS, + READ_RESULT_EOF, + READ_RESULT_UNRECOVERABLE_ERROR + }; + + class WAVReader { + public: + explicit WAVReader(std::string /* path */); + ~WAVReader(); + + [[nodiscard]] inline const std::string& file_path() const { return this->file_path_; } + + [[nodiscard]] FileOpenResult open_file(std::string& /* error */); + void close_file(); + + [[nodiscard]] inline size_t channels() const { return this->channels_; } + [[nodiscard]] inline size_t sample_rate() const { return this->sample_rate_; } + [[nodiscard]] inline size_t total_samples() const { return this->total_samples_; } + [[nodiscard]] inline size_t current_sample_offset() const { return this->current_sample_offset_; } + + /** + * Audio data in interleaved floats reachting from [-1;1]. + * Must contains at least channels * sizeof(float) * sample_count bytes + */ + [[nodiscard]] ReadResult read(void* /* target buffer */, size_t* /* sample count */); + private: + const std::string file_path_; + + std::ifstream is_{}; + + size_t channels_{0}; + size_t sample_rate_{0}; + size_t total_samples_{0}; + size_t current_sample_offset_{0}; + uint8_t bytes_per_sample{0}; /* 1, 2 or 3 (8, 16, 24 bit) */ + }; +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/sounds/SoundPlayer.cpp b/native/serverconnection/src/audio/sounds/SoundPlayer.cpp new file mode 100644 index 0000000..b8f13ae --- /dev/null +++ b/native/serverconnection/src/audio/sounds/SoundPlayer.cpp @@ -0,0 +1,309 @@ +// +// Created by WolverinDEV on 18/03/2020. +// + +#include +#include +#include +#include "./SoundPlayer.h" +#include "../AudioOutput.h" +#include "../file/wav.h" +#include "../../EventLoop.h" +#include "../../logger.h" +#include "../AudioEventLoop.h" +#include "../AudioResampler.h" +#include "../AudioMerger.h" + +#ifdef max + #undef max +#endif + +using namespace tc::audio; + +extern tc::audio::AudioOutput* global_audio_output; +namespace tc::audio::sounds { + class FilePlayer : public event::EventEntry, public std::enable_shared_from_this { + public: + explicit FilePlayer(PlaybackSettings settings) : settings_{std::move(settings)} { + log_trace(category::memory, tr("Allocated FilePlayer instance at {}"), (void*) this); + } + + ~FilePlayer() { + this->finalize(true); + log_trace(category::memory, tr("Deleted FilePlayer instance at {}"), (void*) this); + } + + [[nodiscard]] inline const PlaybackSettings& settings() const { return this->settings_; } + + [[nodiscard]] inline bool is_finished() const { return this->state_ == PLAYER_STATE_UNSET; } + + /* should not be blocking! */ + bool play() { + if(this->state_ != PLAYER_STATE_UNSET) return false; + this->state_ = PLAYER_STATE_INITIALIZE; + + audio::decode_event_loop->schedule(this->shared_from_this()); + return true; + } + + /* should not be blocking! */ + void cancel() { + this->state_ = PLAYER_STATE_CANCELED; + audio::decode_event_loop->schedule(this->shared_from_this()); + } + private: + constexpr static auto kBufferChunkTimespan{0.2}; + + const PlaybackSettings settings_; + std::unique_ptr file_handle{nullptr}; + std::unique_ptr resampler{nullptr}; + std::shared_ptr output_source; + + void* cache_buffer{nullptr}; + + enum { + PLAYER_STATE_INITIALIZE, + PLAYER_STATE_PLAYING, + PLAYER_STATE_AWAIT_FINISH, + PLAYER_STATE_FINISHED, + PLAYER_STATE_CANCELED, + PLAYER_STATE_UNSET + } state_{PLAYER_STATE_UNSET}; + + void finalize(bool is_destructor_call) { + if(this->output_source && global_audio_output) + global_audio_output->delete_source(this->output_source); + if(auto buffer{std::exchange(this->cache_buffer, nullptr)}; buffer) + ::free(buffer); + if(!is_destructor_call) + audio::decode_event_loop->cancel(this->shared_from_this()); + this->state_ = PLAYER_STATE_UNSET; + } + + void event_execute(const std::chrono::system_clock::time_point &) override { + if(this->state_ == PLAYER_STATE_INITIALIZE) { + this->file_handle = std::make_unique(this->settings_.file); + std::string error{}; + if(auto err{this->file_handle->open_file(error)}; err != file::OPEN_RESULT_SUCCESS) { + if(auto callback{this->settings_.callback}; callback) + callback(PlaybackResult::FILE_OPEN_ERROR, error); + this->finalize(false); + return; + } + + if(!global_audio_output) { + if(auto callback{this->settings_.callback}; callback) + callback(PlaybackResult::SOUND_NOT_INITIALIZED, ""); + this->finalize(false); + return; + } + + this->initialize_playback(); + + auto max_samples = (size_t) + std::max(this->output_source->sample_rate, this->file_handle->sample_rate()) * kBufferChunkTimespan * 8 * + std::max(this->file_handle->channels(), this->output_source->channel_count); + this->cache_buffer = ::malloc((size_t) (max_samples * sizeof(float))); + if(!this->cache_buffer) { + if(auto callback{this->settings_.callback}; callback) + callback(PlaybackResult::PLAYBACK_ERROR, "failed to allocate cached buffer"); + + this->finalize(false); + return; + } + this->state_ = PLAYER_STATE_PLAYING; + } + if(this->state_ == PLAYER_STATE_PLAYING) { + if(!this->could_enqueue_next_buffer()) return; + + auto samples_to_read = (size_t) (this->file_handle->sample_rate() * kBufferChunkTimespan); + auto errc = this->file_handle->read(this->cache_buffer, &samples_to_read); + switch (errc) { + case file::READ_RESULT_SUCCESS: + break; + + case file::READ_RESULT_EOF: + this->state_ = PLAYER_STATE_AWAIT_FINISH; + return; + + + case file::READ_RESULT_UNRECOVERABLE_ERROR: + if(auto callback{this->settings_.callback}; callback) + callback(PlaybackResult::PLAYBACK_ERROR, "read resulted in an unrecoverable error"); + + this->finalize(false); + return; + } + + if(!merge::merge_channels_interleaved(this->cache_buffer, this->output_source->channel_count, this->cache_buffer, this->file_handle->channels(), samples_to_read)) { + log_warn(category::audio, tr("failed to merge channels for replaying a sound")); + return; + } + + auto resampled_samples = this->resampler->process(this->cache_buffer, this->cache_buffer, samples_to_read); + if(resampled_samples <= 0) { + log_warn(category::audio, tr("failed to resample file audio buffer ({})"), resampled_samples); + return; + } + + this->output_source->enqueue_samples(this->cache_buffer, resampled_samples); + if(this->could_enqueue_next_buffer()) + audio::decode_event_loop->schedule(this->shared_from_this()); + } else if(this->state_ == PLAYER_STATE_FINISHED || this->state_ == PLAYER_STATE_CANCELED) { + this->finalize(false); + if(auto callback{this->settings_.callback}; callback) + callback(this->state_ == PLAYER_STATE_CANCELED ? PlaybackResult::CANCELED : PlaybackResult::SUCCEEDED, ""); + this->state_ = PLAYER_STATE_UNSET; + return; + } + auto filled_samples = this->output_source->current_latency(); + } + + void initialize_playback() { + assert(this->file_handle); + assert(global_audio_output); + + const auto max_buffer = (size_t) ceil(global_audio_output->sample_rate() * kBufferChunkTimespan * 3); + this->output_source = global_audio_output->create_source(max_buffer); + this->output_source->overflow_strategy = audio::overflow_strategy::ignore; + this->output_source->max_buffered_samples = max_buffer; + this->output_source->min_buffered_samples = (size_t) floor(this->output_source->sample_rate * 0.04); + + auto weak_this = this->weak_from_this(); + this->output_source->on_underflow = [weak_this](size_t sample_count){ + auto self = weak_this.lock(); + if(!self) return false; + + if(self->state_ == PLAYER_STATE_PLAYING) + log_warn(category::audio, tr("Having an audio underflow while playing a sound.")); + else if(self->state_ == PLAYER_STATE_AWAIT_FINISH) + self->state_ = PLAYER_STATE_FINISHED; + audio::decode_event_loop->schedule(self); + return false; + }; + this->output_source->on_read = [weak_this] { + auto self = weak_this.lock(); + if(!self) return; + + if(self->could_enqueue_next_buffer() && self->state_ == PLAYER_STATE_PLAYING) + audio::decode_event_loop->schedule(self); + }; + + this->output_source->on_overflow = [weak_this](size_t count) { + log_warn(category::audio, tr("Having an audio overflow while playing a sound.")); + }; + + this->resampler = std::make_unique(this->file_handle->sample_rate(), this->output_source->sample_rate, this->output_source->channel_count); + } + + + [[nodiscard]] inline size_t cache_buffer_sample_size() const { + return (size_t) (this->output_source->sample_rate * kBufferChunkTimespan); + } + + [[nodiscard]] inline bool could_enqueue_next_buffer() const { + if(!this->output_source) return false; + + const auto current_size = this->output_source->current_latency(); + const auto max_size = this->output_source->max_buffered_samples; + if(current_size > max_size) return false; + + const auto size_left = max_size - current_size; + return size_left >= this->cache_buffer_sample_size(); + } + }; + + std::mutex file_player_mutex{}; + std::deque> file_players{}; + + sound_playback_id playback_sound(const PlaybackSettings& settings) { + if(!audio::initialized()) { + settings.callback(PlaybackResult::SOUND_NOT_INITIALIZED, ""); + return 0; + } + + std::unique_lock fplock{file_player_mutex}; + file_players.erase(std::remove_if(file_players.begin(), file_players.end(), [](const auto& player) { + return player->is_finished(); + }), file_players.end()); + + auto player = std::make_shared(settings); + file_players.push_back(player); + if(!player->play()) { + if(auto callback{settings.callback}; callback) + callback(PlaybackResult::PLAYBACK_ERROR, "failed to start playback."); + return 0; + } + fplock.unlock(); + return (sound_playback_id) &*player; + } + + void cancel_playback(const sound_playback_id& id) { + std::unique_lock fplock{file_player_mutex}; + auto player_it = std::find_if(file_players.begin(), file_players.end(), [&](const auto& player) { return (sound_playback_id) &*player == id; }); + if(player_it == file_players.end()) return; + + auto player = *player_it; + file_players.erase(player_it); + fplock.unlock(); + + player->cancel(); + } +} + +NAN_METHOD(tc::audio::sounds::playback_sound_js) { + if(info.Length() != 1 || !info[0]->IsObject()) { + Nan::ThrowError("invalid arguments"); + return; + } + + auto data = info[0].As(); + auto file = Nan::GetLocal(data, "file"); + auto volume = Nan::GetLocal(data, "volume", Nan::New(1.f)); + v8::Local callback = Nan::GetLocal(data, "callback"); + if(file.IsEmpty() || !file->IsString()) { + Nan::ThrowError("missing file path"); + return; + } + if(volume.IsEmpty() || !volume->IsNumber()) { + Nan::ThrowError("invalid volume"); + return; + } + + PlaybackSettings settings{}; + settings.file = *Nan::Utf8String(file); + settings.volume = volume->Value(); + if(!callback.IsEmpty()) { + if(!callback->IsFunction()) { + Nan::ThrowError("invalid callback function"); + return; + } + + Nan::Global cb{callback}; + auto async_callback = Nan::async_callback([cb = std::move(cb)](PlaybackResult result, std::string error) mutable { + Nan::HandleScope scope{}; + auto callback = cb.Get(Nan::GetCurrentContext()->GetIsolate()).As(); + cb.Reset(); + + v8::Local arguments[2]; + arguments[0] = Nan::New((int) result); + arguments[1] = Nan::LocalStringUTF8(error); + (void) callback->Call(Nan::GetCurrentContext(), Nan::Undefined(), 2, arguments); + }).option_destroyed_execute(true); + + settings.callback = [async_callback](PlaybackResult result, const std::string& error) mutable { + async_callback.call_cpy(result, error); + }; + } + + info.GetReturnValue().Set((uint32_t) playback_sound(settings)); +} + +NAN_METHOD(tc::audio::sounds::cancel_playback_js) { + if(info.Length() != 1 || !info[0]->IsNumber()) { + Nan::ThrowError("invalid arguments"); + return; + } + + cancel_playback(info[0].As()->Value()); +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/sounds/SoundPlayer.h b/native/serverconnection/src/audio/sounds/SoundPlayer.h new file mode 100644 index 0000000..b01dbe6 --- /dev/null +++ b/native/serverconnection/src/audio/sounds/SoundPlayer.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include + +namespace tc::audio::sounds { + typedef uintptr_t sound_playback_id; + + enum struct PlaybackResult { + SUCCEEDED, + CANCELED, + SOUND_NOT_INITIALIZED, + FILE_OPEN_ERROR, + PLAYBACK_ERROR /* has the extra error set */ + }; + + typedef std::function callback_status_t; + struct PlaybackSettings { + std::string file{}; + double volume{1.f}; + /* ATTENTION: This callback may be called within the audio loop! */ + callback_status_t callback{}; + }; + + extern sound_playback_id playback_sound(const PlaybackSettings& /* settings */); + extern void cancel_playback(const sound_playback_id&); + + extern NAN_METHOD(playback_sound_js); + extern NAN_METHOD(cancel_playback_js); +} \ No newline at end of file diff --git a/native/serverconnection/src/bindings.cpp b/native/serverconnection/src/bindings.cpp index 3f0a232..49c1990 100644 --- a/native/serverconnection/src/bindings.cpp +++ b/native/serverconnection/src/bindings.cpp @@ -22,7 +22,8 @@ #include "audio/js/AudioRecorder.h" #include "audio/js/AudioConsumer.h" #include "audio/js/AudioFilter.h" -#include "connection/audio/AudioEventLoop.h" +#include "audio/AudioEventLoop.h" +#include "audio/sounds/SoundPlayer.h" #ifndef WIN32 #include @@ -69,8 +70,8 @@ void testTomMath(){ tc::audio::AudioOutput* global_audio_output; #define ENUM_SET(object, key, value) \ - Nan::DefineOwnProperty(object, Nan::New(key).ToLocalChecked(), Nan::New(value), v8::DontDelete); \ - Nan::Set(object, value, Nan::New(key).ToLocalChecked()); + Nan::DefineOwnProperty(object, Nan::New(key).ToLocalChecked(), Nan::New((uint32_t) value), v8::DontDelete); \ + Nan::Set(object, (uint32_t) value, Nan::New(key).ToLocalChecked()); NAN_MODULE_INIT(init) { logger::initialize_node(); @@ -177,6 +178,24 @@ NAN_MODULE_INIT(init) { audio::recorder::AudioFilterWrapper::Init(namespace_record); Nan::Set(namespace_audio, Nan::New("record").ToLocalChecked(), namespace_record); } + { + auto namespace_sounds = Nan::New(); + Nan::Set(namespace_sounds, Nan::LocalString("playback_sound"), Nan::GetFunction(Nan::New(audio::sounds::playback_sound_js)).ToLocalChecked()); + Nan::Set(namespace_sounds, Nan::LocalString("cancel_playback"), Nan::GetFunction(Nan::New(audio::sounds::cancel_playback_js)).ToLocalChecked()); + + { + auto enum_object = Nan::New(); + ENUM_SET(enum_object, "SUCCEEDED", audio::sounds::PlaybackResult::SUCCEEDED); + ENUM_SET(enum_object, "CANCELED", audio::sounds::PlaybackResult::CANCELED); + ENUM_SET(enum_object, "SOUND_NOT_INITIALIZED", audio::sounds::PlaybackResult::SOUND_NOT_INITIALIZED); + ENUM_SET(enum_object, "FILE_OPEN_ERROR", audio::sounds::PlaybackResult::FILE_OPEN_ERROR); + ENUM_SET(enum_object, "PLAYBACK_ERROR", audio::sounds::PlaybackResult::PLAYBACK_ERROR); + + Nan::DefineOwnProperty(namespace_sounds, Nan::New("PlaybackResult").ToLocalChecked(), enum_object, v8::DontDelete); + } + + Nan::Set(namespace_audio, Nan::New("sounds").ToLocalChecked(), namespace_sounds); + } Nan::Set(target, Nan::New("audio").ToLocalChecked(), namespace_audio); } diff --git a/native/serverconnection/src/connection/audio/AudioEventLoop.h b/native/serverconnection/src/connection/audio/AudioEventLoop.h deleted file mode 100644 index 83ae434..0000000 --- a/native/serverconnection/src/connection/audio/AudioEventLoop.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include "../../EventLoop.h" - -namespace tc { - namespace audio { - extern event::EventExecutor* encode_event_loop; - extern event::EventExecutor* decode_event_loop; - - extern void init_event_loops(); - extern void shutdown_event_loops(); - } -} \ No newline at end of file diff --git a/native/serverconnection/src/connection/audio/AudioSender.cpp b/native/serverconnection/src/connection/audio/AudioSender.cpp index 8361e0b..8303871 100644 --- a/native/serverconnection/src/connection/audio/AudioSender.cpp +++ b/native/serverconnection/src/connection/audio/AudioSender.cpp @@ -2,7 +2,7 @@ #include "VoiceConnection.h" #include "../ServerConnection.h" #include "../../logger.h" -#include "AudioEventLoop.h" +#include "../../audio/AudioEventLoop.h" #include "../../audio/AudioMerger.h" using namespace std; diff --git a/native/serverconnection/src/connection/audio/VoiceClient.cpp b/native/serverconnection/src/connection/audio/VoiceClient.cpp index 99b72a2..b397594 100644 --- a/native/serverconnection/src/connection/audio/VoiceClient.cpp +++ b/native/serverconnection/src/connection/audio/VoiceClient.cpp @@ -4,7 +4,7 @@ #include "../../audio/codec/OpusConverter.h" #include "../../audio/AudioMerger.h" #include "../../audio/js/AudioOutputStream.h" -#include "AudioEventLoop.h" +#include "../../audio/AudioEventLoop.h" #include "../../logger.h" using namespace std; diff --git a/native/serverconnection/src/connection/audio/VoiceClient.h b/native/serverconnection/src/connection/audio/VoiceClient.h index 4a1cda6..49193d5 100644 --- a/native/serverconnection/src/connection/audio/VoiceClient.h +++ b/native/serverconnection/src/connection/audio/VoiceClient.h @@ -56,7 +56,6 @@ namespace tc { template friend inline std::shared_ptr<_Tp> std::static_pointer_cast(const std::shared_ptr<_Up>& __r) noexcept; #endif - //_NODISCARD shared_ptr<_Ty1> static_pointer_cast(shared_ptr<_Ty2>&& _Other) noexcept public: struct state { enum value { diff --git a/native/serverconnection/src/connection/ft/FileTransferObject.cpp b/native/serverconnection/src/connection/ft/FileTransferObject.cpp index 153bf40..8ad9aed 100644 --- a/native/serverconnection/src/connection/ft/FileTransferObject.cpp +++ b/native/serverconnection/src/connection/ft/FileTransferObject.cpp @@ -163,7 +163,7 @@ void TransferObjectWrap::do_wrap(v8::Local object) { ); Nan::Set(object, Nan::New("name").ToLocalChecked(), - Nan::New(this->target()->name().c_str()).ToLocalChecked() + Nan::New(this->target()->name().c_str()).ToLocalChecked() ); if(source) { diff --git a/native/serverconnection/src/logger.h b/native/serverconnection/src/logger.h index f6b2c13..82ba37c 100644 --- a/native/serverconnection/src/logger.h +++ b/native/serverconnection/src/logger.h @@ -4,9 +4,9 @@ #include #include -#include #include #include +#include namespace tc_logger { namespace level = spdlog::level; diff --git a/native/serverconnection/test/js/audio.ts b/native/serverconnection/test/js/audio.ts index 1d00fea..8f9b0b0 100644 --- a/native/serverconnection/test/js/audio.ts +++ b/native/serverconnection/test/js/audio.ts @@ -18,6 +18,8 @@ handle.audio.initialize(() => { const stream = handle.audio.playback.create_stream(); console.log("Own stream: %o", stream); + stream.set_buffer_latency(0.02); + stream.set_buffer_max_latency(0.2); const recorder = handle.audio.record.create_recorder(); const default_input = handle.audio.available_devices().find(e => e.input_default); @@ -34,6 +36,19 @@ handle.audio.initialize(() => { }); }); + setInterval(() => { + handle.audio.sounds.playback_sound({ + file: "D:\\TeaSpeak\\web\\shared\\audio\\speech\\connection.refused.wav", + //file: "D:\\Users\\WolverinDEV\\Downloads\\LRMonoPhase4.wav", + + volume: 1, + callback: (result, message) => { + console.log("Result %s: %s", handle.audio.sounds.PlaybackResult[result], message); + } + }); + }, 500); + + /* setInterval(() => { const elements = handle.audio.available_devices().filter(e => e.input_supported); const dev = elements[Math.floor(Math.random() * elements.length)]; @@ -45,4 +60,5 @@ handle.audio.initialize(() => { }); }); }, 1000); + */ });