From 53367bafa5043cf4d5866fb31c4ab0aea5460d8d Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Sun, 29 Nov 2020 21:06:57 +0100 Subject: [PATCH] Some updates related to video --- github | 2 +- jenkins/create_build.sh | 4 +- modules/core/main-window/index.ts | 2 +- modules/renderer/audio/AudioRecorder.ts | 13 +- .../renderer/connection/ServerConnection.ts | 114 +++++++---- .../renderer/connection/VoiceConnection.ts | 12 ++ .../connection/VoiceConnection.ts.old | 180 ------------------ modules/renderer/hooks/Video.ts | 62 ++++++ modules/renderer/index.ts | 1 + native/serverconnection/exports/exports.d.ts | 1 - 10 files changed, 158 insertions(+), 233 deletions(-) delete mode 100644 modules/renderer/connection/VoiceConnection.ts.old create mode 100644 modules/renderer/hooks/Video.ts diff --git a/github b/github index 7c087d4..30d1bc0 160000 --- a/github +++ b/github @@ -1 +1 @@ -Subproject commit 7c087d46ad75ff641d5862a57ff13f3e860cc8a4 +Subproject commit 30d1bc01979c59d3d869f3be733b8849b173b42c diff --git a/jenkins/create_build.sh b/jenkins/create_build.sh index 08ef2e3..d96f108 100755 --- a/jenkins/create_build.sh +++ b/jenkins/create_build.sh @@ -122,5 +122,5 @@ function deploy_client() { #install_npm #compile_scripts #compile_native -#package_client -deploy_client +package_client +#deploy_client diff --git a/modules/core/main-window/index.ts b/modules/core/main-window/index.ts index 9fefcbc..6948bff 100644 --- a/modules/core/main-window/index.ts +++ b/modules/core/main-window/index.ts @@ -50,7 +50,7 @@ function spawnMainWindow(rendererEntryPoint: string) { mainWindow.on('closed', () => { app.releaseSingleInstanceLock(); - closeURLPreview(); + closeURLPreview().then(undefined); mainWindow = null; dereferenceApp(); diff --git a/modules/renderer/audio/AudioRecorder.ts b/modules/renderer/audio/AudioRecorder.ts index 0a2de80..b83df0b 100644 --- a/modules/renderer/audio/AudioRecorder.ts +++ b/modules/renderer/audio/AudioRecorder.ts @@ -4,9 +4,8 @@ import { InputConsumer, InputConsumerType, InputEvents, - InputStartResult, InputState, - LevelMeter + LevelMeter, MediaStreamRequestResult } from "tc-shared/voice/RecorderBase"; import {audio} from "tc-native/connection"; import {tr} from "tc-shared/i18n/localize"; @@ -60,10 +59,10 @@ export class NativeInput implements AbstractInput { } } - async start(): Promise { + async start(): Promise { if(this.state === InputState.RECORDING) { logWarn(LogCategory.VOICE, tr("Tried to start an input recorder twice.")); - return InputStartResult.EOK; + return MediaStreamRequestResult.EBUSY; } this.state = InputState.INITIALIZING; @@ -72,10 +71,12 @@ export class NativeInput implements AbstractInput { if(state !== "success") { if(state === "invalid-device") { - return InputStartResult.EDEVICEUNKNOWN; + return MediaStreamRequestResult.EDEVICEUNKNOWN; } else if(state === undefined) { throw tr("invalid set device result state"); } + + /* FIXME! */ throw state; } @@ -88,7 +89,7 @@ export class NativeInput implements AbstractInput { })); this.state = InputState.RECORDING; - return InputStartResult.EOK; + return true; } finally { if(this.state === InputState.INITIALIZING) { this.state = InputState.PAUSED; diff --git a/modules/renderer/connection/ServerConnection.ts b/modules/renderer/connection/ServerConnection.ts index 153205e..5fdc196 100644 --- a/modules/renderer/connection/ServerConnection.ts +++ b/modules/renderer/connection/ServerConnection.ts @@ -3,7 +3,7 @@ import { AbstractServerConnection, CommandOptionDefaults, CommandOptions, - ConnectionStateListener, + ConnectionStateListener, ConnectionStatistics, ServerCommand } from "tc-shared/connection/ConnectionBase"; import {CommandResult} from "tc-shared/connection/ServerConnectionDeclaration"; @@ -23,6 +23,9 @@ import {AbstractVoiceConnection} from "tc-shared/connection/VoiceConnection"; import {LogCategory, logDebug, logWarn} from "tc-shared/log"; import {ErrorCode} from "tc-shared/connection/ErrorCode"; import {ServerAddress} from "tc-shared/tree/Server"; +import {VideoConnection} from "tc-shared/connection/VideoConnection"; +import {RTCConnection} from "tc-shared/connection/rtc/Connection"; +import {RtpVideoConnection} from "tc-shared/connection/rtc/video/Connection"; interface ErrorCodeListener { callback: (result: CommandResult) => void; @@ -121,6 +124,7 @@ class ErrorCommandHandler extends AbstractCommandHandler { clearTimeout(listener.timeout); } + this.handle.getRtcConnection().doInitialSetup(); this.errorCodeMapping = {}; } else if(command.command == "notifyconnectioninforequest") { this.handle.send_command("setconnectioninfo", @@ -165,71 +169,82 @@ class ErrorCommandHandler extends AbstractCommandHandler { } export class ServerConnection extends AbstractServerConnection { - private _native_handle: NativeServerConnection; - private readonly _voice_connection: NativeVoiceConnectionWrapper; + private nativeHandle: NativeServerConnection; - private _do_teamspeak: boolean; + private readonly rtcConnection: RTCConnection; + private readonly voiceConnection: NativeVoiceConnectionWrapper; + private readonly videoConnection: VideoConnection; - private readonly _command_handler: NativeConnectionCommandBoss; - private readonly _command_error_handler: ErrorCommandHandler; - private readonly _command_handler_default: ConnectionCommandHandler; + private connectTeamSpeak: boolean; - private _remote_address: ServerAddress; - private _handshake_handler: HandshakeHandler; + private readonly commandHandler: NativeConnectionCommandBoss; + private readonly commandErrorHandler: ErrorCommandHandler; + private readonly defaultCommandHandler: ConnectionCommandHandler; - onconnectionstatechanged: ConnectionStateListener; + private remoteAddress: ServerAddress; + private handshakeHandler: HandshakeHandler; constructor(props: ConnectionHandler) { super(props); - this._command_handler = new NativeConnectionCommandBoss(this); - this._command_error_handler = new ErrorCommandHandler(this); - this._command_handler_default = new ConnectionCommandHandler(this); + this.commandHandler = new NativeConnectionCommandBoss(this); + this.commandErrorHandler = new ErrorCommandHandler(this); + this.defaultCommandHandler = new ConnectionCommandHandler(this); - this._command_handler.register_handler(this._command_error_handler); - this._command_handler.register_handler(this._command_handler_default); + this.rtcConnection = new RTCConnection(this, false); + this.videoConnection = new RtpVideoConnection(this.rtcConnection); - this._native_handle = spawn_native_server_connection(); - this._native_handle.callback_disconnect = reason => { + this.commandHandler.register_handler(this.commandErrorHandler); + this.commandHandler.register_handler(this.defaultCommandHandler); + + this.nativeHandle = spawn_native_server_connection(); + this.nativeHandle.callback_disconnect = reason => { this.client.handleDisconnect(DisconnectReason.CONNECTION_CLOSED, { reason: reason }); }; - this._native_handle.callback_command = (command, args, switches) => { + this.nativeHandle.callback_command = (command, args, switches) => { console.log("Received: %o %o %o", command, args, switches); //FIXME catch error - this._command_handler.invoke_handle({ + this.commandHandler.invoke_handle({ command: command, arguments: args }); }; - this._voice_connection = new NativeVoiceConnectionWrapper(this, this._native_handle._voice_connection); + + this.voiceConnection = new NativeVoiceConnectionWrapper(this, this.nativeHandle._voice_connection); this.command_helper.initialize(); } native_handle() : NativeServerConnection { - return this._native_handle; + return this.nativeHandle; } finalize() { - if(this._native_handle) - destroy_native_server_connection(this._native_handle); - this._native_handle = undefined; + if(this.nativeHandle) { + if(destroy_native_server_connection) { + /* currently not defined but may will be ;) */ + destroy_native_server_connection(this.nativeHandle); + } + this.nativeHandle = undefined; + } + + this.rtcConnection.destroy(); } connect(address: ServerAddress, handshake: HandshakeHandler, timeout?: number): Promise { this.updateConnectionState(ConnectionState.CONNECTING); - this._remote_address = address; - this._handshake_handler = handshake; - this._do_teamspeak = false; + this.remoteAddress = address; + this.handshakeHandler = handshake; + this.connectTeamSpeak = false; handshake.setConnection(this); handshake.initialize(); return new Promise((resolve, reject) => { - this._native_handle.connect({ + this.nativeHandle.connect({ remote_host: address.host, remote_port: address.port, @@ -241,15 +256,15 @@ export class ServerConnection extends AbstractServerConnection { /* required to notify the handle, just a promise reject does not work */ this.client.handleDisconnect(DisconnectReason.CONNECT_FAILURE, error); this.updateConnectionState(ConnectionState.UNCONNECTED); - reject(this._native_handle.error_message(error)); + reject(this.nativeHandle.error_message(error)); return; } else { resolve(); } this.updateConnectionState(ConnectionState.AUTHENTICATING); - console.log("Remote server type: %o (%s)", this._native_handle.server_type, ServerType[this._native_handle.server_type]); - if(this._native_handle.server_type == ServerType.TEAMSPEAK || this._do_teamspeak) { + console.log("Remote server type: %o (%s)", this.nativeHandle.server_type, ServerType[this.nativeHandle.server_type]); + if(this.nativeHandle.server_type == ServerType.TEAMSPEAK || this.connectTeamSpeak) { console.log("Trying to use TeamSpeak's identity system"); this.handshake_handler().on_teamspeak(); } @@ -263,24 +278,24 @@ export class ServerConnection extends AbstractServerConnection { remote_address(): ServerAddress { - return this._remote_address; + return this.remoteAddress; } handshake_handler(): HandshakeHandler { - return this._handshake_handler; + return this.handshakeHandler; } connected(): boolean { - return typeof(this._native_handle) !== "undefined" && this._native_handle.connected(); + return typeof(this.nativeHandle) !== "undefined" && this.nativeHandle.connected(); } disconnect(reason?: string): Promise { console.trace("Disconnect: %s",reason); - return new Promise((resolve, reject) => this._native_handle.disconnect(reason || "", error => { + return new Promise((resolve, reject) => this.nativeHandle.disconnect(reason || "", error => { if(error == 0) resolve(); else - reject(this._native_handle.error_message(error)); + reject(this.nativeHandle.error_message(error)); })); } @@ -289,11 +304,11 @@ export class ServerConnection extends AbstractServerConnection { } getVoiceConnection(): AbstractVoiceConnection { - return this._voice_connection; + return this.voiceConnection; } command_handler_boss(): AbstractCommandHandlerBoss { - return this._command_handler; + return this.commandHandler; } send_command(command: string, data?: any, _options?: CommandOptions): Promise { @@ -313,7 +328,7 @@ export class ServerConnection extends AbstractServerConnection { console.log("Send: %o %o", command, data); const promise = new Promise((resolve, reject) => { - data[0]["return_code"] = this._command_error_handler.generateReturnCode(command, result => { + data[0]["return_code"] = this.commandErrorHandler.generateReturnCode(command, result => { if(result.success) { resolve(result); } else { @@ -322,20 +337,35 @@ export class ServerConnection extends AbstractServerConnection { }); try { - this._native_handle.send_command(command, data, options.flagset || []); + this.nativeHandle.send_command(command, data, options.flagset || []); } catch(error) { reject(tr("failed to send command")); console.warn(tr("Failed to send command: %o"), error); } }); - return this._command_handler_default.proxy_command_promise(promise, options); + return this.defaultCommandHandler.proxy_command_promise(promise, options); } ping(): { native: number; javascript?: number } { return { - native: this._native_handle ? (this._native_handle.current_ping() / 1000) : -2 + native: this.nativeHandle ? (this.nativeHandle.current_ping() / 1000) : -2 }; } + + getControlStatistics(): ConnectionStatistics { + return { + bytesReceived: 0, + bytesSend: 0 + }; + } + + getVideoConnection(): VideoConnection { + return this.videoConnection; + } + + getRtcConnection() : RTCConnection { + return this.rtcConnection; + } } export class NativeConnectionCommandBoss extends AbstractCommandHandlerBoss { diff --git a/modules/renderer/connection/VoiceConnection.ts b/modules/renderer/connection/VoiceConnection.ts index 8a32788..0ed4d6d 100644 --- a/modules/renderer/connection/VoiceConnection.ts +++ b/modules/renderer/connection/VoiceConnection.ts @@ -14,6 +14,7 @@ import {VoicePlayerEvents, VoicePlayerLatencySettings, VoicePlayerState} from "t import {Registry} from "tc-shared/events"; import {LogCategory, logDebug, logInfo, logWarn} from "tc-shared/log"; import {tr} from "tc-shared/i18n/localize"; +import {ConnectionStatistics} from "tc-shared/connection/ConnectionBase"; export class NativeVoiceConnectionWrapper extends AbstractVoiceConnection { private readonly serverConnectionStateChangedListener; @@ -228,6 +229,17 @@ export class NativeVoiceConnectionWrapper extends AbstractVoiceConnection { stopWhisper() { } + + getConnectionStats(): Promise { + return Promise.resolve({ + bytesSend: 0, + bytesReceived: 0 + }); + } + + getRetryTimestamp(): number | 0 { + return Date.now(); + } } class NativeVoiceClientWrapper implements VoiceClient { diff --git a/modules/renderer/connection/VoiceConnection.ts.old b/modules/renderer/connection/VoiceConnection.ts.old deleted file mode 100644 index e9ca45d..0000000 --- a/modules/renderer/connection/VoiceConnection.ts.old +++ /dev/null @@ -1,180 +0,0 @@ -import {ServerConnection} from "./ServerConnection"; -import {NativeVoiceConnection} from "tc-native/connection"; -import {RecorderProfile} from "tc-shared/voice/RecorderProfile"; -import {tr} from "tc-shared/i18n/localize"; -import {LogCategory} from "tc-shared/log"; -import * as log from "tc-shared/log"; -import { - AbstractVoiceConnection, - VoiceConnectionStatus -} from "tc-shared/connection/VoiceConnection"; -import {NativeInput} from "../audio/AudioRecorder"; - -export class VoiceConnection extends AbstractVoiceConnection { - readonly connection: ServerConnection; - readonly handle: NativeVoiceConnection; - - private _audio_source: RecorderProfile; - - constructor(connection: ServerConnection, voice: NativeVoiceConnection) { - super(connection); - this.connection = connection; - this.handle = voice; - } - - setup() { } - - async acquire_voice_recorder(recorder: RecorderProfile | undefined, enforce?: boolean) { - if(this._audio_source === recorder && !enforce) - return; - - if(this._audio_source) - await this._audio_source.unmount(); - - if(recorder) { - if(!(recorder.input instanceof NativeInput)) - throw "Recorder input must be an instance of NativeInput!"; - await recorder.unmount(); - } - - this.handleVoiceEnded(); - this._audio_source = recorder; - - if(recorder) { - recorder.current_handler = this.connection.client; - - recorder.callback_unmount = () => { - this._audio_source = undefined; - this.handle.set_audio_source(undefined); - this.connection.client.update_voice_status(undefined); - }; - - recorder.callback_start = this.on_voice_started.bind(this); - recorder.callback_stop = this.handleVoiceEnded.bind(this); - - (recorder as any).callback_support_change = () => { - this.connection.client.update_voice_status(undefined); - }; - - this.handle.set_audio_source((recorder.input as NativeInput).getNativeConsumer()); - } - this.connection.client.update_voice_status(undefined); - } - - voice_playback_support() : boolean { - return this.connection.connected(); - } - - voice_send_support() : boolean { - return this.connection.connected(); - } - - private current_channel_codec() : number { - const chandler = this.connection.client; - return (chandler.getClient().currentChannel() || {properties: { channel_codec: 4}}).properties.channel_codec; - } - - private handleVoiceEnded() { - const chandler = this.connection.client; - chandler.getClient().speaking = false; - - if(!chandler.connected) - return false; - - if(chandler.isMicrophoneMuted()) - return false; - - console.log(tr("Local voice ended")); - //TODO Send end? (Or is this already an automated thing?) - } - - private on_voice_started() { - const chandler = this.connection.client; - if(chandler.isMicrophoneMuted()) { - /* evil hack due to the settings :D */ - log.warn(LogCategory.VOICE, tr("Received local voice started event, even thou we're muted! Do not send any voice.")); - if(this.handle) { - this.handle.enable_voice_send(false); - } - return; - } - - log.info(LogCategory.VOICE, tr("Local voice started (Native)")); - this.handle.enable_voice_send(true); - - const ch = chandler.getClient(); - if(ch) ch.speaking = true; - } - - getConnectionState(): VoiceConnectionStatus { - return VoiceConnectionStatus.Connected; - } - - voice_recorder(): RecorderProfile { - return this._audio_source; - } - - available_clients(): VoiceClient[] { - return this.handle.available_clients().map(e => Object.assign(e, { - support_latency_settings() { return true; }, - reset_latency_settings() { - const stream = this.get_stream(); - stream.set_buffer_latency(0.080); - stream.set_buffer_max_latency(0.5); - return this.latency_settings(); - }, - latency_settings(settings?: LatencySettings) : LatencySettings { - const stream = this.get_stream(); - if(typeof settings !== "undefined") { - stream.set_buffer_latency(settings.min_buffer / 1000); - stream.set_buffer_max_latency(settings.max_buffer / 100); - } - return { - max_buffer: Math.floor(stream.get_buffer_max_latency() * 1000), - min_buffer: Math.floor(stream.get_buffer_latency() * 1000) - }; - }, - - support_flush() { return true; }, - flush() { - const stream = this.get_stream(); - stream.flush_buffer(); - } - }) as any); /* cast to any because of: Type 'import("/mnt/d/TeaSpeak/client_linux/client/imports/shared-app/connection/ConnectionBase").voice.PlayerState' is not assignable to type 'import("tc-native/connection").PlayerState' */ - } - - find_client(client_id: number) : VoiceClient | undefined { - for(const client of this.available_clients()) - if(client.client_id === client_id) - return client; - return undefined; - } - - unregister_client(client: VoiceClient): Promise { - this.handle.unregister_client(client.client_id); - return Promise.resolve(); - } - - register_client(client_id: number): VoiceClient { - const client = this.handle.register_client(client_id); - const c = this.find_client(client_id); - c.reset_latency_settings(); - return c; - } - - decoding_supported(codec: number): boolean { - return this.handle.decoding_supported(codec); - } - - encoding_supported(codec: number): boolean { - return this.handle.encoding_supported(codec); - } - - get_encoder_codec(): number { - return this.handle.get_encoder_codec(); - } - - set_encoder_codec(codec: number) { - return this.handle.set_encoder_codec(codec); - } -} \ No newline at end of file diff --git a/modules/renderer/hooks/Video.ts b/modules/renderer/hooks/Video.ts new file mode 100644 index 0000000..2317ed9 --- /dev/null +++ b/modules/renderer/hooks/Video.ts @@ -0,0 +1,62 @@ +import * as loader from "tc-loader"; +import {Stage} from "tc-loader"; +import {ScreenCaptureDevice, setVideoDriver, VideoSource} from "tc-shared/video/VideoSource"; +import {WebVideoDriver, WebVideoSource} from "tc-shared/media/Video"; +import {desktopCapturer, remote} from "electron"; +import {requestMediaStreamWithConstraints} from "tc-shared/media/Stream"; +import {tr} from "tc-shared/i18n/localize"; + +loader.register_task(Stage.JAVASCRIPT_INITIALIZING, { + priority: 10, + function: async () => { + const instance = new NativeVideoDriver(); + await instance.initialize(); + setVideoDriver(instance); + }, + name: "Video init" +}); + +class NativeVideoDriver extends WebVideoDriver { + private currentScreenCaptureDevices: ScreenCaptureDevice[]; + + screenQueryAvailable(): boolean { + return true; + } + + async queryScreenCaptureDevices(): Promise { + const sources = await desktopCapturer.getSources({ fetchWindowIcons: true, types: ['window', 'screen'], thumbnailSize: { width: 480, height: 270 } }); + + return this.currentScreenCaptureDevices = sources.map(entry => { + return { + id: entry.id, + name: entry.name, + + type: entry.display_id ? "full-screen" : "window", + + appIcon: entry.appIcon?.toDataURL(), + appPreview: entry.thumbnail?.toDataURL() + } + }) + } + + async createScreenSource(id: string | undefined, allowFocusLoss: boolean): Promise { + const result = await requestMediaStreamWithConstraints({ + mandatory: { + chromeMediaSource: 'desktop', + chromeMediaSourceId: id, + } + } as any, "video"); + + if(typeof result === "string") { + throw result; + } + + if(!allowFocusLoss) { + /* redraw focus to our window since we lost it after requesting the screen capture */ + remote.getCurrentWindow().focus(); + } + + const name = this.currentScreenCaptureDevices.find(e => e.id === id)?.name || tr("Screen device"); + return new WebVideoSource(id, name, result); + } +} \ No newline at end of file diff --git a/modules/renderer/index.ts b/modules/renderer/index.ts index f6e0b7e..66ddf6f 100644 --- a/modules/renderer/index.ts +++ b/modules/renderer/index.ts @@ -166,6 +166,7 @@ loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { await import("./hooks/ChangeLogClient"); await import("./hooks/Backend"); await import("./hooks/MenuBar"); + await import("./hooks/Video"); await import("./UnloadHandler"); await import("./WindowsTrayHandler"); diff --git a/native/serverconnection/exports/exports.d.ts b/native/serverconnection/exports/exports.d.ts index 60affdd..8635121 100644 --- a/native/serverconnection/exports/exports.d.ts +++ b/native/serverconnection/exports/exports.d.ts @@ -83,7 +83,6 @@ declare module "tc-native/connection" { export function spawn_server_connection() : NativeServerConnection; export function destroy_server_connection(connection: NativeServerConnection); - export namespace ft { export interface TransferObject { name: string;