Some updates related to video

github

@@ -1 +1 @@
-Subproject commit 7c087d46ad75ff641d5862a57ff13f3e860cc8a4
+Subproject commit 30d1bc01979c59d3d869f3be733b8849b173b42c

jenkins/create_build.sh

@@ -122,5 +122,5 @@ function deploy_client() {
 #install_npm
 #compile_scripts
 #compile_native
-#package_client
+package_client
-deploy_client
+#deploy_client

modules/core/main-window/index.ts

@@ -50,7 +50,7 @@ function spawnMainWindow(rendererEntryPoint: string) {
 
     mainWindow.on('closed', () => {
         app.releaseSingleInstanceLock();
-        closeURLPreview();
+        closeURLPreview().then(undefined);
         mainWindow = null;
 
         dereferenceApp();

modules/renderer/audio/AudioRecorder.ts

@@ -4,9 +4,8 @@ import {
     InputConsumer,
     InputConsumerType,
     InputEvents,
-    InputStartResult,
     InputState,
-    LevelMeter
+    LevelMeter, MediaStreamRequestResult
 } from "tc-shared/voice/RecorderBase";
 import {audio} from "tc-native/connection";
 import {tr} from "tc-shared/i18n/localize";
@@ -60,10 +59,10 @@ export class NativeInput implements AbstractInput {
         }
     }
 
-    async start(): Promise<InputStartResult> {
+    async start(): Promise<MediaStreamRequestResult | true> {
         if(this.state === InputState.RECORDING) {
             logWarn(LogCategory.VOICE, tr("Tried to start an input recorder twice."));
-            return InputStartResult.EOK;
+            return MediaStreamRequestResult.EBUSY;
         }
 
         this.state = InputState.INITIALIZING;
@@ -72,10 +71,12 @@ export class NativeInput implements AbstractInput {
 
             if(state !== "success") {
                 if(state === "invalid-device") {
-                    return InputStartResult.EDEVICEUNKNOWN;
+                    return MediaStreamRequestResult.EDEVICEUNKNOWN;
                 } else if(state === undefined) {
                     throw tr("invalid set device result state");
                 }
+
+                /* FIXME! */
                 throw state;
             }
 
@@ -88,7 +89,7 @@ export class NativeInput implements AbstractInput {
             }));
 
             this.state = InputState.RECORDING;
-            return InputStartResult.EOK;
+            return true;
         } finally {
             if(this.state === InputState.INITIALIZING) {
                 this.state = InputState.PAUSED;

modules/renderer/connection/ServerConnection.ts

@@ -3,7 +3,7 @@ import {
     AbstractServerConnection,
     CommandOptionDefaults,
     CommandOptions,
-    ConnectionStateListener,
+    ConnectionStateListener, ConnectionStatistics,
     ServerCommand
 } from "tc-shared/connection/ConnectionBase";
 import {CommandResult} from "tc-shared/connection/ServerConnectionDeclaration";
@@ -23,6 +23,9 @@ import {AbstractVoiceConnection} from "tc-shared/connection/VoiceConnection";
 import {LogCategory, logDebug, logWarn} from "tc-shared/log";
 import {ErrorCode} from "tc-shared/connection/ErrorCode";
 import {ServerAddress} from "tc-shared/tree/Server";
+import {VideoConnection} from "tc-shared/connection/VideoConnection";
+import {RTCConnection} from "tc-shared/connection/rtc/Connection";
+import {RtpVideoConnection} from "tc-shared/connection/rtc/video/Connection";
 
 interface ErrorCodeListener {
     callback: (result: CommandResult) => void;
@@ -121,6 +124,7 @@ class ErrorCommandHandler extends AbstractCommandHandler {
                 clearTimeout(listener.timeout);
             }
 
+            this.handle.getRtcConnection().doInitialSetup();
             this.errorCodeMapping = {};
         } else if(command.command == "notifyconnectioninforequest") {
             this.handle.send_command("setconnectioninfo",
@@ -165,71 +169,82 @@ class ErrorCommandHandler extends AbstractCommandHandler {
 }
 
 export class ServerConnection extends AbstractServerConnection {
-    private _native_handle: NativeServerConnection;
+    private nativeHandle: NativeServerConnection;
-    private readonly _voice_connection: NativeVoiceConnectionWrapper;
 
-    private _do_teamspeak: boolean;
+    private readonly rtcConnection: RTCConnection;
+    private readonly voiceConnection: NativeVoiceConnectionWrapper;
+    private readonly videoConnection: VideoConnection;
 
-    private readonly _command_handler: NativeConnectionCommandBoss;
+    private connectTeamSpeak: boolean;
-    private readonly _command_error_handler: ErrorCommandHandler;
-    private readonly _command_handler_default: ConnectionCommandHandler;
 
-    private _remote_address: ServerAddress;
+    private readonly commandHandler: NativeConnectionCommandBoss;
-    private _handshake_handler: HandshakeHandler;
+    private readonly commandErrorHandler: ErrorCommandHandler;
+    private readonly defaultCommandHandler: ConnectionCommandHandler;
 
-    onconnectionstatechanged: ConnectionStateListener;
+    private remoteAddress: ServerAddress;
+    private handshakeHandler: HandshakeHandler;
 
     constructor(props: ConnectionHandler) {
         super(props);
 
-        this._command_handler = new NativeConnectionCommandBoss(this);
+        this.commandHandler = new NativeConnectionCommandBoss(this);
-        this._command_error_handler = new ErrorCommandHandler(this);
+        this.commandErrorHandler = new ErrorCommandHandler(this);
-        this._command_handler_default = new ConnectionCommandHandler(this);
+        this.defaultCommandHandler = new ConnectionCommandHandler(this);
 
-        this._command_handler.register_handler(this._command_error_handler);
+        this.rtcConnection = new RTCConnection(this, false);
-        this._command_handler.register_handler(this._command_handler_default);
+        this.videoConnection = new RtpVideoConnection(this.rtcConnection);
 
-        this._native_handle = spawn_native_server_connection();
+        this.commandHandler.register_handler(this.commandErrorHandler);
-        this._native_handle.callback_disconnect = reason => {
+        this.commandHandler.register_handler(this.defaultCommandHandler);
+
+        this.nativeHandle = spawn_native_server_connection();
+        this.nativeHandle.callback_disconnect = reason => {
             this.client.handleDisconnect(DisconnectReason.CONNECTION_CLOSED, {
                 reason: reason
             });
         };
-        this._native_handle.callback_command = (command, args, switches) => {
+        this.nativeHandle.callback_command = (command, args, switches) => {
             console.log("Received: %o %o %o", command, args, switches);
             //FIXME catch error
 
-            this._command_handler.invoke_handle({
+            this.commandHandler.invoke_handle({
                 command: command,
                 arguments: args
             });
         };
-        this._voice_connection = new NativeVoiceConnectionWrapper(this, this._native_handle._voice_connection);
+
+        this.voiceConnection = new NativeVoiceConnectionWrapper(this, this.nativeHandle._voice_connection);
 
         this.command_helper.initialize();
     }
 
     native_handle() : NativeServerConnection {
-        return this._native_handle;
+        return this.nativeHandle;
     }
 
     finalize() {
-        if(this._native_handle)
+        if(this.nativeHandle) {
-            destroy_native_server_connection(this._native_handle);
+            if(destroy_native_server_connection) {
-        this._native_handle = undefined;
+                /* currently not defined but may will be ;) */
+                destroy_native_server_connection(this.nativeHandle);
+            }
+            this.nativeHandle = undefined;
+        }
+
+        this.rtcConnection.destroy();
     }
 
     connect(address: ServerAddress, handshake: HandshakeHandler, timeout?: number): Promise<void> {
         this.updateConnectionState(ConnectionState.CONNECTING);
 
-        this._remote_address = address;
+        this.remoteAddress = address;
-        this._handshake_handler = handshake;
+        this.handshakeHandler = handshake;
-        this._do_teamspeak = false;
+        this.connectTeamSpeak = false;
         handshake.setConnection(this);
         handshake.initialize();
 
         return new Promise<void>((resolve, reject) => {
-            this._native_handle.connect({
+            this.nativeHandle.connect({
                 remote_host: address.host,
                 remote_port: address.port,
 
@@ -241,15 +256,15 @@ export class ServerConnection extends AbstractServerConnection {
                         /* required to notify the handle, just a promise reject does not work */
                         this.client.handleDisconnect(DisconnectReason.CONNECT_FAILURE, error);
                         this.updateConnectionState(ConnectionState.UNCONNECTED);
-                        reject(this._native_handle.error_message(error));
+                        reject(this.nativeHandle.error_message(error));
                         return;
                     } else {
                         resolve();
                     }
                     this.updateConnectionState(ConnectionState.AUTHENTICATING);
 
-                    console.log("Remote server type: %o (%s)", this._native_handle.server_type, ServerType[this._native_handle.server_type]);
+                    console.log("Remote server type: %o (%s)", this.nativeHandle.server_type, ServerType[this.nativeHandle.server_type]);
-                    if(this._native_handle.server_type == ServerType.TEAMSPEAK || this._do_teamspeak) {
+                    if(this.nativeHandle.server_type == ServerType.TEAMSPEAK || this.connectTeamSpeak) {
                         console.log("Trying to use TeamSpeak's identity system");
                         this.handshake_handler().on_teamspeak();
                     }
@@ -263,24 +278,24 @@ export class ServerConnection extends AbstractServerConnection {
 
 
     remote_address(): ServerAddress {
-        return this._remote_address;
+        return this.remoteAddress;
     }
 
     handshake_handler(): HandshakeHandler {
-        return this._handshake_handler;
+        return this.handshakeHandler;
     }
 
     connected(): boolean {
-        return typeof(this._native_handle) !== "undefined" && this._native_handle.connected();
+        return typeof(this.nativeHandle) !== "undefined" && this.nativeHandle.connected();
     }
 
     disconnect(reason?: string): Promise<void> {
         console.trace("Disconnect: %s",reason);
-        return new Promise<void>((resolve, reject) => this._native_handle.disconnect(reason || "", error => {
+        return new Promise<void>((resolve, reject) => this.nativeHandle.disconnect(reason || "", error => {
             if(error == 0)
                 resolve();
             else
-                reject(this._native_handle.error_message(error));
+                reject(this.nativeHandle.error_message(error));
         }));
     }
 
@@ -289,11 +304,11 @@ export class ServerConnection extends AbstractServerConnection {
     }
 
     getVoiceConnection(): AbstractVoiceConnection {
-        return this._voice_connection;
+        return this.voiceConnection;
     }
 
     command_handler_boss(): AbstractCommandHandlerBoss {
-        return this._command_handler;
+        return this.commandHandler;
     }
 
     send_command(command: string, data?: any, _options?: CommandOptions): Promise<CommandResult> {
@@ -313,7 +328,7 @@ export class ServerConnection extends AbstractServerConnection {
 
         console.log("Send: %o %o", command, data);
         const promise = new Promise<CommandResult>((resolve, reject) => {
-            data[0]["return_code"] = this._command_error_handler.generateReturnCode(command, result => {
+            data[0]["return_code"] = this.commandErrorHandler.generateReturnCode(command, result => {
                 if(result.success) {
                     resolve(result);
                 } else {
@@ -322,20 +337,35 @@ export class ServerConnection extends AbstractServerConnection {
             });
 
             try {
-                this._native_handle.send_command(command, data, options.flagset || []);
+                this.nativeHandle.send_command(command, data, options.flagset || []);
             } catch(error) {
                 reject(tr("failed to send command"));
                 console.warn(tr("Failed to send command: %o"), error);
             }
         });
-        return this._command_handler_default.proxy_command_promise(promise, options);
+        return this.defaultCommandHandler.proxy_command_promise(promise, options);
     }
 
     ping(): { native: number; javascript?: number } {
         return {
-            native: this._native_handle ? (this._native_handle.current_ping() / 1000) : -2
+            native: this.nativeHandle ? (this.nativeHandle.current_ping() / 1000) : -2
         };
     }
+
+    getControlStatistics(): ConnectionStatistics {
+        return {
+            bytesReceived: 0,
+            bytesSend: 0
+        };
+    }
+
+    getVideoConnection(): VideoConnection {
+        return this.videoConnection;
+    }
+
+    getRtcConnection() : RTCConnection {
+        return this.rtcConnection;
+    }
 }
 
 export class NativeConnectionCommandBoss extends AbstractCommandHandlerBoss {

modules/renderer/connection/VoiceConnection.ts

@@ -14,6 +14,7 @@ import {VoicePlayerEvents, VoicePlayerLatencySettings, VoicePlayerState} from "t
 import {Registry} from "tc-shared/events";
 import {LogCategory, logDebug, logInfo, logWarn} from "tc-shared/log";
 import {tr} from "tc-shared/i18n/localize";
+import {ConnectionStatistics} from "tc-shared/connection/ConnectionBase";
 
 export class NativeVoiceConnectionWrapper extends AbstractVoiceConnection {
     private readonly serverConnectionStateChangedListener;
@@ -228,6 +229,17 @@ export class NativeVoiceConnectionWrapper extends AbstractVoiceConnection {
 
     stopWhisper() {
     }
+
+    getConnectionStats(): Promise<ConnectionStatistics> {
+        return Promise.resolve({
+            bytesSend: 0,
+            bytesReceived: 0
+        });
+    }
+
+    getRetryTimestamp(): number | 0 {
+        return Date.now();
+    }
 }
 
 class NativeVoiceClientWrapper implements VoiceClient {

modules/renderer/connection/VoiceConnection.ts.old

@@ -1,180 +0,0 @@
-import {ServerConnection} from "./ServerConnection";
-import {NativeVoiceConnection} from "tc-native/connection";
-import {RecorderProfile} from "tc-shared/voice/RecorderProfile";
-import {tr} from "tc-shared/i18n/localize";
-import {LogCategory} from "tc-shared/log";
-import * as log from "tc-shared/log";
-import {
-    AbstractVoiceConnection,
-    VoiceConnectionStatus
-} from "tc-shared/connection/VoiceConnection";
-import {NativeInput} from "../audio/AudioRecorder";
-
-export class VoiceConnection extends AbstractVoiceConnection {
-    readonly connection: ServerConnection;
-    readonly handle: NativeVoiceConnection;
-
-    private _audio_source: RecorderProfile;
-
-    constructor(connection: ServerConnection, voice: NativeVoiceConnection) {
-        super(connection);
-        this.connection = connection;
-        this.handle = voice;
-    }
-
-    setup() { }
-
-    async acquire_voice_recorder(recorder: RecorderProfile | undefined, enforce?: boolean) {
-        if(this._audio_source === recorder && !enforce)
-            return;
-
-        if(this._audio_source)
-            await this._audio_source.unmount();
-
-        if(recorder) {
-            if(!(recorder.input instanceof NativeInput))
-                throw "Recorder input must be an instance of NativeInput!";
-            await recorder.unmount();
-        }
-
-        this.handleVoiceEnded();
-        this._audio_source = recorder;
-
-        if(recorder) {
-            recorder.current_handler = this.connection.client;
-
-            recorder.callback_unmount = () => {
-                this._audio_source = undefined;
-                this.handle.set_audio_source(undefined);
-                this.connection.client.update_voice_status(undefined);
-            };
-
-            recorder.callback_start = this.on_voice_started.bind(this);
-            recorder.callback_stop = this.handleVoiceEnded.bind(this);
-
-            (recorder as any).callback_support_change = () => {
-                this.connection.client.update_voice_status(undefined);
-            };
- 
-            this.handle.set_audio_source((recorder.input as NativeInput).getNativeConsumer());
-        }
-        this.connection.client.update_voice_status(undefined);
-    }
-
-    voice_playback_support() : boolean {
-        return this.connection.connected();
-    }
-
-    voice_send_support() : boolean {
-        return this.connection.connected();
-    }
-
-    private current_channel_codec() : number {
-        const chandler = this.connection.client;
-        return (chandler.getClient().currentChannel() || {properties: { channel_codec: 4}}).properties.channel_codec;
-    }
-
-    private handleVoiceEnded() {
-        const chandler = this.connection.client;
-        chandler.getClient().speaking = false;
-
-        if(!chandler.connected)
-            return false;
-
-        if(chandler.isMicrophoneMuted())
-            return false;
-
-        console.log(tr("Local voice ended"));
-        //TODO Send end? (Or is this already an automated thing?)
-    }
-
-    private on_voice_started() {
-        const chandler = this.connection.client;
-        if(chandler.isMicrophoneMuted()) {
-            /* evil hack due to the settings :D */
-            log.warn(LogCategory.VOICE, tr("Received local voice started event, even thou we're muted! Do not send any voice."));
-            if(this.handle) {
-                this.handle.enable_voice_send(false);
-            }
-            return;
-        }
-
-        log.info(LogCategory.VOICE, tr("Local voice started (Native)"));
-        this.handle.enable_voice_send(true);
-
-        const ch = chandler.getClient();
-        if(ch) ch.speaking = true;
-    }
-
-    getConnectionState(): VoiceConnectionStatus {
-        return VoiceConnectionStatus.Connected;
-    }
-
-    voice_recorder(): RecorderProfile {
-        return this._audio_source;
-    }
-
-    available_clients(): VoiceClient[] {
-        return this.handle.available_clients().map(e => Object.assign(e, {
-            support_latency_settings() { return true; },
-            reset_latency_settings() {
-                const stream = this.get_stream();
-                stream.set_buffer_latency(0.080);
-                stream.set_buffer_max_latency(0.5);
-                return this.latency_settings();
-            },
-            latency_settings(settings?: LatencySettings) : LatencySettings {
-                const stream = this.get_stream();
-                if(typeof settings !== "undefined") {
-                    stream.set_buffer_latency(settings.min_buffer / 1000);
-                    stream.set_buffer_max_latency(settings.max_buffer / 100);
-                }
-                return {
-                    max_buffer: Math.floor(stream.get_buffer_max_latency() * 1000),
-                    min_buffer: Math.floor(stream.get_buffer_latency() * 1000)
-                };
-            },
-
-            support_flush() { return true; },
-            flush() {
-                const stream = this.get_stream();
-                stream.flush_buffer();
-            }
-        }) as any); /* cast to any because of: Type 'import("/mnt/d/TeaSpeak/client_linux/client/imports/shared-app/connection/ConnectionBase").voice.PlayerState' is not assignable to type 'import("tc-native/connection").PlayerState' */
-    }
-
-    find_client(client_id: number) : VoiceClient | undefined {
-        for(const client of this.available_clients())
-            if(client.client_id === client_id)
-                return client;
-        return undefined;
-    }
-
-    unregister_client(client: VoiceClient): Promise<void> {
-        this.handle.unregister_client(client.client_id);
-        return Promise.resolve();
-    }
-
-    register_client(client_id: number): VoiceClient {
-        const client = this.handle.register_client(client_id);
-        const c = this.find_client(client_id);
-        c.reset_latency_settings();
-        return c;
-    }
-
-    decoding_supported(codec: number): boolean {
-        return this.handle.decoding_supported(codec);
-    }
-
-    encoding_supported(codec: number): boolean {
-        return this.handle.encoding_supported(codec);
-    }
-
-    get_encoder_codec(): number {
-        return this.handle.get_encoder_codec();
-    }
-
-    set_encoder_codec(codec: number) {
-        return this.handle.set_encoder_codec(codec);
-    }
-}

modules/renderer/hooks/Video.ts

@@ -0,0 +1,62 @@
+import * as loader from "tc-loader";
+import {Stage} from "tc-loader";
+import {ScreenCaptureDevice, setVideoDriver, VideoSource} from "tc-shared/video/VideoSource";
+import {WebVideoDriver, WebVideoSource} from "tc-shared/media/Video";
+import {desktopCapturer, remote} from "electron";
+import {requestMediaStreamWithConstraints} from "tc-shared/media/Stream";
+import {tr} from "tc-shared/i18n/localize";
+
+loader.register_task(Stage.JAVASCRIPT_INITIALIZING, {
+    priority: 10,
+    function: async () => {
+        const instance = new NativeVideoDriver();
+        await instance.initialize();
+        setVideoDriver(instance);
+    },
+    name: "Video init"
+});
+
+class NativeVideoDriver extends WebVideoDriver {
+    private currentScreenCaptureDevices: ScreenCaptureDevice[];
+
+    screenQueryAvailable(): boolean {
+        return true;
+    }
+
+    async queryScreenCaptureDevices(): Promise<ScreenCaptureDevice[]> {
+        const sources = await desktopCapturer.getSources({ fetchWindowIcons: true, types: ['window', 'screen'], thumbnailSize: { width: 480, height: 270 } });
+
+        return this.currentScreenCaptureDevices = sources.map(entry => {
+            return {
+                id: entry.id,
+                name: entry.name,
+
+                type: entry.display_id ? "full-screen" : "window",
+
+                appIcon: entry.appIcon?.toDataURL(),
+                appPreview: entry.thumbnail?.toDataURL()
+            }
+        })
+    }
+
+    async createScreenSource(id: string | undefined, allowFocusLoss: boolean): Promise<VideoSource> {
+        const result = await requestMediaStreamWithConstraints({
+            mandatory: {
+                chromeMediaSource: 'desktop',
+                chromeMediaSourceId: id,
+            }
+        } as any, "video");
+
+        if(typeof result === "string") {
+            throw result;
+        }
+
+        if(!allowFocusLoss) {
+            /* redraw focus to our window since we lost it after requesting the screen capture */
+            remote.getCurrentWindow().focus();
+        }
+
+        const name = this.currentScreenCaptureDevices.find(e => e.id === id)?.name || tr("Screen device");
+        return new WebVideoSource(id, name, result);
+    }
+}

modules/renderer/index.ts

@@ -166,6 +166,7 @@ loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, {
             await import("./hooks/ChangeLogClient");
             await import("./hooks/Backend");
             await import("./hooks/MenuBar");
+            await import("./hooks/Video");
 
             await import("./UnloadHandler");
             await import("./WindowsTrayHandler");

native/serverconnection/exports/exports.d.ts

@@ -83,7 +83,6 @@ declare module "tc-native/connection" {
     export function spawn_server_connection() : NativeServerConnection;
     export function destroy_server_connection(connection: NativeServerConnection);
 
-
     export namespace ft {
         export interface TransferObject {
             name: string;