TeaSpeak-Client/modules/core/ui-loader/loader.ts

import {is_debug} from "../main-window";
import * as moment from "moment";
import * as request from "request";
import * as querystring from "querystring";
import * as fs from "fs-extra";
import * as os from "os";
const UUID = require('pure-uuid');
import * as path from "path";
import * as zlib from "zlib";
import * as tar from "tar-stream";
import {Arguments, processArguments} from "../../shared/process-arguments";
import {parse_version} from "../../shared/version";

import * as electron from "electron";
import MessageBoxOptions = Electron.MessageBoxOptions;
import {current_version, execute_graphical} from "../app-updater";
import * as local_ui_cache from "./local_ui_cache";
import {WriteStream} from "fs";

const TIMEOUT = 30000;

interface RemoteURL {
    (): string;
    cached?: string;
}
const remote_url: RemoteURL = () => {
    if(remote_url.cached)
        return remote_url.cached;
    const default_path = is_debug ? "http://localhost/home/TeaSpeak/Web-Client/client-api/environment/" : "https://clientapi.teaspeak.de/";
    return remote_url.cached = (processArguments.has_value(...Arguments.SERVER_URL) ? processArguments.value(...Arguments.SERVER_URL) : default_path);
};

export interface VersionedFile {
    name: string,
    hash: string,
    path: string,
    type: string,

    local_url: () => Promise<string>
}

function generate_tmp() : Promise<string> {
    if(generate_tmp.promise) return generate_tmp.promise;

    return (generate_tmp.promise = fs.mkdtemp(path.join(os.tmpdir(), "TeaClient-")).then(path => {
        process.on('exit', event => {
            try {
                if(fs.pathExistsSync(path))
                    fs.removeSync(path);
            } catch (e) {
                console.warn("Failed to delete temp directory: %o", e);
            }
        });
        
        global["browser-root"] = path;
        console.log("Local browser path: %s", path);
        return Promise.resolve(path);
    }));
}

namespace generate_tmp {
    export let promise: Promise<string>;
}

function get_raw_app_files() : Promise<VersionedFile[]> {
    return new Promise<VersionedFile[]>((resolve, reject) => {
        const url = remote_url() + "api.php?" + querystring.stringify({
            type: "files",
        });
        console.debug("Requesting file list from %s", url);
        request.get(url, {
            timeout: TIMEOUT
        }, (error, response, body: string) => {
            if(error) {
                reject(error);
                return;
            }

            if(!response) {
                reject("missing response object");
                return;
            }

            if(response.statusCode != 200) { setImmediate(reject, "invalid status code " + response.statusCode + " for " + url); return; }
            if(parseInt(response.headers["info-version"] as string) != 1 && !processArguments.has_flag(Arguments.UPDATER_UI_IGNORE_VERSION)) { setImmediate(reject, "Invalid response version (" + response.headers["info-version"] + "). Update your app manually!"); return; }
            if(!body) {
                setImmediate(reject, "invalid body. (Missing)");
                return;
            }
            let result: VersionedFile[] = [];

            body.split("\n").forEach(entry => {
                if(entry.length == 0) return;

                let info = entry.split("\t");
                if(info[0] == "type") return;

                result.push({
                    type: info[0],
                    hash: info[1],
                    path: info[2],
                    name: info[3]
                } as VersionedFile);
            });
            setImmediate(resolve, result);
        });
    });
}

async function download_raw_app_files() : Promise<VersionedFile[]> {
    const local_temp_path = await generate_tmp();
    return get_raw_app_files().then(response => {
        for(let file of response) {
            const full_path = path.join(local_temp_path, file.path, file.name);
            file.local_url = () => fs.mkdirs(path.dirname(full_path)).then(() => new Promise<string>((resolve, reject) => {
                const write_stream = fs.createWriteStream(full_path);
                request.get(remote_url() + "api.php?" + querystring.stringify({
                    type: "file",
                    path: file.path,
                    name: file.name
                }), {
                    timeout: TIMEOUT
                }).on('response', function(response) {
                    if(response.statusCode != 200) {
                        setImmediate(reject, "invalid status code " + response.statusCode + " for file " + file.name + " (" + file.path + ")");
                        return;
                    }
                }).on('complete', event => {
                }).on('error', error => {
                    try { write_stream.close(); } catch (e) { }
                    setImmediate(reject, error);
                }).pipe(write_stream)
                .on('finish', event => {
                    try { write_stream.close(); } catch (e) { }
                    setImmediate(resolve, file.path + "/" + file.name);
                }).on('error', error => {
                    try { write_stream.close(); } catch (e) { }
                    setImmediate(reject, error);
                });
            }));
        }
        return Promise.resolve(response);
    }).catch(error => {
        console.log("Failed to get file list: %o", error);
        return Promise.reject("Failed to get file list (" + error + ")");
    })
}

async function client_shipped_ui() : Promise<local_ui_cache.CachedUIPack | undefined> {
    const app_path = electron.app.getAppPath();
    if(!app_path.endsWith(".asar"))
        return undefined;

    const base_path = path.join(path.dirname(app_path), "ui");
    //console.debug("Looking for client shipped UI pack at %s", base_path);
    if(!(await fs.pathExists(base_path)))
        return undefined;

    const info: {
        channel: string,
        version: string,
        git_hash: string,
        required_client: string,
        timestamp: number,
        filename: string
    } = await fs.readJson(path.join(base_path, "default_ui_info.json")) as any;

    return {
        download_timestamp: info.timestamp * 1000,
        status: "valid",
        invalid_reason: undefined,
        local_checksum: "none",
        local_file_path: path.join(path.join(path.dirname(app_path), "ui"), info.filename),
        pack_info: {
            channel: info.channel,
            min_client_version: info.required_client,
            timestamp: info.timestamp * 1000,
            version: info.version,
            versions_hash: info.git_hash
        }
    };
}

async function query_ui_pack_versions() : Promise<local_ui_cache.UIPackInfo[]> {
    const url = remote_url() + "api.php?" + querystring.stringify({
        type: "ui-info"
    });
    console.debug("Loading UI pack information (URL: %s)", url);

    let body = await new Promise<string>((resolve, reject) => request.get(url, { timeout: TIMEOUT }, (error, response, body: string) => {
        if(error)
            reject(error);
        else if(!response)
            reject("missing response object");
        else {
            if(response.statusCode !== 200)
                reject(response.statusCode + " " + response.statusMessage);
            else if(!body)
                reject("missing body in response");
            else
                resolve(body);
        }
    }));

    let response;
    try {
        response = JSON.parse(body);
    } catch (error) {
        console.error("Received unparsable response for UI pack info. Response: %s", body);
        throw "failed to parse response";
    }

    if(!response["success"])
        throw "request failed: " + (response["msg"] || "unknown error");

    if(!Array.isArray(response["versions"])) {
        console.error("Response object misses 'versions' tag or has an invalid value. Object: %o", response);
        throw "response contains invalid data";
    }

    let ui_versions: local_ui_cache.UIPackInfo[] = [];
    for(const entry of response["versions"]) {
        ui_versions.push({
            channel: entry["channel"],
            versions_hash: entry["git-ref"],
            version: entry["version"],
            timestamp: parseInt(entry["timestamp"]) * 1000, /* server provices that stuff in seconds */
            min_client_version: entry["required_client"]
        });
    }

    return ui_versions;
}

async function download_ui_pack(version: local_ui_cache.UIPackInfo) : Promise<local_ui_cache.CachedUIPack> {
    const target_file = path.join(local_ui_cache.cache_path(), version.channel + "_" + version.versions_hash + "_" + version.timestamp + ".tar.gz");
    if(await fs.pathExists(target_file)) {
        try {
            await fs.remove(target_file);
        } catch (error) {
            console.error("Tried to download UI version %s, but we failed to delete the old file: %o", version.versions_hash, error);
            throw "failed to delete old file";
        }
    }
    try {
        await fs.mkdirp(path.dirname(target_file));
    } catch (error) {
        console.error("Failed to create target UI pack download directory at %s: %o", path.dirname(target_file), error);
        throw "failed to create target directories";
    }

    await new Promise((resolve, reject) => {
        let fstream: WriteStream;
        try {
            request.get(remote_url() + "api.php?" + querystring.stringify({
                "type": "ui-download",
                "git-ref": version.versions_hash,
                "version": version.version,
                "timestamp": Math.floor(version.timestamp / 1000), /* remote server has only the timestamp in seconds*/
                "channel": version.channel
            }), {
                timeout: TIMEOUT
            }).on('response', function(response: request.Response) {
                if(response.statusCode != 200)
                    reject(response.statusCode + " " + response.statusMessage);
            }).on('error', error => {
                reject(error);
            }).pipe(fstream = fs.createWriteStream(target_file)).on('finish', () => {
                try { fstream.close(); } catch (e) { }

                resolve();
            });
        } catch (error) {
            try { fstream.close(); } catch (e) { }

            reject(error);
        }
    });

    try {
        const cache = await local_ui_cache.load();
        const info: local_ui_cache.CachedUIPack = {
            pack_info: version,
            local_file_path: target_file,
            local_checksum: "none", //TODO!
            invalid_reason: undefined,
            status: "valid",
            download_timestamp: Date.now()
        };
        cache.cached_ui_packs.push(info);
        await local_ui_cache.save();
        return info;
    } catch (error) {
        console.error("Failed to register downloaded UI pack to the UI cache: %o", error);
        throw "failed to register downloaded UI pack to the UI cache";
    }
}

async function ui_pack_usable(version: local_ui_cache.CachedUIPack) : Promise<boolean> {
    if(version.status !== "valid") return false;
    return await fs.pathExists(version.local_file_path);
}

async function unpack_local_ui_pack(version: local_ui_cache.CachedUIPack) : Promise<string> {
    if(!await ui_pack_usable(version))
        throw "UI pack has been invalidated";

    const target_directory = await generate_tmp();
    if(!await fs.pathExists(target_directory))
        throw "failed to create temporary directory";

    const gunzip = zlib.createGunzip();
    const extract = tar.extract();
    let fpipe: fs.ReadStream;

    try {
        fpipe = fs.createReadStream(version.local_file_path);
    } catch (error) {
        console.error("Failed to open UI pack at %s: %o", version.local_file_path, error);
        throw "failed to open UI pack";
    }

    extract.on('entry', function(header: tar.Headers, stream, next) {
        if(header.type == 'file') {
            const target_file = path.join(target_directory, header.name);
            if(!fs.existsSync(path.dirname(target_file))) fs.mkdirsSync(path.dirname(target_file));

            stream.on('end', () => setImmediate(next));
            const wfpipe = fs.createWriteStream(target_file);
            stream.pipe(wfpipe);
        } else if(header.type == 'directory') {
            if(fs.existsSync(path.join(target_directory, header.name)))
                setImmediate(next);
            fs.mkdirs(path.join(target_directory, header.name)).catch(error => {
                console.warn("Failed to create unpacking dir " + path.join(target_directory, header.name));
                console.error(error);
            }).then(() => setImmediate(next));
        } else {
            console.warn("Invalid ui tar ball entry type (" + header.type + ")");
            return;
        }
    });

    const finish_promise = new Promise((resolve, reject) => {
        gunzip.on('error', event => {
            reject(event);
        });

        extract.on('finish', resolve);
        extract.on('error', event => {
            if(!event) return;
            reject(event);
        });

        fpipe.pipe(gunzip).pipe(extract);
    });

    try {
        await finish_promise;
    } catch(error) {
        console.error("Failed to extract UI files to %s: %o", target_directory, error);
        throw "failed to unpack the UI pack";
    }

    return target_directory;
}

async function load_files_from_dev_server(channel: string, stats_update: (message: string, index: number) => any) : Promise<String> {
    stats_update("Fetching files", 0);
    let files: VersionedFile[];
    try {
        files = await download_raw_app_files()
    } catch (error) {
        console.log("Failed to fetch raw UI file list: %o", error);
        let msg;
        if(error instanceof Error)
            msg = error.message;
        else if(typeof error === "string")
            msg = error;
        throw "failed to get file list" + (msg ? " (" + msg + ")" : "");
    }


    const max_simultaneously_downloads = 8;
    let pending_files: VersionedFile[] = files.slice(0);
    let current_downloads: {[key: string]: Promise<void>} = {};

    const update_download_status = () => {
        const indicator = (pending_files.length + Object.keys(current_downloads).length) / files.length;
        stats_update("Downloading raw UI files", 1 - indicator);
    };
    update_download_status();

    let errors: { file: VersionedFile; error: any }[] = [];
    while(pending_files.length > 0) {
        while(pending_files.length > 0 && Object.keys(current_downloads).length < max_simultaneously_downloads) {
            const file = pending_files.pop();
            current_downloads[file.hash] = file.local_url().catch(error => {
                errors.push({ file: file, error: error});
            }).then(() => {
                delete current_downloads[file.hash];
            });
        }

        update_download_status();
        await Promise.race(Object.keys(current_downloads).map(e => current_downloads[e]));

        if(errors.length > 0)
            break;
    }

    /* await full finish */
    while(Object.keys(current_downloads).length > 0) {
        update_download_status();
        await Promise.race(Object.keys(current_downloads).map(e => current_downloads[e]));
    }

    if(errors.length > 0) {
        console.log("Failed to load UI files (%d):", errors.length);
        for(const error of errors)
            console.error(" - %s: %o", path.join(error.file.path + error.file.name), error.error);
        throw "failed to download file " + path.join(errors[0].file.path + errors[0].file.name) + " (" + errors[0].error + ")\nView console for a full error report.";
    }

    console.log("Successfully loaded UI files from remote server.");
    /* generate_tmp has already been called an its the file destination */
    return path.join(await generate_tmp(), "index.html"); /* entry point */
}

async function stream_files_from_dev_server(channel: string, stats_update: (message: string, index: number) => any) : Promise<string> {
    return remote_url() + "index.html";
}

async function load_bundles_ui_pack(channel: string, stats_update: (message: string, index: number) => any) : Promise<String> {
    stats_update("Query local UI pack info", .33);
    const bundles_ui = await client_shipped_ui();
    if(!bundles_ui) throw "client has no bundled UI pack";

    stats_update("Unpacking bundled UI", .66);
    const result = await unpack_local_ui_pack(bundles_ui);
    stats_update("Local UI pack loaded", 1);
    console.log("Loaded bundles UI pack successfully. Version: {timestamp: %d, hash: %s}", bundles_ui.pack_info.timestamp, bundles_ui.pack_info.versions_hash);
    return path.join(result, "index.html");
}

async function load_cached_or_remote_ui_pack(channel: string, stats_update: (message: string, index: number) => any, ignore_new_version_timestamp: boolean) : Promise<String> {
    stats_update("Fetching info", 0);
    const ui_cache = await local_ui_cache.load();
    const bundles_ui = await client_shipped_ui();
    const client_version = await current_version();

    let available_versions: local_ui_cache.CachedUIPack[] = ui_cache.cached_ui_packs.filter(e => {
        if(e.status !== "valid")
            return false;

        if(bundles_ui) {
            if(e.pack_info.timestamp <= bundles_ui.download_timestamp)
                return false;
        }

        const required_version = parse_version(e.pack_info.min_client_version);
        return client_version.in_dev() || client_version.newer_than(required_version) || client_version.equals(required_version);
    });
    if(processArguments.has_flag(Arguments.UPDATER_UI_NO_CACHE)) {
        console.log("Ignoring local UI cache");
        available_versions = [];
    }

    let remote_version_dropped = false;
    /* remote version gathering */
    remote_loader: {
        stats_update("Loading remote info", .25);
        let remote_versions: local_ui_cache.UIPackInfo[];
        try {
            remote_versions = await query_ui_pack_versions();
        } catch (error) {
            if(available_versions.length === 0)
                throw "failed to query remote UI packs: " + error;
            console.error("Failed to query remote UI packs: %o", error);
            break remote_loader;
        }

        stats_update("Parsing UI packs", .40);
        const remote_version = remote_versions.find(e => e.channel === channel);
        if(!remote_version && available_versions.length === 0)
            throw "no UI pack available for channel " + channel;

        let newest_local_version = available_versions.map(e => e.pack_info.timestamp).reduce((a, b) => Math.max(a, b), bundles_ui ? bundles_ui.download_timestamp : 0);
        console.log("Remote version %d, Local version %d", remote_version.timestamp, newest_local_version);
        const required_version = parse_version(remote_version.min_client_version);
        if(required_version.newer_than(client_version) && !is_debug) {
            const result = await electron.dialog.showMessageBox({
                type: "question",
                message:
                    "Your client is outdated.\n" +
                    "Newer UI packs (>= " + remote_version.version + ", " + remote_version.versions_hash + ") require client " + remote_version.min_client_version + "\n" +
                    "Do you want to update your client?",
                title: "Client outdated!",
                buttons: ["Update client", available_versions.length === 0 ? "Close client" : "Ignore and use last possible"]
            } as MessageBoxOptions);

            if(result.response == 0) {
                if(!await execute_graphical(channel, true))
                    throw "Client outdated an no suitable UI pack versions found";
                else
                    return;
            } else {
                if(available_versions.length === 0) {
                    electron.app.exit(1);
                    return;
                }
            }
        } else if(remote_version.timestamp <= newest_local_version && !ignore_new_version_timestamp) {
            /* We've already a equal or newer version. Don't use the remote version */
            remote_version_dropped = !!bundles_ui && remote_version.timestamp > bundles_ui.download_timestamp; /* if remote is older than current bundled version its def. not a drop */
        } else {
            /* update is possible because the timestamp is newer than out latest local version */
            try {
                console.log("Downloading UI pack version (%d) %s. Forced: %s. Newest local version: %d", remote_version.timestamp,
                    remote_version.versions_hash, ignore_new_version_timestamp ? "true" : "false", newest_local_version);
                stats_update("Downloading new UI pack", .55);
                available_versions.push(await download_ui_pack(remote_version));
            } catch (error) {
                console.error("Failed to download new UI pack: %o", error);
            }
        }
    }

    stats_update("Unpacking UI", .70);
    available_versions.sort((a, b) => a.pack_info.timestamp - b.pack_info.timestamp);

    /* Only invalidate the version if any other succeeded to load. Else we might fucked up (no permission to write etc) */
    let invalidate_versions: local_ui_cache.CachedUIPack[] = [];
    const do_invalidate_versions = async () => {
        if(invalidate_versions.length > 0) {
            for(const version of invalidate_versions) {
                version.invalid_reason = "failed to unpack";
                version.status = "invalid";
            }
            await local_ui_cache.save();
        }
    };

    while(available_versions.length > 0) {
        const pack = available_versions.pop();
        console.log("Trying to load UI pack from %s (%s). Downloaded at %s", moment(pack.pack_info.timestamp).format("llll"), pack.pack_info.versions_hash, moment(pack.download_timestamp).format("llll"));

        try {
            const target = await unpack_local_ui_pack(pack);
            stats_update("UI pack loaded", 1);
            await do_invalidate_versions();
            return path.join(target, "index.html");
        } catch (error) {
            invalidate_versions.push(pack);
            console.log("Failed to unpack UI pack: %o", error);
        }
    }

    if(remote_version_dropped) {
        /* try again, but this time enforce a remote download */
        const result = await load_cached_or_remote_ui_pack(channel, stats_update, true);
        await do_invalidate_versions(); /* new UI pack seems to be successfully loaded */
        return result; /* if not succeeded an exception will be thrown */
    }

    throw "Failed to load any UI pack (local and remote)\nView the console for more details.\n";
}

enum UILoaderMethod {
    PACK,
    BUNDLED_PACK,
    RAW_FILES,
    DEVELOP_SERVER
}

export async function load_files(channel: string, stats_update: (message: string, index: number) => any) : Promise<String> {
    let enforced_loading_method = parseInt(processArguments.has_value(Arguments.UPDATER_UI_LOAD_TYPE) ? processArguments.value(Arguments.UPDATER_UI_LOAD_TYPE) : "-1") as UILoaderMethod;

    if(typeof UILoaderMethod[enforced_loading_method] !== "undefined") {
        switch (enforced_loading_method) {
            case UILoaderMethod.PACK:
                return await load_cached_or_remote_ui_pack(channel, stats_update, false);

            case UILoaderMethod.BUNDLED_PACK:
                return await load_bundles_ui_pack(channel, stats_update);

            case UILoaderMethod.RAW_FILES:
                return await load_files_from_dev_server(channel, stats_update);

            case UILoaderMethod.DEVELOP_SERVER:
                return await stream_files_from_dev_server(channel, stats_update);
        }
    }

    let first_error;
    if(is_debug) {
        try {
            return await load_files_from_dev_server(channel, stats_update);
        } catch(error) {
            console.warn("Failed to load raw UI files: %o", error);
            first_error = first_error || error;
        }
    }

    try {
        return await load_cached_or_remote_ui_pack(channel, stats_update, false);
    } catch(error) {
        console.warn("Failed to load cached/remote UI pack: %o", error);
        first_error = first_error || error;
    }

    try {
        return await load_bundles_ui_pack(channel, stats_update);
    } catch(error) {
        console.warn("Failed to load bundles UI pack: %o", error);
        first_error = first_error || error;
    }

    throw first_error;
}