radioconda/rerender.py

#!/usr/bin/env python3
import contextlib
import pathlib
import shutil
from typing import Any, Dict, Optional

import conda_lock
from conda_package_streaming.package_streaming import stream_conda_component
from conda_package_streaming.url import conda_reader_for_url
import diff_match_patch
import yaml


def name_from_pkg_spec(spec: str):
    return (
        spec.split(sep=None, maxsplit=1)[0]
        .split(sep="=", maxsplit=1)[0]
        .split(sep="::", maxsplit=1)[-1]
    )


def write_env_file(
    env_dict: Dict[str, Any],
    file_path: pathlib.Path,
    name: Optional[str] = None,
    version: Optional[str] = None,
    platform: Optional[str] = None,
    variables: Optional[dict] = None,
):
    if name:
        env_dict["name"] = name
    if version:
        env_dict["version"] = version
    if platform:
        env_dict["platform"] = platform
    if variables:
        env_dict["variables"] = variables
    with file_path.open("w") as f:
        yaml.safe_dump(env_dict, stream=f)

    return env_dict


def render_metapackage_environments(
    lockfile_path: pathlib.Path,
    requested_pkg_names: Dict[str, Any],
    name: str,
    version: str,
    output_dir: pathlib.Path,
) -> None:
    lock_content = conda_lock.conda_lock.parse_conda_lock_file(lockfile_path)
    lock_work_dir = lockfile_path.parent

    # render main env spec into environment file for creating metapackage
    conda_lock.conda_lock.do_render(
        lockfile=lock_content,
        kinds=("env",),
        filename_template=f"{lock_work_dir}/{name}-{{platform}}.metapackage",
    )
    # process and save rendered platform env files to the output directory
    for platform_env_yaml_path in lock_work_dir.glob("*.metapackage.yml"):
        platform_env_yaml_name = platform_env_yaml_path.name.partition(".")[0]
        platform = platform_env_yaml_name.split(sep="-", maxsplit=1)[1]

        with platform_env_yaml_path.open("r") as f:
            platform_env_dict = yaml.safe_load(f)

        dependencies = sorted(platform_env_dict["dependencies"])
        # filter the dependency list by the explicitly listed package names
        platform_env_dict["dependencies"] = [
            spec
            for spec in dependencies
            if name_from_pkg_spec(spec) in requested_pkg_names
        ]

        if platform.startswith("win"):
            variables = dict(
                GR_PREFIX="", GRC_BLOCKS_PATH="", UHD_PKG_PATH="", VOLK_PREFIX=""
            )
        else:
            variables = None
        write_env_file(
            env_dict=platform_env_dict,
            file_path=output_dir / f"{platform_env_yaml_name}.yml",
            name=name,
            version=version,
            platform=platform,
            variables=variables,
        )


def render_constructors(
    lockfile_path: pathlib.Path,
    requested_pkg_names: Dict[str, Any],
    name: str,
    version: str,
    company: str,
    license_file: pathlib.Path,
    output_dir: pathlib.Path,
    builder_lockfile_path: pathlib.Path,
) -> None:
    lock_content = conda_lock.conda_lock.parse_conda_lock_file(lockfile_path)
    lock_work_dir = lockfile_path.parent

    builder_lock_content = conda_lock.conda_lock.parse_conda_lock_file(
        builder_lockfile_path
    )
    constructor_lockdeps = [
        lockdep
        for lockdep in builder_lock_content.package
        if lockdep.name == "constructor"
    ]

    # render main + installer env specs into environment file for creating installer
    conda_lock.conda_lock.do_render(
        lockfile=lock_content,
        kinds=("env",),
        filename_template=f"{lock_work_dir}/{name}-{{platform}}.constructor",
        extras=("installer",),
    )

    for platform_env_yaml_path in lock_work_dir.glob("*.constructor.yml"):
        constructor_name = platform_env_yaml_path.name.partition(".")[0]
        platform = constructor_name.split(sep="-", maxsplit=1)[1]

        with platform_env_yaml_path.open("r") as f:
            platform_env_dict = yaml.safe_load(f)

        # filter requested_pkg_names by locked environment to account for selectors
        platform_env_pkg_names = [
            name_from_pkg_spec(spec) for spec in platform_env_dict["dependencies"]
        ]
        user_requested_specs = [
            name for name in requested_pkg_names if name in platform_env_pkg_names
        ]

        construct_dict = dict(
            name=name,
            version=version,
            company=company,
            channels=platform_env_dict["channels"],
            specs=sorted(platform_env_dict["dependencies"]),
            user_requested_specs=user_requested_specs,
            initialize_by_default=False if platform.startswith("win") else True,
            installer_type="all",
            keep_pkgs=True,
            license_file="LICENSE",
            register_python_default=False,
            write_condarc=True,
            condarc=dict(
                channels=platform_env_dict["channels"],
                channel_priority="strict",
            ),
        )
        if platform.startswith("win"):
            construct_dict["post_install"] = "post_install.bat"
            # point to template that we generate at build time with a patch over default
            construct_dict["nsis_template"] = "main.nsi.tmpl"
        else:
            construct_dict["post_install"] = "post_install.sh"

        constructor_dir = output_dir / constructor_name
        if constructor_dir.exists():
            shutil.rmtree(constructor_dir)
        constructor_dir.mkdir(parents=True)

        # copy license to the constructor directory
        shutil.copy(license_file, constructor_dir / "LICENSE")

        # write the post_install scripts referenced in the construct dict
        if platform.startswith("win"):
            with (constructor_dir / "post_install.bat").open("w") as f:
                f.write(
                    "\n".join(
                        (
                            r'echo {"env_vars": {"GR_PREFIX": "", "GRC_BLOCKS_PATH": "", "UHD_PKG_PATH": "", "VOLK_PREFIX": ""}}>%PREFIX%\conda-meta\state',
                            r"del /q %PREFIX%\pkgs\*.tar.bz2",
                            r"del /q %PREFIX%\pkgs\*.conda",
                            "exit 0",
                            "",
                        )
                    )
                )
        else:
            with (constructor_dir / "post_install.sh").open("w") as f:
                f.write(
                    "\n".join(
                        (
                            "#!/bin/sh",
                            f'PREFIX="${{PREFIX:-$2/{name}}}"',
                            r"rm -f $PREFIX/pkgs/*.tar.bz2 $PREFIX/pkgs/*.conda",
                            "exit 0",
                            "",
                        )
                    )
                )

        construct_yaml_path = constructor_dir / "construct.yaml"
        with construct_yaml_path.open("w") as f:
            yaml.safe_dump(construct_dict, stream=f)

        if platform.startswith("win"):
            # patch constructor's nsis template
            constructor_platform_lockdeps = [
                lockdep
                for lockdep in constructor_lockdeps
                if lockdep.platform == platform
            ]
            if constructor_platform_lockdeps:
                lockdep = constructor_platform_lockdeps[0]

                # get the NSIS template that comes with the locked constructor package
                constructor_filename, constructor_pkg = conda_reader_for_url(
                    lockdep.url
                )
                with contextlib.closing(constructor_pkg):
                    for tar, member in stream_conda_component(
                        constructor_filename, constructor_pkg, component="pkg"
                    ):
                        if (
                            member.name
                            == "site-packages/constructor/nsis/main.nsi.tmpl"
                        ):
                            locked_nsi_tmpl = tar.extractfile(member).read().decode()
                            break
                # read the original and custom NSIS templates
                local_constructor_nsis = pathlib.Path("constructor") / "nsis"
                with (local_constructor_nsis / "main.nsi.tmpl.orig").open("r") as f:
                    orig_nsi_tmpl = f.read()
                with (local_constructor_nsis / "main.nsi.tmpl").open("r") as f:
                    custom_nsi_tmpl = f.read()

                # get patch from original NSIS template to locked version we will be
                # building with and apply it to the custom template
                dmp = diff_match_patch.diff_match_patch()
                line_orig, line_locked, line_array = dmp.diff_linesToChars(
                    orig_nsi_tmpl, locked_nsi_tmpl
                )
                diffs = dmp.diff_main(line_orig, line_locked, checklines=False)
                dmp.diff_charsToLines(diffs, line_array)
                patches = dmp.patch_make(orig_nsi_tmpl, diffs)
                patched_nsi_tmpl, results = dmp.patch_apply(patches, custom_nsi_tmpl)
                if not all(results):
                    raise RuntimeError("Conflicts found when patching NSIS template")

                # write patched template to constructor dir
                with (constructor_dir / "main.nsi.tmpl").open("w") as f:
                    f.write(patched_nsi_tmpl)

                # update orig and custom with locked and patched
                with (local_constructor_nsis / "main.nsi.tmpl.orig").open("w") as f:
                    f.write(locked_nsi_tmpl)
                with (local_constructor_nsis / "main.nsi.tmpl").open("w") as f:
                    f.write(patched_nsi_tmpl)


def render(
    environment_file: pathlib.Path,
    installer_environment_file: pathlib.Path,
    builder_environment_file: pathlib.Path,
    version: str,
    company: str,
    license_file: pathlib.Path,
    output_dir: pathlib.Path,
    conda_exe: pathlib.Path,
    dirty: Optional[bool] = False,
    keep_workdir: Optional[bool] = False,
) -> None:
    with environment_file.open("r") as f:
        env_yaml_data = yaml.safe_load(f)
    with installer_environment_file.open("r") as f:
        base_env_yaml_data = yaml.safe_load(f)

    env_name = env_yaml_data["name"]
    env_pkg_names = [name_from_pkg_spec(spec) for spec in env_yaml_data["dependencies"]]
    base_env_pkg_names = [
        name_from_pkg_spec(spec) for spec in base_env_yaml_data["dependencies"]
    ]

    if not license_file.exists():
        raise ValueError(f"Cannot find license file: {license_file}")

    if output_dir.exists() and not dirty:
        shutil.rmtree(output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    # working dir for conda-lock outputs that we use as intermediates
    lock_work_dir = output_dir / "lockwork"
    lock_work_dir.mkdir(parents=True, exist_ok=True)

    # create the locked build environment specification
    builder_lockfile_path = output_dir / "buildenv.conda-lock.yml"
    conda_lock.conda_lock.run_lock(
        environment_files=[builder_environment_file],
        conda_exe=conda_exe,
        mamba=True,
        micromamba=True,
        kinds=("lock",),
        lockfile_path=builder_lockfile_path,
    )

    # read environment files and create the lock file
    lockfile_path = lock_work_dir / f"{env_name}.conda-lock.yml"
    conda_lock.conda_lock.run_lock(
        environment_files=[environment_file, installer_environment_file],
        conda_exe=conda_exe,
        mamba=True,
        micromamba=True,
        kinds=("lock",),
        lockfile_path=lockfile_path,
    )

    # render main environment specs into explicit .lock files for reproducibility
    lock_content = conda_lock.conda_lock.parse_conda_lock_file(lockfile_path)
    conda_lock.conda_lock.do_render(
        lockfile=lock_content,
        kinds=("explicit",),
        filename_template=f"{output_dir}/{env_name}-{{platform}}.lock",
    )

    # create the environment specification files for the metapackages
    render_metapackage_environments(
        lockfile_path=lockfile_path,
        requested_pkg_names=env_pkg_names,
        name=env_name,
        version=version,
        output_dir=output_dir,
    )

    # create the rendered constructor directories
    render_constructors(
        lockfile_path=lockfile_path,
        requested_pkg_names=sorted(env_pkg_names + base_env_pkg_names),
        name=env_name,
        version=version,
        company=company,
        license_file=license_file,
        output_dir=output_dir,
        builder_lockfile_path=builder_lockfile_path,
    )

    # clean up conda-lock work dir
    if not keep_workdir:
        shutil.rmtree(lock_work_dir)


if __name__ == "__main__":
    import argparse
    import datetime
    import os

    cwd = pathlib.Path(".").absolute()
    here = pathlib.Path(__file__).parent.absolute().relative_to(cwd)
    distname = os.getenv("DISTNAME", "radioconda")
    source = "/".join(
        (
            os.getenv("GITHUB_SERVER_URL", "https://github.com"),
            os.getenv("GITHUB_REPOSITORY", "ryanvolz/radioconda"),
        )
    )

    dt = datetime.datetime.now()
    version = dt.strftime("%Y.%m.%d")

    parser = argparse.ArgumentParser(
        description=(
            "Re-render installer specification directories to be used by conda"
            " constructor."
        )
    )
    parser.add_argument(
        "environment_file",
        type=pathlib.Path,
        nargs="?",
        default=here / f"{distname}.yaml",
        help=(
            "YAML file defining a distribution, with a 'name' string and"
            " 'channels', 'platforms', and 'dependencies' lists."
            " (default: %(default)s)"
        ),
    )
    parser.add_argument(
        "installer_environment_file",
        type=pathlib.Path,
        nargs="?",
        default=here / f"{distname}_installer.yaml",
        help=(
            "YAML file defining additional packages for the installer, with a 'name'"
            " string and 'channels' and 'dependencies' lists."
            " (default: %(default)s)"
        ),
    )
    parser.add_argument(
        "builder_environment_file",
        type=pathlib.Path,
        nargs="?",
        default=here / "buildenv.yaml",
        help=(
            "YAML file defining the builder environment for running conda-constructor,"
            " with a 'name' string and 'channels', 'platforms', and 'dependencies'"
            " lists."
            " (default: %(default)s)"
        ),
    )
    parser.add_argument(
        "-v",
        "--version",
        type=str,
        default=version,
        help=(
            "Version tag for the installer, defaults to the current date."
            " (default: %(default)s)"
        ),
    )
    parser.add_argument(
        "--company",
        type=str,
        default=source,
        help=(
            "Name of the company/entity who is responsible for the installer."
            " (default: %(default)s)"
        ),
    )
    parser.add_argument(
        "-l",
        "--license_file",
        type=pathlib.Path,
        default=here / "LICENSE",
        help=(
            "File containing the license that applies to the installer."
            " (default: %(default)s)"
        ),
    )
    parser.add_argument(
        "-o",
        "--output_dir",
        type=pathlib.Path,
        default=here / "installer_specs",
        help=(
            "Output directory in which the installer specifications will be rendered."
            " (default: %(default)s)"
        ),
    )
    parser.add_argument(
        "--dirty",
        action="store_true",
        default=False,
        help=("Do not clean up output_dir before rendering. (default: %(default)s)"),
    )
    parser.add_argument(
        "--keep-workdir",
        action="store_true",
        default=False,
        help=(
            "Keep conda-lock working directory ({output_dir}/lockwork) of intermediate"
            " outputs. (default: %(default)s)"
        ),
    )
    parser.add_argument(
        "--conda-exe",
        type=pathlib.Path,
        default=None,
        help=(
            "Path to the conda (or mamba or micromamba) executable to use."
            " (default: search for conda/mamba/micromamba)"
        ),
    )

    args = parser.parse_args()

    render(
        environment_file=args.environment_file,
        installer_environment_file=args.installer_environment_file,
        builder_environment_file=args.builder_environment_file,
        version=args.version,
        company=args.company,
        license_file=args.license_file,
        output_dir=args.output_dir,
        conda_exe=args.conda_exe,
        dirty=args.dirty,
        keep_workdir=args.keep_workdir,
    )