diff --git a/radioconda.yaml b/radioconda.yaml index f629fce..7738063 100644 --- a/radioconda.yaml +++ b/radioconda.yaml @@ -1,4 +1,5 @@ name: radioconda +category: main channels: - conda-forge - ryanvolz diff --git a/radioconda_installer.yaml b/radioconda_installer.yaml index cb2940e..9cfdd34 100644 --- a/radioconda_installer.yaml +++ b/radioconda_installer.yaml @@ -1,7 +1,8 @@ name: radioconda_installer +category: installer channels: - conda-forge - - ryanvolz # [win] + - ryanvolz dependencies: - mamba - radioconda_console_shortcut # [win] diff --git a/rerender.py b/rerender.py index 30d3aba..8286902 100755 --- a/rerender.py +++ b/rerender.py @@ -1,56 +1,34 @@ #!/usr/bin/env python3 import pathlib import shutil -from typing import List, Optional +from typing import Any, Dict, Optional import conda_lock import yaml def name_from_pkg_spec(spec: str): - return spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0] - - -def lock_env_spec( - lock_spec: conda_lock.src_parser.LockSpecification, conda_exe: str -) -> conda_lock.src_parser.LockSpecification: - create_env_dict = conda_lock.conda_lock.solve_specs_for_arch( - conda=conda_exe, - channels=lock_spec.channels, - specs=lock_spec.specs, - platform=lock_spec.platform, + return ( + spec.split(sep=None, maxsplit=1)[0] + .split(sep="=", maxsplit=1)[0] + .split(sep="::", maxsplit=1)[-1] ) - pkgs = create_env_dict["actions"]["LINK"] - locked_specs = ["{name}={version}={build_string}".format(**pkg) for pkg in pkgs] - - locked_env_spec = conda_lock.src_parser.LockSpecification( - specs=sorted(locked_specs), - channels=lock_spec.channels, - platform=lock_spec.platform, - virtual_package_repo=conda_lock.virtual_package.default_virtual_package_repodata(), - ) - - return locked_env_spec def write_env_file( - env_spec: conda_lock.src_parser.LockSpecification, + env_dict: Dict[str, Any], file_path: pathlib.Path, name: Optional[str] = None, version: Optional[str] = None, + platform: Optional[str] = None, variables: Optional[dict] = None, ): - env_dict = dict( - name=name, - version=version, - platform=env_spec.platform, - channels=env_spec.channels, - dependencies=env_spec.specs, - ) if name: env_dict["name"] = name if version: env_dict["version"] = version + if platform: + env_dict["platform"] = platform if variables: env_dict["variables"] = variables with file_path.open("w") as f: @@ -59,142 +37,37 @@ def write_env_file( return env_dict -def write_lock_file( - lock_spec: conda_lock.src_parser.LockSpecification, - file_path: pathlib.Path, - conda_exe: str, -): - lockfile_contents = conda_lock.conda_lock.create_lockfile_from_spec( - conda=conda_exe, spec=lock_spec, kind="explicit" - ) - - def sanitize_lockfile_line(line): - line = line.strip() - if line == "": - return "#" - else: - return line - - lockfile_contents = [sanitize_lockfile_line(ln) for ln in lockfile_contents] - - with file_path.open("w") as f: - f.write("\n".join(lockfile_contents) + "\n") - - return lockfile_contents - - -def render_constructor( - lock_spec: conda_lock.src_parser.LockSpecification, +def render_metapackage_environments( + lockfile_path: pathlib.Path, + requested_pkg_names: Dict[str, Any], name: str, version: str, - company: str, - license_file: pathlib.Path, output_dir: pathlib.Path, -) -> dict: - platform = lock_spec.platform - constructor_name = f"{name}-{platform}" +) -> None: + lock_content = conda_lock.conda_lock.parse_conda_lock_file(lockfile_path) + lock_work_dir = lockfile_path.parent - construct_dict = dict( - name=name, - version=version, - company=company, - condarc=dict( - channels=lock_spec.channels, - channel_priority="strict", - ), - channels=lock_spec.channels, - specs=lock_spec.specs, - initialize_by_default=False if platform.startswith("win") else True, - installer_type="all", - keep_pkgs=True, - license_file="LICENSE", - register_python_default=False, - write_condarc=True, + # render main env spec into environment file for creating metapackage + conda_lock.conda_lock.do_render( + lockfile=lock_content, + kinds=("env",), + filename_template=f"{lock_work_dir}/{name}-{{platform}}.metapackage", ) - if platform.startswith("win"): - construct_dict["post_install"] = "post_install.bat" - # point to template that we generate at build time with a patch over default - construct_dict["nsis_template"] = "main.nsi.tmpl" - else: - construct_dict["post_install"] = "post_install.sh" + # process and save rendered platform env files to the output directory + for platform_env_yaml_path in lock_work_dir.glob("*.metapackage.yml"): + platform_env_yaml_name = platform_env_yaml_path.name.partition(".")[0] + platform = platform_env_yaml_name.split(sep="-", maxsplit=1)[1] - constructor_dir = output_dir / constructor_name - if constructor_dir.exists(): - shutil.rmtree(constructor_dir) - constructor_dir.mkdir(parents=True) + with platform_env_yaml_path.open("r") as f: + platform_env_dict = yaml.safe_load(f) - # copy license to the constructor directory - shutil.copy(license_file, constructor_dir / "LICENSE") - - # write the post_install scripts referenced in the construct dict - if platform.startswith("win"): - with (constructor_dir / "post_install.bat").open("w") as f: - f.write( - "\n".join( - ( - r'echo {"env_vars": {"GR_PREFIX": "", "GRC_BLOCKS_PATH": "", "UHD_PKG_PATH": "", "VOLK_PREFIX": ""}}>%PREFIX%\conda-meta\state', - r"del /q %PREFIX%\pkgs\*.tar.bz2", - "exit 0", - "", - ) - ) - ) - else: - with (constructor_dir / "post_install.sh").open("w") as f: - f.write( - "\n".join( - ( - "#!/bin/sh", - f'PREFIX="${{PREFIX:-$2/{name}}}"', - r"rm -f $PREFIX/pkgs/*.tar.bz2", - "exit 0", - "", - ) - ) - ) - - construct_yaml_path = constructor_dir / "construct.yaml" - with construct_yaml_path.open("w") as f: - yaml.safe_dump(construct_dict, stream=f) - - return construct_dict - - -def render_platforms( - environment_file: pathlib.Path, - installer_environment_file: pathlib.Path, - version: str, - company: str, - license_file: pathlib.Path, - output_dir: pathlib.Path, - conda_exe: str, -) -> dict: - with environment_file.open("r") as f: - env_yaml_data = yaml.safe_load(f) - - env_name = env_yaml_data["name"] - platforms = env_yaml_data["platforms"] - - if not license_file.exists(): - raise ValueError(f"Cannot find license file: {license_file}") - - if output_dir.exists(): - shutil.rmtree(output_dir) - output_dir.mkdir(parents=True) - - rendered_platforms = {} - - for platform in platforms: - output_name = f"{env_name}-{platform}" - - # get the environment specification for the list of packages from the env file - env_spec = conda_lock.conda_lock.parse_environment_file( - environment_file=environment_file, platform=platform - ) - env_pkg_names = [name_from_pkg_spec(spec) for spec in env_spec.specs] - - # lock the full environment specification to specific versions and builds - locked_env_spec = lock_env_spec(env_spec, conda_exe) + dependencies = sorted(platform_env_dict["dependencies"]) + # filter the dependency list by the explicitly listed package names + platform_env_dict["dependencies"] = [ + spec + for spec in dependencies + if name_from_pkg_spec(spec) in requested_pkg_names + ] if platform.startswith("win"): variables = dict( @@ -202,87 +75,192 @@ def render_platforms( ) else: variables = None - - # write the full environment specification to a lock file (to install from file) - lockfile_contents = write_lock_file( - lock_spec=locked_env_spec, - file_path=output_dir / f"{output_name}.lock", - conda_exe=conda_exe, - ) - - # filter the full package spec by the explicitly listed package names - metapackage_pkg_specs = [ - spec - for spec in locked_env_spec.specs - if name_from_pkg_spec(spec) in env_pkg_names - ] - metapackage_spec = conda_lock.src_parser.LockSpecification( - specs=metapackage_pkg_specs, - channels=locked_env_spec.channels, - platform=locked_env_spec.platform, - ) - - # write the filtered environment spec to a yaml file (to build metapackage) - locked_env_dict = write_env_file( - env_spec=metapackage_spec, - file_path=output_dir / f"{output_name}.yml", - name=env_name, + write_env_file( + env_dict=platform_env_dict, + file_path=output_dir / f"{platform_env_yaml_name}.yml", + name=name, version=version, + platform=platform, variables=variables, ) - # add installer-only (base environment) packages and lock those too - installer_pkg_spec = conda_lock.conda_lock.parse_environment_file( - environment_file=installer_environment_file, platform=platform - ) - installer_spec = conda_lock.src_parser.LockSpecification( - specs=sorted(locked_env_spec.specs + installer_pkg_spec.specs), - channels=sorted( - set(locked_env_spec.channels) | set(installer_pkg_spec.channels) - ), - platform=locked_env_spec.platform, - ) - locked_installer_spec = lock_env_spec(installer_spec, conda_exe) - # get a set of only the packages to put in the constructor specification - # taken from the installer-only list and those explicitly selected originally - constructor_pkg_names = set( - name_from_pkg_spec(spec) - for spec in env_spec.specs + installer_pkg_spec.specs - ) +def render_constructors( + lockfile_path: pathlib.Path, + requested_pkg_names: Dict[str, Any], + name: str, + version: str, + company: str, + license_file: pathlib.Path, + output_dir: pathlib.Path, +) -> None: + lock_content = conda_lock.conda_lock.parse_conda_lock_file(lockfile_path) + lock_work_dir = lockfile_path.parent - # filter the installer spec by the constructor package names - constructor_pkg_specs = [ - spec - for spec in locked_installer_spec.specs - if name_from_pkg_spec(spec) in constructor_pkg_names + # render main + installer env specs into environment file for creating installer + conda_lock.conda_lock.do_render( + lockfile=lock_content, + kinds=("env",), + filename_template=f"{lock_work_dir}/{name}-{{platform}}.constructor", + extras=("installer",), + ) + + for platform_env_yaml_path in lock_work_dir.glob("*.constructor.yml"): + constructor_name = platform_env_yaml_path.name.partition(".")[0] + platform = constructor_name.split(sep="-", maxsplit=1)[1] + + with platform_env_yaml_path.open("r") as f: + platform_env_dict = yaml.safe_load(f) + + # filter requested_pkg_names by locked environment to account for selectors + platform_env_pkg_names = [ + name_from_pkg_spec(spec) for spec in platform_env_dict["dependencies"] + ] + user_requested_specs = [ + name for name in requested_pkg_names if name in platform_env_pkg_names ] - constructor_spec = conda_lock.src_parser.LockSpecification( - specs=constructor_pkg_specs, - channels=locked_installer_spec.channels, - platform=locked_installer_spec.platform, - ) - # create the rendered constructor directory - constructor_dict = render_constructor( - lock_spec=constructor_spec, - name=env_name, + construct_dict = dict( + name=name, version=version, company=company, - license_file=license_file, - output_dir=output_dir, + channels=platform_env_dict["channels"], + specs=sorted(platform_env_dict["dependencies"]), + user_requested_specs=user_requested_specs, + initialize_by_default=False if platform.startswith("win") else True, + installer_type="all", + keep_pkgs=True, + license_file="LICENSE", + register_python_default=False, + write_condarc=True, + condarc=dict( + channels=platform_env_dict["channels"], + channel_priority="strict", + ), ) + if platform.startswith("win"): + construct_dict["post_install"] = "post_install.bat" + # point to template that we generate at build time with a patch over default + construct_dict["nsis_template"] = "main.nsi.tmpl" + else: + construct_dict["post_install"] = "post_install.sh" - # aggregate output - rendered_platforms[output_name] = dict( - locked_env_spec=locked_env_spec, - locked_env_dict=locked_env_dict, - lockfile_contents=lockfile_contents, - locked_installer_spec=locked_installer_spec, - constructor_dict=constructor_dict, - ) + constructor_dir = output_dir / constructor_name + if constructor_dir.exists(): + shutil.rmtree(constructor_dir) + constructor_dir.mkdir(parents=True) - return rendered_platforms + # copy license to the constructor directory + shutil.copy(license_file, constructor_dir / "LICENSE") + + # write the post_install scripts referenced in the construct dict + if platform.startswith("win"): + with (constructor_dir / "post_install.bat").open("w") as f: + f.write( + "\n".join( + ( + r'echo {"env_vars": {"GR_PREFIX": "", "GRC_BLOCKS_PATH": "", "UHD_PKG_PATH": "", "VOLK_PREFIX": ""}}>%PREFIX%\conda-meta\state', + r"del /q %PREFIX%\pkgs\*.tar.bz2", + r"del /q %PREFIX%\pkgs\*.conda", + "exit 0", + "", + ) + ) + ) + else: + with (constructor_dir / "post_install.sh").open("w") as f: + f.write( + "\n".join( + ( + "#!/bin/sh", + f'PREFIX="${{PREFIX:-$2/{name}}}"', + r"rm -f $PREFIX/pkgs/*.tar.bz2 $PREFIX/pkgs/*.conda", + "exit 0", + "", + ) + ) + ) + + construct_yaml_path = constructor_dir / "construct.yaml" + with construct_yaml_path.open("w") as f: + yaml.safe_dump(construct_dict, stream=f) + + +def render( + environment_file: pathlib.Path, + installer_environment_file: pathlib.Path, + version: str, + company: str, + license_file: pathlib.Path, + output_dir: pathlib.Path, + conda_exe: pathlib.Path, + dirty: Optional[bool] = False, + keep_workdir: Optional[bool] = False, +) -> None: + with environment_file.open("r") as f: + env_yaml_data = yaml.safe_load(f) + with installer_environment_file.open("r") as f: + base_env_yaml_data = yaml.safe_load(f) + + env_name = env_yaml_data["name"] + env_pkg_names = [name_from_pkg_spec(spec) for spec in env_yaml_data["dependencies"]] + base_env_pkg_names = [ + name_from_pkg_spec(spec) for spec in base_env_yaml_data["dependencies"] + ] + + if not license_file.exists(): + raise ValueError(f"Cannot find license file: {license_file}") + + if output_dir.exists() and not dirty: + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # working dir for conda-lock outputs that we use as intermediates + lock_work_dir = output_dir / "lockwork" + lock_work_dir.mkdir(parents=True, exist_ok=True) + + # read environment files and create the lock file + lockfile_path = lock_work_dir / f"{env_name}.conda-lock.yml" + conda_lock.conda_lock.run_lock( + environment_files=[environment_file, installer_environment_file], + conda_exe=conda_exe, + mamba=True, + micromamba=True, + kinds=("lock",), + lockfile_path=lockfile_path, + ) + + # render main environment specs into explicit .lock files for reproducibility + lock_content = conda_lock.conda_lock.parse_conda_lock_file(lockfile_path) + conda_lock.conda_lock.do_render( + lockfile=lock_content, + kinds=("explicit",), + filename_template=f"{output_dir}/{env_name}-{{platform}}.lock", + ) + + # create the environment specification files for the metapackages + render_metapackage_environments( + lockfile_path=lockfile_path, + requested_pkg_names=env_pkg_names, + name=env_name, + version=version, + output_dir=output_dir, + ) + + # create the rendered constructor directories + render_constructors( + lockfile_path=lockfile_path, + requested_pkg_names=sorted(env_pkg_names + base_env_pkg_names), + name=env_name, + version=version, + company=company, + license_file=license_file, + output_dir=output_dir, + ) + + # clean up conda-lock work dir + if not keep_workdir: + shutil.rmtree(lock_work_dir) if __name__ == "__main__": @@ -370,9 +348,24 @@ if __name__ == "__main__": " (default: %(default)s)" ), ) + parser.add_argument( + "--dirty", + action="store_true", + default=False, + help=("Do not clean up output_dir before rendering. (default: %(default)s)"), + ) + parser.add_argument( + "--keep-workdir", + action="store_true", + default=False, + help=( + "Keep conda-lock working directory ({output_dir}/lockwork) of intermediate" + " outputs. (default: %(default)s)" + ), + ) parser.add_argument( "--conda-exe", - type=str, + type=pathlib.Path, default=None, help=( "Path to the conda (or mamba or micromamba) executable to use." @@ -382,16 +375,14 @@ if __name__ == "__main__": args = parser.parse_args() - conda_exe = conda_lock.conda_lock.determine_conda_executable( - conda_executable=args.conda_exe, mamba=True, micromamba=True - ) - - constructor_specs = render_platforms( + render( environment_file=args.environment_file, installer_environment_file=args.installer_environment_file, version=args.version, company=args.company, license_file=args.license_file, output_dir=args.output_dir, - conda_exe=conda_exe, + conda_exe=args.conda_exe, + dirty=args.dirty, + keep_workdir=args.keep_workdir, )