diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..62332b3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# github helper pieces to make some files not show up in diffs automatically +installer_specs/*.lock linguist-generated=true diff --git a/.github/workflows/build_radioconda.yml b/.github/workflows/build_radioconda.yml index 6d7673b..c20007f 100644 --- a/.github/workflows/build_radioconda.yml +++ b/.github/workflows/build_radioconda.yml @@ -64,14 +64,14 @@ jobs: env: PLATFORM: ${{ matrix.PLATFORM }} run: | - python build_metapackage.py installer_specs/$DISTNAME-$PLATFORM.txt + python build_metapackage.py - name: Copy lock file and list built installers and packages shell: bash env: PLATFORM: ${{ matrix.PLATFORM }} run: | - cp installer_specs/$DISTNAME-$PLATFORM.txt dist/ + cp installer_specs/$DISTNAME-$PLATFORM.lock dist/ ls -lhR dist - name: Test installer (sh) diff --git a/README.md b/README.md index a9a8eec..27ff511 100644 --- a/README.md +++ b/README.md @@ -104,15 +104,15 @@ To install a particular release version, substitute the desired version number a mamba install -c ryanvolz radioconda=20NN.NN.NN -### Install from environment file +### Install from environment lock file -You can also install from the released environment file (on Windows): +You can also install from the released environment lock file (on Windows): - mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-win-64.txt + mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-win-64.lock (on Linux/macOS): - mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-$(conda info | sed -n -e 's/^.*platform : //p').txt + mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-$(conda info | sed -n -e 's/^.*platform : //p').lock ## Additional Installation for Device Support diff --git a/build_metapackage.py b/build_metapackage.py index c50bf78..c281bb9 100755 --- a/build_metapackage.py +++ b/build_metapackage.py @@ -1,26 +1,49 @@ #!/usr/bin/env python3 import pathlib -import re +from typing import List -comment_re = re.compile(r"^\s*#\s*(?P.*)\s*$") -key_value_re = re.compile(r"^(?P.*):\s*(?P.*)\s*$") +import yaml -def read_lock_file(lock_file: pathlib.Path) -> dict: - with lock_file.open("r") as f: - lines = f.read().splitlines() +def read_env_file( + env_file: pathlib.Path, + fallback_name: str, + fallback_version: str, + fallback_platform: str, + fallback_channels: List[str], +) -> dict: + with env_file.open("r") as f: + env_dict = yaml.safe_load(f) - lock_dict = dict(specs=[]) - for line in lines: - comment_match = comment_re.match(line) - if comment_match: - m = key_value_re.match(comment_match.group("comment")) - if m: - lock_dict[m.group("key")] = m.group("value") - else: - lock_dict["specs"].append(line) + env_dict.setdefault("name", fallback_name) + env_dict.setdefault("version", fallback_version) + env_dict.setdefault("platform", fallback_platform) + env_dict.setdefault("channels", fallback_channels) - return lock_dict + return env_dict + + +def get_conda_metapackage_cmdline( + env_dict: dict, home: str, license_id: str, summary: str +): + cmdline = [ + "conda", + "metapackage", + env_dict["name"], + env_dict["version"], + "--no-anaconda-upload", + "--home", + home, + "--license", + license_id, + "--summary", + summary, + ] + for channel in env_dict["channels"]: + cmdline.extend(["--channel", channel]) + cmdline.extend(["--dependencies"] + env_dict["dependencies"]) + + return cmdline if __name__ == "__main__": @@ -54,12 +77,12 @@ if __name__ == "__main__": ) ) parser.add_argument( - "lock_file", + "env_file", type=pathlib.Path, nargs="?", - default=here / "installer_specs" / f"{distname}-{platform}.txt", + default=here / "installer_specs" / f"{distname}-{platform}.yml", help=( - "Environment lock file for a particular platform" + "Environment yaml file for a particular platform" " (name ends in the platform identifier)." " (default: %(default)s)" ), @@ -92,45 +115,32 @@ if __name__ == "__main__": args, metapackage_args = parser.parse_known_args() - lock_dict = read_lock_file(args.lock_file) + env_dict = read_env_file( + args.env_file, + fallback_name=distname, + fallback_version="0", + fallback_platform=platform, + fallback_channels=["conda-forge"], + ) - name = lock_dict.get("name", distname) - version = lock_dict.get("version", "0") - platform = lock_dict.get("platform", platform) + cmdline = get_conda_metapackage_cmdline( + env_dict=env_dict, home=args.home, license_id=args.license, summary=args.summary + ) + cmdline.extend(metapackage_args) env = os.environ.copy() - env["CONDA_SUBDIR"] = platform + env["CONDA_SUBDIR"] = env_dict["platform"] - channels = [c.strip() for c in lock_dict.get("channels", "conda-forge").split(",")] - - conda_metapackage_cmdline = [ - "conda", - "metapackage", - name, - version, - "--no-anaconda-upload", - "--home", - args.home, - "--license", - args.license, - "--summary", - args.summary, - ] - for channel in channels: - conda_metapackage_cmdline.extend(["--channel", channel]) - conda_metapackage_cmdline.extend(["--dependencies"] + lock_dict["specs"]) - conda_metapackage_cmdline.extend(metapackage_args) - - proc = subprocess.run(conda_metapackage_cmdline, env=env) + proc = subprocess.run(cmdline, env=env) try: proc.check_returncode() except subprocess.CalledProcessError: sys.exit(1) - bldpkgs_dir = pathlib.Path(conda_build_config.bldpkgs_dir) - pkg_paths = list(bldpkgs_dir.glob(f"{name}-{version}*.bz2")) - pkg_out_dir = args.output_dir / platform + bldpkgs_dir = pathlib.Path(conda_build_config.croot) / env_dict["platform"] + pkg_paths = list(bldpkgs_dir.glob(f"{env_dict['name']}-{env_dict['version']}*.bz2")) + pkg_out_dir = args.output_dir / env_dict["platform"] pkg_out_dir.mkdir(parents=True, exist_ok=True) for pkg in pkg_paths: diff --git a/rerender.py b/rerender.py index 1bbe4b2..e767cff 100755 --- a/rerender.py +++ b/rerender.py @@ -32,26 +32,51 @@ def lock_env_spec( return locked_env_spec -def write_lock_file( - lock_spec: conda_lock.src_parser.LockSpecification, - lock_file_path: pathlib.Path, +def write_env_file( + env_spec: conda_lock.src_parser.LockSpecification, + file_path: pathlib.Path, name: Optional[str] = None, version: Optional[str] = None, - channels: Optional[List[str]] = None, ): - lockfile_contents = [ - f"# platform: {lock_spec.platform}", - f"# env_hash: {lock_spec.env_hash()}", - ] + env_dict = dict( + name=name, + version=version, + platform=env_spec.platform, + channels=env_spec.channels, + dependencies=env_spec.specs, + ) if name: - lockfile_contents.append(f"# name: {name}") + env_dict["name"] = name if version: - lockfile_contents.append(f"# version: {version}") - if channels: - lockfile_contents.append(f"# channels: {','.join(channels)}") - lockfile_contents.extend(lock_spec.specs) - with lock_file_path.open("w") as f: - f.write("\n".join(lockfile_contents)) + env_dict["version"] = version + with file_path.open("w") as f: + yaml.safe_dump(env_dict, stream=f) + + return env_dict + + +def write_lock_file( + lock_spec: conda_lock.src_parser.LockSpecification, + file_path: pathlib.Path, + conda_exe: str, +): + lockfile_contents = conda_lock.conda_lock.create_lockfile_from_spec( + channels=lock_spec.channels, conda=conda_exe, spec=lock_spec + ) + + def sanitize_lockfile_line(line): + line = line.strip() + if line == "": + return "#" + else: + return line + + lockfile_contents = [sanitize_lockfile_line(ln) for ln in lockfile_contents] + + with file_path.open("w") as f: + f.write("\n".join(lockfile_contents) + "\n") + + return lockfile_contents def render_constructor( @@ -151,14 +176,19 @@ def render_platforms( # lock the full environment specification to specific versions and builds locked_env_spec = lock_env_spec(env_spec, conda_exe) - # write the full environment specification to a lock file - lock_file_path = output_dir / f"{output_name}.txt" - write_lock_file( - locked_env_spec, - lock_file_path, + # write the full environment specification to a yaml file (to build metapackage) + locked_env_dict = write_env_file( + env_spec=locked_env_spec, + file_path=output_dir / f"{output_name}.yml", name=env_name, version=version, - channels=locked_env_spec.channels, + ) + + # write the full environment specification to a lock file (to install from file) + lockfile_contents = write_lock_file( + lock_spec=locked_env_spec, + file_path=output_dir / f"{output_name}.lock", + conda_exe=conda_exe, ) # add installer-only (base environment) packages and lock those too @@ -200,6 +230,8 @@ def render_platforms( # aggregate output rendered_platforms[output_name] = dict( locked_env_spec=locked_env_spec, + locked_env_dict=locked_env_dict, + lockfile_contents=lockfile_contents, locked_installer_spec=locked_installer_spec, constructor_dict=constructor_dict, )