shway
/
radioconda
mirror of https://github.com/ryanvolz/radioconda.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch to explicit lockfile and build metapackage from an env yaml.

This commit is contained in:
Ryan Volz 2021-05-21 16:00:54 -04:00
parent be10dc29cd
commit 3e3da4374e
5 changed files with 119 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitattributes vendored Normal file
View File

@ -0,0 +1,2 @@
# github helper pieces to make some files not show up in diffs automatically
installer_specs/*.lock linguist-generated=true

4
.github/workflows/build_radioconda.yml vendored
View File

@ -64,14 +64,14 @@ jobs:
env:
PLATFORM: ${{ matrix.PLATFORM }}
run: |
python build_metapackage.py installer_specs/$DISTNAME-$PLATFORM.txt
python build_metapackage.py
- name: Copy lock file and list built installers and packages
shell: bash
env:
PLATFORM: ${{ matrix.PLATFORM }}
run: |
cp installer_specs/$DISTNAME-$PLATFORM.txt dist/
cp installer_specs/$DISTNAME-$PLATFORM.lock dist/
ls -lhR dist
- name: Test installer (sh)

8
README.md
View File

@ -104,15 +104,15 @@ To install a particular release version, substitute the desired version number a
mamba install -c ryanvolz radioconda=20NN.NN.NN
### Install from environment file
### Install from environment lock file
You can also install from the released environment file (on Windows):
You can also install from the released environment lock file (on Windows):
mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-win-64.txt
mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-win-64.lock
(on Linux/macOS):
mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-$(conda info | sed -n -e 's/^.*platform : //p').txt
mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-$(conda info | sed -n -e 's/^.*platform : //p').lock
## Additional Installation for Device Support

106
build_metapackage.py
View File

@ -1,26 +1,49 @@
#!/usr/bin/env python3
import pathlib
import re
from typing import List
comment_re = re.compile(r"^\s*#\s*(?P<comment>.*)\s*$")
key_value_re = re.compile(r"^(?P<key>.*):\s*(?P<value>.*)\s*$")
import yaml
def read_lock_file(lock_file: pathlib.Path) -> dict:
with lock_file.open("r") as f:
lines = f.read().splitlines()
def read_env_file(
env_file: pathlib.Path,
fallback_name: str,
fallback_version: str,
fallback_platform: str,
fallback_channels: List[str],
) -> dict:
with env_file.open("r") as f:
env_dict = yaml.safe_load(f)
lock_dict = dict(specs=[])
for line in lines:
comment_match = comment_re.match(line)
if comment_match:
m = key_value_re.match(comment_match.group("comment"))
if m:
lock_dict[m.group("key")] = m.group("value")
else:
lock_dict["specs"].append(line)
env_dict.setdefault("name", fallback_name)
env_dict.setdefault("version", fallback_version)
env_dict.setdefault("platform", fallback_platform)
env_dict.setdefault("channels", fallback_channels)
return lock_dict
return env_dict
def get_conda_metapackage_cmdline(
env_dict: dict, home: str, license_id: str, summary: str
):
cmdline = [
"conda",
"metapackage",
env_dict["name"],
env_dict["version"],
"--no-anaconda-upload",
"--home",
home,
"--license",
license_id,
"--summary",
summary,
]
for channel in env_dict["channels"]:
cmdline.extend(["--channel", channel])
cmdline.extend(["--dependencies"] + env_dict["dependencies"])
return cmdline
if __name__ == "__main__":
@ -54,12 +77,12 @@ if __name__ == "__main__":
)
)
parser.add_argument(
"lock_file",
"env_file",
type=pathlib.Path,
nargs="?",
default=here / "installer_specs" / f"{distname}-{platform}.txt",
default=here / "installer_specs" / f"{distname}-{platform}.yml",
help=(
"Environment lock file for a particular platform"
"Environment yaml file for a particular platform"
" (name ends in the platform identifier)."
" (default: %(default)s)"
),
@ -92,45 +115,32 @@ if __name__ == "__main__":
args, metapackage_args = parser.parse_known_args()
lock_dict = read_lock_file(args.lock_file)
env_dict = read_env_file(
args.env_file,
fallback_name=distname,
fallback_version="0",
fallback_platform=platform,
fallback_channels=["conda-forge"],
)
name = lock_dict.get("name", distname)
version = lock_dict.get("version", "0")
platform = lock_dict.get("platform", platform)
cmdline = get_conda_metapackage_cmdline(
env_dict=env_dict, home=args.home, license_id=args.license, summary=args.summary
)
cmdline.extend(metapackage_args)
env = os.environ.copy()
env["CONDA_SUBDIR"] = platform
env["CONDA_SUBDIR"] = env_dict["platform"]
channels = [c.strip() for c in lock_dict.get("channels", "conda-forge").split(",")]
conda_metapackage_cmdline = [
"conda",
"metapackage",
name,
version,
"--no-anaconda-upload",
"--home",
args.home,
"--license",
args.license,
"--summary",
args.summary,
]
for channel in channels:
conda_metapackage_cmdline.extend(["--channel", channel])
conda_metapackage_cmdline.extend(["--dependencies"] + lock_dict["specs"])
conda_metapackage_cmdline.extend(metapackage_args)
proc = subprocess.run(conda_metapackage_cmdline, env=env)
proc = subprocess.run(cmdline, env=env)
try:
proc.check_returncode()
except subprocess.CalledProcessError:
sys.exit(1)
bldpkgs_dir = pathlib.Path(conda_build_config.bldpkgs_dir)
pkg_paths = list(bldpkgs_dir.glob(f"{name}-{version}*.bz2"))
pkg_out_dir = args.output_dir / platform
bldpkgs_dir = pathlib.Path(conda_build_config.croot) / env_dict["platform"]
pkg_paths = list(bldpkgs_dir.glob(f"{env_dict['name']}-{env_dict['version']}*.bz2"))
pkg_out_dir = args.output_dir / env_dict["platform"]
pkg_out_dir.mkdir(parents=True, exist_ok=True)
for pkg in pkg_paths:

74
rerender.py
View File

@ -32,26 +32,51 @@ def lock_env_spec(
return locked_env_spec
def write_lock_file(
lock_spec: conda_lock.src_parser.LockSpecification,
lock_file_path: pathlib.Path,
def write_env_file(
env_spec: conda_lock.src_parser.LockSpecification,
file_path: pathlib.Path,
name: Optional[str] = None,
version: Optional[str] = None,
channels: Optional[List[str]] = None,
):
lockfile_contents = [
f"# platform: {lock_spec.platform}",
f"# env_hash: {lock_spec.env_hash()}",
]
env_dict = dict(
name=name,
version=version,
platform=env_spec.platform,
channels=env_spec.channels,
dependencies=env_spec.specs,
)
if name:
lockfile_contents.append(f"# name: {name}")
env_dict["name"] = name
if version:
lockfile_contents.append(f"# version: {version}")
if channels:
lockfile_contents.append(f"# channels: {','.join(channels)}")
lockfile_contents.extend(lock_spec.specs)
with lock_file_path.open("w") as f:
f.write("\n".join(lockfile_contents))
env_dict["version"] = version
with file_path.open("w") as f:
yaml.safe_dump(env_dict, stream=f)
return env_dict
def write_lock_file(
lock_spec: conda_lock.src_parser.LockSpecification,
file_path: pathlib.Path,
conda_exe: str,
):
lockfile_contents = conda_lock.conda_lock.create_lockfile_from_spec(
channels=lock_spec.channels, conda=conda_exe, spec=lock_spec
)
def sanitize_lockfile_line(line):
line = line.strip()
if line == "":
return "#"
else:
return line
lockfile_contents = [sanitize_lockfile_line(ln) for ln in lockfile_contents]
with file_path.open("w") as f:
f.write("\n".join(lockfile_contents) + "\n")
return lockfile_contents
def render_constructor(
@ -151,14 +176,19 @@ def render_platforms(
# lock the full environment specification to specific versions and builds
locked_env_spec = lock_env_spec(env_spec, conda_exe)
# write the full environment specification to a lock file
lock_file_path = output_dir / f"{output_name}.txt"
write_lock_file(
locked_env_spec,
lock_file_path,
# write the full environment specification to a yaml file (to build metapackage)
locked_env_dict = write_env_file(
env_spec=locked_env_spec,
file_path=output_dir / f"{output_name}.yml",
name=env_name,
version=version,
channels=locked_env_spec.channels,
)
# write the full environment specification to a lock file (to install from file)
lockfile_contents = write_lock_file(
lock_spec=locked_env_spec,
file_path=output_dir / f"{output_name}.lock",
conda_exe=conda_exe,
)
# add installer-only (base environment) packages and lock those too
@ -200,6 +230,8 @@ def render_platforms(
# aggregate output
rendered_platforms[output_name] = dict(
locked_env_spec=locked_env_spec,
locked_env_dict=locked_env_dict,
lockfile_contents=lockfile_contents,
locked_installer_spec=locked_installer_spec,
constructor_dict=constructor_dict,
)