mirror of
https://github.com/ryanvolz/radioconda.git
synced 2024-09-19 12:06:34 -04:00
Separate env packages and installer-only (base) packages.
This makes the rendered lock files more useful for users (they won't install conda/mamba in a non-base environment) and will allow for the creation of a metapackage for each release.
This commit is contained in:
parent
25a3056881
commit
5553635a29
2
.github/workflows/build_radioconda.yml
vendored
2
.github/workflows/build_radioconda.yml
vendored
@ -69,7 +69,7 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
PLATFORM: ${{ matrix.PLATFORM }}
|
PLATFORM: ${{ matrix.PLATFORM }}
|
||||||
run: |
|
run: |
|
||||||
cp installer_specs/$DISTNAME-$PLATFORM/$DISTNAME-$PLATFORM.txt dist/
|
cp installer_specs/$DISTNAME-$PLATFORM.txt dist/
|
||||||
ls -lh dist
|
ls -lh dist
|
||||||
|
|
||||||
- name: Test installer (sh)
|
- name: Test installer (sh)
|
||||||
|
@ -7,19 +7,13 @@ platforms:
|
|||||||
- osx-64
|
- osx-64
|
||||||
- win-64
|
- win-64
|
||||||
dependencies:
|
dependencies:
|
||||||
- mamba
|
|
||||||
- ipython
|
|
||||||
- python
|
|
||||||
# restrict to python 3.8 on Windows for Windows 7 compatibility
|
|
||||||
- python 3.8.* # [win]
|
|
||||||
- radioconda_console_shortcut # [win]
|
|
||||||
|
|
||||||
- digital_rf
|
- digital_rf
|
||||||
- gnuradio 3.8.*
|
- gnuradio 3.8.*
|
||||||
- gnuradio-osmosdr
|
- gnuradio-osmosdr
|
||||||
- gnuradio-satellites
|
- gnuradio-satellites
|
||||||
- gnuradio-soapy
|
- gnuradio-soapy
|
||||||
- gqrx
|
- gqrx
|
||||||
|
- ipython
|
||||||
- libiio
|
- libiio
|
||||||
- libm2k
|
- libm2k
|
||||||
- limesuite
|
- limesuite
|
||||||
@ -27,6 +21,10 @@ dependencies:
|
|||||||
- numpy
|
- numpy
|
||||||
- pandas
|
- pandas
|
||||||
- pyadi-iio
|
- pyadi-iio
|
||||||
|
- python
|
||||||
|
# restrict to python 3.8 on Windows for Windows 7 compatibility
|
||||||
|
- python 3.8.* # [win]
|
||||||
|
- radioconda_console_shortcut # [win]
|
||||||
- rtl-sdr
|
- rtl-sdr
|
||||||
- scipy
|
- scipy
|
||||||
- soapysdr
|
- soapysdr
|
||||||
|
244
rerender.py
244
rerender.py
@ -1,12 +1,17 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import pathlib
|
import pathlib
|
||||||
import shutil
|
import shutil
|
||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
import conda_lock
|
import conda_lock
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
def render_lock_spec(
|
def name_from_pkg_spec(spec: str):
|
||||||
|
return spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def lock_env_spec(
|
||||||
lock_spec: conda_lock.src_parser.LockSpecification, conda_exe: str
|
lock_spec: conda_lock.src_parser.LockSpecification, conda_exe: str
|
||||||
) -> conda_lock.src_parser.LockSpecification:
|
) -> conda_lock.src_parser.LockSpecification:
|
||||||
create_env_dict = conda_lock.conda_lock.solve_specs_for_arch(
|
create_env_dict = conda_lock.conda_lock.solve_specs_for_arch(
|
||||||
@ -16,36 +21,104 @@ def render_lock_spec(
|
|||||||
platform=lock_spec.platform,
|
platform=lock_spec.platform,
|
||||||
)
|
)
|
||||||
pkgs = create_env_dict["actions"]["LINK"]
|
pkgs = create_env_dict["actions"]["LINK"]
|
||||||
spec_names = set(
|
locked_specs = ["{name}={version}={build_string}".format(**pkg) for pkg in pkgs]
|
||||||
spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0]
|
|
||||||
for spec in lock_spec.specs
|
|
||||||
)
|
|
||||||
|
|
||||||
rendered_specs = []
|
locked_env_spec = conda_lock.src_parser.LockSpecification(
|
||||||
rendered_dep_specs = []
|
specs=sorted(locked_specs),
|
||||||
for pkg in pkgs:
|
|
||||||
pkg_spec = "{name}={version}={build_string}".format(**pkg)
|
|
||||||
if pkg["name"] in spec_names:
|
|
||||||
rendered_specs.append(pkg_spec)
|
|
||||||
else:
|
|
||||||
rendered_dep_specs.append(pkg_spec)
|
|
||||||
|
|
||||||
rendered_lock_spec = conda_lock.src_parser.LockSpecification(
|
|
||||||
specs=sorted(rendered_specs),
|
|
||||||
channels=lock_spec.channels,
|
|
||||||
platform=lock_spec.platform,
|
|
||||||
)
|
|
||||||
rendered_full_lock_spec = conda_lock.src_parser.LockSpecification(
|
|
||||||
specs=sorted(rendered_specs + rendered_dep_specs),
|
|
||||||
channels=lock_spec.channels,
|
channels=lock_spec.channels,
|
||||||
platform=lock_spec.platform,
|
platform=lock_spec.platform,
|
||||||
)
|
)
|
||||||
|
|
||||||
return rendered_lock_spec, rendered_full_lock_spec
|
return locked_env_spec
|
||||||
|
|
||||||
|
|
||||||
def render_constructor_specs(
|
def write_lock_file(
|
||||||
|
lock_spec: conda_lock.src_parser.LockSpecification,
|
||||||
|
lock_file_path: pathlib.Path,
|
||||||
|
name: Optional[str] = None,
|
||||||
|
version: Optional[str] = None,
|
||||||
|
channels: Optional[List[str]] = None,
|
||||||
|
):
|
||||||
|
lockfile_contents = [
|
||||||
|
f"# platform: {lock_spec.platform}",
|
||||||
|
f"# env_hash: {lock_spec.env_hash()}",
|
||||||
|
]
|
||||||
|
if name:
|
||||||
|
lockfile_contents.append(f"# name: {name}")
|
||||||
|
if version:
|
||||||
|
lockfile_contents.append(f"# version: {version}")
|
||||||
|
if channels:
|
||||||
|
lockfile_contents.append(f"# channels: {','.join(channels)}")
|
||||||
|
lockfile_contents.extend(lock_spec.specs)
|
||||||
|
with lock_file_path.open("w") as f:
|
||||||
|
f.write("\n".join(lockfile_contents))
|
||||||
|
|
||||||
|
|
||||||
|
def render_constructor(
|
||||||
|
lock_spec: conda_lock.src_parser.LockSpecification,
|
||||||
|
name: str,
|
||||||
|
version: str,
|
||||||
|
company: str,
|
||||||
|
license_file: pathlib.Path,
|
||||||
|
output_dir: pathlib.Path,
|
||||||
|
) -> dict:
|
||||||
|
platform = lock_spec.platform
|
||||||
|
constructor_name = f"{name}-{platform}"
|
||||||
|
|
||||||
|
construct_dict = dict(
|
||||||
|
name=name,
|
||||||
|
version=version,
|
||||||
|
company=company,
|
||||||
|
channels=lock_spec.channels,
|
||||||
|
specs=lock_spec.specs,
|
||||||
|
initialize_by_default=True,
|
||||||
|
installer_type="all",
|
||||||
|
keep_pkgs=True,
|
||||||
|
license_file="LICENSE",
|
||||||
|
register_python_default=False,
|
||||||
|
write_condarc=True,
|
||||||
|
)
|
||||||
|
if platform.startswith("win"):
|
||||||
|
construct_dict["post_install"] = "post_install.bat"
|
||||||
|
else:
|
||||||
|
construct_dict["post_install"] = "post_install.sh"
|
||||||
|
|
||||||
|
constructor_dir = output_dir / constructor_name
|
||||||
|
if constructor_dir.exists():
|
||||||
|
shutil.rmtree(constructor_dir)
|
||||||
|
constructor_dir.mkdir(parents=True)
|
||||||
|
|
||||||
|
# copy license to the constructor directory
|
||||||
|
shutil.copy(license_file, constructor_dir / "LICENSE")
|
||||||
|
|
||||||
|
# write the post_install scripts referenced in the construct dict
|
||||||
|
if platform.startswith("win"):
|
||||||
|
with (constructor_dir / "post_install.bat").open("w") as f:
|
||||||
|
f.write("\n".join((r"del /q %PREFIX%\pkgs\*.tar.bz2", "exit 0", "")))
|
||||||
|
else:
|
||||||
|
with (constructor_dir / "post_install.sh").open("w") as f:
|
||||||
|
f.write(
|
||||||
|
"\n".join(
|
||||||
|
(
|
||||||
|
"#!/bin/sh",
|
||||||
|
f'PREFIX="${{PREFIX:-$2/{name}}}"',
|
||||||
|
r"rm -f $PREFIX/pkgs/*.tar.bz2",
|
||||||
|
"exit 0",
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
construct_yaml_path = constructor_dir / "construct.yaml"
|
||||||
|
with construct_yaml_path.open("w") as f:
|
||||||
|
yaml.safe_dump(construct_dict, stream=f)
|
||||||
|
|
||||||
|
return construct_dict
|
||||||
|
|
||||||
|
|
||||||
|
def render_platforms(
|
||||||
environment_file: pathlib.Path,
|
environment_file: pathlib.Path,
|
||||||
|
installer_pkg_specs: List[str],
|
||||||
version: str,
|
version: str,
|
||||||
company: str,
|
company: str,
|
||||||
license_file: pathlib.Path,
|
license_file: pathlib.Path,
|
||||||
@ -55,85 +128,83 @@ def render_constructor_specs(
|
|||||||
with environment_file.open("r") as f:
|
with environment_file.open("r") as f:
|
||||||
env_yaml_data = yaml.safe_load(f)
|
env_yaml_data = yaml.safe_load(f)
|
||||||
|
|
||||||
installer_name = env_yaml_data["name"]
|
env_name = env_yaml_data["name"]
|
||||||
platforms = env_yaml_data["platforms"]
|
platforms = env_yaml_data["platforms"]
|
||||||
|
|
||||||
if not license_file.exists():
|
if not license_file.exists():
|
||||||
raise ValueError(f"Cannot find license file: {license_file}")
|
raise ValueError(f"Cannot find license file: {license_file}")
|
||||||
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
if output_dir.exists():
|
||||||
|
shutil.rmtree(output_dir)
|
||||||
|
output_dir.mkdir(parents=True)
|
||||||
|
|
||||||
constructor_specs = {}
|
rendered_platforms = {}
|
||||||
|
|
||||||
for platform in platforms:
|
for platform in platforms:
|
||||||
constructor_name = f"{installer_name}-{platform}"
|
output_name = f"{env_name}-{platform}"
|
||||||
|
|
||||||
lock_spec = conda_lock.conda_lock.parse_environment_file(
|
# get the environment specification for the list of packages from the env file
|
||||||
|
env_spec = conda_lock.conda_lock.parse_environment_file(
|
||||||
environment_file=environment_file, platform=platform
|
environment_file=environment_file, platform=platform
|
||||||
)
|
)
|
||||||
rendered_lock_spec, rendered_full_lock_spec = render_lock_spec(
|
|
||||||
lock_spec, conda_exe
|
# lock the full environment specification to specific versions and builds
|
||||||
|
locked_env_spec = lock_env_spec(env_spec, conda_exe)
|
||||||
|
|
||||||
|
# write the full environment specification to a lock file
|
||||||
|
lock_file_path = output_dir / f"{output_name}.txt"
|
||||||
|
write_lock_file(
|
||||||
|
locked_env_spec,
|
||||||
|
lock_file_path,
|
||||||
|
name=env_name,
|
||||||
|
version=version,
|
||||||
|
channels=locked_env_spec.channels,
|
||||||
)
|
)
|
||||||
construct_dict = dict(
|
|
||||||
name=installer_name,
|
# add installer-only (base environment) packages and lock those too
|
||||||
|
installer_spec = conda_lock.src_parser.LockSpecification(
|
||||||
|
specs=sorted(locked_env_spec.specs + installer_pkg_specs),
|
||||||
|
channels=locked_env_spec.channels,
|
||||||
|
platform=locked_env_spec.platform,
|
||||||
|
)
|
||||||
|
locked_installer_spec = lock_env_spec(installer_spec, conda_exe)
|
||||||
|
|
||||||
|
# get a set of only the packages to put in the constructor specification
|
||||||
|
# taken from the installer-only list and those explicitly selected originally
|
||||||
|
constructor_pkg_names = set(
|
||||||
|
name_from_pkg_spec(spec) for spec in env_spec.specs + installer_pkg_specs
|
||||||
|
)
|
||||||
|
|
||||||
|
# filter the installer spec by the constructor package names
|
||||||
|
constructor_pkg_specs = [
|
||||||
|
spec
|
||||||
|
for spec in locked_installer_spec.specs
|
||||||
|
if name_from_pkg_spec(spec) in constructor_pkg_names
|
||||||
|
]
|
||||||
|
constructor_spec = conda_lock.src_parser.LockSpecification(
|
||||||
|
specs=constructor_pkg_specs,
|
||||||
|
channels=locked_installer_spec.channels,
|
||||||
|
platform=locked_installer_spec.platform,
|
||||||
|
)
|
||||||
|
|
||||||
|
# create the rendered constructor directory
|
||||||
|
constructor_dict = render_constructor(
|
||||||
|
lock_spec=constructor_spec,
|
||||||
|
name=env_name,
|
||||||
version=version,
|
version=version,
|
||||||
company=company,
|
company=company,
|
||||||
channels=rendered_lock_spec.channels,
|
license_file=license_file,
|
||||||
specs=rendered_lock_spec.specs,
|
output_dir=output_dir,
|
||||||
initialize_by_default=True,
|
|
||||||
installer_type="all",
|
|
||||||
keep_pkgs=True,
|
|
||||||
license_file="LICENSE",
|
|
||||||
register_python_default=False,
|
|
||||||
write_condarc=True,
|
|
||||||
)
|
)
|
||||||
if platform.startswith("win"):
|
|
||||||
construct_dict["post_install"] = "post_install.bat"
|
|
||||||
else:
|
|
||||||
construct_dict["post_install"] = "post_install.sh"
|
|
||||||
|
|
||||||
constructor_specs[constructor_name] = construct_dict
|
# aggregate output
|
||||||
|
rendered_platforms[output_name] = dict(
|
||||||
|
locked_env_spec=locked_env_spec,
|
||||||
|
locked_installer_spec=locked_installer_spec,
|
||||||
|
constructor_dict=constructor_dict,
|
||||||
|
)
|
||||||
|
|
||||||
constructor_dir = output_dir / constructor_name
|
return rendered_platforms
|
||||||
if constructor_dir.exists():
|
|
||||||
shutil.rmtree(constructor_dir)
|
|
||||||
constructor_dir.mkdir(parents=True)
|
|
||||||
|
|
||||||
# copy license to the constructor directory
|
|
||||||
shutil.copy(license_file, constructor_dir / "LICENSE")
|
|
||||||
|
|
||||||
# write the post_install scripts referenced in the construct dict
|
|
||||||
if platform.startswith("win"):
|
|
||||||
with (constructor_dir / "post_install.bat").open("w") as f:
|
|
||||||
f.write("\n".join((r"del /q %PREFIX%\pkgs\*.tar.bz2", "exit 0", "")))
|
|
||||||
else:
|
|
||||||
with (constructor_dir / "post_install.sh").open("w") as f:
|
|
||||||
f.write(
|
|
||||||
"\n".join(
|
|
||||||
(
|
|
||||||
"#!/bin/sh",
|
|
||||||
f'PREFIX="${{PREFIX:-$2/{installer_name}}}"',
|
|
||||||
r"rm -f $PREFIX/pkgs/*.tar.bz2",
|
|
||||||
"exit 0",
|
|
||||||
"",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
construct_yaml_path = constructor_dir / "construct.yaml"
|
|
||||||
with construct_yaml_path.open("w") as f:
|
|
||||||
yaml.safe_dump(construct_dict, stream=f)
|
|
||||||
|
|
||||||
lockfile_contents = [
|
|
||||||
f"# platform: {rendered_full_lock_spec.platform}",
|
|
||||||
f"# env_hash: {rendered_full_lock_spec.env_hash()}",
|
|
||||||
]
|
|
||||||
lockfile_contents.extend(rendered_full_lock_spec.specs)
|
|
||||||
lock_file_path = constructor_dir / f"{constructor_name}.txt"
|
|
||||||
with lock_file_path.open("w") as f:
|
|
||||||
f.write("\n".join(lockfile_contents))
|
|
||||||
|
|
||||||
return constructor_specs
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
@ -221,8 +292,9 @@ if __name__ == "__main__":
|
|||||||
conda_executable=args.conda_exe, mamba=True, micromamba=True
|
conda_executable=args.conda_exe, mamba=True, micromamba=True
|
||||||
)
|
)
|
||||||
|
|
||||||
constructor_specs = render_constructor_specs(
|
constructor_specs = render_platforms(
|
||||||
environment_file=args.environment_file,
|
environment_file=args.environment_file,
|
||||||
|
installer_pkg_specs=["mamba"],
|
||||||
version=args.version,
|
version=args.version,
|
||||||
company=args.company,
|
company=args.company,
|
||||||
license_file=args.license_file,
|
license_file=args.license_file,
|
||||||
|
Loading…
Reference in New Issue
Block a user