diff --git a/.github/workflows/build_radioconda.yml b/.github/workflows/build_radioconda.yml index 5111136..8542322 100644 --- a/.github/workflows/build_radioconda.yml +++ b/.github/workflows/build_radioconda.yml @@ -6,10 +6,13 @@ on: pull_request: paths: - "installer_specs/**" + workflow_dispatch: env: DISTNAME: radioconda - SOURCE: github.com/ryanvolz/radioconda + LICENSE_ID: BSD-3-Clause + METAPACKAGE_LABEL: dev + METAPACKAGE_SUMMARY: Metapackage to install the radioconda package set. jobs: build: @@ -45,31 +48,31 @@ jobs: with: environment-file: buildenv.yaml - - name: Build installer (bash) - if: runner.os != 'Windows' + - name: Build installer + shell: bash -l {0} + env: + PLATFORM: ${{ matrix.PLATFORM }} + OS_NAME: ${{ matrix.OS_NAME }} + run: | + if [ "$OS_NAME" == "Windows" ]; then + PATH=$CONDA_PREFIX/NSIS:$PATH + fi + python build_installer.py -v + + - name: Build metapackage shell: bash -l {0} env: PLATFORM: ${{ matrix.PLATFORM }} run: | - python build_installer.py -v + python build_metapackage.py installer_specs/$DISTNAME-$PLATFORM.txt - - name: Build installer (cmd.exe) - if: runner.os == 'Windows' - shell: powershell - env: - PLATFORM: ${{ matrix.PLATFORM }} - run: | - $Env:Path = "$Env:CONDA_PREFIX\NSIS;$Env:Path" - echo $Env:Path - python build_installer.py -v - - - name: Copy spec and list built installers + - name: Copy lock file and list built installers and packages shell: bash env: PLATFORM: ${{ matrix.PLATFORM }} run: | - cp installer_specs/$DISTNAME-$PLATFORM/$DISTNAME-$PLATFORM.txt dist/ - ls -lh dist + cp installer_specs/$DISTNAME-$PLATFORM.txt dist/ + ls -lhR dist - name: Test installer (sh) if: contains(matrix.OS_NAME, 'Linux') || contains(matrix.OS_NAME, 'MacOSX') @@ -86,15 +89,30 @@ jobs: - name: Test installer (pkg) if: contains(matrix.OS_NAME, 'MacOSX') - continue-on-error: true # this test doesn't work yet shell: bash env: OS_NAME: ${{ matrix.OS_NAME }} ARCH: ${{ matrix.ARCH }} TARGET_VOLUME: CurrentUserHomeDirectory - INSTALL_PATH: ~/Applications/${{ env.DISTNAME }} + INSTALL_PATH: ${{ github.workspace }}/../../../${{ env.DISTNAME }} run: | - installer -verbose -pkg dist/$DISTNAME-*-$OS_NAME-$ARCH.pkg -target $TARGET_VOLUME + cat >pkg-choices.xml < + + + + + attributeSetting + 0 + choiceAttribute + selected + choiceIdentifier + io.continuum.pkg.pathupdate + + + + EOF + installer -verbose -dumplog -applyChoiceChangesXML pkg-choices.xml -pkg dist/$DISTNAME-*-$OS_NAME-$ARCH.pkg -target $TARGET_VOLUME eval "$($INSTALL_PATH/bin/conda shell.bash hook)" conda info conda list @@ -145,3 +163,12 @@ jobs: tag: ${{ github.ref }} overwrite: true file_glob: true + + - name: Upload metapackage to Anaconda.org + if: startsWith(github.ref, 'refs/tags/') + shell: bash -l {0} + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + run: | + micromamba --help + anaconda upload -l $METAPACKAGE_LABEL --skip-existing dist/conda-bld/**/* diff --git a/README.md b/README.md index aceb43d..a9a8eec 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,19 @@ Once you have radioconda installed, you can stay up to date for all packages wit ### Upgrade to latest release -To install the latest release in particular, run (on Windows): +To install the latest release in particular, run + + mamba upgrade -c ryanvolz radioconda + +### Install a particular release + +To install a particular release version, substitute the desired version number and run + + mamba install -c ryanvolz radioconda=20NN.NN.NN + +### Install from environment file + +You can also install from the released environment file (on Windows): mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-win-64.txt @@ -102,16 +114,6 @@ To install the latest release in particular, run (on Windows): mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-$(conda info | sed -n -e 's/^.*platform : //p').txt -### Install a particular release - -To install the package versions associated with a particular release, substitute the release number and run the following (on Windows): - - mamba install --file https://github.com/ryanvolz/radioconda/releases/download/20NN.NN.NN/radioconda-win-64.txt - -(on Linux/macOS): - - mamba install --file https://github.com/ryanvolz/radioconda/releases/download/20NN.NN.NN/radioconda-$(conda info | sed -n -e 's/^.*platform : //p').txt - ## Additional Installation for Device Support To use particular software radio devices, it might be necessary to install additional drivers or firmware. Find your device below and follow the instructions. (Help add to this section by filing an issue if the instructions don't work or you have additional instructions to add!) diff --git a/build_metapackage.py b/build_metapackage.py new file mode 100755 index 0000000..c50bf78 --- /dev/null +++ b/build_metapackage.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +import pathlib +import re + +comment_re = re.compile(r"^\s*#\s*(?P.*)\s*$") +key_value_re = re.compile(r"^(?P.*):\s*(?P.*)\s*$") + + +def read_lock_file(lock_file: pathlib.Path) -> dict: + with lock_file.open("r") as f: + lines = f.read().splitlines() + + lock_dict = dict(specs=[]) + for line in lines: + comment_match = comment_re.match(line) + if comment_match: + m = key_value_re.match(comment_match.group("comment")) + if m: + lock_dict[m.group("key")] = m.group("value") + else: + lock_dict["specs"].append(line) + + return lock_dict + + +if __name__ == "__main__": + import argparse + import os + import subprocess + import shutil + import sys + + import conda_build.config + + conda_build_config = conda_build.config.Config() + + cwd = pathlib.Path(".").absolute() + here = pathlib.Path(__file__).parent.absolute().relative_to(cwd) + distname = os.getenv("DISTNAME", "radioconda") + platform = os.getenv("PLATFORM", conda_build_config.subdir) + source = "/".join( + ( + os.getenv("GITHUB_SERVER_URL", "https://github.com"), + os.getenv("GITHUB_REPOSITORY", "ryanvolz/radioconda"), + ) + ) + license_id = os.getenv("LICENSE_ID", "BSD-3-Clause") + summary = os.getenv("METAPACKAGE_SUMMARY", f"Metapackage for {distname}.") + + parser = argparse.ArgumentParser( + description=( + "Build environment metapackage using conda-build." + " Additional command-line options will be passed to conda metapackage." + ) + ) + parser.add_argument( + "lock_file", + type=pathlib.Path, + nargs="?", + default=here / "installer_specs" / f"{distname}-{platform}.txt", + help=( + "Environment lock file for a particular platform" + " (name ends in the platform identifier)." + " (default: %(default)s)" + ), + ) + parser.add_argument( + "-o", + "--output_dir", + type=pathlib.Path, + default=here / "dist" / "conda-bld", + help=( + "Output directory in which the metapackage will be placed." + " (default: %(default)s)" + ), + ) + parser.add_argument( + "--home", + default=source, + help="The homepage for the metapackage. (default: %(default)s)", + ) + parser.add_argument( + "--license", + default=license_id, + help="The SPDX license identifier for the metapackage. (default: %(default)s)", + ) + parser.add_argument( + "--summary", + default=summary, + help="Summary of the package. (default: %(default)s)", + ) + + args, metapackage_args = parser.parse_known_args() + + lock_dict = read_lock_file(args.lock_file) + + name = lock_dict.get("name", distname) + version = lock_dict.get("version", "0") + platform = lock_dict.get("platform", platform) + + env = os.environ.copy() + env["CONDA_SUBDIR"] = platform + + channels = [c.strip() for c in lock_dict.get("channels", "conda-forge").split(",")] + + conda_metapackage_cmdline = [ + "conda", + "metapackage", + name, + version, + "--no-anaconda-upload", + "--home", + args.home, + "--license", + args.license, + "--summary", + args.summary, + ] + for channel in channels: + conda_metapackage_cmdline.extend(["--channel", channel]) + conda_metapackage_cmdline.extend(["--dependencies"] + lock_dict["specs"]) + conda_metapackage_cmdline.extend(metapackage_args) + + proc = subprocess.run(conda_metapackage_cmdline, env=env) + + try: + proc.check_returncode() + except subprocess.CalledProcessError: + sys.exit(1) + + bldpkgs_dir = pathlib.Path(conda_build_config.bldpkgs_dir) + pkg_paths = list(bldpkgs_dir.glob(f"{name}-{version}*.bz2")) + pkg_out_dir = args.output_dir / platform + pkg_out_dir.mkdir(parents=True, exist_ok=True) + + for pkg in pkg_paths: + shutil.copy(pkg, pkg_out_dir) diff --git a/buildenv.yaml b/buildenv.yaml index f394ecd..399b0c9 100644 --- a/buildenv.yaml +++ b/buildenv.yaml @@ -2,4 +2,6 @@ name: buildenv channels: - conda-forge dependencies: + - anaconda-client + - conda-build - constructor diff --git a/radioconda.yaml b/radioconda.yaml index 37d8609..10ed2ca 100644 --- a/radioconda.yaml +++ b/radioconda.yaml @@ -7,19 +7,13 @@ platforms: - osx-64 - win-64 dependencies: - - mamba - - ipython - - python - # restrict to python 3.8 on Windows for Windows 7 compatibility - - python 3.8.* # [win] - - radioconda_console_shortcut # [win] - - digital_rf - gnuradio 3.9.* - gnuradio-osmosdr - gnuradio-satellites - gnuradio-soapy - gqrx + - ipython - libiio - libm2k - limesuite @@ -27,6 +21,10 @@ dependencies: - numpy - pandas - pyadi-iio + - python + # restrict to python 3.8 on Windows for Windows 7 compatibility + - python 3.8.* # [win] + - radioconda_console_shortcut # [win] - rtl-sdr - scipy - soapysdr diff --git a/rerender.py b/rerender.py index 26ce1ab..1bbe4b2 100755 --- a/rerender.py +++ b/rerender.py @@ -1,12 +1,17 @@ #!/usr/bin/env python3 import pathlib import shutil +from typing import List, Optional import conda_lock import yaml -def render_lock_spec( +def name_from_pkg_spec(spec: str): + return spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0] + + +def lock_env_spec( lock_spec: conda_lock.src_parser.LockSpecification, conda_exe: str ) -> conda_lock.src_parser.LockSpecification: create_env_dict = conda_lock.conda_lock.solve_specs_for_arch( @@ -16,36 +21,104 @@ def render_lock_spec( platform=lock_spec.platform, ) pkgs = create_env_dict["actions"]["LINK"] - spec_names = set( - spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0] - for spec in lock_spec.specs - ) + locked_specs = ["{name}={version}={build_string}".format(**pkg) for pkg in pkgs] - rendered_specs = [] - rendered_dep_specs = [] - for pkg in pkgs: - pkg_spec = "{name}={version}={build_string}".format(**pkg) - if pkg["name"] in spec_names: - rendered_specs.append(pkg_spec) - else: - rendered_dep_specs.append(pkg_spec) - - rendered_lock_spec = conda_lock.src_parser.LockSpecification( - specs=sorted(rendered_specs), - channels=lock_spec.channels, - platform=lock_spec.platform, - ) - rendered_full_lock_spec = conda_lock.src_parser.LockSpecification( - specs=sorted(rendered_specs + rendered_dep_specs), + locked_env_spec = conda_lock.src_parser.LockSpecification( + specs=sorted(locked_specs), channels=lock_spec.channels, platform=lock_spec.platform, ) - return rendered_lock_spec, rendered_full_lock_spec + return locked_env_spec -def render_constructor_specs( +def write_lock_file( + lock_spec: conda_lock.src_parser.LockSpecification, + lock_file_path: pathlib.Path, + name: Optional[str] = None, + version: Optional[str] = None, + channels: Optional[List[str]] = None, +): + lockfile_contents = [ + f"# platform: {lock_spec.platform}", + f"# env_hash: {lock_spec.env_hash()}", + ] + if name: + lockfile_contents.append(f"# name: {name}") + if version: + lockfile_contents.append(f"# version: {version}") + if channels: + lockfile_contents.append(f"# channels: {','.join(channels)}") + lockfile_contents.extend(lock_spec.specs) + with lock_file_path.open("w") as f: + f.write("\n".join(lockfile_contents)) + + +def render_constructor( + lock_spec: conda_lock.src_parser.LockSpecification, + name: str, + version: str, + company: str, + license_file: pathlib.Path, + output_dir: pathlib.Path, +) -> dict: + platform = lock_spec.platform + constructor_name = f"{name}-{platform}" + + construct_dict = dict( + name=name, + version=version, + company=company, + channels=lock_spec.channels, + specs=lock_spec.specs, + initialize_by_default=True, + installer_type="all", + keep_pkgs=True, + license_file="LICENSE", + register_python_default=False, + write_condarc=True, + ) + if platform.startswith("win"): + construct_dict["post_install"] = "post_install.bat" + else: + construct_dict["post_install"] = "post_install.sh" + + constructor_dir = output_dir / constructor_name + if constructor_dir.exists(): + shutil.rmtree(constructor_dir) + constructor_dir.mkdir(parents=True) + + # copy license to the constructor directory + shutil.copy(license_file, constructor_dir / "LICENSE") + + # write the post_install scripts referenced in the construct dict + if platform.startswith("win"): + with (constructor_dir / "post_install.bat").open("w") as f: + f.write("\n".join((r"del /q %PREFIX%\pkgs\*.tar.bz2", "exit 0", ""))) + else: + with (constructor_dir / "post_install.sh").open("w") as f: + f.write( + "\n".join( + ( + "#!/bin/sh", + f'PREFIX="${{PREFIX:-$2/{name}}}"', + r"rm -f $PREFIX/pkgs/*.tar.bz2", + "exit 0", + "", + ) + ) + ) + + construct_yaml_path = constructor_dir / "construct.yaml" + with construct_yaml_path.open("w") as f: + yaml.safe_dump(construct_dict, stream=f) + + return construct_dict + + +def render_platforms( environment_file: pathlib.Path, + installer_pkg_specs: List[str], version: str, company: str, license_file: pathlib.Path, @@ -55,75 +128,83 @@ def render_constructor_specs( with environment_file.open("r") as f: env_yaml_data = yaml.safe_load(f) - installer_name = env_yaml_data["name"] + env_name = env_yaml_data["name"] platforms = env_yaml_data["platforms"] if not license_file.exists(): raise ValueError(f"Cannot find license file: {license_file}") - output_dir.mkdir(parents=True, exist_ok=True) + if output_dir.exists(): + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True) - constructor_specs = {} + rendered_platforms = {} for platform in platforms: - constructor_name = f"{installer_name}-{platform}" + output_name = f"{env_name}-{platform}" - lock_spec = conda_lock.conda_lock.parse_environment_file( + # get the environment specification for the list of packages from the env file + env_spec = conda_lock.conda_lock.parse_environment_file( environment_file=environment_file, platform=platform ) - rendered_lock_spec, rendered_full_lock_spec = render_lock_spec( - lock_spec, conda_exe + + # lock the full environment specification to specific versions and builds + locked_env_spec = lock_env_spec(env_spec, conda_exe) + + # write the full environment specification to a lock file + lock_file_path = output_dir / f"{output_name}.txt" + write_lock_file( + locked_env_spec, + lock_file_path, + name=env_name, + version=version, + channels=locked_env_spec.channels, ) - construct_dict = dict( - name=installer_name, + + # add installer-only (base environment) packages and lock those too + installer_spec = conda_lock.src_parser.LockSpecification( + specs=sorted(locked_env_spec.specs + installer_pkg_specs), + channels=locked_env_spec.channels, + platform=locked_env_spec.platform, + ) + locked_installer_spec = lock_env_spec(installer_spec, conda_exe) + + # get a set of only the packages to put in the constructor specification + # taken from the installer-only list and those explicitly selected originally + constructor_pkg_names = set( + name_from_pkg_spec(spec) for spec in env_spec.specs + installer_pkg_specs + ) + + # filter the installer spec by the constructor package names + constructor_pkg_specs = [ + spec + for spec in locked_installer_spec.specs + if name_from_pkg_spec(spec) in constructor_pkg_names + ] + constructor_spec = conda_lock.src_parser.LockSpecification( + specs=constructor_pkg_specs, + channels=locked_installer_spec.channels, + platform=locked_installer_spec.platform, + ) + + # create the rendered constructor directory + constructor_dict = render_constructor( + lock_spec=constructor_spec, + name=env_name, version=version, company=company, - channels=rendered_lock_spec.channels, - specs=rendered_lock_spec.specs, - initialize_by_default=True, - installer_type="all", - keep_pkgs=True, - license_file="LICENSE", - register_python_default=False, - write_condarc=True, + license_file=license_file, + output_dir=output_dir, ) - if platform.startswith("win"): - construct_dict["post_install"] = "post_install.bat" - else: - construct_dict["post_install"] = "post_install.sh" - constructor_specs[constructor_name] = construct_dict + # aggregate output + rendered_platforms[output_name] = dict( + locked_env_spec=locked_env_spec, + locked_installer_spec=locked_installer_spec, + constructor_dict=constructor_dict, + ) - constructor_dir = output_dir / constructor_name - if constructor_dir.exists(): - shutil.rmtree(constructor_dir) - constructor_dir.mkdir(parents=True) - - # copy license to the constructor directory - shutil.copy(license_file, constructor_dir / "LICENSE") - - # write the post_install scripts referenced in the construct dict - if platform.startswith("win"): - with (constructor_dir / "post_install.bat").open("w") as f: - f.writelines((r"del /q %PREFIX%\pkgs\*.tar.bz2", "")) - else: - with (constructor_dir / "post_install.sh").open("w") as f: - f.writelines((r"rm -f $PREFIX/pkgs/*.tar.bz2", "")) - - construct_yaml_path = constructor_dir / "construct.yaml" - with construct_yaml_path.open("w") as f: - yaml.safe_dump(construct_dict, stream=f) - - lockfile_contents = [ - f"# platform: {rendered_full_lock_spec.platform}", - f"# env_hash: {rendered_full_lock_spec.env_hash()}", - ] - lockfile_contents.extend(rendered_full_lock_spec.specs) - lock_file_path = constructor_dir / f"{constructor_name}.txt" - with lock_file_path.open("w") as f: - f.writelines(s + "\n" for s in lockfile_contents) - - return constructor_specs + return rendered_platforms if __name__ == "__main__": @@ -134,7 +215,12 @@ if __name__ == "__main__": cwd = pathlib.Path(".").absolute() here = pathlib.Path(__file__).parent.absolute().relative_to(cwd) distname = os.getenv("DISTNAME", "radioconda") - source = os.getenv("SOURCE", "github.com/ryanvolz/radioconda") + source = "/".join( + ( + os.getenv("GITHUB_SERVER_URL", "https://github.com"), + os.getenv("GITHUB_REPOSITORY", "ryanvolz/radioconda"), + ) + ) dt = datetime.datetime.now() version = dt.strftime("%Y.%m.%d") @@ -211,8 +297,9 @@ if __name__ == "__main__": conda_executable=args.conda_exe, mamba=True, micromamba=True ) - constructor_specs = render_constructor_specs( + constructor_specs = render_platforms( environment_file=args.environment_file, + installer_pkg_specs=["mamba"], version=args.version, company=args.company, license_file=args.license_file,