Merge branch 'next' into dev

.github/workflows/build_radioconda.yml

@@ -6,10 +6,13 @@ on:
   pull_request:
     paths:
       - "installer_specs/**"
+  workflow_dispatch:
 
 env:
   DISTNAME: radioconda
-  SOURCE: github.com/ryanvolz/radioconda
+  LICENSE_ID: BSD-3-Clause
+  METAPACKAGE_LABEL: dev
+  METAPACKAGE_SUMMARY: Metapackage to install the radioconda package set.
 
 jobs:
   build:
@@ -45,31 +48,31 @@ jobs:
         with:
           environment-file: buildenv.yaml
 
-      - name: Build installer (bash)
+      - name: Build installer
-        if: runner.os != 'Windows'
+        shell: bash -l {0}
+        env:
+          PLATFORM: ${{ matrix.PLATFORM }}
+          OS_NAME: ${{ matrix.OS_NAME }}
+        run: |
+          if [ "$OS_NAME" == "Windows" ]; then
+            PATH=$CONDA_PREFIX/NSIS:$PATH
+          fi
+          python build_installer.py -v
+
+      - name: Build metapackage
         shell: bash -l {0}
         env:
           PLATFORM: ${{ matrix.PLATFORM }}
         run: |
-          python build_installer.py -v
+          python build_metapackage.py installer_specs/$DISTNAME-$PLATFORM.txt
 
-      - name: Build installer (cmd.exe)
+      - name: Copy lock file and list built installers and packages
-        if: runner.os == 'Windows'
-        shell: powershell
-        env:
-          PLATFORM: ${{ matrix.PLATFORM }}
-        run: |
-          $Env:Path = "$Env:CONDA_PREFIX\NSIS;$Env:Path"
-          echo $Env:Path
-          python build_installer.py -v
-
-      - name: Copy spec and list built installers
         shell: bash
         env:
           PLATFORM: ${{ matrix.PLATFORM }}
         run: |
-          cp installer_specs/$DISTNAME-$PLATFORM/$DISTNAME-$PLATFORM.txt dist/
+          cp installer_specs/$DISTNAME-$PLATFORM.txt dist/
-          ls -lh dist
+          ls -lhR dist
 
       - name: Test installer (sh)
         if: contains(matrix.OS_NAME, 'Linux') || contains(matrix.OS_NAME, 'MacOSX')
@@ -86,15 +89,30 @@ jobs:
 
       - name: Test installer (pkg)
         if: contains(matrix.OS_NAME, 'MacOSX')
-        continue-on-error: true # this test doesn't work yet
         shell: bash
         env:
           OS_NAME: ${{ matrix.OS_NAME }}
           ARCH: ${{ matrix.ARCH }}
           TARGET_VOLUME: CurrentUserHomeDirectory
-          INSTALL_PATH: ~/Applications/${{ env.DISTNAME }}
+          INSTALL_PATH: ${{ github.workspace }}/../../../${{ env.DISTNAME }}
         run: |
-          installer -verbose -pkg dist/$DISTNAME-*-$OS_NAME-$ARCH.pkg -target $TARGET_VOLUME
+          cat >pkg-choices.xml <<EOF
+          <?xml version="1.0" encoding="UTF-8"?>
+          <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+          <plist version="1.0">
+          <array>
+              <dict>
+                  <key>attributeSetting</key>
+                  <integer>0</integer>
+                  <key>choiceAttribute</key>
+                  <string>selected</string>
+                  <key>choiceIdentifier</key>
+                  <string>io.continuum.pkg.pathupdate</string>
+              </dict>
+          </array>
+          </plist>
+          EOF
+          installer -verbose -dumplog -applyChoiceChangesXML pkg-choices.xml -pkg dist/$DISTNAME-*-$OS_NAME-$ARCH.pkg -target $TARGET_VOLUME
           eval "$($INSTALL_PATH/bin/conda shell.bash hook)"
           conda info
           conda list
@@ -145,3 +163,12 @@ jobs:
           tag: ${{ github.ref }}
           overwrite: true
           file_glob: true
+
+      - name: Upload metapackage to Anaconda.org
+        if: startsWith(github.ref, 'refs/tags/')
+        shell: bash -l {0}
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+        run: |
+          micromamba --help
+          anaconda upload -l $METAPACKAGE_LABEL --skip-existing dist/conda-bld/**/*

README.md

@@ -94,7 +94,19 @@ Once you have radioconda installed, you can stay up to date for all packages wit
 
 ### Upgrade to latest release
 
-To install the latest release in particular, run (on Windows):
+To install the latest release in particular, run
+
+    mamba upgrade -c ryanvolz radioconda
+
+### Install a particular release
+
+To install a particular release version, substitute the desired version number and run
+
+    mamba install -c ryanvolz radioconda=20NN.NN.NN
+
+### Install from environment file
+
+You can also install from the released environment file (on Windows):
 
     mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-win-64.txt
 
@@ -102,16 +114,6 @@ To install the latest release in particular, run (on Windows):
 
     mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-$(conda info | sed -n -e 's/^.*platform : //p').txt
 
-### Install a particular release
-
-To install the package versions associated with a particular release, substitute the release number and run the following (on Windows):
-
-    mamba install --file https://github.com/ryanvolz/radioconda/releases/download/20NN.NN.NN/radioconda-win-64.txt
-
-(on Linux/macOS):
-
-    mamba install --file https://github.com/ryanvolz/radioconda/releases/download/20NN.NN.NN/radioconda-$(conda info | sed -n -e 's/^.*platform : //p').txt
-
 ## Additional Installation for Device Support
 
 To use particular software radio devices, it might be necessary to install additional drivers or firmware. Find your device below and follow the instructions. (Help add to this section by filing an issue if the instructions don't work or you have additional instructions to add!)

build_metapackage.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+import pathlib
+import re
+
+comment_re = re.compile(r"^\s*#\s*(?P<comment>.*)\s*$")
+key_value_re = re.compile(r"^(?P<key>.*):\s*(?P<value>.*)\s*$")
+
+
+def read_lock_file(lock_file: pathlib.Path) -> dict:
+    with lock_file.open("r") as f:
+        lines = f.read().splitlines()
+
+    lock_dict = dict(specs=[])
+    for line in lines:
+        comment_match = comment_re.match(line)
+        if comment_match:
+            m = key_value_re.match(comment_match.group("comment"))
+            if m:
+                lock_dict[m.group("key")] = m.group("value")
+        else:
+            lock_dict["specs"].append(line)
+
+    return lock_dict
+
+
+if __name__ == "__main__":
+    import argparse
+    import os
+    import subprocess
+    import shutil
+    import sys
+
+    import conda_build.config
+
+    conda_build_config = conda_build.config.Config()
+
+    cwd = pathlib.Path(".").absolute()
+    here = pathlib.Path(__file__).parent.absolute().relative_to(cwd)
+    distname = os.getenv("DISTNAME", "radioconda")
+    platform = os.getenv("PLATFORM", conda_build_config.subdir)
+    source = "/".join(
+        (
+            os.getenv("GITHUB_SERVER_URL", "https://github.com"),
+            os.getenv("GITHUB_REPOSITORY", "ryanvolz/radioconda"),
+        )
+    )
+    license_id = os.getenv("LICENSE_ID", "BSD-3-Clause")
+    summary = os.getenv("METAPACKAGE_SUMMARY", f"Metapackage for {distname}.")
+
+    parser = argparse.ArgumentParser(
+        description=(
+            "Build environment metapackage using conda-build."
+            " Additional command-line options will be passed to conda metapackage."
+        )
+    )
+    parser.add_argument(
+        "lock_file",
+        type=pathlib.Path,
+        nargs="?",
+        default=here / "installer_specs" / f"{distname}-{platform}.txt",
+        help=(
+            "Environment lock file for a particular platform"
+            " (name ends in the platform identifier)."
+            " (default: %(default)s)"
+        ),
+    )
+    parser.add_argument(
+        "-o",
+        "--output_dir",
+        type=pathlib.Path,
+        default=here / "dist" / "conda-bld",
+        help=(
+            "Output directory in which the metapackage will be placed."
+            " (default: %(default)s)"
+        ),
+    )
+    parser.add_argument(
+        "--home",
+        default=source,
+        help="The homepage for the metapackage. (default: %(default)s)",
+    )
+    parser.add_argument(
+        "--license",
+        default=license_id,
+        help="The SPDX license identifier for the metapackage. (default: %(default)s)",
+    )
+    parser.add_argument(
+        "--summary",
+        default=summary,
+        help="Summary of the package. (default: %(default)s)",
+    )
+
+    args, metapackage_args = parser.parse_known_args()
+
+    lock_dict = read_lock_file(args.lock_file)
+
+    name = lock_dict.get("name", distname)
+    version = lock_dict.get("version", "0")
+    platform = lock_dict.get("platform", platform)
+
+    env = os.environ.copy()
+    env["CONDA_SUBDIR"] = platform
+
+    channels = [c.strip() for c in lock_dict.get("channels", "conda-forge").split(",")]
+
+    conda_metapackage_cmdline = [
+        "conda",
+        "metapackage",
+        name,
+        version,
+        "--no-anaconda-upload",
+        "--home",
+        args.home,
+        "--license",
+        args.license,
+        "--summary",
+        args.summary,
+    ]
+    for channel in channels:
+        conda_metapackage_cmdline.extend(["--channel", channel])
+    conda_metapackage_cmdline.extend(["--dependencies"] + lock_dict["specs"])
+    conda_metapackage_cmdline.extend(metapackage_args)
+
+    proc = subprocess.run(conda_metapackage_cmdline, env=env)
+
+    try:
+        proc.check_returncode()
+    except subprocess.CalledProcessError:
+        sys.exit(1)
+
+    bldpkgs_dir = pathlib.Path(conda_build_config.bldpkgs_dir)
+    pkg_paths = list(bldpkgs_dir.glob(f"{name}-{version}*.bz2"))
+    pkg_out_dir = args.output_dir / platform
+    pkg_out_dir.mkdir(parents=True, exist_ok=True)
+
+    for pkg in pkg_paths:
+        shutil.copy(pkg, pkg_out_dir)

buildenv.yaml

@@ -2,4 +2,6 @@ name: buildenv
 channels:
   - conda-forge
 dependencies:
+  - anaconda-client
+  - conda-build
   - constructor

radioconda.yaml

@@ -7,19 +7,13 @@ platforms:
   - osx-64
   - win-64
 dependencies:
-  - mamba
-  - ipython
-  - python
-  # restrict to python 3.8 on Windows for Windows 7 compatibility
-  - python 3.8.* # [win]
-  - radioconda_console_shortcut # [win]
-
   - digital_rf
   - gnuradio 3.9.*
   - gnuradio-osmosdr
   - gnuradio-satellites
   - gnuradio-soapy
   - gqrx
+  - ipython
   - libiio
   - libm2k
   - limesuite
@@ -27,6 +21,10 @@ dependencies:
   - numpy
   - pandas
   - pyadi-iio
+  - python
+  # restrict to python 3.8 on Windows for Windows 7 compatibility
+  - python 3.8.* # [win]
+  - radioconda_console_shortcut # [win]
   - rtl-sdr
   - scipy
   - soapysdr

rerender.py

@@ -1,12 +1,17 @@
 #!/usr/bin/env python3
 import pathlib
 import shutil
+from typing import List, Optional
 
 import conda_lock
 import yaml
 
 
-def render_lock_spec(
+def name_from_pkg_spec(spec: str):
+    return spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0]
+
+
+def lock_env_spec(
     lock_spec: conda_lock.src_parser.LockSpecification, conda_exe: str
 ) -> conda_lock.src_parser.LockSpecification:
     create_env_dict = conda_lock.conda_lock.solve_specs_for_arch(
@@ -16,36 +21,104 @@ def render_lock_spec(
         platform=lock_spec.platform,
     )
     pkgs = create_env_dict["actions"]["LINK"]
-    spec_names = set(
+    locked_specs = ["{name}={version}={build_string}".format(**pkg) for pkg in pkgs]
-        spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0]
-        for spec in lock_spec.specs
-    )
 
-    rendered_specs = []
+    locked_env_spec = conda_lock.src_parser.LockSpecification(
-    rendered_dep_specs = []
+        specs=sorted(locked_specs),
-    for pkg in pkgs:
-        pkg_spec = "{name}={version}={build_string}".format(**pkg)
-        if pkg["name"] in spec_names:
-            rendered_specs.append(pkg_spec)
-        else:
-            rendered_dep_specs.append(pkg_spec)
-
-    rendered_lock_spec = conda_lock.src_parser.LockSpecification(
-        specs=sorted(rendered_specs),
-        channels=lock_spec.channels,
-        platform=lock_spec.platform,
-    )
-    rendered_full_lock_spec = conda_lock.src_parser.LockSpecification(
-        specs=sorted(rendered_specs + rendered_dep_specs),
         channels=lock_spec.channels,
         platform=lock_spec.platform,
     )
 
-    return rendered_lock_spec, rendered_full_lock_spec
+    return locked_env_spec
 
 
-def render_constructor_specs(
+def write_lock_file(
+    lock_spec: conda_lock.src_parser.LockSpecification,
+    lock_file_path: pathlib.Path,
+    name: Optional[str] = None,
+    version: Optional[str] = None,
+    channels: Optional[List[str]] = None,
+):
+    lockfile_contents = [
+        f"# platform: {lock_spec.platform}",
+        f"# env_hash: {lock_spec.env_hash()}",
+    ]
+    if name:
+        lockfile_contents.append(f"# name: {name}")
+    if version:
+        lockfile_contents.append(f"# version: {version}")
+    if channels:
+        lockfile_contents.append(f"# channels: {','.join(channels)}")
+    lockfile_contents.extend(lock_spec.specs)
+    with lock_file_path.open("w") as f:
+        f.write("\n".join(lockfile_contents))
+
+
+def render_constructor(
+    lock_spec: conda_lock.src_parser.LockSpecification,
+    name: str,
+    version: str,
+    company: str,
+    license_file: pathlib.Path,
+    output_dir: pathlib.Path,
+) -> dict:
+    platform = lock_spec.platform
+    constructor_name = f"{name}-{platform}"
+
+    construct_dict = dict(
+        name=name,
+        version=version,
+        company=company,
+        channels=lock_spec.channels,
+        specs=lock_spec.specs,
+        initialize_by_default=True,
+        installer_type="all",
+        keep_pkgs=True,
+        license_file="LICENSE",
+        register_python_default=False,
+        write_condarc=True,
+    )
+    if platform.startswith("win"):
+        construct_dict["post_install"] = "post_install.bat"
+    else:
+        construct_dict["post_install"] = "post_install.sh"
+
+    constructor_dir = output_dir / constructor_name
+    if constructor_dir.exists():
+        shutil.rmtree(constructor_dir)
+    constructor_dir.mkdir(parents=True)
+
+    # copy license to the constructor directory
+    shutil.copy(license_file, constructor_dir / "LICENSE")
+
+    # write the post_install scripts referenced in the construct dict
+    if platform.startswith("win"):
+        with (constructor_dir / "post_install.bat").open("w") as f:
+            f.write("\n".join((r"del /q %PREFIX%\pkgs\*.tar.bz2", "exit 0", "")))
+    else:
+        with (constructor_dir / "post_install.sh").open("w") as f:
+            f.write(
+                "\n".join(
+                    (
+                        "#!/bin/sh",
+                        f'PREFIX="${{PREFIX:-$2/{name}}}"',
+                        r"rm -f $PREFIX/pkgs/*.tar.bz2",
+                        "exit 0",
+                        "",
+                    )
+                )
+            )
+
+    construct_yaml_path = constructor_dir / "construct.yaml"
+    with construct_yaml_path.open("w") as f:
+        yaml.safe_dump(construct_dict, stream=f)
+
+    return construct_dict
+
+
+def render_platforms(
     environment_file: pathlib.Path,
+    installer_pkg_specs: List[str],
     version: str,
     company: str,
     license_file: pathlib.Path,
@@ -55,75 +128,83 @@ def render_constructor_specs(
     with environment_file.open("r") as f:
         env_yaml_data = yaml.safe_load(f)
 
-    installer_name = env_yaml_data["name"]
+    env_name = env_yaml_data["name"]
     platforms = env_yaml_data["platforms"]
 
     if not license_file.exists():
         raise ValueError(f"Cannot find license file: {license_file}")
 
-    output_dir.mkdir(parents=True, exist_ok=True)
+    if output_dir.exists():
+        shutil.rmtree(output_dir)
+    output_dir.mkdir(parents=True)
 
-    constructor_specs = {}
+    rendered_platforms = {}
 
     for platform in platforms:
-        constructor_name = f"{installer_name}-{platform}"
+        output_name = f"{env_name}-{platform}"
 
-        lock_spec = conda_lock.conda_lock.parse_environment_file(
+        # get the environment specification for the list of packages from the env file
+        env_spec = conda_lock.conda_lock.parse_environment_file(
             environment_file=environment_file, platform=platform
         )
-        rendered_lock_spec, rendered_full_lock_spec = render_lock_spec(
+
-            lock_spec, conda_exe
+        # lock the full environment specification to specific versions and builds
+        locked_env_spec = lock_env_spec(env_spec, conda_exe)
+
+        # write the full environment specification to a lock file
+        lock_file_path = output_dir / f"{output_name}.txt"
+        write_lock_file(
+            locked_env_spec,
+            lock_file_path,
+            name=env_name,
+            version=version,
+            channels=locked_env_spec.channels,
         )
-        construct_dict = dict(
+
-            name=installer_name,
+        # add installer-only (base environment) packages and lock those too
+        installer_spec = conda_lock.src_parser.LockSpecification(
+            specs=sorted(locked_env_spec.specs + installer_pkg_specs),
+            channels=locked_env_spec.channels,
+            platform=locked_env_spec.platform,
+        )
+        locked_installer_spec = lock_env_spec(installer_spec, conda_exe)
+
+        # get a set of only the packages to put in the constructor specification
+        # taken from the installer-only list and those explicitly selected originally
+        constructor_pkg_names = set(
+            name_from_pkg_spec(spec) for spec in env_spec.specs + installer_pkg_specs
+        )
+
+        # filter the installer spec by the constructor package names
+        constructor_pkg_specs = [
+            spec
+            for spec in locked_installer_spec.specs
+            if name_from_pkg_spec(spec) in constructor_pkg_names
+        ]
+        constructor_spec = conda_lock.src_parser.LockSpecification(
+            specs=constructor_pkg_specs,
+            channels=locked_installer_spec.channels,
+            platform=locked_installer_spec.platform,
+        )
+
+        # create the rendered constructor directory
+        constructor_dict = render_constructor(
+            lock_spec=constructor_spec,
+            name=env_name,
             version=version,
             company=company,
-            channels=rendered_lock_spec.channels,
+            license_file=license_file,
-            specs=rendered_lock_spec.specs,
+            output_dir=output_dir,
-            initialize_by_default=True,
-            installer_type="all",
-            keep_pkgs=True,
-            license_file="LICENSE",
-            register_python_default=False,
-            write_condarc=True,
         )
-        if platform.startswith("win"):
-            construct_dict["post_install"] = "post_install.bat"
-        else:
-            construct_dict["post_install"] = "post_install.sh"
 
-        constructor_specs[constructor_name] = construct_dict
+        # aggregate output
+        rendered_platforms[output_name] = dict(
+            locked_env_spec=locked_env_spec,
+            locked_installer_spec=locked_installer_spec,
+            constructor_dict=constructor_dict,
+        )
 
-        constructor_dir = output_dir / constructor_name
+    return rendered_platforms
-        if constructor_dir.exists():
-            shutil.rmtree(constructor_dir)
-        constructor_dir.mkdir(parents=True)
-
-        # copy license to the constructor directory
-        shutil.copy(license_file, constructor_dir / "LICENSE")
-
-        # write the post_install scripts referenced in the construct dict
-        if platform.startswith("win"):
-            with (constructor_dir / "post_install.bat").open("w") as f:
-                f.writelines((r"del /q %PREFIX%\pkgs\*.tar.bz2", ""))
-        else:
-            with (constructor_dir / "post_install.sh").open("w") as f:
-                f.writelines((r"rm -f $PREFIX/pkgs/*.tar.bz2", ""))
-
-        construct_yaml_path = constructor_dir / "construct.yaml"
-        with construct_yaml_path.open("w") as f:
-            yaml.safe_dump(construct_dict, stream=f)
-
-        lockfile_contents = [
-            f"# platform: {rendered_full_lock_spec.platform}",
-            f"# env_hash: {rendered_full_lock_spec.env_hash()}",
-        ]
-        lockfile_contents.extend(rendered_full_lock_spec.specs)
-        lock_file_path = constructor_dir / f"{constructor_name}.txt"
-        with lock_file_path.open("w") as f:
-            f.writelines(s + "\n" for s in lockfile_contents)
-
-    return constructor_specs
 
 
 if __name__ == "__main__":
@@ -134,7 +215,12 @@ if __name__ == "__main__":
     cwd = pathlib.Path(".").absolute()
     here = pathlib.Path(__file__).parent.absolute().relative_to(cwd)
     distname = os.getenv("DISTNAME", "radioconda")
-    source = os.getenv("SOURCE", "github.com/ryanvolz/radioconda")
+    source = "/".join(
+        (
+            os.getenv("GITHUB_SERVER_URL", "https://github.com"),
+            os.getenv("GITHUB_REPOSITORY", "ryanvolz/radioconda"),
+        )
+    )
 
     dt = datetime.datetime.now()
     version = dt.strftime("%Y.%m.%d")
@@ -211,8 +297,9 @@ if __name__ == "__main__":
         conda_executable=args.conda_exe, mamba=True, micromamba=True
     )
 
-    constructor_specs = render_constructor_specs(
+    constructor_specs = render_platforms(
         environment_file=args.environment_file,
+        installer_pkg_specs=["mamba"],
         version=args.version,
         company=args.company,
         license_file=args.license_file,