Separate env packages and installer-only (base) packages.

This makes the rendered lock files more useful for users (they won't
install conda/mamba in a non-base environment) and will allow for the
creation of a metapackage for each release.

.github/workflows/build_radioconda.yml

@@ -69,7 +69,7 @@ jobs:
         env:
           PLATFORM: ${{ matrix.PLATFORM }}
         run: |
-          cp installer_specs/$DISTNAME-$PLATFORM/$DISTNAME-$PLATFORM.txt dist/
+          cp installer_specs/$DISTNAME-$PLATFORM.txt dist/
           ls -lh dist
 
       - name: Test installer (sh)

radioconda.yaml

@@ -7,19 +7,13 @@ platforms:
   - osx-64
   - win-64
 dependencies:
-  - mamba
-  - ipython
-  - python
-  # restrict to python 3.8 on Windows for Windows 7 compatibility
-  - python 3.8.* # [win]
-  - radioconda_console_shortcut # [win]
-
   - digital_rf
   - gnuradio 3.8.*
   - gnuradio-osmosdr
   - gnuradio-satellites
   - gnuradio-soapy
   - gqrx
+  - ipython
   - libiio
   - libm2k
   - limesuite
@@ -27,6 +21,10 @@ dependencies:
   - numpy
   - pandas
   - pyadi-iio
+  - python
+  # restrict to python 3.8 on Windows for Windows 7 compatibility
+  - python 3.8.* # [win]
+  - radioconda_console_shortcut # [win]
   - rtl-sdr
   - scipy
   - soapysdr

rerender.py

@@ -1,12 +1,17 @@
 #!/usr/bin/env python3
 import pathlib
 import shutil
+from typing import List, Optional
 
 import conda_lock
 import yaml
 
 
-def render_lock_spec(
+def name_from_pkg_spec(spec: str):
+    return spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0]
+
+
+def lock_env_spec(
     lock_spec: conda_lock.src_parser.LockSpecification, conda_exe: str
 ) -> conda_lock.src_parser.LockSpecification:
     create_env_dict = conda_lock.conda_lock.solve_specs_for_arch(
@@ -16,70 +21,56 @@ def render_lock_spec(
         platform=lock_spec.platform,
     )
     pkgs = create_env_dict["actions"]["LINK"]
-    spec_names = set(
-        spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0]
-        for spec in lock_spec.specs
-    )
+    locked_specs = ["{name}={version}={build_string}".format(**pkg) for pkg in pkgs]
 
-    rendered_specs = []
-    rendered_dep_specs = []
-    for pkg in pkgs:
-        pkg_spec = "{name}={version}={build_string}".format(**pkg)
-        if pkg["name"] in spec_names:
-            rendered_specs.append(pkg_spec)
-        else:
-            rendered_dep_specs.append(pkg_spec)
-
-    rendered_lock_spec = conda_lock.src_parser.LockSpecification(
-        specs=sorted(rendered_specs),
-        channels=lock_spec.channels,
-        platform=lock_spec.platform,
-    )
-    rendered_full_lock_spec = conda_lock.src_parser.LockSpecification(
-        specs=sorted(rendered_specs + rendered_dep_specs),
+    locked_env_spec = conda_lock.src_parser.LockSpecification(
+        specs=sorted(locked_specs),
         channels=lock_spec.channels,
         platform=lock_spec.platform,
     )
 
-    return rendered_lock_spec, rendered_full_lock_spec
+    return locked_env_spec
 
 
-def render_constructor_specs(
-    environment_file: pathlib.Path,
+def write_lock_file(
+    lock_spec: conda_lock.src_parser.LockSpecification,
+    lock_file_path: pathlib.Path,
+    name: Optional[str] = None,
+    version: Optional[str] = None,
+    channels: Optional[List[str]] = None,
+):
+    lockfile_contents = [
+        f"# platform: {lock_spec.platform}",
+        f"# env_hash: {lock_spec.env_hash()}",
+    ]
+    if name:
+        lockfile_contents.append(f"# name: {name}")
+    if version:
+        lockfile_contents.append(f"# version: {version}")
+    if channels:
+        lockfile_contents.append(f"# channels: {','.join(channels)}")
+    lockfile_contents.extend(lock_spec.specs)
+    with lock_file_path.open("w") as f:
+        f.write("\n".join(lockfile_contents))
+
+
+def render_constructor(
+    lock_spec: conda_lock.src_parser.LockSpecification,
+    name: str,
     version: str,
     company: str,
     license_file: pathlib.Path,
     output_dir: pathlib.Path,
-    conda_exe: str,
 ) -> dict:
-    with environment_file.open("r") as f:
-        env_yaml_data = yaml.safe_load(f)
+    platform = lock_spec.platform
+    constructor_name = f"{name}-{platform}"
 
-    installer_name = env_yaml_data["name"]
-    platforms = env_yaml_data["platforms"]
-
-    if not license_file.exists():
-        raise ValueError(f"Cannot find license file: {license_file}")
-
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    constructor_specs = {}
-
-    for platform in platforms:
-        constructor_name = f"{installer_name}-{platform}"
-
-        lock_spec = conda_lock.conda_lock.parse_environment_file(
-            environment_file=environment_file, platform=platform
-        )
-        rendered_lock_spec, rendered_full_lock_spec = render_lock_spec(
-            lock_spec, conda_exe
-        )
     construct_dict = dict(
-            name=installer_name,
+        name=name,
         version=version,
         company=company,
-            channels=rendered_lock_spec.channels,
-            specs=rendered_lock_spec.specs,
+        channels=lock_spec.channels,
+        specs=lock_spec.specs,
         initialize_by_default=True,
         installer_type="all",
         keep_pkgs=True,
@@ -92,8 +83,6 @@ def render_constructor_specs(
     else:
         construct_dict["post_install"] = "post_install.sh"
 
-        constructor_specs[constructor_name] = construct_dict
-
     constructor_dir = output_dir / constructor_name
     if constructor_dir.exists():
         shutil.rmtree(constructor_dir)
@@ -112,7 +101,7 @@ def render_constructor_specs(
                 "\n".join(
                     (
                         "#!/bin/sh",
-                            f'PREFIX="${{PREFIX:-$2/{installer_name}}}"',
+                        f'PREFIX="${{PREFIX:-$2/{name}}}"',
                         r"rm -f $PREFIX/pkgs/*.tar.bz2",
                         "exit 0",
                         "",
@@ -124,16 +113,98 @@ def render_constructor_specs(
     with construct_yaml_path.open("w") as f:
         yaml.safe_dump(construct_dict, stream=f)
 
-        lockfile_contents = [
-            f"# platform: {rendered_full_lock_spec.platform}",
-            f"# env_hash: {rendered_full_lock_spec.env_hash()}",
-        ]
-        lockfile_contents.extend(rendered_full_lock_spec.specs)
-        lock_file_path = constructor_dir / f"{constructor_name}.txt"
-        with lock_file_path.open("w") as f:
-            f.write("\n".join(lockfile_contents))
+    return construct_dict
 
-    return constructor_specs
+
+def render_platforms(
+    environment_file: pathlib.Path,
+    installer_pkg_specs: List[str],
+    version: str,
+    company: str,
+    license_file: pathlib.Path,
+    output_dir: pathlib.Path,
+    conda_exe: str,
+) -> dict:
+    with environment_file.open("r") as f:
+        env_yaml_data = yaml.safe_load(f)
+
+    env_name = env_yaml_data["name"]
+    platforms = env_yaml_data["platforms"]
+
+    if not license_file.exists():
+        raise ValueError(f"Cannot find license file: {license_file}")
+
+    if output_dir.exists():
+        shutil.rmtree(output_dir)
+    output_dir.mkdir(parents=True)
+
+    rendered_platforms = {}
+
+    for platform in platforms:
+        output_name = f"{env_name}-{platform}"
+
+        # get the environment specification for the list of packages from the env file
+        env_spec = conda_lock.conda_lock.parse_environment_file(
+            environment_file=environment_file, platform=platform
+        )
+
+        # lock the full environment specification to specific versions and builds
+        locked_env_spec = lock_env_spec(env_spec, conda_exe)
+
+        # write the full environment specification to a lock file
+        lock_file_path = output_dir / f"{output_name}.txt"
+        write_lock_file(
+            locked_env_spec,
+            lock_file_path,
+            name=env_name,
+            version=version,
+            channels=locked_env_spec.channels,
+        )
+
+        # add installer-only (base environment) packages and lock those too
+        installer_spec = conda_lock.src_parser.LockSpecification(
+            specs=sorted(locked_env_spec.specs + installer_pkg_specs),
+            channels=locked_env_spec.channels,
+            platform=locked_env_spec.platform,
+        )
+        locked_installer_spec = lock_env_spec(installer_spec, conda_exe)
+
+        # get a set of only the packages to put in the constructor specification
+        # taken from the installer-only list and those explicitly selected originally
+        constructor_pkg_names = set(
+            name_from_pkg_spec(spec) for spec in env_spec.specs + installer_pkg_specs
+        )
+
+        # filter the installer spec by the constructor package names
+        constructor_pkg_specs = [
+            spec
+            for spec in locked_installer_spec.specs
+            if name_from_pkg_spec(spec) in constructor_pkg_names
+        ]
+        constructor_spec = conda_lock.src_parser.LockSpecification(
+            specs=constructor_pkg_specs,
+            channels=locked_installer_spec.channels,
+            platform=locked_installer_spec.platform,
+        )
+
+        # create the rendered constructor directory
+        constructor_dict = render_constructor(
+            lock_spec=constructor_spec,
+            name=env_name,
+            version=version,
+            company=company,
+            license_file=license_file,
+            output_dir=output_dir,
+        )
+
+        # aggregate output
+        rendered_platforms[output_name] = dict(
+            locked_env_spec=locked_env_spec,
+            locked_installer_spec=locked_installer_spec,
+            constructor_dict=constructor_dict,
+        )
+
+    return rendered_platforms
 
 
 if __name__ == "__main__":
@@ -221,8 +292,9 @@ if __name__ == "__main__":
         conda_executable=args.conda_exe, mamba=True, micromamba=True
     )
 
-    constructor_specs = render_constructor_specs(
+    constructor_specs = render_platforms(
         environment_file=args.environment_file,
+        installer_pkg_specs=["mamba"],
         version=args.version,
         company=args.company,
         license_file=args.license_file,