shway/radioconda
1
0
Fork 0
mirror of https://github.com/ryanvolz/radioconda.git synced 2024-09-19 12:06:34 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'next' into dev

Browse Source
This commit is contained in:
Ryan Volz 2021-05-21 13:33:39 -04:00
commit 3206dab3c6
6 changed files with 368 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
.github/workflows/build_radioconda.yml vendored
View File

@ -6,10 +6,13 @@ on:
pull_request:
paths:
- "installer_specs/**"
workflow_dispatch:
env:
DISTNAME: radioconda
SOURCE: github.com/ryanvolz/radioconda
LICENSE_ID: BSD-3-Clause
METAPACKAGE_LABEL: dev
METAPACKAGE_SUMMARY: Metapackage to install the radioconda package set.
jobs:
build:
@ -45,31 +48,31 @@ jobs:
with:
environment-file: buildenv.yaml
- name: Build installer (bash)
if: runner.os != 'Windows'
- name: Build installer
shell: bash -l {0}
env:
PLATFORM: ${{ matrix.PLATFORM }}
OS_NAME: ${{ matrix.OS_NAME }}
run: |
if [ "$OS_NAME" == "Windows" ]; then
PATH=$CONDA_PREFIX/NSIS:$PATH
fi
python build_installer.py -v
- name: Build metapackage
shell: bash -l {0}
env:
PLATFORM: ${{ matrix.PLATFORM }}
run: |
python build_installer.py -v
python build_metapackage.py installer_specs/$DISTNAME-$PLATFORM.txt
- name: Build installer (cmd.exe)
if: runner.os == 'Windows'
shell: powershell
env:
PLATFORM: ${{ matrix.PLATFORM }}
run: |
$Env:Path = "$Env:CONDA_PREFIX\NSIS;$Env:Path"
echo $Env:Path
python build_installer.py -v
- name: Copy spec and list built installers
- name: Copy lock file and list built installers and packages
shell: bash
env:
PLATFORM: ${{ matrix.PLATFORM }}
run: |
cp installer_specs/$DISTNAME-$PLATFORM/$DISTNAME-$PLATFORM.txt dist/
ls -lh dist
cp installer_specs/$DISTNAME-$PLATFORM.txt dist/
ls -lhR dist
- name: Test installer (sh)
if: contains(matrix.OS_NAME, 'Linux') || contains(matrix.OS_NAME, 'MacOSX')
@ -86,15 +89,30 @@ jobs:
- name: Test installer (pkg)
if: contains(matrix.OS_NAME, 'MacOSX')
continue-on-error: true # this test doesn't work yet
shell: bash
env:
OS_NAME: ${{ matrix.OS_NAME }}
ARCH: ${{ matrix.ARCH }}
TARGET_VOLUME: CurrentUserHomeDirectory
INSTALL_PATH: ~/Applications/${{ env.DISTNAME }}
INSTALL_PATH: ${{ github.workspace }}/../../../${{ env.DISTNAME }}
run: |
installer -verbose -pkg dist/$DISTNAME-*-$OS_NAME-$ARCH.pkg -target $TARGET_VOLUME
cat >pkg-choices.xml <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<array>
<dict>
<key>attributeSetting</key>
<integer>0</integer>
<key>choiceAttribute</key>
<string>selected</string>
<key>choiceIdentifier</key>
<string>io.continuum.pkg.pathupdate</string>
</dict>
</array>
</plist>
EOF
installer -verbose -dumplog -applyChoiceChangesXML pkg-choices.xml -pkg dist/$DISTNAME-*-$OS_NAME-$ARCH.pkg -target $TARGET_VOLUME
eval "$($INSTALL_PATH/bin/conda shell.bash hook)"
conda info
conda list
@ -145,3 +163,12 @@ jobs:
tag: ${{ github.ref }}
overwrite: true
file_glob: true
- name: Upload metapackage to Anaconda.org
if: startsWith(github.ref, 'refs/tags/')
shell: bash -l {0}
env:
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
run: |
micromamba --help
anaconda upload -l $METAPACKAGE_LABEL --skip-existing dist/conda-bld/**/*

24
README.md
View File

@ -94,7 +94,19 @@ Once you have radioconda installed, you can stay up to date for all packages wit
### Upgrade to latest release
To install the latest release in particular, run (on Windows):
To install the latest release in particular, run
mamba upgrade -c ryanvolz radioconda
### Install a particular release
To install a particular release version, substitute the desired version number and run
mamba install -c ryanvolz radioconda=20NN.NN.NN
### Install from environment file
You can also install from the released environment file (on Windows):
mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-win-64.txt
@ -102,16 +114,6 @@ To install the latest release in particular, run (on Windows):
mamba install --file https://github.com/ryanvolz/radioconda/releases/latest/download/radioconda-$(conda info | sed -n -e 's/^.*platform : //p').txt
### Install a particular release
To install the package versions associated with a particular release, substitute the release number and run the following (on Windows):
mamba install --file https://github.com/ryanvolz/radioconda/releases/download/20NN.NN.NN/radioconda-win-64.txt
(on Linux/macOS):
mamba install --file https://github.com/ryanvolz/radioconda/releases/download/20NN.NN.NN/radioconda-$(conda info | sed -n -e 's/^.*platform : //p').txt
## Additional Installation for Device Support
To use particular software radio devices, it might be necessary to install additional drivers or firmware. Find your device below and follow the instructions. (Help add to this section by filing an issue if the instructions don't work or you have additional instructions to add!)

137
build_metapackage.py Executable file
View File

@ -0,0 +1,137 @@
#!/usr/bin/env python3
import pathlib
import re
comment_re = re.compile(r"^\s*#\s*(?P<comment>.*)\s*$")
key_value_re = re.compile(r"^(?P<key>.*):\s*(?P<value>.*)\s*$")
def read_lock_file(lock_file: pathlib.Path) -> dict:
with lock_file.open("r") as f:
lines = f.read().splitlines()
lock_dict = dict(specs=[])
for line in lines:
comment_match = comment_re.match(line)
if comment_match:
m = key_value_re.match(comment_match.group("comment"))
if m:
lock_dict[m.group("key")] = m.group("value")
else:
lock_dict["specs"].append(line)
return lock_dict
if __name__ == "__main__":
import argparse
import os
import subprocess
import shutil
import sys
import conda_build.config
conda_build_config = conda_build.config.Config()
cwd = pathlib.Path(".").absolute()
here = pathlib.Path(__file__).parent.absolute().relative_to(cwd)
distname = os.getenv("DISTNAME", "radioconda")
platform = os.getenv("PLATFORM", conda_build_config.subdir)
source = "/".join(
(
os.getenv("GITHUB_SERVER_URL", "https://github.com"),
os.getenv("GITHUB_REPOSITORY", "ryanvolz/radioconda"),
)
)
license_id = os.getenv("LICENSE_ID", "BSD-3-Clause")
summary = os.getenv("METAPACKAGE_SUMMARY", f"Metapackage for {distname}.")
parser = argparse.ArgumentParser(
description=(
"Build environment metapackage using conda-build."
" Additional command-line options will be passed to conda metapackage."
)
)
parser.add_argument(
"lock_file",
type=pathlib.Path,
nargs="?",
default=here / "installer_specs" / f"{distname}-{platform}.txt",
help=(
"Environment lock file for a particular platform"
" (name ends in the platform identifier)."
" (default: %(default)s)"
),
)
parser.add_argument(
"-o",
"--output_dir",
type=pathlib.Path,
default=here / "dist" / "conda-bld",
help=(
"Output directory in which the metapackage will be placed."
" (default: %(default)s)"
),
)
parser.add_argument(
"--home",
default=source,
help="The homepage for the metapackage. (default: %(default)s)",
)
parser.add_argument(
"--license",
default=license_id,
help="The SPDX license identifier for the metapackage. (default: %(default)s)",
)
parser.add_argument(
"--summary",
default=summary,
help="Summary of the package. (default: %(default)s)",
)
args, metapackage_args = parser.parse_known_args()
lock_dict = read_lock_file(args.lock_file)
name = lock_dict.get("name", distname)
version = lock_dict.get("version", "0")
platform = lock_dict.get("platform", platform)
env = os.environ.copy()
env["CONDA_SUBDIR"] = platform
channels = [c.strip() for c in lock_dict.get("channels", "conda-forge").split(",")]
conda_metapackage_cmdline = [
"conda",
"metapackage",
name,
version,
"--no-anaconda-upload",
"--home",
args.home,
"--license",
args.license,
"--summary",
args.summary,
]
for channel in channels:
conda_metapackage_cmdline.extend(["--channel", channel])
conda_metapackage_cmdline.extend(["--dependencies"] + lock_dict["specs"])
conda_metapackage_cmdline.extend(metapackage_args)
proc = subprocess.run(conda_metapackage_cmdline, env=env)
try:
proc.check_returncode()
except subprocess.CalledProcessError:
sys.exit(1)
bldpkgs_dir = pathlib.Path(conda_build_config.bldpkgs_dir)
pkg_paths = list(bldpkgs_dir.glob(f"{name}-{version}*.bz2"))
pkg_out_dir = args.output_dir / platform
pkg_out_dir.mkdir(parents=True, exist_ok=True)
for pkg in pkg_paths:
shutil.copy(pkg, pkg_out_dir)

2
buildenv.yaml
View File

@ -2,4 +2,6 @@ name: buildenv
channels:
- conda-forge
dependencies:
- anaconda-client
- conda-build
- constructor

12
radioconda.yaml
View File

@ -7,19 +7,13 @@ platforms:
- osx-64
- win-64
dependencies:
- mamba
- ipython
- python
# restrict to python 3.8 on Windows for Windows 7 compatibility
- python 3.8.* # [win]
- radioconda_console_shortcut # [win]
- digital_rf
- gnuradio 3.9.*
- gnuradio-osmosdr
- gnuradio-satellites
- gnuradio-soapy
- gqrx
- ipython
- libiio
- libm2k
- limesuite
@ -27,6 +21,10 @@ dependencies:
- numpy
- pandas
- pyadi-iio
- python
# restrict to python 3.8 on Windows for Windows 7 compatibility
- python 3.8.* # [win]
- radioconda_console_shortcut # [win]
- rtl-sdr
- scipy
- soapysdr

241
rerender.py
View File

@ -1,12 +1,17 @@
#!/usr/bin/env python3
import pathlib
import shutil
from typing import List, Optional
import conda_lock
import yaml
def render_lock_spec(
def name_from_pkg_spec(spec: str):
return spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0]
def lock_env_spec(
lock_spec: conda_lock.src_parser.LockSpecification, conda_exe: str
) -> conda_lock.src_parser.LockSpecification:
create_env_dict = conda_lock.conda_lock.solve_specs_for_arch(
@ -16,36 +21,104 @@ def render_lock_spec(
platform=lock_spec.platform,
)
pkgs = create_env_dict["actions"]["LINK"]
spec_names = set(
spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0]
for spec in lock_spec.specs
)
locked_specs = ["{name}={version}={build_string}".format(**pkg) for pkg in pkgs]
rendered_specs = []
rendered_dep_specs = []
for pkg in pkgs:
pkg_spec = "{name}={version}={build_string}".format(**pkg)
if pkg["name"] in spec_names:
rendered_specs.append(pkg_spec)
else:
rendered_dep_specs.append(pkg_spec)
rendered_lock_spec = conda_lock.src_parser.LockSpecification(
specs=sorted(rendered_specs),
channels=lock_spec.channels,
platform=lock_spec.platform,
)
rendered_full_lock_spec = conda_lock.src_parser.LockSpecification(
specs=sorted(rendered_specs + rendered_dep_specs),
locked_env_spec = conda_lock.src_parser.LockSpecification(
specs=sorted(locked_specs),
channels=lock_spec.channels,
platform=lock_spec.platform,
)
return rendered_lock_spec, rendered_full_lock_spec
return locked_env_spec
def render_constructor_specs(
def write_lock_file(
lock_spec: conda_lock.src_parser.LockSpecification,
lock_file_path: pathlib.Path,
name: Optional[str] = None,
version: Optional[str] = None,
channels: Optional[List[str]] = None,
):
lockfile_contents = [
f"# platform: {lock_spec.platform}",
f"# env_hash: {lock_spec.env_hash()}",
]
if name:
lockfile_contents.append(f"# name: {name}")
if version:
lockfile_contents.append(f"# version: {version}")
if channels:
lockfile_contents.append(f"# channels: {','.join(channels)}")
lockfile_contents.extend(lock_spec.specs)
with lock_file_path.open("w") as f:
f.write("\n".join(lockfile_contents))
def render_constructor(
lock_spec: conda_lock.src_parser.LockSpecification,
name: str,
version: str,
company: str,
license_file: pathlib.Path,
output_dir: pathlib.Path,
) -> dict:
platform = lock_spec.platform
constructor_name = f"{name}-{platform}"
construct_dict = dict(
name=name,
version=version,
company=company,
channels=lock_spec.channels,
specs=lock_spec.specs,
initialize_by_default=True,
installer_type="all",
keep_pkgs=True,
license_file="LICENSE",
register_python_default=False,
write_condarc=True,
)
if platform.startswith("win"):
construct_dict["post_install"] = "post_install.bat"
else:
construct_dict["post_install"] = "post_install.sh"
constructor_dir = output_dir / constructor_name
if constructor_dir.exists():
shutil.rmtree(constructor_dir)
constructor_dir.mkdir(parents=True)
# copy license to the constructor directory
shutil.copy(license_file, constructor_dir / "LICENSE")
# write the post_install scripts referenced in the construct dict
if platform.startswith("win"):
with (constructor_dir / "post_install.bat").open("w") as f:
f.write("\n".join((r"del /q %PREFIX%\pkgs\*.tar.bz2", "exit 0", "")))
else:
with (constructor_dir / "post_install.sh").open("w") as f:
f.write(
"\n".join(
(
"#!/bin/sh",
f'PREFIX="${{PREFIX:-$2/{name}}}"',
r"rm -f $PREFIX/pkgs/*.tar.bz2",
"exit 0",
"",
)
)
)
construct_yaml_path = constructor_dir / "construct.yaml"
with construct_yaml_path.open("w") as f:
yaml.safe_dump(construct_dict, stream=f)
return construct_dict
def render_platforms(
environment_file: pathlib.Path,
installer_pkg_specs: List[str],
version: str,
company: str,
license_file: pathlib.Path,
@ -55,75 +128,83 @@ def render_constructor_specs(
with environment_file.open("r") as f:
env_yaml_data = yaml.safe_load(f)
installer_name = env_yaml_data["name"]
env_name = env_yaml_data["name"]
platforms = env_yaml_data["platforms"]
if not license_file.exists():
raise ValueError(f"Cannot find license file: {license_file}")
output_dir.mkdir(parents=True, exist_ok=True)
if output_dir.exists():
shutil.rmtree(output_dir)
output_dir.mkdir(parents=True)
constructor_specs = {}
rendered_platforms = {}
for platform in platforms:
constructor_name = f"{installer_name}-{platform}"
output_name = f"{env_name}-{platform}"
lock_spec = conda_lock.conda_lock.parse_environment_file(
# get the environment specification for the list of packages from the env file
env_spec = conda_lock.conda_lock.parse_environment_file(
environment_file=environment_file, platform=platform
)
rendered_lock_spec, rendered_full_lock_spec = render_lock_spec(
lock_spec, conda_exe
# lock the full environment specification to specific versions and builds
locked_env_spec = lock_env_spec(env_spec, conda_exe)
# write the full environment specification to a lock file
lock_file_path = output_dir / f"{output_name}.txt"
write_lock_file(
locked_env_spec,
lock_file_path,
name=env_name,
version=version,
channels=locked_env_spec.channels,
)
construct_dict = dict(
name=installer_name,
# add installer-only (base environment) packages and lock those too
installer_spec = conda_lock.src_parser.LockSpecification(
specs=sorted(locked_env_spec.specs + installer_pkg_specs),
channels=locked_env_spec.channels,
platform=locked_env_spec.platform,
)
locked_installer_spec = lock_env_spec(installer_spec, conda_exe)
# get a set of only the packages to put in the constructor specification
# taken from the installer-only list and those explicitly selected originally
constructor_pkg_names = set(
name_from_pkg_spec(spec) for spec in env_spec.specs + installer_pkg_specs
)
# filter the installer spec by the constructor package names
constructor_pkg_specs = [
spec
for spec in locked_installer_spec.specs
if name_from_pkg_spec(spec) in constructor_pkg_names
]
constructor_spec = conda_lock.src_parser.LockSpecification(
specs=constructor_pkg_specs,
channels=locked_installer_spec.channels,
platform=locked_installer_spec.platform,
)
# create the rendered constructor directory
constructor_dict = render_constructor(
lock_spec=constructor_spec,
name=env_name,
version=version,
company=company,
channels=rendered_lock_spec.channels,
specs=rendered_lock_spec.specs,
initialize_by_default=True,
installer_type="all",
keep_pkgs=True,
license_file="LICENSE",
register_python_default=False,
write_condarc=True,
license_file=license_file,
output_dir=output_dir,
)
if platform.startswith("win"):
construct_dict["post_install"] = "post_install.bat"
else:
construct_dict["post_install"] = "post_install.sh"
constructor_specs[constructor_name] = construct_dict
# aggregate output
rendered_platforms[output_name] = dict(
locked_env_spec=locked_env_spec,
locked_installer_spec=locked_installer_spec,
constructor_dict=constructor_dict,
)
constructor_dir = output_dir / constructor_name
if constructor_dir.exists():
shutil.rmtree(constructor_dir)
constructor_dir.mkdir(parents=True)
# copy license to the constructor directory
shutil.copy(license_file, constructor_dir / "LICENSE")
# write the post_install scripts referenced in the construct dict
if platform.startswith("win"):
with (constructor_dir / "post_install.bat").open("w") as f:
f.writelines((r"del /q %PREFIX%\pkgs\*.tar.bz2", ""))
else:
with (constructor_dir / "post_install.sh").open("w") as f:
f.writelines((r"rm -f $PREFIX/pkgs/*.tar.bz2", ""))
construct_yaml_path = constructor_dir / "construct.yaml"
with construct_yaml_path.open("w") as f:
yaml.safe_dump(construct_dict, stream=f)
lockfile_contents = [
f"# platform: {rendered_full_lock_spec.platform}",
f"# env_hash: {rendered_full_lock_spec.env_hash()}",
]
lockfile_contents.extend(rendered_full_lock_spec.specs)
lock_file_path = constructor_dir / f"{constructor_name}.txt"
with lock_file_path.open("w") as f:
f.writelines(s + "\n" for s in lockfile_contents)
return constructor_specs
return rendered_platforms
if __name__ == "__main__":
@ -134,7 +215,12 @@ if __name__ == "__main__":
cwd = pathlib.Path(".").absolute()
here = pathlib.Path(__file__).parent.absolute().relative_to(cwd)
distname = os.getenv("DISTNAME", "radioconda")
source = os.getenv("SOURCE", "github.com/ryanvolz/radioconda")
source = "/".join(
(
os.getenv("GITHUB_SERVER_URL", "https://github.com"),
os.getenv("GITHUB_REPOSITORY", "ryanvolz/radioconda"),
)
)
dt = datetime.datetime.now()
version = dt.strftime("%Y.%m.%d")
@ -211,8 +297,9 @@ if __name__ == "__main__":
conda_executable=args.conda_exe, mamba=True, micromamba=True
)
constructor_specs = render_constructor_specs(
constructor_specs = render_platforms(
environment_file=args.environment_file,
installer_pkg_specs=["mamba"],
version=args.version,
company=args.company,
license_file=args.license_file,