diff --git a/aprsd/cmds/healthcheck.py b/aprsd/cmds/healthcheck.py index feec6d0..8820fc7 100644 --- a/aprsd/cmds/healthcheck.py +++ b/aprsd/cmds/healthcheck.py @@ -13,9 +13,9 @@ from oslo_config import cfg from rich.console import Console import aprsd -from aprsd import ( +from aprsd import ( # noqa: F401 cli_helper, - conf, # noqa + conf, ) # local imports here diff --git a/aprsd/cmds/list_plugins.py b/aprsd/cmds/list_plugins.py index 82507c9..75b7256 100644 --- a/aprsd/cmds/list_plugins.py +++ b/aprsd/cmds/list_plugins.py @@ -4,10 +4,8 @@ import inspect import logging import os import pkgutil -import re import sys from traceback import print_tb -from urllib.parse import urljoin import click import requests @@ -162,13 +160,15 @@ def show_built_in_plugins(console): console.print(table) -def _get_pypi_packages(): +def _get_pypi_packages_OLD(): query = "aprsd" snippets = [] s = requests.Session() + c = Console() for page in range(1, 3): params = {"q": query, "page": page} r = s.get(PYPI_URL, params=params) + c.print(f"r = {r.text}") soup = BeautifulSoup(r.text, "html.parser") snippets += soup.select('a[class*="snippet"]') if not hasattr(s, "start_url"): @@ -177,8 +177,34 @@ def _get_pypi_packages(): return snippets +def _get_pypi_packages(): + if simple_r := requests.get( + "https://pypi.org/simple", + headers={"Accept": "application/vnd.pypi.simple.v1+json"}, + ): + simple_response = simple_r.json() + else: + simple_response = {} + + key = "aprsd" + matches = [ + p["name"] for p in simple_response["projects"] if p["name"].startswith(key) + ] + + packages = [] + for pkg in matches: + # Get info for first match + if r := requests.get( + f"https://pypi.org/pypi/{pkg}/json", + headers={"Accept": "application/json"}, + ): + packages.append(r.json()) + + return packages + + def show_pypi_plugins(installed_plugins, console): - snippets = _get_pypi_packages() + packages = _get_pypi_packages() title = Text.assemble( ("Pypi.org APRSD Installable Plugin Packages\n\n", "bold magenta"), @@ -193,33 +219,21 @@ def show_pypi_plugins(installed_plugins, console): table.add_column("Version", style="yellow", justify="center") table.add_column("Released", style="bold green", justify="center") table.add_column("Installed?", style="red", justify="center") - for snippet in snippets: - link = urljoin(PYPI_URL, snippet.get("href")) - package = re.sub( - r"\s+", " ", snippet.select_one('span[class*="name"]').text.strip() - ) - version = re.sub( - r"\s+", " ", snippet.select_one('span[class*="version"]').text.strip() - ) - created = re.sub( - r"\s+", " ", snippet.select_one('span[class*="created"]').text.strip() - ) - description = re.sub( - r"\s+", " ", snippet.select_one('p[class*="description"]').text.strip() - ) - emoji = ":open_file_folder:" + emoji = ":open_file_folder:" + for package in packages: + link = package["info"]["package_url"] + version = package["info"]["version"] + package_name = package["info"]["name"] + description = package["info"]["summary"] + created = package["releases"][version][0]["upload_time"] - if "aprsd-" not in package or "-plugin" not in package: + if "aprsd-" not in package_name or "-plugin" not in package_name: continue - under = package.replace("-", "_") - if under in installed_plugins: - installed = "Yes" - else: - installed = "No" - + under = package_name.replace("-", "_") + installed = "Yes" if under in installed_plugins else "No" table.add_row( - f"[link={link}]{emoji}[/link] {package}", + f"[link={link}]{emoji}[/link] {package_name}", description, version, created, @@ -231,7 +245,7 @@ def show_pypi_plugins(installed_plugins, console): def show_pypi_extensions(installed_extensions, console): - snippets = _get_pypi_packages() + packages = _get_pypi_packages() title = Text.assemble( ("Pypi.org APRSD Installable Extension Packages\n\n", "bold magenta"), @@ -245,33 +259,21 @@ def show_pypi_extensions(installed_extensions, console): table.add_column("Version", style="yellow", justify="center") table.add_column("Released", style="bold green", justify="center") table.add_column("Installed?", style="red", justify="center") - for snippet in snippets: - link = urljoin(PYPI_URL, snippet.get("href")) - package = re.sub( - r"\s+", " ", snippet.select_one('span[class*="name"]').text.strip() - ) - version = re.sub( - r"\s+", " ", snippet.select_one('span[class*="version"]').text.strip() - ) - created = re.sub( - r"\s+", " ", snippet.select_one('span[class*="created"]').text.strip() - ) - description = re.sub( - r"\s+", " ", snippet.select_one('p[class*="description"]').text.strip() - ) - emoji = ":open_file_folder:" + emoji = ":open_file_folder:" - if "aprsd-" not in package or "-extension" not in package: + for package in packages: + link = package["info"]["package_url"] + version = package["info"]["version"] + package_name = package["info"]["name"] + description = package["info"]["summary"] + created = package["releases"][version][0]["upload_time"] + if "aprsd-" not in package_name or "-extension" not in package_name: continue - under = package.replace("-", "_") - if under in installed_extensions: - installed = "Yes" - else: - installed = "No" - + under = package_name.replace("-", "_") + installed = "Yes" if under in installed_extensions else "No" table.add_row( - f"[link={link}]{emoji}[/link] {package}", + f"[link={link}]{emoji}[/link] {package_name}", description, version, created, @@ -337,5 +339,7 @@ def list_extensions(ctx): with console.status("Show APRSD Extensions") as status: status.update("Fetching pypi.org APRSD Extensions") + + status.update("Looking for installed APRSD Extensions") installed_extensions = get_installed_extensions() show_pypi_extensions(installed_extensions, console) diff --git a/requirements.in b/requirements.in index ca989ed..5c2cddf 100644 --- a/requirements.in +++ b/requirements.in @@ -1,6 +1,4 @@ aprslib>=0.7.0 -# For the list-plugins pypi.org search scraping -beautifulsoup4 click dataclasses-json haversine diff --git a/requirements.txt b/requirements.txt index cc2a453..4d4f2d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,8 +9,6 @@ attrs==24.3.0 # rush ax253==0.1.5.post1 # via kiss3 -beautifulsoup4==4.12.3 - # via -r requirements.in bitarray==3.0.0 # via # ax253 @@ -39,7 +37,7 @@ kiss3==8.0.0 # via -r requirements.in loguru==0.7.3 # via -r requirements.in -marshmallow==3.23.2 +marshmallow==3.23.3 # via dataclasses-json mypy-extensions==1.0.0 # via typing-inspect @@ -78,8 +76,6 @@ rich==12.6.0 # via -r requirements.in rush==2021.4.0 # via -r requirements.in -soupsieve==2.6 - # via beautifulsoup4 stevedore==5.4.0 # via oslo-config thesmuggler==1.0.1 diff --git a/tests/plugins/test_notify.py b/tests/plugins/test_notify.py index ed1c81d..4cb2638 100644 --- a/tests/plugins/test_notify.py +++ b/tests/plugins/test_notify.py @@ -2,9 +2,9 @@ from unittest import mock from oslo_config import cfg -from aprsd import ( +from aprsd import ( # noqa: F401 client, - conf, # noqa: F401 + conf, packets, ) from aprsd.plugins import notify as notify_plugin diff --git a/tests/test_plugin.py b/tests/test_plugin.py index def058f..636a245 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3,8 +3,8 @@ from unittest import mock from oslo_config import cfg -from aprsd import ( - conf, # noqa: F401 +from aprsd import ( # noqa: F401 + conf, packets, plugins, )