From cd62db95c13e54a46577c0d8aa63864342a7bce4 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 8 Dec 2021 17:03:12 -0500 Subject: [PATCH] Added new feature to list-plugins command This patch updates the ouput of the list-plugins command. This also adds the ability to show the available plugins to install that are published packages on pypi.org. This also shows the list of installed packages from pypi.org --- ChangeLog | 1 + aprsd/cmds/completion.py | 2 +- aprsd/cmds/dev.py | 3 +- aprsd/cmds/healthcheck.py | 3 +- aprsd/cmds/list_plugins.py | 201 ++++++++++++++++++++++++++++++++++--- aprsd/cmds/listen.py | 3 +- aprsd/cmds/send_message.py | 3 +- aprsd/cmds/server.py | 3 +- requirements.in | 2 + requirements.txt | 4 + 10 files changed, 200 insertions(+), 25 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0401ce1..e45760c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,7 @@ CHANGES v2.5.6 ------ +* Changelog * Tightened up the packet logging * Added unit tests for USWeatherPlugin, USMetarPlugin * Added test\_location to test LocationPlugin diff --git a/aprsd/cmds/completion.py b/aprsd/cmds/completion.py index 274443b..1356d28 100644 --- a/aprsd/cmds/completion.py +++ b/aprsd/cmds/completion.py @@ -1,7 +1,7 @@ import click import click_completion -from ..aprsd import cli +from aprsd.aprsd import cli CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) diff --git a/aprsd/cmds/dev.py b/aprsd/cmds/dev.py index a6d67dd..e9313b8 100644 --- a/aprsd/cmds/dev.py +++ b/aprsd/cmds/dev.py @@ -9,8 +9,7 @@ import click # local imports here from aprsd import cli_helper, client, messaging, packets, plugin, stats, trace - -from ..aprsd import cli +from aprsd.aprsd import cli LOG = logging.getLogger("APRSD") diff --git a/aprsd/cmds/healthcheck.py b/aprsd/cmds/healthcheck.py index 8b57ace..722e549 100644 --- a/aprsd/cmds/healthcheck.py +++ b/aprsd/cmds/healthcheck.py @@ -14,9 +14,8 @@ import requests import aprsd from aprsd import cli_helper, utils - # local imports here -from ..aprsd import cli +from aprsd.aprsd import cli # setup the global logger diff --git a/aprsd/cmds/list_plugins.py b/aprsd/cmds/list_plugins.py index 94d51e9..78b3e95 100644 --- a/aprsd/cmds/list_plugins.py +++ b/aprsd/cmds/list_plugins.py @@ -1,21 +1,183 @@ +import fnmatch +import importlib import inspect import logging -from textwrap import indent +import os +import pkgutil +import re +import sys +from traceback import print_tb +from urllib.parse import urljoin +from bs4 import BeautifulSoup import click -from tabulate import tabulate +import requests +from rich.console import Console +from rich.table import Table +from rich.text import Text +from thesmuggler import smuggle -from aprsd import cli_helper, plugin +from aprsd import cli_helper +from aprsd import plugin as aprsd_plugin +from aprsd.aprsd import cli from aprsd.plugins import ( email, fortune, location, notify, ping, query, time, version, weather, ) -from ..aprsd import cli - LOG = logging.getLogger("APRSD") +def onerror(name): + print(f"Error importing module {name}") + type, value, traceback = sys.exc_info() + print_tb(traceback) + + +def is_plugin(obj): + for c in inspect.getmro(obj): + if issubclass(c, aprsd_plugin.APRSDPluginBase): + return True + + return False + + +def plugin_type(obj): + for c in inspect.getmro(obj): + if issubclass(c, aprsd_plugin.APRSDRegexCommandPluginBase): + return "RegexCommand" + if issubclass(c, aprsd_plugin.APRSDWatchListPluginBase): + return "WatchList" + if issubclass(c, aprsd_plugin.APRSDPluginBase): + return "APRSDPluginBase" + + return "Unknown" + + +def walk_package(package): + return pkgutil.walk_packages( + package.__path__, + package.__name__ + ".", + onerror=onerror, + ) + + +def get_module_info(package_name, module_name, module_path): + if not os.path.exists(module_path): + return None + + dir_path = os.path.realpath(module_path) + pattern = "*.py" + + obj_list = [] + + for path, _subdirs, files in os.walk(dir_path): + for name in files: + if fnmatch.fnmatch(name, pattern): + module = smuggle(f"{path}/{name}") + for mem_name, obj in inspect.getmembers(module): + if inspect.isclass(obj) and is_plugin(obj): + obj_list.append( + { + "package": package_name, + "name": mem_name, "obj": obj, + "version": obj.version, + "path": f"{'.'.join([module_name, obj.__name__])}", + }, + ) + + return obj_list + + +def get_installed_plugins(): + # installed plugins + ip = {} + for finder, name, ispkg in pkgutil.iter_modules(): + if name.startswith("aprsd_"): + if ispkg: + module = importlib.import_module(name) + pkgs = walk_package(module) + for pkg in pkgs: + pkg_info = get_module_info(module.__name__, pkg.name, module.__path__[0]) + ip[name] = pkg_info + return ip + + +def show_pypi_plugins(installed_plugins, console): + query = "aprsd" + api_url = "https://pypi.org/search/" + snippets = [] + s = requests.Session() + for page in range(1, 3): + params = {"q": query, "page": page} + r = s.get(api_url, params=params) + soup = BeautifulSoup(r.text, "html.parser") + snippets += soup.select('a[class*="snippet"]') + if not hasattr(s, "start_url"): + s.start_url = r.url.rsplit("&page", maxsplit=1).pop(0) + + title = Text.assemble( + ("Pypi.org APRSD Installable Plugin Packages\n\n", "bold magenta"), + ("Install any of the following plugins with ", "bold yellow"), + ("pip install ", "bold white"), + ("", "cyan"), + ) + + table = Table(title=title) + table.add_column("Plugin Package Name", style="cyan", no_wrap=True) + table.add_column("Description", style="yellow") + table.add_column("Version", style="yellow") + table.add_column("Released", style="bold green") + table.add_column("Installed?", style="red") + table.add_column("Version Installed", style="yellow") + for snippet in snippets: + link = urljoin(api_url, snippet.get("href")) + package = re.sub(r"\s+", " ", snippet.select_one('span[class*="name"]').text.strip()) + version = re.sub(r"\s+", " ", snippet.select_one('span[class*="version"]').text.strip()) + released = re.sub(r"\s+", " ", snippet.select_one('span[class*="released"]').text.strip()) + description = re.sub(r"\s+", " ", snippet.select_one('p[class*="description"]').text.strip()) + emoji = ":open_file_folder:" + if "aprsd-" not in package or "-plugin" not in package: + continue + under = package.replace("-", "_") + if under in installed_plugins: + installed = "Yes" + try: + version_installed = installed_plugins[under].__version__ + except Exception: + version_installed = "Unknown" + else: + installed = "No" + version_installed = "Unknown" + table.add_row( + f"[link={link}]{emoji}[/link] {package}", + description, version, released, installed, version_installed, + ) + + console.print("\n") + console.print(table) + return + + +def show_installed_plugins(installed_plugins, console): + if not installed_plugins: + return + + table = Table( + title="[not italic]:snake:[/] [bold][magenta]APRSD Installed 3rd party Plugins [not italic]:snake:[/]", + ) + table.add_column("Package Name", style="white", no_wrap=True) + table.add_column("Plugin Name", style="cyan", no_wrap=True) + table.add_column("Type", style="bold green") + table.add_column("Plugin Path", style="bold blue") + for name in installed_plugins: + for plugin in installed_plugins[name]: + table.add_row(name.replace("_", "-"), plugin["name"], plugin_type(plugin["obj"]), plugin["path"]) + + console.print("\n") + console.print(table) + + @cli.command() @cli_helper.add_options(cli_helper.common_options) @click.pass_context @@ -30,7 +192,7 @@ def list_plugins(ctx): entries = inspect.getmembers(module, inspect.isclass) for entry in entries: cls = entry[1] - if issubclass(cls, plugin.APRSDPluginBase): + if issubclass(cls, aprsd_plugin.APRSDPluginBase): info = { "name": cls.__qualname__, "path": f"{cls.__module__}.{cls.__qualname__}", @@ -39,21 +201,32 @@ def list_plugins(ctx): "short_desc": cls.short_description, } - if issubclass(cls, plugin.APRSDRegexCommandPluginBase): + if issubclass(cls, aprsd_plugin.APRSDRegexCommandPluginBase): info["command_regex"] = cls.command_regex info["type"] = "RegexCommand" - if issubclass(cls, plugin.APRSDWatchListPluginBase): + if issubclass(cls, aprsd_plugin.APRSDWatchListPluginBase): info["type"] = "WatchList" plugins.append(info) - lines = [] - headers = ("Plugin Name", "Plugin Path", "Type", "Info") + plugins = sorted(plugins, key=lambda i: i["name"]) + table = Table( + title="[not italic]:snake:[/] [bold][magenta]APRSD Built-in Plugins [not italic]:snake:[/]", + ) + table.add_column("Plugin Name", style="cyan", no_wrap=True) + table.add_column("Info", style="bold yellow") + table.add_column("Type", style="bold green") + table.add_column("Plugin Path", style="bold blue") for entry in plugins: - lines.append( - (entry["name"], entry["path"], entry["type"], entry["short_desc"]), - ) + table.add_row(entry["name"], entry["short_desc"], entry["type"], entry["path"]) - click.echo(indent(tabulate(lines, headers, disable_numparse=True), " ")) + console = Console() + console.print(table) + + # now find any from pypi? + installed_plugins = get_installed_plugins() + show_pypi_plugins(installed_plugins, console) + + show_installed_plugins(installed_plugins, console) diff --git a/aprsd/cmds/listen.py b/aprsd/cmds/listen.py index 49d424c..7e7d8cd 100644 --- a/aprsd/cmds/listen.py +++ b/aprsd/cmds/listen.py @@ -16,8 +16,7 @@ import aprsd from aprsd import ( cli_helper, client, messaging, packets, stats, threads, trace, utils, ) - -from ..aprsd import cli +from aprsd.aprsd import cli # setup the global logger diff --git a/aprsd/cmds/send_message.py b/aprsd/cmds/send_message.py index d905b24..b2125a0 100644 --- a/aprsd/cmds/send_message.py +++ b/aprsd/cmds/send_message.py @@ -8,8 +8,7 @@ import click import aprsd from aprsd import cli_helper, client, messaging, packets - -from ..aprsd import cli +from aprsd.aprsd import cli LOG = logging.getLogger("APRSD") diff --git a/aprsd/cmds/server.py b/aprsd/cmds/server.py index 48014f9..f271f86 100644 --- a/aprsd/cmds/server.py +++ b/aprsd/cmds/server.py @@ -10,8 +10,7 @@ from aprsd import ( trace, utils, ) from aprsd import aprsd as aprsd_main - -from ..aprsd import cli +from aprsd.aprsd import cli LOG = logging.getLogger("APRSD") diff --git a/requirements.in b/requirements.in index 273d502..9b524b6 100644 --- a/requirements.in +++ b/requirements.in @@ -22,3 +22,5 @@ flask-socketio eventlet tabulate rich +# For the list-plugins pypi.org search scraping +beautifulsoup4 diff --git a/requirements.txt b/requirements.txt index e23f556..79ad4ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,8 @@ aprslib==0.7.0 # via -r requirements.in backoff==1.11.1 # via opencage +beautifulsoup4==4.10.0 + # via -r requirements.in bidict==0.21.2 # via python-socketio certifi==2021.5.30 @@ -112,6 +114,8 @@ six==1.16.0 # imapclient # pyopenssl # signalslot +soupsieve==2.3.1 + # via beautifulsoup4 tabulate==0.8.9 # via -r requirements.in thesmuggler==1.0.1