From 23e3876e7b5d4a4b68101dd824afb2ec8ade5281 Mon Sep 17 00:00:00 2001 From: Hemna Date: Thu, 16 Sep 2021 17:08:30 -0400 Subject: [PATCH 01/12] Refactor utils usage This patch separates out the config from the utils.py utils.py has grown into a catchall for everything and this patch is the start of that cleanup. --- aprsd/client.py | 5 - aprsd/config.py | 367 ++++++++++++++++++++++++++++++++++++++ aprsd/dev.py | 8 +- aprsd/flask.py | 12 +- aprsd/healthcheck.py | 6 +- aprsd/listen.py | 14 +- aprsd/main.py | 47 ++--- aprsd/messaging.py | 16 +- aprsd/plugins/location.py | 4 +- aprsd/plugins/time.py | 10 +- aprsd/plugins/weather.py | 18 +- aprsd/utils.py | 348 ------------------------------------ docker/Dockerfile | 2 +- docker/build.sh | 2 +- tests/test_plugin.py | 4 +- 15 files changed, 435 insertions(+), 428 deletions(-) create mode 100644 aprsd/config.py diff --git a/aprsd/client.py b/aprsd/client.py index dd1e248..3e8e198 100644 --- a/aprsd/client.py +++ b/aprsd/client.py @@ -38,11 +38,6 @@ class Client: if config: self.config = config - def new(self): - obj = super().__new__(Client) - obj.config = self.config - return obj - @property def client(self): if not self.aprs_client: diff --git a/aprsd/config.py b/aprsd/config.py new file mode 100644 index 0000000..1461c2f --- /dev/null +++ b/aprsd/config.py @@ -0,0 +1,367 @@ +import logging +import os +from pathlib import Path +import sys + +import click +import yaml + +from aprsd import utils + + +LOG_LEVELS = { + "CRITICAL": logging.CRITICAL, + "ERROR": logging.ERROR, + "WARNING": logging.WARNING, + "INFO": logging.INFO, + "DEBUG": logging.DEBUG, +} + +DEFAULT_DATE_FORMAT = "%m/%d/%Y %I:%M:%S %p" +DEFAULT_LOG_FORMAT = ( + "[%(asctime)s] [%(threadName)-20.20s] [%(levelname)-5.5s]" + " %(message)s - [%(pathname)s:%(lineno)d]" +) + +QUEUE_DATE_FORMAT = "[%m/%d/%Y] [%I:%M:%S %p]" +QUEUE_LOG_FORMAT = ( + "%(asctime)s [%(threadName)-20.20s] [%(levelname)-5.5s]" + " %(message)s - [%(pathname)s:%(lineno)d]" +) + +CORE_MESSAGE_PLUGINS = [ + "aprsd.plugins.email.EmailPlugin", + "aprsd.plugins.fortune.FortunePlugin", + "aprsd.plugins.location.LocationPlugin", + "aprsd.plugins.ping.PingPlugin", + "aprsd.plugins.query.QueryPlugin", + "aprsd.plugins.stock.StockPlugin", + "aprsd.plugins.time.TimePlugin", + "aprsd.plugins.weather.USWeatherPlugin", + "aprsd.plugins.version.VersionPlugin", +] + +CORE_NOTIFY_PLUGINS = [ + "aprsd.plugins.notify.NotifySeenPlugin", +] + +# an example of what should be in the ~/.aprsd/config.yml +DEFAULT_CONFIG_DICT = { + "ham": {"callsign": "NOCALL"}, + "aprs": { + "enabled": True, + "login": "CALLSIGN", + "password": "00000", + "host": "rotate.aprs2.net", + "port": 14580, + }, + "kiss": { + "tcp": { + "enabled": False, + "host": "direwolf.ip.address", + "port": "8001", + }, + "serial": { + "enabled": False, + "device": "/dev/ttyS0", + "baudrate": 9600, + }, + }, + "aprsd": { + "logfile": "/tmp/aprsd.log", + "logformat": DEFAULT_LOG_FORMAT, + "dateformat": DEFAULT_DATE_FORMAT, + "trace": False, + "enabled_plugins": CORE_MESSAGE_PLUGINS, + "units": "imperial", + "watch_list": { + "enabled": False, + # Who gets the alert? + "alert_callsign": "NOCALL", + # 43200 is 12 hours + "alert_time_seconds": 43200, + # How many packets to save in a ring Buffer + # for a particular callsign + "packet_keep_count": 10, + "callsigns": [], + "enabled_plugins": CORE_NOTIFY_PLUGINS, + }, + "web": { + "enabled": True, + "logging_enabled": True, + "host": "0.0.0.0", + "port": 8001, + "users": { + "admin": "password-here", + }, + }, + "email": { + "enabled": True, + "shortcuts": { + "aa": "5551239999@vtext.com", + "cl": "craiglamparter@somedomain.org", + "wb": "555309@vtext.com", + }, + "smtp": { + "login": "SMTP_USERNAME", + "password": "SMTP_PASSWORD", + "host": "smtp.gmail.com", + "port": 465, + "use_ssl": False, + "debug": False, + }, + "imap": { + "login": "IMAP_USERNAME", + "password": "IMAP_PASSWORD", + "host": "imap.gmail.com", + "port": 993, + "use_ssl": True, + "debug": False, + }, + }, + }, + "services": { + "aprs.fi": {"apiKey": "APIKEYVALUE"}, + "openweathermap": {"apiKey": "APIKEYVALUE"}, + "opencagedata": {"apiKey": "APIKEYVALUE"}, + "avwx": {"base_url": "http://host:port", "apiKey": "APIKEYVALUE"}, + }, +} + +home = str(Path.home()) +DEFAULT_CONFIG_DIR = f"{home}/.config/aprsd/" +DEFAULT_SAVE_FILE = f"{home}/.config/aprsd/aprsd.p" +DEFAULT_CONFIG_FILE = f"{home}/.config/aprsd/aprsd.yml" + + +def add_config_comments(raw_yaml): + end_idx = utils.end_substr(raw_yaml, "aprs:") + if end_idx != -1: + # lets insert a comment + raw_yaml = utils.insert_str( + raw_yaml, + "\n # Set enabled to False if there is no internet connectivity." + "\n # This is useful for a direwolf KISS aprs connection only. " + "\n" + "\n # Get the passcode for your callsign here: " + "\n # https://apps.magicbug.co.uk/passcode", + end_idx, + ) + + end_idx = utils.end_substr(raw_yaml, "aprs.fi:") + if end_idx != -1: + # lets insert a comment + raw_yaml = utils.insert_str( + raw_yaml, + "\n # Get the apiKey from your aprs.fi account here: " + "\n # http://aprs.fi/account", + end_idx, + ) + + end_idx = utils.end_substr(raw_yaml, "opencagedata:") + if end_idx != -1: + # lets insert a comment + raw_yaml = utils.insert_str( + raw_yaml, + "\n # (Optional for TimeOpenCageDataPlugin) " + "\n # Get the apiKey from your opencagedata account here: " + "\n # https://opencagedata.com/dashboard#api-keys", + end_idx, + ) + + end_idx = utils.end_substr(raw_yaml, "openweathermap:") + if end_idx != -1: + # lets insert a comment + raw_yaml = utils.insert_str( + raw_yaml, + "\n # (Optional for OWMWeatherPlugin) " + "\n # Get the apiKey from your " + "\n # openweathermap account here: " + "\n # https://home.openweathermap.org/api_keys", + end_idx, + ) + + end_idx = utils.end_substr(raw_yaml, "avwx:") + if end_idx != -1: + # lets insert a comment + raw_yaml = utils.insert_str( + raw_yaml, + "\n # (Optional for AVWXWeatherPlugin) " + "\n # Use hosted avwx-api here: https://avwx.rest " + "\n # or deploy your own from here: " + "\n # https://github.com/avwx-rest/avwx-api", + end_idx, + ) + + return raw_yaml + + +def dump_default_cfg(): + return add_config_comments( + yaml.dump( + DEFAULT_CONFIG_DICT, + indent=4, + ), + ) + + +def create_default_config(): + """Create a default config file.""" + # make sure the directory location exists + config_file_expanded = os.path.expanduser(DEFAULT_CONFIG_FILE) + config_dir = os.path.dirname(config_file_expanded) + if not os.path.exists(config_dir): + click.echo(f"Config dir '{config_dir}' doesn't exist, creating.") + utils.mkdir_p(config_dir) + with open(config_file_expanded, "w+") as cf: + cf.write(dump_default_cfg()) + + +def get_config(config_file): + """This tries to read the yaml config from .""" + config_file_expanded = os.path.expanduser(config_file) + if os.path.exists(config_file_expanded): + with open(config_file_expanded) as stream: + config = yaml.load(stream, Loader=yaml.FullLoader) + return config + else: + if config_file == DEFAULT_CONFIG_FILE: + click.echo( + f"{config_file_expanded} is missing, creating config file", + ) + create_default_config() + msg = ( + "Default config file created at {}. Please edit with your " + "settings.".format(config_file) + ) + click.echo(msg) + else: + # The user provided a config file path different from the + # Default, so we won't try and create it, just bitch and bail. + msg = f"Custom config file '{config_file}' is missing." + click.echo(msg) + + sys.exit(-1) + + +# This method tries to parse the config yaml file +# and consume the settings. +# If the required params don't exist, +# it will look in the environment +def parse_config(config_file): + # for now we still use globals....ugh + global CONFIG + + def fail(msg): + click.echo(msg) + sys.exit(-1) + + def check_option(config, chain, default_fail=None): + try: + config = check_config_option(config, chain, default_fail=default_fail) + except Exception as ex: + fail(repr(ex)) + else: + return config + + config = get_config(config_file) + + # special check here to make sure user has edited the config file + # and changed the ham callsign + check_option( + config, + [ + "ham", + "callsign", + ], + default_fail=DEFAULT_CONFIG_DICT["ham"]["callsign"], + ) + check_option( + config, + ["services", "aprs.fi", "apiKey"], + default_fail=DEFAULT_CONFIG_DICT["services"]["aprs.fi"]["apiKey"], + ) + check_option( + config, + ["aprs", "login"], + default_fail=DEFAULT_CONFIG_DICT["aprs"]["login"], + ) + check_option( + config, + ["aprs", "password"], + default_fail=DEFAULT_CONFIG_DICT["aprs"]["password"], + ) + + # Ensure they change the admin password + if config["aprsd"]["web"]["enabled"] is True: + check_option( + config, + ["aprsd", "web", "users", "admin"], + default_fail=DEFAULT_CONFIG_DICT["aprsd"]["web"]["users"]["admin"], + ) + + if config["aprsd"]["watch_list"]["enabled"] is True: + check_option( + config, + ["aprsd", "watch_list", "alert_callsign"], + default_fail=DEFAULT_CONFIG_DICT["aprsd"]["watch_list"]["alert_callsign"], + ) + + if config["aprsd"]["email"]["enabled"] is True: + # Check IMAP server settings + check_option(config, ["aprsd", "email", "imap", "host"]) + check_option(config, ["aprsd", "email", "imap", "port"]) + check_option( + config, + ["aprsd", "email", "imap", "login"], + default_fail=DEFAULT_CONFIG_DICT["aprsd"]["email"]["imap"]["login"], + ) + check_option( + config, + ["aprsd", "email", "imap", "password"], + default_fail=DEFAULT_CONFIG_DICT["aprsd"]["email"]["imap"]["password"], + ) + + # Check SMTP server settings + check_option(config, ["aprsd", "email", "smtp", "host"]) + check_option(config, ["aprsd", "email", "smtp", "port"]) + check_option( + config, + ["aprsd", "email", "smtp", "login"], + default_fail=DEFAULT_CONFIG_DICT["aprsd"]["email"]["smtp"]["login"], + ) + check_option( + config, + ["aprsd", "email", "smtp", "password"], + default_fail=DEFAULT_CONFIG_DICT["aprsd"]["email"]["smtp"]["password"], + ) + + return config + + +def conf_option_exists(conf, chain): + _key = chain.pop(0) + if _key in conf: + return conf_option_exists(conf[_key], chain) if chain else conf[_key] + + +def check_config_option(config, chain, default_fail=None): + result = conf_option_exists(config, chain.copy()) + if result is None: + raise Exception( + "'{}' was not in config file".format( + chain, + ), + ) + else: + if default_fail: + if result == default_fail: + # We have to fail and bail if the user hasn't edited + # this config option. + raise Exception( + "Config file needs to be edited from provided defaults for {}.".format( + chain, + ), + ) + else: + return config diff --git a/aprsd/dev.py b/aprsd/dev.py index e93f2f0..01eecc1 100644 --- a/aprsd/dev.py +++ b/aprsd/dev.py @@ -14,7 +14,9 @@ import click_completion # local imports here import aprsd -from aprsd import client, plugin, utils +from aprsd import client +from aprsd import config as aprsd_config +from aprsd import plugin # setup the global logger @@ -156,7 +158,7 @@ def setup_logging(config, loglevel, quiet): "--config", "config_file", show_default=True, - default=utils.DEFAULT_CONFIG_FILE, + default=aprsd_config.DEFAULT_CONFIG_FILE, help="The aprsd config file to use for options.", ) @click.option( @@ -178,7 +180,7 @@ def test_plugin( ): """APRSD Plugin test app.""" - config = utils.parse_config(config_file) + config = aprsd_config.parse_config(config_file) setup_logging(config, loglevel, False) LOG.info(f"Test APRSD PLugin version: {aprsd.__version__}") diff --git a/aprsd/flask.py b/aprsd/flask.py index e8fc0aa..ede31db 100644 --- a/aprsd/flask.py +++ b/aprsd/flask.py @@ -17,9 +17,9 @@ from flask_socketio import Namespace, SocketIO from werkzeug.security import check_password_hash, generate_password_hash import aprsd -from aprsd import ( - client, kissclient, messaging, packets, plugin, stats, threads, utils, -) +from aprsd import client +from aprsd import config as aprsd_config +from aprsd import kissclient, messaging, packets, plugin, stats, threads, utils LOG = logging.getLogger("APRSD") @@ -553,10 +553,10 @@ def setup_logging(config, flask_app, loglevel, quiet): flask_app.logger.disabled = True return - log_level = utils.LOG_LEVELS[loglevel] + log_level = aprsd_config.LOG_LEVELS[loglevel] LOG.setLevel(log_level) - log_format = config["aprsd"].get("logformat", utils.DEFAULT_LOG_FORMAT) - date_format = config["aprsd"].get("dateformat", utils.DEFAULT_DATE_FORMAT) + log_format = config["aprsd"].get("logformat", aprsd_config.DEFAULT_LOG_FORMAT) + date_format = config["aprsd"].get("dateformat", aprsd_config.DEFAULT_DATE_FORMAT) log_formatter = logging.Formatter(fmt=log_format, datefmt=date_format) log_file = config["aprsd"].get("logfile", None) if log_file: diff --git a/aprsd/healthcheck.py b/aprsd/healthcheck.py index 4e4b212..462661c 100644 --- a/aprsd/healthcheck.py +++ b/aprsd/healthcheck.py @@ -19,7 +19,7 @@ import requests # local imports here import aprsd -from aprsd import utils +from aprsd import config as aprsd_config # setup the global logger @@ -172,7 +172,7 @@ def parse_delta_str(s): "--config", "config_file", show_default=True, - default=utils.DEFAULT_CONFIG_FILE, + default=aprsd_config.DEFAULT_CONFIG_FILE, help="The aprsd config file to use for options.", ) @click.option( @@ -191,7 +191,7 @@ def parse_delta_str(s): def check(loglevel, config_file, health_url, timeout): """APRSD Plugin test app.""" - config = utils.parse_config(config_file) + config = aprsd_config.parse_config(config_file) setup_logging(config, loglevel, False) LOG.debug(f"APRSD HealthCheck version: {aprsd.__version__}") diff --git a/aprsd/listen.py b/aprsd/listen.py index 4342447..40d90b7 100644 --- a/aprsd/listen.py +++ b/aprsd/listen.py @@ -36,7 +36,9 @@ import click_completion # local imports here import aprsd -from aprsd import client, messaging, stats, threads, trace, utils +from aprsd import client +from aprsd import config as aprsd_config +from aprsd import messaging, stats, threads, trace, utils # setup the global logger @@ -169,10 +171,10 @@ def signal_handler(sig, frame): # to disable logging to stdout, but still log to file # use the --quiet option on the cmdln def setup_logging(config, loglevel, quiet): - log_level = utils.LOG_LEVELS[loglevel] + log_level = aprsd_config.LOG_LEVELS[loglevel] LOG.setLevel(log_level) - log_format = config["aprsd"].get("logformat", utils.DEFAULT_LOG_FORMAT) - date_format = config["aprsd"].get("dateformat", utils.DEFAULT_DATE_FORMAT) + log_format = config["aprsd"].get("logformat", aprsd_config.DEFAULT_LOG_FORMAT) + date_format = config["aprsd"].get("dateformat", aprsd_config.DEFAULT_DATE_FORMAT) log_formatter = logging.Formatter(fmt=log_format, datefmt=date_format) log_file = config["aprsd"].get("logfile", None) if log_file: @@ -218,7 +220,7 @@ def setup_logging(config, loglevel, quiet): "--config", "config_file", show_default=True, - default=utils.DEFAULT_CONFIG_FILE, + default=aprsd_config.DEFAULT_CONFIG_FILE, help="The aprsd config file to use for options.", ) @click.option( @@ -258,7 +260,7 @@ def listen( """Send a message to a callsign via APRS_IS.""" global got_ack, got_response - config = utils.parse_config(config_file) + config = aprsd_config.parse_config(config_file) if not aprs_login: click.echo("Must set --aprs_login or APRS_LOGIN") return diff --git a/aprsd/main.py b/aprsd/main.py index 6de7358..91b4816 100644 --- a/aprsd/main.py +++ b/aprsd/main.py @@ -37,9 +37,10 @@ import click_completion # local imports here import aprsd from aprsd import ( - client, flask, kissclient, messaging, packets, plugin, stats, threads, - trace, utils, + flask, kissclient, messaging, packets, plugin, stats, threads, trace, utils, ) +from aprsd import client +from aprsd import config as aprsd_config # setup the global logger @@ -48,22 +49,8 @@ LOG = logging.getLogger("APRSD") CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) - flask_enabled = False -# server_event = threading.Event() - -# localization, please edit: -# HOST = "noam.aprs2.net" # north america tier2 servers round robin -# USER = "KM6XXX-9" # callsign of this aprs client with SSID -# PASS = "99999" # google how to generate this -# BASECALLSIGN = "KM6XXX" # callsign of radio in the field to send email -# shortcuts = { -# "aa" : "5551239999@vtext.com", -# "cl" : "craiglamparter@somedomain.org", -# "wb" : "5553909472@vtext.com" -# } - def custom_startswith(string, incomplete): """A custom completion match that supports case insensitive matching.""" @@ -172,10 +159,10 @@ def signal_handler(sig, frame): # to disable logging to stdout, but still log to file # use the --quiet option on the cmdln def setup_logging(config, loglevel, quiet): - log_level = utils.LOG_LEVELS[loglevel] + log_level = aprsd_config.LOG_LEVELS[loglevel] LOG.setLevel(log_level) - log_format = config["aprsd"].get("logformat", utils.DEFAULT_LOG_FORMAT) - date_format = config["aprsd"].get("dateformat", utils.DEFAULT_DATE_FORMAT) + log_format = config["aprsd"].get("logformat", aprsd_config.DEFAULT_LOG_FORMAT) + date_format = config["aprsd"].get("dateformat", aprsd_config.DEFAULT_DATE_FORMAT) log_formatter = logging.Formatter(fmt=log_format, datefmt=date_format) log_file = config["aprsd"].get("logfile", None) if log_file: @@ -196,15 +183,15 @@ def setup_logging(config, loglevel, quiet): imap_logger.addHandler(fh) if ( - utils.check_config_option( + aprsd_config.check_config_option( config, ["aprsd", "web", "enabled"], default_fail=False, ) ): qh = logging.handlers.QueueHandler(threads.logging_queue) q_log_formatter = logging.Formatter( - fmt=utils.QUEUE_LOG_FORMAT, - datefmt=utils.QUEUE_DATE_FORMAT, + fmt=aprsd_config.QUEUE_LOG_FORMAT, + datefmt=aprsd_config.QUEUE_DATE_FORMAT, ) qh.setFormatter(q_log_formatter) LOG.addHandler(qh) @@ -234,11 +221,11 @@ def setup_logging(config, loglevel, quiet): "--config", "config_file", show_default=True, - default=utils.DEFAULT_CONFIG_FILE, + default=aprsd_config.DEFAULT_CONFIG_FILE, help="The aprsd config file to use for options.", ) def check_version(loglevel, config_file): - config = utils.parse_config(config_file) + config = aprsd_config.parse_config(config_file) setup_logging(config, loglevel, False) level, msg = utils._check_version() @@ -251,7 +238,7 @@ def check_version(loglevel, config_file): @main.command() def sample_config(): """This dumps the config to stdout.""" - click.echo(utils.dump_default_cfg()) + click.echo(aprsd_config.dump_default_cfg()) @main.command() @@ -272,7 +259,7 @@ def sample_config(): "--config", "config_file", show_default=True, - default=utils.DEFAULT_CONFIG_FILE, + default=aprsd_config.DEFAULT_CONFIG_FILE, help="The aprsd config file to use for options.", ) @click.option( @@ -312,7 +299,7 @@ def send_message( """Send a message to a callsign via APRS_IS.""" global got_ack, got_response - config = utils.parse_config(config_file) + config = aprsd_config.parse_config(config_file) if not aprs_login: click.echo("Must set --aprs_login or APRS_LOGIN") return @@ -429,7 +416,7 @@ def send_message( "--config", "config_file", show_default=True, - default=utils.DEFAULT_CONFIG_FILE, + default=aprsd_config.DEFAULT_CONFIG_FILE, help="The aprsd config file to use for options.", ) @click.option( @@ -454,7 +441,7 @@ def server( if not quiet: click.echo("Load config") - config = utils.parse_config(config_file) + config = aprsd_config.parse_config(config_file) setup_logging(config, loglevel, quiet) level, msg = utils._check_version() @@ -523,7 +510,7 @@ def server( keepalive = threads.KeepAliveThread(config=config) keepalive.start() - web_enabled = utils.check_config_option(config, ["aprsd", "web", "enabled"], default_fail=False) + web_enabled = aprsd_config.check_config_option(config, ["aprsd", "web", "enabled"], default_fail=False) if web_enabled: flask_enabled = True diff --git a/aprsd/messaging.py b/aprsd/messaging.py index 751272f..1a983a1 100644 --- a/aprsd/messaging.py +++ b/aprsd/messaging.py @@ -9,7 +9,9 @@ import re import threading import time -from aprsd import client, kissclient, packets, stats, threads, trace, utils +from aprsd import client +from aprsd import config as aprsd_config +from aprsd import kissclient, packets, stats, threads, trace LOG = logging.getLogger("APRSD") @@ -113,11 +115,11 @@ class MsgTrack: LOG.debug(f"Save tracker to disk? {len(self)}") if len(self) > 0: LOG.info(f"Saving {len(self)} tracking messages to disk") - pickle.dump(self.dump(), open(utils.DEFAULT_SAVE_FILE, "wb+")) + pickle.dump(self.dump(), open(aprsd_config.DEFAULT_SAVE_FILE, "wb+")) else: LOG.debug( "Nothing to save, flushing old save file '{}'".format( - utils.DEFAULT_SAVE_FILE, + aprsd_config.DEFAULT_SAVE_FILE, ), ) self.flush() @@ -131,8 +133,8 @@ class MsgTrack: return dump def load(self): - if os.path.exists(utils.DEFAULT_SAVE_FILE): - raw = pickle.load(open(utils.DEFAULT_SAVE_FILE, "rb")) + if os.path.exists(aprsd_config.DEFAULT_SAVE_FILE): + raw = pickle.load(open(aprsd_config.DEFAULT_SAVE_FILE, "rb")) if raw: self.track = raw LOG.debug("Loaded MsgTrack dict from disk.") @@ -171,8 +173,8 @@ class MsgTrack: def flush(self): """Nuke the old pickle file that stored the old results from last aprsd run.""" - if os.path.exists(utils.DEFAULT_SAVE_FILE): - pathlib.Path(utils.DEFAULT_SAVE_FILE).unlink() + if os.path.exists(aprsd_config.DEFAULT_SAVE_FILE): + pathlib.Path(aprsd_config.DEFAULT_SAVE_FILE).unlink() with self.lock: self.track = {} diff --git a/aprsd/plugins/location.py b/aprsd/plugins/location.py index 59b3847..80a44e6 100644 --- a/aprsd/plugins/location.py +++ b/aprsd/plugins/location.py @@ -2,7 +2,7 @@ import logging import re import time -from aprsd import plugin, plugin_utils, trace, utils +from aprsd import config, plugin, plugin_utils, trace LOG = logging.getLogger("APRSD") @@ -24,7 +24,7 @@ class LocationPlugin(plugin.APRSDRegexCommandPluginBase): # get last location of a callsign, get descriptive name from weather service try: - utils.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" diff --git a/aprsd/plugins/time.py b/aprsd/plugins/time.py index 81df83a..516643a 100644 --- a/aprsd/plugins/time.py +++ b/aprsd/plugins/time.py @@ -5,7 +5,7 @@ import time from opencage.geocoder import OpenCageGeocode import pytz -from aprsd import fuzzyclock, plugin, plugin_utils, trace, utils +from aprsd import config, fuzzyclock, plugin, plugin_utils, trace LOG = logging.getLogger("APRSD") @@ -64,7 +64,7 @@ class TimeOpenCageDataPlugin(TimePlugin): # get last location of a callsign, get descriptive name from weather service try: - utils.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" @@ -95,7 +95,7 @@ class TimeOpenCageDataPlugin(TimePlugin): lon = aprs_data["entries"][0]["lng"] try: - utils.check_config_option(self.config, "opencagedata", "apiKey") + config.check_config_option(self.config, "opencagedata", "apiKey") except Exception as ex: LOG.error(f"Failed to find config opencage:apiKey {ex}") return "No opencage apiKey found" @@ -130,7 +130,7 @@ class TimeOWMPlugin(TimePlugin): # get last location of a callsign, get descriptive name from weather service try: - utils.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" @@ -160,7 +160,7 @@ class TimeOWMPlugin(TimePlugin): lon = aprs_data["entries"][0]["lng"] try: - utils.check_config_option( + config.check_config_option( self.config, ["services", "openweathermap", "apiKey"], ) diff --git a/aprsd/plugins/weather.py b/aprsd/plugins/weather.py index 8eea849..a1dc2cf 100644 --- a/aprsd/plugins/weather.py +++ b/aprsd/plugins/weather.py @@ -4,7 +4,7 @@ import re import requests -from aprsd import plugin, plugin_utils, trace, utils +from aprsd import config, plugin, plugin_utils, trace LOG = logging.getLogger("APRSD") @@ -34,7 +34,7 @@ class USWeatherPlugin(plugin.APRSDRegexCommandPluginBase): # message = packet.get("message_text", None) # ack = packet.get("msgNo", "0") try: - utils.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" @@ -115,7 +115,7 @@ class USMetarPlugin(plugin.APRSDRegexCommandPluginBase): fromcall = fromcall try: - utils.check_config_option( + config.check_config_option( self.config, ["services", "aprs.fi", "apiKey"], ) @@ -199,7 +199,7 @@ class OWMWeatherPlugin(plugin.APRSDRegexCommandPluginBase): searchcall = fromcall try: - utils.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" @@ -220,7 +220,7 @@ class OWMWeatherPlugin(plugin.APRSDRegexCommandPluginBase): lon = aprs_data["entries"][0]["lng"] try: - utils.check_config_option( + config.check_config_option( self.config, ["services", "openweathermap", "apiKey"], ) @@ -229,7 +229,7 @@ class OWMWeatherPlugin(plugin.APRSDRegexCommandPluginBase): return "No openweathermap apiKey found" try: - utils.check_config_option(self.config, ["aprsd", "units"]) + config.check_config_option(self.config, ["aprsd", "units"]) except Exception: LOG.debug("Couldn't find untis in aprsd:services:units") units = "metric" @@ -323,7 +323,7 @@ class AVWXWeatherPlugin(plugin.APRSDRegexCommandPluginBase): searchcall = fromcall try: - utils.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" @@ -344,13 +344,13 @@ class AVWXWeatherPlugin(plugin.APRSDRegexCommandPluginBase): lon = aprs_data["entries"][0]["lng"] try: - utils.check_config_option(self.config, ["services", "avwx", "apiKey"]) + config.check_config_option(self.config, ["services", "avwx", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config avwx:apiKey {ex}") return "No avwx apiKey found" try: - utils.check_config_option(self.config, ["services", "avwx", "base_url"]) + config.check_config_option(self.config, ["services", "avwx", "base_url"]) except Exception as ex: LOG.debug(f"Didn't find avwx:base_url {ex}") base_url = "https://avwx.rest" diff --git a/aprsd/utils.py b/aprsd/utils.py index 1f55e06..d639582 100644 --- a/aprsd/utils.py +++ b/aprsd/utils.py @@ -3,128 +3,13 @@ import collections import errno import functools -import logging import os -from pathlib import Path import re -import sys import threading -import click import update_checker -import yaml import aprsd -from aprsd import plugin - - -LOG_LEVELS = { - "CRITICAL": logging.CRITICAL, - "ERROR": logging.ERROR, - "WARNING": logging.WARNING, - "INFO": logging.INFO, - "DEBUG": logging.DEBUG, -} - -DEFAULT_DATE_FORMAT = "%m/%d/%Y %I:%M:%S %p" -DEFAULT_LOG_FORMAT = ( - "[%(asctime)s] [%(threadName)-20.20s] [%(levelname)-5.5s]" - " %(message)s - [%(pathname)s:%(lineno)d]" -) - -QUEUE_DATE_FORMAT = "[%m/%d/%Y] [%I:%M:%S %p]" -QUEUE_LOG_FORMAT = ( - "%(asctime)s [%(threadName)-20.20s] [%(levelname)-5.5s]" - " %(message)s - [%(pathname)s:%(lineno)d]" -) - -# an example of what should be in the ~/.aprsd/config.yml -DEFAULT_CONFIG_DICT = { - "ham": {"callsign": "NOCALL"}, - "aprs": { - "enabled": True, - "login": "CALLSIGN", - "password": "00000", - "host": "rotate.aprs2.net", - "port": 14580, - }, - "kiss": { - "tcp": { - "enabled": False, - "host": "direwolf.ip.address", - "port": "8001", - }, - "serial": { - "enabled": False, - "device": "/dev/ttyS0", - "baudrate": 9600, - }, - }, - "aprsd": { - "logfile": "/tmp/aprsd.log", - "logformat": DEFAULT_LOG_FORMAT, - "dateformat": DEFAULT_DATE_FORMAT, - "trace": False, - "enabled_plugins": plugin.CORE_MESSAGE_PLUGINS, - "units": "imperial", - "watch_list": { - "enabled": False, - # Who gets the alert? - "alert_callsign": "NOCALL", - # 43200 is 12 hours - "alert_time_seconds": 43200, - # How many packets to save in a ring Buffer - # for a particular callsign - "packet_keep_count": 10, - "callsigns": [], - "enabled_plugins": plugin.CORE_NOTIFY_PLUGINS, - }, - "web": { - "enabled": True, - "logging_enabled": True, - "host": "0.0.0.0", - "port": 8001, - "users": { - "admin": "password-here", - }, - }, - "email": { - "enabled": True, - "shortcuts": { - "aa": "5551239999@vtext.com", - "cl": "craiglamparter@somedomain.org", - "wb": "555309@vtext.com", - }, - "smtp": { - "login": "SMTP_USERNAME", - "password": "SMTP_PASSWORD", - "host": "smtp.gmail.com", - "port": 465, - "use_ssl": False, - "debug": False, - }, - "imap": { - "login": "IMAP_USERNAME", - "password": "IMAP_PASSWORD", - "host": "imap.gmail.com", - "port": 993, - "use_ssl": True, - "debug": False, - }, - }, - }, - "services": { - "aprs.fi": {"apiKey": "APIKEYVALUE"}, - "openweathermap": {"apiKey": "APIKEYVALUE"}, - "opencagedata": {"apiKey": "APIKEYVALUE"}, - "avwx": {"base_url": "http://host:port", "apiKey": "APIKEYVALUE"}, - }, -} - -home = str(Path.home()) -DEFAULT_CONFIG_DIR = f"{home}/.config/aprsd/" -DEFAULT_SAVE_FILE = f"{home}/.config/aprsd/aprsd.p" -DEFAULT_CONFIG_FILE = f"{home}/.config/aprsd/aprsd.yml" def synchronized(wrapped): @@ -175,239 +60,6 @@ def end_substr(original, substr): return idx -def dump_default_cfg(): - return add_config_comments( - yaml.dump( - DEFAULT_CONFIG_DICT, - indent=4, - ), - ) - - -def add_config_comments(raw_yaml): - end_idx = end_substr(raw_yaml, "aprs:") - if end_idx != -1: - # lets insert a comment - raw_yaml = insert_str( - raw_yaml, - "\n # Set enabled to False if there is no internet connectivity." - "\n # This is useful for a direwolf KISS aprs connection only. " - "\n" - "\n # Get the passcode for your callsign here: " - "\n # https://apps.magicbug.co.uk/passcode", - end_idx, - ) - - end_idx = end_substr(raw_yaml, "aprs.fi:") - if end_idx != -1: - # lets insert a comment - raw_yaml = insert_str( - raw_yaml, - "\n # Get the apiKey from your aprs.fi account here: " - "\n # http://aprs.fi/account", - end_idx, - ) - - end_idx = end_substr(raw_yaml, "opencagedata:") - if end_idx != -1: - # lets insert a comment - raw_yaml = insert_str( - raw_yaml, - "\n # (Optional for TimeOpenCageDataPlugin) " - "\n # Get the apiKey from your opencagedata account here: " - "\n # https://opencagedata.com/dashboard#api-keys", - end_idx, - ) - - end_idx = end_substr(raw_yaml, "openweathermap:") - if end_idx != -1: - # lets insert a comment - raw_yaml = insert_str( - raw_yaml, - "\n # (Optional for OWMWeatherPlugin) " - "\n # Get the apiKey from your " - "\n # openweathermap account here: " - "\n # https://home.openweathermap.org/api_keys", - end_idx, - ) - - end_idx = end_substr(raw_yaml, "avwx:") - if end_idx != -1: - # lets insert a comment - raw_yaml = insert_str( - raw_yaml, - "\n # (Optional for AVWXWeatherPlugin) " - "\n # Use hosted avwx-api here: https://avwx.rest " - "\n # or deploy your own from here: " - "\n # https://github.com/avwx-rest/avwx-api", - end_idx, - ) - - return raw_yaml - - -def create_default_config(): - """Create a default config file.""" - # make sure the directory location exists - config_file_expanded = os.path.expanduser(DEFAULT_CONFIG_FILE) - config_dir = os.path.dirname(config_file_expanded) - if not os.path.exists(config_dir): - click.echo(f"Config dir '{config_dir}' doesn't exist, creating.") - mkdir_p(config_dir) - with open(config_file_expanded, "w+") as cf: - cf.write(dump_default_cfg()) - - -def get_config(config_file): - """This tries to read the yaml config from .""" - config_file_expanded = os.path.expanduser(config_file) - if os.path.exists(config_file_expanded): - with open(config_file_expanded) as stream: - config = yaml.load(stream, Loader=yaml.FullLoader) - return config - else: - if config_file == DEFAULT_CONFIG_FILE: - click.echo( - f"{config_file_expanded} is missing, creating config file", - ) - create_default_config() - msg = ( - "Default config file created at {}. Please edit with your " - "settings.".format(config_file) - ) - click.echo(msg) - else: - # The user provided a config file path different from the - # Default, so we won't try and create it, just bitch and bail. - msg = f"Custom config file '{config_file}' is missing." - click.echo(msg) - - sys.exit(-1) - - -def conf_option_exists(conf, chain): - _key = chain.pop(0) - if _key in conf: - return conf_option_exists(conf[_key], chain) if chain else conf[_key] - - -def check_config_option(config, chain, default_fail=None): - result = conf_option_exists(config, chain.copy()) - if result is None: - raise Exception( - "'{}' was not in config file".format( - chain, - ), - ) - else: - if default_fail: - if result == default_fail: - # We have to fail and bail if the user hasn't edited - # this config option. - raise Exception( - "Config file needs to be edited from provided defaults for {}.".format( - chain, - ), - ) - else: - return config - - -# This method tries to parse the config yaml file -# and consume the settings. -# If the required params don't exist, -# it will look in the environment -def parse_config(config_file): - # for now we still use globals....ugh - global CONFIG - - def fail(msg): - click.echo(msg) - sys.exit(-1) - - def check_option(config, chain, default_fail=None): - try: - config = check_config_option(config, chain, default_fail=default_fail) - except Exception as ex: - fail(repr(ex)) - else: - return config - - config = get_config(config_file) - - # special check here to make sure user has edited the config file - # and changed the ham callsign - check_option( - config, - [ - "ham", - "callsign", - ], - default_fail=DEFAULT_CONFIG_DICT["ham"]["callsign"], - ) - check_option( - config, - ["services", "aprs.fi", "apiKey"], - default_fail=DEFAULT_CONFIG_DICT["services"]["aprs.fi"]["apiKey"], - ) - check_option( - config, - ["aprs", "login"], - default_fail=DEFAULT_CONFIG_DICT["aprs"]["login"], - ) - check_option( - config, - ["aprs", "password"], - default_fail=DEFAULT_CONFIG_DICT["aprs"]["password"], - ) - - # Ensure they change the admin password - if config["aprsd"]["web"]["enabled"] is True: - check_option( - config, - ["aprsd", "web", "users", "admin"], - default_fail=DEFAULT_CONFIG_DICT["aprsd"]["web"]["users"]["admin"], - ) - - if config["aprsd"]["watch_list"]["enabled"] is True: - check_option( - config, - ["aprsd", "watch_list", "alert_callsign"], - default_fail=DEFAULT_CONFIG_DICT["aprsd"]["watch_list"]["alert_callsign"], - ) - - if config["aprsd"]["email"]["enabled"] is True: - # Check IMAP server settings - check_option(config, ["aprsd", "email", "imap", "host"]) - check_option(config, ["aprsd", "email", "imap", "port"]) - check_option( - config, - ["aprsd", "email", "imap", "login"], - default_fail=DEFAULT_CONFIG_DICT["aprsd"]["email"]["imap"]["login"], - ) - check_option( - config, - ["aprsd", "email", "imap", "password"], - default_fail=DEFAULT_CONFIG_DICT["aprsd"]["email"]["imap"]["password"], - ) - - # Check SMTP server settings - check_option(config, ["aprsd", "email", "smtp", "host"]) - check_option(config, ["aprsd", "email", "smtp", "port"]) - check_option( - config, - ["aprsd", "email", "smtp", "login"], - default_fail=DEFAULT_CONFIG_DICT["aprsd"]["email"]["smtp"]["login"], - ) - check_option( - config, - ["aprsd", "email", "smtp", "password"], - default_fail=DEFAULT_CONFIG_DICT["aprsd"]["email"]["smtp"]["password"], - ) - - return config - - def human_size(bytes, units=None): """Returns a human readable string representation of bytes""" if not units: diff --git a/docker/Dockerfile b/docker/Dockerfile index a928482..a07b82f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -28,7 +28,7 @@ RUN addgroup --gid $GID $APRS_USER RUN useradd -m -u $UID -g $APRS_USER $APRS_USER # Install aprsd -RUN /usr/local/bin/pip3 install aprsd==2.3.0 +RUN /usr/local/bin/pip3 install aprsd==2.3.1 # Ensure /config is there with a default config file USER root diff --git a/docker/build.sh b/docker/build.sh index 064b1cd..6ab4edf 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -36,7 +36,7 @@ do esac done -VERSION="2.2.1" +VERSION="2.3.1" if [ $ALL_PLATFORMS -eq 1 ] then diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 7a5d575..df4337e 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -4,7 +4,7 @@ from unittest import mock import pytz import aprsd -from aprsd import messaging, packets, stats, utils +from aprsd import config, messaging, packets, stats from aprsd.fuzzyclock import fuzzy from aprsd.plugins import fortune as fortune_plugin from aprsd.plugins import ping as ping_plugin @@ -19,7 +19,7 @@ class TestPlugin(unittest.TestCase): def setUp(self): self.fromcall = fake.FAKE_FROM_CALLSIGN self.ack = 1 - self.config = utils.DEFAULT_CONFIG_DICT + self.config = config.DEFAULT_CONFIG_DICT self.config["ham"]["callsign"] = self.fromcall self.config["aprs"]["login"] = fake.FAKE_TO_CALLSIGN # Inintialize the stats object with the config From 270be947b59d47ba577091be119f6c8de522f3fd Mon Sep 17 00:00:00 2001 From: Hemna Date: Fri, 17 Sep 2021 09:32:30 -0400 Subject: [PATCH 02/12] Refactored client classes This patch completely refactors and simplifies how the clients are created and used. There is no need now to have a separate KISSRXThread. Since all the custom work for the KISS client is encapsulated in the kiss client itself, the same RX thread and callback mechanism works for both the APRSIS client and KISS Client objects. There is also no need to determine which transport (aprsis vs kiss) is being used at runtime by any of the messages objects. The same API works for both APRSIS and KISS Client objects --- aprsd/client.py | 338 +++++++++-------------- aprsd/clients/aprsis.py | 209 ++++++++++++++ aprsd/{kissclient.py => clients/kiss.py} | 92 ++---- aprsd/flask.py | 14 +- aprsd/main.py | 38 +-- aprsd/messaging.py | 46 +-- aprsd/plugin.py | 2 +- aprsd/threads.py | 107 +------ 8 files changed, 409 insertions(+), 437 deletions(-) create mode 100644 aprsd/clients/aprsis.py rename aprsd/{kissclient.py => clients/kiss.py} (54%) diff --git a/aprsd/client.py b/aprsd/client.py index 3e8e198..3b7ee0e 100644 --- a/aprsd/client.py +++ b/aprsd/client.py @@ -1,26 +1,30 @@ +import abc import logging -import select import time import aprslib -from aprslib import is_py3 -from aprslib.exceptions import ( - ConnectionDrop, ConnectionError, GenericError, LoginError, ParseError, - UnknownFormat, -) +from aprslib.exceptions import LoginError -import aprsd -from aprsd import stats +from aprsd import trace +from aprsd.clients import aprsis, kiss LOG = logging.getLogger("APRSD") +TRANSPORT_APRSIS = "aprsis" +TRANSPORT_TCPKISS = "tcpkiss" +TRANSPORT_SERIALKISS = "serialkiss" + +# Main must create this from the ClientFactory +# object such that it's populated with the +# Correct config +factory = None class Client: """Singleton client class that constructs the aprslib connection.""" _instance = None - aprs_client = None + _client = None config = None connected = False @@ -40,14 +44,49 @@ class Client: @property def client(self): - if not self.aprs_client: - self.aprs_client = self.setup_connection() - return self.aprs_client + if not self._client: + self._client = self.setup_connection() + return self._client def reset(self): """Call this to force a rebuild/reconnect.""" - del self.aprs_client + del self._client + @abc.abstractmethod + def setup_connection(self): + pass + + @staticmethod + @abc.abstractmethod + def is_enabled(config): + pass + + @staticmethod + @abc.abstractmethod + def transport(config): + pass + + @abc.abstractmethod + def decode_packet(self, *args, **kwargs): + pass + + +class APRSISClient(Client): + + @staticmethod + def is_enabled(config): + # Defaults to True if the enabled flag is non existent + return config["aprs"].get("enabled", True) + + @staticmethod + def transport(config): + return TRANSPORT_APRSIS + + def decode_packet(self, *args, **kwargs): + """APRS lib already decodes this.""" + return args[0] + + @trace.trace def setup_connection(self): user = self.config["aprs"]["login"] password = self.config["aprs"]["password"] @@ -55,10 +94,11 @@ class Client: port = self.config["aprs"].get("port", 14580) connected = False backoff = 1 + aprs_client = None while not connected: try: LOG.info("Creating aprslib client") - aprs_client = Aprsdis(user, passwd=password, host=host, port=port) + aprs_client = aprsis.Aprsdis(user, passwd=password, host=host, port=port) # Force the logging to be the same aprs_client.logger = LOG aprs_client.connect() @@ -77,200 +117,96 @@ class Client: return aprs_client -class Aprsdis(aprslib.IS): - """Extend the aprslib class so we can exit properly.""" +class KISSClient(Client): - # flag to tell us to stop - thread_stop = False + @staticmethod + def is_enabled(config): + """Return if tcp or serial KISS is enabled.""" + if "kiss" not in config: + return False - # timeout in seconds - select_timeout = 1 + if "serial" in config["kiss"]: + if config["kiss"]["serial"].get("enabled", False): + return True - def stop(self): - self.thread_stop = True - LOG.info("Shutdown Aprsdis client.") + if "tcp" in config["kiss"]: + if config["kiss"]["tcp"].get("enabled", False): + return True - def send(self, msg): - """Send an APRS Message object.""" - line = str(msg) - self.sendall(line) + @staticmethod + def transport(config): + if "serial" in config["kiss"]: + if config["kiss"]["serial"].get("enabled", False): + return TRANSPORT_SERIALKISS - def _socket_readlines(self, blocking=False): - """ - Generator for complete lines, received from the server - """ - try: - self.sock.setblocking(0) - except OSError as e: - self.logger.error(f"socket error when setblocking(0): {str(e)}") - raise aprslib.ConnectionDrop("connection dropped") + if "tcp" in config["kiss"]: + if config["kiss"]["tcp"].get("enabled", False): + return TRANSPORT_TCPKISS - while not self.thread_stop: - short_buf = b"" - newline = b"\r\n" + def decode_packet(self, *args, **kwargs): + """We get a frame, which has to be decoded.""" + frame = kwargs["frame"] + LOG.debug(f"Got an APRS Frame '{frame}'") + # try and nuke the * from the fromcall sign. + frame.header._source._ch = False + payload = str(frame.payload.decode()) + msg = f"{str(frame.header)}:{payload}" + # msg = frame.tnc2 + LOG.debug(f"Decoding {msg}") - # set a select timeout, so we get a chance to exit - # when user hits CTRL-C - readable, writable, exceptional = select.select( - [self.sock], - [], - [], - self.select_timeout, - ) - if not readable: - if not blocking: - break - else: - continue + packet = aprslib.parse(msg) + return packet - try: - short_buf = self.sock.recv(4096) - - # sock.recv returns empty if the connection drops - if not short_buf: - if not blocking: - # We could just not be blocking, so empty is expected - continue - else: - self.logger.error("socket.recv(): returned empty") - raise aprslib.ConnectionDrop("connection dropped") - except OSError as e: - # self.logger.error("socket error on recv(): %s" % str(e)) - if "Resource temporarily unavailable" in str(e): - if not blocking: - if len(self.buf) == 0: - break - - self.buf += short_buf - - while newline in self.buf: - line, self.buf = self.buf.split(newline, 1) - - yield line - - def _send_login(self): - """ - Sends login string to server - """ - login_str = "user {0} pass {1} vers github.com/craigerl/aprsd {3}{2}\r\n" - login_str = login_str.format( - self.callsign, - self.passwd, - (" filter " + self.filter) if self.filter != "" else "", - aprsd.__version__, - ) - - self.logger.info("Sending login information") - - try: - self._sendall(login_str) - self.sock.settimeout(5) - test = self.sock.recv(len(login_str) + 100) - if is_py3: - test = test.decode("latin-1") - test = test.rstrip() - - self.logger.debug("Server: %s", test) - - a, b, callsign, status, e = test.split(" ", 4) - s = e.split(",") - if len(s): - server_string = s[0].replace("server ", "") - else: - server_string = e.replace("server ", "") - - self.logger.info(f"Connected to {server_string}") - self.server_string = server_string - stats.APRSDStats().set_aprsis_server(server_string) - - if callsign == "": - raise LoginError("Server responded with empty callsign???") - if callsign != self.callsign: - raise LoginError(f"Server: {test}") - if status != "verified," and self.passwd != "-1": - raise LoginError("Password is incorrect") - - if self.passwd == "-1": - self.logger.info("Login successful (receive only)") - else: - self.logger.info("Login successful") - - except LoginError as e: - self.logger.error(str(e)) - self.close() - raise - except Exception as e: - self.close() - self.logger.error(f"Failed to login '{e}'") - raise LoginError("Failed to login") - - def consumer(self, callback, blocking=True, immortal=False, raw=False): - """ - When a position sentence is received, it will be passed to the callback function - - blocking: if true (default), runs forever, otherwise will return after one sentence - You can still exit the loop, by raising StopIteration in the callback function - - immortal: When true, consumer will try to reconnect and stop propagation of Parse exceptions - if false (default), consumer will return - - raw: when true, raw packet is passed to callback, otherwise the result from aprs.parse() - """ - - if not self._connected: - raise ConnectionError("not connected to a server") - - line = b"" - - while True and not self.thread_stop: - try: - for line in self._socket_readlines(blocking): - if line[0:1] != b"#": - if raw: - callback(line) - else: - callback(self._parse(line)) - else: - self.logger.debug("Server: %s", line.decode("utf8")) - stats.APRSDStats().set_aprsis_keepalive() - except ParseError as exp: - self.logger.log( - 11, - "%s\n Packet: %s", - exp, - exp.packet, - ) - except UnknownFormat as exp: - self.logger.log( - 9, - "%s\n Packet: %s", - exp, - exp.packet, - ) - except LoginError as exp: - self.logger.error("%s: %s", exp.__class__.__name__, exp) - except (KeyboardInterrupt, SystemExit): - raise - except (ConnectionDrop, ConnectionError): - self.close() - - if not immortal: - raise - else: - self.connect(blocking=blocking) - continue - except GenericError: - pass - except StopIteration: - break - except Exception: - self.logger.error("APRS Packet: %s", line) - raise - - if not blocking: - break + @trace.trace + def setup_connection(self): + ax25client = kiss.Aioax25Client(self.config) + return ax25client -def get_client(): - cl = Client() - return cl.client +class ClientFactory: + _instance = None + + def __new__(cls, *args, **kwargs): + """This magic turns this into a singleton.""" + if cls._instance is None: + cls._instance = super().__new__(cls) + # Put any initialization here. + return cls._instance + + def __init__(self, config): + self.config = config + self._builders = {} + + def register(self, key, builder): + self._builders[key] = builder + + def create(self, key=None): + if not key: + if APRSISClient.is_enabled(self.config): + key = TRANSPORT_APRSIS + elif KISSClient.is_enabled(self.config): + key = KISSClient.transport(self.config) + + LOG.debug(f"GET client {key}") + builder = self._builders.get(key) + if not builder: + raise ValueError(key) + return builder(self.config) + + def is_client_enabled(self): + """Make sure at least one client is enabled.""" + enabled = False + for key in self._builders.keys(): + enabled |= self._builders[key].is_enabled(self.config) + + return enabled + + @staticmethod + def setup(config): + """Create and register all possible client objects.""" + global factory + + factory = ClientFactory(config) + factory.register(TRANSPORT_APRSIS, APRSISClient) + factory.register(TRANSPORT_TCPKISS, KISSClient) + factory.register(TRANSPORT_SERIALKISS, KISSClient) diff --git a/aprsd/clients/aprsis.py b/aprsd/clients/aprsis.py new file mode 100644 index 0000000..ac7bdac --- /dev/null +++ b/aprsd/clients/aprsis.py @@ -0,0 +1,209 @@ +import logging +import select + +import aprslib +from aprslib import is_py3 +from aprslib.exceptions import ( + ConnectionDrop, ConnectionError, GenericError, LoginError, ParseError, + UnknownFormat, +) + +import aprsd +from aprsd import stats + + +LOG = logging.getLogger("APRSD") + + +class Aprsdis(aprslib.IS): + """Extend the aprslib class so we can exit properly.""" + + # flag to tell us to stop + thread_stop = False + + # timeout in seconds + select_timeout = 1 + + def stop(self): + self.thread_stop = True + LOG.info("Shutdown Aprsdis client.") + + def send(self, msg): + """Send an APRS Message object.""" + line = str(msg) + self.sendall(line) + + def _socket_readlines(self, blocking=False): + """ + Generator for complete lines, received from the server + """ + try: + self.sock.setblocking(0) + except OSError as e: + self.logger.error(f"socket error when setblocking(0): {str(e)}") + raise aprslib.ConnectionDrop("connection dropped") + + while not self.thread_stop: + short_buf = b"" + newline = b"\r\n" + + # set a select timeout, so we get a chance to exit + # when user hits CTRL-C + readable, writable, exceptional = select.select( + [self.sock], + [], + [], + self.select_timeout, + ) + if not readable: + if not blocking: + break + else: + continue + + try: + short_buf = self.sock.recv(4096) + + # sock.recv returns empty if the connection drops + if not short_buf: + if not blocking: + # We could just not be blocking, so empty is expected + continue + else: + self.logger.error("socket.recv(): returned empty") + raise aprslib.ConnectionDrop("connection dropped") + except OSError as e: + # self.logger.error("socket error on recv(): %s" % str(e)) + if "Resource temporarily unavailable" in str(e): + if not blocking: + if len(self.buf) == 0: + break + + self.buf += short_buf + + while newline in self.buf: + line, self.buf = self.buf.split(newline, 1) + + yield line + + def _send_login(self): + """ + Sends login string to server + """ + login_str = "user {0} pass {1} vers github.com/craigerl/aprsd {3}{2}\r\n" + login_str = login_str.format( + self.callsign, + self.passwd, + (" filter " + self.filter) if self.filter != "" else "", + aprsd.__version__, + ) + + self.logger.info("Sending login information") + + try: + self._sendall(login_str) + self.sock.settimeout(5) + test = self.sock.recv(len(login_str) + 100) + if is_py3: + test = test.decode("latin-1") + test = test.rstrip() + + self.logger.debug("Server: %s", test) + + a, b, callsign, status, e = test.split(" ", 4) + s = e.split(",") + if len(s): + server_string = s[0].replace("server ", "") + else: + server_string = e.replace("server ", "") + + self.logger.info(f"Connected to {server_string}") + self.server_string = server_string + stats.APRSDStats().set_aprsis_server(server_string) + + if callsign == "": + raise LoginError("Server responded with empty callsign???") + if callsign != self.callsign: + raise LoginError(f"Server: {test}") + if status != "verified," and self.passwd != "-1": + raise LoginError("Password is incorrect") + + if self.passwd == "-1": + self.logger.info("Login successful (receive only)") + else: + self.logger.info("Login successful") + + except LoginError as e: + self.logger.error(str(e)) + self.close() + raise + except Exception as e: + self.close() + self.logger.error(f"Failed to login '{e}'") + raise LoginError("Failed to login") + + def consumer(self, callback, blocking=True, immortal=False, raw=False): + """ + When a position sentence is received, it will be passed to the callback function + + blocking: if true (default), runs forever, otherwise will return after one sentence + You can still exit the loop, by raising StopIteration in the callback function + + immortal: When true, consumer will try to reconnect and stop propagation of Parse exceptions + if false (default), consumer will return + + raw: when true, raw packet is passed to callback, otherwise the result from aprs.parse() + """ + + if not self._connected: + raise ConnectionError("not connected to a server") + + line = b"" + + while True and not self.thread_stop: + try: + for line in self._socket_readlines(blocking): + if line[0:1] != b"#": + if raw: + callback(line) + else: + callback(self._parse(line)) + else: + self.logger.debug("Server: %s", line.decode("utf8")) + stats.APRSDStats().set_aprsis_keepalive() + except ParseError as exp: + self.logger.log( + 11, + "%s\n Packet: %s", + exp, + exp.packet, + ) + except UnknownFormat as exp: + self.logger.log( + 9, + "%s\n Packet: %s", + exp, + exp.packet, + ) + except LoginError as exp: + self.logger.error("%s: %s", exp.__class__.__name__, exp) + except (KeyboardInterrupt, SystemExit): + raise + except (ConnectionDrop, ConnectionError): + self.close() + + if not immortal: + raise + else: + self.connect(blocking=blocking) + continue + except GenericError: + pass + except StopIteration: + break + except Exception: + self.logger.error("APRS Packet: %s", line) + raise + + if not blocking: + break diff --git a/aprsd/kissclient.py b/aprsd/clients/kiss.py similarity index 54% rename from aprsd/kissclient.py rename to aprsd/clients/kiss.py index bed12c2..7ea1ce6 100644 --- a/aprsd/kissclient.py +++ b/aprsd/clients/kiss.py @@ -5,83 +5,20 @@ from aioax25 import interface from aioax25 import kiss as kiss from aioax25.aprs import APRSInterface -from aprsd import trace - -TRANSPORT_TCPKISS = "tcpkiss" -TRANSPORT_SERIALKISS = "serialkiss" LOG = logging.getLogger("APRSD") -class KISSClient: - - _instance = None - config = None - ax25client = None - loop = None - - def __new__(cls, *args, **kwargs): - """Singleton for this class.""" - if cls._instance is None: - cls._instance = super().__new__(cls) - # initialize shit here - return cls._instance - - def __init__(self, config=None): - if config: - self.config = config - - @staticmethod - def kiss_enabled(config): - """Return if tcp or serial KISS is enabled.""" - if "kiss" not in config: - return False - - if "serial" in config["kiss"]: - if config["kiss"]["serial"].get("enabled", False): - return True - - if "tcp" in config["kiss"]: - if config["kiss"]["tcp"].get("enabled", False): - return True - - @staticmethod - def transport(config): - if "serial" in config["kiss"]: - if config["kiss"]["serial"].get("enabled", False): - return TRANSPORT_SERIALKISS - - if "tcp" in config["kiss"]: - if config["kiss"]["tcp"].get("enabled", False): - return TRANSPORT_TCPKISS - - @property - def client(self): - if not self.ax25client: - self.ax25client = self.setup_connection() - return self.ax25client - - def reset(self): - """Call this to fore a rebuild/reconnect.""" - self.ax25client.stop() - del self.ax25client - - @trace.trace - def setup_connection(self): - ax25client = Aioax25Client(self.config) - LOG.debug("Complete") - return ax25client - - class Aioax25Client: def __init__(self, config): self.config = config + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + self.loop = asyncio.get_event_loop() self.setup() def setup(self): # we can be TCP kiss or Serial kiss - - self.loop = asyncio.get_event_loop() if "serial" in self.config["kiss"] and self.config["kiss"]["serial"].get( "enabled", False, @@ -131,10 +68,20 @@ class Aioax25Client: self.kissdev._close() self.loop.stop() - def consumer(self, callback, callsign=None): - if not callsign: - callsign = self.config["ham"]["callsign"] - self.aprsint.bind(callback=callback, callsign="WB4BOR", ssid=12, regex=False) + def set_filter(self, filter): + # This does nothing right now. + pass + + def consumer(self, callback, blocking=True, immortal=False, raw=False): + callsign = self.config["kiss"]["callsign"] + call = callsign.split("-") + if len(call) > 1: + callsign = call[0] + ssid = int(call[1]) + else: + ssid = 0 + self.aprsint.bind(callback=callback, callsign=callsign, ssid=ssid, regex=False) + self.loop.run_forever() def send(self, msg): """Send an APRS Message object.""" @@ -145,8 +92,3 @@ class Aioax25Client: path=["WIDE1-1", "WIDE2-1"], oneshot=True, ) - - -def get_client(): - cl = KISSClient() - return cl.client diff --git a/aprsd/flask.py b/aprsd/flask.py index ede31db..ce5ab36 100644 --- a/aprsd/flask.py +++ b/aprsd/flask.py @@ -19,7 +19,8 @@ from werkzeug.security import check_password_hash, generate_password_hash import aprsd from aprsd import client from aprsd import config as aprsd_config -from aprsd import kissclient, messaging, packets, plugin, stats, threads, utils +from aprsd import messaging, packets, plugin, stats, threads, utils +from aprsd.clients import aprsis LOG = logging.getLogger("APRSD") @@ -136,7 +137,8 @@ class SendMessageThread(threads.APRSDThread): while not connected: try: LOG.info("Creating aprslib client") - aprs_client = client.Aprsdis( + + aprs_client = aprsis.Aprsdis( user, passwd=password, host=host, @@ -312,16 +314,16 @@ class APRSDFlask(flask_classful.FlaskView): ) else: # We might be connected to a KISS socket? - if kissclient.KISSClient.kiss_enabled(self.config): - transport = kissclient.KISSClient.transport(self.config) - if transport == kissclient.TRANSPORT_TCPKISS: + if client.KISSClient.kiss_enabled(self.config): + transport = client.KISSClient.transport(self.config) + if transport == client.TRANSPORT_TCPKISS: aprs_connection = ( "TCPKISS://{}:{}".format( self.config["kiss"]["tcp"]["host"], self.config["kiss"]["tcp"]["port"], ) ) - elif transport == kissclient.TRANSPORT_SERIALKISS: + elif transport == client.TRANSPORT_SERIALKISS: aprs_connection = ( "SerialKISS://{}@{} baud".format( self.config["kiss"]["serial"]["device"], diff --git a/aprsd/main.py b/aprsd/main.py index 91b4816..3bebc80 100644 --- a/aprsd/main.py +++ b/aprsd/main.py @@ -37,7 +37,7 @@ import click_completion # local imports here import aprsd from aprsd import ( - flask, kissclient, messaging, packets, plugin, stats, threads, trace, utils, + flask, messaging, packets, plugin, stats, threads, trace, utils, ) from aprsd import client from aprsd import config as aprsd_config @@ -463,23 +463,13 @@ def server( trace.setup_tracing(["method", "api"]) stats.APRSDStats(config) - if config["aprs"].get("enabled", True): - try: - cl = client.Client(config) - cl.client - except LoginError: - sys.exit(-1) - - rx_thread = threads.APRSDRXThread( - msg_queues=threads.msg_queues, - config=config, - ) - rx_thread.start() - else: - LOG.info( - "APRS network connection Not Enabled in config. This is" - " for setups without internet connectivity.", - ) + # Initialize the client factory and create + # The correct client object ready for use + client.ClientFactory.setup(config) + # Make sure we have 1 client transport enabled + if not client.factory.is_client_enabled(): + LOG.error("No Clients are enabled in config.") + sys.exit(-1) # Create the initial PM singleton and Register plugins plugin_manager = plugin.PluginManager(config) @@ -497,13 +487,11 @@ def server( packets.PacketList(config=config) packets.WatchList(config=config) - if kissclient.KISSClient.kiss_enabled(config): - kcl = kissclient.KISSClient(config=config) - # This initializes the client object. - kcl.client - - kissrx_thread = threads.KISSRXThread(msg_queues=threads.msg_queues, config=config) - kissrx_thread.start() + rx_thread = threads.APRSDRXThread( + msg_queues=threads.msg_queues, + config=config, + ) + rx_thread.start() messaging.MsgTrack().restart() diff --git a/aprsd/messaging.py b/aprsd/messaging.py index 1a983a1..f80574c 100644 --- a/aprsd/messaging.py +++ b/aprsd/messaging.py @@ -11,7 +11,7 @@ import time from aprsd import client from aprsd import config as aprsd_config -from aprsd import kissclient, packets, stats, threads, trace +from aprsd import packets, stats, threads LOG = logging.getLogger("APRSD") @@ -20,10 +20,6 @@ LOG = logging.getLogger("APRSD") # and it's ok, but don't send a usage string back NULL_MESSAGE = -1 -MESSAGE_TRANSPORT_TCPKISS = "tcpkiss" -MESSAGE_TRANSPORT_SERIALKISS = "serialkiss" -MESSAGE_TRANSPORT_APRSIS = "aprsis" - class MsgTrack: """Class to keep track of outstanding text messages. @@ -36,13 +32,6 @@ class MsgTrack: automatically adds itself to this class. When the ack is recieved from the radio, the message object is removed from this class. - - # TODO(hemna) - When aprsd is asked to quit this class should be serialized and - saved to disk/db to keep track of the state of outstanding messages. - When aprsd is started, it should try and fetch the saved state, - and reloaded to a live state. - """ _instance = None @@ -241,7 +230,6 @@ class Message(metaclass=abc.ABCMeta): fromcall, tocall, msg_id=None, - transport=MESSAGE_TRANSPORT_APRSIS, ): self.fromcall = fromcall self.tocall = tocall @@ -250,18 +238,11 @@ class Message(metaclass=abc.ABCMeta): c.increment() msg_id = c.value self.id = msg_id - self.transport = transport @abc.abstractmethod def send(self): """Child class must declare.""" - def get_transport(self): - if self.transport == MESSAGE_TRANSPORT_APRSIS: - return client.get_client() - elif self.transport == MESSAGE_TRANSPORT_TCPKISS: - return kissclient.get_client() - class RawMessage(Message): """Send a raw message. @@ -273,8 +254,8 @@ class RawMessage(Message): message = None - def __init__(self, message, transport=MESSAGE_TRANSPORT_APRSIS): - super().__init__(None, None, msg_id=None, transport=transport) + def __init__(self, message): + super().__init__(None, None, msg_id=None) self.message = message def dict(self): @@ -303,7 +284,7 @@ class RawMessage(Message): def send_direct(self, aprsis_client=None): """Send a message without a separate thread.""" - cl = self.get_transport() + cl = client.factory.create().client log_message( "Sending Message Direct", str(self).rstrip("\n"), @@ -312,7 +293,7 @@ class RawMessage(Message): fromcall=self.fromcall, ) cl.send(self) - stats.APRSDStats().msgs_sent_inc() + stats.APRSDStats().msgs_tx_inc() class TextMessage(Message): @@ -327,9 +308,8 @@ class TextMessage(Message): message, msg_id=None, allow_delay=True, - transport=MESSAGE_TRANSPORT_APRSIS, ): - super().__init__(fromcall, tocall, msg_id, transport=transport) + super().__init__(fromcall, tocall, msg_id) self.message = message # do we try and save this message for later if we don't get # an ack? Some messages we don't want to do this ever. @@ -386,7 +366,7 @@ class TextMessage(Message): if aprsis_client: cl = aprsis_client else: - cl = self.get_transport() + cl = client.factory.create().client log_message( "Sending Message Direct", str(self).rstrip("\n"), @@ -424,7 +404,6 @@ class SendMessageThread(threads.APRSDThread): LOG.info("Message Send Complete via Ack.") return False else: - cl = msg.get_transport() send_now = False if msg.last_send_attempt == msg.retry_count: # we reached the send limit, don't send again @@ -455,6 +434,7 @@ class SendMessageThread(threads.APRSDThread): retry_number=msg.last_send_attempt, msg_num=msg.id, ) + cl = client.factory.create().client cl.send(msg) stats.APRSDStats().msgs_tx_inc() packets.PacketList().add(msg.dict()) @@ -469,8 +449,8 @@ class SendMessageThread(threads.APRSDThread): class AckMessage(Message): """Class for building Acks and sending them.""" - def __init__(self, fromcall, tocall, msg_id, transport=MESSAGE_TRANSPORT_APRSIS): - super().__init__(fromcall, tocall, msg_id=msg_id, transport=transport) + def __init__(self, fromcall, tocall, msg_id): + super().__init__(fromcall, tocall, msg_id=msg_id) def dict(self): now = datetime.datetime.now() @@ -509,7 +489,7 @@ class AckMessage(Message): if aprsis_client: cl = aprsis_client else: - cl = self.get_transport() + cl = client.factory.create().client log_message( "Sending ack", str(self).rstrip("\n"), @@ -526,10 +506,8 @@ class SendAckThread(threads.APRSDThread): self.ack = ack super().__init__(f"SendAck-{self.ack.id}") - @trace.trace def loop(self): """Separate thread to send acks with retries.""" - LOG.debug("SendAckThread loop start") send_now = False if self.ack.last_send_attempt == self.ack.retry_count: # we reached the send limit, don't send again @@ -554,7 +532,7 @@ class SendAckThread(threads.APRSDThread): send_now = True if send_now: - cl = self.ack.get_transport() + cl = client.factory.create().client log_message( "Sending ack", str(self.ack).rstrip("\n"), diff --git a/aprsd/plugin.py b/aprsd/plugin.py index fe5efee..d7fb0bc 100644 --- a/aprsd/plugin.py +++ b/aprsd/plugin.py @@ -158,7 +158,7 @@ class APRSDWatchListPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): ) # make sure the timeout is set or this doesn't work if watch_list: - aprs_client = client.get_client() + aprs_client = client.factory.create().client filter_str = "b/{}".format("/".join(watch_list)) aprs_client.set_filter(filter_str) else: diff --git a/aprsd/threads.py b/aprsd/threads.py index 1132c0a..8cf75f0 100644 --- a/aprsd/threads.py +++ b/aprsd/threads.py @@ -8,7 +8,7 @@ import tracemalloc import aprslib -from aprsd import client, kissclient, messaging, packets, plugin, stats, utils +from aprsd import client, messaging, packets, plugin, stats, utils LOG = logging.getLogger("APRSD") @@ -137,9 +137,9 @@ class KeepAliveThread(APRSDThread): if delta > self.max_delta: # We haven't gotten a keepalive from aprs-is in a while # reset the connection.a - if not kissclient.KISSClient.kiss_enabled(self.config): + if not client.KISSClient.is_enabled(self.config): LOG.warning("Resetting connection to APRS-IS.") - client.Client().reset() + client.factory.create().reset() # Check version every hour delta = now - self.checker_time @@ -158,13 +158,13 @@ class APRSDRXThread(APRSDThread): super().__init__("RX_MSG") self.msg_queues = msg_queues self.config = config + self._client = client.factory.create() def stop(self): self.thread_stop = True - client.get_client().stop() + client.factory.create().client.stop() def loop(self): - aprs_client = client.get_client() # setup the consumer of messages and block until a messages try: @@ -177,7 +177,9 @@ class APRSDRXThread(APRSDThread): # and the aprslib developer didn't want to allow a PR to add # kwargs. :( # https://github.com/rossengeorgiev/aprs-python/pull/56 - aprs_client.consumer(self.process_packet, raw=False, blocking=False) + self._client.client.consumer( + self.process_packet, raw=False, blocking=False, + ) except aprslib.exceptions.ConnectionDrop: LOG.error("Connection dropped, reconnecting") @@ -185,21 +187,21 @@ class APRSDRXThread(APRSDThread): # Force the deletion of the client object connected to aprs # This will cause a reconnect, next time client.get_client() # is called - client.Client().reset() + self._client.reset() # Continue to loop return True - def process_packet(self, packet): + def process_packet(self, *args, **kwargs): + packet = self._client.decode_packet(*args, **kwargs) thread = APRSDProcessPacketThread(packet=packet, config=self.config) thread.start() class APRSDProcessPacketThread(APRSDThread): - def __init__(self, packet, config, transport="aprsis"): + def __init__(self, packet, config): self.packet = packet self.config = config - self.transport = transport name = self.packet["raw"][:10] super().__init__(f"RX_PACKET-{name}") @@ -254,7 +256,6 @@ class APRSDProcessPacketThread(APRSDThread): self.config["aprs"]["login"], fromcall, msg_id=msg_id, - transport=self.transport, ) ack.send() @@ -275,7 +276,6 @@ class APRSDProcessPacketThread(APRSDThread): self.config["aprs"]["login"], fromcall, subreply, - transport=self.transport, ) msg.send() elif isinstance(reply, messaging.Message): @@ -296,7 +296,6 @@ class APRSDProcessPacketThread(APRSDThread): self.config["aprs"]["login"], fromcall, reply, - transport=self.transport, ) msg.send() @@ -309,7 +308,6 @@ class APRSDProcessPacketThread(APRSDThread): self.config["aprs"]["login"], fromcall, reply, - transport=self.transport, ) msg.send() except Exception as ex: @@ -321,88 +319,7 @@ class APRSDProcessPacketThread(APRSDThread): self.config["aprs"]["login"], fromcall, reply, - transport=self.transport, ) msg.send() LOG.debug("Packet processing complete") - - -class APRSDTXThread(APRSDThread): - def __init__(self, msg_queues, config): - super().__init__("TX_MSG") - self.msg_queues = msg_queues - self.config = config - - def loop(self): - try: - msg = self.msg_queues["tx"].get(timeout=1) - msg.send() - except queue.Empty: - pass - # Continue to loop - return True - - -class KISSRXThread(APRSDThread): - """Thread that connects to direwolf's TCPKISS interface. - - All Packets are processed and sent back out the direwolf - interface instead of the aprs-is server. - - """ - - def __init__(self, msg_queues, config): - super().__init__("KISSRX_MSG") - self.msg_queues = msg_queues - self.config = config - - def stop(self): - self.thread_stop = True - kissclient.get_client().stop() - - def loop(self): - kiss_client = kissclient.get_client() - - # setup the consumer of messages and block until a messages - try: - # This will register a packet consumer with aprslib - # When new packets come in the consumer will process - # the packet - - # Do a partial here because the consumer signature doesn't allow - # For kwargs to be passed in to the consumer func we declare - # and the aprslib developer didn't want to allow a PR to add - # kwargs. :( - # https://github.com/rossengeorgiev/aprs-python/pull/56 - kiss_client.consumer(self.process_packet, callsign=self.config["kiss"]["callsign"]) - kiss_client.loop.run_forever() - - except aprslib.exceptions.ConnectionDrop: - LOG.error("Connection dropped, reconnecting") - time.sleep(5) - # Force the deletion of the client object connected to aprs - # This will cause a reconnect, next time client.get_client() - # is called - client.Client().reset() - # Continue to loop - - def process_packet(self, interface, frame): - """Process a packet recieved from aprs-is server.""" - - LOG.debug(f"Got an APRS Frame '{frame}'") - # try and nuke the * from the fromcall sign. - frame.header._source._ch = False - payload = str(frame.payload.decode()) - msg = f"{str(frame.header)}:{payload}" - # msg = frame.tnc2 - LOG.debug(f"Decoding {msg}") - - packet = aprslib.parse(msg) - LOG.debug(packet) - thread = APRSDProcessPacketThread( - packet=packet, config=self.config, - transport=messaging.MESSAGE_TRANSPORT_TCPKISS, - ) - thread.start() - return From a6ed7b894b07d30a26394c3325af68ecca0afcd9 Mon Sep 17 00:00:00 2001 From: Hemna Date: Mon, 4 Oct 2021 11:36:13 -0400 Subject: [PATCH 03/12] Fixed email plugin's use of globals The email plugin was still using globals for tracking the check_email_delay as well as the config. This patch creates a new singleton thread safe mechanism for check_email_delay with the EmailInfo class. --- aprsd/plugins/email.py | 136 +++++++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 54 deletions(-) diff --git a/aprsd/plugins/email.py b/aprsd/plugins/email.py index 7784c11..cc84cb9 100644 --- a/aprsd/plugins/email.py +++ b/aprsd/plugins/email.py @@ -5,6 +5,7 @@ import imaplib import logging import re import smtplib +import threading import time import imapclient @@ -15,9 +16,45 @@ from aprsd import messaging, plugin, stats, threads, trace LOG = logging.getLogger("APRSD") -# This gets forced set from main.py prior to being used internally -CONFIG = {} -check_email_delay = 60 + +class EmailInfo: + """A singleton thread safe mechanism for the global check_email_delay. + + This has to be done because we have 2 separate threads that access + the delay value. + 1) when EmailPlugin runs from a user message and + 2) when the background EmailThread runs to check email. + + Access the check email delay with + EmailInfo().delay + + Set it with + EmailInfo().delay = 100 + or + EmailInfo().delay += 10 + + """ + + _instance = None + + def __new__(cls, *args, **kwargs): + """This magic turns this into a singleton.""" + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance.lock = threading.Lock() + cls._instance._delay = 60 + return cls._instance + + @property + def delay(self): + with self.lock: + return self._delay + + @delay.setter + def delay(self, val): + with self.lock: + self._delay = val + class EmailPlugin(plugin.APRSDRegexCommandPluginBase): @@ -34,8 +71,6 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): def setup(self): """Ensure that email is enabled and start the thread.""" - global CONFIG - CONFIG = self.config email_enabled = self.config["aprsd"]["email"].get("enabled", False) validation = self.config["aprsd"]["email"].get("validate", False) @@ -81,7 +116,7 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): r = re.search("^-([0-9])[0-9]*$", message) if r is not None: LOG.debug("RESEND EMAIL") - resend_email(r.group(1), fromcall) + resend_email(self.config, r.group(1), fromcall) reply = messaging.NULL_MESSAGE # -user@address.com body of email elif re.search(r"^-([A-Za-z0-9_\-\.@]+) (.*)", message): @@ -91,7 +126,7 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): to_addr = a.group(1) content = a.group(2) - email_address = get_email_from_shortcut(to_addr) + email_address = get_email_from_shortcut(self.config, to_addr) if not email_address: reply = "Bad email address" return reply @@ -114,7 +149,7 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): too_soon = 1 if not too_soon or ack == 0: LOG.info(f"Send email '{content}'") - send_result = email.send_email(to_addr, content) + send_result = send_email(self.config, to_addr, content) reply = messaging.NULL_MESSAGE if send_result != 0: reply = f"-{to_addr} failed" @@ -143,10 +178,9 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): return reply -def _imap_connect(): - global CONFIG - imap_port = CONFIG["aprsd"]["email"]["imap"].get("port", 143) - use_ssl = CONFIG["aprsd"]["email"]["imap"].get("use_ssl", False) +def _imap_connect(config): + imap_port = config["aprsd"]["email"]["imap"].get("port", 143) + use_ssl = config["aprsd"]["email"]["imap"].get("use_ssl", False) # host = CONFIG["aprsd"]["email"]["imap"]["host"] # msg = "{}{}:{}".format("TLS " if use_ssl else "", host, imap_port) # LOG.debug("Connect to IMAP host {} with user '{}'". @@ -154,7 +188,7 @@ def _imap_connect(): try: server = imapclient.IMAPClient( - CONFIG["aprsd"]["email"]["imap"]["host"], + config["aprsd"]["email"]["imap"]["host"], port=imap_port, use_uid=True, ssl=use_ssl, @@ -166,8 +200,8 @@ def _imap_connect(): try: server.login( - CONFIG["aprsd"]["email"]["imap"]["login"], - CONFIG["aprsd"]["email"]["imap"]["password"], + config["aprsd"]["email"]["imap"]["login"], + config["aprsd"]["email"]["imap"]["password"], ) except (imaplib.IMAP4.error, Exception) as e: msg = getattr(e, "message", repr(e)) @@ -183,15 +217,15 @@ def _imap_connect(): return server -def _smtp_connect(): - host = CONFIG["aprsd"]["email"]["smtp"]["host"] - smtp_port = CONFIG["aprsd"]["email"]["smtp"]["port"] - use_ssl = CONFIG["aprsd"]["email"]["smtp"].get("use_ssl", False) +def _smtp_connect(config): + host = config["aprsd"]["email"]["smtp"]["host"] + smtp_port = config["aprsd"]["email"]["smtp"]["port"] + use_ssl = config["aprsd"]["email"]["smtp"].get("use_ssl", False) msg = "{}{}:{}".format("SSL " if use_ssl else "", host, smtp_port) LOG.debug( "Connect to SMTP host {} with user '{}'".format( msg, - CONFIG["aprsd"]["email"]["imap"]["login"], + config["aprsd"]["email"]["imap"]["login"], ), ) @@ -214,15 +248,15 @@ def _smtp_connect(): LOG.debug(f"Connected to smtp host {msg}") - debug = CONFIG["aprsd"]["email"]["smtp"].get("debug", False) + debug = config["aprsd"]["email"]["smtp"].get("debug", False) if debug: server.set_debuglevel(5) server.sendmail = trace.trace(server.sendmail) try: server.login( - CONFIG["aprsd"]["email"]["smtp"]["login"], - CONFIG["aprsd"]["email"]["smtp"]["password"], + config["aprsd"]["email"]["smtp"]["login"], + config["aprsd"]["email"]["smtp"]["password"], ) except Exception: LOG.error("Couldn't connect to SMTP Server") @@ -273,9 +307,9 @@ def validate_shortcuts(config): ) -def get_email_from_shortcut(addr): - if CONFIG["aprsd"]["email"].get("shortcuts", False): - return CONFIG["aprsd"]["email"]["shortcuts"].get(addr, addr) +def get_email_from_shortcut(config, addr): + if config["aprsd"]["email"].get("shortcuts", False): + return config["aprsd"]["email"]["shortcuts"].get(addr, addr) else: return addr @@ -286,9 +320,9 @@ def validate_email_config(config, disable_validation=False): This helps with failing early during startup. """ LOG.info("Checking IMAP configuration") - imap_server = _imap_connect() + imap_server = _imap_connect(config) LOG.info("Checking SMTP configuration") - smtp_server = _smtp_connect() + smtp_server = _smtp_connect(config) # Now validate and flag any shortcuts as invalid if not disable_validation: @@ -398,34 +432,32 @@ def parse_email(msgid, data, server): @trace.trace -def send_email(to_addr, content): - global check_email_delay - - shortcuts = CONFIG["aprsd"]["email"]["shortcuts"] - email_address = get_email_from_shortcut(to_addr) +def send_email(config, to_addr, content): + shortcuts = config["aprsd"]["email"]["shortcuts"] + email_address = get_email_from_shortcut(config, to_addr) LOG.info("Sending Email_________________") if to_addr in shortcuts: LOG.info("To : " + to_addr) to_addr = email_address LOG.info(" (" + to_addr + ")") - subject = CONFIG["ham"]["callsign"] + subject = config["ham"]["callsign"] # content = content + "\n\n(NOTE: reply with one line)" LOG.info("Subject : " + subject) LOG.info("Body : " + content) # check email more often since there's activity right now - check_email_delay = 60 + EmailInfo().delay = 60 msg = MIMEText(content) msg["Subject"] = subject - msg["From"] = CONFIG["aprsd"]["email"]["smtp"]["login"] + msg["From"] = config["aprsd"]["email"]["smtp"]["login"] msg["To"] = to_addr server = _smtp_connect() if server: try: server.sendmail( - CONFIG["aprsd"]["email"]["smtp"]["login"], + config["aprsd"]["email"]["smtp"]["login"], [to_addr], msg.as_string(), ) @@ -440,20 +472,19 @@ def send_email(to_addr, content): @trace.trace -def resend_email(count, fromcall): - global check_email_delay +def resend_email(config, count, fromcall): date = datetime.datetime.now() month = date.strftime("%B")[:3] # Nov, Mar, Apr day = date.day year = date.year today = f"{day}-{month}-{year}" - shortcuts = CONFIG["aprsd"]["email"]["shortcuts"] + shortcuts = config["aprsd"]["email"]["shortcuts"] # swap key/value shortcuts_inverted = {v: k for k, v in shortcuts.items()} try: - server = _imap_connect() + server = _imap_connect(config) except Exception as e: LOG.exception("Failed to Connect to IMAP. Cannot resend email ", e) return @@ -493,7 +524,7 @@ def resend_email(count, fromcall): reply = "-" + from_addr + " * " + body.decode(errors="ignore") # messaging.send_message(fromcall, reply) msg = messaging.TextMessage( - CONFIG["aprs"]["login"], + config["aprs"]["login"], fromcall, reply, ) @@ -515,11 +546,11 @@ def resend_email(count, fromcall): str(s).zfill(2), ) # messaging.send_message(fromcall, reply) - msg = messaging.TextMessage(CONFIG["aprs"]["login"], fromcall, reply) + msg = messaging.TextMessage(config["aprs"]["login"], fromcall, reply) msg.send() # check email more often since we're resending one now - check_email_delay = 60 + EmailInfo().delay = 60 server.logout() # end resend_email() @@ -533,27 +564,24 @@ class APRSDEmailThread(threads.APRSDThread): self.past = datetime.datetime.now() def loop(self): - global check_email_delay - - check_email_delay = 60 time.sleep(5) stats.APRSDStats().email_thread_update() # always sleep for 5 seconds and see if we need to check email # This allows CTRL-C to stop the execution of this loop sooner # than check_email_delay time now = datetime.datetime.now() - if now - self.past > datetime.timedelta(seconds=check_email_delay): + if now - self.past > datetime.timedelta(seconds=EmailInfo().delay): # It's time to check email # slowly increase delay every iteration, max out at 300 seconds # any send/receive/resend activity will reset this to 60 seconds - if check_email_delay < 300: - check_email_delay += 1 + if EmailInfo().delay < 300: + EmailInfo().delay += 1 LOG.debug( - "check_email_delay is " + str(check_email_delay) + " seconds", + f"check_email_delay is {EmailInfo().delay} seconds ", ) - shortcuts = CONFIG["aprsd"]["email"]["shortcuts"] + shortcuts = self.config["aprsd"]["email"]["shortcuts"] # swap key/value shortcuts_inverted = {v: k for k, v in shortcuts.items()} @@ -564,7 +592,7 @@ class APRSDEmailThread(threads.APRSDThread): today = f"{day}-{month}-{year}" try: - server = _imap_connect() + server = _imap_connect(self.config) except Exception as e: LOG.exception("IMAP failed to connect.", e) return True @@ -658,7 +686,7 @@ class APRSDEmailThread(threads.APRSDThread): LOG.exception("Couldn't remove seen flag from email", e) # check email more often since we just received an email - check_email_delay = 60 + EmailInfo().delay = 60 # reset clock LOG.debug("Done looping over Server.fetch, logging out.") From 491644ece68f251c939fe43483bd39647b4cfa10 Mon Sep 17 00:00:00 2001 From: Hemna Date: Mon, 4 Oct 2021 15:22:10 -0400 Subject: [PATCH 04/12] Added new Config object. The config object now has builtin dot notation getter with default config.get("some.path.here", default="Not found") --- aprsd/client.py | 20 +++---- aprsd/config.py | 108 +++++++++++++++++++++++--------------- aprsd/main.py | 13 ++--- aprsd/plugins/email.py | 1 - aprsd/plugins/location.py | 4 +- tests/test_email.py | 16 +++--- 6 files changed, 86 insertions(+), 76 deletions(-) diff --git a/aprsd/client.py b/aprsd/client.py index 3b7ee0e..75eff1f 100644 --- a/aprsd/client.py +++ b/aprsd/client.py @@ -125,23 +125,19 @@ class KISSClient(Client): if "kiss" not in config: return False - if "serial" in config["kiss"]: - if config["kiss"]["serial"].get("enabled", False): - return True + if config.get("kiss.serial.enabled", default=False): + return True - if "tcp" in config["kiss"]: - if config["kiss"]["tcp"].get("enabled", False): - return True + if config.get("kiss.tcp.enabled", default=False): + return True @staticmethod def transport(config): - if "serial" in config["kiss"]: - if config["kiss"]["serial"].get("enabled", False): - return TRANSPORT_SERIALKISS + if config.get("kiss.serial.enabled", default=False): + return TRANSPORT_SERIALKISS - if "tcp" in config["kiss"]: - if config["kiss"]["tcp"].get("enabled", False): - return TRANSPORT_TCPKISS + if config.get("kiss.tcp.enabled", default=False): + return TRANSPORT_TCPKISS def decode_packet(self, *args, **kwargs): """We get a frame, which has to be decoded.""" diff --git a/aprsd/config.py b/aprsd/config.py index 1461c2f..eb43fb5 100644 --- a/aprsd/config.py +++ b/aprsd/config.py @@ -1,3 +1,4 @@ +import collections import logging import os from pathlib import Path @@ -134,6 +135,60 @@ DEFAULT_SAVE_FILE = f"{home}/.config/aprsd/aprsd.p" DEFAULT_CONFIG_FILE = f"{home}/.config/aprsd/aprsd.yml" +class Config(collections.UserDict): + def _get(self, d, keys, default=None): + """ + Example: + d = {'meta': {'status': 'OK', 'status_code': 200}} + deep_get(d, ['meta', 'status_code']) # => 200 + deep_get(d, ['garbage', 'status_code']) # => None + deep_get(d, ['meta', 'garbage'], default='-') # => '-' + + """ + if type(keys) is str and "." in keys: + keys = keys.split(".") + + assert type(keys) is list + if d is None: + return default + + if not keys: + return d + + if type(d) is str: + return default + + return self._get(d.get(keys[0]), keys[1:], default) + + def get(self, path, default=None): + return self._get(self.data, path, default=default) + + def exists(self, path): + """See if a conf value exists.""" + test = "-3.14TEST41.3-" + return (self.get(path, default=test) != test) + + def check_option(self, path, default_fail=None): + """Make sure the config option doesn't have default value.""" + if not self.exists(path): + raise Exception( + "Option '{}' was not in config file".format( + path, + ), + ) + + val = self.get(path) + if val == default_fail: + # We have to fail and bail if the user hasn't edited + # this config option. + raise Exception( + "Config file needs to be changed from provided" + " defaults for '{}'".format( + path, + ), + ) + + def add_config_comments(raw_yaml): end_idx = utils.end_substr(raw_yaml, "aprs:") if end_idx != -1: @@ -223,7 +278,7 @@ def get_config(config_file): if os.path.exists(config_file_expanded): with open(config_file_expanded) as stream: config = yaml.load(stream, Loader=yaml.FullLoader) - return config + return Config(config) else: if config_file == DEFAULT_CONFIG_FILE: click.echo( @@ -249,33 +304,28 @@ def get_config(config_file): # If the required params don't exist, # it will look in the environment def parse_config(config_file): - # for now we still use globals....ugh - global CONFIG + config = get_config(config_file) def fail(msg): click.echo(msg) sys.exit(-1) - def check_option(config, chain, default_fail=None): + def check_option(config, path, default_fail=None): try: - config = check_config_option(config, chain, default_fail=default_fail) + config.check_option(path, default_fail=default_fail) except Exception as ex: fail(repr(ex)) else: return config - config = get_config(config_file) - # special check here to make sure user has edited the config file # and changed the ham callsign check_option( config, - [ - "ham", - "callsign", - ], + "ham.callsign", default_fail=DEFAULT_CONFIG_DICT["ham"]["callsign"], ) + check_option( config, ["services", "aprs.fi", "apiKey"], @@ -283,7 +333,7 @@ def parse_config(config_file): ) check_option( config, - ["aprs", "login"], + "aprs.login", default_fail=DEFAULT_CONFIG_DICT["aprs"]["login"], ) check_option( @@ -293,21 +343,21 @@ def parse_config(config_file): ) # Ensure they change the admin password - if config["aprsd"]["web"]["enabled"] is True: + if config.get("aprsd.web.enabled") is True: check_option( config, ["aprsd", "web", "users", "admin"], default_fail=DEFAULT_CONFIG_DICT["aprsd"]["web"]["users"]["admin"], ) - if config["aprsd"]["watch_list"]["enabled"] is True: + if config.get("aprsd.watch_list.enabled") is True: check_option( config, ["aprsd", "watch_list", "alert_callsign"], default_fail=DEFAULT_CONFIG_DICT["aprsd"]["watch_list"]["alert_callsign"], ) - if config["aprsd"]["email"]["enabled"] is True: + if config.get("aprsd.email.enabled") is True: # Check IMAP server settings check_option(config, ["aprsd", "email", "imap", "host"]) check_option(config, ["aprsd", "email", "imap", "port"]) @@ -337,31 +387,3 @@ def parse_config(config_file): ) return config - - -def conf_option_exists(conf, chain): - _key = chain.pop(0) - if _key in conf: - return conf_option_exists(conf[_key], chain) if chain else conf[_key] - - -def check_config_option(config, chain, default_fail=None): - result = conf_option_exists(config, chain.copy()) - if result is None: - raise Exception( - "'{}' was not in config file".format( - chain, - ), - ) - else: - if default_fail: - if result == default_fail: - # We have to fail and bail if the user hasn't edited - # this config option. - raise Exception( - "Config file needs to be edited from provided defaults for {}.".format( - chain, - ), - ) - else: - return config diff --git a/aprsd/main.py b/aprsd/main.py index 3bebc80..010d8ad 100644 --- a/aprsd/main.py +++ b/aprsd/main.py @@ -174,20 +174,13 @@ def setup_logging(config, loglevel, quiet): LOG.addHandler(fh) imap_logger = None - if config["aprsd"]["email"].get("enabled", False) and config["aprsd"]["email"][ - "imap" - ].get("debug", False): + if config.get("aprsd.email.enabled", default=False) and config.get("aprsd.email.imap.debug", default=False): imap_logger = logging.getLogger("imapclient.imaplib") imap_logger.setLevel(log_level) imap_logger.addHandler(fh) - if ( - aprsd_config.check_config_option( - config, ["aprsd", "web", "enabled"], - default_fail=False, - ) - ): + if config.get("aprsd.web.enabled", default=False): qh = logging.handlers.QueueHandler(threads.logging_queue) q_log_formatter = logging.Formatter( fmt=aprsd_config.QUEUE_LOG_FORMAT, @@ -498,7 +491,7 @@ def server( keepalive = threads.KeepAliveThread(config=config) keepalive.start() - web_enabled = aprsd_config.check_config_option(config, ["aprsd", "web", "enabled"], default_fail=False) + web_enabled = config.get("aprsd.web.enabled", default=False) if web_enabled: flask_enabled = True diff --git a/aprsd/plugins/email.py b/aprsd/plugins/email.py index cc84cb9..a8a297a 100644 --- a/aprsd/plugins/email.py +++ b/aprsd/plugins/email.py @@ -56,7 +56,6 @@ class EmailInfo: self._delay = val - class EmailPlugin(plugin.APRSDRegexCommandPluginBase): """Email Plugin.""" diff --git a/aprsd/plugins/location.py b/aprsd/plugins/location.py index 80a44e6..6b9835e 100644 --- a/aprsd/plugins/location.py +++ b/aprsd/plugins/location.py @@ -2,7 +2,7 @@ import logging import re import time -from aprsd import config, plugin, plugin_utils, trace +from aprsd import plugin, plugin_utils, trace LOG = logging.getLogger("APRSD") @@ -24,7 +24,7 @@ class LocationPlugin(plugin.APRSDRegexCommandPluginBase): # get last location of a callsign, get descriptive name from weather service try: - config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + self.config.check_option(["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" diff --git a/tests/test_email.py b/tests/test_email.py index 6a4532f..4cd1528 100644 --- a/tests/test_email.py +++ b/tests/test_email.py @@ -5,21 +5,21 @@ from aprsd.plugins import email class TestEmail(unittest.TestCase): def test_get_email_from_shortcut(self): - email.CONFIG = {"aprsd": {"email": {"shortcuts": {}}}} + config = {"aprsd": {"email": {"shortcuts": {}}}} email_address = "something@something.com" addr = f"-{email_address}" - actual = email.get_email_from_shortcut(addr) + actual = email.get_email_from_shortcut(config, addr) self.assertEqual(addr, actual) - email.CONFIG = {"aprsd": {"email": {"nothing": "nothing"}}} - actual = email.get_email_from_shortcut(addr) + config = {"aprsd": {"email": {"nothing": "nothing"}}} + actual = email.get_email_from_shortcut(config, addr) self.assertEqual(addr, actual) - email.CONFIG = {"aprsd": {"email": {"shortcuts": {"not_used": "empty"}}}} - actual = email.get_email_from_shortcut(addr) + config = {"aprsd": {"email": {"shortcuts": {"not_used": "empty"}}}} + actual = email.get_email_from_shortcut(config, addr) self.assertEqual(addr, actual) - email.CONFIG = {"aprsd": {"email": {"shortcuts": {"-wb": email_address}}}} + config = {"aprsd": {"email": {"shortcuts": {"-wb": email_address}}}} short = "-wb" - actual = email.get_email_from_shortcut(short) + actual = email.get_email_from_shortcut(config, short) self.assertEqual(email_address, actual) From e3c5c7b4081eb2c6403195f756483f1f8a093cec Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 6 Oct 2021 12:08:29 -0400 Subject: [PATCH 05/12] Fixed notify plugins The notify base filter() was missing the @hookimpl --- aprsd/plugin.py | 49 ++++++++++++++++++++++++----------------- aprsd/plugins/notify.py | 6 +---- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/aprsd/plugin.py b/aprsd/plugin.py index d7fb0bc..2e95020 100644 --- a/aprsd/plugin.py +++ b/aprsd/plugin.py @@ -17,9 +17,6 @@ from aprsd import client, messaging, packets, threads # setup the global logger LOG = logging.getLogger("APRSD") -hookspec = pluggy.HookspecMarker("aprsd") -hookimpl = pluggy.HookimplMarker("aprsd") - CORE_MESSAGE_PLUGINS = [ "aprsd.plugins.email.EmailPlugin", "aprsd.plugins.fortune.FortunePlugin", @@ -36,8 +33,11 @@ CORE_NOTIFY_PLUGINS = [ "aprsd.plugins.notify.NotifySeenPlugin", ] +hookspec = pluggy.HookspecMarker("aprsd") +hookimpl = pluggy.HookimplMarker("aprsd") -class APRSDCommandSpec: + +class APRSDPluginSpec: """A hook specification namespace.""" @hookspec @@ -62,11 +62,8 @@ class APRSDPluginBase(metaclass=abc.ABCMeta): self.config = config self.message_counter = 0 self.setup() - threads = self.create_threads() - if threads: - self.threads = threads - if self.threads: - self.start_threads() + self.threads = self.create_threads() + self.start_threads() def start_threads(self): if self.enabled and self.threads: @@ -96,11 +93,6 @@ class APRSDPluginBase(metaclass=abc.ABCMeta): def message_count(self): return self.message_counter - @property - def version(self): - """Version""" - raise NotImplementedError - @abc.abstractmethod def setup(self): """Do any plugin setup here.""" @@ -122,7 +114,6 @@ class APRSDPluginBase(metaclass=abc.ABCMeta): if isinstance(thread, threads.APRSDThread): thread.stop() - @hookimpl @abc.abstractmethod def filter(self, packet): pass @@ -164,14 +155,22 @@ class APRSDWatchListPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): else: LOG.warning("Watch list enabled, but no callsigns set.") + @hookimpl def filter(self, packet): + result = messaging.NULL_MESSAGE if self.enabled: wl = packets.WatchList() - result = messaging.NULL_MESSAGE if wl.callsign_in_watchlist(packet["from"]): # packet is from a callsign in the watch list self.rx_inc() - result = self.process() + try: + result = self.process(packet) + except Exception as ex: + LOG.error( + "Plugin {} failed to process packet {}".format( + self.__class__, ex, + ), + ) if result: self.tx_inc() wl.update_seen(packet) @@ -221,7 +220,14 @@ class APRSDRegexCommandPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): if re.search(self.command_regex, message): self.rx_inc() if self.enabled: - result = self.process(packet) + try: + result = self.process(packet) + except Exception as ex: + LOG.error( + "Plugin {} failed to process packet {}".format( + self.__class__, ex, + ), + ) if result: self.tx_inc() else: @@ -255,6 +261,10 @@ class PluginManager: if config: self.config = config + def _init(self): + self._pluggy_pm = pluggy.PluginManager("aprsd") + self._pluggy_pm.add_hookspecs(APRSDPluginSpec) + def load_plugins_from_path(self, module_path): if not os.path.exists(module_path): LOG.error(f"plugin path '{module_path}' doesn't exist.") @@ -356,8 +366,7 @@ class PluginManager: LOG.info("Loading APRSD Plugins") enabled_plugins = self.config["aprsd"].get("enabled_plugins", None) - self._pluggy_pm = pluggy.PluginManager("aprsd") - self._pluggy_pm.add_hookspecs(APRSDCommandSpec) + self._init() if enabled_plugins: for p_name in enabled_plugins: self._load_plugin(p_name) diff --git a/aprsd/plugins/notify.py b/aprsd/plugins/notify.py index c3806fa..8a31563 100644 --- a/aprsd/plugins/notify.py +++ b/aprsd/plugins/notify.py @@ -17,12 +17,8 @@ class NotifySeenPlugin(plugin.APRSDWatchListPluginBase): version = "1.0" - def __init__(self, config): - """The aprsd config object is stored.""" - super().__init__(config) - def process(self, packet): - LOG.info("BaseNotifyPlugin") + LOG.info("NotifySeenPlugin") notify_callsign = self.config["aprsd"]["watch_list"]["alert_callsign"] fromcall = packet.get("from") From c097c31258ecf8d2c23193b8045d0b12b227b2a1 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 6 Oct 2021 12:09:52 -0400 Subject: [PATCH 06/12] Updated dev to use plugin manager Also ensure that main creates the client prior to starting the plugins. --- aprsd/dev.py | 4 +++- aprsd/main.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/aprsd/dev.py b/aprsd/dev.py index 01eecc1..23fb608 100644 --- a/aprsd/dev.py +++ b/aprsd/dev.py @@ -190,7 +190,9 @@ def test_plugin( client.Client(config) pm = plugin.PluginManager(config) + pm._init() obj = pm._create_class(plugin_path, plugin.APRSDPluginBase, config=config) + pm._pluggy_pm.register(obj) login = config["aprs"]["login"] packet = { @@ -200,7 +202,7 @@ def test_plugin( "msgNo": 1, } - reply = obj.filter(packet) + reply = pm.run(packet) # Plugin might have threads, so lets stop them so we can exit. obj.stop_threads() LOG.info(f"Result = '{reply}'") diff --git a/aprsd/main.py b/aprsd/main.py index 010d8ad..af88327 100644 --- a/aprsd/main.py +++ b/aprsd/main.py @@ -464,6 +464,9 @@ def server( LOG.error("No Clients are enabled in config.") sys.exit(-1) + # Creates the client object + client.factory.create().client + # Create the initial PM singleton and Register plugins plugin_manager = plugin.PluginManager(config) plugin_manager.setup_plugins() From fdc8c0cd66f6c4662e60b3a7259b792848deb747 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 6 Oct 2021 12:12:49 -0400 Subject: [PATCH 07/12] Increase email delay to +10 This patch updates the increasing of the email check delay to += 10 seconds instead of +1. --- aprsd/plugins/email.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aprsd/plugins/email.py b/aprsd/plugins/email.py index a8a297a..ffd9c32 100644 --- a/aprsd/plugins/email.py +++ b/aprsd/plugins/email.py @@ -575,7 +575,7 @@ class APRSDEmailThread(threads.APRSDThread): # slowly increase delay every iteration, max out at 300 seconds # any send/receive/resend activity will reset this to 60 seconds if EmailInfo().delay < 300: - EmailInfo().delay += 1 + EmailInfo().delay += 10 LOG.debug( f"check_email_delay is {EmailInfo().delay} seconds ", ) From 30671cbdbc6b6f95af0574fe7fdba8d8d6c3474f Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 6 Oct 2021 12:54:29 -0400 Subject: [PATCH 08/12] Added some server startup info logs This patch adds some general info logs around starting the client connection as well as loading the plugins. --- aprsd/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aprsd/main.py b/aprsd/main.py index af88327..3e911da 100644 --- a/aprsd/main.py +++ b/aprsd/main.py @@ -465,9 +465,11 @@ def server( sys.exit(-1) # Creates the client object + LOG.info("Creating client connection") client.factory.create().client # Create the initial PM singleton and Register plugins + LOG.info("Loading Plugin Manager and registering plugins") plugin_manager = plugin.PluginManager(config) plugin_manager.setup_plugins() From f8d87d05bbf5e8f3861c42199907b210c1d9e91b Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 6 Oct 2021 15:17:09 -0400 Subject: [PATCH 09/12] Fixed issue with flask config Flask was trying to serialize the UserDict object. Use the data (dict) inside of it instead. --- aprsd/flask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aprsd/flask.py b/aprsd/flask.py index ce5ab36..bf1d9bc 100644 --- a/aprsd/flask.py +++ b/aprsd/flask.py @@ -340,7 +340,7 @@ class APRSDFlask(flask_classful.FlaskView): aprs_connection=aprs_connection, callsign=self.config["aprs"]["login"], version=aprsd.__version__, - config_json=json.dumps(self.config), + config_json=json.dumps(self.config.data), watch_count=watch_count, watch_age=watch_age, plugin_count=plugin_count, From 725bb2fe35d6adf0cd56e2a866bee0233c46534a Mon Sep 17 00:00:00 2001 From: Hemna Date: Thu, 7 Oct 2021 10:05:19 -0400 Subject: [PATCH 10/12] Fixed send-message after config/client rework This patch fixes the send-message from the command line ability after the complete rework of the client classes. --- aprsd/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aprsd/main.py b/aprsd/main.py index 3e911da..fea3d84 100644 --- a/aprsd/main.py +++ b/aprsd/main.py @@ -351,8 +351,8 @@ def send_message( sys.exit(0) try: - cl = client.Client(config) - cl.setup_connection() + client.ClientFactory.setup(config) + client.factory.create().client except LoginError: sys.exit(-1) @@ -379,7 +379,7 @@ def send_message( # This will register a packet consumer with aprslib # When new packets come in the consumer will process # the packet - aprs_client = client.get_client() + aprs_client = client.factory.create().client aprs_client.consumer(rx_packet, raw=False) except aprslib.exceptions.ConnectionDrop: LOG.error("Connection dropped, reconnecting") @@ -387,7 +387,7 @@ def send_message( # Force the deletion of the client object connected to aprs # This will cause a reconnect, next time client.get_client() # is called - cl.reset() + aprs_client.reset() # main() ### From 5b17228811d303c62fe82d85a4ddf14e15d87934 Mon Sep 17 00:00:00 2001 From: Hemna Date: Thu, 7 Oct 2021 10:11:48 -0400 Subject: [PATCH 11/12] removed usage of config.check_config_option check_config_option has been superceeded by the config UserDict object's ability to see if a config option exists. --- aprsd/plugins/time.py | 11 +++++------ aprsd/plugins/weather.py | 24 +++++++++--------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/aprsd/plugins/time.py b/aprsd/plugins/time.py index 516643a..ad660ac 100644 --- a/aprsd/plugins/time.py +++ b/aprsd/plugins/time.py @@ -5,7 +5,7 @@ import time from opencage.geocoder import OpenCageGeocode import pytz -from aprsd import config, fuzzyclock, plugin, plugin_utils, trace +from aprsd import fuzzyclock, plugin, plugin_utils, trace LOG = logging.getLogger("APRSD") @@ -64,7 +64,7 @@ class TimeOpenCageDataPlugin(TimePlugin): # get last location of a callsign, get descriptive name from weather service try: - config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + self.config.exists(["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" @@ -95,7 +95,7 @@ class TimeOpenCageDataPlugin(TimePlugin): lon = aprs_data["entries"][0]["lng"] try: - config.check_config_option(self.config, "opencagedata", "apiKey") + self.config.exists("opencagedata.apiKey") except Exception as ex: LOG.error(f"Failed to find config opencage:apiKey {ex}") return "No opencage apiKey found" @@ -130,7 +130,7 @@ class TimeOWMPlugin(TimePlugin): # get last location of a callsign, get descriptive name from weather service try: - config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + self.config.exists(["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" @@ -160,8 +160,7 @@ class TimeOWMPlugin(TimePlugin): lon = aprs_data["entries"][0]["lng"] try: - config.check_config_option( - self.config, + self.config.exists( ["services", "openweathermap", "apiKey"], ) except Exception as ex: diff --git a/aprsd/plugins/weather.py b/aprsd/plugins/weather.py index a1dc2cf..21364ab 100644 --- a/aprsd/plugins/weather.py +++ b/aprsd/plugins/weather.py @@ -4,7 +4,7 @@ import re import requests -from aprsd import config, plugin, plugin_utils, trace +from aprsd import plugin, plugin_utils, trace LOG = logging.getLogger("APRSD") @@ -34,7 +34,7 @@ class USWeatherPlugin(plugin.APRSDRegexCommandPluginBase): # message = packet.get("message_text", None) # ack = packet.get("msgNo", "0") try: - config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + self.config.exists(["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" @@ -115,10 +115,7 @@ class USMetarPlugin(plugin.APRSDRegexCommandPluginBase): fromcall = fromcall try: - config.check_config_option( - self.config, - ["services", "aprs.fi", "apiKey"], - ) + self.config.exists(["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" @@ -199,7 +196,7 @@ class OWMWeatherPlugin(plugin.APRSDRegexCommandPluginBase): searchcall = fromcall try: - config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + self.config.exists(["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" @@ -220,16 +217,13 @@ class OWMWeatherPlugin(plugin.APRSDRegexCommandPluginBase): lon = aprs_data["entries"][0]["lng"] try: - config.check_config_option( - self.config, - ["services", "openweathermap", "apiKey"], - ) + self.config.exists(["services", "openweathermap", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config openweathermap:apiKey {ex}") return "No openweathermap apiKey found" try: - config.check_config_option(self.config, ["aprsd", "units"]) + self.config.exists(["aprsd", "units"]) except Exception: LOG.debug("Couldn't find untis in aprsd:services:units") units = "metric" @@ -323,7 +317,7 @@ class AVWXWeatherPlugin(plugin.APRSDRegexCommandPluginBase): searchcall = fromcall try: - config.check_config_option(self.config, ["services", "aprs.fi", "apiKey"]) + self.config.exists(["services", "aprs.fi", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config aprs.fi:apikey {ex}") return "No aprs.fi apikey found" @@ -344,13 +338,13 @@ class AVWXWeatherPlugin(plugin.APRSDRegexCommandPluginBase): lon = aprs_data["entries"][0]["lng"] try: - config.check_config_option(self.config, ["services", "avwx", "apiKey"]) + self.config.exists(["services", "avwx", "apiKey"]) except Exception as ex: LOG.error(f"Failed to find config avwx:apiKey {ex}") return "No avwx apiKey found" try: - config.check_config_option(self.config, ["services", "avwx", "base_url"]) + self.config.exists(self.config, ["services", "avwx", "base_url"]) except Exception as ex: LOG.debug(f"Didn't find avwx:base_url {ex}") base_url = "https://avwx.rest" From db9cbf51dfff9cee20bd545b06c69375e6e76589 Mon Sep 17 00:00:00 2001 From: Hemna Date: Thu, 7 Oct 2021 10:56:09 -0400 Subject: [PATCH 12/12] Updated build.sh This patch forces the rebuild of the docker buildx build container. Also makes the tag, version available from cmdln --- docker/Dockerfile-dev | 4 ++-- docker/build.sh | 26 +++++++++++++++++++------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile-dev b/docker/Dockerfile-dev index 171d006..2f0937e 100644 --- a/docker/Dockerfile-dev +++ b/docker/Dockerfile-dev @@ -1,14 +1,14 @@ FROM python:3.8-slim as aprsd # Dockerfile for building a container during aprsd development. -ARG BRANCH +ARG branch ARG UID ARG GID ENV APRS_USER=aprs ENV HOME=/home/aprs ENV APRSD=http://github.com/craigerl/aprsd.git -ENV APRSD_BRANCH=${BRANCH:-master} +ENV APRSD_BRANCH=${branch:-master} ENV VIRTUAL_ENV=$HOME/.venv3 ENV UID=${UID:-1000} ENV GID=${GID:-1000} diff --git a/docker/build.sh b/docker/build.sh index 6ab4edf..68f6dbc 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -15,14 +15,18 @@ EOF ALL_PLATFORMS=0 DEV=0 -TAG="master" +TAG="latest" +BRANCH="master" -while getopts “t:da” OPTION +while getopts “t:dab:” OPTION do case $OPTION in t) TAG=$OPTARG ;; + b) + BRANCH=$OPTARG + ;; a) ALL_PLATFORMS=1 ;; @@ -45,20 +49,28 @@ else PLATFORMS="linux/amd64" fi +echo "Build with tag=${TAG} BRANCH=${BRANCH} dev?=${DEV} platforms?=${PLATFORMS}" + + +echo "Destroying old multiarch build container" +docker buildx rm multiarch +echo "Creating new buildx container" +docker buildx create --name multiarch --platform linux/arm/v7,linux/arm/v6,linux/arm64,linux/amd64 --config ./buildkit.toml --use --driver-opt image=moby/buildkit:master + if [ $DEV -eq 1 ] then + echo "Build -DEV- with tag=${TAG} BRANCH=${BRANCH} platforms?=${PLATFORMS}" # Use this script to locally build the docker image docker buildx build --push --platform $PLATFORMS \ -t harbor.hemna.com/hemna6969/aprsd:$TAG \ - -f Dockerfile-dev --no-cache . + -f Dockerfile-dev --build-arg branch=$BRANCH --no-cache . else # Use this script to locally build the docker image + echo "Build with tag=${TAG} BRANCH=${BRANCH} platforms?=${PLATFORMS}" docker buildx build --push --platform $PLATFORMS \ -t hemna6969/aprsd:$VERSION \ - -t hemna6969/aprsd:latest \ - -t harbor.hemna.com/hemna6969/aprsd:latest \ + -t hemna6969/aprsd:$TAG \ + -t harbor.hemna.com/hemna6969/aprsd:$TAG \ -t harbor.hemna.com/hemna6969/aprsd:$VERSION \ -f Dockerfile . - - fi