Added basic service abstraction for weather

Since there are many weather services that provide an API
for fetching weather, and some don't work in other countries,
this patch adds a new service abstraction for weather.

the user configures which weather service they want to use in the config
file, then that service is loaded at start time, and the weather plugin
uses the WeatherService object to fetch the weather for a lat,lon combo.
The WeatherService itself calls the configured service object that
fetches and returns the weather.   There is only 1 weather service
in this patch, which is the same as it used to be.  calling
forecast.weather.gov, which is a US government API.

ChangeLog

@@ -1,9 +1,15 @@
 CHANGES
 =======
 
+v1.5.1
+------
+
+* Updated README to fix pypi page
+
 v1.5.0
 ------
 
+* Updated Changelog for v1.5.0 release
 * Fix tox tests
 * fix usage statement
 * Enabled some emailthread messages and added timestamp

INSTALL.txt

@@ -13,12 +13,12 @@ cd ~/aprsd2
 git clone https://github.com/craigerl/aprsd.git
 cd aprsd
 pip install -e .
-cd ~/.venv_aprsd
+cd ~/.venv_aprsd/bin
 
-./bin/aprsd  # generates a config.yml template on first run
+./aprsd sample-config    # generates a config.yml template
 
-vi ~/.aprsd/config.yml
+vi ~/.config/aprsd/config.yml   # copy/edit config here
 
-./bin/aprsd
+./aprsd server
 
 # profit! #

Makefile

@@ -33,7 +33,7 @@ clean:
 
 test: dev
 	.venv/bin/pre-commit run --all-files
-	tox -p
+	tox -p all
 
 build: test
 	rm -rf dist/*

README.rst

@@ -36,11 +36,13 @@ provide responding to messages to check email, get location, ping,
 time of day, get weather, and fortune telling as well as version information
 of aprsd itself.
 
+Documentation: https://aprsd.readthedocs.io
+
 
 APRSD Overview Diagram
 ----------------------
 
-.. image:: https://raw.githubusercontent.com/craigerl/aprsd/crager-stable/docs/_static/aprsd_overview.svg?sanitize=true
+.. image:: https://raw.githubusercontent.com/craigerl/aprsd/master/docs/_static/aprsd_overview.svg?sanitize=true
 
 
 Typical use case

aprsd/dev.py

@@ -0,0 +1,198 @@
+#
+#  Dev.py is used to help develop plugins
+#
+#
+# python included libs
+import logging
+from logging import NullHandler
+from logging.handlers import RotatingFileHandler
+import os
+import sys
+
+# local imports here
+import aprsd
+from aprsd import client, email, plugin, service, utils
+import click
+import click_completion
+
+# setup the global logger
+# logging.basicConfig(level=logging.DEBUG) # level=10
+LOG = logging.getLogger("APRSD")
+
+LOG_LEVELS = {
+    "CRITICAL": logging.CRITICAL,
+    "ERROR": logging.ERROR,
+    "WARNING": logging.WARNING,
+    "INFO": logging.INFO,
+    "DEBUG": logging.DEBUG,
+}
+
+CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
+
+
+def custom_startswith(string, incomplete):
+    """A custom completion match that supports case insensitive matching."""
+    if os.environ.get("_CLICK_COMPLETION_COMMAND_CASE_INSENSITIVE_COMPLETE"):
+        string = string.lower()
+        incomplete = incomplete.lower()
+    return string.startswith(incomplete)
+
+
+click_completion.core.startswith = custom_startswith
+click_completion.init()
+
+
+cmd_help = """Shell completion for click-completion-command
+Available shell types:
+\b
+  %s
+Default type: auto
+""" % "\n  ".join(
+    "{:<12} {}".format(k, click_completion.core.shells[k])
+    for k in sorted(click_completion.core.shells.keys())
+)
+
+
+@click.group(help=cmd_help, context_settings=CONTEXT_SETTINGS)
+@click.version_option()
+def main():
+    pass
+
+
+@main.command()
+@click.option(
+    "-i",
+    "--case-insensitive/--no-case-insensitive",
+    help="Case insensitive completion",
+)
+@click.argument(
+    "shell",
+    required=False,
+    type=click_completion.DocumentedChoice(click_completion.core.shells),
+)
+def show(shell, case_insensitive):
+    """Show the click-completion-command completion code"""
+    extra_env = (
+        {"_CLICK_COMPLETION_COMMAND_CASE_INSENSITIVE_COMPLETE": "ON"}
+        if case_insensitive
+        else {}
+    )
+    click.echo(click_completion.core.get_code(shell, extra_env=extra_env))
+
+
+@main.command()
+@click.option(
+    "--append/--overwrite",
+    help="Append the completion code to the file",
+    default=None,
+)
+@click.option(
+    "-i",
+    "--case-insensitive/--no-case-insensitive",
+    help="Case insensitive completion",
+)
+@click.argument(
+    "shell",
+    required=False,
+    type=click_completion.DocumentedChoice(click_completion.core.shells),
+)
+@click.argument("path", required=False)
+def install(append, case_insensitive, shell, path):
+    """Install the click-completion-command completion"""
+    extra_env = (
+        {"_CLICK_COMPLETION_COMMAND_CASE_INSENSITIVE_COMPLETE": "ON"}
+        if case_insensitive
+        else {}
+    )
+    shell, path = click_completion.core.install(
+        shell=shell,
+        path=path,
+        append=append,
+        extra_env=extra_env,
+    )
+    click.echo("{} completion installed in {}".format(shell, path))
+
+
+# Setup the logging faciility
+# to disable logging to stdout, but still log to file
+# use the --quiet option on the cmdln
+def setup_logging(config, loglevel, quiet):
+    log_level = LOG_LEVELS[loglevel]
+    LOG.setLevel(log_level)
+    log_format = "[%(asctime)s] [%(threadName)-12s] [%(levelname)-5.5s]" " %(message)s"
+    date_format = "%m/%d/%Y %I:%M:%S %p"
+    log_formatter = logging.Formatter(fmt=log_format, datefmt=date_format)
+    log_file = config["aprs"].get("logfile", None)
+    if log_file:
+        fh = RotatingFileHandler(log_file, maxBytes=(10248576 * 5), backupCount=4)
+    else:
+        fh = NullHandler()
+
+    fh.setFormatter(log_formatter)
+    LOG.addHandler(fh)
+
+    if not quiet:
+        sh = logging.StreamHandler(sys.stdout)
+        sh.setFormatter(log_formatter)
+        LOG.addHandler(sh)
+
+
+@main.command()
+@click.option(
+    "--loglevel",
+    default="DEBUG",
+    show_default=True,
+    type=click.Choice(
+        ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"],
+        case_sensitive=False,
+    ),
+    show_choices=True,
+    help="The log level to use for aprsd.log",
+)
+@click.option(
+    "-c",
+    "--config",
+    "config_file",
+    show_default=True,
+    default=utils.DEFAULT_CONFIG_FILE,
+    help="The aprsd config file to use for options.",
+)
+@click.option(
+    "-p",
+    "--plugin",
+    "plugin_path",
+    show_default=True,
+    default="aprsd.plugins.wx.WxPlugin",
+    help="The plugin to run",
+)
+@click.argument("fromcall")
+@click.argument("message", nargs=-1, required=True)
+def test_plugin(
+    loglevel,
+    config_file,
+    plugin_path,
+    fromcall,
+    message,
+):
+    """APRSD Plugin test app."""
+
+    config = utils.parse_config(config_file)
+    email.CONFIG = config
+
+    setup_logging(config, loglevel, False)
+    LOG.info("Test APRSD PLugin version: {}".format(aprsd.__version__))
+    if type(message) is tuple:
+        message = " ".join(message)
+    LOG.info("P'{}'  F'{}'   C'{}'".format(plugin_path, fromcall, message))
+    client.Client(config)
+    service.WeatherService(config)
+
+    pm = plugin.PluginManager(config)
+    obj = pm._create_class(plugin_path, plugin.APRSDPluginBase, config=config)
+
+    reply = obj.run(fromcall, message, 1)
+    LOG.info("Result = '{}'".format(reply))
+
+
+if __name__ == "__main__":
+    main()

aprsd/main.py

@@ -32,7 +32,7 @@ import time
 
 # local imports here
 import aprsd
-from aprsd import client, email, messaging, plugin, threads, utils
+from aprsd import client, email, messaging, plugin, service, threads, utils
 import aprslib
 from aprslib.exceptions import LoginError
 import click
@@ -443,6 +443,9 @@ def server(
         LOG.debug("Loading saved MsgTrack object.")
         messaging.MsgTrack().load()
 
+    LOG.info("Loading weather service")
+    service.WeatherService(config)
+
     rx_msg_queue = queue.Queue(maxsize=20)
     tx_msg_queue = queue.Queue(maxsize=20)
     msg_queues = {"rx": rx_msg_queue, "tx": tx_msg_queue}

aprsd/plugin.py

@@ -24,6 +24,7 @@ CORE_PLUGINS = [
     "aprsd.plugins.query.QueryPlugin",
     "aprsd.plugins.time.TimePlugin",
     "aprsd.plugins.weather.WeatherPlugin",
+    "aprsd.plugins.weather.WxPlugin",
     "aprsd.plugins.version.VersionPlugin",
 ]
 
@@ -59,7 +60,9 @@ class APRSDPluginBase(metaclass=abc.ABCMeta):
 
     @hookimpl
     def run(self, fromcall, message, ack):
+        LOG.debug("F({}) M({})".format(fromcall, message))
         if re.search(self.command_regex, message):
+            LOG.debug("call command F{} M{}".format(fromcall, message))
             return self.command(fromcall, message, ack)
 
     @abc.abstractmethod

aprsd/plugin_utils.py

@@ -0,0 +1,53 @@
+#  Utilities for plugins to use
+import logging
+
+import requests
+
+LOG = logging.getLogger("APRSD")
+
+
+def get_aprs_fi(api_key, callsign):
+    LOG.info("Fetch aprs.fi location for '{}'".format(callsign))
+    try:
+        url = (
+            "http://api.aprs.fi/api/get?"
+            "&what=loc&apikey={}&format=json"
+            "&name={}".format(api_key, callsign)
+        )
+        response = requests.get(url)
+    except Exception:
+        raise Exception("Failed to get aprs.fi location")
+    else:
+        response.raise_for_status()
+        return response
+
+
+def get_weather_gov_for_gps(lat, lon):
+    LOG.debug("Fetch station at {}, {}".format(lat, lon))
+    try:
+        url2 = (
+            "https://forecast.weather.gov/MapClick.php?lat=%s"
+            "&lon=%s&FcstType=json" % (lat, lon)
+        )
+        LOG.debug("Fetching weather '{}'".format(url2))
+        response = requests.get(url2)
+    except Exception as e:
+        LOG.error(e)
+        raise Exception("Failed to get weather")
+    else:
+        response.raise_for_status()
+        return response
+
+
+def get_weather_gov_metar(station):
+    LOG.debug("Fetch metar for station '{}'".format(station))
+    try:
+        url = "https://api.weather.gov/stations/{}/observations/latest".format(
+            station,
+        )
+        response = requests.get(url)
+    except Exception:
+        raise Exception("Failed to fetch metar")
+    else:
+        response.raise_for_status()
+        return response

aprsd/plugins/fortune.py

@@ -32,6 +32,12 @@ class FortunePlugin(plugin.APRSDPluginBase):
                 timeout=3,
                 universal_newlines=True,
             )
+            output = (
+                output.replace("\r", "")
+                .replace("\n", "")
+                .replace("  ", "")
+                .replace("\t", " ")
+            )
         except subprocess.CalledProcessError as ex:
             reply = "Fortune command failed '{}'".format(ex.output)
         else:

aprsd/plugins/query.py

@@ -11,7 +11,7 @@ class QueryPlugin(plugin.APRSDPluginBase):
     """Query command."""
 
     version = "1.0"
-    command_regex = r"^\?.*"
+    command_regex = r"^\!.*"
     command_name = "query"
 
     def command(self, fromcall, message, ack):
@@ -28,8 +28,8 @@ class QueryPlugin(plugin.APRSDPluginBase):
         # only I can do admin commands
         if re.search(searchstring, fromcall):
 
-            # resend last N most recent:  "?3"
+            # resend last N most recent:  "!3"
-            r = re.search(r"^\?([0-9]).*", message)
+            r = re.search(r"^\!([0-9]).*", message)
             if r is not None:
                 if len(tracker) > 0:
                     last_n = r.group(1)
@@ -41,8 +41,8 @@ class QueryPlugin(plugin.APRSDPluginBase):
                     LOG.debug(reply)
                 return reply
 
-            # resend all:   "?a"
+            # resend all:   "!a"
-            r = re.search(r"^\?[aA].*", message)
+            r = re.search(r"^\![aA].*", message)
             if r is not None:
                 if len(tracker) > 0:
                     reply = messaging.NULL_MESSAGE
@@ -53,8 +53,8 @@ class QueryPlugin(plugin.APRSDPluginBase):
                     LOG.debug(reply)
                 return reply
 
-            # delete all:   "?d"
+            # delete all:   "!d"
-            r = re.search(r"^\?[dD].*", message)
+            r = re.search(r"^\![dD].*", message)
             if r is not None:
                 reply = "Deleted ALL pending msgs."
                 LOG.debug(reply)

aprsd/plugins/weather.py

@@ -1,8 +1,8 @@
 import json
 import logging
+import re
 
-from aprsd import plugin
+from aprsd import plugin, plugin_utils, service
-import requests
 
 LOG = logging.getLogger("APRSD")
 
@@ -17,38 +17,95 @@ class WeatherPlugin(plugin.APRSDPluginBase):
     def command(self, fromcall, message, ack):
         LOG.info("Weather Plugin")
         api_key = self.config["aprs.fi"]["apiKey"]
+
+        # Fetching weather for someone else?
+        a = re.search(r"^.*\s+(.*)", message)
+        if a is not None:
+            searchcall = a.group(1)
+        else:
+            searchcall = fromcall
+
         try:
-            url = (
+            resp = plugin_utils.get_aprs_fi(api_key, searchcall)
-                "http://api.aprs.fi/api/get?"
+        except Exception as e:
-                "&what=loc&apikey={}&format=json"
+            LOG.debug("Weather failed with:  {}".format(str(e)))
-                "&name={}".format(api_key, fromcall)
+            reply = "Unable to find you (send beacon?)"
-            )
+        else:
-            response = requests.get(url)
+            aprs_data = json.loads(resp.text)
-            # aprs_data = json.loads(response.read())
-            aprs_data = json.loads(response.text)
             lat = aprs_data["entries"][0]["lat"]
             lon = aprs_data["entries"][0]["lng"]
-            url2 = (
+
-                "https://forecast.weather.gov/MapClick.php?lat=%s"
+            try:
-                "&lon=%s&FcstType=json" % (lat, lon)
+                wx_service = service.WeatherService(self.config)
-            )
+                reply = wx_service.forecast_short(lat, lon)
-            response2 = requests.get(url2)
+                # resp = plugin_utils.get_weather_gov_for_gps(lat, lon)
-            # wx_data = json.loads(response2.read())
+            except Exception as e:
-            wx_data = json.loads(response2.text)
+                LOG.debug("Weather failed with:  {}".format(str(e)))
-            reply = (
+                return "Unable to Lookup weather"
-                "%sF(%sF/%sF) %s. %s, %s."
+            else:
-                % (
+                # wx_data = json.loads(resp.text)
-                    wx_data["currentobservation"]["Temp"],
+
-                    wx_data["data"]["temperature"][0],
+                LOG.debug("reply: '{}' ".format(reply))
-                    wx_data["data"]["temperature"][1],
+                return reply
-                    wx_data["data"]["weather"][0],
+
-                    wx_data["time"]["startPeriodName"][1],
+
-                    wx_data["data"]["weather"][1],
+class WxPlugin(WeatherPlugin):
-                )
+    """METAR Command"""
-            ).rstrip()
+
-            LOG.debug("reply: '{}' ".format(reply))
+    version = "1.0"
-        except Exception as e:
+    command_regex = "^[mx]"
-            LOG.debug("Weather failed with:  " + "%s" % str(e))
+    command_name = "wx (Metar)"
-            reply = "Unable to find you (send beacon?)"
+
+    def command(self, fromcall, message, ack):
+        LOG.info("WX Plugin '{}'".format(message))
+        api_key = self.config["aprs.fi"]["apiKey"]
+        a = re.search(r"^.*\s+(.*)", message)
+        if a is not None:
+            searchcall = a.group(1)
+            station = searchcall.upper()
+            try:
+                resp = plugin_utils.get_weather_gov_metar(station)
+            except Exception as e:
+                LOG.debug("Weather failed with:  {}".format(str(e)))
+                reply = "Unable to find station METAR"
+            else:
+                station_data = json.loads(resp.text)
+                reply = station_data["properties"]["rawMessage"]
+
+            return reply
+        else:
+            # if no second argument, search for calling station
+            fromcall = fromcall
+            try:
+                resp = plugin_utils.get_aprs_fi(api_key, fromcall)
+            except Exception as e:
+                LOG.debug("Weather failed with:  {}".format(str(e)))
+                reply = "Unable to find you (send beacon?)"
+            else:
+                aprs_data = json.loads(resp.text)
+                lat = aprs_data["entries"][0]["lat"]
+                lon = aprs_data["entries"][0]["lng"]
+
+                try:
+                    resp = self.get_weather_gov_for_gps(lat, lon)
+                except Exception as e:
+                    LOG.debug("Weather failed with:  {}".format(str(e)))
+                    reply = "Unable to find you (send beacon?)"
+                else:
+                    wx_data = json.loads(resp.text)
+
+                    if wx_data["location"]["metar"]:
+                        station = wx_data["location"]["metar"]
+                        try:
+                            resp = self.get_metar(station)
+                        except Exception as e:
+                            LOG.debug("Weather failed with:  {}".format(str(e)))
+                            reply = "Failed to get Metar"
+                        else:
+                            station_data = json.loads(resp.text)
+                            reply = station_data["properties"]["rawMessage"]
+                    else:
+                        # Couldn't find a station
+                        reply = "No Metar station found"
 
         return reply

aprsd/service.py

@@ -0,0 +1,52 @@
+#  Base services class
+# this is the service mechanism used to manage
+# weather and location services from the config.
+# There are many weather and location services
+# that we could support.
+import abc
+import logging
+
+from aprsd import utils, weather
+
+LOG = logging.getLogger("APRSD")
+
+
+class APRSDService(metaclass=abc.ABCMeta):
+
+    config = None
+
+    def __init__(self, config):
+        LOG.debug("Service set config")
+        self.config = config
+        self.load()
+
+    @abc.abstractmethod
+    def load(self):
+        """Load and configure the service"""
+        pass
+
+
+class WeatherService(APRSDService):
+    _instance = None
+    wx = None
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            # Put any init here
+        return cls._instance
+
+    def load(self):
+        """Load the correct weather """
+        wx_shortcut = self.config["aprsd"]["services"].get(
+            "weather",
+            weather.DEFAULT_PROVIDER,
+        )
+        wx_class = weather.PROVIDER_MAPPING[wx_shortcut]
+        self.wx = utils.create_class(wx_class, weather.APRSDWeather, config=self.config)
+
+    def forecast_short(self, lat, lon):
+        return self.wx.forecast_short(lat, lon)
+
+    def forecast_raw(self, lat, lon):
+        return self.wx.forecast_raw(lat, lon)

aprsd/utils.py

@@ -2,12 +2,14 @@
 
 import errno
 import functools
+import importlib
+import logging
 import os
 from pathlib import Path
 import sys
 import threading
 
-from aprsd import plugin
+from aprsd import plugin, weather
 import click
 import yaml
 
@@ -44,6 +46,9 @@ DEFAULT_CONFIG_DICT = {
     "aprsd": {
         "plugin_dir": "~/.config/aprsd/plugins",
         "enabled_plugins": plugin.CORE_PLUGINS,
+        "services": {
+            "weather": weather.PROVIDER_MAPPING,
+        },
     },
 }
 
@@ -52,6 +57,8 @@ DEFAULT_CONFIG_DIR = "{}/.config/aprsd/".format(home)
 DEFAULT_SAVE_FILE = "{}/.config/aprsd/aprsd.p".format(home)
 DEFAULT_CONFIG_FILE = "{}/.config/aprsd/aprsd.yml".format(home)
 
+LOG = logging.getLogger("APRSD")
+
 
 def synchronized(wrapped):
     lock = threading.Lock()
@@ -222,3 +229,35 @@ def parse_config(config_file):
     check_option(config, "smtp", "password")
 
     return config
+
+
+def create_class(module_class_string, super_cls: type = None, **kwargs):
+    """
+    Method to create a class from a fqn python string.
+    :param module_class_string: full name of the class to create an object of
+    :param super_cls: expected super class for validity, None if bypass
+    :param kwargs: parameters to pass
+    :return:
+    """
+    module_name, class_name = module_class_string.rsplit(".", 1)
+    try:
+        module = importlib.import_module(module_name)
+    except Exception as ex:
+        LOG.error("Failed to load Plugin '{}' : '{}'".format(module_name, ex))
+        return
+
+    assert hasattr(module, class_name), "class {} is not in {}".format(
+        class_name,
+        module_name,
+    )
+    # click.echo('reading class {} from module {}'.format(
+    #     class_name, module_name))
+    cls = getattr(module, class_name)
+    if super_cls is not None:
+        assert issubclass(cls, super_cls), "class {} should inherit from {}".format(
+            class_name,
+            super_cls.__name__,
+        )
+    # click.echo('initialising {} with params {}'.format(class_name, kwargs))
+    obj = cls(**kwargs)
+    return obj

aprsd/weather.py

@@ -0,0 +1,66 @@
+import abc
+import json
+import logging
+
+import requests
+
+LOG = logging.getLogger("APRSD")
+
+DEFAULT_PROVIDER = "us-gov"
+PROVIDER_MAPPING = {
+    "us-gov": "aprsd.weather.USWeatherGov",
+}
+
+
+class APRSDWeather(metaclass=abc.ABCMeta):
+    confg = None
+
+    def __init__(self, config):
+        self.config = config
+
+    @abc.abstractmethod
+    def forecast_raw(self, lat, lon):
+        """Get a raw forecast json for latitude, longitude.
+
+        The format of the json response is entirely
+        depentent on the service itself.
+        """
+        pass
+
+    @abc.abstractmethod
+    def forecast_short(self, lat, lon):
+        """Get a short form forecast for latitude, longitude."""
+        pass
+
+
+class USWeatherGov(APRSDWeather):
+    def forecast_raw(self, lat, lon):
+        LOG.debug("Fetch station at {}, {}".format(lat, lon))
+        try:
+            url2 = (
+                "https://forecast.weather.gov/MapClick.php?lat=%s"
+                "&lon=%s&FcstType=json" % (lat, lon)
+            )
+            LOG.debug("Fetching weather '{}'".format(url2))
+            response = requests.get(url2)
+        except Exception as e:
+            LOG.error(e)
+            raise Exception("Failed to get weather")
+        else:
+            response.raise_for_status()
+            return json.loads(response.text)
+
+    def forecast_short(self, lat, lon):
+        """Return a short string for the forecast."""
+        wx_data = self.forecast_raw(lat, lon)
+        reply = (
+            "{}F({}F/{}F) {}. {}, {}.".format(
+                wx_data["currentobservation"]["Temp"],
+                wx_data["data"]["temperature"][0],
+                wx_data["data"]["temperature"][1],
+                wx_data["data"]["weather"][0],
+                wx_data["time"]["startPeriodName"][1],
+                wx_data["data"]["weather"][1],
+            )
+        ).rstrip()
+        return reply

setup.cfg

@@ -35,6 +35,7 @@ packages =
 [entry_points]
 console_scripts =
     aprsd = aprsd.main:main
+    aprsd-dev = aprsd.dev:main
     fake_aprs = aprsd.fake_aprs:main
 
 [build_sphinx]

tests/test_plugin.py

@@ -41,7 +41,7 @@ class TestPlugin(unittest.TestCase):
 
     @mock.patch("aprsd.messaging.MsgTrack.flush")
     def test_query_flush(self, mock_flush):
-        message = "?delete"
+        message = "!delete"
         query = query_plugin.QueryPlugin(self.config)
 
         expected = "Deleted ALL pending msgs."
@@ -53,7 +53,7 @@ class TestPlugin(unittest.TestCase):
     def test_query_restart_delayed(self, mock_restart):
         track = messaging.MsgTrack()
         track.track = {}
-        message = "?4"
+        message = "!4"
         query = query_plugin.QueryPlugin(self.config)
 
         expected = "No pending msgs to resend"