1
0
mirror of https://github.com/craigerl/aprsd.git synced 2024-10-31 15:07:13 -04:00

Added healthcheck app

This patch adds the healthcheck app that uses the flask stats url
to fetch the internal stats of a running aprsd server.  If the server is
up the stats will return and be checked for 'healthy' status.
IF the url fails to return, healthcheck will exit with -1.  You can use
this script to restart aprsd if healthcheck exits with -1 status.

There is a check against the email thread.  The email thread updates a
deadman's timer every 5 seconds.   If that time gets older than 5
minutes, then healthcheck will say that's a failure and exit with -1.

You can call healthcheck and restart aprsd if it fails (exit -1)
This commit is contained in:
Hemna 2021-01-22 12:36:40 -05:00
parent f932c203d7
commit c870207a96
2 changed files with 223 additions and 0 deletions

222
aprsd/healthcheck.py Normal file
View File

@ -0,0 +1,222 @@
#
# Used to fetch the stats url and determine if
# aprsd server is 'healthy'
#
#
# python included libs
import datetime
import json
import logging
from logging import NullHandler
from logging.handlers import RotatingFileHandler
import os
import re
import sys
# local imports here
import aprsd
from aprsd import utils
import click
import click_completion
import requests
# setup the global logger
# logging.basicConfig(level=logging.DEBUG) # level=10
LOG = logging.getLogger("APRSD")
LOG_LEVELS = {
"CRITICAL": logging.CRITICAL,
"ERROR": logging.ERROR,
"WARNING": logging.WARNING,
"INFO": logging.INFO,
"DEBUG": logging.DEBUG,
}
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
def custom_startswith(string, incomplete):
"""A custom completion match that supports case insensitive matching."""
if os.environ.get("_CLICK_COMPLETION_COMMAND_CASE_INSENSITIVE_COMPLETE"):
string = string.lower()
incomplete = incomplete.lower()
return string.startswith(incomplete)
click_completion.core.startswith = custom_startswith
click_completion.init()
cmd_help = """Shell completion for click-completion-command
Available shell types:
\b
%s
Default type: auto
""" % "\n ".join(
"{:<12} {}".format(k, click_completion.core.shells[k])
for k in sorted(click_completion.core.shells.keys())
)
@click.group(help=cmd_help, context_settings=CONTEXT_SETTINGS)
@click.version_option()
def main():
pass
@main.command()
@click.option(
"-i",
"--case-insensitive/--no-case-insensitive",
help="Case insensitive completion",
)
@click.argument(
"shell",
required=False,
type=click_completion.DocumentedChoice(click_completion.core.shells),
)
def show(shell, case_insensitive):
"""Show the click-completion-command completion code"""
extra_env = (
{"_CLICK_COMPLETION_COMMAND_CASE_INSENSITIVE_COMPLETE": "ON"}
if case_insensitive
else {}
)
click.echo(click_completion.core.get_code(shell, extra_env=extra_env))
@main.command()
@click.option(
"--append/--overwrite",
help="Append the completion code to the file",
default=None,
)
@click.option(
"-i",
"--case-insensitive/--no-case-insensitive",
help="Case insensitive completion",
)
@click.argument(
"shell",
required=False,
type=click_completion.DocumentedChoice(click_completion.core.shells),
)
@click.argument("path", required=False)
def install(append, case_insensitive, shell, path):
"""Install the click-completion-command completion"""
extra_env = (
{"_CLICK_COMPLETION_COMMAND_CASE_INSENSITIVE_COMPLETE": "ON"}
if case_insensitive
else {}
)
shell, path = click_completion.core.install(
shell=shell,
path=path,
append=append,
extra_env=extra_env,
)
click.echo("{} completion installed in {}".format(shell, path))
# Setup the logging faciility
# to disable logging to stdout, but still log to file
# use the --quiet option on the cmdln
def setup_logging(config, loglevel, quiet):
log_level = LOG_LEVELS[loglevel]
LOG.setLevel(log_level)
log_format = "[%(asctime)s] [%(threadName)-12s] [%(levelname)-5.5s]" " %(message)s"
date_format = "%m/%d/%Y %I:%M:%S %p"
log_formatter = logging.Formatter(fmt=log_format, datefmt=date_format)
log_file = config["aprs"].get("logfile", None)
if log_file:
fh = RotatingFileHandler(log_file, maxBytes=(10248576 * 5), backupCount=4)
else:
fh = NullHandler()
fh.setFormatter(log_formatter)
LOG.addHandler(fh)
if not quiet:
sh = logging.StreamHandler(sys.stdout)
sh.setFormatter(log_formatter)
LOG.addHandler(sh)
def parse_delta_str(s):
if "day" in s:
m = re.match(
r"(?P<days>[-\d]+) day[s]*, (?P<hours>\d+):(?P<minutes>\d+):(?P<seconds>\d[\.\d+]*)",
s,
)
else:
m = re.match(r"(?P<hours>\d+):(?P<minutes>\d+):(?P<seconds>\d[\.\d+]*)", s)
return {key: float(val) for key, val in m.groupdict().items()}
@main.command()
@click.option(
"--loglevel",
default="INFO",
show_default=True,
type=click.Choice(
["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"],
case_sensitive=False,
),
show_choices=True,
help="The log level to use for aprsd.log",
)
@click.option(
"-c",
"--config",
"config_file",
show_default=True,
default=utils.DEFAULT_CONFIG_FILE,
help="The aprsd config file to use for options.",
)
@click.option(
"--url",
"health_url",
show_default=True,
default="http://localhost:8001/stats",
help="The aprsd url to call for checking health/stats",
)
@click.option(
"--timeout",
show_default=True,
default=3,
help="How long to wait for healtcheck url to come back",
)
def check(loglevel, config_file, health_url, timeout):
"""APRSD Plugin test app."""
config = utils.parse_config(config_file)
setup_logging(config, loglevel, False)
LOG.debug("APRSD HealthCheck version: {}".format(aprsd.__version__))
try:
url = health_url
response = requests.get(url, timeout=timeout)
response.raise_for_status()
except Exception as ex:
LOG.error("Failed to fetch healthcheck url '{}' : '{}'".format(url, ex))
sys.exit(-1)
else:
stats = json.loads(response.text)
LOG.debug(stats)
email_thread_last_update = stats["stats"]["email"]["thread_last_update"]
delta = parse_delta_str(email_thread_last_update)
d = datetime.timedelta(**delta)
max_timeout = {"hours": 0.0, "minutes": 5, "seconds": 0}
max_delta = datetime.timedelta(**max_timeout)
if d > max_delta:
LOG.error("Email thread is very old! {}".format(d))
sys.exit(-1)
sys.exit(0)
if __name__ == "__main__":
main()

View File

@ -36,6 +36,7 @@ packages =
console_scripts =
aprsd = aprsd.main:main
aprsd-dev = aprsd.dev:main
aprsd-healthcheck = aprsd.healthcheck:main
fake_aprs = aprsd.fake_aprs:main
[build_sphinx]