From bf8d2c6088d9c09783b8d68de4c88fe77fbbc9f5 Mon Sep 17 00:00:00 2001 From: Hemna Date: Thu, 1 Apr 2021 23:12:25 -0400 Subject: [PATCH] Reworked the stats dict output and healthcheck This patch reworks the stats object dict and includes more data. Also includes aprsis last update timestamp (from last recieved message). This is used to help determine if the aprsis server connection is still alive and well. --- aprsd/client.py | 71 ++++++++++++++++++++++++++++++++-- aprsd/flask.py | 21 ++++------ aprsd/healthcheck.py | 9 +++++ aprsd/stats.py | 43 +++++++++++++++++++- aprsd/threads.py | 16 +++++--- aprsd/utils.py | 7 ++++ aprsd/web/templates/index.html | 8 ++-- 7 files changed, 148 insertions(+), 27 deletions(-) diff --git a/aprsd/client.py b/aprsd/client.py index 6dea8e5..92e9c98 100644 --- a/aprsd/client.py +++ b/aprsd/client.py @@ -3,9 +3,17 @@ import select import time import aprsd +from aprsd import stats import aprslib from aprslib import is_py3 -from aprslib.exceptions import LoginError +from aprslib.exceptions import ( + ConnectionDrop, + ConnectionError, + GenericError, + LoginError, + ParseError, + UnknownFormat, +) LOG = logging.getLogger("APRSD") @@ -163,6 +171,7 @@ class Aprsdis(aprslib.IS): self.logger.info("Connected to {}".format(server_string)) self.server_string = server_string + stats.APRSDStats().set_aprsis_server(server_string) if callsign == "": raise LoginError("Server responded with empty callsign???") @@ -180,11 +189,67 @@ class Aprsdis(aprslib.IS): self.logger.error(str(e)) self.close() raise - except Exception: + except Exception as e: self.close() - self.logger.error("Failed to login") + self.logger.error("Failed to login '{}'".format(e)) raise LoginError("Failed to login") + def consumer(self, callback, blocking=True, immortal=False, raw=False): + """ + When a position sentence is received, it will be passed to the callback function + + blocking: if true (default), runs forever, otherwise will return after one sentence + You can still exit the loop, by raising StopIteration in the callback function + + immortal: When true, consumer will try to reconnect and stop propagation of Parse exceptions + if false (default), consumer will return + + raw: when true, raw packet is passed to callback, otherwise the result from aprs.parse() + """ + + if not self._connected: + raise ConnectionError("not connected to a server") + + line = b"" + + while True: + try: + for line in self._socket_readlines(blocking): + if line[0:1] != b"#": + if raw: + callback(line) + else: + callback(self._parse(line)) + else: + self.logger.debug("Server: %s", line.decode("utf8")) + stats.APRSDStats().set_aprsis_keepalive() + except ParseError as exp: + self.logger.log(11, "%s\n Packet: %s", exp.args[0], exp.args[1]) + except UnknownFormat as exp: + self.logger.log(9, "%s\n Packet: %s", exp.args[0], exp.args[1]) + except LoginError as exp: + self.logger.error("%s: %s", exp.__class__.__name__, exp.args[0]) + except (KeyboardInterrupt, SystemExit): + raise + except (ConnectionDrop, ConnectionError): + self.close() + + if not immortal: + raise + else: + self.connect(blocking=blocking) + continue + except GenericError: + pass + except StopIteration: + break + except Exception: + self.logger.error("APRS Packet: %s", line) + raise + + if not blocking: + break + def get_client(): cl = Client() diff --git a/aprsd/flask.py b/aprsd/flask.py index a1bdb00..2f0eee6 100644 --- a/aprsd/flask.py +++ b/aprsd/flask.py @@ -4,10 +4,8 @@ import logging from logging import NullHandler from logging.handlers import RotatingFileHandler import sys -import tracemalloc -import aprsd -from aprsd import client, messaging, plugin, stats, utils +from aprsd import messaging, plugin, stats, utils import flask import flask_classful from flask_httpauth import HTTPBasicAuth @@ -81,20 +79,15 @@ class APRSDFlask(flask_classful.FlaskView): stats_obj = stats.APRSDStats() track = messaging.MsgTrack() now = datetime.datetime.now() - current, peak = tracemalloc.get_traced_memory() - cl = client.Client() - server_string = cl.client.server_string + + time_format = "%m-%d-%Y %H:%M:%S" + + stats_dict = stats_obj.stats() result = { - "version": aprsd.__version__, - "aprsis_server": server_string, - "callsign": self.config["aprs"]["login"], - "uptime": stats_obj.uptime, + "time": now.strftime(time_format), "size_tracker": len(track), - "stats": stats_obj.stats(), - "time": now.strftime("%m-%d-%Y %H:%M:%S"), - "memory_current": current, - "memory_peak": peak, + "stats": stats_dict, } return result diff --git a/aprsd/healthcheck.py b/aprsd/healthcheck.py index db2f97e..35e3a6e 100644 --- a/aprsd/healthcheck.py +++ b/aprsd/healthcheck.py @@ -215,6 +215,15 @@ def check(loglevel, config_file, health_url, timeout): LOG.error("Email thread is very old! {}".format(d)) sys.exit(-1) + aprsis_last_update = stats["stats"]["aprs-is"]["last_update"] + delta = parse_delta_str(aprsis_last_update) + d = datetime.timedelta(**delta) + max_timeout = {"hours": 0.0, "minutes": 5, "seconds": 0} + max_delta = datetime.timedelta(**max_timeout) + if d > max_delta: + LOG.error("APRS-IS last update is very old! {}".format(d)) + sys.exit(-1) + sys.exit(0) diff --git a/aprsd/stats.py b/aprsd/stats.py index 4b4ce1e..6729564 100644 --- a/aprsd/stats.py +++ b/aprsd/stats.py @@ -2,6 +2,9 @@ import datetime import logging import threading +import aprsd +from aprsd import utils + LOG = logging.getLogger("APRSD") @@ -12,6 +15,7 @@ class APRSDStats: config = None start_time = None + _aprsis_keepalive = None _msgs_tracked = 0 _msgs_tx = 0 @@ -35,6 +39,7 @@ class APRSDStats: # any initializetion here cls._instance.lock = threading.Lock() cls._instance.start_time = datetime.datetime.now() + cls._instance._aprsis_keepalive = datetime.datetime.now() return cls._instance def __init__(self, config=None): @@ -53,7 +58,7 @@ class APRSDStats: def set_memory(self, memory): with self.lock: - self._mem_curent = memory + self._mem_current = memory @property def memory_peak(self): @@ -64,6 +69,24 @@ class APRSDStats: with self.lock: self._mem_peak = memory + @property + def aprsis_server(self): + with self.lock: + return self._aprsis_server + + def set_aprsis_server(self, server): + with self.lock: + self._aprsis_server = server + + @property + def aprsis_keepalive(self): + with self.lock: + return self._aprsis_keepalive + + def set_aprsis_keepalive(self): + with self.lock: + self._aprsis_keepalive = datetime.datetime.now() + @property def msgs_tx(self): with self.lock: @@ -152,7 +175,25 @@ class APRSDStats: else: last_update = "never" + if self._aprsis_keepalive: + last_aprsis_keepalive = str(now - self._aprsis_keepalive) + else: + last_aprsis_keepalive = "never" + stats = { + "aprsd": { + "version": aprsd.__version__, + "uptime": self.uptime, + "memory_current": self.memory, + "memory_current_str": utils.human_size(self.memory), + "memory_peak": self.memory_peak, + "memory_peak_str": utils.human_size(self.memory_peak), + }, + "aprs-is": { + "server": self.aprsis_server, + "callsign": self.config["aprs"]["login"], + "last_update": last_aprsis_keepalive, + }, "messages": { "tracked": self.msgs_tracked, "sent": self.msgs_tx, diff --git a/aprsd/threads.py b/aprsd/threads.py index 03b4fc7..01439ee 100644 --- a/aprsd/threads.py +++ b/aprsd/threads.py @@ -1,12 +1,13 @@ import abc import datetime +import gc import logging import queue import threading import time import tracemalloc -from aprsd import client, messaging, plugin, stats, trace +from aprsd import client, messaging, plugin, stats, trace, utils import aprslib LOG = logging.getLogger("APRSD") @@ -74,28 +75,33 @@ class KeepAliveThread(APRSDThread): def loop(self): if self.cntr % 6 == 0: + nuked = gc.collect() tracker = messaging.MsgTrack() stats_obj = stats.APRSDStats() now = datetime.datetime.now() - last_email = stats.APRSDStats().email_thread_time + last_email = stats_obj.email_thread_time if last_email: email_thread_time = str(now - last_email) else: email_thread_time = "N/A" + last_msg_time = str(now - stats_obj.aprsis_keepalive) + current, peak = tracemalloc.get_traced_memory() stats_obj.set_memory(current) stats_obj.set_memory_peak(peak) LOG.debug( "Uptime ({}) Tracker({}) " - "Msgs: TX:{} RX:{} EmailThread: {} RAM: Current:{} Peak:{}".format( + "Msgs: TX:{} RX:{} Last: {} - EmailThread: {} - RAM: Current:{} Peak:{} Nuked: {}".format( stats_obj.uptime, len(tracker), stats_obj.msgs_tx, stats_obj.msgs_rx, + last_msg_time, email_thread_time, - current, - peak, + utils.human_size(current), + utils.human_size(peak), + nuked, ), ) self.cntr += 1 diff --git a/aprsd/utils.py b/aprsd/utils.py index 0be68f9..5373fa0 100644 --- a/aprsd/utils.py +++ b/aprsd/utils.py @@ -361,3 +361,10 @@ def parse_config(config_file): ) return config + + +def human_size(bytes, units=None): + """ Returns a human readable string representation of bytes """ + if not units: + units = [" bytes", "KB", "MB", "GB", "TB", "PB", "EB"] + return str(bytes) + units[0] if bytes < 1024 else human_size(bytes >> 10, units[1:]) diff --git a/aprsd/web/templates/index.html b/aprsd/web/templates/index.html index e4263b8..614d6a4 100644 --- a/aprsd/web/templates/index.html +++ b/aprsd/web/templates/index.html @@ -139,14 +139,14 @@ } function update_stats( data ) { - $("#version").text( data["version"] ); - $("#aprsis").text( "APRS-IS Server: " + data["aprsis_server"] ); - $("#uptime").text( "uptime: " + data["uptime"] ); + $("#version").text( data["stats"]["aprsd"]["version"] ); + $("#aprsis").text( "APRS-IS Server: " + data["stats"]["aprs-is"]["server"] ); + $("#uptime").text( "uptime: " + data["stats"]["aprsd"]["uptime"] ); const html_pretty = Prism.highlight(JSON.stringify(data, null, '\t'), Prism.languages.json, 'json'); $("#jsonstats").html(html_pretty); //$("#jsonstats").effect("highlight", {color: "#333333"}, 800); //console.log(data); - updateDualData(memory_chart, data["time"], data["memory_peak"], data["memory_current"]); + updateDualData(memory_chart, data["time"], data["stats"]["aprsd"]["memory_peak"], data["stats"]["aprsd"]["memory_current"]); updateQuadData(message_chart, data["time"], data["stats"]["messages"]["sent"], data["stats"]["messages"]["recieved"], data["stats"]["messages"]["ack_sent"], data["stats"]["messages"]["ack_recieved"]); }