mirror of https://github.com/craigerl/aprsd.git
Reworked the stats dict output and healthcheck
This patch reworks the stats object dict and includes more data. Also includes aprsis last update timestamp (from last recieved message). This is used to help determine if the aprsis server connection is still alive and well.
This commit is contained in:
parent
123266c9ad
commit
bf8d2c6088
|
@ -3,9 +3,17 @@ import select
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import aprsd
|
import aprsd
|
||||||
|
from aprsd import stats
|
||||||
import aprslib
|
import aprslib
|
||||||
from aprslib import is_py3
|
from aprslib import is_py3
|
||||||
from aprslib.exceptions import LoginError
|
from aprslib.exceptions import (
|
||||||
|
ConnectionDrop,
|
||||||
|
ConnectionError,
|
||||||
|
GenericError,
|
||||||
|
LoginError,
|
||||||
|
ParseError,
|
||||||
|
UnknownFormat,
|
||||||
|
)
|
||||||
|
|
||||||
LOG = logging.getLogger("APRSD")
|
LOG = logging.getLogger("APRSD")
|
||||||
|
|
||||||
|
@ -163,6 +171,7 @@ class Aprsdis(aprslib.IS):
|
||||||
|
|
||||||
self.logger.info("Connected to {}".format(server_string))
|
self.logger.info("Connected to {}".format(server_string))
|
||||||
self.server_string = server_string
|
self.server_string = server_string
|
||||||
|
stats.APRSDStats().set_aprsis_server(server_string)
|
||||||
|
|
||||||
if callsign == "":
|
if callsign == "":
|
||||||
raise LoginError("Server responded with empty callsign???")
|
raise LoginError("Server responded with empty callsign???")
|
||||||
|
@ -180,11 +189,67 @@ class Aprsdis(aprslib.IS):
|
||||||
self.logger.error(str(e))
|
self.logger.error(str(e))
|
||||||
self.close()
|
self.close()
|
||||||
raise
|
raise
|
||||||
except Exception:
|
except Exception as e:
|
||||||
self.close()
|
self.close()
|
||||||
self.logger.error("Failed to login")
|
self.logger.error("Failed to login '{}'".format(e))
|
||||||
raise LoginError("Failed to login")
|
raise LoginError("Failed to login")
|
||||||
|
|
||||||
|
def consumer(self, callback, blocking=True, immortal=False, raw=False):
|
||||||
|
"""
|
||||||
|
When a position sentence is received, it will be passed to the callback function
|
||||||
|
|
||||||
|
blocking: if true (default), runs forever, otherwise will return after one sentence
|
||||||
|
You can still exit the loop, by raising StopIteration in the callback function
|
||||||
|
|
||||||
|
immortal: When true, consumer will try to reconnect and stop propagation of Parse exceptions
|
||||||
|
if false (default), consumer will return
|
||||||
|
|
||||||
|
raw: when true, raw packet is passed to callback, otherwise the result from aprs.parse()
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not self._connected:
|
||||||
|
raise ConnectionError("not connected to a server")
|
||||||
|
|
||||||
|
line = b""
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
for line in self._socket_readlines(blocking):
|
||||||
|
if line[0:1] != b"#":
|
||||||
|
if raw:
|
||||||
|
callback(line)
|
||||||
|
else:
|
||||||
|
callback(self._parse(line))
|
||||||
|
else:
|
||||||
|
self.logger.debug("Server: %s", line.decode("utf8"))
|
||||||
|
stats.APRSDStats().set_aprsis_keepalive()
|
||||||
|
except ParseError as exp:
|
||||||
|
self.logger.log(11, "%s\n Packet: %s", exp.args[0], exp.args[1])
|
||||||
|
except UnknownFormat as exp:
|
||||||
|
self.logger.log(9, "%s\n Packet: %s", exp.args[0], exp.args[1])
|
||||||
|
except LoginError as exp:
|
||||||
|
self.logger.error("%s: %s", exp.__class__.__name__, exp.args[0])
|
||||||
|
except (KeyboardInterrupt, SystemExit):
|
||||||
|
raise
|
||||||
|
except (ConnectionDrop, ConnectionError):
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
if not immortal:
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
self.connect(blocking=blocking)
|
||||||
|
continue
|
||||||
|
except GenericError:
|
||||||
|
pass
|
||||||
|
except StopIteration:
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
self.logger.error("APRS Packet: %s", line)
|
||||||
|
raise
|
||||||
|
|
||||||
|
if not blocking:
|
||||||
|
break
|
||||||
|
|
||||||
|
|
||||||
def get_client():
|
def get_client():
|
||||||
cl = Client()
|
cl = Client()
|
||||||
|
|
|
@ -4,10 +4,8 @@ import logging
|
||||||
from logging import NullHandler
|
from logging import NullHandler
|
||||||
from logging.handlers import RotatingFileHandler
|
from logging.handlers import RotatingFileHandler
|
||||||
import sys
|
import sys
|
||||||
import tracemalloc
|
|
||||||
|
|
||||||
import aprsd
|
from aprsd import messaging, plugin, stats, utils
|
||||||
from aprsd import client, messaging, plugin, stats, utils
|
|
||||||
import flask
|
import flask
|
||||||
import flask_classful
|
import flask_classful
|
||||||
from flask_httpauth import HTTPBasicAuth
|
from flask_httpauth import HTTPBasicAuth
|
||||||
|
@ -81,20 +79,15 @@ class APRSDFlask(flask_classful.FlaskView):
|
||||||
stats_obj = stats.APRSDStats()
|
stats_obj = stats.APRSDStats()
|
||||||
track = messaging.MsgTrack()
|
track = messaging.MsgTrack()
|
||||||
now = datetime.datetime.now()
|
now = datetime.datetime.now()
|
||||||
current, peak = tracemalloc.get_traced_memory()
|
|
||||||
cl = client.Client()
|
time_format = "%m-%d-%Y %H:%M:%S"
|
||||||
server_string = cl.client.server_string
|
|
||||||
|
stats_dict = stats_obj.stats()
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
"version": aprsd.__version__,
|
"time": now.strftime(time_format),
|
||||||
"aprsis_server": server_string,
|
|
||||||
"callsign": self.config["aprs"]["login"],
|
|
||||||
"uptime": stats_obj.uptime,
|
|
||||||
"size_tracker": len(track),
|
"size_tracker": len(track),
|
||||||
"stats": stats_obj.stats(),
|
"stats": stats_dict,
|
||||||
"time": now.strftime("%m-%d-%Y %H:%M:%S"),
|
|
||||||
"memory_current": current,
|
|
||||||
"memory_peak": peak,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
|
@ -215,6 +215,15 @@ def check(loglevel, config_file, health_url, timeout):
|
||||||
LOG.error("Email thread is very old! {}".format(d))
|
LOG.error("Email thread is very old! {}".format(d))
|
||||||
sys.exit(-1)
|
sys.exit(-1)
|
||||||
|
|
||||||
|
aprsis_last_update = stats["stats"]["aprs-is"]["last_update"]
|
||||||
|
delta = parse_delta_str(aprsis_last_update)
|
||||||
|
d = datetime.timedelta(**delta)
|
||||||
|
max_timeout = {"hours": 0.0, "minutes": 5, "seconds": 0}
|
||||||
|
max_delta = datetime.timedelta(**max_timeout)
|
||||||
|
if d > max_delta:
|
||||||
|
LOG.error("APRS-IS last update is very old! {}".format(d))
|
||||||
|
sys.exit(-1)
|
||||||
|
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,9 @@ import datetime
|
||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
|
import aprsd
|
||||||
|
from aprsd import utils
|
||||||
|
|
||||||
LOG = logging.getLogger("APRSD")
|
LOG = logging.getLogger("APRSD")
|
||||||
|
|
||||||
|
|
||||||
|
@ -12,6 +15,7 @@ class APRSDStats:
|
||||||
config = None
|
config = None
|
||||||
|
|
||||||
start_time = None
|
start_time = None
|
||||||
|
_aprsis_keepalive = None
|
||||||
|
|
||||||
_msgs_tracked = 0
|
_msgs_tracked = 0
|
||||||
_msgs_tx = 0
|
_msgs_tx = 0
|
||||||
|
@ -35,6 +39,7 @@ class APRSDStats:
|
||||||
# any initializetion here
|
# any initializetion here
|
||||||
cls._instance.lock = threading.Lock()
|
cls._instance.lock = threading.Lock()
|
||||||
cls._instance.start_time = datetime.datetime.now()
|
cls._instance.start_time = datetime.datetime.now()
|
||||||
|
cls._instance._aprsis_keepalive = datetime.datetime.now()
|
||||||
return cls._instance
|
return cls._instance
|
||||||
|
|
||||||
def __init__(self, config=None):
|
def __init__(self, config=None):
|
||||||
|
@ -53,7 +58,7 @@ class APRSDStats:
|
||||||
|
|
||||||
def set_memory(self, memory):
|
def set_memory(self, memory):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
self._mem_curent = memory
|
self._mem_current = memory
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def memory_peak(self):
|
def memory_peak(self):
|
||||||
|
@ -64,6 +69,24 @@ class APRSDStats:
|
||||||
with self.lock:
|
with self.lock:
|
||||||
self._mem_peak = memory
|
self._mem_peak = memory
|
||||||
|
|
||||||
|
@property
|
||||||
|
def aprsis_server(self):
|
||||||
|
with self.lock:
|
||||||
|
return self._aprsis_server
|
||||||
|
|
||||||
|
def set_aprsis_server(self, server):
|
||||||
|
with self.lock:
|
||||||
|
self._aprsis_server = server
|
||||||
|
|
||||||
|
@property
|
||||||
|
def aprsis_keepalive(self):
|
||||||
|
with self.lock:
|
||||||
|
return self._aprsis_keepalive
|
||||||
|
|
||||||
|
def set_aprsis_keepalive(self):
|
||||||
|
with self.lock:
|
||||||
|
self._aprsis_keepalive = datetime.datetime.now()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def msgs_tx(self):
|
def msgs_tx(self):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
|
@ -152,7 +175,25 @@ class APRSDStats:
|
||||||
else:
|
else:
|
||||||
last_update = "never"
|
last_update = "never"
|
||||||
|
|
||||||
|
if self._aprsis_keepalive:
|
||||||
|
last_aprsis_keepalive = str(now - self._aprsis_keepalive)
|
||||||
|
else:
|
||||||
|
last_aprsis_keepalive = "never"
|
||||||
|
|
||||||
stats = {
|
stats = {
|
||||||
|
"aprsd": {
|
||||||
|
"version": aprsd.__version__,
|
||||||
|
"uptime": self.uptime,
|
||||||
|
"memory_current": self.memory,
|
||||||
|
"memory_current_str": utils.human_size(self.memory),
|
||||||
|
"memory_peak": self.memory_peak,
|
||||||
|
"memory_peak_str": utils.human_size(self.memory_peak),
|
||||||
|
},
|
||||||
|
"aprs-is": {
|
||||||
|
"server": self.aprsis_server,
|
||||||
|
"callsign": self.config["aprs"]["login"],
|
||||||
|
"last_update": last_aprsis_keepalive,
|
||||||
|
},
|
||||||
"messages": {
|
"messages": {
|
||||||
"tracked": self.msgs_tracked,
|
"tracked": self.msgs_tracked,
|
||||||
"sent": self.msgs_tx,
|
"sent": self.msgs_tx,
|
||||||
|
|
|
@ -1,12 +1,13 @@
|
||||||
import abc
|
import abc
|
||||||
import datetime
|
import datetime
|
||||||
|
import gc
|
||||||
import logging
|
import logging
|
||||||
import queue
|
import queue
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import tracemalloc
|
import tracemalloc
|
||||||
|
|
||||||
from aprsd import client, messaging, plugin, stats, trace
|
from aprsd import client, messaging, plugin, stats, trace, utils
|
||||||
import aprslib
|
import aprslib
|
||||||
|
|
||||||
LOG = logging.getLogger("APRSD")
|
LOG = logging.getLogger("APRSD")
|
||||||
|
@ -74,28 +75,33 @@ class KeepAliveThread(APRSDThread):
|
||||||
|
|
||||||
def loop(self):
|
def loop(self):
|
||||||
if self.cntr % 6 == 0:
|
if self.cntr % 6 == 0:
|
||||||
|
nuked = gc.collect()
|
||||||
tracker = messaging.MsgTrack()
|
tracker = messaging.MsgTrack()
|
||||||
stats_obj = stats.APRSDStats()
|
stats_obj = stats.APRSDStats()
|
||||||
now = datetime.datetime.now()
|
now = datetime.datetime.now()
|
||||||
last_email = stats.APRSDStats().email_thread_time
|
last_email = stats_obj.email_thread_time
|
||||||
if last_email:
|
if last_email:
|
||||||
email_thread_time = str(now - last_email)
|
email_thread_time = str(now - last_email)
|
||||||
else:
|
else:
|
||||||
email_thread_time = "N/A"
|
email_thread_time = "N/A"
|
||||||
|
|
||||||
|
last_msg_time = str(now - stats_obj.aprsis_keepalive)
|
||||||
|
|
||||||
current, peak = tracemalloc.get_traced_memory()
|
current, peak = tracemalloc.get_traced_memory()
|
||||||
stats_obj.set_memory(current)
|
stats_obj.set_memory(current)
|
||||||
stats_obj.set_memory_peak(peak)
|
stats_obj.set_memory_peak(peak)
|
||||||
LOG.debug(
|
LOG.debug(
|
||||||
"Uptime ({}) Tracker({}) "
|
"Uptime ({}) Tracker({}) "
|
||||||
"Msgs: TX:{} RX:{} EmailThread: {} RAM: Current:{} Peak:{}".format(
|
"Msgs: TX:{} RX:{} Last: {} - EmailThread: {} - RAM: Current:{} Peak:{} Nuked: {}".format(
|
||||||
stats_obj.uptime,
|
stats_obj.uptime,
|
||||||
len(tracker),
|
len(tracker),
|
||||||
stats_obj.msgs_tx,
|
stats_obj.msgs_tx,
|
||||||
stats_obj.msgs_rx,
|
stats_obj.msgs_rx,
|
||||||
|
last_msg_time,
|
||||||
email_thread_time,
|
email_thread_time,
|
||||||
current,
|
utils.human_size(current),
|
||||||
peak,
|
utils.human_size(peak),
|
||||||
|
nuked,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
self.cntr += 1
|
self.cntr += 1
|
||||||
|
|
|
@ -361,3 +361,10 @@ def parse_config(config_file):
|
||||||
)
|
)
|
||||||
|
|
||||||
return config
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def human_size(bytes, units=None):
|
||||||
|
""" Returns a human readable string representation of bytes """
|
||||||
|
if not units:
|
||||||
|
units = [" bytes", "KB", "MB", "GB", "TB", "PB", "EB"]
|
||||||
|
return str(bytes) + units[0] if bytes < 1024 else human_size(bytes >> 10, units[1:])
|
||||||
|
|
|
@ -139,14 +139,14 @@
|
||||||
}
|
}
|
||||||
|
|
||||||
function update_stats( data ) {
|
function update_stats( data ) {
|
||||||
$("#version").text( data["version"] );
|
$("#version").text( data["stats"]["aprsd"]["version"] );
|
||||||
$("#aprsis").text( "APRS-IS Server: " + data["aprsis_server"] );
|
$("#aprsis").text( "APRS-IS Server: " + data["stats"]["aprs-is"]["server"] );
|
||||||
$("#uptime").text( "uptime: " + data["uptime"] );
|
$("#uptime").text( "uptime: " + data["stats"]["aprsd"]["uptime"] );
|
||||||
const html_pretty = Prism.highlight(JSON.stringify(data, null, '\t'), Prism.languages.json, 'json');
|
const html_pretty = Prism.highlight(JSON.stringify(data, null, '\t'), Prism.languages.json, 'json');
|
||||||
$("#jsonstats").html(html_pretty);
|
$("#jsonstats").html(html_pretty);
|
||||||
//$("#jsonstats").effect("highlight", {color: "#333333"}, 800);
|
//$("#jsonstats").effect("highlight", {color: "#333333"}, 800);
|
||||||
//console.log(data);
|
//console.log(data);
|
||||||
updateDualData(memory_chart, data["time"], data["memory_peak"], data["memory_current"]);
|
updateDualData(memory_chart, data["time"], data["stats"]["aprsd"]["memory_peak"], data["stats"]["aprsd"]["memory_current"]);
|
||||||
updateQuadData(message_chart, data["time"], data["stats"]["messages"]["sent"], data["stats"]["messages"]["recieved"], data["stats"]["messages"]["ack_sent"], data["stats"]["messages"]["ack_recieved"]);
|
updateQuadData(message_chart, data["time"], data["stats"]["messages"]["sent"], data["stats"]["messages"]["recieved"], data["stats"]["messages"]["ack_sent"], data["stats"]["messages"]["ack_recieved"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue