Reworked the admin graphs

This patch fixes some bugs wth the rpc for packets as well
as reworks the admin graphs to use echarts.
This commit is contained in:
Hemna 2023-11-17 11:36:39 -05:00
parent fe1ebf2ec1
commit 763c9ab897
7 changed files with 501 additions and 25 deletions

View File

@ -70,7 +70,7 @@ def main():
# First import all the possible commands for the CLI
# The commands themselves live in the cmds directory
from .cmds import ( # noqa
completion, dev, fetch_stats, healthcheck, list_plugins, listen,
completion, config, dev, fetch_stats, healthcheck, list_plugins, listen,
send_message, server, webchat,
)
cli(auto_envvar_prefix="APRSD")
@ -145,6 +145,8 @@ def sample_config(ctx):
if not sys.argv[1:]:
raise SystemExit
raise
LOG.warning(conf.namespace)
return
generator.generate(conf)

View File

@ -108,6 +108,8 @@ class Packet(metaclass=abc.ABCMeta):
# hash=False
#)
last_send_time: float = field(repr=False, default=0, compare=False, hash=False)
last_send_attempt: int = field(repr=False, default=0, compare=False, hash=False)
# Do we allow this packet to be saved to send later?
allow_delay: bool = field(repr=False, default=True, compare=False, hash=False)
path: List[str] = field(default_factory=list, compare=False, hash=False)

View File

@ -19,11 +19,12 @@ class PacketList(MutableMapping):
lock = threading.Lock()
_total_rx: int = 0
_total_tx: int = 0
types = {}
def __new__(cls, *args, **kwargs):
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._maxlen = 1000
cls._maxlen = 100
cls.d = OrderedDict()
return cls._instance
@ -32,6 +33,10 @@ class PacketList(MutableMapping):
"""Add a packet that was received."""
self._total_rx += 1
self._add(packet)
ptype = packet.__class__.__name__
if not ptype in self.types:
self.types[ptype] = {"tx": 0, "rx": 0}
self.types[ptype]["rx"] += 1
seen_list.SeenList().update_seen(packet)
stats.APRSDStats().rx(packet)
@ -40,6 +45,10 @@ class PacketList(MutableMapping):
"""Add a packet that was received."""
self._total_tx += 1
self._add(packet)
ptype = packet.__class__.__name__
if not ptype in self.types:
self.types[ptype] = {"tx": 0, "rx": 0}
self.types[ptype]["tx"] += 1
seen_list.SeenList().update_seen(packet)
stats.APRSDStats().tx(packet)
@ -50,6 +59,10 @@ class PacketList(MutableMapping):
def _add(self, packet):
self[packet.key] = packet
def copy(self):
return self.d.copy()
@property
def maxlen(self):
return self._maxlen

View File

@ -65,6 +65,7 @@ class PacketTrack(objectstore.ObjectStoreMixin):
@wrapt.synchronized(lock)
def add(self, packet):
key = packet.msgNo
packet._last_send_attempt = 0
self.data[key] = packet
self.total_tracked += 1
@ -83,7 +84,7 @@ class PacketTrack(objectstore.ObjectStoreMixin):
"""Walk the list of messages and restart them if any."""
for key in self.data.keys():
pkt = self.data[key]
if pkt.last_send_attempt < pkt.retry_count:
if pkt._last_send_attempt < pkt.retry_count:
tx.send(pkt)
def _resend(self, packet):

View File

@ -0,0 +1,403 @@
var packet_list = {};
var tx_data = [];
var rx_data = [];
var packet_types_data = {};
var mem_current = []
var mem_peak = []
function start_charts() {
console.log("start_charts() called");
// Initialize the echarts instance based on the prepared dom
create_packets_chart();
create_packets_types_chart();
create_messages_chart();
create_ack_chart();
create_memory_chart();
}
function create_packets_chart() {
// The packets totals TX/RX chart.
pkt_c_canvas = document.getElementById('packetsChart');
packets_chart = echarts.init(pkt_c_canvas);
// Specify the configuration items and data for the chart
var option = {
title: {
text: 'APRS Packet totals'
},
legend: {},
tooltip : {
trigger: 'axis'
},
toolbox: {
show : true,
feature : {
mark : {show: true},
dataView : {show: true, readOnly: true},
magicType : {show: true, type: ['line', 'bar']},
restore : {show: true},
saveAsImage : {show: true}
}
},
calculable : true,
xAxis: { type: 'time' },
yAxis: { },
series: [
{
name: 'tx',
type: 'line',
smooth: true,
color: 'red',
encode: {
x: 'timestamp',
y: 'tx' // refer sensor 1 value
}
},{
name: 'rx',
type: 'line',
smooth: true,
encode: {
x: 'timestamp',
y: 'rx'
}
}]
};
// Display the chart using the configuration items and data just specified.
packets_chart.setOption(option);
}
function create_packets_types_chart() {
// The packets types chart
pkt_types_canvas = document.getElementById('packetTypesChart');
packet_types_chart = echarts.init(pkt_types_canvas);
// The series and data are built and updated on the fly
// as packets come in.
var option = {
title: {
text: 'Packet Types'
},
legend: {},
tooltip : {
trigger: 'axis'
},
toolbox: {
show : true,
feature : {
mark : {show: true},
dataView : {show: true, readOnly: true},
magicType : {show: true, type: ['line', 'bar']},
restore : {show: true},
saveAsImage : {show: true}
}
},
calculable : true,
xAxis: { type: 'time' },
yAxis: { },
}
packet_types_chart.setOption(option);
}
function create_messages_chart() {
msg_c_canvas = document.getElementById('messagesChart');
message_chart = echarts.init(msg_c_canvas);
// Specify the configuration items and data for the chart
var option = {
title: {
text: 'Message Packets'
},
legend: {},
tooltip: {
trigger: 'axis'
},
toolbox: {
show: true,
feature: {
mark : {show: true},
dataView : {show: true, readOnly: true},
magicType : {show: true, type: ['line', 'bar']},
restore : {show: true},
saveAsImage : {show: true}
}
},
calculable: true,
xAxis: { type: 'time' },
yAxis: { },
series: [
{
name: 'tx',
type: 'line',
smooth: true,
color: 'red',
encode: {
x: 'timestamp',
y: 'tx' // refer sensor 1 value
}
},{
name: 'rx',
type: 'line',
smooth: true,
encode: {
x: 'timestamp',
y: 'rx'
}
}]
};
// Display the chart using the configuration items and data just specified.
message_chart.setOption(option);
}
function create_ack_chart() {
ack_canvas = document.getElementById('acksChart');
ack_chart = echarts.init(ack_canvas);
// Specify the configuration items and data for the chart
var option = {
title: {
text: 'Ack Packets'
},
legend: {},
tooltip: {
trigger: 'axis'
},
toolbox: {
show: true,
feature: {
mark : {show: true},
dataView : {show: true, readOnly: false},
magicType : {show: true, type: ['line', 'bar']},
restore : {show: true},
saveAsImage : {show: true}
}
},
calculable: true,
xAxis: { type: 'time' },
yAxis: { },
series: [
{
name: 'tx',
type: 'line',
smooth: true,
color: 'red',
encode: {
x: 'timestamp',
y: 'tx' // refer sensor 1 value
}
},{
name: 'rx',
type: 'line',
smooth: true,
encode: {
x: 'timestamp',
y: 'rx'
}
}]
};
ack_chart.setOption(option);
}
function create_memory_chart() {
ack_canvas = document.getElementById('memChart');
memory_chart = echarts.init(ack_canvas);
// Specify the configuration items and data for the chart
var option = {
title: {
text: 'Memory Usage'
},
legend: {},
tooltip: {
trigger: 'axis'
},
toolbox: {
show: true,
feature: {
mark : {show: true},
dataView : {show: true, readOnly: false},
magicType : {show: true, type: ['line', 'bar']},
restore : {show: true},
saveAsImage : {show: true}
}
},
calculable: true,
xAxis: { type: 'time' },
yAxis: { },
series: [
{
name: 'current',
type: 'line',
smooth: true,
color: 'red',
encode: {
x: 'timestamp',
y: 'current' // refer sensor 1 value
}
},{
name: 'peak',
type: 'line',
smooth: true,
encode: {
x: 'timestamp',
y: 'peak'
}
}]
};
memory_chart.setOption(option);
}
function updatePacketData(chart, time, first, second) {
tx_data.push([time, first]);
rx_data.push([time, second]);
option = {
series: [
{
name: 'tx',
data: tx_data,
},
{
name: 'rx',
data: rx_data,
}
]
}
chart.setOption(option);
}
function updatePacketTypesData(time, typesdata) {
//The options series is created on the fly each time based on
//the packet types we have in the data
var series = []
for (const k in typesdata) {
tx = [time, typesdata[k]["tx"]]
rx = [time, typesdata[k]["rx"]]
if (packet_types_data.hasOwnProperty(k)) {
packet_types_data[k]["tx"].push(tx)
packet_types_data[k]["rx"].push(rx)
} else {
packet_types_data[k] = {'tx': [tx], 'rx': [rx]}
}
}
}
function updatePacketTypesChart() {
series = []
for (const k in packet_types_data) {
entry = {
name: k+"tx",
data: packet_types_data[k]["tx"],
type: 'line',
smooth: true,
encode: {
x: 'timestamp',
y: k+'tx' // refer sensor 1 value
}
}
series.push(entry)
entry = {
name: k+"rx",
data: packet_types_data[k]["rx"],
type: 'line',
smooth: true,
encode: {
x: 'timestamp',
y: k+'rx' // refer sensor 1 value
}
}
series.push(entry)
}
option = {
series: series
}
console.log(option)
packet_types_chart.setOption(option);
}
function updateTypeChart(chart, key) {
//Generic function to update a packet type chart
if (! packet_types_data.hasOwnProperty(key)) {
return;
}
if (! packet_types_data[key].hasOwnProperty('tx')) {
return;
}
var option = {
series: [{
name: "tx",
data: packet_types_data[key]["tx"],
},
{
name: "rx",
data: packet_types_data[key]["rx"]
}]
}
chart.setOption(option);
}
function updateMemChart(time, current, peak) {
mem_current.push([time, current]);
mem_peak.push([time, peak]);
option = {
series: [
{
name: 'current',
data: mem_current,
},
{
name: 'peak',
data: mem_peak,
}
]
}
memory_chart.setOption(option);
}
function updateMessagesChart() {
updateTypeChart(message_chart, "MessagePacket")
}
function updateAcksChart() {
updateTypeChart(ack_chart, "AckPacket")
}
function update_stats( data ) {
console.log(data);
our_callsign = data["stats"]["aprsd"]["callsign"];
$("#version").text( data["stats"]["aprsd"]["version"] );
$("#aprs_connection").html( data["aprs_connection"] );
$("#uptime").text( "uptime: " + data["stats"]["aprsd"]["uptime"] );
const html_pretty = Prism.highlight(JSON.stringify(data, null, '\t'), Prism.languages.json, 'json');
$("#jsonstats").html(html_pretty);
t = Date.parse(data["time"]);
ts = new Date(t);
updatePacketData(packets_chart, ts, data["stats"]["packets"]["sent"], data["stats"]["packets"]["received"]);
updatePacketTypesData(ts, data["stats"]["packets"]["types"]);
updatePacketTypesChart();
updateMessagesChart();
updateAcksChart();
updateMemChart(ts, data["stats"]["aprsd"]["memory_current"], data["stats"]["aprsd"]["memory_peak"]);
//updateQuadData(message_chart, short_time, data["stats"]["messages"]["sent"], data["stats"]["messages"]["received"], data["stats"]["messages"]["ack_sent"], data["stats"]["messages"]["ack_recieved"]);
//updateDualData(email_chart, short_time, data["stats"]["email"]["sent"], data["stats"]["email"]["recieved"]);
//updateDualData(memory_chart, short_time, data["stats"]["aprsd"]["memory_peak"], data["stats"]["aprsd"]["memory_current"]);
}

View File

@ -6,6 +6,7 @@
<script src="https://cdn.socket.io/4.7.1/socket.io.min.js" integrity="sha512-+NaO7d6gQ1YPxvc/qHIqZEchjGm207SszoNeMgppoqD/67fEqmc1edS8zrbxPD+4RQI3gDgT/83ihpFW61TG/Q==" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.4/dist/Chart.bundle.js"></script>
<script src="https://cdn.jsdelivr.net/npm/echarts@5.4.3/dist/echarts.min.js"></script>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/semantic-ui@2.4.2/dist/semantic.min.css">
<script src="https://cdn.jsdelivr.net/npm/semantic-ui@2.4.2/dist/semantic.min.js"></script>
@ -15,7 +16,7 @@
<link rel="stylesheet" href="/static/css/prism.css">
<script src="/static/js/prism.js"></script>
<script src="/static/js/main.js"></script>
<script src="/static/js/charts.js"></script>
<script src="/static/js/echarts.js"></script>
<script src="/static/js/tabs.js"></script>
<script src="/static/js/send-message.js"></script>
<script src="/static/js/logs.js"></script>
@ -83,6 +84,7 @@
<div class="item" data-tab="plugin-tab">Plugins</div>
<div class="item" data-tab="config-tab">Config</div>
<div class="item" data-tab="log-tab">LogFile</div>
<!-- <div class="item" data-tab="oslo-tab">OSLO CONFIG</div> //-->
<div class="item" data-tab="raw-tab">Raw JSON</div>
</div>
@ -92,25 +94,25 @@
<div class="ui equal width relaxed grid">
<div class="row">
<div class="column">
<div class="ui segment" style="height: 300px">
<canvas id="packetsChart"></canvas>
</div>
</div>
<div class="column">
<div class="ui segment" style="height: 300px">
<canvas id="messageChart"></canvas>
</div>
<div class="ui segment" style="height: 300px" id="packetsChart"></div>
</div>
</div>
<div class="row">
<div class="column">
<div class="ui segment" style="height: 300px">
<canvas id="emailChart"></canvas>
</div>
<div class="ui segment" style="height: 300px" id="packetTypesChart"></div>
</div>
</div>
<div class="row">
<div class="column">
<div class="ui segment" style="height: 300px" id="messagesChart"></div>
</div>
<div class="column">
<div class="ui segment" style="height: 300px">
<canvas id="memChart"></canvas>
<div class="ui segment" style="height: 300px" id="acksChart"></div>
</div>
</div>
<div class="row">
<div class="column">
<div class="ui segment" style="height: 300px" id="memChart">
</div>
</div>
</div>
@ -118,7 +120,7 @@
<div id="stats" class="two column">
<button class="ui button" id="toggleStats">Toggle raw json</button>
<pre id="jsonstats" class="language-json">{{ stats }}</pre>
</div> --!>
</div> //-->
</div>
</div>
@ -164,9 +166,15 @@
<pre id="logContainer" style="height: 600px;overflow-y:auto;overflow-x:auto;"><code id="logtext" class="language-log" ></code></pre>
</div>
<!--
<div class="ui bottom attached tab segment" data-tab="oslo-tab">
<h3 class="ui dividing header">OSLO</h3>
<pre id="osloContainer" style="height:600px;overflow-y:auto;" class="language-json">{{ oslo_out|safe }}</pre>
</div> //-->
<div class="ui bottom attached tab segment" data-tab="raw-tab">
<h3 class="ui dividing header">Raw JSON</h3>
<pre id="jsonstats" class="language-json">{{ stats|safe }}</pre>
<pre id="jsonstats" class="language-yaml" style="height:600px;overflow-y:auto;">{{ stats|safe }}</pre>
</div>
<div class="ui text container">

View File

@ -1,4 +1,6 @@
import datetime
import importlib.metadata as imp
import io
import json
import logging
from logging.handlers import RotatingFileHandler
@ -8,7 +10,7 @@ import flask
from flask import Flask
from flask.logging import default_handler
from flask_httpauth import HTTPBasicAuth
from oslo_config import cfg
from oslo_config import cfg, generator
import socketio
from werkzeug.security import check_password_hash
@ -96,9 +98,17 @@ def _stats():
if packet_list:
rx = packet_list.total_rx()
tx = packet_list.total_tx()
types = {}
types_copy = packet_list.types.copy()
for key in types_copy:
types[str(key)] = dict(types_copy[key])
stats_dict["packets"] = {
"sent": tx,
"received": rx,
"types": types,
}
if track:
size_tracker = len(track)
@ -123,7 +133,6 @@ def stats():
@app.route("/")
def index():
stats = _stats()
LOG.debug(stats)
wl = aprsd_rpc_client.RPCClient().get_watch_list()
if wl and wl.is_enabled():
watch_count = len(wl)
@ -185,6 +194,7 @@ def index():
watch_age=watch_age,
seen_count=seen_count,
plugin_count=plugin_count,
# oslo_out=generate_oslo()
)
@ -205,10 +215,12 @@ def get_packets():
LOG.debug("/packets called")
packet_list = aprsd_rpc_client.RPCClient().get_packet_list()
if packet_list:
packets = packet_list.get()
tmp_list = []
for pkt in packets:
tmp_list.append(pkt.json)
pkts = packet_list.copy()
for key in pkts:
pkt = packet_list.get(key)
if pkt:
tmp_list.append(pkt.json)
return json.dumps(tmp_list)
else:
@ -224,6 +236,35 @@ def plugins():
return "reloaded"
def _get_namespaces():
args = []
all = imp.entry_points()
selected = []
if "oslo.config.opts" in all:
for x in all["oslo.config.opts"]:
if x.group == "oslo.config.opts":
selected.append(x)
for entry in selected:
if "aprsd" in entry.name:
args.append("--namespace")
args.append(entry.name)
return args
def generate_oslo():
CONF.namespace = _get_namespaces()
string_out = io.StringIO()
generator.generate(CONF, string_out)
return string_out.getvalue()
@auth.login_required
@app.route("/oslo")
def oslo():
return generate_oslo()
@auth.login_required
@app.route("/save")
@ -348,6 +389,8 @@ if __name__ == "uwsgi_file_aprsd_wsgi":
log_level = init_app(
log_level="DEBUG",
config_file="/config/aprsd.conf",
# Commented out for local development.
# config_file=cli_helper.DEFAULT_CONFIG_FILE
)
setup_logging(app, log_level)
sio.register_namespace(LoggingNamespace("/logs"))
@ -362,7 +405,11 @@ if __name__ == "aprsd.wsgi":
sio = socketio.Server(logger=True, async_mode=async_mode)
app.wsgi_app = socketio.WSGIApp(sio, app.wsgi_app)
log_level = init_app(config_file="/config/aprsd.conf", log_level="DEBUG")
log_level = init_app(
log_level="DEBUG",
#config_file="/config/aprsd.conf",
config_file = cli_helper.DEFAULT_CONFIG_FILE,
)
setup_logging(app, log_level)
sio.register_namespace(LoggingNamespace("/logs"))
CONF.log_opt_values(LOG, logging.DEBUG)