Reworked the admin graphs

This patch fixes some bugs wth the rpc for packets as well
as reworks the admin graphs to use echarts.

aprsd/main.py

@@ -70,7 +70,7 @@ def main():
     # First import all the possible commands for the CLI
     # The commands themselves live in the cmds directory
     from .cmds import (  # noqa
-        completion, dev, fetch_stats, healthcheck, list_plugins, listen,
+        completion, config, dev, fetch_stats, healthcheck, list_plugins, listen,
         send_message, server, webchat,
     )
     cli(auto_envvar_prefix="APRSD")
@@ -145,6 +145,8 @@ def sample_config(ctx):
         if not sys.argv[1:]:
             raise SystemExit
         raise
+    LOG.warning(conf.namespace)
+    return
     generator.generate(conf)
 
 

aprsd/packets/core.py

@@ -108,6 +108,8 @@ class Packet(metaclass=abc.ABCMeta):
     #    hash=False
     #)
     last_send_time: float = field(repr=False, default=0, compare=False, hash=False)
+    last_send_attempt: int = field(repr=False, default=0, compare=False, hash=False)
+
     # Do we allow this packet to be saved to send later?
     allow_delay: bool = field(repr=False, default=True, compare=False, hash=False)
     path: List[str] = field(default_factory=list, compare=False, hash=False)

aprsd/packets/packet_list.py

@@ -19,11 +19,12 @@ class PacketList(MutableMapping):
     lock = threading.Lock()
     _total_rx: int = 0
     _total_tx: int = 0
+    types = {}
 
     def __new__(cls, *args, **kwargs):
         if cls._instance is None:
             cls._instance = super().__new__(cls)
-            cls._maxlen = 1000
+            cls._maxlen = 100
             cls.d = OrderedDict()
         return cls._instance
 
@@ -32,6 +33,10 @@ class PacketList(MutableMapping):
         """Add a packet that was received."""
         self._total_rx += 1
         self._add(packet)
+        ptype = packet.__class__.__name__
+        if not ptype in self.types:
+            self.types[ptype] = {"tx": 0, "rx": 0}
+        self.types[ptype]["rx"] += 1
         seen_list.SeenList().update_seen(packet)
         stats.APRSDStats().rx(packet)
 
@@ -40,6 +45,10 @@ class PacketList(MutableMapping):
         """Add a packet that was received."""
         self._total_tx += 1
         self._add(packet)
+        ptype = packet.__class__.__name__
+        if not ptype in self.types:
+            self.types[ptype] = {"tx": 0, "rx": 0}
+        self.types[ptype]["tx"] += 1
         seen_list.SeenList().update_seen(packet)
         stats.APRSDStats().tx(packet)
 
@@ -50,6 +59,10 @@ class PacketList(MutableMapping):
     def _add(self, packet):
         self[packet.key] = packet
 
+    def copy(self):
+        return self.d.copy()
+
+
     @property
     def maxlen(self):
         return self._maxlen

aprsd/packets/tracker.py

@@ -65,6 +65,7 @@ class PacketTrack(objectstore.ObjectStoreMixin):
     @wrapt.synchronized(lock)
     def add(self, packet):
         key = packet.msgNo
+        packet._last_send_attempt = 0
         self.data[key] = packet
         self.total_tracked += 1
 
@@ -83,7 +84,7 @@ class PacketTrack(objectstore.ObjectStoreMixin):
         """Walk the list of messages and restart them if any."""
         for key in self.data.keys():
             pkt = self.data[key]
-            if pkt.last_send_attempt < pkt.retry_count:
+            if pkt._last_send_attempt < pkt.retry_count:
                 tx.send(pkt)
 
     def _resend(self, packet):

aprsd/web/admin/static/js/echarts.js

@@ -0,0 +1,403 @@
+var packet_list = {};
+
+var tx_data = [];
+var rx_data = [];
+
+var packet_types_data = {};
+
+var mem_current = []
+var mem_peak = []
+
+
+function start_charts() {
+    console.log("start_charts() called");
+    // Initialize the echarts instance based on the prepared dom
+    create_packets_chart();
+    create_packets_types_chart();
+    create_messages_chart();
+    create_ack_chart();
+    create_memory_chart();
+}
+
+
+function create_packets_chart() {
+    // The packets totals TX/RX chart.
+    pkt_c_canvas = document.getElementById('packetsChart');
+    packets_chart = echarts.init(pkt_c_canvas);
+
+    // Specify the configuration items and data for the chart
+    var option = {
+      title: {
+          text: 'APRS Packet totals'
+      },
+      legend: {},
+      tooltip : {
+        trigger: 'axis'
+      },
+      toolbox: {
+        show : true,
+        feature : {
+           mark : {show: true},
+           dataView : {show: true, readOnly: true},
+           magicType : {show: true, type: ['line', 'bar']},
+           restore : {show: true},
+           saveAsImage : {show: true}
+        }
+      },
+      calculable : true,
+      xAxis: { type: 'time' },
+      yAxis: { },
+      series: [
+          {
+             name: 'tx',
+             type: 'line',
+             smooth: true,
+             color: 'red',
+             encode: {
+               x: 'timestamp',
+               y: 'tx' // refer sensor 1 value
+             }
+          },{
+             name: 'rx',
+             type: 'line',
+             smooth: true,
+             encode: {
+               x: 'timestamp',
+               y: 'rx'
+          }
+      }]
+    };
+
+    // Display the chart using the configuration items and data just specified.
+    packets_chart.setOption(option);
+}
+
+
+function create_packets_types_chart() {
+    // The packets types chart
+    pkt_types_canvas = document.getElementById('packetTypesChart');
+    packet_types_chart = echarts.init(pkt_types_canvas);
+
+    // The series and data are built and updated on the fly
+    // as packets come in.
+    var option = {
+      title: {
+          text: 'Packet Types'
+      },
+      legend: {},
+      tooltip : {
+        trigger: 'axis'
+      },
+      toolbox: {
+        show : true,
+        feature : {
+           mark : {show: true},
+           dataView : {show: true, readOnly: true},
+           magicType : {show: true, type: ['line', 'bar']},
+           restore : {show: true},
+           saveAsImage : {show: true}
+        }
+      },
+      calculable : true,
+      xAxis: { type: 'time' },
+      yAxis: { },
+    }
+
+    packet_types_chart.setOption(option);
+}
+
+
+function create_messages_chart() {
+    msg_c_canvas = document.getElementById('messagesChart');
+    message_chart = echarts.init(msg_c_canvas);
+
+    // Specify the configuration items and data for the chart
+    var option = {
+      title: {
+          text: 'Message Packets'
+      },
+      legend: {},
+      tooltip: {
+        trigger: 'axis'
+      },
+      toolbox: {
+        show: true,
+        feature: {
+           mark : {show: true},
+           dataView : {show: true, readOnly: true},
+           magicType : {show: true, type: ['line', 'bar']},
+           restore : {show: true},
+           saveAsImage : {show: true}
+        }
+      },
+      calculable: true,
+      xAxis: { type: 'time' },
+      yAxis: { },
+      series: [
+          {
+             name: 'tx',
+             type: 'line',
+             smooth: true,
+             color: 'red',
+             encode: {
+               x: 'timestamp',
+               y: 'tx' // refer sensor 1 value
+             }
+          },{
+             name: 'rx',
+             type: 'line',
+             smooth: true,
+             encode: {
+               x: 'timestamp',
+               y: 'rx'
+          }
+      }]
+    };
+
+    // Display the chart using the configuration items and data just specified.
+    message_chart.setOption(option);
+}
+
+function create_ack_chart() {
+    ack_canvas = document.getElementById('acksChart');
+    ack_chart = echarts.init(ack_canvas);
+
+    // Specify the configuration items and data for the chart
+    var option = {
+      title: {
+          text: 'Ack Packets'
+      },
+      legend: {},
+      tooltip: {
+        trigger: 'axis'
+      },
+      toolbox: {
+        show: true,
+        feature: {
+           mark : {show: true},
+           dataView : {show: true, readOnly: false},
+           magicType : {show: true, type: ['line', 'bar']},
+           restore : {show: true},
+           saveAsImage : {show: true}
+        }
+      },
+      calculable: true,
+      xAxis: { type: 'time' },
+      yAxis: { },
+      series: [
+          {
+             name: 'tx',
+             type: 'line',
+             smooth: true,
+             color: 'red',
+             encode: {
+               x: 'timestamp',
+               y: 'tx' // refer sensor 1 value
+             }
+          },{
+             name: 'rx',
+             type: 'line',
+             smooth: true,
+             encode: {
+               x: 'timestamp',
+               y: 'rx'
+          }
+      }]
+    };
+
+    ack_chart.setOption(option);
+}
+
+function create_memory_chart() {
+    ack_canvas = document.getElementById('memChart');
+    memory_chart = echarts.init(ack_canvas);
+
+    // Specify the configuration items and data for the chart
+    var option = {
+      title: {
+          text: 'Memory Usage'
+      },
+      legend: {},
+      tooltip: {
+        trigger: 'axis'
+      },
+      toolbox: {
+        show: true,
+        feature: {
+           mark : {show: true},
+           dataView : {show: true, readOnly: false},
+           magicType : {show: true, type: ['line', 'bar']},
+           restore : {show: true},
+           saveAsImage : {show: true}
+        }
+      },
+      calculable: true,
+      xAxis: { type: 'time' },
+      yAxis: { },
+      series: [
+          {
+             name: 'current',
+             type: 'line',
+             smooth: true,
+             color: 'red',
+             encode: {
+               x: 'timestamp',
+               y: 'current' // refer sensor 1 value
+             }
+          },{
+             name: 'peak',
+             type: 'line',
+             smooth: true,
+             encode: {
+               x: 'timestamp',
+               y: 'peak'
+          }
+      }]
+    };
+
+    memory_chart.setOption(option);
+}
+
+
+
+
+function updatePacketData(chart, time, first, second) {
+    tx_data.push([time, first]);
+    rx_data.push([time, second]);
+    option = {
+        series: [
+          {
+            name: 'tx',
+            data: tx_data,
+          },
+          {
+            name: 'rx',
+            data: rx_data,
+          }
+        ]
+    }
+    chart.setOption(option);
+}
+
+function updatePacketTypesData(time, typesdata) {
+    //The options series is created on the fly each time based on
+    //the packet types we have in the data
+    var series = []
+
+    for (const k in typesdata) {
+        tx = [time, typesdata[k]["tx"]]
+        rx = [time, typesdata[k]["rx"]]
+
+        if (packet_types_data.hasOwnProperty(k)) {
+            packet_types_data[k]["tx"].push(tx)
+            packet_types_data[k]["rx"].push(rx)
+        } else {
+            packet_types_data[k] = {'tx': [tx], 'rx': [rx]}
+        }
+    }
+}
+
+function updatePacketTypesChart() {
+  series = []
+  for (const k in packet_types_data) {
+      entry = {
+          name: k+"tx",
+          data: packet_types_data[k]["tx"],
+          type: 'line',
+          smooth: true,
+          encode: {
+            x: 'timestamp',
+            y: k+'tx' // refer sensor 1 value
+          }
+      }
+      series.push(entry)
+      entry = {
+          name: k+"rx",
+          data: packet_types_data[k]["rx"],
+          type: 'line',
+          smooth: true,
+          encode: {
+            x: 'timestamp',
+            y: k+'rx' // refer sensor 1 value
+          }
+      }
+      series.push(entry)
+  }
+
+  option = {
+      series: series
+  }
+  console.log(option)
+  packet_types_chart.setOption(option);
+}
+
+function updateTypeChart(chart, key)  {
+    //Generic function to update a packet type chart
+    if (! packet_types_data.hasOwnProperty(key)) {
+        return;
+    }
+
+    if (! packet_types_data[key].hasOwnProperty('tx')) {
+        return;
+    }
+    var option = {
+        series: [{
+            name: "tx",
+            data: packet_types_data[key]["tx"],
+        },
+        {
+            name: "rx",
+            data: packet_types_data[key]["rx"]
+        }]
+    }
+
+   chart.setOption(option);
+}
+
+function updateMemChart(time, current, peak) {
+    mem_current.push([time, current]);
+    mem_peak.push([time, peak]);
+    option = {
+        series: [
+          {
+            name: 'current',
+            data: mem_current,
+          },
+          {
+            name: 'peak',
+            data: mem_peak,
+          }
+        ]
+    }
+    memory_chart.setOption(option);
+}
+
+function updateMessagesChart() {
+    updateTypeChart(message_chart, "MessagePacket")
+}
+
+function updateAcksChart() {
+    updateTypeChart(ack_chart, "AckPacket")
+}
+
+function update_stats( data ) {
+    console.log(data);
+    our_callsign = data["stats"]["aprsd"]["callsign"];
+    $("#version").text( data["stats"]["aprsd"]["version"] );
+    $("#aprs_connection").html( data["aprs_connection"] );
+    $("#uptime").text( "uptime: " + data["stats"]["aprsd"]["uptime"] );
+    const html_pretty = Prism.highlight(JSON.stringify(data, null, '\t'), Prism.languages.json, 'json');
+    $("#jsonstats").html(html_pretty);
+
+    t = Date.parse(data["time"]);
+    ts = new Date(t);
+    updatePacketData(packets_chart, ts, data["stats"]["packets"]["sent"], data["stats"]["packets"]["received"]);
+    updatePacketTypesData(ts, data["stats"]["packets"]["types"]);
+    updatePacketTypesChart();
+    updateMessagesChart();
+    updateAcksChart();
+    updateMemChart(ts, data["stats"]["aprsd"]["memory_current"], data["stats"]["aprsd"]["memory_peak"]);
+    //updateQuadData(message_chart, short_time, data["stats"]["messages"]["sent"], data["stats"]["messages"]["received"], data["stats"]["messages"]["ack_sent"], data["stats"]["messages"]["ack_recieved"]);
+    //updateDualData(email_chart, short_time, data["stats"]["email"]["sent"], data["stats"]["email"]["recieved"]);
+    //updateDualData(memory_chart, short_time, data["stats"]["aprsd"]["memory_peak"], data["stats"]["aprsd"]["memory_current"]);
+}

aprsd/web/admin/templates/index.html

@@ -6,6 +6,7 @@
         <script src="https://cdn.socket.io/4.7.1/socket.io.min.js" integrity="sha512-+NaO7d6gQ1YPxvc/qHIqZEchjGm207SszoNeMgppoqD/67fEqmc1edS8zrbxPD+4RQI3gDgT/83ihpFW61TG/Q==" crossorigin="anonymous"></script>
 
         <script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.4/dist/Chart.bundle.js"></script>
+        <script src="https://cdn.jsdelivr.net/npm/echarts@5.4.3/dist/echarts.min.js"></script>
 
         <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/semantic-ui@2.4.2/dist/semantic.min.css">
         <script src="https://cdn.jsdelivr.net/npm/semantic-ui@2.4.2/dist/semantic.min.js"></script>
@@ -15,7 +16,7 @@
         <link rel="stylesheet" href="/static/css/prism.css">
         <script src="/static/js/prism.js"></script>
         <script src="/static/js/main.js"></script>
-        <script src="/static/js/charts.js"></script>
+        <script src="/static/js/echarts.js"></script>
         <script src="/static/js/tabs.js"></script>
         <script src="/static/js/send-message.js"></script>
         <script src="/static/js/logs.js"></script>
@@ -83,6 +84,7 @@
             <div class="item" data-tab="plugin-tab">Plugins</div>
             <div class="item" data-tab="config-tab">Config</div>
             <div class="item" data-tab="log-tab">LogFile</div>
+            <!-- <div class="item" data-tab="oslo-tab">OSLO CONFIG</div> //-->
             <div class="item" data-tab="raw-tab">Raw JSON</div>
         </div>
 
@@ -92,25 +94,25 @@
             <div class="ui equal width relaxed grid">
                 <div class="row">
                     <div class="column">
-                        <div class="ui segment" style="height: 300px">
+                        <div class="ui segment" style="height: 300px" id="packetsChart"></div>
-                            <canvas id="packetsChart"></canvas>
-                        </div>
-                    </div>
-                    <div class="column">
-                        <div class="ui segment" style="height: 300px">
-                            <canvas id="messageChart"></canvas>
-                        </div>
                     </div>
                 </div>
                 <div class="row">
                     <div class="column">
-                        <div class="ui segment" style="height: 300px">
+                        <div class="ui segment" style="height: 300px" id="packetTypesChart"></div>
-                            <canvas id="emailChart"></canvas>
+                    </div>
-                        </div>
+                </div>
+                <div class="row">
+                    <div class="column">
+                        <div class="ui segment" style="height: 300px" id="messagesChart"></div>
                     </div>
                     <div class="column">
-                        <div class="ui segment" style="height: 300px">
+                        <div class="ui segment" style="height: 300px" id="acksChart"></div>
-                            <canvas id="memChart"></canvas>
+                    </div>
+                </div>
+                <div class="row">
+                    <div class="column">
+                        <div class="ui segment" style="height: 300px" id="memChart">
                         </div>
                     </div>
                 </div>
@@ -118,7 +120,7 @@
                     <div id="stats" class="two column">
                         <button class="ui button" id="toggleStats">Toggle raw json</button>
                         <pre id="jsonstats" class="language-json">{{ stats }}</pre>
-                    </div> --!>
+                    </div> //-->
                 </div>
             </div>
 
@@ -164,9 +166,15 @@
             <pre id="logContainer" style="height: 600px;overflow-y:auto;overflow-x:auto;"><code id="logtext" class="language-log" ></code></pre>
         </div>
 
+        <!--
+        <div class="ui bottom attached tab segment" data-tab="oslo-tab">
+            <h3 class="ui dividing header">OSLO</h3>
+            <pre id="osloContainer" style="height:600px;overflow-y:auto;" class="language-json">{{ oslo_out|safe }}</pre>
+        </div> //-->
+
         <div class="ui bottom attached tab segment" data-tab="raw-tab">
           <h3 class="ui dividing header">Raw JSON</h3>
-          <pre id="jsonstats" class="language-json">{{ stats|safe }}</pre>
+          <pre id="jsonstats" class="language-yaml" style="height:600px;overflow-y:auto;">{{ stats|safe }}</pre>
         </div>
 
         <div class="ui text container">

aprsd/wsgi.py

@@ -1,4 +1,6 @@
 import datetime
+import importlib.metadata as imp
+import io
 import json
 import logging
 from logging.handlers import RotatingFileHandler
@@ -8,7 +10,7 @@ import flask
 from flask import Flask
 from flask.logging import default_handler
 from flask_httpauth import HTTPBasicAuth
-from oslo_config import cfg
+from oslo_config import cfg, generator
 import socketio
 from werkzeug.security import check_password_hash
 
@@ -96,9 +98,17 @@ def _stats():
     if packet_list:
         rx = packet_list.total_rx()
         tx = packet_list.total_tx()
+    types = {}
+
+    types_copy = packet_list.types.copy()
+
+    for key in types_copy:
+        types[str(key)] = dict(types_copy[key])
+
     stats_dict["packets"] = {
         "sent": tx,
         "received": rx,
+        "types": types,
     }
     if track:
         size_tracker = len(track)
@@ -123,7 +133,6 @@ def stats():
 @app.route("/")
 def index():
     stats = _stats()
-    LOG.debug(stats)
     wl = aprsd_rpc_client.RPCClient().get_watch_list()
     if wl and wl.is_enabled():
         watch_count = len(wl)
@@ -185,6 +194,7 @@ def index():
         watch_age=watch_age,
         seen_count=seen_count,
         plugin_count=plugin_count,
+        # oslo_out=generate_oslo()
     )
 
 
@@ -205,10 +215,12 @@ def get_packets():
     LOG.debug("/packets called")
     packet_list = aprsd_rpc_client.RPCClient().get_packet_list()
     if packet_list:
-        packets = packet_list.get()
         tmp_list = []
-        for pkt in packets:
+        pkts = packet_list.copy()
-            tmp_list.append(pkt.json)
+        for key in pkts:
+            pkt = packet_list.get(key)
+            if pkt:
+                tmp_list.append(pkt.json)
 
         return json.dumps(tmp_list)
     else:
@@ -224,6 +236,35 @@ def plugins():
 
     return "reloaded"
 
+def _get_namespaces():
+    args = []
+
+    all = imp.entry_points()
+    selected = []
+    if "oslo.config.opts" in all:
+        for x in all["oslo.config.opts"]:
+            if x.group == "oslo.config.opts":
+                selected.append(x)
+    for entry in selected:
+        if "aprsd" in entry.name:
+            args.append("--namespace")
+            args.append(entry.name)
+
+    return args
+
+
+def generate_oslo():
+    CONF.namespace = _get_namespaces()
+    string_out = io.StringIO()
+    generator.generate(CONF, string_out)
+    return string_out.getvalue()
+
+@auth.login_required
+@app.route("/oslo")
+def oslo():
+    return generate_oslo()
+
+
 
 @auth.login_required
 @app.route("/save")
@@ -348,6 +389,8 @@ if __name__ == "uwsgi_file_aprsd_wsgi":
     log_level = init_app(
         log_level="DEBUG",
         config_file="/config/aprsd.conf",
+        # Commented out for local development.
+        # config_file=cli_helper.DEFAULT_CONFIG_FILE
     )
     setup_logging(app, log_level)
     sio.register_namespace(LoggingNamespace("/logs"))
@@ -362,7 +405,11 @@ if __name__ == "aprsd.wsgi":
     sio = socketio.Server(logger=True, async_mode=async_mode)
     app.wsgi_app = socketio.WSGIApp(sio, app.wsgi_app)
 
-    log_level = init_app(config_file="/config/aprsd.conf", log_level="DEBUG")
+    log_level = init_app(
+        log_level="DEBUG",
+        #config_file="/config/aprsd.conf",
+        config_file = cli_helper.DEFAULT_CONFIG_FILE,
+    )
     setup_logging(app, log_level)
     sio.register_namespace(LoggingNamespace("/logs"))
     CONF.log_opt_values(LOG, logging.DEBUG)