From e1183a7e302536d186931cc0f38e2c846b9bc516 Mon Sep 17 00:00:00 2001 From: Hemna Date: Mon, 17 Jul 2023 22:09:18 +0000 Subject: [PATCH 1/4] Remove flask pinning Also removed need for flask-classful. Created new aprsd/wsgi.py for the web admin interface. --- aprsd/wsgi.py | 332 ++++++++++++++++++++++++++++++++++++++++++- dev-requirements.txt | 20 ++- requirements.in | 5 +- requirements.txt | 15 +- 4 files changed, 344 insertions(+), 28 deletions(-) diff --git a/aprsd/wsgi.py b/aprsd/wsgi.py index c5278f5..c68511b 100644 --- a/aprsd/wsgi.py +++ b/aprsd/wsgi.py @@ -1,12 +1,334 @@ +import datetime +import json import logging +from logging.handlers import RotatingFileHandler +import time +import flask +from flask import Flask +from flask.logging import default_handler +from flask_httpauth import HTTPBasicAuth +from flask_socketio import Namespace, SocketIO from oslo_config import cfg +from werkzeug.security import check_password_hash -from aprsd import admin_web -from aprsd import conf # noqa +import aprsd +from aprsd import cli_helper, client, conf, packets, plugin, threads +from aprsd.log import rich as aprsd_logging +from aprsd.rpc import client as aprsd_rpc_client CONF = cfg.CONF -LOG = logging.getLogger("APRSD") -app = None -app = admin_web.create_app() +LOG = logging.getLogger("gunicorn.access") + +auth = HTTPBasicAuth() +users = {} +app = Flask( + "aprsd", + static_url_path="/static", + static_folder="web/admin/static", + template_folder="web/admin/templates", +) +socket_io = SocketIO(app) + + +# HTTPBasicAuth doesn't work on a class method. +# This has to be out here. Rely on the APRSDFlask +# class to initialize the users from the config +@auth.verify_password +def verify_password(username, password): + global users + + if username in users and check_password_hash(users.get(username), password): + return username + + +def _stats(): + track = aprsd_rpc_client.RPCClient().get_packet_track() + now = datetime.datetime.now() + + time_format = "%m-%d-%Y %H:%M:%S" + + stats_dict = aprsd_rpc_client.RPCClient().get_stats_dict() + if not stats_dict: + stats_dict = { + "aprsd": {}, + "aprs-is": {"server": ""}, + "messages": { + "sent": 0, + "received": 0, + }, + "email": { + "sent": 0, + "received": 0, + }, + "seen_list": { + "sent": 0, + "received": 0, + }, + } + + # Convert the watch_list entries to age + wl = aprsd_rpc_client.RPCClient().get_watch_list() + new_list = {} + if wl: + for call in wl.get_all(): + # call_date = datetime.datetime.strptime( + # str(wl.last_seen(call)), + # "%Y-%m-%d %H:%M:%S.%f", + # ) + + # We have to convert the RingBuffer to a real list + # so that json.dumps works. + # pkts = [] + # for pkt in wl.get(call)["packets"].get(): + # pkts.append(pkt) + + new_list[call] = { + "last": wl.age(call), + # "packets": pkts + } + + stats_dict["aprsd"]["watch_list"] = new_list + packet_list = aprsd_rpc_client.RPCClient().get_packet_list() + rx = tx = 0 + if packet_list: + rx = packet_list.total_rx() + tx = packet_list.total_tx() + stats_dict["packets"] = { + "sent": tx, + "received": rx, + } + if track: + size_tracker = len(track) + else: + size_tracker = 0 + + result = { + "time": now.strftime(time_format), + "size_tracker": size_tracker, + "stats": stats_dict, + } + + return result + + +@app.route("/stats") +def stats(): + LOG.debug("/stats called") + return json.dumps(_stats()) + + +@auth.login_required +@app.route("/") +def index(): + stats = _stats() + LOG.debug(stats) + wl = aprsd_rpc_client.RPCClient().get_watch_list() + if wl and wl.is_enabled(): + watch_count = len(wl) + watch_age = wl.max_delta() + else: + watch_count = 0 + watch_age = 0 + + sl = aprsd_rpc_client.RPCClient().get_seen_list() + if sl: + seen_count = len(sl) + else: + seen_count = 0 + + pm = plugin.PluginManager() + plugins = pm.get_plugins() + plugin_count = len(plugins) + + if CONF.aprs_network.enabled: + transport = "aprs-is" + aprs_connection = ( + "APRS-IS Server: " + "{}".format(stats["stats"]["aprs-is"]["server"]) + ) + else: + # We might be connected to a KISS socket? + if client.KISSClient.kiss_enabled(): + transport = client.KISSClient.transport() + if transport == client.TRANSPORT_TCPKISS: + aprs_connection = ( + "TCPKISS://{}:{}".format( + CONF.kiss_tcp.host, + CONF.kiss_tcp.port, + ) + ) + elif transport == client.TRANSPORT_SERIALKISS: + aprs_connection = ( + "SerialKISS://{}@{} baud".format( + CONF.kiss_serial.device, + CONF.kiss_serial.baudrate, + ) + ) + + stats["transport"] = transport + stats["aprs_connection"] = aprs_connection + entries = conf.conf_to_dict() + + return flask.render_template( + "index.html", + initial_stats=stats, + aprs_connection=aprs_connection, + callsign=CONF.callsign, + version=aprsd.__version__, + config_json=json.dumps( + entries, indent=4, + sort_keys=True, default=str, + ), + watch_count=watch_count, + watch_age=watch_age, + seen_count=seen_count, + plugin_count=plugin_count, + ) + +@auth.login_required +def messages(): + track = packets.PacketTrack() + msgs = [] + for id in track: + LOG.info(track[id].dict()) + msgs.append(track[id].dict()) + + return flask.render_template("messages.html", messages=json.dumps(msgs)) + +@auth.login_required +@app.route("/packets") +def packets(): + LOG.debug("/packets called") + packet_list = aprsd_rpc_client.RPCClient().get_packet_list() + if packet_list: + packets = packet_list.get() + tmp_list = [] + for pkt in packets: + tmp_list.append(pkt.json) + + return json.dumps(tmp_list) + else: + return json.dumps([]) + +@auth.login_required +@app.route("/plugins") +def plugins(): + LOG.debug("/plugins called") + pm = plugin.PluginManager() + pm.reload_plugins() + + return "reloaded" + +@auth.login_required +@app.route("/save") +def save(): + """Save the existing queue to disk.""" + track = packets.PacketTrack() + track.save() + return json.dumps({"messages": "saved"}) + + + +class LogUpdateThread(threads.APRSDThread): + + def __init__(self): + super().__init__("LogUpdate") + + def loop(self): + global socket_io + + if socket_io: + log_entries = aprsd_rpc_client.RPCClient().get_log_entries() + + if log_entries: + for entry in log_entries: + socket_io.emit( + "log_entry", entry, + namespace="/logs", + ) + + time.sleep(5) + return True + + +class LoggingNamespace(Namespace): + log_thread = None + + def on_connect(self): + global socket_io + socket_io.emit( + "connected", {"data": "/logs Connected"}, + namespace="/logs", + ) + self.log_thread = LogUpdateThread() + self.log_thread.start() + + def on_disconnect(self): + LOG.debug("LOG Disconnected") + if self.log_thread: + self.log_thread.stop() + + +def setup_logging(flask_app, loglevel): + flask_log = logging.getLogger("werkzeug") + flask_app.logger.removeHandler(default_handler) + flask_log.removeHandler(default_handler) + LOG.handlers = [] + + log_level = conf.log.LOG_LEVELS[loglevel] + LOG.setLevel(log_level) + date_format = CONF.logging.date_format + flask_app.logger.disabled = True + gunicorn_err = logging.getLogger("gunicorn.error") + + if CONF.logging.rich_logging: + log_format = "%(message)s" + log_formatter = logging.Formatter(fmt=log_format, datefmt=date_format) + rh = aprsd_logging.APRSDRichHandler( + show_thread=True, thread_width=15, + rich_tracebacks=True, omit_repeated_times=False, + ) + rh.setFormatter(log_formatter) + LOG.addHandler(rh) + + log_file = CONF.logging.logfile + + if log_file: + log_format = CONF.logging.logformat + log_formatter = logging.Formatter(fmt=log_format, datefmt=date_format) + fh = RotatingFileHandler( + log_file, maxBytes=(10248576 * 5), + backupCount=4, + ) + fh.setFormatter(log_formatter) + LOG.addHandler(fh) + + gunicorn_err.handlers = LOG.handlers + + +def init_app(config_file=None, log_level=None): + default_config_file = cli_helper.DEFAULT_CONFIG_FILE + if not config_file: + config_file = default_config_file + + CONF( + [], project="aprsd", version=aprsd.__version__, + default_config_files=[config_file], + ) + + if not log_level: + log_level = CONF.logging.log_level + + return log_level + + +print(f"APP {__name__}") +if __name__ == "__main__": + socket_io.run(app) + +if __name__ == "aprsd.wsgi": + log_level = init_app(config_file="~/.config/aprsd/aprsd.conf", log_level="DEBUG") + socket_io.on_namespace(LoggingNamespace("/logs")) + setup_logging(app, log_level) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0f485d9..9dd9368 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --annotation-style=line dev-requirements.in @@ -31,13 +31,12 @@ gray==0.13.0 # via -r dev-requirements.in identify==2.5.24 # via pre-commit idna==3.4 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 # via sphinx -importlib-resources==6.0.0 # via fixit, jsonschema, jsonschema-specifications +importlib-resources==6.0.0 # via fixit iniconfig==2.0.0 # via pytest isort==5.12.0 # via -r dev-requirements.in, gray jinja2==3.1.2 # via sphinx -jsonschema==4.18.3 # via fixit -jsonschema-specifications==2023.6.1 # via jsonschema +jsonschema==4.18.4 # via fixit +jsonschema-specifications==2023.7.1 # via jsonschema libcst==1.0.1 # via fixit markupsafe==2.1.3 # via jinja2 mccabe==0.7.0 # via flake8 @@ -48,7 +47,6 @@ packaging==23.1 # via black, build, pyproject-api, pytest, sphinx, tox pathspec==0.11.1 # via black pep8-naming==0.13.3 # via -r dev-requirements.in pip-tools==7.0.0 # via -r dev-requirements.in -pkgutil-resolve-name==1.3.10 # via jsonschema platformdirs==3.9.1 # via black, tox, virtualenv pluggy==1.2.0 # via pytest, tox pre-commit==3.3.3 # via -r dev-requirements.in @@ -59,13 +57,12 @@ pyproject-api==1.5.3 # via tox pyproject-hooks==1.0.0 # via build pytest==7.4.0 # via -r dev-requirements.in, pytest-cov pytest-cov==4.1.0 # via -r dev-requirements.in -pytz==2023.3 # via babel pyupgrade==3.9.0 # via gray -pyyaml==6.0 # via fixit, libcst, pre-commit -referencing==0.29.1 # via jsonschema, jsonschema-specifications +pyyaml==6.0.1 # via fixit, libcst, pre-commit +referencing==0.30.0 # via jsonschema, jsonschema-specifications requests==2.31.0 # via sphinx rich==12.6.0 # via gray -rpds-py==0.8.11 # via jsonschema, referencing +rpds-py==0.9.2 # via jsonschema, referencing snowballstemmer==2.2.0 # via sphinx sphinx==7.0.1 # via -r dev-requirements.in sphinxcontrib-applehelp==1.0.4 # via sphinx @@ -78,14 +75,13 @@ tokenize-rt==5.1.0 # via add-trailing-comma, pyupgrade toml==0.10.2 # via autoflake tomli==2.0.1 # via black, build, coverage, mypy, pip-tools, pyproject-api, pyproject-hooks, pytest, tox tox==4.6.4 # via -r dev-requirements.in -typing-extensions==4.7.1 # via black, libcst, mypy, rich, typing-inspect +typing-extensions==4.7.1 # via libcst, mypy, typing-inspect typing-inspect==0.9.0 # via libcst unify==0.5 # via gray untokenize==0.1.1 # via unify urllib3==2.0.3 # via requests virtualenv==20.24.0 # via pre-commit, tox wheel==0.40.0 # via pip-tools -zipp==3.16.2 # via importlib-metadata, importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements.in b/requirements.in index 766f757..dc69a76 100644 --- a/requirements.in +++ b/requirements.in @@ -2,9 +2,8 @@ aprslib>=0.7.0 click click-params click-completion -flask==2.1.2 -werkzeug==2.1.2 -flask-classful +flask +werkzeug flask-httpauth imapclient pluggy diff --git a/requirements.txt b/requirements.txt index c6200f1..4ab36c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --annotation-style=line requirements.in @@ -11,6 +11,7 @@ ax253==0.1.5.post1 # via kiss3 beautifulsoup4==4.12.2 # via -r requirements.in bidict==0.22.1 # via python-socketio bitarray==2.7.6 # via ax253, kiss3 +blinker==1.6.2 # via flask certifi==2023.5.7 # via httpcore, requests cffi==1.15.1 # via cryptography charset-normalizer==3.2.0 # via requests @@ -26,8 +27,7 @@ decorator==5.1.1 # via validators dnspython==2.4.0 # via eventlet eventlet==0.33.3 # via -r requirements.in exceptiongroup==1.1.2 # via anyio -flask==2.1.2 # via -r requirements.in, flask-classful, flask-httpauth, flask-socketio -flask-classful==0.14.2 # via -r requirements.in +flask==2.3.2 # via -r requirements.in, flask-httpauth, flask-socketio flask-httpauth==4.8.0 # via -r requirements.in flask-socketio==5.3.4 # via -r requirements.in geographiclib==2.0 # via geopy @@ -37,11 +37,11 @@ h11==0.14.0 # via httpcore httpcore==0.17.3 # via dnspython idna==3.4 # via anyio, requests imapclient==2.3.1 # via -r requirements.in -importlib-metadata==6.8.0 # via ax253, flask, kiss3 +importlib-metadata==6.8.0 # via ax253, kiss3 itsdangerous==2.1.2 # via flask jinja2==3.1.2 # via click-completion, flask kiss3==8.0.0 # via -r requirements.in -markupsafe==2.1.3 # via jinja2 +markupsafe==2.1.3 # via jinja2, werkzeug netaddr==0.8.0 # via oslo-config oslo-config==9.1.1 # via -r requirements.in oslo-i18n==6.0.0 # via oslo-config @@ -56,7 +56,7 @@ pyserial-asyncio==0.6 # via kiss3 python-engineio==4.5.1 # via python-socketio python-socketio==5.8.0 # via flask-socketio pytz==2023.3 # via -r requirements.in -pyyaml==6.0 # via -r requirements.in, oslo-config +pyyaml==6.0.1 # via -r requirements.in, oslo-config requests==2.31.0 # via -r requirements.in, oslo-config, update-checker rfc3986==2.0.0 # via oslo-config rich==12.6.0 # via -r requirements.in @@ -69,12 +69,11 @@ soupsieve==2.4.1 # via beautifulsoup4 stevedore==5.1.0 # via oslo-config tabulate==0.9.0 # via -r requirements.in thesmuggler==1.0.1 # via -r requirements.in -typing-extensions==4.7.1 # via rich ua-parser==0.18.0 # via user-agents update-checker==0.18.0 # via -r requirements.in urllib3==2.0.3 # via requests user-agents==2.2.0 # via -r requirements.in validators==0.20.0 # via click-params -werkzeug==2.1.2 # via -r requirements.in, flask +werkzeug==2.3.6 # via -r requirements.in, flask wrapt==1.15.0 # via -r requirements.in, debtcollector zipp==3.16.2 # via importlib-metadata From 6a6e854caf18fc01b6f38badbffe38272c73508c Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 19 Jul 2023 11:27:34 -0400 Subject: [PATCH 2/4] Removed flask-classful from webchat This patch removed the dependency on flask-classful. This required making all of the flask web routing non class based. This patch also changes the aprsis class to allow retries for failed connections when the aprsis servers are full and not responding to login requests. --- aprsd/client.py | 3 +- aprsd/clients/aprsis.py | 17 +-- aprsd/cmds/webchat.py | 216 ++++++++++++++++++------------------- aprsd/wsgi.py | 7 +- tests/cmds/test_webchat.py | 4 +- 5 files changed, 127 insertions(+), 120 deletions(-) diff --git a/aprsd/client.py b/aprsd/client.py index 9c9522d..08df034 100644 --- a/aprsd/client.py +++ b/aprsd/client.py @@ -152,9 +152,10 @@ class APRSISClient(Client): except LoginError as e: LOG.error(f"Failed to login to APRS-IS Server '{e}'") connected = False - raise e + time.sleep(backoff) except Exception as e: LOG.error(f"Unable to connect to APRS-IS server. '{e}' ") + connected = False time.sleep(backoff) backoff = backoff * 2 continue diff --git a/aprsd/clients/aprsis.py b/aprsd/clients/aprsis.py index 5ba53b1..aa1a9ba 100644 --- a/aprsd/clients/aprsis.py +++ b/aprsd/clients/aprsis.py @@ -112,23 +112,23 @@ class Aprsdis(aprslib.IS): self._sendall(login_str) self.sock.settimeout(5) test = self.sock.recv(len(login_str) + 100) + self.logger.debug("Server: '%s'", test) if is_py3: test = test.decode("latin-1") test = test.rstrip() - self.logger.debug("Server: %s", test) + self.logger.debug("Server: '%s'", test) - a, b, callsign, status, e = test.split(" ", 4) + if not test: + raise LoginError(f"Server Response Empty: '{test}'") + + _, _, callsign, status, e = test.split(" ", 4) s = e.split(",") if len(s): server_string = s[0].replace("server ", "") else: server_string = e.replace("server ", "") - self.logger.info(f"Connected to {server_string}") - self.server_string = server_string - stats.APRSDStats().set_aprsis_server(server_string) - if callsign == "": raise LoginError("Server responded with empty callsign???") if callsign != self.callsign: @@ -141,6 +141,10 @@ class Aprsdis(aprslib.IS): else: self.logger.info("Login successful") + self.logger.info(f"Connected to {server_string}") + self.server_string = server_string + stats.APRSDStats().set_aprsis_server(server_string) + except LoginError as e: self.logger.error(str(e)) self.close() @@ -148,6 +152,7 @@ class Aprsdis(aprslib.IS): except Exception as e: self.close() self.logger.error(f"Failed to login '{e}'") + self.logger.exception(e) raise LoginError("Failed to login") def consumer(self, callback, blocking=True, immortal=False, raw=False): diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index c0e719d..fbdc87d 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -12,7 +12,6 @@ import click import flask from flask import request from flask.logging import default_handler -import flask_classful from flask_httpauth import HTTPBasicAuth from flask_socketio import Namespace, SocketIO from oslo_config import cfg @@ -31,9 +30,16 @@ from aprsd.utils import objectstore, trace CONF = cfg.CONF LOG = logging.getLogger("APRSD") auth = HTTPBasicAuth() -users = None +users = {} socketio = None +flask_app = flask.Flask( + "aprsd", + static_url_path="/static", + static_folder="web/chat/static", + template_folder="web/chat/templates", +) + def signal_handler(sig, frame): @@ -174,106 +180,108 @@ class WebChatProcessPacketThread(rx.APRSDProcessPacketThread): ) -class WebChatFlask(flask_classful.FlaskView): +def set_config(): + global users - def set_config(self): - global users - self.users = {} - user = CONF.admin.user - self.users[user] = generate_password_hash(CONF.admin.password) - users = self.users - def _get_transport(self, stats): - if CONF.aprs_network.enabled: - transport = "aprs-is" - aprs_connection = ( - "APRS-IS Server: " - "{}".format(stats["stats"]["aprs-is"]["server"]) - ) - else: - # We might be connected to a KISS socket? - if client.KISSClient.is_enabled(): - transport = client.KISSClient.transport() - if transport == client.TRANSPORT_TCPKISS: - aprs_connection = ( - "TCPKISS://{}:{}".format( - CONF.kiss_tcp.host, - CONF.kiss_tcp.port, - ) - ) - elif transport == client.TRANSPORT_SERIALKISS: - # for pep8 violation - aprs_connection = ( - "SerialKISS://{}@{} baud".format( - CONF.kiss_serial.device, - CONF.kiss_serial.baudrate, - ), - ) - - return transport, aprs_connection - - @auth.login_required - def index(self): - ua_str = request.headers.get("User-Agent") - # this takes about 2 seconds :( - user_agent = ua_parse(ua_str) - LOG.debug(f"Is mobile? {user_agent.is_mobile}") - stats = self._stats() - - if user_agent.is_mobile: - html_template = "mobile.html" - else: - html_template = "index.html" - - # For development - # html_template = "mobile.html" - - LOG.debug(f"Template {html_template}") - - transport, aprs_connection = self._get_transport(stats) - LOG.debug(f"transport {transport} aprs_connection {aprs_connection}") - - stats["transport"] = transport - stats["aprs_connection"] = aprs_connection - LOG.debug(f"initial stats = {stats}") - - return flask.render_template( - html_template, - initial_stats=stats, - aprs_connection=aprs_connection, - callsign=CONF.callsign, - version=aprsd.__version__, +def _get_transport(stats): + if CONF.aprs_network.enabled: + transport = "aprs-is" + aprs_connection = ( + "APRS-IS Server: " + "{}".format(stats["stats"]["aprs-is"]["server"]) ) + else: + # We might be connected to a KISS socket? + if client.KISSClient.is_enabled(): + transport = client.KISSClient.transport() + if transport == client.TRANSPORT_TCPKISS: + aprs_connection = ( + "TCPKISS://{}:{}".format( + CONF.kiss_tcp.host, + CONF.kiss_tcp.port, + ) + ) + elif transport == client.TRANSPORT_SERIALKISS: + # for pep8 violation + aprs_connection = ( + "SerialKISS://{}@{} baud".format( + CONF.kiss_serial.device, + CONF.kiss_serial.baudrate, + ), + ) - @auth.login_required - def send_message_status(self): - LOG.debug(request) - msgs = SentMessages() - info = msgs.get_all() - return json.dumps(info) + return transport, aprs_connection - def _stats(self): - stats_obj = stats.APRSDStats() - now = datetime.datetime.now() - time_format = "%m-%d-%Y %H:%M:%S" - stats_dict = stats_obj.stats() - # Webchat doesnt need these - del stats_dict["aprsd"]["watch_list"] - del stats_dict["aprsd"]["seen_list"] - # del stats_dict["email"] - # del stats_dict["plugins"] - # del stats_dict["messages"] +@auth.login_required +@flask_app.route("/") +def index(): + ua_str = request.headers.get("User-Agent") + # this takes about 2 seconds :( + user_agent = ua_parse(ua_str) + LOG.debug(f"Is mobile? {user_agent.is_mobile}") + stats = _stats() - result = { - "time": now.strftime(time_format), - "stats": stats_dict, - } + if user_agent.is_mobile: + html_template = "mobile.html" + else: + html_template = "index.html" - return result + # For development + # html_template = "mobile.html" - def stats(self): - return json.dumps(self._stats()) + LOG.debug(f"Template {html_template}") + + transport, aprs_connection = _get_transport(stats) + LOG.debug(f"transport {transport} aprs_connection {aprs_connection}") + + stats["transport"] = transport + stats["aprs_connection"] = aprs_connection + LOG.debug(f"initial stats = {stats}") + + return flask.render_template( + html_template, + initial_stats=stats, + aprs_connection=aprs_connection, + callsign=CONF.callsign, + version=aprsd.__version__, + ) + + +@auth.login_required +@flask_app.route("//send-message-status") +def send_message_status(): + LOG.debug(request) + msgs = SentMessages() + info = msgs.get_all() + return json.dumps(info) + + +def _stats(): + stats_obj = stats.APRSDStats() + now = datetime.datetime.now() + + time_format = "%m-%d-%Y %H:%M:%S" + stats_dict = stats_obj.stats() + # Webchat doesnt need these + del stats_dict["aprsd"]["watch_list"] + del stats_dict["aprsd"]["seen_list"] + # del stats_dict["email"] + # del stats_dict["plugins"] + # del stats_dict["messages"] + + result = { + "time": now.strftime(time_format), + "stats": stats_dict, + } + + return result + + +@flask_app.route("/stats") +def get_stats(): + return json.dumps(_stats()) class SendMessageNamespace(Namespace): @@ -377,21 +385,9 @@ def setup_logging(flask_app, loglevel, quiet): @trace.trace def init_flask(loglevel, quiet): - global socketio + global socketio, flask_app - flask_app = flask.Flask( - "aprsd", - static_url_path="/static", - static_folder="web/chat/static", - template_folder="web/chat/templates", - ) setup_logging(flask_app, loglevel, quiet) - server = WebChatFlask() - server.set_config() - flask_app.route("/", methods=["GET"])(server.index) - flask_app.route("/stats", methods=["GET"])(server.stats) - # flask_app.route("/send-message", methods=["GET"])(server.send_message) - flask_app.route("/send-message-status", methods=["GET"])(server.send_message_status) socketio = SocketIO( flask_app, logger=False, engineio_logger=False, @@ -407,7 +403,7 @@ def init_flask(loglevel, quiet): "/sendmsg", ), ) - return socketio, flask_app + return socketio # main() ### @@ -448,6 +444,8 @@ def webchat(ctx, flush, port): LOG.info(f"APRSD Started version: {aprsd.__version__}") CONF.log_opt_values(LOG, logging.DEBUG) + user = CONF.admin.user + users[user] = generate_password_hash(CONF.admin.password) # Initialize the client factory and create # The correct client object ready for use @@ -466,7 +464,7 @@ def webchat(ctx, flush, port): packets.WatchList() packets.SeenList() - (socketio, app) = init_flask(loglevel, quiet) + socketio = init_flask(loglevel, quiet) rx_thread = rx.APRSDPluginRXThread( packet_queue=threads.packet_queue, ) @@ -482,7 +480,7 @@ def webchat(ctx, flush, port): keepalive.start() LOG.info("Start socketio.run()") socketio.run( - app, + flask_app, ssl_context="adhoc", host=CONF.admin.web_ip, port=port, diff --git a/aprsd/wsgi.py b/aprsd/wsgi.py index c68511b..17ffbe3 100644 --- a/aprsd/wsgi.py +++ b/aprsd/wsgi.py @@ -187,6 +187,7 @@ def index(): plugin_count=plugin_count, ) + @auth.login_required def messages(): track = packets.PacketTrack() @@ -197,9 +198,10 @@ def messages(): return flask.render_template("messages.html", messages=json.dumps(msgs)) + @auth.login_required @app.route("/packets") -def packets(): +def get_packets(): LOG.debug("/packets called") packet_list = aprsd_rpc_client.RPCClient().get_packet_list() if packet_list: @@ -212,6 +214,7 @@ def packets(): else: return json.dumps([]) + @auth.login_required @app.route("/plugins") def plugins(): @@ -221,6 +224,7 @@ def plugins(): return "reloaded" + @auth.login_required @app.route("/save") def save(): @@ -230,7 +234,6 @@ def save(): return json.dumps({"messages": "saved"}) - class LogUpdateThread(threads.APRSDThread): def __init__(self): diff --git a/tests/cmds/test_webchat.py b/tests/cmds/test_webchat.py index 3a0103d..53deb0d 100644 --- a/tests/cmds/test_webchat.py +++ b/tests/cmds/test_webchat.py @@ -39,9 +39,9 @@ class TestSendMessageCommand(unittest.TestCase): CliRunner() self.config_and_init() - socketio, flask_app = webchat.init_flask("DEBUG", False) + socketio = webchat.init_flask("DEBUG", False) self.assertIsInstance(socketio, flask_socketio.SocketIO) - self.assertIsInstance(flask_app, flask.Flask) + self.assertIsInstance(webchat.flask_app, flask.Flask) @mock.patch("aprsd.packets.tracker.PacketTrack.remove") @mock.patch("aprsd.cmds.webchat.socketio") From fa452cc773b4aacd4ecb56900e1f954887dea999 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 19 Jul 2023 18:50:42 +0000 Subject: [PATCH 3/4] Update docker bin/admin.sh This patch uses the wsgi.py instead of admin_Web.py --- docker/bin/admin.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/bin/admin.sh b/docker/bin/admin.sh index 3fb5235..7709d5d 100755 --- a/docker/bin/admin.sh +++ b/docker/bin/admin.sh @@ -27,5 +27,6 @@ if [ ! -e "$APRSD_CONFIG" ]; then fi export COLUMNS=200 -exec gunicorn -b :8000 --workers 4 "aprsd.admin_web:create_app(config_file='$APRSD_CONFIG', log_level='$LOG_LEVEL')" +#exec gunicorn -b :8000 --workers 4 "aprsd.admin_web:create_app(config_file='$APRSD_CONFIG', log_level='$LOG_LEVEL')" +exec gunicorn -b :8000 --workers 4 "aprsd.wsgi:app" #exec aprsd listen -c $APRSD_CONFIG --loglevel ${LOG_LEVEL} ${APRSD_LOAD_PLUGINS} ${APRSD_LISTEN_FILTER} From d3a93b735deb222d9a2c27015a413973c810f4d0 Mon Sep 17 00:00:00 2001 From: Hemna Date: Thu, 20 Jul 2023 14:44:46 -0400 Subject: [PATCH 4/4] Added timing after each thread loop This is to help keep track of which non-blocking threads are still alive. The RPC Server thread blocks, so the time will always increase. --- .github/workflows/release_build.yml | 2 +- aprsd/client.py | 2 ++ aprsd/threads/aprsd.py | 7 +++++++ aprsd/threads/keep_alive.py | 9 +++++++-- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_build.yml b/.github/workflows/release_build.yml index 70e5c93..a3838cf 100644 --- a/.github/workflows/release_build.yml +++ b/.github/workflows/release_build.yml @@ -39,7 +39,7 @@ jobs: uses: docker/build-push-action@v3 with: context: "{{defaultContext}}:docker" - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 file: ./Dockerfile build-args: | VERSION=${{ inputs.aprsd_version }} diff --git a/aprsd/client.py b/aprsd/client.py index 08df034..f110236 100644 --- a/aprsd/client.py +++ b/aprsd/client.py @@ -62,6 +62,8 @@ class Client: """Call this to force a rebuild/reconnect.""" if self._client: del self._client + else: + LOG.warning("Client not initialized, nothing to reset.") @abc.abstractmethod def setup_connection(self): diff --git a/aprsd/threads/aprsd.py b/aprsd/threads/aprsd.py index a6f7446..51d7960 100644 --- a/aprsd/threads/aprsd.py +++ b/aprsd/threads/aprsd.py @@ -1,4 +1,5 @@ import abc +import datetime import logging import threading @@ -50,6 +51,7 @@ class APRSDThread(threading.Thread, metaclass=abc.ABCMeta): super().__init__(name=name) self.thread_stop = False APRSDThreadList().add(self) + self._last_loop = datetime.datetime.now() def _should_quit(self): """ see if we have a quit message from the global queue.""" @@ -70,10 +72,15 @@ class APRSDThread(threading.Thread, metaclass=abc.ABCMeta): out = f"Thread <{self.__class__.__name__}({self.name}) Alive? {self.is_alive()}>" return out + def loop_age(self): + """How old is the last loop call?""" + return datetime.datetime.now() - self._last_loop + def run(self): LOG.debug("Starting") while not self._should_quit(): can_loop = self.loop() + self._last_loop = datetime.datetime.now() if not can_loop: self.stop() self._cleanup() diff --git a/aprsd/threads/keep_alive.py b/aprsd/threads/keep_alive.py index 0aae05c..9ea3282 100644 --- a/aprsd/threads/keep_alive.py +++ b/aprsd/threads/keep_alive.py @@ -68,8 +68,13 @@ class KeepAliveThread(APRSDThread): thread_info = {} for thread in thread_list.threads_list: alive = thread.is_alive() - thread_out.append(f"{thread.__class__.__name__}:{alive}") - thread_info[thread.__class__.__name__] = alive + age = thread.loop_age() + key = thread.__class__.__name__ + thread_out.append(f"{key}:{alive}:{age}") + if key not in thread_info: + thread_info[key] = {} + thread_info[key]["alive"] = alive + thread_info[key]["age"] = age if not alive: LOG.error(f"Thread {thread}") LOG.info(",".join(thread_out))