From 89576a3c43ca8b44c67183eeccfb5019d480bb9d Mon Sep 17 00:00:00 2001 From: Hemna Date: Sun, 23 Jul 2023 18:54:23 -0400 Subject: [PATCH 1/3] replacement of flask-socketio with python-socketio This patch starts the work to replace flask-socketio with python-socketio so that uwsgi can be used instead of gunicorn. uwsgi can support websockets. Have to rework webchat command next --- aprsd/rpc/client.py | 4 +- aprsd/web/admin/templates/index.html | 2 +- aprsd/wsgi.py | 81 ++++++++++++++++++---------- dev-requirements.txt | 16 +++--- docker/bin/admin.sh | 5 +- requirements.in | 2 + requirements.txt | 18 ++++--- 7 files changed, 82 insertions(+), 46 deletions(-) diff --git a/aprsd/rpc/client.py b/aprsd/rpc/client.py index cd01f11..6741e4d 100644 --- a/aprsd/rpc/client.py +++ b/aprsd/rpc/client.py @@ -56,7 +56,7 @@ class RPCClient: config={}, ipv6=False, keepalive=False, authorizer=None, ): - print(f"Connecting to RPC host {host}:{port}") + LOG.info(f"Connecting to RPC host '{host}:{port}'") try: s = rpc.AuthSocketStream.connect( host, port, ipv6=ipv6, keepalive=keepalive, @@ -64,7 +64,7 @@ class RPCClient: ) return rpyc.utils.factory.connect_stream(s, service, config=config) except ConnectionRefusedError: - LOG.error(f"Failed to connect to RPC host {host}") + LOG.error(f"Failed to connect to RPC host '{host}:{port}'") return None def get_rpc_client(self): diff --git a/aprsd/web/admin/templates/index.html b/aprsd/web/admin/templates/index.html index ec452bc..fe9bac1 100644 --- a/aprsd/web/admin/templates/index.html +++ b/aprsd/web/admin/templates/index.html @@ -3,7 +3,7 @@ - + diff --git a/aprsd/wsgi.py b/aprsd/wsgi.py index 98ebf53..ba254a2 100644 --- a/aprsd/wsgi.py +++ b/aprsd/wsgi.py @@ -8,8 +8,8 @@ import flask from flask import Flask from flask.logging import default_handler from flask_httpauth import HTTPBasicAuth -from flask_socketio import Namespace, SocketIO from oslo_config import cfg +import socketio from werkzeug.security import check_password_hash import aprsd @@ -29,7 +29,8 @@ app = Flask( static_folder="web/admin/static", template_folder="web/admin/templates", ) -socket_io = SocketIO(app) +bg_thread = None +app.config["SECRET_KEY"] = "secret!" # HTTPBasicAuth doesn't work on a class method. @@ -119,7 +120,6 @@ def stats(): return json.dumps(_stats()) -@auth.login_required @app.route("/") def index(): stats = _stats() @@ -240,51 +240,46 @@ class LogUpdateThread(threads.APRSDThread): super().__init__("LogUpdate") def loop(self): - global socket_io - - if socket_io: + if sio: log_entries = aprsd_rpc_client.RPCClient().get_log_entries() if log_entries: + LOG.info(f"Sending log entries! {len(log_entries)}") for entry in log_entries: - socket_io.emit( + sio.emit( "log_entry", entry, namespace="/logs", ) - time.sleep(5) return True -class LoggingNamespace(Namespace): +class LoggingNamespace(socketio.Namespace): log_thread = None - def on_connect(self): - global socket_io - socket_io.emit( + def on_connect(self, sid, environ): + global sio + LOG.debug(f"LOG on_connect {sid}") + sio.emit( "connected", {"data": "/logs Connected"}, namespace="/logs", ) self.log_thread = LogUpdateThread() self.log_thread.start() - def on_disconnect(self): - LOG.debug("LOG Disconnected") + def on_disconnect(self, sid): + LOG.debug(f"LOG Disconnected {sid}") if self.log_thread: self.log_thread.stop() def setup_logging(flask_app, loglevel): - flask_log = logging.getLogger("werkzeug") - flask_app.logger.removeHandler(default_handler) - flask_log.removeHandler(default_handler) - LOG.handlers = [] - + global app, LOG log_level = conf.log.LOG_LEVELS[loglevel] - LOG.setLevel(log_level) + app.logger.setLevel(log_level) + flask_app.logger.removeHandler(default_handler) + date_format = CONF.logging.date_format - flask_app.logger.disabled = True - gunicorn_err = logging.getLogger("gunicorn.error") if CONF.logging.rich_logging: log_format = "%(message)s" @@ -294,7 +289,7 @@ def setup_logging(flask_app, loglevel): rich_tracebacks=True, omit_repeated_times=False, ) rh.setFormatter(log_formatter) - LOG.addHandler(rh) + app.logger.addHandler(rh) log_file = CONF.logging.logfile @@ -306,9 +301,9 @@ def setup_logging(flask_app, loglevel): backupCount=4, ) fh.setFormatter(log_formatter) - LOG.addHandler(fh) + app.logger.addHandler(fh) - gunicorn_err.handlers = LOG.handlers + LOG = app.logger def init_app(config_file=None, log_level=None): @@ -327,11 +322,41 @@ def init_app(config_file=None, log_level=None): return log_level -print(f"APP {__name__}") if __name__ == "__main__": - socket_io.run(app) + async_mode = "threading" + sio = socketio.Server(logger=True, async_mode=async_mode) + app.wsgi_app = socketio.WSGIApp(sio, app.wsgi_app) + log_level = init_app(log_level="DEBUG") + setup_logging(app, log_level) + sio.register_namespace(LoggingNamespace("/logs")) + CONF.log_opt_values(LOG, logging.DEBUG) + app.run(threaded=True, debug=True, port=8100) + + +if __name__ == "uwsgi_file_aprsd_wsgi": + # Start with + # uwsgi --http :8000 --gevent 1000 --http-websockets --master -w aprsd.wsgi --callable app + + async_mode = "gevent_uwsgi" + sio = socketio.Server(logger=True, async_mode=async_mode) + app.wsgi_app = socketio.WSGIApp(sio, app.wsgi_app) + log_level = init_app( + log_level="DEBUG", + config_file="/config/aprsd.conf", + ) + setup_logging(app, log_level) + sio.register_namespace(LoggingNamespace("/logs")) + CONF.log_opt_values(LOG, logging.DEBUG) + if __name__ == "aprsd.wsgi": + # set async_mode to 'threading', 'eventlet', 'gevent' or 'gevent_uwsgi' to + # force a mode else, the best mode is selected automatically from what's + # installed + async_mode = "threading" + sio = socketio.Server(logger=True, async_mode=async_mode) + app.wsgi_app = socketio.WSGIApp(sio, app.wsgi_app) + log_level = init_app(config_file="/config/aprsd.conf", log_level="DEBUG") - socket_io.on_namespace(LoggingNamespace("/logs")) + sio.on_namespace(LoggingNamespace("/logs")) setup_logging(app, log_level) diff --git a/dev-requirements.txt b/dev-requirements.txt index 9dd9368..87574f1 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -12,14 +12,14 @@ babel==2.12.1 # via sphinx black==23.7.0 # via gray build==0.10.0 # via pip-tools cachetools==5.3.1 # via tox -certifi==2023.5.7 # via requests +certifi==2023.7.22 # via requests cfgv==3.3.1 # via pre-commit chardet==5.1.0 # via tox charset-normalizer==3.2.0 # via requests -click==8.1.5 # via black, pip-tools +click==8.1.6 # via black, pip-tools colorama==0.4.6 # via tox commonmark==0.9.1 # via rich -configargparse==1.5.5 # via gray +configargparse==1.7 # via gray coverage[toml]==7.2.7 # via pytest-cov distlib==0.3.7 # via virtualenv docutils==0.20.1 # via sphinx @@ -28,7 +28,7 @@ filelock==3.12.2 # via tox, virtualenv fixit==0.1.4 # via gray flake8==6.0.0 # via -r dev-requirements.in, fixit, pep8-naming gray==0.13.0 # via -r dev-requirements.in -identify==2.5.24 # via pre-commit +identify==2.5.26 # via pre-commit idna==3.4 # via requests imagesize==1.4.1 # via sphinx importlib-resources==6.0.0 # via fixit @@ -46,7 +46,7 @@ nodeenv==1.8.0 # via pre-commit packaging==23.1 # via black, build, pyproject-api, pytest, sphinx, tox pathspec==0.11.1 # via black pep8-naming==0.13.3 # via -r dev-requirements.in -pip-tools==7.0.0 # via -r dev-requirements.in +pip-tools==7.1.0 # via -r dev-requirements.in platformdirs==3.9.1 # via black, tox, virtualenv pluggy==1.2.0 # via pytest, tox pre-commit==3.3.3 # via -r dev-requirements.in @@ -79,9 +79,9 @@ typing-extensions==4.7.1 # via libcst, mypy, typing-inspect typing-inspect==0.9.0 # via libcst unify==0.5 # via gray untokenize==0.1.1 # via unify -urllib3==2.0.3 # via requests -virtualenv==20.24.0 # via pre-commit, tox -wheel==0.40.0 # via pip-tools +urllib3==2.0.4 # via requests +virtualenv==20.24.1 # via pre-commit, tox +wheel==0.41.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/docker/bin/admin.sh b/docker/bin/admin.sh index 7709d5d..b52c130 100755 --- a/docker/bin/admin.sh +++ b/docker/bin/admin.sh @@ -13,6 +13,8 @@ if [ ! -z "${APRSD_PLUGINS}" ]; then done fi +pip3 install gevent uwsgi + if [ -z "${LOG_LEVEL}" ] || [[ ! "${LOG_LEVEL}" =~ ^(CRITICAL|ERROR|WARNING|INFO)$ ]]; then LOG_LEVEL="DEBUG" fi @@ -28,5 +30,6 @@ fi export COLUMNS=200 #exec gunicorn -b :8000 --workers 4 "aprsd.admin_web:create_app(config_file='$APRSD_CONFIG', log_level='$LOG_LEVEL')" -exec gunicorn -b :8000 --workers 4 "aprsd.wsgi:app" +# exec gunicorn -b :8000 --workers 4 "aprsd.wsgi:app" +exec uwsgi --http :8000 --gevent 1000 --http-websockets --master -w aprsd.wsgi --callable app #exec aprsd listen -c $APRSD_CONFIG --loglevel ${LOG_LEVEL} ${APRSD_LOAD_PLUGINS} ${APRSD_LISTEN_FILTER} diff --git a/requirements.in b/requirements.in index dc69a76..2f3965a 100644 --- a/requirements.in +++ b/requirements.in @@ -15,6 +15,8 @@ six thesmuggler update_checker flask-socketio +python-socketio +gevent eventlet tabulate # Pinned due to gray needing 12.6.0 diff --git a/requirements.txt b/requirements.txt index 4ab36c4..ebc7bd1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,12 +10,12 @@ attrs==23.1.0 # via -r requirements.in, ax253, kiss3, rush ax253==0.1.5.post1 # via kiss3 beautifulsoup4==4.12.2 # via -r requirements.in bidict==0.22.1 # via python-socketio -bitarray==2.7.6 # via ax253, kiss3 +bitarray==2.8.0 # via ax253, kiss3 blinker==1.6.2 # via flask -certifi==2023.5.7 # via httpcore, requests +certifi==2023.7.22 # via httpcore, requests cffi==1.15.1 # via cryptography charset-normalizer==3.2.0 # via requests -click==8.1.5 # via -r requirements.in, click-completion, click-params, flask +click==8.1.6 # via -r requirements.in, click-completion, click-params, flask click-completion==0.5.2 # via -r requirements.in click-params==0.4.1 # via -r requirements.in commonmark==0.9.1 # via rich @@ -32,7 +32,8 @@ flask-httpauth==4.8.0 # via -r requirements.in flask-socketio==5.3.4 # via -r requirements.in geographiclib==2.0 # via geopy geopy==2.3.0 # via -r requirements.in -greenlet==2.0.2 # via eventlet +gevent==23.7.0 # via -r requirements.in +greenlet==2.0.2 # via eventlet, gevent h11==0.14.0 # via httpcore httpcore==0.17.3 # via dnspython idna==3.4 # via anyio, requests @@ -54,7 +55,7 @@ pyopenssl==23.2.0 # via -r requirements.in pyserial==3.5 # via pyserial-asyncio pyserial-asyncio==0.6 # via kiss3 python-engineio==4.5.1 # via python-socketio -python-socketio==5.8.0 # via flask-socketio +python-socketio==5.8.0 # via -r requirements.in, flask-socketio pytz==2023.3 # via -r requirements.in pyyaml==6.0.1 # via -r requirements.in, oslo-config requests==2.31.0 # via -r requirements.in, oslo-config, update-checker @@ -71,9 +72,14 @@ tabulate==0.9.0 # via -r requirements.in thesmuggler==1.0.1 # via -r requirements.in ua-parser==0.18.0 # via user-agents update-checker==0.18.0 # via -r requirements.in -urllib3==2.0.3 # via requests +urllib3==2.0.4 # via requests user-agents==2.2.0 # via -r requirements.in validators==0.20.0 # via click-params werkzeug==2.3.6 # via -r requirements.in, flask wrapt==1.15.0 # via -r requirements.in, debtcollector zipp==3.16.2 # via importlib-metadata +zope-event==5.0 # via gevent +zope-interface==6.0 # via gevent + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From e51a50154459a5f50b3faa7784f68555bb442952 Mon Sep 17 00:00:00 2001 From: Hemna Date: Sun, 23 Jul 2023 19:19:55 -0400 Subject: [PATCH 2/3] change port to 8000 --- aprsd/wsgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aprsd/wsgi.py b/aprsd/wsgi.py index ba254a2..f7b9241 100644 --- a/aprsd/wsgi.py +++ b/aprsd/wsgi.py @@ -330,7 +330,7 @@ if __name__ == "__main__": setup_logging(app, log_level) sio.register_namespace(LoggingNamespace("/logs")) CONF.log_opt_values(LOG, logging.DEBUG) - app.run(threaded=True, debug=True, port=8100) + app.run(threaded=True, debug=True, port=8000) if __name__ == "uwsgi_file_aprsd_wsgi": From 4664ead9e7b833f610b5c04acebbc9cced4447d5 Mon Sep 17 00:00:00 2001 From: Hemna Date: Sun, 23 Jul 2023 19:34:02 -0400 Subject: [PATCH 3/3] Fixed pep8 --- aprsd/rpc/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aprsd/rpc/client.py b/aprsd/rpc/client.py index 6741e4d..985dc2e 100644 --- a/aprsd/rpc/client.py +++ b/aprsd/rpc/client.py @@ -52,9 +52,9 @@ class RPCClient: LOG.debug(f"RPC Client: {self.ip}:{self.port} {self.magic_word}") def _rpyc_connect( - self, host, port, service=rpyc.VoidService, - config={}, ipv6=False, - keepalive=False, authorizer=None, ): + self, host, port, service=rpyc.VoidService, + config={}, ipv6=False, + keepalive=False, authorizer=None, ): LOG.info(f"Connecting to RPC host '{host}:{port}'") try: