replacement of flask-socketio with python-socketio

This patch starts the work to replace flask-socketio with python-socketio so that uwsgi can be used instead of gunicorn. uwsgi can support websockets. Have to rework webchat command next
2023-07-23 18:54:23 -04:00 · 2023-07-23 18:54:23 -04:00 · 89576a3c43
parent 5383b698ea
commit 89576a3c43
7 changed files with 82 additions and 46 deletions
--- a/aprsd/rpc/client.py
+++ b/aprsd/rpc/client.py
@ -56,7 +56,7 @@ class RPCClient:
        config={}, ipv6=False,
        keepalive=False, authorizer=None, ):

-        print(f"Connecting to RPC host {host}:{port}")
+        LOG.info(f"Connecting to RPC host '{host}:{port}'")
        try:
            s = rpc.AuthSocketStream.connect(
                host, port, ipv6=ipv6, keepalive=keepalive,
@ -64,7 +64,7 @@ class RPCClient:
            )
            return rpyc.utils.factory.connect_stream(s, service, config=config)
        except ConnectionRefusedError:
-            LOG.error(f"Failed to connect to RPC host {host}")
+            LOG.error(f"Failed to connect to RPC host '{host}:{port}'")
            return None

    def get_rpc_client(self):
--- a/aprsd/web/admin/templates/index.html
+++ b/aprsd/web/admin/templates/index.html
@ -3,7 +3,7 @@
        <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
        <link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/themes/smoothness/jquery-ui.css">
        <script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"></script>
-        <script src="https://cdn.socket.io/4.1.2/socket.io.min.js" integrity="sha384-toS6mmwu70G0fw54EGlWWeA4z3dyJ+dlXBtSURSKN4vyRFOcxd3Bzjj/AoOwY+Rg" crossorigin="anonymous"></script>
+        <script src="https://cdn.socket.io/4.7.1/socket.io.min.js" integrity="sha512-+NaO7d6gQ1YPxvc/qHIqZEchjGm207SszoNeMgppoqD/67fEqmc1edS8zrbxPD+4RQI3gDgT/83ihpFW61TG/Q==" crossorigin="anonymous"></script>

        <script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.4/dist/Chart.bundle.js"></script>

--- a/aprsd/wsgi.py
+++ b/aprsd/wsgi.py
@ -8,8 +8,8 @@ import flask
 from flask import Flask
 from flask.logging import default_handler
 from flask_httpauth import HTTPBasicAuth
-from flask_socketio import Namespace, SocketIO
 from oslo_config import cfg
+import socketio
 from werkzeug.security import check_password_hash

 import aprsd
@ -29,7 +29,8 @@ app = Flask(
    static_folder="web/admin/static",
    template_folder="web/admin/templates",
 )
-socket_io = SocketIO(app)
+bg_thread = None
+app.config["SECRET_KEY"] = "secret!"


 # HTTPBasicAuth doesn't work on a class method.
@ -119,7 +120,6 @@ def stats():
    return json.dumps(_stats())


-@auth.login_required
@app.route("/")
 def index():
    stats = _stats()
@ -240,51 +240,46 @@ class LogUpdateThread(threads.APRSDThread):
        super().__init__("LogUpdate")

    def loop(self):
-        global socket_io
-
-        if socket_io:
+        if sio:
            log_entries = aprsd_rpc_client.RPCClient().get_log_entries()

            if log_entries:
+                LOG.info(f"Sending log entries! {len(log_entries)}")
                for entry in log_entries:
-                    socket_io.emit(
+                    sio.emit(
                        "log_entry", entry,
                        namespace="/logs",
                    )
-
        time.sleep(5)
        return True


-class LoggingNamespace(Namespace):
+class LoggingNamespace(socketio.Namespace):
    log_thread = None

-    def on_connect(self):
-        global socket_io
-        socket_io.emit(
+    def on_connect(self, sid, environ):
+        global sio
+        LOG.debug(f"LOG on_connect {sid}")
+        sio.emit(
            "connected", {"data": "/logs Connected"},
            namespace="/logs",
        )
        self.log_thread = LogUpdateThread()
        self.log_thread.start()

-    def on_disconnect(self):
-        LOG.debug("LOG Disconnected")
+    def on_disconnect(self, sid):
+        LOG.debug(f"LOG Disconnected {sid}")
        if self.log_thread:
            self.log_thread.stop()


 def setup_logging(flask_app, loglevel):
-    flask_log = logging.getLogger("werkzeug")
-    flask_app.logger.removeHandler(default_handler)
-    flask_log.removeHandler(default_handler)
-    LOG.handlers = []
-
+    global app, LOG
    log_level = conf.log.LOG_LEVELS[loglevel]
-    LOG.setLevel(log_level)
+    app.logger.setLevel(log_level)
+    flask_app.logger.removeHandler(default_handler)
+
    date_format = CONF.logging.date_format
-    flask_app.logger.disabled = True
-    gunicorn_err = logging.getLogger("gunicorn.error")

    if CONF.logging.rich_logging:
        log_format = "%(message)s"
@ -294,7 +289,7 @@ def setup_logging(flask_app, loglevel):
            rich_tracebacks=True, omit_repeated_times=False,
        )
        rh.setFormatter(log_formatter)
-        LOG.addHandler(rh)
+        app.logger.addHandler(rh)

    log_file = CONF.logging.logfile

@ -306,9 +301,9 @@ def setup_logging(flask_app, loglevel):
            backupCount=4,
        )
        fh.setFormatter(log_formatter)
-        LOG.addHandler(fh)
+        app.logger.addHandler(fh)

-    gunicorn_err.handlers = LOG.handlers
+    LOG = app.logger


 def init_app(config_file=None, log_level=None):
@ -327,11 +322,41 @@ def init_app(config_file=None, log_level=None):
    return log_level


-print(f"APP {__name__}")
 if __name__ == "__main__":
-    socket_io.run(app)
+    async_mode = "threading"
+    sio = socketio.Server(logger=True, async_mode=async_mode)
+    app.wsgi_app = socketio.WSGIApp(sio, app.wsgi_app)
+    log_level = init_app(log_level="DEBUG")
+    setup_logging(app, log_level)
+    sio.register_namespace(LoggingNamespace("/logs"))
+    CONF.log_opt_values(LOG, logging.DEBUG)
+    app.run(threaded=True, debug=True, port=8100)
+
+
+if __name__ == "uwsgi_file_aprsd_wsgi":
+    # Start with
+    # uwsgi --http :8000 --gevent 1000 --http-websockets --master -w aprsd.wsgi --callable app
+
+    async_mode = "gevent_uwsgi"
+    sio = socketio.Server(logger=True, async_mode=async_mode)
+    app.wsgi_app = socketio.WSGIApp(sio, app.wsgi_app)
+    log_level = init_app(
+        log_level="DEBUG",
+        config_file="/config/aprsd.conf",
+    )
+    setup_logging(app, log_level)
+    sio.register_namespace(LoggingNamespace("/logs"))
+    CONF.log_opt_values(LOG, logging.DEBUG)
+

 if __name__ == "aprsd.wsgi":
+    # set async_mode to 'threading', 'eventlet', 'gevent' or 'gevent_uwsgi' to
+    # force a mode else, the best mode is selected automatically from what's
+    # installed
+    async_mode = "threading"
+    sio = socketio.Server(logger=True, async_mode=async_mode)
+    app.wsgi_app = socketio.WSGIApp(sio, app.wsgi_app)
+
    log_level = init_app(config_file="/config/aprsd.conf", log_level="DEBUG")
-    socket_io.on_namespace(LoggingNamespace("/logs"))
+    sio.on_namespace(LoggingNamespace("/logs"))
    setup_logging(app, log_level)
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@ -12,14 +12,14 @@ babel==2.12.1             # via sphinx
 black==23.7.0             # via gray
 build==0.10.0             # via pip-tools
 cachetools==5.3.1         # via tox
-certifi==2023.5.7         # via requests
+certifi==2023.7.22        # via requests
 cfgv==3.3.1               # via pre-commit
 chardet==5.1.0            # via tox
 charset-normalizer==3.2.0  # via requests
-click==8.1.5              # via black, pip-tools
+click==8.1.6              # via black, pip-tools
 colorama==0.4.6           # via tox
 commonmark==0.9.1         # via rich
-configargparse==1.5.5     # via gray
+configargparse==1.7       # via gray
 coverage[toml]==7.2.7     # via pytest-cov
 distlib==0.3.7            # via virtualenv
 docutils==0.20.1          # via sphinx
@ -28,7 +28,7 @@ filelock==3.12.2          # via tox, virtualenv
 fixit==0.1.4              # via gray
 flake8==6.0.0             # via -r dev-requirements.in, fixit, pep8-naming
 gray==0.13.0              # via -r dev-requirements.in
-identify==2.5.24          # via pre-commit
+identify==2.5.26          # via pre-commit
 idna==3.4                 # via requests
 imagesize==1.4.1          # via sphinx
 importlib-resources==6.0.0  # via fixit
@ -46,7 +46,7 @@ nodeenv==1.8.0            # via pre-commit
 packaging==23.1           # via black, build, pyproject-api, pytest, sphinx, tox
 pathspec==0.11.1          # via black
 pep8-naming==0.13.3       # via -r dev-requirements.in
-pip-tools==7.0.0          # via -r dev-requirements.in
+pip-tools==7.1.0          # via -r dev-requirements.in
 platformdirs==3.9.1       # via black, tox, virtualenv
 pluggy==1.2.0             # via pytest, tox
 pre-commit==3.3.3         # via -r dev-requirements.in
@ -79,9 +79,9 @@ typing-extensions==4.7.1  # via libcst, mypy, typing-inspect
 typing-inspect==0.9.0     # via libcst
 unify==0.5                # via gray
 untokenize==0.1.1         # via unify
-urllib3==2.0.3            # via requests
-virtualenv==20.24.0       # via pre-commit, tox
-wheel==0.40.0             # via pip-tools
+urllib3==2.0.4            # via requests
+virtualenv==20.24.1       # via pre-commit, tox
+wheel==0.41.0             # via pip-tools

 # The following packages are considered to be unsafe in a requirements file:
 # pip
--- a/docker/bin/admin.sh
+++ b/docker/bin/admin.sh
@ -13,6 +13,8 @@ if [ ! -z "${APRSD_PLUGINS}" ]; then
    done
 fi

+pip3 install gevent uwsgi
+
 if [ -z "${LOG_LEVEL}" ] || [[ ! "${LOG_LEVEL}" =~ ^(CRITICAL|ERROR|WARNING|INFO)$ ]]; then
    LOG_LEVEL="DEBUG"
 fi
@ -28,5 +30,6 @@ fi

 export COLUMNS=200
 #exec gunicorn -b :8000 --workers 4 "aprsd.admin_web:create_app(config_file='$APRSD_CONFIG', log_level='$LOG_LEVEL')"
-exec gunicorn -b :8000 --workers 4 "aprsd.wsgi:app"
+# exec gunicorn -b :8000 --workers 4 "aprsd.wsgi:app"
+exec uwsgi --http :8000 --gevent 1000 --http-websockets --master -w aprsd.wsgi --callable app
 #exec aprsd listen -c $APRSD_CONFIG --loglevel ${LOG_LEVEL} ${APRSD_LOAD_PLUGINS} ${APRSD_LISTEN_FILTER}
--- a/requirements.in
+++ b/requirements.in
@ -15,6 +15,8 @@ six
 thesmuggler
 update_checker
 flask-socketio
+python-socketio
+gevent
 eventlet
 tabulate
 # Pinned due to gray needing 12.6.0
--- a/requirements.txt
+++ b/requirements.txt
@ -10,12 +10,12 @@ attrs==23.1.0             # via -r requirements.in, ax253, kiss3, rush
 ax253==0.1.5.post1        # via kiss3
 beautifulsoup4==4.12.2    # via -r requirements.in
 bidict==0.22.1            # via python-socketio
-bitarray==2.7.6           # via ax253, kiss3
+bitarray==2.8.0           # via ax253, kiss3
 blinker==1.6.2            # via flask
-certifi==2023.5.7         # via httpcore, requests
+certifi==2023.7.22        # via httpcore, requests
 cffi==1.15.1              # via cryptography
 charset-normalizer==3.2.0  # via requests
-click==8.1.5              # via -r requirements.in, click-completion, click-params, flask
+click==8.1.6              # via -r requirements.in, click-completion, click-params, flask
 click-completion==0.5.2   # via -r requirements.in
 click-params==0.4.1       # via -r requirements.in
 commonmark==0.9.1         # via rich
@ -32,7 +32,8 @@ flask-httpauth==4.8.0     # via -r requirements.in
 flask-socketio==5.3.4     # via -r requirements.in
 geographiclib==2.0        # via geopy
 geopy==2.3.0              # via -r requirements.in
-greenlet==2.0.2           # via eventlet
+gevent==23.7.0            # via -r requirements.in
+greenlet==2.0.2           # via eventlet, gevent
 h11==0.14.0               # via httpcore
 httpcore==0.17.3          # via dnspython
 idna==3.4                 # via anyio, requests
@ -54,7 +55,7 @@ pyopenssl==23.2.0         # via -r requirements.in
 pyserial==3.5             # via pyserial-asyncio
 pyserial-asyncio==0.6     # via kiss3
 python-engineio==4.5.1    # via python-socketio
-python-socketio==5.8.0    # via flask-socketio
+python-socketio==5.8.0    # via -r requirements.in, flask-socketio
 pytz==2023.3              # via -r requirements.in
 pyyaml==6.0.1             # via -r requirements.in, oslo-config
 requests==2.31.0          # via -r requirements.in, oslo-config, update-checker
@ -71,9 +72,14 @@ tabulate==0.9.0           # via -r requirements.in
 thesmuggler==1.0.1        # via -r requirements.in
 ua-parser==0.18.0         # via user-agents
 update-checker==0.18.0    # via -r requirements.in
-urllib3==2.0.3            # via requests
+urllib3==2.0.4            # via requests
 user-agents==2.2.0        # via -r requirements.in
 validators==0.20.0        # via click-params
 werkzeug==2.3.6           # via -r requirements.in, flask
 wrapt==1.15.0             # via -r requirements.in, debtcollector
 zipp==3.16.2              # via importlib-metadata
+zope-event==5.0           # via gevent
+zope-interface==6.0       # via gevent
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools