From fe0d71de4d75979968a67e6ff1122a11a303c3bc Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 8 Jul 2023 17:30:22 -0400 Subject: [PATCH] Replace ratelimiter with rush This patch replaces the ratelimiter library with rush for rate limiting as the ratelimiter package doesn't work with python 3.11. This patch also refactors the flask.pu to admin_web.py and aprsd.py to main.py --- Makefile | 4 ++-- aprsd/{flask.py => admin_web.py} | 34 ++++++++++++++++++++-------- aprsd/cmds/completion.py | 2 +- aprsd/cmds/dev.py | 2 +- aprsd/cmds/healthcheck.py | 2 +- aprsd/cmds/list_plugins.py | 2 +- aprsd/cmds/listen.py | 2 +- aprsd/cmds/send_message.py | 2 +- aprsd/cmds/server.py | 7 +++--- aprsd/cmds/webchat.py | 2 +- aprsd/{aprsd.py => main.py} | 0 aprsd/threads/tx.py | 30 ++++++++++++++++++++----- dev-requirements.txt | 38 +++++++++++++++++--------------- requirements.in | 2 +- requirements.txt | 16 +++++++------- setup.cfg | 2 +- tests/cmds/test_send_message.py | 2 +- 17 files changed, 93 insertions(+), 56 deletions(-) rename aprsd/{flask.py => admin_web.py} (94%) rename aprsd/{aprsd.py => main.py} (100%) diff --git a/Makefile b/Makefile index 0063236..1eda795 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,8 @@ include Makefile.venv Makefile.venv: curl \ -o Makefile.fetched \ - -L "https://github.com/sio/Makefile.venv/raw/v2022.07.20/Makefile.venv" - echo "147b164f0cbbbe4a2740dcca6c9adb6e9d8d15b895be3998697aa6a821a277d8 *Makefile.fetched" \ + -L "https://raw.githubusercontent.com/sio/Makefile.venv/master/Makefile.venv" + echo " fb48375ed1fd19e41e0cdcf51a4a0c6d1010dfe03b672ffc4c26a91878544f82 *Makefile.fetched" \ | sha256sum --check - \ && mv Makefile.fetched Makefile.venv diff --git a/aprsd/flask.py b/aprsd/admin_web.py similarity index 94% rename from aprsd/flask.py rename to aprsd/admin_web.py index 199df01..6cc0e76 100644 --- a/aprsd/flask.py +++ b/aprsd/admin_web.py @@ -335,13 +335,29 @@ def init_flask(loglevel, quiet): return socketio, flask_app +def create_app(config_file=None, log_level=None, gunicorn=False): + global socketio + global app + + default_config_file = cli_helper.DEFAULT_CONFIG_FILE + if not config_file: + config_file = default_config_file + + CONF( + [], project="aprsd", version=aprsd.__version__, + default_config_files=[config_file], + ) + + if not log_level: + log_level = CONF.logging.log_level + + if gunicorn: + socketio, app = init_flask(log_level, False) + setup_logging(app, log_level, False) + return app + else: + return socketio + + if __name__ == "aprsd.flask": - try: - default_config_file = cli_helper.DEFAULT_CONFIG_FILE - CONF( - [], project="aprsd", version=aprsd.__version__, - default_config_files=[default_config_file], - ) - except cfg.ConfigFilesNotFoundError: - pass - sio, app = init_flask("DEBUG", False) + sio, app = create_app() diff --git a/aprsd/cmds/completion.py b/aprsd/cmds/completion.py index 1356d28..3022d40 100644 --- a/aprsd/cmds/completion.py +++ b/aprsd/cmds/completion.py @@ -1,7 +1,7 @@ import click import click_completion -from aprsd.aprsd import cli +from aprsd.main import cli CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) diff --git a/aprsd/cmds/dev.py b/aprsd/cmds/dev.py index 6e41db6..f624836 100644 --- a/aprsd/cmds/dev.py +++ b/aprsd/cmds/dev.py @@ -10,7 +10,7 @@ from oslo_config import cfg # local imports here from aprsd import cli_helper, client, conf, packets, plugin -from aprsd.aprsd import cli +from aprsd.main import cli from aprsd.utils import trace diff --git a/aprsd/cmds/healthcheck.py b/aprsd/cmds/healthcheck.py index 1bf7c3b..b63a6d0 100644 --- a/aprsd/cmds/healthcheck.py +++ b/aprsd/cmds/healthcheck.py @@ -16,7 +16,7 @@ import aprsd from aprsd import cli_helper, utils from aprsd import conf # noqa # local imports here -from aprsd.aprsd import cli +from aprsd.main import cli from aprsd.rpc import client as aprsd_rpc_client diff --git a/aprsd/cmds/list_plugins.py b/aprsd/cmds/list_plugins.py index e8f448d..ac4b6cd 100644 --- a/aprsd/cmds/list_plugins.py +++ b/aprsd/cmds/list_plugins.py @@ -19,7 +19,7 @@ from thesmuggler import smuggle from aprsd import cli_helper from aprsd import plugin as aprsd_plugin -from aprsd.aprsd import cli +from aprsd.main import cli from aprsd.plugins import ( email, fortune, location, notify, ping, query, time, version, weather, ) diff --git a/aprsd/cmds/listen.py b/aprsd/cmds/listen.py index 6137548..e496013 100644 --- a/aprsd/cmds/listen.py +++ b/aprsd/cmds/listen.py @@ -16,7 +16,7 @@ from rich.console import Console # local imports here import aprsd from aprsd import cli_helper, client, packets, plugin, stats, threads -from aprsd.aprsd import cli +from aprsd.main import cli from aprsd.rpc import server as rpc_server from aprsd.threads import rx diff --git a/aprsd/cmds/send_message.py b/aprsd/cmds/send_message.py index c830448..1da30a2 100644 --- a/aprsd/cmds/send_message.py +++ b/aprsd/cmds/send_message.py @@ -10,7 +10,7 @@ from oslo_config import cfg import aprsd from aprsd import cli_helper, client, packets from aprsd import conf # noqa : F401 -from aprsd.aprsd import cli +from aprsd.main import cli from aprsd.threads import tx diff --git a/aprsd/cmds/server.py b/aprsd/cmds/server.py index 94b929d..af8f797 100644 --- a/aprsd/cmds/server.py +++ b/aprsd/cmds/server.py @@ -6,9 +6,10 @@ import click from oslo_config import cfg import aprsd -from aprsd import aprsd as aprsd_main -from aprsd import cli_helper, client, packets, plugin, threads, utils -from aprsd.aprsd import cli +from aprsd import cli_helper, client +from aprsd import main as aprsd_main +from aprsd import packets, plugin, threads, utils +from aprsd.main import cli from aprsd.rpc import server as rpc_server from aprsd.threads import rx diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index ff90f3f..8467377 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -22,8 +22,8 @@ import wrapt import aprsd from aprsd import cli_helper, client, conf, packets, stats, threads, utils -from aprsd.aprsd import cli from aprsd.logging import rich as aprsd_logging +from aprsd.main import cli from aprsd.threads import rx, tx from aprsd.utils import objectstore, trace diff --git a/aprsd/aprsd.py b/aprsd/main.py similarity index 100% rename from aprsd/aprsd.py rename to aprsd/main.py diff --git a/aprsd/threads/tx.py b/aprsd/threads/tx.py index 5dea4dd..0ceb355 100644 --- a/aprsd/threads/tx.py +++ b/aprsd/threads/tx.py @@ -3,7 +3,10 @@ import logging import time from oslo_config import cfg -from ratelimiter import RateLimiter +from rush import quota, throttle +from rush.contrib import decorator +from rush.limiters import periodic +from rush.stores import dictionary from aprsd import client from aprsd import conf # noqa @@ -14,10 +17,25 @@ from aprsd.packets import core, tracker CONF = cfg.CONF LOG = logging.getLogger("APRSD") +msg_t = throttle.Throttle( + limiter=periodic.PeriodicLimiter( + store=dictionary.DictionaryStore(), + ), + rate=quota.Quota.per_second( + count=CONF.msg_rate_limit_period, + ), +) +ack_t = throttle.Throttle( + limiter=periodic.PeriodicLimiter( + store=dictionary.DictionaryStore(), + ), + rate=quota.Quota.per_second( + count=CONF.ack_rate_limit_period, + ), +) -def limited(until): - duration = int(round(until - time.time())) - LOG.debug(f"Rate limited, sleeping for {duration:d} seconds") +msg_throttle_decorator = decorator.ThrottleDecorator(throttle=msg_t) +ack_throttle_decorator = decorator.ThrottleDecorator(throttle=ack_t) def send(packet: core.Packet, direct=False, aprs_client=None): @@ -31,7 +49,7 @@ def send(packet: core.Packet, direct=False, aprs_client=None): _send_packet(packet, direct=direct, aprs_client=aprs_client) -@RateLimiter(max_calls=1, period=CONF.msg_rate_limit_period, callback=limited) +@msg_throttle_decorator.sleep_and_retry def _send_packet(packet: core.Packet, direct=False, aprs_client=None): if not direct: thread = SendPacketThread(packet=packet) @@ -40,7 +58,7 @@ def _send_packet(packet: core.Packet, direct=False, aprs_client=None): _send_direct(packet, aprs_client=aprs_client) -@RateLimiter(max_calls=1, period=CONF.ack_rate_limit_period, callback=limited) +@ack_throttle_decorator.sleep_and_retry def _send_ack(packet: core.AckPacket, direct=False, aprs_client=None): if not direct: thread = SendAckThread(packet=packet) diff --git a/dev-requirements.txt b/dev-requirements.txt index 1cfffda..c92e21b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,9 +4,9 @@ # # pip-compile --annotation-style=line --resolver=backtracking dev-requirements.in # -add-trailing-comma==2.5.1 # via gray +add-trailing-comma==3.0.0 # via gray alabaster==0.7.13 # via sphinx -attrs==23.1.0 # via jsonschema +attrs==23.1.0 # via jsonschema, referencing autoflake==1.5.3 # via gray babel==2.12.1 # via sphinx black==23.3.0 # via gray @@ -15,15 +15,15 @@ cachetools==5.3.1 # via tox certifi==2023.5.7 # via requests cfgv==3.3.1 # via pre-commit chardet==5.1.0 # via tox -charset-normalizer==3.1.0 # via requests -click==8.1.3 # via black, pip-tools +charset-normalizer==3.2.0 # via requests +click==8.1.4 # via black, pip-tools colorama==0.4.6 # via tox commonmark==0.9.1 # via rich -configargparse==1.5.3 # via gray +configargparse==1.5.5 # via gray coverage[toml]==7.2.7 # via pytest-cov distlib==0.3.6 # via virtualenv docutils==0.20.1 # via sphinx -exceptiongroup==1.1.1 # via pytest +exceptiongroup==1.1.2 # via pytest filelock==3.12.2 # via tox, virtualenv fixit==0.1.4 # via gray flake8==6.0.0 # via -r dev-requirements.in, fixit, pep8-naming @@ -31,36 +31,38 @@ gray==0.13.0 # via -r dev-requirements.in identify==2.5.24 # via pre-commit idna==3.4 # via requests imagesize==1.4.1 # via sphinx -importlib-resources==5.12.0 # via fixit +importlib-resources==6.0.0 # via fixit iniconfig==2.0.0 # via pytest isort==5.12.0 # via -r dev-requirements.in, gray jinja2==3.1.2 # via sphinx -jsonschema==4.17.3 # via fixit +jsonschema==4.18.0 # via fixit +jsonschema-specifications==2023.6.1 # via jsonschema libcst==1.0.1 # via fixit markupsafe==2.1.3 # via jinja2 mccabe==0.7.0 # via flake8 -mypy==1.4.0 # via -r dev-requirements.in +mypy==1.4.1 # via -r dev-requirements.in mypy-extensions==1.0.0 # via black, mypy, typing-inspect nodeenv==1.8.0 # via pre-commit packaging==23.1 # via black, build, pyproject-api, pytest, sphinx, tox pathspec==0.11.1 # via black pep8-naming==0.13.3 # via -r dev-requirements.in -pip-tools==6.13.0 # via -r dev-requirements.in -platformdirs==3.7.0 # via black, tox, virtualenv +pip-tools==6.14.0 # via -r dev-requirements.in +platformdirs==3.8.1 # via black, tox, virtualenv pluggy==1.2.0 # via pytest, tox pre-commit==3.3.3 # via -r dev-requirements.in pycodestyle==2.10.0 # via flake8 pyflakes==3.0.1 # via autoflake, flake8 pygments==2.15.1 # via rich, sphinx -pyproject-api==1.5.2 # via tox +pyproject-api==1.5.3 # via tox pyproject-hooks==1.0.0 # via build -pyrsistent==0.19.3 # via jsonschema -pytest==7.3.2 # via -r dev-requirements.in, pytest-cov +pytest==7.4.0 # via -r dev-requirements.in, pytest-cov pytest-cov==4.1.0 # via -r dev-requirements.in -pyupgrade==3.7.0 # via gray +pyupgrade==3.9.0 # via gray pyyaml==6.0 # via fixit, libcst, pre-commit +referencing==0.29.1 # via jsonschema, jsonschema-specifications requests==2.31.0 # via sphinx rich==12.6.0 # via gray +rpds-py==0.8.10 # via jsonschema, referencing snowballstemmer==2.2.0 # via sphinx sphinx==7.0.1 # via -r dev-requirements.in sphinxcontrib-applehelp==1.0.4 # via sphinx @@ -71,9 +73,9 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx tokenize-rt==5.1.0 # via add-trailing-comma, pyupgrade toml==0.10.2 # via autoflake -tomli==2.0.1 # via black, build, coverage, mypy, pyproject-api, pyproject-hooks, pytest, tox -tox==4.6.3 # via -r dev-requirements.in -typing-extensions==4.6.3 # via libcst, mypy, typing-inspect +tomli==2.0.1 # via black, build, coverage, mypy, pip-tools, pyproject-api, pyproject-hooks, pytest, tox +tox==4.6.4 # via -r dev-requirements.in +typing-extensions==4.7.1 # via libcst, mypy, typing-inspect typing-inspect==0.9.0 # via libcst unify==0.5 # via gray untokenize==0.1.1 # via unify diff --git a/requirements.in b/requirements.in index ea69046..bec86f7 100644 --- a/requirements.in +++ b/requirements.in @@ -36,5 +36,5 @@ rpyc # raspi cryptography==38.0.1 shellingham==1.5.0.post1 -ratelimiter geopy +rush diff --git a/requirements.txt b/requirements.txt index 10a7118..e6b174d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,15 +5,15 @@ # pip-compile --annotation-style=line --resolver=backtracking requirements.in # aprslib==0.7.2 # via -r requirements.in -attrs==23.1.0 # via -r requirements.in, ax253, kiss3 +attrs==23.1.0 # via -r requirements.in, ax253, kiss3, rush ax253==0.1.5.post1 # via kiss3 beautifulsoup4==4.12.2 # via -r requirements.in bidict==0.22.1 # via python-socketio -bitarray==2.7.5 # via ax253, kiss3 +bitarray==2.7.6 # via ax253, kiss3 certifi==2023.5.7 # via requests cffi==1.15.1 # via cryptography -charset-normalizer==3.1.0 # via requests -click==8.1.3 # via -r requirements.in, click-completion, flask +charset-normalizer==3.2.0 # via requests +click==8.1.4 # via -r requirements.in, click-completion, flask click-completion==0.5.2 # via -r requirements.in commonmark==0.9.1 # via rich cryptography==38.0.1 # via -r requirements.in, pyopenssl @@ -31,7 +31,7 @@ geopy==2.3.0 # via -r requirements.in greenlet==2.0.2 # via eventlet idna==3.4 # via requests imapclient==2.3.1 # via -r requirements.in -importlib-metadata==6.7.0 # via ax253, kiss3 +importlib-metadata==6.8.0 # via ax253, kiss3 itsdangerous==2.1.2 # via flask jinja2==3.1.2 # via click-completion, flask kiss3==8.0.0 # via -r requirements.in @@ -47,22 +47,22 @@ pygments==2.15.1 # via rich pyopenssl==23.2.0 # via -r requirements.in pyserial==3.5 # via pyserial-asyncio pyserial-asyncio==0.6 # via kiss3 -python-engineio==4.4.1 # via python-socketio +python-engineio==4.5.1 # via python-socketio python-socketio==5.8.0 # via flask-socketio pytz==2023.3 # via -r requirements.in pyyaml==6.0 # via -r requirements.in, oslo-config -ratelimiter==1.2.0.post0 # via -r requirements.in requests==2.31.0 # via -r requirements.in, oslo-config, update-checker rfc3986==2.0.0 # via oslo-config rich==12.6.0 # via -r requirements.in rpyc==5.3.1 # via -r requirements.in +rush==2021.4.0 # via -r requirements.in shellingham==1.5.0.post1 # via -r requirements.in, click-completion six==1.16.0 # via -r requirements.in, click-completion, eventlet, imapclient soupsieve==2.4.1 # via beautifulsoup4 stevedore==5.1.0 # via oslo-config tabulate==0.9.0 # via -r requirements.in thesmuggler==1.0.1 # via -r requirements.in -ua-parser==0.16.1 # via user-agents +ua-parser==0.18.0 # via user-agents update-checker==0.18.0 # via -r requirements.in urllib3==2.0.3 # via requests user-agents==2.2.0 # via -r requirements.in diff --git a/setup.cfg b/setup.cfg index 9af2e82..ac77dc6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,7 @@ packages = [entry_points] console_scripts = - aprsd = aprsd.aprsd:main + aprsd = aprsd.main:main oslo.config.opts = aprsd.conf = aprsd.conf.opts:list_opts oslo.config.opts.defaults = diff --git a/tests/cmds/test_send_message.py b/tests/cmds/test_send_message.py index fbeef57..d862282 100644 --- a/tests/cmds/test_send_message.py +++ b/tests/cmds/test_send_message.py @@ -6,8 +6,8 @@ from click.testing import CliRunner from oslo_config import cfg from aprsd import conf # noqa : F401 -from aprsd.aprsd import cli from aprsd.cmds import send_message # noqa +from aprsd.main import cli from .. import fake