From d0f53c563f4186510bede7b81456c3cd6059069c Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 09:20:55 -0400 Subject: [PATCH 01/26] Update tox for mypy runs --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index cdce1e7..eed3e9d 100644 --- a/tox.ini +++ b/tox.ini @@ -94,7 +94,7 @@ skip_install = true deps = -r{toxinidir}/requirements.txt -r{toxinidir}/dev-requirements.txt commands = - mypy aprsd + mypy --ignore-missing-imports --install-types aprsd [testenv:pre-commit] skip_install = true From 12dab284cb6aef896a8f885f5574ed5a8ac9a0b4 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 09:22:48 -0400 Subject: [PATCH 02/26] Start to add types --- aprsd/wsgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aprsd/wsgi.py b/aprsd/wsgi.py index e81ee89..04d7464 100644 --- a/aprsd/wsgi.py +++ b/aprsd/wsgi.py @@ -22,7 +22,7 @@ CONF = cfg.CONF LOG = logging.getLogger("gunicorn.access") auth = HTTPBasicAuth() -users = {} +users: dict[str, str] = {} app = Flask( "aprsd", static_url_path="/static", From 29b4b04eee071b0e65298759ad685275280d1256 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 09:23:27 -0400 Subject: [PATCH 03/26] No need to synchronize on stats this patch updates the stats object to remove the synchronize on calling stats. each property on the stats object are already synchronized. --- aprsd/stats.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aprsd/stats.py b/aprsd/stats.py index 5c8380a..cb20eb8 100644 --- a/aprsd/stats.py +++ b/aprsd/stats.py @@ -174,7 +174,6 @@ class APRSDStats: def email_thread_update(self): self._email_thread_last_time = datetime.datetime.now() - @wrapt.synchronized(lock) def stats(self): now = datetime.datetime.now() if self._email_thread_last_time: From d80277c9d8f882297ee35ffa0d1174150c290b1c Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 09:25:36 -0400 Subject: [PATCH 04/26] Moved Threads list for mypy This patch moves the APRSDThreadList to the bottom of the file so that we can specify the type in the threads_list member for mypy. --- aprsd/threads/aprsd.py | 73 +++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/aprsd/threads/aprsd.py b/aprsd/threads/aprsd.py index 51d7960..306abd5 100644 --- a/aprsd/threads/aprsd.py +++ b/aprsd/threads/aprsd.py @@ -2,6 +2,7 @@ import abc import datetime import logging import threading +from typing import List import wrapt @@ -9,42 +10,6 @@ import wrapt LOG = logging.getLogger("APRSD") -class APRSDThreadList: - """Singleton class that keeps track of application wide threads.""" - - _instance = None - - threads_list = [] - lock = threading.Lock() - - def __new__(cls, *args, **kwargs): - if cls._instance is None: - cls._instance = super().__new__(cls) - cls.threads_list = [] - return cls._instance - - @wrapt.synchronized(lock) - def add(self, thread_obj): - self.threads_list.append(thread_obj) - - @wrapt.synchronized(lock) - def remove(self, thread_obj): - self.threads_list.remove(thread_obj) - - @wrapt.synchronized(lock) - def stop_all(self): - """Iterate over all threads and call stop on them.""" - for th in self.threads_list: - LOG.info(f"Stopping Thread {th.name}") - if hasattr(th, "packet"): - LOG.info(F"{th.name} packet {th.packet}") - th.stop() - - @wrapt.synchronized(lock) - def __len__(self): - return len(self.threads_list) - - class APRSDThread(threading.Thread, metaclass=abc.ABCMeta): def __init__(self, name): @@ -86,3 +51,39 @@ class APRSDThread(threading.Thread, metaclass=abc.ABCMeta): self._cleanup() APRSDThreadList().remove(self) LOG.debug("Exiting") + + +class APRSDThreadList: + """Singleton class that keeps track of application wide threads.""" + + _instance = None + + threads_list: List[APRSDThread] = [] + lock = threading.Lock() + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super().__new__(cls) + cls.threads_list = [] + return cls._instance + + @wrapt.synchronized(lock) + def add(self, thread_obj): + self.threads_list.append(thread_obj) + + @wrapt.synchronized(lock) + def remove(self, thread_obj): + self.threads_list.remove(thread_obj) + + @wrapt.synchronized(lock) + def stop_all(self): + """Iterate over all threads and call stop on them.""" + for th in self.threads_list: + LOG.info(f"Stopping Thread {th.name}") + if hasattr(th, "packet"): + LOG.info(F"{th.name} packet {th.packet}") + th.stop() + + @wrapt.synchronized(lock) + def __len__(self): + return len(self.threads_list) From 9c77ca26be8af072aa80236338450a1de26b6e09 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 09:28:01 -0400 Subject: [PATCH 05/26] Added type setting on pluging.py for mypy --- aprsd/plugin.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/aprsd/plugin.py b/aprsd/plugin.py index 0d164f5..905d583 100644 --- a/aprsd/plugin.py +++ b/aprsd/plugin.py @@ -65,7 +65,7 @@ class APRSDPluginBase(metaclass=abc.ABCMeta): self.threads = self.create_threads() or [] self.start_threads() - def start_threads(self): + def start_threads(self) -> None: if self.enabled and self.threads: if not isinstance(self.threads, list): self.threads = [self.threads] @@ -90,10 +90,10 @@ class APRSDPluginBase(metaclass=abc.ABCMeta): ) @property - def message_count(self): + def message_count(self) -> int: return self.message_counter - def help(self): + def help(self) -> str: return "Help!" @abc.abstractmethod @@ -118,11 +118,11 @@ class APRSDPluginBase(metaclass=abc.ABCMeta): thread.stop() @abc.abstractmethod - def filter(self, packet: packets.core.Packet): + def filter(self, packet: type[packets.core.Packet]) -> str | packets.core.MessagePacket: pass @abc.abstractmethod - def process(self, packet: packets.core.Packet): + def process(self, packet: type[packets.core.Packet]): """This is called when the filter passes.""" @@ -154,7 +154,7 @@ class APRSDWatchListPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): LOG.warning("Watch list enabled, but no callsigns set.") @hookimpl - def filter(self, packet: packets.core.Packet): + def filter(self, packet: type[packets.core.Packet]) -> str | packets.core.MessagePacket: result = packets.NULL_MESSAGE if self.enabled: wl = watch_list.WatchList() @@ -206,7 +206,7 @@ class APRSDRegexCommandPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): self.enabled = True @hookimpl - def filter(self, packet: packets.core.MessagePacket): + def filter(self, packet: packets.core.MessagePacket) -> str | packets.core.MessagePacket: LOG.info(f"{self.__class__.__name__} called") if not self.enabled: result = f"{self.__class__.__name__} isn't enabled" From 90f212e6dc15835d162d28d2deb579149202d525 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 09:29:09 -0400 Subject: [PATCH 06/26] small refactor of stats usage in version plugin --- aprsd/plugins/version.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/aprsd/plugins/version.py b/aprsd/plugins/version.py index 2b4b552..32037a0 100644 --- a/aprsd/plugins/version.py +++ b/aprsd/plugins/version.py @@ -23,9 +23,7 @@ class VersionPlugin(plugin.APRSDRegexCommandPluginBase): # fromcall = packet.get("from") # message = packet.get("message_text", None) # ack = packet.get("msgNo", "0") - stats_obj = stats.APRSDStats() - s = stats_obj.stats() - print(s) + s = stats.APRSDStats().stats() return "APRSD ver:{} uptime:{}".format( aprsd.__version__, s["aprsd"]["uptime"], From 6f1d6b412209aea6bffad37083e4921dff9594f5 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 09:30:27 -0400 Subject: [PATCH 07/26] removed print --- aprsd/plugins/weather.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aprsd/plugins/weather.py b/aprsd/plugins/weather.py index 4af33e4..1e57115 100644 --- a/aprsd/plugins/weather.py +++ b/aprsd/plugins/weather.py @@ -110,7 +110,6 @@ class USMetarPlugin(plugin.APRSDRegexCommandPluginBase, plugin.APRSFIKEYMixin): @trace.trace def process(self, packet): - print("FISTY") fromcall = packet.get("from") message = packet.get("message_text", None) # ack = packet.get("msgNo", "0") From 1477e61b0fbcea4f7eda8faba7901517a89e1b0f Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 09:34:31 -0400 Subject: [PATCH 08/26] Refactored packets this patch removes the need for dacite2 package for creating packet objects from the aprslib decoded packet dictionary. moved the factory method from the base Packet object to the core module. --- aprsd/client.py | 4 +- aprsd/clients/fake.py | 2 +- aprsd/packets/__init__.py | 2 +- aprsd/packets/core.py | 401 ++++++++++++++++++++-------------- requirements.in | 1 - requirements.txt | 7 +- tests/cmds/test_webchat.py | 7 +- tests/fake.py | 15 +- tests/plugins/test_weather.py | 2 +- tests/test_packets.py | 34 ++- tests/test_plugin.py | 5 +- 11 files changed, 298 insertions(+), 182 deletions(-) diff --git a/aprsd/client.py b/aprsd/client.py index 3cfed8a..6a8b26d 100644 --- a/aprsd/client.py +++ b/aprsd/client.py @@ -137,7 +137,7 @@ class APRSISClient(Client): def decode_packet(self, *args, **kwargs): """APRS lib already decodes this.""" - return core.Packet.factory(args[0]) + return core.factory(args[0]) def setup_connection(self): user = CONF.aprs_network.login @@ -238,7 +238,7 @@ class KISSClient(Client): # LOG.debug(f"Decoding {msg}") raw = aprslib.parse(str(frame)) - packet = core.Packet.factory(raw) + packet = core.factory(raw) if isinstance(packet, core.ThirdParty): return packet.subpacket else: diff --git a/aprsd/clients/fake.py b/aprsd/clients/fake.py index b468808..c93a6c3 100644 --- a/aprsd/clients/fake.py +++ b/aprsd/clients/fake.py @@ -67,7 +67,7 @@ class APRSDFakeClient(metaclass=trace.TraceWrapperMetaclass): # Generate packets here? raw = "GTOWN>APDW16,WIDE1-1,WIDE2-1:}KM6LYW-9>APZ100,TCPIP,GTOWN*::KM6LYW :KM6LYW: 19 Miles SW" pkt_raw = aprslib.parse(raw) - pkt = core.Packet.factory(pkt_raw) + pkt = core.factory(pkt_raw) callback(packet=pkt) LOG.debug(f"END blocking FAKE consumer {self}") time.sleep(8) diff --git a/aprsd/packets/__init__.py b/aprsd/packets/__init__.py index dcb2064..17ecb89 100644 --- a/aprsd/packets/__init__.py +++ b/aprsd/packets/__init__.py @@ -1,6 +1,6 @@ from aprsd.packets.core import ( # noqa: F401 AckPacket, BeaconPacket, GPSPacket, MessagePacket, MicEPacket, Packet, - RejectPacket, StatusPacket, WeatherPacket, + RejectPacket, StatusPacket, ThirdPartyPacket, WeatherPacket, factory, ) from aprsd.packets.packet_list import PacketList # noqa: F401 from aprsd.packets.seen_list import SeenList # noqa: F401 diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 17b1234..560423d 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -1,14 +1,12 @@ -import abc from dataclasses import asdict, dataclass, field from datetime import datetime import logging import re import time # Due to a failure in python 3.8 -from typing import List +from typing import List, Optional -import dacite -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from aprsd.utils import counter @@ -19,7 +17,8 @@ PACKET_TYPE_MESSAGE = "message" PACKET_TYPE_ACK = "ack" PACKET_TYPE_REJECT = "reject" PACKET_TYPE_MICE = "mic-e" -PACKET_TYPE_WX = "weather" +PACKET_TYPE_WX = "wx" +PACKET_TYPE_WEATHER = "weather" PACKET_TYPE_OBJECT = "object" PACKET_TYPE_UNKNOWN = "unknown" PACKET_TYPE_STATUS = "status" @@ -52,68 +51,65 @@ def _init_msgNo(): # noqa: N802 return c.value -def factory_from_dict(packet_dict): - pkt_type = get_packet_type(packet_dict) - if pkt_type: - cls = TYPE_LOOKUP[pkt_type] - return cls.from_dict(packet_dict) +def _translate_fields(raw: dict) -> dict: + translate_fields = { + "from": "from_call", + "to": "to_call", + } + # First translate some fields + for key in translate_fields: + if key in raw: + raw[translate_fields[key]] = raw[key] + del raw[key] + + # addresse overrides to_call + if "addresse" in raw: + raw["to_call"] = raw["addresse"] + + return raw -def factory_from_json(packet_dict): - pkt_type = get_packet_type(packet_dict) - if pkt_type: - return TYPE_LOOKUP[pkt_type].from_json(packet_dict) - - -@dataclass_json @dataclass(unsafe_hash=True) -class Packet(metaclass=abc.ABCMeta): - from_call: str = field(default=None) - to_call: str = field(default=None) - addresse: str = field(default=None) - format: str = field(default=None) +class Packet(DataClassJsonMixin): + _type: str = field(default="Packet", hash=False) + from_call: Optional[str] = field(default=None) + to_call: Optional[str] = field(default=None) + addresse: Optional[str] = field(default=None) + format: Optional[str] = field(default=None) msgNo: str = field(default_factory=_init_msgNo) # noqa: N815 - packet_type: str = field(default=None) + packet_type: Optional[str] = field(default=None) timestamp: float = field(default_factory=_init_timestamp, compare=False, hash=False) # Holds the raw text string to be sent over the wire # or holds the raw string from input packet - raw: str = field(default=None, compare=False, hash=False) + raw: Optional[str] = field(default=None, compare=False, hash=False) raw_dict: dict = field(repr=False, default_factory=lambda: {}, compare=False, hash=False) # Built by calling prepare(). raw needs this built first. - payload: str = field(default=None) + payload: Optional[str] = field(default=None) # Fields related to sending packets out send_count: int = field(repr=False, default=0, compare=False, hash=False) retry_count: int = field(repr=False, default=3, compare=False, hash=False) - # last_send_time: datetime = field( - # metadata=dc_json_config( - # encoder=datetime.isoformat, - # decoder=datetime.fromisoformat, - # ), - # repr=True, - # default_factory=_init_send_time, - # compare=False, - # hash=False - # ) last_send_time: float = field(repr=False, default=0, compare=False, hash=False) last_send_attempt: int = field(repr=False, default=0, compare=False, hash=False) # Do we allow this packet to be saved to send later? allow_delay: bool = field(repr=False, default=True, compare=False, hash=False) path: List[str] = field(default_factory=list, compare=False, hash=False) - via: str = field(default=None, compare=False, hash=False) - - def __post__init__(self): - LOG.warning(f"POST INIT {self}") + via: Optional[str] = field(default=None, compare=False, hash=False) @property - def json(self): - """ - get the json formated string - """ + def json(self) -> str: + """get the json formated string""" + # comes from the DataClassJsonMixin return self.to_json() - def get(self, key, default=None): + @property + def dict(self) -> dict: + """get the dict formated string""" + # comes from the DataClassJsonMixin + return self.to_dict() + + def get(self, key: str, default: Optional[str] = None): """Emulate a getter on a dict.""" if hasattr(self, key): return getattr(self, key) @@ -121,116 +117,37 @@ class Packet(metaclass=abc.ABCMeta): return default @property - def key(self): + def key(self) -> str: """Build a key for finding this packet in a dict.""" return f"{self.from_call}:{self.addresse}:{self.msgNo}" - def update_timestamp(self): + def update_timestamp(self) -> None: self.timestamp = _init_timestamp() - def prepare(self): + def prepare(self) -> None: """Do stuff here that is needed prior to sending over the air.""" # now build the raw message for sending self._build_payload() self._build_raw() - def _build_payload(self): + def _build_payload(self) -> None: """The payload is the non headers portion of the packet.""" + if not self.to_call: + raise ValueError("to_call isn't set. Must set to_call before calling prepare()") msg = self._filter_for_send().rstrip("\n") self.payload = ( f":{self.to_call.ljust(9)}" f":{msg}" ) - def _build_raw(self): + def _build_raw(self) -> None: """Build the self.raw which is what is sent over the air.""" self.raw = "{}>APZ100:{}".format( self.from_call, self.payload, ) - @staticmethod - def factory(raw_packet): - """Factory method to create a packet from a raw packet string.""" - raw = raw_packet - raw["raw_dict"] = raw.copy() - translate_fields = { - "from": "from_call", - "to": "to_call", - } - # First translate some fields - for key in translate_fields: - if key in raw: - raw[translate_fields[key]] = raw[key] - del raw[key] - - if "addresse" in raw: - raw["to_call"] = raw["addresse"] - - packet_type = get_packet_type(raw) - raw["packet_type"] = packet_type - class_name = TYPE_LOOKUP[packet_type] - if packet_type == PACKET_TYPE_THIRDPARTY: - # We have an encapsulated packet! - # So we need to decode it and return the inner packet - # as the packet we are going to process. - # This is a recursive call to the factory - subpacket_raw = raw["subpacket"] - subpacket = Packet.factory(subpacket_raw) - del raw["subpacket"] - # raw["subpacket"] = subpacket - packet = dacite.from_dict(data_class=class_name, data=raw) - packet.subpacket = subpacket - return packet - - if packet_type == PACKET_TYPE_UNKNOWN: - # Try and figure it out here - if "latitude" in raw: - class_name = GPSPacket - - if packet_type == PACKET_TYPE_WX: - # the weather information is in a dict - # this brings those values out to the outer dict - - for key in raw["weather"]: - raw[key] = raw["weather"][key] - - # If we have the broken aprslib, then we need to - # Convert the course and speed to wind_speed and wind_direction - # aprslib issue #80 - # https://github.com/rossengeorgiev/aprs-python/issues/80 - # Wind speed and course is option in the SPEC. - # For some reason aprslib multiplies the speed by 1.852. - if "wind_speed" not in raw and "wind_direction" not in raw: - # Most likely this is the broken aprslib - # So we need to convert the wind_gust speed - raw["wind_gust"] = round(raw.get("wind_gust", 0) / 0.44704, 3) - if "wind_speed" not in raw: - wind_speed = raw.get("speed") - if wind_speed: - raw["wind_speed"] = round(wind_speed / 1.852, 3) - raw["weather"]["wind_speed"] = raw["wind_speed"] - if "speed" in raw: - del raw["speed"] - # Let's adjust the rain numbers as well, since it's wrong - raw["rain_1h"] = round((raw.get("rain_1h", 0) / .254) * .01, 3) - raw["weather"]["rain_1h"] = raw["rain_1h"] - raw["rain_24h"] = round((raw.get("rain_24h", 0) / .254) * .01, 3) - raw["weather"]["rain_24h"] = raw["rain_24h"] - raw["rain_since_midnight"] = round((raw.get("rain_since_midnight", 0) / .254) * .01, 3) - raw["weather"]["rain_since_midnight"] = raw["rain_since_midnight"] - - if "wind_direction" not in raw: - wind_direction = raw.get("course") - if wind_direction: - raw["wind_direction"] = wind_direction - raw["weather"]["wind_direction"] = raw["wind_direction"] - if "course" in raw: - del raw["course"] - - return dacite.from_dict(data_class=class_name, data=raw) - - def log(self, header=None): + def log(self, header: Optional[str] = None) -> None: """LOG a packet to the logfile.""" asdict(self) log_list = ["\n"] @@ -273,29 +190,39 @@ class Packet(metaclass=abc.ABCMeta): # and ftm400-send is max 64. setting this to # 67 displays 64 on the ftm400. (+3 {01 suffix) # feature req: break long ones into two msgs + if not self.raw: + raise ValueError("No message text to send. call prepare() first.") + message = self.raw[:67] # We all miss George Carlin return re.sub("fuck|shit|cunt|piss|cock|bitch", "****", message) - def __str__(self): + def __str__(self) -> str: """Show the raw version of the packet""" self.prepare() + if not self.raw: + raise ValueError("self.raw is unset") return self.raw - def __repr__(self): + def __repr__(self) -> str: """Build the repr version of the packet.""" repr = ( f"{self.__class__.__name__}:" f" From: {self.from_call} " - " To: " + f" To: {self.to_call}" ) - return repr @dataclass(unsafe_hash=True) class AckPacket(Packet): - response: str = field(default=None) + _type: str = field(default="AckPacket", hash=False) + response: Optional[str] = field(default=None) + + @staticmethod + def from_aprslib_dict(raw: dict) -> "AckPacket": + raw = _translate_fields(raw) + return AckPacket(**raw) def __post__init__(self): if self.response: @@ -307,7 +234,13 @@ class AckPacket(Packet): @dataclass(unsafe_hash=True) class RejectPacket(Packet): - response: str = field(default=None) + _type: str = field(default="RejectPacket", hash=False) + response: Optional[str] = field(default=None) + + @staticmethod + def from_aprslib_dict(raw: dict) -> "RejectPacket": + raw = _translate_fields(raw) + return RejectPacket(**raw) def __post__init__(self): if self.response: @@ -317,10 +250,10 @@ class RejectPacket(Packet): self.payload = f":{self.to_call.ljust(9)} :rej{self.msgNo}" -@dataclass_json @dataclass(unsafe_hash=True) class MessagePacket(Packet): - message_text: str = field(default=None) + _type: str = field(default="MessagePacket", hash=False) + message_text: Optional[str] = field(default=None) def _filter_for_send(self) -> str: """Filter and format message string for FCC.""" @@ -328,6 +261,9 @@ class MessagePacket(Packet): # and ftm400-send is max 64. setting this to # 67 displays 64 on the ftm400. (+3 {01 suffix) # feature req: break long ones into two msgs + if not self.message_text: + raise ValueError("No message text to send. Populate message_text field.") + message = self.message_text[:67] # We all miss George Carlin return re.sub("fuck|shit|cunt|piss|cock|bitch", "****", message) @@ -339,12 +275,23 @@ class MessagePacket(Packet): str(self.msgNo), ) + @staticmethod + def from_aprslib_dict(raw: dict) -> "MessagePacket": + raw = _translate_fields(raw) + return MessagePacket(**raw) + @dataclass(unsafe_hash=True) class StatusPacket(Packet): - status: str = field(default=None) + _type: str = field(default="StatusPacket", hash=False) + status: Optional[str] = field(default=None) messagecapable: bool = field(default=False) - comment: str = field(default=None) + comment: Optional[str] = field(default=None) + + @staticmethod + def from_aprslib_dict(raw: dict) -> "StatusPacket": + raw = _translate_fields(raw) + return StatusPacket(**raw) def _build_payload(self): raise NotImplementedError @@ -352,14 +299,27 @@ class StatusPacket(Packet): @dataclass(unsafe_hash=True) class GPSPacket(Packet): + _type: str = field(default="GPSPacket", hash=False) latitude: float = field(default=0.00) longitude: float = field(default=0.00) altitude: float = field(default=0.00) rng: float = field(default=0.00) posambiguity: int = field(default=0) - comment: str = field(default=None) + messagecapable: bool = field(default=False) + comment: Optional[str] = field(default=None) symbol: str = field(default="l") symbol_table: str = field(default="/") + raw_timestamp: Optional[str] = field(default=None) + object_name: Optional[str] = field(default=None) + object_format: Optional[str] = field(default=None) + alive: Optional[bool] = field(default=None) + course: Optional[int] = field(default=None) + speed: Optional[float] = field(default=None) + + @staticmethod + def from_aprslib_dict(raw: dict) -> "GPSPacket": + raw = _translate_fields(raw) + return GPSPacket(**raw) def decdeg2dms(self, degrees_decimal): is_positive = degrees_decimal >= 0 @@ -467,6 +427,13 @@ class GPSPacket(Packet): @dataclass(unsafe_hash=True) class BeaconPacket(GPSPacket): + _type: str = field(default="BeaconPacket", hash=False) + + @staticmethod + def from_aprslib_dict(raw: dict) -> "BeaconPacket": + raw = _translate_fields(raw) + return BeaconPacket(**raw) + def _build_payload(self): """The payload is the non headers portion of the packet.""" time_zulu = self._build_time_zulu() @@ -487,9 +454,10 @@ class BeaconPacket(GPSPacket): @dataclass class MicEPacket(GPSPacket): + _type: str = field(default="MicEPacket", hash=False) messagecapable: bool = False - mbits: str = None - mtype: str = None + mbits: Optional[str] = None + mtype: Optional[str] = None # in MPH speed: float = 0.00 # 0 to 360 @@ -501,14 +469,20 @@ class MicEPacket(GPSPacket): @dataclass class ObjectPacket(GPSPacket): + _type: str = field(default="ObjectPacket", hash=False) alive: bool = True - raw_timestamp: str = None + raw_timestamp: Optional[str] = None symbol: str = field(default="r") # in MPH speed: float = 0.00 # 0 to 360 course: int = 0 + @staticmethod + def from_aprslib_dict(raw: dict) -> "ObjectPacket": + raw = _translate_fields(raw) + return ObjectPacket(**raw) + def _build_payload(self): time_zulu = self._build_time_zulu() lat = self.convert_latitude(self.latitude) @@ -541,6 +515,7 @@ class ObjectPacket(GPSPacket): @dataclass() class WeatherPacket(GPSPacket): + _type: str = field(default="WeatherPacket", hash=False) symbol: str = "_" wind_speed: float = 0.00 wind_direction: int = 0 @@ -552,7 +527,57 @@ class WeatherPacket(GPSPacket): rain_since_midnight: float = 0.00 humidity: int = 0 pressure: float = 0.00 - comment: str = None + comment: Optional[str] = field(default=None) + luminosity: Optional[int] = field(default=None) + wx_raw_timestamp: Optional[str] = field(default=None) + course: Optional[int] = field(default=None) + speed: Optional[float] = field(default=None) + + @staticmethod + def from_aprslib_dict(raw: dict) -> "WeatherPacket": + """Create from a dictionary that has come directly from aprslib parse""" + # Because from is a reserved word in python, we need to translate it + # from -> from_call and to -> to_call + raw = _translate_fields(raw) + + for key in raw["weather"]: + raw[key] = raw["weather"][key] + + # If we have the broken aprslib, then we need to + # Convert the course and speed to wind_speed and wind_direction + # aprslib issue #80 + # https://github.com/rossengeorgiev/aprs-python/issues/80 + # Wind speed and course is option in the SPEC. + # For some reason aprslib multiplies the speed by 1.852. + if "wind_speed" not in raw and "wind_direction" not in raw: + # Most likely this is the broken aprslib + # So we need to convert the wind_gust speed + raw["wind_gust"] = round(raw.get("wind_gust", 0) / 0.44704, 3) + if "wind_speed" not in raw: + wind_speed = raw.get("speed") + if wind_speed: + raw["wind_speed"] = round(wind_speed / 1.852, 3) + raw["weather"]["wind_speed"] = raw["wind_speed"] + if "speed" in raw: + del raw["speed"] + # Let's adjust the rain numbers as well, since it's wrong + raw["rain_1h"] = round((raw.get("rain_1h", 0) / .254) * .01, 3) + raw["weather"]["rain_1h"] = raw["rain_1h"] + raw["rain_24h"] = round((raw.get("rain_24h", 0) / .254) * .01, 3) + raw["weather"]["rain_24h"] = raw["rain_24h"] + raw["rain_since_midnight"] = round((raw.get("rain_since_midnight", 0) / .254) * .01, 3) + raw["weather"]["rain_since_midnight"] = raw["rain_since_midnight"] + + if "wind_direction" not in raw: + wind_direction = raw.get("course") + if wind_direction: + raw["wind_direction"] = wind_direction + raw["weather"]["wind_direction"] = raw["wind_direction"] + if "course" in raw: + del raw["course"] + + del raw["weather"] + return WeatherPacket(**raw) def _build_payload(self): """Build an uncompressed weather packet @@ -614,9 +639,11 @@ class WeatherPacket(GPSPacket): ) -class ThirdParty(Packet): +@dataclass() +class ThirdPartyPacket(Packet): + _type: str = "ThirdPartyPacket" # Holds the encapsulated packet - subpacket: Packet = field(default=None, compare=True, hash=False) + subpacket: Optional[type[Packet]] = field(default=None, compare=True, hash=False) def __repr__(self): """Build the repr version of the packet.""" @@ -629,26 +656,50 @@ class ThirdParty(Packet): return repr_str + @staticmethod + def from_aprslib_dict(raw: dict) -> "ThirdPartyPacket": + """Create from a dictionary that has come directly from aprslib parse""" + # Because from is a reserved word in python, we need to translate it + # from -> from_call and to -> to_call + raw = _translate_fields(raw) + subpacket = raw.get("subpacket") + del raw["subpacket"] + return ThirdPartyPacket(**raw, subpacket=factory(subpacket)) -TYPE_LOOKUP = { + +TYPE_LOOKUP: dict[str, type[Packet]] = { PACKET_TYPE_WX: WeatherPacket, + PACKET_TYPE_WEATHER: WeatherPacket, PACKET_TYPE_MESSAGE: MessagePacket, PACKET_TYPE_ACK: AckPacket, PACKET_TYPE_REJECT: RejectPacket, PACKET_TYPE_MICE: MicEPacket, PACKET_TYPE_OBJECT: ObjectPacket, PACKET_TYPE_STATUS: StatusPacket, - PACKET_TYPE_BEACON: GPSPacket, + PACKET_TYPE_BEACON: BeaconPacket, PACKET_TYPE_UNKNOWN: Packet, - PACKET_TYPE_THIRDPARTY: ThirdParty, + PACKET_TYPE_THIRDPARTY: ThirdPartyPacket, +} + +OBJ_LOOKUP: dict[str, type[Packet]] = { + "MessagePacket": MessagePacket, + "AckPacket": AckPacket, + "RejectPacket": RejectPacket, + "MicEPacket": MicEPacket, + "ObjectPacket": ObjectPacket, + "StatusPacket": StatusPacket, + "GPSPacket": GPSPacket, + "BeaconPacket": BeaconPacket, + "WeatherPacket": WeatherPacket, + "ThirdPartyPacket": ThirdPartyPacket, } -def get_packet_type(packet: dict): +def get_packet_type(packet: dict) -> str: """Decode the packet type from the packet.""" - pkt_format = packet.get("format", None) - msg_response = packet.get("response", None) + pkt_format = packet.get("format") + msg_response = packet.get("response") packet_type = PACKET_TYPE_UNKNOWN if pkt_format == "message" and msg_response == "ack": packet_type = PACKET_TYPE_ACK @@ -664,9 +715,11 @@ def get_packet_type(packet: dict): packet_type = PACKET_TYPE_STATUS elif pkt_format == PACKET_TYPE_BEACON: packet_type = PACKET_TYPE_BEACON + elif pkt_format == PACKET_TYPE_WX: + packet_type = PACKET_TYPE_WEATHER elif pkt_format == PACKET_TYPE_UNCOMPRESSED: - if packet.get("symbol", None) == "_": - packet_type = PACKET_TYPE_WX + if packet.get("symbol") == "_": + packet_type = PACKET_TYPE_WEATHER elif pkt_format == PACKET_TYPE_THIRDPARTY: packet_type = PACKET_TYPE_THIRDPARTY @@ -676,13 +729,43 @@ def get_packet_type(packet: dict): return packet_type -def is_message_packet(packet): +def is_message_packet(packet: dict) -> bool: return get_packet_type(packet) == PACKET_TYPE_MESSAGE -def is_ack_packet(packet): +def is_ack_packet(packet: dict) -> bool: return get_packet_type(packet) == PACKET_TYPE_ACK -def is_mice_packet(packet): +def is_mice_packet(packet: dict) -> bool: return get_packet_type(packet) == PACKET_TYPE_MICE + + +def factory(raw_packet: dict) -> type[Packet]: + """Factory method to create a packet from a raw packet string.""" + raw = raw_packet + if "_type" in raw: + cls = globals()[raw["_type"]] + return cls.from_dict(raw) + + raw["raw_dict"] = raw.copy() + raw = _translate_fields(raw) + + packet_type = get_packet_type(raw) + + raw["packet_type"] = packet_type + class_name = TYPE_LOOKUP[packet_type] + if packet_type == PACKET_TYPE_WX: + # the weather information is in a dict + # this brings those values out to the outer dict + class_name = WeatherPacket + elif packet_type == PACKET_TYPE_OBJECT and "weather" in raw: + class_name = WeatherPacket + elif packet_type == PACKET_TYPE_UNKNOWN: + # Try and figure it out here + if "latitude" in raw: + class_name = GPSPacket + else: + raise Exception(f"Unknown packet type {packet_type} {raw}") + + return class_name.from_aprslib_dict(raw) diff --git a/requirements.in b/requirements.in index e5599fb..dcd231d 100644 --- a/requirements.in +++ b/requirements.in @@ -28,7 +28,6 @@ wrapt kiss3 attrs dataclasses -dacite2 oslo.config rpyc>=6.0.0 # Pin this here so it doesn't require a compile on diff --git a/requirements.txt b/requirements.txt index 6122800..1799187 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,6 @@ click==8.1.7 # via -r requirements.in, click-completion, click-para click-completion==0.5.2 # via -r requirements.in click-params==0.5.0 # via -r requirements.in commonmark==0.9.1 # via rich -dacite2==2.0.0 # via -r requirements.in dataclasses==0.6 # via -r requirements.in dataclasses-json==0.6.4 # via -r requirements.in debtcollector==3.0.0 # via oslo-config @@ -34,7 +33,7 @@ greenlet==3.0.3 # via eventlet, gevent h11==0.14.0 # via wsproto idna==3.6 # via requests imapclient==3.0.1 # via -r requirements.in -importlib-metadata==7.0.1 # via ax253, kiss3 +importlib-metadata==7.0.2 # via ax253, kiss3 itsdangerous==2.1.2 # via flask jinja2==3.1.3 # via click-completion, flask kiss3==8.0.0 # via -r requirements.in @@ -45,7 +44,7 @@ mypy-extensions==1.0.0 # via typing-inspect netaddr==1.2.1 # via oslo-config oslo-config==9.4.0 # via -r requirements.in oslo-i18n==6.3.0 # via oslo-config -packaging==23.2 # via marshmallow +packaging==24.0 # via marshmallow pbr==6.0.0 # via -r requirements.in, oslo-i18n, stevedore pluggy==1.4.0 # via -r requirements.in plumbum==1.8.2 # via rpyc @@ -76,7 +75,7 @@ validators==0.22.0 # via click-params werkzeug==3.0.1 # via -r requirements.in, flask wrapt==1.16.0 # via -r requirements.in, debtcollector, deprecated wsproto==1.2.0 # via simple-websocket -zipp==3.17.0 # via importlib-metadata +zipp==3.18.1 # via importlib-metadata zope-event==5.0 # via gevent zope-interface==6.2 # via gevent diff --git a/tests/cmds/test_webchat.py b/tests/cmds/test_webchat.py index 88e38e1..8280899 100644 --- a/tests/cmds/test_webchat.py +++ b/tests/cmds/test_webchat.py @@ -51,11 +51,8 @@ class TestSendMessageCommand(unittest.TestCase): ): self.config_and_init() mock_socketio.emit = mock.MagicMock() - packet = fake.fake_packet( - message="blah", - msg_number=1, - message_format=core.PACKET_TYPE_ACK, - ) + # Create an ACK packet + packet = fake.fake_ack_packet() mock_queue = mock.MagicMock() socketio = mock.MagicMock() wcp = webchat.WebChatProcessPacketThread(mock_queue, socketio) diff --git a/tests/fake.py b/tests/fake.py index 1912157..4a7c816 100644 --- a/tests/fake.py +++ b/tests/fake.py @@ -1,4 +1,4 @@ -from aprsd import packets, plugin, threads +from aprsd import plugin, threads from aprsd.packets import core @@ -13,6 +13,7 @@ def fake_packet( message=None, msg_number=None, message_format=core.PACKET_TYPE_MESSAGE, + response=None, ): packet_dict = { "from": fromcall, @@ -27,7 +28,17 @@ def fake_packet( if msg_number: packet_dict["msgNo"] = str(msg_number) - return packets.Packet.factory(packet_dict) + if response: + packet_dict["response"] = response + + return core.factory(packet_dict) + + +def fake_ack_packet(): + return fake_packet( + msg_number=12, + response=core.PACKET_TYPE_ACK, + ) class FakeBaseNoThreadsPlugin(plugin.APRSDPluginBase): diff --git a/tests/plugins/test_weather.py b/tests/plugins/test_weather.py index 8a85e0b..c8d814d 100644 --- a/tests/plugins/test_weather.py +++ b/tests/plugins/test_weather.py @@ -11,7 +11,7 @@ from .. import fake, test_plugin CONF = cfg.CONF -class TestUSWeatherPluginPlugin(test_plugin.TestPlugin): +class TestUSWeatherPlugin(test_plugin.TestPlugin): def test_not_enabled_missing_aprs_fi_key(self): # When the aprs.fi api key isn't set, then diff --git a/tests/test_packets.py b/tests/test_packets.py index 7b8fdca..15f1433 100644 --- a/tests/test_packets.py +++ b/tests/test_packets.py @@ -1,6 +1,8 @@ import unittest from unittest import mock +import aprslib + from aprsd import packets from aprsd.packets import core @@ -55,7 +57,7 @@ class TestPluginBase(unittest.TestCase): def test_packet_factory(self): pkt_dict = self._fake_dict() - pkt = packets.Packet.factory(pkt_dict) + pkt = packets.factory(pkt_dict) self.assertIsInstance(pkt, packets.MessagePacket) self.assertEqual(fake.FAKE_FROM_CALLSIGN, pkt.from_call) @@ -71,7 +73,7 @@ class TestPluginBase(unittest.TestCase): "comment": "Home!", } pkt_dict["format"] = core.PACKET_TYPE_UNCOMPRESSED - pkt = packets.Packet.factory(pkt_dict) + pkt = packets.factory(pkt_dict) self.assertIsInstance(pkt, packets.WeatherPacket) @mock.patch("aprsd.packets.core.GPSPacket._build_time_zulu") @@ -100,3 +102,31 @@ class TestPluginBase(unittest.TestCase): wx.prepare() expected = "KFAKE>KMINE,WIDE1-1,WIDE2-1:@221450z0.0/0.0_000/000g000t000r001p000P000h00b00000" self.assertEqual(expected, wx.raw) + + def test_beacon_factory(self): + """Test to ensure a beacon packet is created.""" + packet_raw = "WB4BOR-12>APZ100,WIDE2-1:@161647z3724.15N107847.58W$ APRSD WebChat" + packet_dict = aprslib.parse(packet_raw) + packet = packets.factory(packet_dict) + self.assertIsInstance(packet, packets.BeaconPacket) + + def test_reject_factory(self): + """Test to ensure a reject packet is created.""" + packet_raw = "HB9FDL-1>APK102,HB9FM-4*,WIDE2,qAR,HB9FEF-11::REPEAT :rej4139" + packet_dict = aprslib.parse(packet_raw) + packet = packets.factory(packet_dict) + self.assertIsInstance(packet, packets.RejectPacket) + + def test_thirdparty_factory(self): + """Test to ensure a third party packet is created.""" + packet_raw = "GTOWN>APDW16,WIDE1-1,WIDE2-1:}KM6LYW-9>APZ100,TCPIP,GTOWN*::KM6LYW :KM6LYW: 19 Miles SW" + packet_dict = aprslib.parse(packet_raw) + packet = packets.factory(packet_dict) + self.assertIsInstance(packet, packets.ThirdPartyPacket) + + def test_weather_factory(self): + """Test to ensure a weather packet is created.""" + packet_raw = "FW9222>APRS,TCPXX*,qAX,CWOP-6:@122025z2953.94N/08423.77W_232/003g006t084r000p032P000h80b10157L745.DsWLL" + packet_dict = aprslib.parse(packet_raw) + packet = packets.factory(packet_dict) + self.assertIsInstance(packet, packets.WeatherPacket) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index e69ede3..e9a17eb 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -45,7 +45,6 @@ class TestPluginManager(unittest.TestCase): self.assertEqual([], plugin_list) pm.setup_plugins() plugin_list = pm.get_plugins() - print(plugin_list) self.assertIsInstance(plugin_list, list) self.assertIsInstance( plugin_list[0], @@ -163,9 +162,7 @@ class TestPluginBase(TestPlugin): self.assertEqual(expected, actual) mock_process.assert_not_called() - packet = fake.fake_packet( - message_format=core.PACKET_TYPE_ACK, - ) + packet = fake.fake_ack_packet() expected = packets.NULL_MESSAGE actual = p.filter(packet) self.assertEqual(expected, actual) From 00e185b4e7ca5abed9f56282924e183b1201ea33 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 15:36:26 -0400 Subject: [PATCH 09/26] Fixed some tox errors related to mypy --- aprsd/plugin.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/aprsd/plugin.py b/aprsd/plugin.py index 905d583..c760d56 100644 --- a/aprsd/plugin.py +++ b/aprsd/plugin.py @@ -1,4 +1,5 @@ -# The base plugin class +from __future__ import annotations + import abc import importlib import inspect @@ -42,7 +43,7 @@ class APRSDPluginSpec: """A hook specification namespace.""" @hookspec - def filter(self, packet: packets.core.Packet): + def filter(self, packet: type[packets.Packet]): """My special little hook that you can customize.""" @@ -118,11 +119,11 @@ class APRSDPluginBase(metaclass=abc.ABCMeta): thread.stop() @abc.abstractmethod - def filter(self, packet: type[packets.core.Packet]) -> str | packets.core.MessagePacket: + def filter(self, packet: type[packets.Packet]) -> str | packets.MessagePacket: pass @abc.abstractmethod - def process(self, packet: type[packets.core.Packet]): + def process(self, packet: type[packets.Packet]): """This is called when the filter passes.""" @@ -154,7 +155,7 @@ class APRSDWatchListPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): LOG.warning("Watch list enabled, but no callsigns set.") @hookimpl - def filter(self, packet: type[packets.core.Packet]) -> str | packets.core.MessagePacket: + def filter(self, packet: type[packets.Packet]) -> str | packets.MessagePacket: result = packets.NULL_MESSAGE if self.enabled: wl = watch_list.WatchList() @@ -206,14 +207,14 @@ class APRSDRegexCommandPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): self.enabled = True @hookimpl - def filter(self, packet: packets.core.MessagePacket) -> str | packets.core.MessagePacket: + def filter(self, packet: packets.MessagePacket) -> str | packets.MessagePacket: LOG.info(f"{self.__class__.__name__} called") if not self.enabled: result = f"{self.__class__.__name__} isn't enabled" LOG.warning(result) return result - if not isinstance(packet, packets.core.MessagePacket): + if not isinstance(packet, packets.MessagePacket): LOG.warning(f"{self.__class__.__name__} Got a {packet.__class__.__name__} ignoring") return packets.NULL_MESSAGE @@ -226,7 +227,7 @@ class APRSDRegexCommandPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): # and is an APRS message format and has a message. if ( tocall == CONF.callsign - and isinstance(packet, packets.core.MessagePacket) + and isinstance(packet, packets.MessagePacket) and message ): if re.search(self.command_regex, message, re.IGNORECASE): @@ -269,7 +270,7 @@ class HelpPlugin(APRSDRegexCommandPluginBase): def help(self): return "Help: send APRS help or help " - def process(self, packet: packets.core.MessagePacket): + def process(self, packet: packets.MessagePacket): LOG.info("HelpPlugin") # fromcall = packet.get("from") message = packet.message_text @@ -469,12 +470,12 @@ class PluginManager: LOG.info("Completed Plugin Loading.") - def run(self, packet: packets.core.MessagePacket): + def run(self, packet: packets.MessagePacket): """Execute all the plugins run method.""" with self.lock: return self._pluggy_pm.hook.filter(packet=packet) - def run_watchlist(self, packet: packets.core.Packet): + def run_watchlist(self, packet: packets.Packet): with self.lock: return self._watchlist_pm.hook.filter(packet=packet) From eada5e9ce25ea388913af699b3042b13156b91f3 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 15:52:01 -0400 Subject: [PATCH 10/26] updated dev-requirements --- dev-requirements.txt | 277 ++++++++++++------------------------------- 1 file changed, 74 insertions(+), 203 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index dec7d53..cdcab43 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,209 +4,80 @@ # # pip-compile --annotation-style=line dev-requirements.in # -add-trailing-comma==3.1.0 - # via gray -alabaster==0.7.16 - # via sphinx -autoflake==1.5.3 - # via gray -babel==2.14.0 - # via sphinx -black==24.3.0 - # via gray -build==1.1.1 - # via pip-tools -cachetools==5.3.3 - # via tox -certifi==2024.2.2 - # via requests -cfgv==3.4.0 - # via pre-commit -chardet==5.2.0 - # via tox -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # black - # fixit - # moreorless - # pip-tools -colorama==0.4.6 - # via tox -commonmark==0.9.1 - # via rich -configargparse==1.7 - # via gray -coverage[toml]==7.4.3 - # via pytest-cov -distlib==0.3.8 - # via virtualenv -docutils==0.20.1 - # via sphinx -exceptiongroup==1.2.0 - # via pytest -filelock==3.13.1 - # via - # tox - # virtualenv -fixit==2.1.0 - # via gray -flake8==7.0.0 - # via - # -r dev-requirements.in - # pep8-naming -gray==0.14.0 - # via -r dev-requirements.in -identify==2.5.35 - # via pre-commit -idna==3.6 - # via requests -imagesize==1.4.1 - # via sphinx -iniconfig==2.0.0 - # via pytest -isort==5.13.2 - # via - # -r dev-requirements.in - # gray -jinja2==3.1.3 - # via sphinx -libcst==1.2.0 - # via fixit -markupsafe==2.1.5 - # via jinja2 -mccabe==0.7.0 - # via flake8 -moreorless==0.4.0 - # via fixit -mypy==1.8.0 - # via -r dev-requirements.in -mypy-extensions==1.0.0 - # via - # black - # mypy - # typing-inspect -nodeenv==1.8.0 - # via pre-commit -packaging==23.2 - # via - # black - # build - # fixit - # pyproject-api - # pytest - # sphinx - # tox -pathspec==0.12.1 - # via - # black - # trailrunner -pep8-naming==0.13.3 - # via -r dev-requirements.in -pip-tools==7.4.1 - # via -r dev-requirements.in -platformdirs==4.2.0 - # via - # black - # tox - # virtualenv -pluggy==1.4.0 - # via - # pytest - # tox -pre-commit==3.6.2 - # via -r dev-requirements.in -pycodestyle==2.11.1 - # via flake8 -pyflakes==3.2.0 - # via - # autoflake - # flake8 -pygments==2.17.2 - # via - # rich - # sphinx -pyproject-api==1.6.1 - # via tox -pyproject-hooks==1.0.0 - # via - # build - # pip-tools -pytest==8.0.2 - # via - # -r dev-requirements.in - # pytest-cov -pytest-cov==4.1.0 - # via -r dev-requirements.in -pyupgrade==3.15.1 - # via gray -pyyaml==6.0.1 - # via - # libcst - # pre-commit -requests==2.31.0 - # via sphinx -rich==12.6.0 - # via gray -snowballstemmer==2.2.0 - # via sphinx -sphinx==7.2.6 - # via -r dev-requirements.in -sphinxcontrib-applehelp==1.0.8 - # via sphinx -sphinxcontrib-devhelp==1.0.6 - # via sphinx -sphinxcontrib-htmlhelp==2.0.5 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.7 - # via sphinx -sphinxcontrib-serializinghtml==1.1.10 - # via sphinx -tokenize-rt==5.2.0 - # via - # add-trailing-comma - # pyupgrade -toml==0.10.2 - # via autoflake -tomli==2.0.1 - # via - # black - # build - # coverage - # fixit - # mypy - # pip-tools - # pyproject-api - # pyproject-hooks - # pytest - # tox -tox==4.14.0 - # via -r dev-requirements.in -trailrunner==1.4.0 - # via fixit -typing-extensions==4.10.0 - # via - # black - # libcst - # mypy - # typing-inspect -typing-inspect==0.9.0 - # via libcst -unify==0.5 - # via gray -untokenize==0.1.1 - # via unify -urllib3==2.2.1 - # via requests -virtualenv==20.25.1 - # via - # pre-commit - # tox -wheel==0.42.0 - # via pip-tools +add-trailing-comma==3.1.0 # via gray +alabaster==0.7.16 # via sphinx +autoflake==1.5.3 # via gray +babel==2.14.0 # via sphinx +black==24.3.0 # via gray +build==1.1.1 # via pip-tools +cachetools==5.3.3 # via tox +certifi==2024.2.2 # via requests +cfgv==3.4.0 # via pre-commit +chardet==5.2.0 # via tox +charset-normalizer==3.3.2 # via requests +click==8.1.7 # via black, fixit, moreorless, pip-tools +colorama==0.4.6 # via tox +commonmark==0.9.1 # via rich +configargparse==1.7 # via gray +coverage[toml]==7.4.4 # via pytest-cov +distlib==0.3.8 # via virtualenv +docutils==0.20.1 # via sphinx +exceptiongroup==1.2.0 # via pytest +filelock==3.13.1 # via tox, virtualenv +fixit==2.1.0 # via gray +flake8==7.0.0 # via -r dev-requirements.in, pep8-naming +gray==0.14.0 # via -r dev-requirements.in +identify==2.5.35 # via pre-commit +idna==3.6 # via requests +imagesize==1.4.1 # via sphinx +iniconfig==2.0.0 # via pytest +isort==5.13.2 # via -r dev-requirements.in, gray +jinja2==3.1.3 # via sphinx +libcst==1.2.0 # via fixit +markupsafe==2.1.5 # via jinja2 +mccabe==0.7.0 # via flake8 +moreorless==0.4.0 # via fixit +mypy==1.9.0 # via -r dev-requirements.in +mypy-extensions==1.0.0 # via black, mypy, typing-inspect +nodeenv==1.8.0 # via pre-commit +packaging==24.0 # via black, build, fixit, pyproject-api, pytest, sphinx, tox +pathspec==0.12.1 # via black, trailrunner +pep8-naming==0.13.3 # via -r dev-requirements.in +pip-tools==7.4.1 # via -r dev-requirements.in +platformdirs==4.2.0 # via black, tox, virtualenv +pluggy==1.4.0 # via pytest, tox +pre-commit==3.6.2 # via -r dev-requirements.in +pycodestyle==2.11.1 # via flake8 +pyflakes==3.2.0 # via autoflake, flake8 +pygments==2.17.2 # via rich, sphinx +pyproject-api==1.6.1 # via tox +pyproject-hooks==1.0.0 # via build, pip-tools +pytest==8.1.1 # via -r dev-requirements.in, pytest-cov +pytest-cov==4.1.0 # via -r dev-requirements.in +pyupgrade==3.15.1 # via gray +pyyaml==6.0.1 # via libcst, pre-commit +requests==2.31.0 # via sphinx +rich==12.6.0 # via gray +snowballstemmer==2.2.0 # via sphinx +sphinx==7.2.6 # via -r dev-requirements.in +sphinxcontrib-applehelp==1.0.8 # via sphinx +sphinxcontrib-devhelp==1.0.6 # via sphinx +sphinxcontrib-htmlhelp==2.0.5 # via sphinx +sphinxcontrib-jsmath==1.0.1 # via sphinx +sphinxcontrib-qthelp==1.0.7 # via sphinx +sphinxcontrib-serializinghtml==1.1.10 # via sphinx +tokenize-rt==5.2.0 # via add-trailing-comma, pyupgrade +toml==0.10.2 # via autoflake +tomli==2.0.1 # via black, build, coverage, fixit, mypy, pip-tools, pyproject-api, pyproject-hooks, pytest, tox +tox==4.14.1 # via -r dev-requirements.in +trailrunner==1.4.0 # via fixit +typing-extensions==4.10.0 # via black, libcst, mypy, typing-inspect +typing-inspect==0.9.0 # via libcst +unify==0.5 # via gray +untokenize==0.1.1 # via unify +urllib3==2.2.1 # via requests +virtualenv==20.25.1 # via pre-commit, tox +wheel==0.43.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: # pip From 386d2bea6260e6b0ea63aec0b26151e2a9602984 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 16:12:18 -0400 Subject: [PATCH 11/26] Fix for micE packet decoding with mbits --- aprsd/packets/core.py | 8 ++++++-- tests/test_packets.py | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 560423d..9337844 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -463,8 +463,10 @@ class MicEPacket(GPSPacket): # 0 to 360 course: int = 0 - def _build_payload(self): - raise NotImplementedError + @staticmethod + def from_aprslib_dict(raw: dict) -> "MicEPacket": + raw = _translate_fields(raw) + return MicEPacket(**raw) @dataclass @@ -768,4 +770,6 @@ def factory(raw_packet: dict) -> type[Packet]: else: raise Exception(f"Unknown packet type {packet_type} {raw}") + print(f"factory({packet_type}): {class_name} {raw}") + return class_name.from_aprslib_dict(raw) diff --git a/tests/test_packets.py b/tests/test_packets.py index 15f1433..9080591 100644 --- a/tests/test_packets.py +++ b/tests/test_packets.py @@ -130,3 +130,9 @@ class TestPluginBase(unittest.TestCase): packet_dict = aprslib.parse(packet_raw) packet = packets.factory(packet_dict) self.assertIsInstance(packet, packets.WeatherPacket) + + def test_mice_factory(self): + packet_raw = 'kh2sr-15>S7TSYR,WIDE1-1,WIDE2-1,qAO,KO6KL-1:`1`7\x1c\x1c.#/`"4,}QuirkyQRP 4.6V 35.3C S06' + packet_dict = aprslib.parse(packet_raw) + packet = packets.factory(packet_dict) + self.assertIsInstance(packet, packets.MicEPacket) From e386e91f6e58e63a405eb2365809501c52a05abd Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 20 Mar 2024 21:46:43 -0400 Subject: [PATCH 12/26] Eliminated need for from_aprslib_dict This patch eliminates the need for a custom static method on each Packetclass to convert an aprslib raw decoded dictionary -> correct Packet class. This now uses the built in dataclasses_json from_dict() mixin with an override for both the WeatherPacket and the ThirdPartyPacket. This patch also adds the TelemetryPacket and adds some missing members to a few of the classes from test runs decoding all packets from APRS-IS -> Packet classes. Also adds some verification for packets in test_packets --- aprsd/packets/core.py | 146 ++++++++++++++++++++++-------------------- tests/test_packets.py | 29 +++++++++ 2 files changed, 104 insertions(+), 71 deletions(-) diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 9337844..93eaa50 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -4,13 +4,17 @@ import logging import re import time # Due to a failure in python 3.8 -from typing import List, Optional +from typing import Any, List, Optional, Type, TypeVar, Union -from dataclasses_json import DataClassJsonMixin +from dataclasses_json import DataClassJsonMixin, dataclass_json from aprsd.utils import counter +# For mypy to be happy +A = TypeVar("A", bound="DataClassJsonMixin") +Json = Union[dict, list, str, int, float, bool, None] + LOG = logging.getLogger("APRSD") PACKET_TYPE_MESSAGE = "message" @@ -24,6 +28,7 @@ PACKET_TYPE_UNKNOWN = "unknown" PACKET_TYPE_STATUS = "status" PACKET_TYPE_BEACON = "beacon" PACKET_TYPE_THIRDPARTY = "thirdparty" +PACKET_TYPE_TELEMETRY = "telemetry-message" PACKET_TYPE_UNCOMPRESSED = "uncompressed" NO_DATE = datetime(1900, 10, 24) @@ -69,8 +74,9 @@ def _translate_fields(raw: dict) -> dict: return raw +@dataclass_json @dataclass(unsafe_hash=True) -class Packet(DataClassJsonMixin): +class Packet: _type: str = field(default="Packet", hash=False) from_call: Optional[str] = field(default=None) to_call: Optional[str] = field(default=None) @@ -101,13 +107,13 @@ class Packet(DataClassJsonMixin): def json(self) -> str: """get the json formated string""" # comes from the DataClassJsonMixin - return self.to_json() + return self.to_json() # type: ignore @property def dict(self) -> dict: """get the dict formated string""" # comes from the DataClassJsonMixin - return self.to_dict() + return self.to_dict() # type: ignore def get(self, key: str, default: Optional[str] = None): """Emulate a getter on a dict.""" @@ -214,16 +220,12 @@ class Packet(DataClassJsonMixin): return repr +@dataclass_json @dataclass(unsafe_hash=True) class AckPacket(Packet): _type: str = field(default="AckPacket", hash=False) response: Optional[str] = field(default=None) - @staticmethod - def from_aprslib_dict(raw: dict) -> "AckPacket": - raw = _translate_fields(raw) - return AckPacket(**raw) - def __post__init__(self): if self.response: LOG.warning("Response set!") @@ -232,16 +234,12 @@ class AckPacket(Packet): self.payload = f":{self.to_call.ljust(9)}:ack{self.msgNo}" +@dataclass_json @dataclass(unsafe_hash=True) class RejectPacket(Packet): _type: str = field(default="RejectPacket", hash=False) response: Optional[str] = field(default=None) - @staticmethod - def from_aprslib_dict(raw: dict) -> "RejectPacket": - raw = _translate_fields(raw) - return RejectPacket(**raw) - def __post__init__(self): if self.response: LOG.warning("Response set!") @@ -250,6 +248,7 @@ class RejectPacket(Packet): self.payload = f":{self.to_call.ljust(9)} :rej{self.msgNo}" +@dataclass_json @dataclass(unsafe_hash=True) class MessagePacket(Packet): _type: str = field(default="MessagePacket", hash=False) @@ -275,28 +274,21 @@ class MessagePacket(Packet): str(self.msgNo), ) - @staticmethod - def from_aprslib_dict(raw: dict) -> "MessagePacket": - raw = _translate_fields(raw) - return MessagePacket(**raw) - +@dataclass_json @dataclass(unsafe_hash=True) class StatusPacket(Packet): _type: str = field(default="StatusPacket", hash=False) status: Optional[str] = field(default=None) messagecapable: bool = field(default=False) comment: Optional[str] = field(default=None) - - @staticmethod - def from_aprslib_dict(raw: dict) -> "StatusPacket": - raw = _translate_fields(raw) - return StatusPacket(**raw) + raw_timestamp: Optional[str] = field(default=None) def _build_payload(self): raise NotImplementedError +@dataclass_json @dataclass(unsafe_hash=True) class GPSPacket(Packet): _type: str = field(default="GPSPacket", hash=False) @@ -315,11 +307,15 @@ class GPSPacket(Packet): alive: Optional[bool] = field(default=None) course: Optional[int] = field(default=None) speed: Optional[float] = field(default=None) - - @staticmethod - def from_aprslib_dict(raw: dict) -> "GPSPacket": - raw = _translate_fields(raw) - return GPSPacket(**raw) + phg: Optional[str] = field(default=None) + phg_power: Optional[int] = field(default=None) + phg_height: Optional[float] = field(default=None) + phg_gain: Optional[int] = field(default=None) + phg_dir: Optional[str] = field(default=None) + phg_range: Optional[float] = field(default=None) + phg_rate: Optional[int] = field(default=None) + # http://www.aprs.org/datum.txt + daodatumbyte: Optional[str] = field(default=None) def decdeg2dms(self, degrees_decimal): is_positive = degrees_decimal >= 0 @@ -425,15 +421,11 @@ class GPSPacket(Packet): ) +@dataclass_json @dataclass(unsafe_hash=True) class BeaconPacket(GPSPacket): _type: str = field(default="BeaconPacket", hash=False) - @staticmethod - def from_aprslib_dict(raw: dict) -> "BeaconPacket": - raw = _translate_fields(raw) - return BeaconPacket(**raw) - def _build_payload(self): """The payload is the non headers portion of the packet.""" time_zulu = self._build_time_zulu() @@ -452,23 +444,37 @@ class BeaconPacket(GPSPacket): ) +@dataclass_json @dataclass class MicEPacket(GPSPacket): _type: str = field(default="MicEPacket", hash=False) messagecapable: bool = False mbits: Optional[str] = None mtype: Optional[str] = None + telemetry: Optional[dict] = field(default=None) # in MPH speed: float = 0.00 # 0 to 360 course: int = 0 - @staticmethod - def from_aprslib_dict(raw: dict) -> "MicEPacket": - raw = _translate_fields(raw) - return MicEPacket(**raw) + +@dataclass_json +@dataclass +class TelemetryPacket(GPSPacket): + _type: str = field(default="TelemetryPacket", hash=False) + messagecapable: bool = False + mbits: Optional[str] = None + mtype: Optional[str] = None + telemetry: Optional[dict] = field(default=None) + tPARM: Optional[list[str]] = field(default=None) # noqa: N815 + tUNIT: Optional[list[str]] = field(default=None) # noqa: N815 + # in MPH + speed: float = 0.00 + # 0 to 360 + course: int = 0 +@dataclass_json @dataclass class ObjectPacket(GPSPacket): _type: str = field(default="ObjectPacket", hash=False) @@ -480,11 +486,6 @@ class ObjectPacket(GPSPacket): # 0 to 360 course: int = 0 - @staticmethod - def from_aprslib_dict(raw: dict) -> "ObjectPacket": - raw = _translate_fields(raw) - return ObjectPacket(**raw) - def _build_payload(self): time_zulu = self._build_time_zulu() lat = self.convert_latitude(self.latitude) @@ -516,7 +517,7 @@ class ObjectPacket(GPSPacket): @dataclass() -class WeatherPacket(GPSPacket): +class WeatherPacket(GPSPacket, DataClassJsonMixin): _type: str = field(default="WeatherPacket", hash=False) symbol: str = "_" wind_speed: float = 0.00 @@ -535,13 +536,7 @@ class WeatherPacket(GPSPacket): course: Optional[int] = field(default=None) speed: Optional[float] = field(default=None) - @staticmethod - def from_aprslib_dict(raw: dict) -> "WeatherPacket": - """Create from a dictionary that has come directly from aprslib parse""" - # Because from is a reserved word in python, we need to translate it - # from -> from_call and to -> to_call - raw = _translate_fields(raw) - + def _translate(self, raw: dict) -> dict: for key in raw["weather"]: raw[key] = raw["weather"][key] @@ -579,7 +574,13 @@ class WeatherPacket(GPSPacket): del raw["course"] del raw["weather"] - return WeatherPacket(**raw) + return raw + + @classmethod + def from_dict(cls: Type[A], kvs: Json, *, infer_missing=False) -> A: + """Create from a dictionary that has come directly from aprslib parse""" + raw = cls._translate(cls, kvs) # type: ignore + return super().from_dict(raw) def _build_payload(self): """Build an uncompressed weather packet @@ -642,7 +643,7 @@ class WeatherPacket(GPSPacket): @dataclass() -class ThirdPartyPacket(Packet): +class ThirdPartyPacket(Packet, DataClassJsonMixin): _type: str = "ThirdPartyPacket" # Holds the encapsulated packet subpacket: Optional[type[Packet]] = field(default=None, compare=True, hash=False) @@ -658,15 +659,11 @@ class ThirdPartyPacket(Packet): return repr_str - @staticmethod - def from_aprslib_dict(raw: dict) -> "ThirdPartyPacket": - """Create from a dictionary that has come directly from aprslib parse""" - # Because from is a reserved word in python, we need to translate it - # from -> from_call and to -> to_call - raw = _translate_fields(raw) - subpacket = raw.get("subpacket") - del raw["subpacket"] - return ThirdPartyPacket(**raw, subpacket=factory(subpacket)) + @classmethod + def from_dict(cls: Type[A], kvs: Json, *, infer_missing=False) -> A: + obj = super().from_dict(kvs) + obj.subpacket = factory(obj.subpacket) # type: ignore + return obj TYPE_LOOKUP: dict[str, type[Packet]] = { @@ -681,6 +678,7 @@ TYPE_LOOKUP: dict[str, type[Packet]] = { PACKET_TYPE_BEACON: BeaconPacket, PACKET_TYPE_UNKNOWN: Packet, PACKET_TYPE_THIRDPARTY: ThirdPartyPacket, + PACKET_TYPE_TELEMETRY: TelemetryPacket, } OBJ_LOOKUP: dict[str, type[Packet]] = { @@ -694,6 +692,7 @@ OBJ_LOOKUP: dict[str, type[Packet]] = { "BeaconPacket": BeaconPacket, "WeatherPacket": WeatherPacket, "ThirdPartyPacket": ThirdPartyPacket, + "TelemetryPacket": TelemetryPacket, } @@ -717,6 +716,8 @@ def get_packet_type(packet: dict) -> str: packet_type = PACKET_TYPE_STATUS elif pkt_format == PACKET_TYPE_BEACON: packet_type = PACKET_TYPE_BEACON + elif pkt_format == PACKET_TYPE_TELEMETRY: + packet_type = PACKET_TYPE_TELEMETRY elif pkt_format == PACKET_TYPE_WX: packet_type = PACKET_TYPE_WEATHER elif pkt_format == PACKET_TYPE_UNCOMPRESSED: @@ -739,11 +740,11 @@ def is_ack_packet(packet: dict) -> bool: return get_packet_type(packet) == PACKET_TYPE_ACK -def is_mice_packet(packet: dict) -> bool: +def is_mice_packet(packet: dict[Any, Any]) -> bool: return get_packet_type(packet) == PACKET_TYPE_MICE -def factory(raw_packet: dict) -> type[Packet]: +def factory(raw_packet: dict[Any, Any]) -> type[Packet]: """Factory method to create a packet from a raw packet string.""" raw = raw_packet if "_type" in raw: @@ -756,20 +757,23 @@ def factory(raw_packet: dict) -> type[Packet]: packet_type = get_packet_type(raw) raw["packet_type"] = packet_type - class_name = TYPE_LOOKUP[packet_type] + packet_class = TYPE_LOOKUP[packet_type] if packet_type == PACKET_TYPE_WX: # the weather information is in a dict # this brings those values out to the outer dict - class_name = WeatherPacket + packet_class = WeatherPacket elif packet_type == PACKET_TYPE_OBJECT and "weather" in raw: - class_name = WeatherPacket + packet_class = WeatherPacket elif packet_type == PACKET_TYPE_UNKNOWN: # Try and figure it out here if "latitude" in raw: - class_name = GPSPacket + packet_class = GPSPacket else: + LOG.error(f"Unknown packet type {packet_type}") + LOG.error(raw) raise Exception(f"Unknown packet type {packet_type} {raw}") - print(f"factory({packet_type}): {class_name} {raw}") + # LOG.info(f"factory({packet_type}):({raw.get('from_call')}) {packet_class.__class__} {raw}") + # LOG.error(packet_class) - return class_name.from_aprslib_dict(raw) + return packet_class().from_dict(raw) # type: ignore diff --git a/tests/test_packets.py b/tests/test_packets.py index 9080591..e8e3679 100644 --- a/tests/test_packets.py +++ b/tests/test_packets.py @@ -110,6 +110,11 @@ class TestPluginBase(unittest.TestCase): packet = packets.factory(packet_dict) self.assertIsInstance(packet, packets.BeaconPacket) + packet_raw = "kd8mey-10>APRS,TCPIP*,qAC,T2SYDNEY:=4247.80N/08539.00WrPHG1210/Making 220 Great Again Allstar# 552191" + packet_dict = aprslib.parse(packet_raw) + packet = packets.factory(packet_dict) + self.assertIsInstance(packet, packets.BeaconPacket) + def test_reject_factory(self): """Test to ensure a reject packet is created.""" packet_raw = "HB9FDL-1>APK102,HB9FM-4*,WIDE2,qAR,HB9FEF-11::REPEAT :rej4139" @@ -117,6 +122,12 @@ class TestPluginBase(unittest.TestCase): packet = packets.factory(packet_dict) self.assertIsInstance(packet, packets.RejectPacket) + self.assertEqual("4139", packet.msgNo) + self.assertEqual("HB9FDL-1", packet.from_call) + self.assertEqual("REPEAT", packet.to_call) + self.assertEqual("reject", packet.packet_type) + self.assertIsNone(packet.payload) + def test_thirdparty_factory(self): """Test to ensure a third party packet is created.""" packet_raw = "GTOWN>APDW16,WIDE1-1,WIDE2-1:}KM6LYW-9>APZ100,TCPIP,GTOWN*::KM6LYW :KM6LYW: 19 Miles SW" @@ -131,8 +142,26 @@ class TestPluginBase(unittest.TestCase): packet = packets.factory(packet_dict) self.assertIsInstance(packet, packets.WeatherPacket) + self.assertEqual(28.88888888888889, packet.temperature) + self.assertEqual(0.0, packet.rain_1h) + self.assertEqual(1015.7, packet.pressure) + self.assertEqual(80, packet.humidity) + self.assertEqual(745, packet.luminosity) + self.assertEqual(3.0, packet.wind_speed) + self.assertEqual(232, packet.wind_direction) + self.assertEqual(6.0, packet.wind_gust) + self.assertEqual(29.899, packet.latitude) + self.assertEqual(-84.39616666666667, packet.longitude) + def test_mice_factory(self): packet_raw = 'kh2sr-15>S7TSYR,WIDE1-1,WIDE2-1,qAO,KO6KL-1:`1`7\x1c\x1c.#/`"4,}QuirkyQRP 4.6V 35.3C S06' packet_dict = aprslib.parse(packet_raw) packet = packets.factory(packet_dict) self.assertIsInstance(packet, packets.MicEPacket) + + # Packet with telemetry and DAO + # http://www.aprs.org/datum.txt + packet_raw = 'KD9YIL>T0PX9W,WIDE1-1,WIDE2-1,qAO,NU9R-10:`sB,l#P>/\'"6+}|#*%U\'a|!whl!|3' + packet_dict = aprslib.parse(packet_raw) + packet = packets.factory(packet_dict) + self.assertIsInstance(packet, packets.MicEPacket) From 9858955d34b88003285c4c588eacb32ca3a4ce67 Mon Sep 17 00:00:00 2001 From: Hemna Date: Fri, 22 Mar 2024 23:16:00 -0400 Subject: [PATCH 13/26] Don't call stats so often in webchat --- aprsd/web/chat/static/js/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aprsd/web/chat/static/js/main.js b/aprsd/web/chat/static/js/main.js index dc27310..a0c505c 100644 --- a/aprsd/web/chat/static/js/main.js +++ b/aprsd/web/chat/static/js/main.js @@ -37,7 +37,7 @@ function start_update() { update_stats(data); }, complete: function() { - setTimeout(statsworker, 10000); + setTimeout(statsworker, 60000); } }); })(); From f450238348e6869f5ec04d247d6825f719582a5f Mon Sep 17 00:00:00 2001 From: Hemna Date: Fri, 22 Mar 2024 23:18:47 -0400 Subject: [PATCH 14/26] Added missing packet types in listen command This patch adds some missing packet objects for the listen command. Also moves the keepalive startup a little later --- aprsd/cmds/listen.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/aprsd/cmds/listen.py b/aprsd/cmds/listen.py index 92c0007..eaa69f9 100644 --- a/aprsd/cmds/listen.py +++ b/aprsd/cmds/listen.py @@ -17,6 +17,7 @@ from rich.console import Console import aprsd from aprsd import cli_helper, client, packets, plugin, stats, threads from aprsd.main import cli +from aprsd.packets import log as packet_log from aprsd.rpc import server as rpc_server from aprsd.threads import rx @@ -53,27 +54,31 @@ class APRSDListenThread(rx.APRSDRXThread): filters = { packets.Packet.__name__: packets.Packet, packets.AckPacket.__name__: packets.AckPacket, + packets.BeaconPacket.__name__: packets.BeaconPacket, packets.GPSPacket.__name__: packets.GPSPacket, packets.MessagePacket.__name__: packets.MessagePacket, packets.MicEPacket.__name__: packets.MicEPacket, + packets.ObjectPacket.__name__: packets.ObjectPacket, + packets.StatusPacket.__name__: packets.StatusPacket, + packets.ThirdPartyPacket.__name__: packets.ThirdPartyPacket, packets.WeatherPacket.__name__: packets.WeatherPacket, + packets.UnknownPacket.__name__: packets.UnknownPacket, } if self.packet_filter: filter_class = filters[self.packet_filter] if isinstance(packet, filter_class): - packet.log(header="RX") + packet_log.log(packet) if self.plugin_manager: # Don't do anything with the reply # This is the listen only command. self.plugin_manager.run(packet) else: + packet_log.log(packet) if self.plugin_manager: # Don't do anything with the reply. # This is the listen only command. self.plugin_manager.run(packet) - else: - packet.log(header="RX") packets.PacketList().rx(packet) @@ -96,11 +101,16 @@ class APRSDListenThread(rx.APRSDRXThread): "--packet-filter", type=click.Choice( [ - packets.Packet.__name__, packets.AckPacket.__name__, + packets.BeaconPacket.__name__, packets.GPSPacket.__name__, packets.MicEPacket.__name__, packets.MessagePacket.__name__, + packets.ObjectPacket.__name__, + packets.RejectPacket.__name__, + packets.StatusPacket.__name__, + packets.ThirdPartyPacket.__name__, + packets.UnknownPacket.__name__, packets.WeatherPacket.__name__, ], case_sensitive=False, @@ -180,7 +190,7 @@ def listen( aprs_client.set_filter(filter) keepalive = threads.KeepAliveThread() - keepalive.start() + #keepalive.start() if CONF.rpc_settings.enabled: rpc = rpc_server.APRSDRPCThread() @@ -205,6 +215,7 @@ def listen( ) LOG.debug("Start APRSDListenThread") listen_thread.start() + keepalive.start() LOG.debug("keepalive Join") keepalive.join() LOG.debug("listen_thread Join") From 127d3b3f267b9f547a1c593ecbdbaf510d633fd5 Mon Sep 17 00:00:00 2001 From: Hemna Date: Fri, 22 Mar 2024 23:19:54 -0400 Subject: [PATCH 15/26] Fixed some logging in webchat --- aprsd/cmds/webchat.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index d6bd1f3..18a2385 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -30,7 +30,7 @@ from aprsd.utils import trace CONF = cfg.CONF -LOG = logging.getLogger("APRSD") +LOG = logging.getLogger() auth = HTTPBasicAuth() users = {} socketio = None @@ -335,7 +335,6 @@ class WebChatProcessPacketThread(rx.APRSDProcessPacketThread): def process_our_message_packet(self, packet: packets.MessagePacket): global callsign_locations - LOG.info(f"process MessagePacket {repr(packet)}") # ok lets see if we have the location for the # person we just sent a message to. from_call = packet.get("from_call").upper() @@ -624,7 +623,7 @@ def webchat(ctx, flush, port): LOG.info(msg) LOG.info(f"APRSD Started version: {aprsd.__version__}") - CONF.log_opt_values(LOG, logging.DEBUG) + CONF.log_opt_values(logging.getLogger(), logging.DEBUG) user = CONF.admin.user users[user] = generate_password_hash(CONF.admin.password) if not port: From 1e6c483002be894c8862dd218b7ab7a52b65d2ef Mon Sep 17 00:00:00 2001 From: Hemna Date: Fri, 22 Mar 2024 23:20:16 -0400 Subject: [PATCH 16/26] Completely redo logging of packets!! refactored all logging of packets. Packet class now doesn't do logging. the format of the packet log now lives on a single line with colors. Created a new packet property called human_info, which creates a string for the payload of each packet type in a human readable format. TODO: need to create a config option to allow showing the older style of multiline logs for packets. --- aprsd/log/log.py | 5 +- aprsd/packets/__init__.py | 5 +- aprsd/packets/core.py | 225 +++++++++++++++++++++++++++----------- aprsd/plugin.py | 2 +- aprsd/threads/rx.py | 7 +- aprsd/threads/tx.py | 8 +- 6 files changed, 180 insertions(+), 72 deletions(-) diff --git a/aprsd/log/log.py b/aprsd/log/log.py index e1ee3e7..c64fbb4 100644 --- a/aprsd/log/log.py +++ b/aprsd/log/log.py @@ -10,7 +10,8 @@ from aprsd.conf import log as conf_log CONF = cfg.CONF -LOG = logging.getLogger("APRSD") +#LOG = logging.getLogger("APRSD") +LOG = logger logging_queue = queue.Queue() @@ -70,6 +71,8 @@ def setup_logging(loglevel=None, quiet=False): { "sink": sys.stdout, "serialize": False, "format": CONF.logging.logformat, + "colorize": True, + "level": log_level, }, ] if CONF.logging.logfile: diff --git a/aprsd/packets/__init__.py b/aprsd/packets/__init__.py index 17ecb89..1335637 100644 --- a/aprsd/packets/__init__.py +++ b/aprsd/packets/__init__.py @@ -1,6 +1,7 @@ from aprsd.packets.core import ( # noqa: F401 - AckPacket, BeaconPacket, GPSPacket, MessagePacket, MicEPacket, Packet, - RejectPacket, StatusPacket, ThirdPartyPacket, WeatherPacket, factory, + AckPacket, BeaconPacket, GPSPacket, MessagePacket, MicEPacket, ObjectPacket, + Packet, RejectPacket, StatusPacket, ThirdPartyPacket, UnknownPacket, + WeatherPacket, factory, ) from aprsd.packets.packet_list import PacketList # noqa: F401 from aprsd.packets.seen_list import SeenList # noqa: F401 diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 93eaa50..3a7e0d9 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -1,4 +1,4 @@ -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from datetime import datetime import logging import re @@ -6,7 +6,10 @@ import time # Due to a failure in python 3.8 from typing import Any, List, Optional, Type, TypeVar, Union -from dataclasses_json import DataClassJsonMixin, dataclass_json +from dataclasses_json import ( + CatchAll, DataClassJsonMixin, Undefined, dataclass_json, +) +from loguru import logger from aprsd.utils import counter @@ -15,8 +18,10 @@ from aprsd.utils import counter A = TypeVar("A", bound="DataClassJsonMixin") Json = Union[dict, list, str, int, float, bool, None] -LOG = logging.getLogger("APRSD") +LOG = logging.getLogger() +LOGU = logger +PACKET_TYPE_BULLETIN = "bulletin" PACKET_TYPE_MESSAGE = "message" PACKET_TYPE_ACK = "ack" PACKET_TYPE_REJECT = "reject" @@ -82,7 +87,7 @@ class Packet: to_call: Optional[str] = field(default=None) addresse: Optional[str] = field(default=None) format: Optional[str] = field(default=None) - msgNo: str = field(default_factory=_init_msgNo) # noqa: N815 + msgNo: Optional[str] = field(default=None) # noqa: N815 packet_type: Optional[str] = field(default=None) timestamp: float = field(default_factory=_init_timestamp, compare=False, hash=False) # Holds the raw text string to be sent over the wire @@ -130,9 +135,23 @@ class Packet: def update_timestamp(self) -> None: self.timestamp = _init_timestamp() + @property + def human_info(self) -> str: + """Build a human readable string for this packet. + + This doesn't include the from to and type, but just + the human readable payload. + """ + self.prepare() + msg = self._filter_for_send().rstrip("\n") + return msg + + def prepare(self) -> None: """Do stuff here that is needed prior to sending over the air.""" # now build the raw message for sending + if not self.msgNo: + self.msgNo = _init_msgNo() self._build_payload() self._build_raw() @@ -153,43 +172,6 @@ class Packet: self.payload, ) - def log(self, header: Optional[str] = None) -> None: - """LOG a packet to the logfile.""" - asdict(self) - log_list = ["\n"] - name = self.__class__.__name__ - if header: - if "tx" in header.lower(): - log_list.append( - f"{header}________({name} " - f"TX:{self.send_count+1} of {self.retry_count})", - ) - else: - log_list.append(f"{header}________({name})") - # log_list.append(f" Packet : {self.__class__.__name__}") - log_list.append(f" Raw : {self.raw}") - if self.to_call: - log_list.append(f" To : {self.to_call}") - if self.from_call: - log_list.append(f" From : {self.from_call}") - if hasattr(self, "path") and self.path: - log_list.append(f" Path : {'=>'.join(self.path)}") - if hasattr(self, "via") and self.via: - log_list.append(f" VIA : {self.via}") - - elif isinstance(self, MessagePacket): - log_list.append(f" Message : {self.message_text}") - - if hasattr(self, "comment") and self.comment: - log_list.append(f" Comment : {self.comment}") - - if self.msgNo: - log_list.append(f" Msg # : {self.msgNo}") - log_list.append(f"{header}________({name})") - - LOG.info("\n".join(log_list)) - LOG.debug(repr(self)) - def _filter_for_send(self) -> str: """Filter and format message string for FCC.""" # max? ftm400 displays 64, raw msg shows 74 @@ -284,8 +266,31 @@ class StatusPacket(Packet): comment: Optional[str] = field(default=None) raw_timestamp: Optional[str] = field(default=None) + def _filter_for_send(self) -> str: + """Filter and format message string for FCC.""" + # max? ftm400 displays 64, raw msg shows 74 + # and ftm400-send is max 64. setting this to + # 67 displays 64 on the ftm400. (+3 {01 suffix) + # feature req: break long ones into two msgs + if not self.status and not self.comment: + self.status = "None" + + message = self.status[:67] + # We all miss George Carlin + return re.sub("fuck|shit|cunt|piss|cock|bitch", "****", message) + def _build_payload(self): - raise NotImplementedError + self.payload = ":{}:{}{{{}".format( + self.to_call.ljust(9), + self._filter_for_send().rstrip("\n"), + str(self.msgNo), + ) + + @property + def human_info(self) -> str: + self.prepare() + return self.status + @dataclass_json @@ -420,6 +425,25 @@ class GPSPacket(Packet): f"{self.payload}" ) + @property + def human_info(self) -> str: + #LOG.warning(self.to_dict()) + h_str = [] + h_str.append(f"Lat:{self.latitude:03.3f}") + h_str.append(f"Lon:{self.longitude:03.3f}") + if self.altitude: + h_str.append(f"Altitude {self.altitude:03.0f}") + if self.speed: + h_str.append(f"Speed {self.speed:03.0f}MPH") + if self.course: + h_str.append(f"Course {self.course:03.0f}") + if self.rng: + h_str.append(f"RNG {self.rng:03.0f}") + if self.phg: + h_str.append(f"PHG {self.phg}") + + return " ".join(h_str) + @dataclass_json @dataclass(unsafe_hash=True) @@ -443,6 +467,14 @@ class BeaconPacket(GPSPacket): f"{self.payload}" ) + @property + def human_info(self) -> str: + h_str = [] + h_str.append(f"Lat:{self.latitude:03.3f}") + h_str.append(f"Lon:{self.longitude:03.3f}") + h_str.append(f"{self.comment}") + return " ".join(h_str) + @dataclass_json @dataclass @@ -457,6 +489,10 @@ class MicEPacket(GPSPacket): # 0 to 360 course: int = 0 + @property + def human_info(self) -> str: + h_info = super().human_info + return f"{h_info} {self.mbits} mbits" @dataclass_json @dataclass @@ -473,6 +509,11 @@ class TelemetryPacket(GPSPacket): # 0 to 360 course: int = 0 + @property + def human_info(self) -> str: + h_info = super().human_info + return f"{h_info} {self.telemetry}" + @dataclass_json @dataclass @@ -515,6 +556,11 @@ class ObjectPacket(GPSPacket): f"{self.payload}" ) + @property + def human_info(self) -> str: + h_info = super().human_info + return f"{h_info} {self.comment}" + @dataclass() class WeatherPacket(GPSPacket, DataClassJsonMixin): @@ -582,6 +628,17 @@ class WeatherPacket(GPSPacket, DataClassJsonMixin): raw = cls._translate(cls, kvs) # type: ignore return super().from_dict(raw) + @property + def human_info(self) -> str: + h_str = [] + h_str.append(f"Temp {self.temperature:03.0f}F") + h_str.append(f"Humidity {self.humidity}%") + h_str.append(f"Wind {self.wind_speed:03.0f}MPH@{self.wind_direction}") + h_str.append(f"Pressure {self.pressure}mb") + h_str.append(f"Rain {self.rain_24h}in/24hr") + + return " ".join(h_str) + def _build_payload(self): """Build an uncompressed weather packet @@ -665,8 +722,56 @@ class ThirdPartyPacket(Packet, DataClassJsonMixin): obj.subpacket = factory(obj.subpacket) # type: ignore return obj + @property + def human_info(self) -> str: + sub_info = self.subpacket.human_info + return f"{self.from_call}->{self.to_call} {sub_info}" + + +@dataclass_json +@dataclass +class BulletinPacket(Packet): + _type: str = "BulletinPacket" + # Holds the encapsulated packet + bid: Optional[str] = field(default="1") + message_text: Optional[str] = field(default=None) + + @property + def human_info(self) -> str: + return f"BLN{self.bid} {self.message_text}" + + +@dataclass_json(undefined=Undefined.INCLUDE) +@dataclass +class UnknownPacket: + """Catchall Packet for things we don't know about. + + All of the unknown attributes are stored in the unknown_fields + """ + unknown_fields: CatchAll + _type: str = "UnknownPacket" + from_call: Optional[str] = field(default=None) + to_call: Optional[str] = field(default=None) + msgNo: str = field(default_factory=_init_msgNo) # noqa: N815 + format: Optional[str] = field(default=None) + raw: Optional[str] = field(default=None) + raw_dict: dict = field(repr=False, default_factory=lambda: {}, compare=False, hash=False) + path: List[str] = field(default_factory=list, compare=False, hash=False) + packet_type: Optional[str] = field(default=None) + via: Optional[str] = field(default=None, compare=False, hash=False) + + @property + def key(self) -> str: + """Build a key for finding this packet in a dict.""" + return f"{self.from_call}:{self.packet_type}:{self.to_call}" + + @property + def human_info(self) -> str: + return str(self.unknown_fields) + TYPE_LOOKUP: dict[str, type[Packet]] = { + PACKET_TYPE_BULLETIN: BulletinPacket, PACKET_TYPE_WX: WeatherPacket, PACKET_TYPE_WEATHER: WeatherPacket, PACKET_TYPE_MESSAGE: MessagePacket, @@ -676,25 +781,11 @@ TYPE_LOOKUP: dict[str, type[Packet]] = { PACKET_TYPE_OBJECT: ObjectPacket, PACKET_TYPE_STATUS: StatusPacket, PACKET_TYPE_BEACON: BeaconPacket, - PACKET_TYPE_UNKNOWN: Packet, + PACKET_TYPE_UNKNOWN: UnknownPacket, PACKET_TYPE_THIRDPARTY: ThirdPartyPacket, PACKET_TYPE_TELEMETRY: TelemetryPacket, } -OBJ_LOOKUP: dict[str, type[Packet]] = { - "MessagePacket": MessagePacket, - "AckPacket": AckPacket, - "RejectPacket": RejectPacket, - "MicEPacket": MicEPacket, - "ObjectPacket": ObjectPacket, - "StatusPacket": StatusPacket, - "GPSPacket": GPSPacket, - "BeaconPacket": BeaconPacket, - "WeatherPacket": WeatherPacket, - "ThirdPartyPacket": ThirdPartyPacket, - "TelemetryPacket": TelemetryPacket, -} - def get_packet_type(packet: dict) -> str: """Decode the packet type from the packet.""" @@ -714,6 +805,8 @@ def get_packet_type(packet: dict) -> str: packet_type = PACKET_TYPE_OBJECT elif pkt_format == "status": packet_type = PACKET_TYPE_STATUS + elif pkt_format == PACKET_TYPE_BULLETIN: + packet_type = PACKET_TYPE_BULLETIN elif pkt_format == PACKET_TYPE_BEACON: packet_type = PACKET_TYPE_BEACON elif pkt_format == PACKET_TYPE_TELEMETRY: @@ -729,6 +822,8 @@ def get_packet_type(packet: dict) -> str: if packet_type == PACKET_TYPE_UNKNOWN: if "latitude" in packet: packet_type = PACKET_TYPE_BEACON + else: + packet_type = PACKET_TYPE_UNKNOWN return packet_type @@ -769,11 +864,17 @@ def factory(raw_packet: dict[Any, Any]) -> type[Packet]: if "latitude" in raw: packet_class = GPSPacket else: - LOG.error(f"Unknown packet type {packet_type}") - LOG.error(raw) - raise Exception(f"Unknown packet type {packet_type} {raw}") + LOG.warning(f"Unknown packet type {packet_type}") + LOG.warning(raw) + packet_class = UnknownPacket - # LOG.info(f"factory({packet_type}):({raw.get('from_call')}) {packet_class.__class__} {raw}") - # LOG.error(packet_class) + raw.get("addresse", raw.get("to_call")) + + # TODO: Find a global way to enable/disable this + # LOGU.opt(colors=True).info( + # f"factory({packet_type: <8}):" + # f"({packet_class.__name__: <13}): " + # f"{raw.get('from_call'): <9} -> {to: <9}") + #LOG.info(raw.get('msgNo')) return packet_class().from_dict(raw) # type: ignore diff --git a/aprsd/plugin.py b/aprsd/plugin.py index c760d56..f73e0c6 100644 --- a/aprsd/plugin.py +++ b/aprsd/plugin.py @@ -208,7 +208,7 @@ class APRSDRegexCommandPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): @hookimpl def filter(self, packet: packets.MessagePacket) -> str | packets.MessagePacket: - LOG.info(f"{self.__class__.__name__} called") + LOG.debug(f"{self.__class__.__name__} called") if not self.enabled: result = f"{self.__class__.__name__} isn't enabled" LOG.warning(result) diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index 3353523..5613368 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -7,6 +7,7 @@ import aprslib from oslo_config import cfg from aprsd import client, packets, plugin +from aprsd.packets import log as packet_log from aprsd.threads import APRSDThread, tx @@ -80,7 +81,7 @@ class APRSDDupeRXThread(APRSDRXThread): """ packet = self._client.decode_packet(*args, **kwargs) # LOG.debug(raw) - packet.log(header="RX") + packet_log.log(packet) if isinstance(packet, packets.AckPacket): # We don't need to drop AckPackets, those should be @@ -142,14 +143,14 @@ class APRSDProcessPacketThread(APRSDThread): def process_ack_packet(self, packet): """We got an ack for a message, no need to resend it.""" ack_num = packet.msgNo - LOG.info(f"Got ack for message {ack_num}") + LOG.debug(f"Got ack for message {ack_num}") pkt_tracker = packets.PacketTrack() pkt_tracker.remove(ack_num) def process_reject_packet(self, packet): """We got a reject message for a packet. Stop sending the message.""" ack_num = packet.msgNo - LOG.info(f"Got REJECT for message {ack_num}") + LOG.debug(f"Got REJECT for message {ack_num}") pkt_tracker = packets.PacketTrack() pkt_tracker.remove(ack_num) diff --git a/aprsd/threads/tx.py b/aprsd/threads/tx.py index bfd95f9..c84b4bf 100644 --- a/aprsd/threads/tx.py +++ b/aprsd/threads/tx.py @@ -10,7 +10,9 @@ from rush.stores import dictionary from aprsd import client from aprsd import conf # noqa from aprsd import threads as aprsd_threads -from aprsd.packets import core, tracker +from aprsd.packets import core +from aprsd.packets import log as packet_log +from aprsd.packets import tracker CONF = cfg.CONF @@ -74,7 +76,7 @@ def _send_direct(packet, aprs_client=None): cl = client.factory.create() packet.update_timestamp() - packet.log(header="TX") + packet_log.log(packet, tx=True) cl.send(packet) @@ -163,7 +165,7 @@ class SendAckThread(aprsd_threads.APRSDThread): if self.packet.send_count == self.packet.retry_count: # we reached the send limit, don't send again # TODO(hemna) - Need to put this in a delayed queue? - LOG.info( + LOG.debug( f"{self.packet.__class__.__name__}" f"({self.packet.msgNo}) " "Send Complete. Max attempts reached" From b461231c008127ede1e4fcbc24067bfd283bef4c Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 23 Mar 2024 10:24:02 -0400 Subject: [PATCH 17/26] Fix some pep8 issues --- aprsd/cmds/listen.py | 3 ++- aprsd/log/log.py | 2 +- aprsd/packets/core.py | 6 ++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/aprsd/cmds/listen.py b/aprsd/cmds/listen.py index eaa69f9..93564ac 100644 --- a/aprsd/cmds/listen.py +++ b/aprsd/cmds/listen.py @@ -190,7 +190,7 @@ def listen( aprs_client.set_filter(filter) keepalive = threads.KeepAliveThread() - #keepalive.start() + # keepalive.start() if CONF.rpc_settings.enabled: rpc = rpc_server.APRSDRPCThread() @@ -215,6 +215,7 @@ def listen( ) LOG.debug("Start APRSDListenThread") listen_thread.start() + keepalive.start() LOG.debug("keepalive Join") keepalive.join() diff --git a/aprsd/log/log.py b/aprsd/log/log.py index c64fbb4..149c84b 100644 --- a/aprsd/log/log.py +++ b/aprsd/log/log.py @@ -10,7 +10,7 @@ from aprsd.conf import log as conf_log CONF = cfg.CONF -#LOG = logging.getLogger("APRSD") +# LOG = logging.getLogger("APRSD") LOG = logger logging_queue = queue.Queue() diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 3a7e0d9..336e504 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -146,7 +146,6 @@ class Packet: msg = self._filter_for_send().rstrip("\n") return msg - def prepare(self) -> None: """Do stuff here that is needed prior to sending over the air.""" # now build the raw message for sending @@ -292,7 +291,6 @@ class StatusPacket(Packet): return self.status - @dataclass_json @dataclass(unsafe_hash=True) class GPSPacket(Packet): @@ -427,7 +425,6 @@ class GPSPacket(Packet): @property def human_info(self) -> str: - #LOG.warning(self.to_dict()) h_str = [] h_str.append(f"Lat:{self.latitude:03.3f}") h_str.append(f"Lon:{self.longitude:03.3f}") @@ -494,6 +491,7 @@ class MicEPacket(GPSPacket): h_info = super().human_info return f"{h_info} {self.mbits} mbits" + @dataclass_json @dataclass class TelemetryPacket(GPSPacket): @@ -875,6 +873,6 @@ def factory(raw_packet: dict[Any, Any]) -> type[Packet]: # f"factory({packet_type: <8}):" # f"({packet_class.__name__: <13}): " # f"{raw.get('from_call'): <9} -> {to: <9}") - #LOG.info(raw.get('msgNo')) + # LOG.info(raw.get('msgNo')) return packet_class().from_dict(raw) # type: ignore From 9e26df26d6da1298cc3dfaead7579cb19854448c Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 23 Mar 2024 10:46:17 -0400 Subject: [PATCH 18/26] Added new webchat config option for logging This patch adds a new config option for the webchat command to disable url request logging. --- aprsd/cmds/webchat.py | 3 --- aprsd/conf/common.py | 5 +++++ aprsd/log/log.py | 24 +++++++++++++++++++++--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index 18a2385..a059e8f 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -22,7 +22,6 @@ import aprsd from aprsd import ( cli_helper, client, packets, plugin_utils, stats, threads, utils, ) -from aprsd.log import log from aprsd.main import cli from aprsd.threads import aprsd as aprsd_threads from aprsd.threads import rx, tx @@ -571,8 +570,6 @@ class SendMessageNamespace(Namespace): def init_flask(loglevel, quiet): global socketio, flask_app - log.setup_logging(loglevel, quiet) - socketio = SocketIO( flask_app, logger=False, engineio_logger=False, async_mode="threading", diff --git a/aprsd/conf/common.py b/aprsd/conf/common.py index dd2794a..dc27776 100644 --- a/aprsd/conf/common.py +++ b/aprsd/conf/common.py @@ -225,6 +225,11 @@ webchat_opts = [ default=None, help="Longitude for the GPS Beacon button. If not set, the button will not be enabled.", ), + cfg.BoolOpt( + "disable_url_request_logging", + default=False, + help="Disable the logging of url requests in the webchat command.", + ), ] registry_opts = [ diff --git a/aprsd/log/log.py b/aprsd/log/log.py index 149c84b..692b4ec 100644 --- a/aprsd/log/log.py +++ b/aprsd/log/log.py @@ -36,6 +36,7 @@ class InterceptHandler(logging.Handler): # to disable log to stdout, but still log to file # use the --quiet option on the cmdln def setup_logging(loglevel=None, quiet=False): + print(f"setup_logging: loglevel={loglevel}, quiet={quiet}") if not loglevel: log_level = CONF.logging.log_level else: @@ -54,9 +55,13 @@ def setup_logging(loglevel=None, quiet=False): "aprslib.parsing", "aprslib.exceptions", ] + webserver_list = [ + "werkzeug", + "werkzeug._internal", + ] # We don't really want to see the aprslib parsing debug output. - disable_list = imap_list + aprslib_list + disable_list = imap_list + aprslib_list + webserver_list # remove every other logger's handlers # and propagate to root logger @@ -67,9 +72,17 @@ def setup_logging(loglevel=None, quiet=False): else: logging.getLogger(name).propagate = True + if CONF.webchat.disable_url_request_logging: + for name in webserver_list: + logging.getLogger(name).handlers = [] + logging.getLogger(name).propagate = True + logging.getLogger(name).setLevel(logging.ERROR) + + handlers = [ { - "sink": sys.stdout, "serialize": False, + "sink": sys.stdout, + "serialize": False, "format": CONF.logging.logformat, "colorize": True, "level": log_level, @@ -78,8 +91,11 @@ def setup_logging(loglevel=None, quiet=False): if CONF.logging.logfile: handlers.append( { - "sink": CONF.logging.logfile, "serialize": False, + "sink": CONF.logging.logfile, + "serialize": False, "format": CONF.logging.logformat, + "colorize": False, + "level": log_level, }, ) @@ -93,6 +109,8 @@ def setup_logging(loglevel=None, quiet=False): { "sink": qh, "serialize": False, "format": CONF.logging.logformat, + "level": log_level, + "colorize": False, }, ) From da7b7124d7982e8d24a8e7b22bd8276f679dffe4 Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 23 Mar 2024 10:54:10 -0400 Subject: [PATCH 19/26] Some packet cleanup --- aprsd/log/log.py | 1 - aprsd/packets/core.py | 26 +++++++------------------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/aprsd/log/log.py b/aprsd/log/log.py index 692b4ec..26c4f4e 100644 --- a/aprsd/log/log.py +++ b/aprsd/log/log.py @@ -78,7 +78,6 @@ def setup_logging(loglevel=None, quiet=False): logging.getLogger(name).propagate = True logging.getLogger(name).setLevel(logging.ERROR) - handlers = [ { "sink": sys.stdout, diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 336e504..06187c7 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -108,18 +108,6 @@ class Packet: path: List[str] = field(default_factory=list, compare=False, hash=False) via: Optional[str] = field(default=None, compare=False, hash=False) - @property - def json(self) -> str: - """get the json formated string""" - # comes from the DataClassJsonMixin - return self.to_json() # type: ignore - - @property - def dict(self) -> dict: - """get the dict formated string""" - # comes from the DataClassJsonMixin - return self.to_dict() # type: ignore - def get(self, key: str, default: Optional[str] = None): """Emulate a getter on a dict.""" if hasattr(self, key): @@ -474,7 +462,7 @@ class BeaconPacket(GPSPacket): @dataclass_json -@dataclass +@dataclass(unsafe_hash=True) class MicEPacket(GPSPacket): _type: str = field(default="MicEPacket", hash=False) messagecapable: bool = False @@ -493,7 +481,7 @@ class MicEPacket(GPSPacket): @dataclass_json -@dataclass +@dataclass(unsafe_hash=True) class TelemetryPacket(GPSPacket): _type: str = field(default="TelemetryPacket", hash=False) messagecapable: bool = False @@ -514,7 +502,7 @@ class TelemetryPacket(GPSPacket): @dataclass_json -@dataclass +@dataclass(unsafe_hash=True) class ObjectPacket(GPSPacket): _type: str = field(default="ObjectPacket", hash=False) alive: bool = True @@ -560,7 +548,7 @@ class ObjectPacket(GPSPacket): return f"{h_info} {self.comment}" -@dataclass() +@dataclass(unsafe_hash=True) class WeatherPacket(GPSPacket, DataClassJsonMixin): _type: str = field(default="WeatherPacket", hash=False) symbol: str = "_" @@ -697,7 +685,7 @@ class WeatherPacket(GPSPacket, DataClassJsonMixin): ) -@dataclass() +@dataclass(unsafe_hash=True) class ThirdPartyPacket(Packet, DataClassJsonMixin): _type: str = "ThirdPartyPacket" # Holds the encapsulated packet @@ -727,7 +715,7 @@ class ThirdPartyPacket(Packet, DataClassJsonMixin): @dataclass_json -@dataclass +@dataclass(unsafe_hash=True) class BulletinPacket(Packet): _type: str = "BulletinPacket" # Holds the encapsulated packet @@ -740,7 +728,7 @@ class BulletinPacket(Packet): @dataclass_json(undefined=Undefined.INCLUDE) -@dataclass +@dataclass(unsafe_hash=True) class UnknownPacket: """Catchall Packet for things we don't know about. From c581dc5020b140955e005cea41eb094f85c1d250 Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 23 Mar 2024 11:47:45 -0400 Subject: [PATCH 20/26] Added new config option log_packet_format This new DEFAULT group option specifies what format to use when logging a packet. --- aprsd/conf/common.py | 8 +++ aprsd/packets/log.py | 125 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 aprsd/packets/log.py diff --git a/aprsd/conf/common.py b/aprsd/conf/common.py index dc27776..877d49a 100644 --- a/aprsd/conf/common.py +++ b/aprsd/conf/common.py @@ -101,6 +101,14 @@ aprsd_opts = [ default=None, help="Longitude for the GPS Beacon button. If not set, the button will not be enabled.", ), + cfg.StrOpt( + "log_packet_format", + choices=["compact", "multiline", "both"], + default="compact", + help="When logging packets 'compact' will use a single line formatted for each packet." + "'multiline' will use multiple lines for each packet and is the traditional format." + "both will log both compact and multiline.", + ), ] watch_list_opts = [ diff --git a/aprsd/packets/log.py b/aprsd/packets/log.py new file mode 100644 index 0000000..b46e957 --- /dev/null +++ b/aprsd/packets/log.py @@ -0,0 +1,125 @@ +import logging +from typing import Optional + +from loguru import logger +from oslo_config import cfg + +from aprsd.packets.core import AckPacket, RejectPacket + + +LOG = logging.getLogger() +LOGU = logger +CONF = cfg.CONF + +FROM_COLOR = "fg #C70039" +TO_COLOR = "fg #D033FF" +TX_COLOR = "red" +RX_COLOR = "green" +PACKET_COLOR = "cyan" + + +def log_multiline(packet, tx: Optional[bool] = False, header: Optional[bool] = True) -> None: + """LOG a packet to the logfile.""" + if CONF.log_packet_format == "compact": + return + # asdict(packet) + logit = ["\n"] + name = packet.__class__.__name__ + if header: + if tx: + header_str = f"<{TX_COLOR}>TX" + logit.append( + f"{header_str}________(<{PACKET_COLOR}>{name} " + f"TX:{packet.send_count + 1} of {packet.retry_count})", + ) + else: + header_str = f"<{RX_COLOR}>RX" + logit.append( + f"{header_str}________(<{PACKET_COLOR}>{name})", + ) + + else: + header_str = "" + logit.append(f"__________(<{PACKET_COLOR}>{name})") + # log_list.append(f" Packet : {packet.__class__.__name__}") + if packet.msgNo: + logit.append(f" Msg # : {packet.msgNo}") + if packet.from_call: + logit.append(f" From : <{FROM_COLOR}>{packet.from_call}") + if packet.to_call: + logit.append(f" To : <{TO_COLOR}>{packet.to_call}") + if hasattr(packet, "path") and packet.path: + logit.append(f" Path : {'=>'.join(packet.path)}") + if hasattr(packet, "via") and packet.via: + logit.append(f" VIA : {packet.via}") + + if not isinstance(packet, AckPacket) and not isinstance(packet, RejectPacket): + msg = packet.human_info + + if msg: + msg = msg.replace("<", "\\<") + logit.append(f" Info : {msg}") + + raw = packet.raw.replace("<", "\\<") + logit.append(f" Raw : {raw}") + logit.append(f"{header_str}________(<{PACKET_COLOR}>{name})") + + LOGU.opt(colors=True).info("\n".join(logit)) + LOG.debug(repr(packet)) + + +def log(packet, tx: Optional[bool] = False, header: Optional[bool] = True) -> None: + if CONF.log_packet_format == "multiline": + log_multiline(packet, tx, header) + return + + logit = [] + name = packet.__class__.__name__ + + if header: + if tx: + via_color = "red" + arrow = f"<{via_color}>->" + logit.append( + f"TX {arrow} " + f"{name}" + f":{packet.msgNo}" + f" ({packet.send_count + 1} of {packet.retry_count})", + ) + else: + via_color = "fg #828282" + arrow = f"<{via_color}>->" + left_arrow = f"<{via_color}><-" + logit.append( + f"RX {left_arrow} " + f"{name}" + f":{packet.msgNo}", + ) + else: + via_color = "green" + arrow = f"<{via_color}>->" + logit.append( + f"{name}" + f":{packet.msgNo}", + ) + + tmp = None + if packet.path: + tmp = f"{arrow}".join(packet.path) + f"{arrow} " + + logit.append( + f"<{FROM_COLOR}>{packet.from_call} {arrow}" + f"{tmp if tmp else ' '}" + f"<{TO_COLOR}>{packet.to_call}", + ) + + if not isinstance(packet, AckPacket) and not isinstance(packet, RejectPacket): + logit.append(":") + msg = packet.human_info + + if msg: + msg = msg.replace("<", "\\<") + logit.append(f"{msg}") + + LOGU.opt(colors=True).info(" ".join(logit)) + log_multiline(packet, tx, header) From f4356e4a2075c2d2d5342024848ef739840f5309 Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 23 Mar 2024 13:00:51 -0400 Subject: [PATCH 21/26] Show comment in multiline packet output This patch adds the comment for a packet if it exists in the multiline log output --- aprsd/packets/log.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aprsd/packets/log.py b/aprsd/packets/log.py index b46e957..74ef36c 100644 --- a/aprsd/packets/log.py +++ b/aprsd/packets/log.py @@ -60,6 +60,10 @@ def log_multiline(packet, tx: Optional[bool] = False, header: Optional[bool] = T msg = msg.replace("<", "\\<") logit.append(f" Info : {msg}") + if hasattr(packet, "comment") and packet.comment: + logit.append(f" Comment : {packet.comment}") + + raw = packet.raw.replace("<", "\\<") logit.append(f" Raw : {raw}") logit.append(f"{header_str}________(<{PACKET_COLOR}>{name})") From f53df2498890c252fdc06e1973e060f1e7d1957a Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 23 Mar 2024 16:59:33 -0400 Subject: [PATCH 22/26] More packet cleanup and tests --- aprsd/cmds/webchat.py | 9 +- aprsd/packets/__init__.py | 6 +- aprsd/packets/core.py | 207 +++++++++++--------------------------- aprsd/packets/log.py | 1 - tests/test_packets.py | 120 +++++++++++++++++++++- 5 files changed, 185 insertions(+), 158 deletions(-) diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index a059e8f..0a3f68e 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -7,7 +7,6 @@ import sys import threading import time -from aprslib import util as aprslib_util import click import flask from flask import request @@ -539,10 +538,10 @@ class SendMessageNamespace(Namespace): def on_gps(self, data): LOG.debug(f"WS on_GPS: {data}") - lat = aprslib_util.latitude_to_ddm(data["latitude"]) - long = aprslib_util.longitude_to_ddm(data["longitude"]) - LOG.debug(f"Lat DDM {lat}") - LOG.debug(f"Long DDM {long}") + lat = data["latitude"] + long = data["longitude"] + LOG.debug(f"Lat {lat}") + LOG.debug(f"Long {long}") tx.send( packets.GPSPacket( diff --git a/aprsd/packets/__init__.py b/aprsd/packets/__init__.py index 1335637..c83d002 100644 --- a/aprsd/packets/__init__.py +++ b/aprsd/packets/__init__.py @@ -1,7 +1,7 @@ from aprsd.packets.core import ( # noqa: F401 - AckPacket, BeaconPacket, GPSPacket, MessagePacket, MicEPacket, ObjectPacket, - Packet, RejectPacket, StatusPacket, ThirdPartyPacket, UnknownPacket, - WeatherPacket, factory, + AckPacket, BeaconPacket, BulletinPacket, GPSPacket, MessagePacket, + MicEPacket, ObjectPacket, Packet, RejectPacket, StatusPacket, + ThirdPartyPacket, UnknownPacket, WeatherPacket, factory, ) from aprsd.packets.packet_list import PacketList # noqa: F401 from aprsd.packets.seen_list import SeenList # noqa: F401 diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 06187c7..9372000 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -6,6 +6,7 @@ import time # Due to a failure in python 3.8 from typing import Any, List, Optional, Type, TypeVar, Union +from aprslib import util as aprslib_util from dataclasses_json import ( CatchAll, DataClassJsonMixin, Undefined, dataclass_json, ) @@ -131,7 +132,7 @@ class Packet: the human readable payload. """ self.prepare() - msg = self._filter_for_send().rstrip("\n") + msg = self._filter_for_send(self.raw).rstrip("\n") return msg def prepare(self) -> None: @@ -146,10 +147,10 @@ class Packet: """The payload is the non headers portion of the packet.""" if not self.to_call: raise ValueError("to_call isn't set. Must set to_call before calling prepare()") - msg = self._filter_for_send().rstrip("\n") + + # The base packet class has no real payload self.payload = ( f":{self.to_call.ljust(9)}" - f":{msg}" ) def _build_raw(self) -> None: @@ -159,16 +160,16 @@ class Packet: self.payload, ) - def _filter_for_send(self) -> str: + def _filter_for_send(self, msg) -> str: """Filter and format message string for FCC.""" # max? ftm400 displays 64, raw msg shows 74 # and ftm400-send is max 64. setting this to # 67 displays 64 on the ftm400. (+3 {01 suffix) # feature req: break long ones into two msgs - if not self.raw: + if not msg: raise ValueError("No message text to send. call prepare() first.") - message = self.raw[:67] + message = msg[:67] # We all miss George Carlin return re.sub("fuck|shit|cunt|piss|cock|bitch", "****", message) @@ -193,14 +194,28 @@ class Packet: @dataclass(unsafe_hash=True) class AckPacket(Packet): _type: str = field(default="AckPacket", hash=False) - response: Optional[str] = field(default=None) - - def __post__init__(self): - if self.response: - LOG.warning("Response set!") def _build_payload(self): - self.payload = f":{self.to_call.ljust(9)}:ack{self.msgNo}" + self.payload = f":{self.to_call: <9}:ack{self.msgNo}" + + +@dataclass_json +@dataclass(unsafe_hash=True) +class BulletinPacket(Packet): + _type: str = "BulletinPacket" + # Holds the encapsulated packet + bid: Optional[str] = field(default="1") + message_text: Optional[str] = field(default=None) + + @property + def human_info(self) -> str: + return f"BLN{self.bid} {self.message_text}" + + def _build_payload(self) -> None: + self.payload = ( + f":BLN{self.bid:<9}" + f":{self.message_text}" + ) @dataclass_json @@ -214,7 +229,7 @@ class RejectPacket(Packet): LOG.warning("Response set!") def _build_payload(self): - self.payload = f":{self.to_call.ljust(9)} :rej{self.msgNo}" + self.payload = f":{self.to_call: <9}:rej{self.msgNo}" @dataclass_json @@ -223,23 +238,10 @@ class MessagePacket(Packet): _type: str = field(default="MessagePacket", hash=False) message_text: Optional[str] = field(default=None) - def _filter_for_send(self) -> str: - """Filter and format message string for FCC.""" - # max? ftm400 displays 64, raw msg shows 74 - # and ftm400-send is max 64. setting this to - # 67 displays 64 on the ftm400. (+3 {01 suffix) - # feature req: break long ones into two msgs - if not self.message_text: - raise ValueError("No message text to send. Populate message_text field.") - - message = self.message_text[:67] - # We all miss George Carlin - return re.sub("fuck|shit|cunt|piss|cock|bitch", "****", message) - def _build_payload(self): self.payload = ":{}:{}{{{}".format( self.to_call.ljust(9), - self._filter_for_send().rstrip("\n"), + self._filter_for_send(self.message_text).rstrip("\n"), str(self.msgNo), ) @@ -253,23 +255,10 @@ class StatusPacket(Packet): comment: Optional[str] = field(default=None) raw_timestamp: Optional[str] = field(default=None) - def _filter_for_send(self) -> str: - """Filter and format message string for FCC.""" - # max? ftm400 displays 64, raw msg shows 74 - # and ftm400-send is max 64. setting this to - # 67 displays 64 on the ftm400. (+3 {01 suffix) - # feature req: break long ones into two msgs - if not self.status and not self.comment: - self.status = "None" - - message = self.status[:67] - # We all miss George Carlin - return re.sub("fuck|shit|cunt|piss|cock|bitch", "****", message) - def _build_payload(self): self.payload = ":{}:{}{{{}".format( self.to_call.ljust(9), - self._filter_for_send().rstrip("\n"), + self._filter_for_send(self.status).rstrip("\n"), str(self.msgNo), ) @@ -308,102 +297,29 @@ class GPSPacket(Packet): # http://www.aprs.org/datum.txt daodatumbyte: Optional[str] = field(default=None) - def decdeg2dms(self, degrees_decimal): - is_positive = degrees_decimal >= 0 - degrees_decimal = abs(degrees_decimal) - minutes, seconds = divmod(degrees_decimal * 3600, 60) - degrees, minutes = divmod(minutes, 60) - degrees = degrees if is_positive else -degrees - - degrees = str(int(degrees)).replace("-", "0") - minutes = str(int(minutes)).replace("-", "0") - seconds = str(int(round(seconds * 0.01, 2) * 100)) - - return {"degrees": degrees, "minutes": minutes, "seconds": seconds} - - def decdeg2dmm_m(self, degrees_decimal): - is_positive = degrees_decimal >= 0 - degrees_decimal = abs(degrees_decimal) - minutes, seconds = divmod(degrees_decimal * 3600, 60) - degrees, minutes = divmod(minutes, 60) - degrees = degrees if is_positive else -degrees - - degrees = abs(int(degrees)) - minutes = int(round(minutes + (seconds / 60), 2)) - hundredths = round(seconds / 60, 2) - - return { - "degrees": degrees, "minutes": minutes, "seconds": seconds, - "hundredths": hundredths, - } - - def convert_latitude(self, degrees_decimal): - det = self.decdeg2dmm_m(degrees_decimal) - if degrees_decimal > 0: - direction = "N" - else: - direction = "S" - - degrees = str(det.get("degrees")).zfill(2) - minutes = str(det.get("minutes")).zfill(2) - seconds = det.get("seconds") - hun = det.get("hundredths") - hundredths = f"{hun:.2f}".split(".")[1] - - LOG.debug( - f"LAT degress {degrees} minutes {str(minutes)} " - f"seconds {seconds} hundredths {hundredths} direction {direction}", - ) - - lat = f"{degrees}{str(minutes)}.{hundredths}{direction}" - return lat - - def convert_longitude(self, degrees_decimal): - det = self.decdeg2dmm_m(degrees_decimal) - if degrees_decimal > 0: - direction = "E" - else: - direction = "W" - - degrees = str(det.get("degrees")).zfill(3) - minutes = str(det.get("minutes")).zfill(2) - seconds = det.get("seconds") - hun = det.get("hundredths") - hundredths = f"{hun:.2f}".split(".")[1] - - LOG.debug( - f"LON degress {degrees} minutes {str(minutes)} " - f"seconds {seconds} hundredths {hundredths} direction {direction}", - ) - - lon = f"{degrees}{str(minutes)}.{hundredths}{direction}" - return lon - def _build_time_zulu(self): """Build the timestamp in UTC/zulu.""" if self.timestamp: - local_dt = datetime.fromtimestamp(self.timestamp) - else: - local_dt = datetime.now() - self.timestamp = datetime.timestamp(local_dt) - - utc_offset_timedelta = datetime.utcnow() - local_dt - result_utc_datetime = local_dt + utc_offset_timedelta - time_zulu = result_utc_datetime.strftime("%d%H%M") - return time_zulu + return datetime.utcfromtimestamp(self.timestamp).strftime("%d%H%M") def _build_payload(self): """The payload is the non headers portion of the packet.""" time_zulu = self._build_time_zulu() - lat = self.latitude - long = self.longitude - self.payload = ( - f"@{time_zulu}z{lat}{self.symbol_table}" - f"{long}{self.symbol}" - ) + lat = aprslib_util.latitude_to_ddm(self.latitude) + long = aprslib_util.longitude_to_ddm(self.longitude) + payload = [ + "@" if self.timestamp else "!", + time_zulu, + lat, + self.symbol_table, + long, + self.symbol, + ] if self.comment: - self.payload = f"{self.payload}{self.comment}" + payload.append(self._filter_for_send(self.comment)) + + self.payload = "".join(payload) def _build_raw(self): self.raw = ( @@ -438,14 +354,20 @@ class BeaconPacket(GPSPacket): def _build_payload(self): """The payload is the non headers portion of the packet.""" time_zulu = self._build_time_zulu() - lat = self.convert_latitude(self.latitude) - long = self.convert_longitude(self.longitude) + lat = aprslib_util.latitude_to_ddm(self.latitude) + lon = aprslib_util.longitude_to_ddm(self.longitude) self.payload = ( f"@{time_zulu}z{lat}{self.symbol_table}" - f"{long}{self.symbol}APRSD Beacon" + f"{lon}" ) + if self.comment: + comment = self._filter_for_send(self.comment) + self.payload = f"{self.payload}{self.symbol}{comment}" + else: + self.payload = f"{self.payload}{self.symbol}APRSD Beacon" + def _build_raw(self): self.raw = ( f"{self.from_call}>APZ100:" @@ -515,8 +437,8 @@ class ObjectPacket(GPSPacket): def _build_payload(self): time_zulu = self._build_time_zulu() - lat = self.convert_latitude(self.latitude) - long = self.convert_longitude(self.longitude) + lat = aprslib_util.latitude_to_ddm(self.latitude) + long = aprslib_util.longitude_to_ddm(self.longitude) self.payload = ( f"*{time_zulu}z{lat}{self.symbol_table}" @@ -524,7 +446,8 @@ class ObjectPacket(GPSPacket): ) if self.comment: - self.payload = f"{self.payload}{self.comment}" + comment = self._filter_for_send(self.comment) + self.payload = f"{self.payload}{comment}" def _build_raw(self): """ @@ -674,7 +597,8 @@ class WeatherPacket(GPSPacket, DataClassJsonMixin): f"b{self.pressure:05.0f}", ] if self.comment: - contents.append(self.comment) + comment = self.filter_for_send(self.comment) + contents.append(comment) self.payload = "".join(contents) def _build_raw(self): @@ -714,19 +638,6 @@ class ThirdPartyPacket(Packet, DataClassJsonMixin): return f"{self.from_call}->{self.to_call} {sub_info}" -@dataclass_json -@dataclass(unsafe_hash=True) -class BulletinPacket(Packet): - _type: str = "BulletinPacket" - # Holds the encapsulated packet - bid: Optional[str] = field(default="1") - message_text: Optional[str] = field(default=None) - - @property - def human_info(self) -> str: - return f"BLN{self.bid} {self.message_text}" - - @dataclass_json(undefined=Undefined.INCLUDE) @dataclass(unsafe_hash=True) class UnknownPacket: diff --git a/aprsd/packets/log.py b/aprsd/packets/log.py index 74ef36c..ff4704a 100644 --- a/aprsd/packets/log.py +++ b/aprsd/packets/log.py @@ -63,7 +63,6 @@ def log_multiline(packet, tx: Optional[bool] = False, header: Optional[bool] = T if hasattr(packet, "comment") and packet.comment: logit.append(f" Comment : {packet.comment}") - raw = packet.raw.replace("<", "\\<") logit.append(f" Raw : {raw}") logit.append(f"{header_str}________(<{PACKET_COLOR}>{name})") diff --git a/tests/test_packets.py b/tests/test_packets.py index e8e3679..a9774e0 100644 --- a/tests/test_packets.py +++ b/tests/test_packets.py @@ -2,6 +2,7 @@ import unittest from unittest import mock import aprslib +from aprslib import util as aprslib_util from aprsd import packets from aprsd.packets import core @@ -9,7 +10,7 @@ from aprsd.packets import core from . import fake -class TestPluginBase(unittest.TestCase): +class TestPacketBase(unittest.TestCase): def _fake_dict( self, @@ -165,3 +166,120 @@ class TestPluginBase(unittest.TestCase): packet_dict = aprslib.parse(packet_raw) packet = packets.factory(packet_dict) self.assertIsInstance(packet, packets.MicEPacket) + + def test_ack_format(self): + """Test the ack packet format.""" + ack = packets.AckPacket( + from_call=fake.FAKE_FROM_CALLSIGN, + to_call=fake.FAKE_TO_CALLSIGN, + msgNo=123, + ) + + expected = f"{fake.FAKE_FROM_CALLSIGN}>APZ100::{fake.FAKE_TO_CALLSIGN:<9}:ack123" + self.assertEqual(expected, str(ack)) + + def test_reject_format(self): + """Test the reject packet format.""" + reject = packets.RejectPacket( + from_call=fake.FAKE_FROM_CALLSIGN, + to_call=fake.FAKE_TO_CALLSIGN, + msgNo=123, + ) + + expected = f"{fake.FAKE_FROM_CALLSIGN}>APZ100::{fake.FAKE_TO_CALLSIGN:<9}:rej123" + self.assertEqual(expected, str(reject)) + + def test_beacon_format(self): + """Test the beacon packet format.""" + lat = 28.123456 + lon = -80.123456 + ts = 1711219496.6426 + comment = "My Beacon Comment" + packet = packets.BeaconPacket( + from_call=fake.FAKE_FROM_CALLSIGN, + to_call=fake.FAKE_TO_CALLSIGN, + latitude=lat, + longitude=lon, + timestamp=ts, + symbol=">", + comment=comment, + ) + + expected_lat = aprslib_util.latitude_to_ddm(lat) + expected_lon = aprslib_util.longitude_to_ddm(lon) + expected = f"KFAKE>APZ100:@231844z{expected_lat}/{expected_lon}>{comment}" + self.assertEqual(expected, str(packet)) + + def test_beacon_format_no_comment(self): + """Test the beacon packet format.""" + lat = 28.123456 + lon = -80.123456 + ts = 1711219496.6426 + packet = packets.BeaconPacket( + from_call=fake.FAKE_FROM_CALLSIGN, + to_call=fake.FAKE_TO_CALLSIGN, + latitude=lat, + longitude=lon, + timestamp=ts, + symbol=">", + ) + empty_comment = "APRSD Beacon" + + expected_lat = aprslib_util.latitude_to_ddm(lat) + expected_lon = aprslib_util.longitude_to_ddm(lon) + expected = f"KFAKE>APZ100:@231844z{expected_lat}/{expected_lon}>{empty_comment}" + self.assertEqual(expected, str(packet)) + + def test_bulletin_format(self): + """Test the bulletin packet format.""" + # bulletin id = 0 + bid = 0 + packet = packets.BulletinPacket( + from_call=fake.FAKE_FROM_CALLSIGN, + message_text="My Bulletin Message", + bid=0, + ) + + expected = f"{fake.FAKE_FROM_CALLSIGN}>APZ100::BLN{bid:<9}:{packet.message_text}" + self.assertEqual(expected, str(packet)) + + # bulletin id = 1 + bid = 1 + txt = "((((((( CX2SA - Salto Uruguay ))))))) http://www.cx2sa.org" + packet = packets.BulletinPacket( + from_call=fake.FAKE_FROM_CALLSIGN, + message_text=txt, + bid=1, + ) + + expected = f"{fake.FAKE_FROM_CALLSIGN}>APZ100::BLN{bid:<9}:{txt}" + self.assertEqual(expected, str(packet)) + + def test_message_format(self): + """Test the message packet format.""" + + message = "My Message" + msgno = "ABX" + packet = packets.MessagePacket( + from_call=fake.FAKE_FROM_CALLSIGN, + to_call=fake.FAKE_TO_CALLSIGN, + message_text=message, + msgNo=msgno, + ) + + expected = f"{fake.FAKE_FROM_CALLSIGN}>APZ100::{fake.FAKE_TO_CALLSIGN:<9}:{message}{{{msgno}" + self.assertEqual(expected, str(packet)) + + # test with bad words + # Currently fails with mixed case + message = "My cunt piss fuck text" + exp_msg = "My **** **** **** text" + msgno = "ABX" + packet = packets.MessagePacket( + from_call=fake.FAKE_FROM_CALLSIGN, + to_call=fake.FAKE_TO_CALLSIGN, + message_text=message, + msgNo=msgno, + ) + expected = f"{fake.FAKE_FROM_CALLSIGN}>APZ100::{fake.FAKE_TO_CALLSIGN:<9}:{exp_msg}{{{msgno}" + self.assertEqual(expected, str(packet)) From 873fc066080cb588658e56574100dd5200e1edbc Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 23 Mar 2024 17:53:01 -0400 Subject: [PATCH 23/26] added packet counter random int The packet counter now starts at a random number between 1 and 9999 instead of always at 1. --- aprsd/packets/core.py | 2 +- aprsd/utils/counter.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 9372000..861cabf 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -167,7 +167,7 @@ class Packet: # 67 displays 64 on the ftm400. (+3 {01 suffix) # feature req: break long ones into two msgs if not msg: - raise ValueError("No message text to send. call prepare() first.") + return "" message = msg[:67] # We all miss George Carlin diff --git a/aprsd/utils/counter.py b/aprsd/utils/counter.py index 5f569f4..30b6b75 100644 --- a/aprsd/utils/counter.py +++ b/aprsd/utils/counter.py @@ -1,9 +1,13 @@ from multiprocessing import RawValue +import random import threading import wrapt +MAX_PACKET_ID = 9999 + + class PacketCounter: """ Global Packet id counter class. @@ -17,19 +21,18 @@ class PacketCounter: """ _instance = None - max_count = 9999 lock = threading.Lock() def __new__(cls, *args, **kwargs): """Make this a singleton class.""" if cls._instance is None: cls._instance = super().__new__(cls, *args, **kwargs) - cls._instance.val = RawValue("i", 1) + cls._instance.val = RawValue("i", random.randint(1, MAX_PACKET_ID)) return cls._instance @wrapt.synchronized(lock) def increment(self): - if self.val.value == self.max_count: + if self.val.value == MAX_PACKET_ID: self.val.value = 1 else: self.val.value += 1 From f400c6004e74b71a482429fc56d7595796fb5748 Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 23 Mar 2024 18:02:01 -0400 Subject: [PATCH 24/26] Fix for filtering curse words This patch adds a fix for filtering out curse words. This adds a flag to the regex to ignore case! --- aprsd/packets/core.py | 5 ++++- tests/test_packets.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 861cabf..812f6b8 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -171,7 +171,10 @@ class Packet: message = msg[:67] # We all miss George Carlin - return re.sub("fuck|shit|cunt|piss|cock|bitch", "****", message) + return re.sub( + "fuck|shit|cunt|piss|cock|bitch", "****", + message, flags=re.IGNORECASE, + ) def __str__(self) -> str: """Show the raw version of the packet""" diff --git a/tests/test_packets.py b/tests/test_packets.py index a9774e0..9fe7990 100644 --- a/tests/test_packets.py +++ b/tests/test_packets.py @@ -272,8 +272,8 @@ class TestPacketBase(unittest.TestCase): # test with bad words # Currently fails with mixed case - message = "My cunt piss fuck text" - exp_msg = "My **** **** **** text" + message = "My cunt piss fuck shIt text" + exp_msg = "My **** **** **** **** text" msgno = "ABX" packet = packets.MessagePacket( from_call=fake.FAKE_FROM_CALLSIGN, From c0623596cd2253e4f9bc6e0d20dd524bde2bc948 Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 23 Mar 2024 19:27:23 -0400 Subject: [PATCH 25/26] Change debug log color this patch changes the debug log color from dark blue to grey --- aprsd/log/log.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aprsd/log/log.py b/aprsd/log/log.py index 26c4f4e..fcdb620 100644 --- a/aprsd/log/log.py +++ b/aprsd/log/log.py @@ -115,3 +115,4 @@ def setup_logging(loglevel=None, quiet=False): # configure loguru logger.configure(handlers=handlers) + logger.level("DEBUG", color="") From 0321cb6cf10e7194eddd8c44c3ddce254de6b504 Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 23 Mar 2024 21:06:20 -0400 Subject: [PATCH 26/26] Put packet.json back in --- aprsd/packets/core.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 812f6b8..1665e29 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -109,6 +109,14 @@ class Packet: path: List[str] = field(default_factory=list, compare=False, hash=False) via: Optional[str] = field(default=None, compare=False, hash=False) + @property + def json(self): + """get the json formated string. + + This is used soley by the rpc server to return json over the wire. + """ + return self.to_json() + def get(self, key: str, default: Optional[str] = None): """Emulate a getter on a dict.""" if hasattr(self, key):