From e33eecbc4e916ea6d7beb131b6fc002fd4a5a501 Mon Sep 17 00:00:00 2001 From: Ian Renton Date: Sun, 20 Sep 2026 19:08:05 +0100 Subject: [PATCH] Fix mojibake in DX cluster and RBN spot comments Both providers always decoded telnet bytes as Latin-1, which silently mangles nodes that send UTF-8 (Latin-1 decode never raises, so mixed encodings across cluster nodes went unnoticed). Added a shared decode_telnet_bytes() helper that tries UTF-8 first and falls back to Latin-1, since a byte stream that happens to be valid multi-byte UTF-8 is essentially never accidental Latin-1 text. Co-Authored-By: Claude Sonnet 5 --- providers/spot/dxcluster.py | 4 ++-- providers/spot/rbn.py | 4 ++-- providers/spot/spot_provider.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/providers/spot/dxcluster.py b/providers/spot/dxcluster.py index 4413207..95f564e 100644 --- a/providers/spot/dxcluster.py +++ b/providers/spot/dxcluster.py @@ -9,7 +9,7 @@ import telnetlib3 from core.config import SERVER_OWNER_CALLSIGN from data.spot import Spot -from providers.spot.spot_provider import SpotProvider +from providers.spot.spot_provider import SpotProvider, decode_telnet_bytes logger = logging.getLogger(__name__) @@ -96,7 +96,7 @@ class DXCluster(SpotProvider): try: # Check new telnet info against regular expression telnet_output = self._telnet.read_until("\n".encode("latin-1")) - match = self._spot_line_pattern.match(telnet_output.decode("latin-1")) + match = self._spot_line_pattern.match(decode_telnet_bytes(telnet_output)) if match: spot_time = datetime.strptime(match.group(5), "%H%MZ").replace(tzinfo=pytz.UTC) spot_datetime = datetime.combine( diff --git a/providers/spot/rbn.py b/providers/spot/rbn.py index b4f7617..4f022d5 100644 --- a/providers/spot/rbn.py +++ b/providers/spot/rbn.py @@ -9,7 +9,7 @@ import telnetlib3 from core.config import SERVER_OWNER_CALLSIGN from data.spot import Spot -from providers.spot.spot_provider import SpotProvider +from providers.spot.spot_provider import SpotProvider, decode_telnet_bytes logger = logging.getLogger(__name__) @@ -81,7 +81,7 @@ class RBN(SpotProvider): try: # Check new telnet info against regular expression telnet_output = self._telnet.read_until("\n".encode("latin-1")) - match = self._LINE_PATTERN.match(telnet_output.decode("latin-1")) + match = self._LINE_PATTERN.match(decode_telnet_bytes(telnet_output)) if match: spot_time = datetime.strptime(match.group(5), "%H%MZ").replace(tzinfo=pytz.UTC) spot_datetime = datetime.combine( diff --git a/providers/spot/spot_provider.py b/providers/spot/spot_provider.py index 88df0a1..286af27 100644 --- a/providers/spot/spot_provider.py +++ b/providers/spot/spot_provider.py @@ -5,6 +5,18 @@ import pytz from core.data_store import DATA_STORE +def decode_telnet_bytes(data: bytes) -> str: + """Decode a line of text received from a telnet connection. DX cluster and RBN nodes are inconsistent about the + character encoding they use for spot comments: most send UTF-8, but some older ones send Latin-1/CP1252. Try + UTF-8 first, since valid multi-byte UTF-8 sequences are very unlikely to occur by chance in Latin-1 text, then + fall back to Latin-1, which can decode any byte sequence without raising.""" + + try: + return data.decode("utf-8") + except UnicodeDecodeError: + return data.decode("latin-1") + + class SpotProvider: """Generic spot provider class. Subclasses of this query the individual APIs for data."""