Allow spotting upstream to cluster. #39

providers/spot/dxcluster.py

@@ -3,9 +3,12 @@ import re
 from datetime import datetime
 
 import pytz
+import telnetlib3
 
 from core.config import SERVER_OWNER_CALLSIGN
+from core.utils import decode_telnet_bytes
 from data.spot import Spot
+from providers.spot.spot_provider import SpotSubmissionError
 from providers.spot.telnet_spot_provider import TelnetSpotProvider
 
 logger = logging.getLogger(__name__)
@@ -24,6 +27,9 @@ class DXCluster(TelnetSpotProvider):
         re.IGNORECASE,
     )
 
+    # Timeout for each step of the telnet conversation when submitting a spot
+    _SUBMIT_TIMEOUT_SEC = 10
+
     def __init__(self, provider_config):
         """Constructor requires hostname and port"""
 
@@ -58,3 +64,73 @@ class DXCluster(TelnetSpotProvider):
             comment=match.group(4).strip(),
             time=spot_datetime.timestamp(),
         )
+
+    def can_submit_spot(self, activity):
+        # Clusters accept spots for any activity, or for no activity at all.
+        return True
+
+    def submit_spot(self, spot, credentials):
+        """Submit a spot to the cluster. This makes a new telnet connection to the same node that we receive spots from,
+        logs in as the spotter, sends the spot using the DX command, then logs out. If anything goes wrong we raise an
+        error which can then be logged by the caller and returned to the user as an error message in the API JSON / web
+        UI."""
+
+        command = self._build_dx_command(spot)
+        telnet = telnetlib3.Telnet(self._host, self._port, timeout=self._SUBMIT_TIMEOUT_SEC)
+        try:
+            # Wait for login prompt and log in
+            output = telnet.read_until(self._login_prompt.encode("latin-1"), timeout=self._SUBMIT_TIMEOUT_SEC)
+            if self._login_prompt.encode("latin-1") not in output:
+                logger.debug(f"{self.name} response while waiting for login prompt: {decode_telnet_bytes(output)}")
+                raise SpotSubmissionError(f"{self.name} did not give the expected login prompt.")
+            telnet.write(f"{spot.de_call}\n".encode("latin-1"))
+
+            # Wait for the command prompt. This is a line like "M0TRT de W3LPL 27-Sep-2026 0945Z dxspider >". We can't
+            # just wait for ">" as the welcome message might contain that character.
+            call = re.escape(spot.de_call).encode("latin-1")
+            prompt = re.compile(rb"(?im)^" + call + rb" de \S+.*>\s*$")
+            index, _, output = telnet.expect([prompt], timeout=self._SUBMIT_TIMEOUT_SEC)
+            if index < 0:
+                logger.debug(f"{self.name} response while waiting for command prompt: {decode_telnet_bytes(output)}")
+                raise SpotSubmissionError(f"{self.name} did not accept a login as {spot.de_call}.")
+
+            # Send the spot.
+            logger.debug(f"Submitting spot to {self.name} as {spot.de_call}: {command}")
+            telnet.write(f"{command}\n".encode("latin-1"))
+
+            # Log out
+            telnet.write(b"bye\n")
+        finally:
+            telnet.close()
+
+    @staticmethod
+    def _build_dx_command(spot):
+        """Build the "DX <freq kHz> <dx call> <comment>" command used to submit a spot to the cluster. The mode, and
+        any activities and references e.g. "SSB POTA GB-1234 tnx 73"."""
+
+        freq = spot.freq
+        if not freq or not spot.dx_call:
+            raise ValueError("A frequency and DX callsign are required to submit a spot to a cluster.")
+
+        comment_parts = []
+        # Mode
+        if spot.mode:
+            comment_parts.append(spot.mode)
+        # Build list of activities with refs like "POTA GB-0001 SOTA G/SC-001"
+        activities_with_refs = set()
+        for activity_ref in spot.activity_refs:
+            if activity_ref.id:
+                comment_parts.append(f"{activity_ref.activity} {activity_ref.id}")
+                activities_with_refs.add(activity_ref.activity)
+        # If we have activities that don't have refs, add them to the comment
+        comment_parts.extend(str(a) for a in spot.activities if a not in activities_with_refs)
+        # Add the actual spot comment
+        if spot.comment:
+            comment_parts.append(spot.comment)
+
+        # Build command
+        command = f"DX {freq / 1000.0:.1f} {spot.dx_call} {' '.join(comment_parts)}"
+
+        # Strip anything that isn't printable ASCII, so that e.g. a newline in the comment can't be used to send
+        # exploit the system and send additional commands to the cluster as the spotter.
+        return re.sub(r"\s+", " ", re.sub(r"[^\x20-\x7E]", " ", command)).strip()

providers/spot/gma.py

@@ -158,8 +158,9 @@ class GMA(HTTPSpotProvider):
 
         return new_spots
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.GMA
+        return False  # return activity == ActivityName.GMA
 
     def submit_spot(self, spot, credentials):
         # TODO: Implement.

providers/spot/hema.py

@@ -89,8 +89,9 @@ class HEMA(HTTPSpotProvider):
                 logger.warning("Connection error when accessing HEMA spots API.")
         return new_spots
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.HEMA
+        return False  # return activity == ActivityName.HEMA
 
     def submit_spot(self, spot, credentials):
         # TODO: Implement. Currently blocked awaiting their API team to make a change to allow us to spot with a

providers/spot/parksnpeaks.py

@@ -104,8 +104,9 @@ class ParksNPeaks(HTTPSpotProvider):
                 new_spots.append(spot)
         return new_spots
 
+    # Upstream spotting disabled until properly tested
     def can_submit_spot(self, activity):
-        return activity in self.SUBMITTABLE_ACTIVITIES
+        return False  # return activity in self.SUBMITTABLE_ACTIVITIES
 
     def submit_spot(self, spot, credentials):
         # TODO test this works

providers/spot/pota.py

@@ -41,7 +41,7 @@ class POTA(HTTPSpotProvider):
                         name=source_spot["name"],
                         latitude=source_spot["latitude"],
                         longitude=source_spot["longitude"],
-                        ref_type=ActivityRefType.PARK
+                        ref_type=ActivityRefType.PARK,
                     )
                 ],
                 time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S")
@@ -57,8 +57,9 @@ class POTA(HTTPSpotProvider):
             new_spots.append(spot)
         return new_spots
 
+    # Upstream spotting disabled until properly tested
     def can_submit_spot(self, activity):
-        return activity == ActivityName.POTA
+        return False  # return activity == ActivityName.POTA
 
     def submit_spot(self, spot, credentials):
         ref_id = spot.activity_refs[0].id if spot.activity_refs else None

providers/spot/sota.py

@@ -83,8 +83,9 @@ class SOTA(HTTPSpotProvider):
                 logger.warning("Timeout when accessing SOTA spots API.")
         return new_spots
 
+    # Upstream spotting disabled until properly tested
     def can_submit_spot(self, activity):
-        return activity == ActivityName.SOTA
+        return False  # return activity == ActivityName.SOTA
 
     def submit_spot(self, spot, credentials):
         # TODO test this method works

providers/spot/spot_provider.py

@@ -5,6 +5,12 @@ import pytz
 from core.data_store import DATA_STORE
 
 
+class SpotSubmissionError(Exception):
+    """Raised by a provider's submit_spot() when the upstream service refused or didn't accept the spot. The message
+    is returned in the API / showed on the web UI. This is probably a user error, so we log a warning rather than a big
+    stack trace like we would if we got some unexpected exception."""
+
+
 class SpotProvider:
     """Generic spot provider class. Subclasses of this query the individual APIs for data."""
 
@@ -61,7 +67,8 @@ class SpotProvider:
         raise NotImplementedError("Subclasses must implement this method")
 
     def can_submit_spot(self, activity):
-        """Return True if this provider supports submitting spots upstream for the given activity."""
+        """Return True if this provider supports submitting spots upstream for the given activity. Activity can be None,
+        to check whether this provider supports submitting spots that have no activity."""
 
         return False
 

providers/spot/tiles.py

@@ -69,7 +69,7 @@ class Tiles(HTTPSpotProvider):
                         name=source_spot["maidenhead_grid"],
                         latitude=source_spot["latitude"],
                         longitude=source_spot["longitude"],
-                        ref_type=ActivityRefType.GRID
+                        ref_type=ActivityRefType.GRID,
                     )
                 ],
                 time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
@@ -84,8 +84,9 @@ class Tiles(HTTPSpotProvider):
             new_spots.append(spot)
         return new_spots
 
+    # Upstream spotting disabled until properly tested
     def can_submit_spot(self, activity):
-        return activity == ActivityName.TILES
+        return False  # return activity == ActivityName.TILES
 
     def submit_spot(self, spot, credentials):
         # Tiles on the air currently only supports *self* spots

providers/spot/wota.py

@@ -114,8 +114,9 @@ class WOTA(HTTPSpotProvider):
 
         return new_spots
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.WOTA
+        return False  # return activity == ActivityName.WOTA
 
     def submit_spot(self, spot, credentials):
         # TODO Ask M5TEA if he's happy to share how this is done from his app

providers/spot/wwbota.py

@@ -52,8 +52,9 @@ class WWBOTA(SSESpotProvider):
         # WWBOTA does support a special "Test" spot type, we need to avoid adding that.
         return spot if source_spot["type"] != "Test" else None
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.WWBOTA
+        return False  # return activity == ActivityName.WWBOTA
 
     def submit_spot(self, spot, credentials):
         # TODO: Implement. WWBOTA API docs cover this: https://api.wwbota.org/#tag/Spots/operation/create_spot_spots__post

providers/spot/wwff.py

@@ -38,7 +38,7 @@ class WWFF(HTTPSpotProvider):
                         name=source_spot["reference_name"],
                         latitude=source_spot["latitude"],
                         longitude=source_spot["longitude"],
-                        ref_type=ActivityRefType.PARK
+                        ref_type=ActivityRefType.PARK,
                     )
                 ],
                 time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
@@ -51,8 +51,9 @@ class WWFF(HTTPSpotProvider):
             new_spots.append(spot)
         return new_spots
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.WWFF
+        return False  # return activity == ActivityName.WWFF
 
     def submit_spot(self, spot, credentials):
         # TODO: Implement. Spotting to WWFF should be possible, need to look up the Spotline docs or copy approach from

providers/spot/zlota.py

@@ -51,8 +51,9 @@ class ZLOTA(HTTPSpotProvider):
             new_spots.append(spot)
         return new_spots
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.ZLOTA
+        return False  # return activity == ActivityName.ZLOTA
 
     def submit_spot(self, spot, credentials):
         # TODO: Implement. Spotting to ZLOTA is supported via POST, see https://ontheair.nz/api