Allow spotting upstream to cluster. #39

config-example.yml

@@ -350,9 +350,10 @@ max_alert_age_sec: 604800
 # Allow submitting spots to the Spothole API?
 allow_spotting: true
 
-# Allow upstream submission of spots to external providers (POTA, SOTA, etc.) via the API?
+# Allow upstream submission of spots to external providers via the API? Currently only DX cluster providers support
-# Requires allow_spotting to also be true. Set to false to only accept spots into the local
+# this, and will log in using the spotter's callsign to send the spot. Upstream spotting to POTA, SOTA etc. is a work
-# Spothole database, without forwarding them to any external service.
+# in progress. Requires allow_spotting to also be true. Set to false to only accept spots into the local Spothole
+# database, without forwarding them to any external service.
 allow_upstream_spotting: false
 
 # Google reCAPTCHA v2 keys for CAPTCHA protection on upstream spot submission. Both keys must be set to enable CAPTCHA.

providers/spot/dxcluster.py

@@ -3,9 +3,12 @@ import re
 from datetime import datetime
 
 import pytz
+import telnetlib3
 
 from core.config import SERVER_OWNER_CALLSIGN
+from core.utils import decode_telnet_bytes
 from data.spot import Spot
+from providers.spot.spot_provider import SpotSubmissionError
 from providers.spot.telnet_spot_provider import TelnetSpotProvider
 
 logger = logging.getLogger(__name__)
@@ -24,6 +27,9 @@ class DXCluster(TelnetSpotProvider):
         re.IGNORECASE,
     )
 
+    # Timeout for each step of the telnet conversation when submitting a spot
+    _SUBMIT_TIMEOUT_SEC = 10
+
     def __init__(self, provider_config):
         """Constructor requires hostname and port"""
 
@@ -58,3 +64,73 @@ class DXCluster(TelnetSpotProvider):
             comment=match.group(4).strip(),
             time=spot_datetime.timestamp(),
         )
+
+    def can_submit_spot(self, activity):
+        # Clusters accept spots for any activity, or for no activity at all.
+        return True
+
+    def submit_spot(self, spot, credentials):
+        """Submit a spot to the cluster. This makes a new telnet connection to the same node that we receive spots from,
+        logs in as the spotter, sends the spot using the DX command, then logs out. If anything goes wrong we raise an
+        error which can then be logged by the caller and returned to the user as an error message in the API JSON / web
+        UI."""
+
+        command = self._build_dx_command(spot)
+        telnet = telnetlib3.Telnet(self._host, self._port, timeout=self._SUBMIT_TIMEOUT_SEC)
+        try:
+            # Wait for login prompt and log in
+            output = telnet.read_until(self._login_prompt.encode("latin-1"), timeout=self._SUBMIT_TIMEOUT_SEC)
+            if self._login_prompt.encode("latin-1") not in output:
+                logger.debug(f"{self.name} response while waiting for login prompt: {decode_telnet_bytes(output)}")
+                raise SpotSubmissionError(f"{self.name} did not give the expected login prompt.")
+            telnet.write(f"{spot.de_call}\n".encode("latin-1"))
+
+            # Wait for the command prompt. This is a line like "M0TRT de W3LPL 27-Sep-2026 0945Z dxspider >". We can't
+            # just wait for ">" as the welcome message might contain that character.
+            call = re.escape(spot.de_call).encode("latin-1")
+            prompt = re.compile(rb"(?im)^" + call + rb" de \S+.*>\s*$")
+            index, _, output = telnet.expect([prompt], timeout=self._SUBMIT_TIMEOUT_SEC)
+            if index < 0:
+                logger.debug(f"{self.name} response while waiting for command prompt: {decode_telnet_bytes(output)}")
+                raise SpotSubmissionError(f"{self.name} did not accept a login as {spot.de_call}.")
+
+            # Send the spot.
+            logger.debug(f"Submitting spot to {self.name} as {spot.de_call}: {command}")
+            telnet.write(f"{command}\n".encode("latin-1"))
+
+            # Log out
+            telnet.write(b"bye\n")
+        finally:
+            telnet.close()
+
+    @staticmethod
+    def _build_dx_command(spot):
+        """Build the "DX <freq kHz> <dx call> <comment>" command used to submit a spot to the cluster. The mode, and
+        any activities and references e.g. "SSB POTA GB-1234 tnx 73"."""
+
+        freq = spot.freq
+        if not freq or not spot.dx_call:
+            raise ValueError("A frequency and DX callsign are required to submit a spot to a cluster.")
+
+        comment_parts = []
+        # Mode
+        if spot.mode:
+            comment_parts.append(spot.mode)
+        # Build list of activities with refs like "POTA GB-0001 SOTA G/SC-001"
+        activities_with_refs = set()
+        for activity_ref in spot.activity_refs:
+            if activity_ref.id:
+                comment_parts.append(f"{activity_ref.activity} {activity_ref.id}")
+                activities_with_refs.add(activity_ref.activity)
+        # If we have activities that don't have refs, add them to the comment
+        comment_parts.extend(str(a) for a in spot.activities if a not in activities_with_refs)
+        # Add the actual spot comment
+        if spot.comment:
+            comment_parts.append(spot.comment)
+
+        # Build command
+        command = f"DX {freq / 1000.0:.1f} {spot.dx_call} {' '.join(comment_parts)}"
+
+        # Strip anything that isn't printable ASCII, so that e.g. a newline in the comment can't be used to send
+        # exploit the system and send additional commands to the cluster as the spotter.
+        return re.sub(r"\s+", " ", re.sub(r"[^\x20-\x7E]", " ", command)).strip()

providers/spot/gma.py

@@ -158,8 +158,9 @@ class GMA(HTTPSpotProvider):
 
         return new_spots
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.GMA
+        return False  # return activity == ActivityName.GMA
 
     def submit_spot(self, spot, credentials):
         # TODO: Implement.

providers/spot/hema.py

@@ -89,8 +89,9 @@ class HEMA(HTTPSpotProvider):
                 logger.warning("Connection error when accessing HEMA spots API.")
         return new_spots
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.HEMA
+        return False  # return activity == ActivityName.HEMA
 
     def submit_spot(self, spot, credentials):
         # TODO: Implement. Currently blocked awaiting their API team to make a change to allow us to spot with a

providers/spot/parksnpeaks.py

@@ -104,8 +104,9 @@ class ParksNPeaks(HTTPSpotProvider):
                 new_spots.append(spot)
         return new_spots
 
+    # Upstream spotting disabled until properly tested
     def can_submit_spot(self, activity):
-        return activity in self.SUBMITTABLE_ACTIVITIES
+        return False  # return activity in self.SUBMITTABLE_ACTIVITIES
 
     def submit_spot(self, spot, credentials):
         # TODO test this works

providers/spot/pota.py

@@ -41,7 +41,7 @@ class POTA(HTTPSpotProvider):
                         name=source_spot["name"],
                         latitude=source_spot["latitude"],
                         longitude=source_spot["longitude"],
-                        ref_type=ActivityRefType.PARK
+                        ref_type=ActivityRefType.PARK,
                     )
                 ],
                 time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S")
@@ -57,8 +57,9 @@ class POTA(HTTPSpotProvider):
             new_spots.append(spot)
         return new_spots
 
+    # Upstream spotting disabled until properly tested
     def can_submit_spot(self, activity):
-        return activity == ActivityName.POTA
+        return False  # return activity == ActivityName.POTA
 
     def submit_spot(self, spot, credentials):
         ref_id = spot.activity_refs[0].id if spot.activity_refs else None

providers/spot/sota.py

@@ -83,8 +83,9 @@ class SOTA(HTTPSpotProvider):
                 logger.warning("Timeout when accessing SOTA spots API.")
         return new_spots
 
+    # Upstream spotting disabled until properly tested
     def can_submit_spot(self, activity):
-        return activity == ActivityName.SOTA
+        return False  # return activity == ActivityName.SOTA
 
     def submit_spot(self, spot, credentials):
         # TODO test this method works

providers/spot/spot_provider.py

@@ -5,6 +5,12 @@ import pytz
 from core.data_store import DATA_STORE
 
 
+class SpotSubmissionError(Exception):
+    """Raised by a provider's submit_spot() when the upstream service refused or didn't accept the spot. The message
+    is returned in the API / showed on the web UI. This is probably a user error, so we log a warning rather than a big
+    stack trace like we would if we got some unexpected exception."""
+
+
 class SpotProvider:
     """Generic spot provider class. Subclasses of this query the individual APIs for data."""
 
@@ -61,7 +67,8 @@ class SpotProvider:
         raise NotImplementedError("Subclasses must implement this method")
 
     def can_submit_spot(self, activity):
-        """Return True if this provider supports submitting spots upstream for the given activity."""
+        """Return True if this provider supports submitting spots upstream for the given activity. Activity can be None,
+        to check whether this provider supports submitting spots that have no activity."""
 
         return False
 

providers/spot/tiles.py

@@ -69,7 +69,7 @@ class Tiles(HTTPSpotProvider):
                         name=source_spot["maidenhead_grid"],
                         latitude=source_spot["latitude"],
                         longitude=source_spot["longitude"],
-                        ref_type=ActivityRefType.GRID
+                        ref_type=ActivityRefType.GRID,
                     )
                 ],
                 time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
@@ -84,8 +84,9 @@ class Tiles(HTTPSpotProvider):
             new_spots.append(spot)
         return new_spots
 
+    # Upstream spotting disabled until properly tested
     def can_submit_spot(self, activity):
-        return activity == ActivityName.TILES
+        return False  # return activity == ActivityName.TILES
 
     def submit_spot(self, spot, credentials):
         # Tiles on the air currently only supports *self* spots

providers/spot/wota.py

@@ -114,8 +114,9 @@ class WOTA(HTTPSpotProvider):
 
         return new_spots
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.WOTA
+        return False  # return activity == ActivityName.WOTA
 
     def submit_spot(self, spot, credentials):
         # TODO Ask M5TEA if he's happy to share how this is done from his app

providers/spot/wwbota.py

@@ -52,8 +52,9 @@ class WWBOTA(SSESpotProvider):
         # WWBOTA does support a special "Test" spot type, we need to avoid adding that.
         return spot if source_spot["type"] != "Test" else None
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.WWBOTA
+        return False  # return activity == ActivityName.WWBOTA
 
     def submit_spot(self, spot, credentials):
         # TODO: Implement. WWBOTA API docs cover this: https://api.wwbota.org/#tag/Spots/operation/create_spot_spots__post

providers/spot/wwff.py

@@ -38,7 +38,7 @@ class WWFF(HTTPSpotProvider):
                         name=source_spot["reference_name"],
                         latitude=source_spot["latitude"],
                         longitude=source_spot["longitude"],
-                        ref_type=ActivityRefType.PARK
+                        ref_type=ActivityRefType.PARK,
                     )
                 ],
                 time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
@@ -51,8 +51,9 @@ class WWFF(HTTPSpotProvider):
             new_spots.append(spot)
         return new_spots
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.WWFF
+        return False  # return activity == ActivityName.WWFF
 
     def submit_spot(self, spot, credentials):
         # TODO: Implement. Spotting to WWFF should be possible, need to look up the Spotline docs or copy approach from

providers/spot/zlota.py

@@ -51,8 +51,9 @@ class ZLOTA(HTTPSpotProvider):
             new_spots.append(spot)
         return new_spots
 
+    # Upstream spotting disabled until implemented
     def can_submit_spot(self, activity):
-        return activity == ActivityName.ZLOTA
+        return False  # return activity == ActivityName.ZLOTA
 
     def submit_spot(self, spot, credentials):
         # TODO: Implement. Spotting to ZLOTA is supported via POST, see https://ontheair.nz/api

static/js/add-spot.js

@@ -196,7 +196,7 @@ function addSpot() {
             data: JSON.stringify({spot, handling}),
             contentType: 'application/json',
             type: 'POST',
-            timeout: 10000,
+            timeout: 60000,
             success: async function (result) {
                 // Reset CAPTCHA for next use
                 if (window._recaptchaWidgetId !== undefined) {

telnetserver/telnetserver.py

@@ -115,7 +115,7 @@ class TelnetServer:
                     break
 
                 input_buffer, command = self._consume_input(input_buffer, data)
-                if command == "exit":
+                if command == "exit" or command == "bye":
                     writer.write(b"Goodbye!\r\n")
                     await writer.drain()
                     # Exit the while read loop, this will disconnect the client.

templates/add-spot.html

@@ -3,13 +3,15 @@
 
 <div id="add-spot-intro-box" class="permanently-dismissible-box mt-3">
     <div class="alert alert-primary alert-dismissible fade show" role="alert">
-        <!-- TODO Remove when feature available -->
+        <i class="fa-solid fa-circle-info"></i> <strong>Adding spots to Spothole</strong><br/>
-        <i class="fa-solid fa-circle-info"></i> <strong>Adding spots to Spothole</strong><br/>This page is implemented
+        {% if web_ui_options["allow_upstream_spotting"] %}
-        as a proof of concept for adding spots to the Spothole system. Currently, spots added in this way are only
+        Spots added here are visible within Spothole. You can also choose to send your spot "upstream" to the DX
-        visible within Spothole and are not sent "upstream" to DX clusters or xOTA spotting sites. The functionality
+        cluster, in which case it is posted using the callsign you enter in "Your Call". Sending spots upstream to other
-        might be extended to include this in future if there is demand for it. If you'd like this to be added, please
+        spotting sites such as POTA and SOTA is currently a work in progress.
-        give a thumbs-up on <a href="https://git.ianrenton.com/ian/spothole/issues/39" target="_new" class="alert-link">issue
+        {% else %}
-        #39</a> or get in touch via email.
+        Spots added here are only visible within Spothole. This server does not allow spots to be sent "upstream" to DX
+        clusters or other spotting sites such as POTA and SOTA.
+        {% end %}
         <button type="button" id="add-spot-intro-box-dismiss" class="btn-close" data-bs-dismiss="alert"
                 aria-label="Close"></button>
     </div>
@@ -136,7 +138,7 @@
     const ALLOW_UPSTREAM_SPOTTING = {% raw safe_json_dumps(web_ui_options["allow_upstream_spotting"]) %};
 </script>
 
-<script src="/static/js/add-spot.js?v=1790443101"></script>
+<script src="/static/js/add-spot.js?v=1790503760"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -87,7 +87,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1790443102"></script>
+<script src="/static/js/alerts.js?v=1790503760"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -82,8 +82,8 @@
     const BANDS = {% raw safe_json_dumps(options["bands"]) %};
 </script>
 
-<script src="/static/js/spotsbandsandmap.js?v=1790443101"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1790503760"></script>
-<script src="/static/js/bands.js?v=1790443101"></script>
+<script src="/static/js/bands.js?v=1790503760"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1790443101" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1790503760" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -16,10 +16,10 @@
     window.fetchEventSource = fetchEventSource;
 </script>
 
-<script src="/static/js/utils.js?v=1790443101"></script>
+<script src="/static/js/utils.js?v=1790503760"></script>
-<script src="/static/js/ui-ham.js?v=1790443101"></script>
+<script src="/static/js/ui-ham.js?v=1790503760"></script>
-<script src="/static/js/geo.js?v=1790443101"></script>
+<script src="/static/js/geo.js?v=1790503760"></script>
-<script src="/static/js/common.js?v=1790443101"></script>
+<script src="/static/js/common.js?v=1790503760"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1790443101"></script>
+<script src="/static/js/conditions.js?v=1790503760"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/help/usage/telnet.html

@@ -7,9 +7,11 @@
     data into a traditional desktop logging program. The data Spothole produces is compatible with DXSpider, and
     therefore many loggers should accept it with no problems.</p>
 <p>This is a relatively new feature however, so if you run into problems, please do let me know.</p>
-<p>The one caveat with the Spothole telnet server is that <em>it does not support any commands</em>, other than <code>exit</code>.
+<p>The one caveat with the Spothole telnet server is that <em>it does not support any commands</em>, other than
+    <code>exit</code> or <code>bye</code>.
     You cannot therefore issue commands to retrieve past entries, set up filters, etc. If you need to filter the data,
-    many desktop logging programs support this client-side, so hopefully this is not too big of an obstacle.
+    many desktop logging programs support this client-side, so hopefully this is not too big of an obstacle. You also
+    can't currently send spots to Spothole using the Telnet server.
 </p>
 <p>To access Spothole via telnet, set up your logger with the server address <code>{{ telnet_server_address }}</code>
     and port <code>{{ telnet_server_port }}</code>. You can also access it from a terminal with <code>telnet {{

templates/help/usage/web.html

@@ -127,7 +127,10 @@
 
 <h3><i class="fa-solid fa-comment"></i> Add Spot</h3>
 
-<p>The Add Spot page allows you to add a new spot into the Spothole system directly. Currently, this stays within
+<p>The Add Spot page allows you to add a new spot into the Spothole system directly. If the server allows it, a "Send
-    Spothole and is not sent "upstream" to DX clusters, POTA, SOTA etc. regardless of any activity you select.</p>
+    spot to..." checkbox will also appear, allowing you to send your spot "upstream" to the DX cluster that Spothole
+    receives spots from. Spothole will log in to the cluster using the callsign you entered in "Your Call", so if you
+    are already connected to the same cluster node with that callsign, you may need to use a different SSID. Sending
+    spots upstream to POTA, SOTA etc. is not yet supported.</p>
 
 {% end %}

templates/map.html

@@ -115,8 +115,8 @@
     const CARTODB_API_KEY = "{{ web_ui_options.get('cartodb_api_key', '') }}";
 </script>
 
-<script src="/static/js/spotsbandsandmap.js?v=1790443102"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1790503760"></script>
-<script src="/static/js/map.js?v=1790443102"></script>
+<script src="/static/js/map.js?v=1790503760"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -127,8 +127,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1790443101"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1790503760"></script>
-<script src="/static/js/spots.js?v=1790443101"></script>
+<script src="/static/js/spots.js?v=1790503760"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -96,7 +96,7 @@
     </div>
 </div>
 
-<script src="/static/js/status.js?v=1790443101"></script>
+<script src="/static/js/status.js?v=1790503760"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");

webserver/handlers/api/addspot.py

@@ -6,6 +6,7 @@ from typing import Any
 import requests
 import tornado
 from tornado import httputil
+from tornado.ioloop import IOLoop
 from tornado.web import Application
 
 from core.activity_utils import get_ref_regex_for_activity
@@ -13,7 +14,9 @@ from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRE
 from core.constants import UNKNOWN_BAND
 from core.utils import infer_band_from_freq, safe_json_dumps
 from data.spot import Spot
-from providers.spot.spot_provider import SpotProvider
+from providers.spot.dxcluster import DXCluster
+from providers.spot.spot_provider import SpotProvider, SpotSubmissionError
+from providers.spot.tiles import Tiles
 
 logger = logging.getLogger(__name__)
 
@@ -37,7 +40,10 @@ class APISpotHandler(tornado.web.RequestHandler):
         self._spots = spots
         self._spot_providers = spot_providers or []
 
-    def post(self):
+    async def post(self):
+        """Handle the post request with spot data. This is async because it could go on to submit spots to providers
+        which could be slow, so we need to stop it blocking the whole web server"""
+
         try:
             # Reject if not allowed
             if not ALLOW_SPOTTING:
@@ -77,10 +83,6 @@ class APISpotHandler(tornado.web.RequestHandler):
             upstream_credentials = handling.get("upstream_credentials", {})
             captcha_token = handling.get("captcha_token", None)
 
-            # Spothole v2.0 release only: deny upstream spotting. Spothole API breaking changes were in v2.0 but
-            # functionality is not ready yet. TODO
-            submit_upstream = False
-
             # Verify CAPTCHA if required
             if RECAPTCHA_SECRET_KEY:
                 if not captcha_token:
@@ -89,7 +91,7 @@ class APISpotHandler(tornado.web.RequestHandler):
                     self.set_header("Cache-Control", "no-store")
                     self.set_header("Content-Type", "application/json")
                     return
-                if not self._verify_recaptcha(captcha_token):
+                if not await IOLoop.current().run_in_executor(None, self._verify_recaptcha, captcha_token):
                     self.set_status(422)
                     self.write(safe_json_dumps("Error - CAPTCHA verification failed."))
                     self.set_header("Cache-Control", "no-store")
@@ -166,20 +168,15 @@ class APISpotHandler(tornado.web.RequestHandler):
 
             # Validate upstream submission requirements
             if submit_upstream and upstream_provider_name:
-                if not spot.activities:
+                is_cluster = isinstance(self._find_provider(upstream_provider_name, spot.activities), DXCluster)
-                    # TODO when we allow spotting to cluster upstream, we need to remove this restriction
+                is_tiles = isinstance(self._find_provider(upstream_provider_name, spot.activities), Tiles)
-                    self.set_status(422)
+                if not spot.activity_refs and not is_tiles and not is_cluster:
-                    self.write(safe_json_dumps("Error - an activity must be selected to submit upstream."))
-                    self.set_header("Cache-Control", "no-store")
-                    self.set_header("Content-Type", "application/json")
-                    return
-                if not spot.activity_refs and upstream_provider_name != "Tiles":
                     self.set_status(422)
                     self.write(safe_json_dumps("Error - an activity reference is required to submit upstream."))
                     self.set_header("Cache-Control", "no-store")
                     self.set_header("Content-Type", "application/json")
                     return
-                if not spot.dx_grid and upstream_provider_name == "Tiles":
+                if not spot.dx_grid and is_tiles:
                     self.set_status(422)
                     self.write(
                         safe_json_dumps("Error - a grid reference is required to submit upstream to Tiles on the Air.")
@@ -187,7 +184,7 @@ class APISpotHandler(tornado.web.RequestHandler):
                     self.set_header("Cache-Control", "no-store")
                     self.set_header("Content-Type", "application/json")
                     return
-                if not spot.mode and upstream_provider_name == "Tiles":
+                if not spot.mode and is_tiles:
                     self.set_status(422)
                     self.write(safe_json_dumps("Error - a mode is required to submit upstream to Tiles on the Air."))
                     self.set_header("Cache-Control", "no-store")
@@ -200,19 +197,29 @@ class APISpotHandler(tornado.web.RequestHandler):
                 provider = self._find_provider(upstream_provider_name, spot.activities)
                 if provider:
                     try:
-                        # Submit spot to the upstream provider
+                        # Submit spot to the upstream provider. Run in a separate thread otherwise this blocks the whole
-                        provider.submit_spot(spot, upstream_credentials)
+                        # web server for everyone!
-                        # Trigger a re-poll after 1 second so the spot appears quickly
+                        await IOLoop.current().run_in_executor(None, provider.submit_spot, spot, upstream_credentials)
-                        threading.Timer(1.0, provider.force_poll).start()
+                        # Trigger a re-poll after 3 second so the spot appears quickly. (Submitting to a cluster node is
+                        # slower than this, but we get data as a live stream from cluster anyway, so force_poll does
+                        # nothing in that case. This is really just for the HTTP providers when we submit a spot to them)
+                        threading.Timer(3.0, provider.force_poll).start()
                     except NotImplementedError as e:
                         upstream_warning = str(e)
+                    except SpotSubmissionError as e:
+                        logger.warning(f"Upstream submission to {upstream_provider_name} was not accepted: {e}")
+                        upstream_warning = (
+                            f"Spot was saved locally but upstream submission to {upstream_provider_name} failed: {e}"
+                        )
                     except Exception:
                         logger.exception(f"Failed to submit spot upstream to {upstream_provider_name}")
                         upstream_warning = (
                             f"Spot was saved locally but upstream submission to {upstream_provider_name} failed."
                         )
-                else:
+                elif spot.activities:
                     upstream_warning = f"No enabled provider named '{upstream_provider_name}' supports upstream submission for {', '.join(spot.activities)} spots."
+                else:
+                    upstream_warning = f"No enabled provider named '{upstream_provider_name}' supports upstream submission for spots with no activity."
 
             # If we successfully submitted the spot upstream, don't add it direct to Spothole, otherwise it will be a
             # duplicate with what immediately comes back from the API. But if we weren't asked to send it upstream, or
@@ -239,10 +246,11 @@ class APISpotHandler(tornado.web.RequestHandler):
             self.set_header("Content-Type", "application/json")
 
     def _find_provider(self, provider_name, activities) -> SpotProvider | None:
-        """Find an enabled provider by name that can submit spots for at least one of the given activities."""
+        """Find an enabled provider by name that can submit spots for at least one of the given activities. If there
+        are no activities, find one that can submit spots with no activity."""
 
         for p in self._spot_providers:
-            if p.enabled and p.name == provider_name and any(p.can_submit_spot(a) for a in activities):
+            if p.enabled and p.name == provider_name and any(p.can_submit_spot(a) for a in activities or [None]):
                 return p
         return None
 

webserver/handlers/api/options.py

@@ -21,16 +21,18 @@ def build_options():
     """Build the options data that is returned by the /options API and used to populate the web UI templates. This
     doesn't change at runtime, so it is built once on first use and then cached."""
 
-    # Build a map of activity name -> list of provider names that can submit spots for that activity
+    # Build a map of activity name -> list of provider names that can submit spots for that activity. Spots with no
+    # activity use the empty string as the key as that's effectively what is in the "blank" entry in the drop-down
+    # on the Add Spot page
     spot_submit_providers = {}
-
+    for provider in DATA_PROVIDERS.spot_providers:
-    # Spothole v2.0 - disable this for now, API changes are in but this functionality is not ready yet. TODO
+        if not provider.enabled:
-    # for provider in DATA_PROVIDERS.spot_providers:
+            continue
-    #     if not provider.enabled:
+        for activity_name in [None, *(activity.name for activity in ACTIVITIES.values())]:
-    #         continue
+            if provider.can_submit_spot(activity_name):
-    #     for activity in ACTIVITIES.values():
+                names = spot_submit_providers.setdefault(activity_name or "", [])
-    #         if provider.can_submit_spot(activity.name):
+                if provider.name not in names:
-    #             spot_submit_providers.setdefault(activity.name, []).append(provider.name)
+                    names.append(provider.name)
 
     # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle
     # things that aren't even available.

webserver/webserver.py

@@ -175,7 +175,7 @@ class WebServer:
                 APISpotHandler,
                 {
                     "spots": self._data_store.spots,
-                    "spot_providers": self._data_providers,
+                    "spot_providers": self._data_providers.spot_providers,
                 },
             ),
         ]
@@ -226,7 +226,7 @@ class WebServer:
                 V2APISpotHandler,
                 {
                     "spots": self._data_store.spots,
-                    "spot_providers": self._data_providers,
+                    "spot_providers": self._data_providers.spot_providers,
                 },
             ),
         ]
@@ -277,7 +277,7 @@ class WebServer:
                 V1APISpotHandler,
                 {
                     "spots": self._data_store.spots,
-                    "spot_providers": self._data_providers,
+                    "spot_providers": self._data_providers.spot_providers,
                 },
             ),
         ]