Merge branch 'main' into 95-send-spots-to-xota

# Conflicts:
#	README.md
#	config-example.yml
#	core/config.py
#	server/handlers/api/addspot.py
#	server/handlers/api/options.py
#	static/js/add-spot.js
#	static/js/bands.js
#	static/js/map.js

.idea/spothole.iml

@@ -4,7 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
       <excludeFolder url="file://$MODULE_DIR$/.venv" />
-      <excludeFolder url="file://$MODULE_DIR$/webassets/vendor" />
+      <excludeFolder url="file://$MODULE_DIR$/static/vendor" />
     </content>
     <orderEntry type="jdk" jdkName="Python 3.13 virtualenv at ~/code/spothole/.venv" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />

README.md

@@ -1,4 +1,4 @@
-# ![Spothole](/webassets/img/logo.png)
+# ![Spothole](/static/img/logo.png)
 
 Spothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open
 JSON API as well as a website to browse the data.
@@ -89,10 +89,10 @@ callsign and park data.
 
 Various approaches exist to writing your own client, but in general:
 
-* Refer to the API docs. These are built on an OpenAPI definition file (`/webassets/apidocs/openapi.yml`), which you can
+* Refer to the API docs. These are built on an OpenAPI definition file (`/static/apidocs/openapi.yml`), which you can
   automatically use to generate a client skeleton using various software.
 * Call the main "spots" or "alerts" API endpoints to get the data you want. For example, your app could call
-  `https://spothole.app/api/v1/spots` once every few minutes. Apply filters if necessary.
+  `https://spothole.app/api/v2/spots` once every few minutes. Apply filters if necessary.
 * Call the "options" API to get an idea of which bands, modes etc. the server knows about. You might want to do that
   first before calling the spots/alerts APIs, to allow you to populate your filters correctly.
 * Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "alerts"/"upcoming"
@@ -105,12 +105,12 @@ once every two minutes, so if your client is interested in POTA data there's no
 than that.
 
 If you absolutely must be informed within seconds of a spot arriving in Spothole, please use the SSE endpoints instead,
-e.g. `https://spothole.app/api/v1/spots/stream`.
+e.g. `https://spothole.app/api/v2/spots/stream`.
 
 If you want to handle different types of spot or alert differently within your client, please consider making a single
 request to the Spothole API to retrieve all the data, then filtering on your side. For example, call
-`https://spothole.app/api/v1/spots?sig=POTA,SOTA` rather than making two separate calls to
-`https://spothole.app/api/v1/spots?sig=POTA` and `https://spothole.app/api/v1/spots?sig=SOTA`.
+`https://spothole.app/api/v2/spots?sig=POTA,SOTA` rather than making two separate calls to
+`https://spothole.app/api/v2/spots?sig=POTA` and `https://spothole.app/api/v2/spots?sig=SOTA`.
 
 Remember, here at Spothole Inc. we offer an industry-standard "five nines" uptime on our server, with our own unique
 twist: we don't tell you which side of the decimal point the nines start! (Translation: This is a hobby project.
@@ -318,33 +318,49 @@ To set up nginx as a reverse proxy that sits in front of Spothole, first ensure
 
 Create a file at `/etc/nginx/sites-available/` called `spothole`. Give it the following contents, replacing
 `spothole.app` with the domain name on which you want to run Spothole. If you changed the port on which Spothole runs,
-update that on the "proxy_pass" line too.
+update that on the "proxy_pass" line, and if you installed Spothole somewhere other than `/home/spothole/spothole`,
+adjust the alias location for serving static files.
+
+(The latter section, configuring the nginx server to serve static files directly, improves efficiency because it saves
+Spothole itself from serving JS, CSS etc. files. If you can't do this for some reason, e.g. your nginx and spothole are
+on different computers, you can omit the `location /static/ {}` block.)
 
 ```nginx
 server {
     server_name spothole.app;
 
+    # Global proxy settings
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+    proxy_connect_timeout 10s;
+    proxy_buffering on;
+    
+    # Pass on IP address and host information to Spothole, in case logging this information is required
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Forwarded-Proto $scheme;
+
     # Wellknown area for Lets Encrypt
     location /.well-known/ {
         alias /var/www/html/.well-known/;
     }
 
+    # Load static assets directly from the Spothole static directory
+    location /static/ {
+        alias /home/spothole/spothole/static/;
+        expires 1h;
+        add_header Cache-Control "public, max-age=3600, must-revalidate";
+    }
+    
     # SSE endpoints
-    location ~ ^/api/v1/(spots|alerts)/stream {
+    location ~ ^/api/v2/(spots|alerts)/stream/? {
         proxy_pass http://127.0.0.1:8080;
         
-        # Allow keep-alive
-        proxy_http_version 1.1;
-        proxy_set_header Connection "";
-        
-        # Set correct content type for SSE API calls
-        add_header Content-Type text/event-stream always;
-        
-        # Set remove buffering, remove caching, add suitable timeouts for SSE API calls
+        # Remove buffering, remove caching, add suitable timeouts for SSE API calls
         proxy_buffering off;
         proxy_cache off;
         proxy_read_timeout 24h;
-        proxy_connect_timeout 10s;
         proxy_send_timeout 24h;
         proxy_set_header X-Accel-Buffering no;
         add_header Cache-Control no-store always;
@@ -352,59 +368,28 @@ server {
         # Allow cross-origin requests to API
         proxy_hide_header Access-Control-Allow-Origin;
         add_header Access-Control-Allow-Origin * always; 
-        
-        # Pass on IP address and host information to Spothole, in case logging this information is required
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto $scheme;
     }
 
     # Other API endpoints
     location /api/ {
         proxy_pass http://127.0.0.1:8080;
         
-        # Allow keep-alive
-        proxy_http_version 1.1;
-        proxy_set_header Connection "";
-        
-        # Set up buffering, remove caching, add suitable timeouts for API calls
-        proxy_buffering on;
+        # Remove buffering, remove caching, add suitable timeouts for API calls
+        proxy_buffering off;
         proxy_cache off;
         proxy_read_timeout 30s;
-        proxy_connect_timeout 10s;
         add_header Cache-Control no-store always;
         
         # Allow cross-origin requests to API
         proxy_hide_header Access-Control-Allow-Origin;
         add_header Access-Control-Allow-Origin * always;
-        
-        # Pass on IP address and host information to Spothole, in case logging this information is required
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto $scheme;
     }
 
-    # Static assets
+    # Templated pages
     location / {
         proxy_pass http://127.0.0.1:8080;
-        
-        # Allow keep-alive
-        proxy_http_version 1.1;
-        proxy_set_header Connection "";
-        
-        # Set up buffering and caching, add suitable timeouts for static asset requests
-        proxy_buffering on;
         proxy_read_timeout 30s;
-        proxy_connect_timeout 10s;
-        add_header Cache-Control "public, max-age=3600, must-revalidate" always;
-        
-        # Pass on IP address and host information to Spothole, in case logging this information is required
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto $scheme;
+        add_header Cache-Control "no-cache, must-revalidate" always;
     }
 }
 ```
@@ -467,12 +452,12 @@ To navigate your way around the source code, this list may help.
 
 *HTML/JS/CSS front-end code*
 
-* `/webassets` - Root for static files served by the web server
-* `/webassets/apidocs` - Contains the OpenAPI spec (`openapi.yml`)
-* `/webassets/css` - CSS files used by the web front-end
-* `/webassets/img` - image files used by the web front-end
-* `/webassets/js` - JavaScript used by the web front-end
-* `/webassets/vendor` - Third-party libraries (CSS, JS, fonts and images)
+* `/static` - Root for static files served by the web server. These are all served from a path starting `/static/`.
+* `/static/apidocs` - Contains the OpenAPI spec (`openapi.yml`)
+* `/static/css` - CSS files used by the web front-end
+* `/static/img` - image files used by the web front-end
+* `/static/js` - JavaScript used by the web front-end
+* `/static/vendor` - Third-party libraries (CSS, JS, fonts and images)
 
 *Miscellaneous*
 
@@ -524,13 +509,16 @@ my knowledge, created by HA8TKS for his CQ and ITU zone layers for Leaflet. `/da
 maps of conference centres.
 
 The project contains a set of flag icons generated using the "Noto Color Emoji" font on a Debian system, in the
-`/webassets/img/flags/` directory.
+`/static/img/flags/` directory.
 
 The software uses a number of Python libraries as listed in `requirements.txt`, and a number of JavaScript libraries.
 This project would not have been possible without these libraries, so many thanks to their developers.
 
 ### Third Party Libraries
 
+A number of third-party libraries are self-hosted in the `/static/vendor/` directory. These files are subject to
+their own licences and are not covered by the overall licence declared in the `LICENSE` file.
+
 A number of third-party libraries are self-hosted in the `/webassets/vendor/` directory. These files are subject to
 their own licences and are not covered by the overall licence declared in the `LICENSE` file.
 

alertproviders/http_alert_provider.py

@@ -4,6 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from alertproviders.alert_provider import AlertProvider
 from core.constants import HTTP_HEADERS
@@ -41,6 +42,8 @@ class HTTPAlertProvider(AlertProvider):
             # Request data from API
             logging.debug("Polling " + self.name + " alert API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            # Check response code was good
+            if http_response.ok:
                 # Pass off to the subclass for processing
                 new_alerts = self._http_response_to_alerts(http_response)
                 # Submit the new alerts for processing. There might not be any alerts for the less popular programs.
@@ -50,7 +53,14 @@ class HTTPAlertProvider(AlertProvider):
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
                 logging.debug("Received data from " + self.name + " alert API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} alerts API.")
 
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} alerts API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} alerts API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP JSON Alert Provider (" + self.name + ")")

config-example.yml

@@ -181,7 +181,7 @@ alert-providers:
 
 
 # Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band
-# conditions, etc.) and make it available via the /api/v1/solar endpoint.
+# conditions, etc.) and make it available via the /api/v2/solar endpoint.
 solar-condition-providers:
   - class: "HamQSL"
     name: "HamQSL"
@@ -211,10 +211,28 @@ clublog-api-key: ""
 # Allow submitting spots to the Spothole API?
 allow-spotting: true
 
+# Allow upstream submission of spots to external providers (POTA, SOTA, etc.) via the API?
+# Requires allow-spotting to also be true. Set to false to only accept spots into the local
+# Spothole database, without forwarding them to any external service.
+allow-upstream-spotting: true
+
+# Google reCAPTCHA v2 keys for CAPTCHA protection on upstream spot submission. Both keys must be set to enable CAPTCHA.
+# Leave both empty to disable CAPTCHA (e.g. for a private/trusted server) or if allow-spotting is false, in which case
+# they will do nothing. Note that with CAPTCHA enabled, this will prevent third-party clients submitting spots through
+# Spothole unless the clients are web-based, use the same site key, have their domains enabled in your reCAPTCHA config,
+# and of course their user solves the CAPTCHA.
+# You can sign up for reCAPTCHA at https://www.google.com/recaptcha/
+recaptcha-site-key: ""
+recaptcha-secret-key: ""
+
 # Log web requests? Useful to see what your users are requesting in terms of pages and API endpoints, but will fill up
 # your log quickly on a popular server.
 log-web-requests: false
 
+# Minimum severity of log statements to print to the system log. Defaults to INFO, change to DEBUG if you need more
+# information.
+log-level: INFO
+
 # Options for the web UI.
 web-ui-options:
   spot-count: [ 10, 25, 50, 100 ]

core/cache_utils.py

@@ -8,8 +8,11 @@ from requests_cache import CachedSession
 # of time has passed. This is used throughout Spothole to cache data that does not change
 # rapidly. The ThreadSafeSession construct here protects it against some multithreading
 # contention weirdness we sometimes used to see on startup where the cache was hammered
-# pretty hard.
-_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30))
+# pretty hard. The expanded list of allowable_codes ensures we also cache and return 400-type
+# responses, e.g "this SOTA summit ref doesn't actually exist", to avoid hammering remote
+# servers for data they've told us they can't provide.
+_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30),
+                         allowable_codes=(200, 400, 401, 403, 404))
 _lock = threading.Lock()
 
 

core/config.py

@@ -14,14 +14,18 @@ with open("config.yml") as f:
     config = yaml.safe_load(f)
 logging.info("Loaded config.")
 
-BASE_URL = config["base-url"]
-MAX_SPOT_AGE = config["max-spot-age-sec"]
-MAX_ALERT_AGE = config["max-alert-age-sec"]
-SERVER_OWNER_CALLSIGN = config["server-owner-callsign"]
-WEB_SERVER_PORT = config["web-server-port"]
-ALLOW_SPOTTING = config["allow-spotting"]
-WEB_UI_OPTIONS = config["web-ui-options"]
+BASE_URL = config.get("base-url", "http://localhost:8080")
+MAX_SPOT_AGE = config.get("max-spot-age-sec", 3600)
+MAX_ALERT_AGE = config.get("max-alert-age-sec", 604800)
+SERVER_OWNER_CALLSIGN = config.get("server-owner-callsign", "N0CALL")
+WEB_SERVER_PORT = config.get("web-server-port", 8080)
+ALLOW_SPOTTING = config.get("allow-spotting", True)
+ALLOW_UPSTREAM_SPOTTING = config.get("allow-upstream-spotting", True)
+WEB_UI_OPTIONS = config.get("web-ui-options", {})
 API_ONLY_MODE = config.get("api-only-mode", False)
+RECAPTCHA_SECRET_KEY = config.get("recaptcha-secret-key", "")
+RECAPTCHA_SITE_KEY = config.get("recaptcha-site-key", "")
+LOG_LEVEL = config.get("log-level", "INFO")
 LOG_WEB_REQUESTS = config.get("log-web-requests", False)
 
 # For ease of config, each spot provider owns its own config about whether it should be enabled by default in the web UI
@@ -29,6 +33,9 @@ LOG_WEB_REQUESTS = config.get("log-web-requests", False)
 WEB_UI_OPTIONS["spot-providers-enabled-by-default"] = [p["name"] for p in config["spot-providers"] if p["enabled"] and (
         "enabled-by-default-in-web-ui" not in p or p["enabled-by-default-in-web-ui"])]
 # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from
-# one of our proviers. We set that to also be enabled by default.
+# one of our proviers. We set that to also be enabled by default. We can also include the reCaptcha site key so the UI
+# can access it.
 if ALLOW_SPOTTING:
     WEB_UI_OPTIONS["spot-providers-enabled-by-default"].append("API")
+    WEB_UI_OPTIONS["recaptcha-site-key"] = RECAPTCHA_SITE_KEY
+WEB_UI_OPTIONS["allow-upstream-spotting"] = ALLOW_SPOTTING and ALLOW_UPSTREAM_SPOTTING

core/constants.py

@@ -11,7 +11,7 @@ HAMQTH_PRG = ("Spothole v" + SOFTWARE_VERSION + " operated by " + SERVER_OWNER_C
 
 # Special Interest Groups
 SIGS = [
-    SIG(name="POTA", comment_names=["POTA"], description="Parks on the Air", ref_regex=r"[A-Z]{2}\-\d{4,5}"),
+    SIG(name="POTA", comment_names=["POTA"], description="Parks on the Air", ref_regex=r"[A-Z]{2}\-\d{4,5}|K\-TEST"),
     SIG(name="SOTA", comment_names=["SOTA"], description="Summits on the Air", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}"),
     SIG(name="WWFF", comment_names=["WWFF"], description="World Wide Flora & Fauna", ref_regex=r"[A-Z0-9]{1,3}FF\-\d{4}"),
     SIG(name="GMA", comment_names=["GMA"], description="Global Mountain Activity", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}"),
@@ -28,7 +28,7 @@ SIGS = [
     SIG(name="BOTA", comment_names=[], description="Beaches on the Air"),
     SIG(name="KRMNPA", comment_names=["KRMNPA"], description="Keith Roget Memorial National Parks Award"),
     SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"LL[A-Z]{2}\-\d{4}"),
-    SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2}R\-\d{4}"),
+    SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2,3}R\-\d{4}"),
     SIG(name="Tiles", comment_names=[], description="Tiles on the Air", ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}"),
     SIG(name="WAB", comment_names=["WAB"], description="Worked All Britain", ref_regex=r"[A-Z]{1,2}[0-9]{2}"),
     SIG(name="WAI", comment_names=["WAI"], description="Worked All Ireland", ref_regex=r"[A-Z][0-9]{2}"),
@@ -38,10 +38,12 @@ SIGS = [
 
 # Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
 CW_MODES = ["CW"]
-PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "M17"]
+PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "FUSION", "M17"]
 DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "PSK", "OLIVIA", "PKT", "MSK144"]
 ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
 MODE_TYPES = ["CW", "PHONE", "DATA"]
+SSB_SUB_MODES = ["USB", "LSB"]
+DV_SUB_MODES = ["DMR", "DSTAR", "C4FM", "FUSION", "M17"]
 
 # Mode aliases. Sometimes we get spots with a mode described in a different way that is effectively the same as a mode
 # we already know, or we want to normalise things for consistency. The lookup table for this is here. Incoming spots
@@ -102,5 +104,6 @@ PROPAGATION_MODES = {
     "AU": "Aurora",
     "MS": "Meteor scatter",
     "RS": "Rain scatter",
-    "AS": "Aircraft scatter"
+    "AS": "Aircraft scatter",
+    "SAT": "Satellite"
 }

core/lookup_helper.py

@@ -5,13 +5,13 @@ import re
 import urllib.parse
 from datetime import timedelta
 
-import requests
 import xmltodict
 from diskcache import Cache
 from pyhamtools import LookupLib, Callinfo, callinfo
 from pyhamtools.exceptions import APIKeyMissingError
 from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 from requests_cache import CachedSession
 
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
@@ -143,13 +143,21 @@ class LookupHelper:
         try:
             logging.info("Downloading Country-files.com cty.plist...")
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
-                                                      headers=HTTP_HEADERS).text
+                                                      headers=HTTP_HEADERS)
 
+            if response.ok:
                 with open(self._country_files_cty_plist_download_location, "w") as f:
-                f.write(response)
+                    f.write(response.text)
                     f.flush()
                     return True
+            else:
+                logging.warning(f"HTTP {response.status_code} when downloading Country-files.com cty.plist.")
+                return False
 
+        except ConnectionError:
+            logging.warning(f"Connection error when downloading Clublog cty.xml.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when downloading Clublog cty.xml.")
         except Exception as e:
             logging.error("Exception when downloading Clublog cty.xml", e)
             return False
@@ -161,13 +169,21 @@ class LookupHelper:
             logging.info("Downloading dxcc.json...")
             response = SEMI_STATIC_URL_DATA_CACHE.get(
                 "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
-                headers=HTTP_HEADERS).text
+                headers=HTTP_HEADERS)
 
+            if response.ok:
                 with open(self._dxcc_json_download_location, "w") as f:
-                f.write(response)
+                    f.write(response.text)
                     f.flush()
                     return True
+            else:
+                logging.warning(f"HTTP {response.status_code} when downloading dxcc.json.")
+                return False
 
+        except ConnectionError:
+            logging.warning(f"Connection error when downloading dxcc.json.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when downloading dxcc.json.")
         except Exception as e:
             logging.error("Exception when downloading dxcc.json", e)
             return False
@@ -490,25 +506,47 @@ class LookupHelper:
 
         # Try the call as given, then fall back to the base call (strips /P, /M etc.)
         calls_to_try = [call]
+        try:
             home_call = callinfo.Callinfo.get_homecall(call)
             if home_call != call:
                 calls_to_try.append(home_call)
+        except ValueError:
+            logging.debug("Could not look up home call for callsign %s", call)
 
         for lookup_call in calls_to_try:
             try:
-                lookup_response = requests.get(
+                response = SEMI_STATIC_URL_DATA_CACHE.get(
                     self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
-                    headers=HTTP_HEADERS, timeout=10).content
-                raw = xmltodict.parse(lookup_response).get("QRZDatabase", {}).get("Callsign")
-                if raw:
-                    data = _normalize_qrz_data(raw)
+                    headers=HTTP_HEADERS, timeout=10)
+                if response.ok:
+                    qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
+                    if qrz_response:
+                        if "Callsign" in qrz_response:
+                            data = _normalize_qrz_data(qrz_response.get("Callsign"))
                             self._qrz_callsign_data_cache.add(call, data, expire=604800)  # 1 week in seconds
                             return data
+                        elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
+                            # Errors here are normally just "callsign not in database", no need to log that ourselves
+                            # above debug level.
+                            logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call,
+                                          qrz_response.get("Session").get("Error"))
+
+                    elif not response.from_cache:
+                        logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call)
+                elif not response.from_cache:
+                    logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call)
+
             except (KeyError, ValueError):
                 continue
+            except ConnectionError:
+                logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call)
+                continue
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call)
+                continue
             except Exception:
-                logging.error("Exception when looking up QRZ data")
-                return None
+                logging.error("Exception when looking up callsign %s using QRZ", lookup_call, exc_info=True)
+                continue
 
         # Not found in QRZ; cache None so we don't keep retrying
         self._qrz_callsign_data_cache.add(call, None, expire=604800)  # 1 week in seconds
@@ -546,23 +584,36 @@ class LookupHelper:
 
         # Try the call as given, then fall back to the base call (strips /P, /M etc.)
         calls_to_try = [call]
+        try:
             home_call = callinfo.Callinfo.get_homecall(call)
             if home_call != call:
                 calls_to_try.append(home_call)
+        except ValueError:
+            logging.debug("Could not look up home call for callsign %s", call)
 
         for lookup_call in calls_to_try:
             try:
-                lookup_data = SEMI_STATIC_URL_DATA_CACHE.get(
+                response = SEMI_STATIC_URL_DATA_CACHE.get(
                     self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
-                        lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS).content
-                data = xmltodict.parse(lookup_data)["HamQTH"]["search"]
+                        lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
+                if response.ok:
+                    data = xmltodict.parse(response.content)["HamQTH"]["search"]
                     self._hamqth_callsign_data_cache.add(call, data, expire=604800)  # 1 week in seconds
                     return data
+                elif not response.from_cache:
+                    logging.warning("HTTP %d looking up callsign %s using HamQTH", response.status_code, lookup_call)
+
             except (KeyError, ValueError):
                 continue
+            except ConnectionError:
+                logging.warning(f"Connection error when looking up callsign %s using HamQTH", lookup_call)
+                continue
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when looking up callsign %s using HamQTH", lookup_call)
+                continue
             except Exception:
-                logging.error("Exception when looking up HamQTH data")
-                return None
+                logging.error("Exception when looking up callsign %s using HamQTH", lookup_call, exc_info=True)
+                continue
 
         # Not found in HamQTH; cache None so we don't keep retrying
         self._hamqth_callsign_data_cache.add(call, None, expire=604800)  # 1 week in seconds

core/sig_utils.py

@@ -2,6 +2,7 @@ import csv
 import logging
 
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import SIGS, HTTP_HEADERS
@@ -11,6 +12,14 @@ from core.geo_utils import wab_wai_square_to_lat_lon
 # file in Spothole and load it on startup.
 with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
     _DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
+# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
+# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
+# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
+_WWFF_INDEX_CACHE = {}
+_SIOTA_INDEX_CACHE = {}
+_WOTA_INDEX_CACHE = {}
+_ZLOTA_INDEX_CACHE = {}
+_LLOTA_INDEX_CACHE = {}
 
 
 def get_ref_regex_for_sig(sig):
@@ -45,9 +54,8 @@ def populate_sig_ref_info(sig_ref):
     try:
         if sig.upper() == "POTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if data:
                     fullname = str(data["name"]) if "name" in data else None
                     if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
@@ -57,12 +65,16 @@ def populate_sig_ref_info(sig_ref):
                     sig_ref.grid = data["grid6"] if "grid6" in data else None
                     sig_ref.latitude = data["latitude"] if "latitude" in data else None
                     sig_ref.longitude = data["longitude"] if "longitude" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "SOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if data:
                     sig_ref.name = data["name"] if "name" in data else None
                     sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
@@ -70,67 +82,108 @@ def populate_sig_ref_info(sig_ref):
                     sig_ref.latitude = data["latitude"] if "latitude" in data else None
                     sig_ref.longitude = data["longitude"] if "longitude" in data else None
                     sig_ref.activation_score = data["points"] if "points" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "WWBOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if data:
                     sig_ref.name = data["name"] if "name" in data else None
                     sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
                     sig_ref.grid = data["locator"] if "locator" in data else None
                     sig_ref.latitude = data["lat"] if "lat" in data else None
                     sig_ref.longitude = data["long"] if "long" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if data:
                     sig_ref.name = data["name"] if "name" in data else None
                     sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
                     sig_ref.grid = data["locator"] if "locator" in data else None
-                sig_ref.latitude = data["latitude"] if "latitude" in data else None
-                sig_ref.longitude = data["longitude"] if "longitude" in data else None
+
+                    # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
+                    # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
+                    # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
+                    # just use the single point we have.
+                    if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
+                            "lat2") is not None and data.get("lng2") is not None:
+                        sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
+                        sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
+                    else:
+                        sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
+                        sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "WWFF":
-            wwff_response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
                                                       headers=HTTP_HEADERS)
-            if not wwff_response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", wwff_response.status_code, sig, ref_id)
-                return sig_ref
-            wwff_index = {row["reference"]: row for row in csv.DictReader(wwff_response.content.decode().splitlines())}
-            row = wwff_index.get(ref_id)
+            if response.ok:
+                global _WWFF_INDEX_CACHE
+                if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
+                    # New data from WWFF, update our internal map
+                    _WWFF_INDEX_CACHE = {row["reference"]: row for row in
+                                         csv.DictReader(response.content.decode().splitlines())}
+                row = _WWFF_INDEX_CACHE.get(ref_id)
                 if row:
                     sig_ref.name = row["name"] if "name" in row else None
                     sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
                     sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
                     sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
-                sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row["longitude"] != "-" else None
+                    sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row[
+                        "longitude"] != "-" else None
+                elif not response.from_cache:
+                    logging.warning("WWFF database did not contain data for ref %s", ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "SIOTA":
-            siota_response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
                                                       headers=HTTP_HEADERS)
-            if not siota_response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", siota_response.status_code, sig, ref_id)
-                return sig_ref
-            siota_index = {row["SILO_CODE"]: row for row in
-                           csv.DictReader(siota_response.content.decode().splitlines())}
-            row = siota_index.get(ref_id)
+            if response.ok:
+                global _SIOTA_INDEX_CACHE
+                if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
+                    # New data from SIOTA, update our internal map
+                    _SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
+                                          csv.DictReader(response.content.decode().splitlines())}
+                row = _SIOTA_INDEX_CACHE.get(ref_id)
                 if row:
                     sig_ref.name = row["NAME"] if "NAME" in row else None
                     sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
                     sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
                     sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
+                elif not response.from_cache:
+                    logging.warning("SIOTA database did not contain data for ref %s", ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "WOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if data:
-                for feature in data.get("features", []):
-                    if feature["properties"]["wotaId"] == ref_id:
+                    global _WOTA_INDEX_CACHE
+                    if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from WOTA, update our internal map
+                        _WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
+                                             data.get("features", [])}
+                    feature = _WOTA_INDEX_CACHE.get(ref_id)
+                    if feature:
                         sig_ref.name = feature["properties"]["title"]
                         # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
                         # added to them
@@ -141,48 +194,68 @@ def populate_sig_ref_info(sig_ref):
                         sig_ref.grid = feature["properties"]["qthLocator"]
                         sig_ref.latitude = feature["geometry"]["coordinates"][1]
                         sig_ref.longitude = feature["geometry"]["coordinates"][0]
-                        break
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "ZLOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if isinstance(data, list):
-                for asset in data:
-                    if asset["code"] == ref_id:
+                    global _ZLOTA_INDEX_CACHE
+                    if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from ZLOTA, update our internal map
+                        _ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
+                    asset = _ZLOTA_INDEX_CACHE.get(ref_id)
+                    if asset:
                         sig_ref.name = asset["name"]
-                        sig_ref.url = "https://ontheair.nz/assets/ZLI_OT-030" + ref_id.replace("/", "_")
+                        sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
                         try:
                             sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
                         except:
                             logging.debug("Invalid lat/lon received for reference")
                             sig_ref.latitude = asset["y"]
                             sig_ref.longitude = asset["x"]
-                        break
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "BOTA":
             if not sig_ref.name:
                 sig_ref.name = sig_ref.id
             sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
+
         elif sig.upper() == "LLOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if isinstance(data, list):
-                for ref in data:
-                    if ref["reference_code"] == ref_id:
+                    global _LLOTA_INDEX_CACHE
+                    if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from LLOTA, update our internal map
+                        _LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
+                    ref = _LLOTA_INDEX_CACHE.get(ref_id)
+                    if ref:
                         sig_ref.name = str(ref["name"])
                         sig_ref.url = "https://llota.app/list/ref/" + ref_id
                         sig_ref.grid = str(ref["grid_locator"])
                         ll = locator_to_latlong(sig_ref.grid)
                         sig_ref.latitude = ll[0]
                         sig_ref.longitude = ll[1]
-                        break
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "WWTOTA":
             if not sig_ref.name:
                 sig_ref.name = sig_ref.id
             sig_ref.url = "https://wwtota.com/seznam/karta_rozhledny.php?ref=" + str(sig_ref.name)
+
         elif sig.upper() == "TILES":
             # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
             if not sig_ref.name:
@@ -193,6 +266,7 @@ def populate_sig_ref_info(sig_ref):
                 ll = locator_to_latlong(str(sig_ref.grid))
                 sig_ref.latitude = ll[0]
                 sig_ref.longitude = ll[1]
+
         elif sig.upper() == "WAB" or sig.upper() == "WAI":
             ll = wab_wai_square_to_lat_lon(ref_id)
             if ll:
@@ -202,21 +276,31 @@ def populate_sig_ref_info(sig_ref):
                     sig_ref.latitude = ll[0]
                     sig_ref.longitude = ll[1]
                 except:
-                    logging.debug("Invalid lat/lon received for reference")
+                    logging.warning("Invalid lat/lon received for WAB/WAI reference")
+
         elif sig.upper() == "DME":
             # Zero-pad to 5 digits to match our source data
             row = _DME_INDEX.get(ref_id.zfill(5))
             if row:
                 sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
-                sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get("LATITUD_ETRS89_REGCAN95") else None
-                sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get("LONGITUD_ETRS89_REGCAN95") else None
+                sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LATITUD_ETRS89_REGCAN95") else None
+                sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LONGITUD_ETRS89_REGCAN95") else None
                 if sig_ref.latitude and sig_ref.longitude:
                     try:
                         sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6)
                     except Exception:
-                        logging.debug("Invalid lat/lon received for reference")
+                        logging.warning("Invalid lat/lon received for DME reference")
+            else:
+                logging.warning("DME database did not contain data for ref %s", ref_id)
+
+    except ConnectionError:
+        logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
+    except (ConnectTimeout, ReadTimeout):
+        logging.warning(f"Timeout when looking up sig_ref info for " + sig + " ref " + ref_id)
     except Exception:
-        logging.warning("Failed to look up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
+        logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
     return sig_ref
 
 

data/lookup_credentials.py

@@ -12,15 +12,15 @@ class LookupCredentials:
     hamqth_session_id: str = ""  # alternative to username/password
 
 
-def extract_credentials(query_params):
-    """Build a LookupCredentials from HTTP query params; returns None if no usable credentials are present."""
+def extract_credentials(headers):
+    """Build a LookupCredentials from HTTP request headers; returns None if no usable credentials are present."""
     creds = LookupCredentials(
-        qrz_username=query_params.get("qrz_username", ""),
-        qrz_password=query_params.get("qrz_password", ""),
-        qrz_session_key=query_params.get("qrz_session_key", ""),
-        hamqth_username=query_params.get("hamqth_username", ""),
-        hamqth_password=query_params.get("hamqth_password", ""),
-        hamqth_session_id=query_params.get("hamqth_session_id", ""),
+        qrz_username=headers.get("X-QRZ-Username", ""),
+        qrz_password=headers.get("X-QRZ-Password", ""),
+        qrz_session_key=headers.get("X-QRZ-Session-Key", ""),
+        hamqth_username=headers.get("X-HamQTH-Username", ""),
+        hamqth_password=headers.get("X-HamQTH-Password", ""),
+        hamqth_session_id=headers.get("X-HamQTH-Session-ID", ""),
     )
     has_qrz = creds.qrz_session_key or (creds.qrz_username and creds.qrz_password)
     has_hamqth = creds.hamqth_session_id or (creds.hamqth_username and creds.hamqth_password)

data/spot.py

@@ -133,6 +133,17 @@ class Spot:
     # The ID the source gave it, if any.
     source_id: str | None = None
 
+    def __post_init__(self):
+        """Normalise fields that don't survive a plain dict to Spot conversion. This is used in the "add spot" API
+        endpoint where the client is submitting JSON, and we want to recreate a full Spot object, including nested
+        objects such as the sig_refs list.."""
+
+        if self.sig_refs:
+            self.sig_refs = [
+                sig_ref if isinstance(sig_ref, SIGRef) else SIGRef(**sig_ref)
+                for sig_ref in self.sig_refs
+            ]
+
     def infer_missing(self, credentials=None):
         """Infer missing parameters where possible"""
 

server/handlers/api/addspot.py

@@ -1,35 +1,40 @@
-import json
 import logging
 import re
+import threading
 from datetime import datetime
 from typing import Any
 
 import pytz
+import requests
 import tornado
 from tornado import httputil
 from tornado.web import Application
 
-from core.config import ALLOW_SPOTTING, MAX_SPOT_AGE
+from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, MAX_SPOT_AGE, RECAPTCHA_SECRET_KEY
 from core.constants import UNKNOWN_BAND
 from core.lookup_helper import infer_band_from_freq
 from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import safe_json_dumps
-from data.sig_ref import SIGRef
 from data.spot import Spot
+from spotproviders.spot_provider import SpotProvider
+
+RECAPTCHA_VERIFY_URL = "https://www.google.com/recaptcha/api/siteverify"
 
 
 class APISpotHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v1/spot (POST)"""
+    """API request handler for /api/v2/spot (POST)"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._spots = None
         self._web_server_metrics = None
+        self._spot_providers = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, spots, web_server_metrics):
+    def initialize(self, spots, web_server_metrics, spot_providers=None):
         self._spots = spots
         self._web_server_metrics = web_server_metrics
+        self._spot_providers = spot_providers or []
 
     def post(self):
         try:
@@ -64,17 +69,38 @@ class APISpotHandler(tornado.web.RequestHandler):
                 self.set_header("Content-Type", "application/json")
                 return
 
-            # Read in the request body as JSON then convert to a Spot object
-            json_spot = tornado.escape.json_decode(post_data)
-            spot = Spot(**json_spot)
+            # Read in the request body as JSON
+            json_body = tornado.escape.json_decode(post_data)
 
-            # Converting to a spot object this way won't have coped with sig_ref objects, so fix that. (Would be nice to
-            # redo this in a functional style)
-            if spot.sig_refs:
-                real_sig_refs = []
-                for dict_obj in spot.sig_refs:
-                    real_sig_refs.append(json.loads(json.dumps(dict_obj), object_hook=lambda d: SIGRef(**d)))
-                spot.sig_refs = real_sig_refs
+            # Extract the "spot" and "handling" sub-objects from the request body
+            spot_data = json_body.get("spot", {})
+            handling = json_body.get("handling", {})
+
+            # Extract individual parameters that say how this spot should be handled by the server
+            submit_upstream = handling.get("submit_upstream", False)
+            upstream_provider_name = handling.get("upstream_provider", None)
+            upstream_credentials = handling.get("upstream_credentials", {})
+            captcha_token = handling.get("captcha_token", None)
+
+            # Verify CAPTCHA if required
+            if RECAPTCHA_SECRET_KEY:
+                if not captcha_token:
+                    self.set_status(422)
+                    self.write(json.dumps("Error - CAPTCHA token is required for spot submission.",
+                                          default=serialize_everything))
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+                if not self._verify_recaptcha(captcha_token):
+                    self.set_status(422)
+                    self.write(json.dumps("Error - CAPTCHA verification failed.",
+                                          default=serialize_everything))
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+
+            # Convert spot field to a Spot object
+            spot = Spot(**spot_data)
 
             # Reject if no timestamp, frequency, dx_call or de_call
             if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
@@ -126,19 +152,105 @@ class APISpotHandler(tornado.web.RequestHandler):
                 self.set_header("Content-Type", "application/json")
                 return
 
-            # infer missing data, and add it to our database.
-            spot.source = "API"
+            # Reject upstream submission if not permitted
+            if submit_upstream and not ALLOW_UPSTREAM_SPOTTING:
+                self.set_status(403)
+                self.write(json.dumps("Error - this server does not allow upstream spot submission.",
+                                      default=serialize_everything))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Validate upstream submission requirements
+            if submit_upstream and upstream_provider_name:
+                if not spot.sig:
+                    self.set_status(422)
+                    self.write(json.dumps("Error - a SIG must be selected to submit upstream.",
+                                          default=serialize_everything))
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+                if not spot.sig_refs and upstream_provider_name != "Tiles":
+                    self.set_status(422)
+                    self.write(json.dumps("Error - a SIG reference is required to submit upstream.",
+                                          default=serialize_everything))
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+                if not spot.dx_grid and upstream_provider_name == "Tiles":
+                    self.set_status(422)
+                    self.write(json.dumps("Error - a grid reference is required to submit upstream to Tiles on the Air.",
+                                          default=serialize_everything))
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+                if not spot.mode and upstream_provider_name == "Tiles":
+                    self.set_status(422)
+                    self.write(json.dumps("Error - a mode is required to submit upstream to Tiles on the Air.",
+                                          default=serialize_everything))
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+
+            # Submit upstream if requested
+            upstream_warning = None
+            if submit_upstream and upstream_provider_name:
+                provider = self._find_provider(upstream_provider_name, spot.sig)
+                if provider:
+                    try:
+                        # Submit spot to the upstream provider
+                        provider.submit_spot(spot, upstream_credentials)
+                        # Trigger a re-poll after 1 second so the spot appears quickly
+                        threading.Timer(1.0, provider.force_poll).start()
+                    except NotImplementedError as e:
+                        upstream_warning = str(e)
+                    except Exception as e:
+                        logging.warning("Failed to submit spot upstream to " + upstream_provider_name + ": " + str(e))
+                        upstream_warning = "Spot was saved locally but upstream submission to " + upstream_provider_name + " failed: " + str(
+                            e)
+                else:
+                    upstream_warning = "No enabled provider named '" + upstream_provider_name + "' supports upstream submission for " + spot.sig + " spots."
+
+            # If we successfully submitted the spot upstream, don't add it direct to Spothole, otherwise it will be a
+            # duplicate with what immediately comes back from the API. But if we weren't asked to send it upstream, or
+            # we were but it failed, we should still add it to our database anyway.
+            if not submit_upstream or upstream_warning:
                 spot.infer_missing()
                 self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
 
+            if upstream_warning:
+                self.write(json.dumps("Warning - " + upstream_warning, default=serialize_everything))
+                self.set_status(201)
+            else:
                 self.write(safe_json_dumps("OK"))
                 self.set_status(201)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
 
         except Exception as e:
-            logging.error(e)
+            logging.error("Exception when handling client request to add spot API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
+
+    def _find_provider(self, provider_name, sig) -> SpotProvider | None:
+        """Find an enabled provider by name that can submit spots for the given SIG."""
+
+        for p in self._spot_providers:
+            if p.enabled and p.name == provider_name and p.can_submit_spot(sig):
+                return p
+        return None
+
+    @staticmethod
+    def _verify_recaptcha(token):
+        """Verify a Google reCAPTCHA v2 token. Returns True if valid."""
+
+        try:
+            response = requests.post(RECAPTCHA_VERIFY_URL,
+                                     data={"secret": RECAPTCHA_SECRET_KEY, "response": token},
+                                     timeout=(5, 10))
+            return response.ok and response.json().get("success", False)
+        except Exception as e:
+            logging.warning("reCAPTCHA verification request failed: " + str(e))
+            return False

server/handlers/api/alerts.py

@@ -19,7 +19,7 @@ SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
 
 
 class APIAlertsHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v1/alerts"""
+    """API request handler for /api/v2/alerts"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._alerts = None
@@ -52,18 +52,17 @@ class APIAlertsHandler(tornado.web.RequestHandler):
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
 
             # Fetch all alerts matching the query, then optionally enrich with online data
-            credentials = extract_credentials(query_params)
+            credentials = extract_credentials(self.request.headers)
             data = get_alert_list_with_filters(self._alerts, query_params)
             if credentials:
                 data = self._enrich(data, credentials)
             self.write(safe_json_dumps(data))
             self.set_status(200)
         except ValueError as e:
-            logging.error(e)
             self.write(safe_json_dumps("Bad request - " + str(e)))
             self.set_status(400)
         except Exception as e:
-            logging.error(e)
+            logging.error("Exception when handling client request to alerts API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
         self.set_header("Cache-Control", "no-store")
@@ -71,7 +70,7 @@ class APIAlertsHandler(tornado.web.RequestHandler):
 
 
 class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
-    """API request handler for /api/v1/alerts/stream"""
+    """API request handler for /api/v2/alerts/stream"""
 
     def __init__(self, application, request, **kwargs: Any):
         self._sse_alert_queues = None
@@ -103,7 +102,7 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
-            self._credentials = extract_credentials(self._query_params)
+            self._credentials = extract_credentials(self.request.headers)
 
             # Create a alert queue and add it to the web server's list. The web server will fill this when alerts arrive
             self._alert_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)

server/handlers/api/dxstats.py

@@ -1,4 +1,5 @@
 import json
+import logging
 from collections import Counter
 from datetime import datetime, timedelta
 from typing import Any
@@ -9,6 +10,7 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
 
 CONTINENTS = ["EU", "NA", "SA", "AS", "AF", "OC", "AN"]
 BANDS = ["160m", "80m", "60m", "40m", "30m", "20m", "17m", "15m", "12m", "10m", "6m"]
@@ -17,7 +19,7 @@ BANDS_SET = frozenset(BANDS)
 
 
 class APIDxStatsHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v1/dxstats"""
+    """API request handler for /api/v2/dxstats"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._spots = None
@@ -29,6 +31,7 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
         self._web_server_metrics = web_server_metrics
 
     def get(self):
+        try:
             self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
             self._web_server_metrics["api_access_counter"] += 1
             self._web_server_metrics["status"] = "OK"
@@ -55,3 +58,8 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to dx stats API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/api/lookups.py

@@ -19,7 +19,7 @@ from data.spot import Spot
 
 
 class APILookupCallHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v1/lookup/call"""
+    """API request handler for /api/v2/lookup/call"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._web_server_metrics = None
@@ -46,7 +46,7 @@ class APILookupCallHandler(tornado.web.RequestHandler):
                 if re.match(r"^[A-Z0-9/\-]*$", call):
                     # Take the callsign, make a "fake spot" so we can run infer_missing() on it, then repack the
                     # resulting data in the correct way for the API response.
-                    credentials = extract_credentials(query_params)
+                    credentials = extract_credentials(self.request.headers)
                     fake_spot = Spot(dx_call=call)
                     fake_spot.infer_missing(credentials)
                     data = {
@@ -74,7 +74,7 @@ class APILookupCallHandler(tornado.web.RequestHandler):
                 self.set_status(422)
 
         except Exception as e:
-            logging.error(e)
+            logging.error("Exception when handling client request to call lookup API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
 
@@ -83,7 +83,7 @@ class APILookupCallHandler(tornado.web.RequestHandler):
 
 
 class APILookupSIGRefHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v1/lookup/sigref"""
+    """API request handler for /api/v2/lookup/sigref"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._web_server_metrics = None
@@ -126,7 +126,7 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
                 self.set_status(422)
 
         except Exception as e:
-            logging.error(e)
+            logging.error("Exception when handling client request to sig ref lookup API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
 
@@ -135,7 +135,7 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
 
 
 class APILookupGridHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v1/lookup/grid"""
+    """API request handler for /api/v2/lookup/grid"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._web_server_metrics = None
@@ -188,7 +188,7 @@ class APILookupGridHandler(tornado.web.RequestHandler):
                 self.set_status(422)
 
         except Exception as e:
-            logging.error(e)
+            logging.error("Exception when handling client request to grid ref lookup API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
 

server/handlers/api/options.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 from typing import Any
 
@@ -13,43 +14,65 @@ from core.utils import safe_json_dumps
 
 
 class APIOptionsHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v1/options"""
+    """API request handler for /api/v2/options"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._status_data = None
         self._web_server_metrics = None
+        self._spot_providers = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, status_data, web_server_metrics):
+    def initialize(self, status_data, web_server_metrics, spot_providers=None):
         self._status_data = status_data
         self._web_server_metrics = web_server_metrics
+        self._spot_providers = spot_providers or []
 
     def get(self):
+        try:
             # Metrics
             self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
             self._web_server_metrics["api_access_counter"] += 1
             self._web_server_metrics["status"] = "OK"
             api_requests_counter.inc()
 
+            # Build a map of SIG name -> list of provider names that can submit spots for that SIG
+        spot_submit_providers = {}
+        for provider in self._spot_providers:
+            if not provider.enabled:
+                continue
+            for sig in SIGS:
+                    if provider.can_submit_spot(sig.name):
+                    spot_submit_providers.setdefault(sig.name, []).append(provider.name)
+
+        # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle
+        # things that aren't even available.
+        spot_sources: list = list(
+            map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["spot_providers"])))
+        alert_sources = list(
+            map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["alert_providers"])))
+        # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from
+        # one of our providers.
+            if ALLOW_SPOTTING:
+                spot_sources.append("API")
+
         options = {"bands": BANDS,
                    "modes": ALL_MODES,
                    "mode_types": MODE_TYPES,
                    "sigs": SIGS,
-                   # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle things that aren't even available.
-                   "spot_sources": list(
-                       map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["spot_providers"]))),
-                   "alert_sources": list(
-                       map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["alert_providers"]))),
+                   "spot_sources": spot_sources,
+                   "alert_sources": alert_sources,
                    "continents": CONTINENTS,
                    "propagation_modes": list(PROPAGATION_MODES.values()),
                    "max_spot_age": MAX_SPOT_AGE,
-                   "spot_allowed": ALLOW_SPOTTING}
-        # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from
-        # one of our proviers.
-        if ALLOW_SPOTTING:
-            options["spot_sources"].append("API")
+                   "spot_allowed": ALLOW_SPOTTING,
+                   "spot_submit_providers": spot_submit_providers}
 
             self.write(safe_json_dumps(options))
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to options API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/api/solar_conditions.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 from typing import Any
 
@@ -7,10 +8,11 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
 
 
 class APISolarConditionsHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v1/solar"""
+    """API request handler for /api/v2/solar"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._solar_conditions = None
@@ -22,6 +24,7 @@ class APISolarConditionsHandler(tornado.web.RequestHandler):
         self._web_server_metrics = web_server_metrics
 
     def get(self):
+        try:
             # Metrics
             self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
             self._web_server_metrics["api_access_counter"] += 1
@@ -32,3 +35,8 @@ class APISolarConditionsHandler(tornado.web.RequestHandler):
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to solar conditions API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/api/spots.py

@@ -19,7 +19,7 @@ SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
 
 
 class APISpotsHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v1/spots"""
+    """API request handler for /api/v2/spots"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._spots = None
@@ -52,18 +52,17 @@ class APISpotsHandler(tornado.web.RequestHandler):
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
 
             # Fetch all spots matching the query, then optionally enrich with online data
-            credentials = extract_credentials(query_params)
+            credentials = extract_credentials(self.request.headers)
             data = get_spot_list_with_filters(self._spots, query_params)
             if credentials:
                 data = self._enrich(data, credentials)
             self.write(safe_json_dumps(data))
             self.set_status(200)
         except ValueError as e:
-            logging.error(e)
             self.write(safe_json_dumps("Bad request - " + str(e)))
             self.set_status(400)
         except Exception as e:
-            logging.error(e)
+            logging.error("Excedption when handling client request to spots API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
         self.set_header("Cache-Control", "no-store")
@@ -71,7 +70,7 @@ class APISpotsHandler(tornado.web.RequestHandler):
 
 
 class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
-    """API request handler for /api/v1/spots/stream"""
+    """API request handler for /api/v2/spots/stream"""
 
     def __init__(self, application, request, **kwargs: Any):
         self._sse_spot_queues = None
@@ -105,7 +104,7 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
-            self._credentials = extract_credentials(self._query_params)
+            self._credentials = extract_credentials(self.request.headers)
 
             # Create a spot queue and add it to the web server's list. The web server will fill this when spots arrive
             self._spot_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)

server/handlers/api/status.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 from typing import Any
 
@@ -11,7 +12,7 @@ from core.utils import safe_json_dumps
 
 
 class APIStatusHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v1/status"""
+    """API request handler for /api/v2/status"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._status_data = None
@@ -23,6 +24,7 @@ class APIStatusHandler(tornado.web.RequestHandler):
         self._web_server_metrics = web_server_metrics
 
     def get(self):
+        try:
             # Metrics
             self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
             self._web_server_metrics["api_access_counter"] += 1
@@ -33,3 +35,8 @@ class APIStatusHandler(tornado.web.RequestHandler):
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to status API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/api/v1_compatability.py

@@ -0,0 +1,31 @@
+import json
+
+import tornado
+
+from core.utils import serialize_everything
+
+
+class V1GoneHandler(tornado.web.RequestHandler):
+    """Returns 410 Gone with a message for any endpoints in the old API that have breaking changes in the new one or
+    have been retired."""
+
+    def post(self):
+        self.set_status(410)
+        self.write(json.dumps(
+            "This API endpoint has a breaking change or has been removed in the current version of the Spothole API. Please see /apidocs for details of the current API version and the endpoints available.",
+            default=serialize_everything
+        ))
+        self.set_header("Cache-Control", "no-store")
+        self.set_header("Content-Type", "application/json")
+
+
+class V1RedirectHandler(tornado.web.RequestHandler):
+    """Returns 308 Permanent Redirect from any path in the old API to the new one, where there were no breaking changes."""
+
+    def get(self, path):
+        new_url = "/api/v2/" + path
+        if self.request.query:
+            new_url += "?" + self.request.query
+        self.set_status(308)
+        self.set_header("Location", new_url)
+        self.finish()

server/handlers/manifesthandler.py

@@ -0,0 +1,11 @@
+import tornado.web
+
+from core.config import BASE_URL
+
+
+class ManifestHandler(tornado.web.RequestHandler):
+    """Handler for manifest.webmanifest, which needs BASE_URL inserted and a custom content-type"""
+
+    def get(self):
+        self.set_header("Content-Type", "application/manifest+json; charset=UTF-8")
+        self.render("manifest.webmanifest", baseurl=BASE_URL)

server/webserver.py

@@ -5,9 +5,10 @@ import os
 import tornado
 from tornado.web import StaticFileHandler
 
-from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS
+from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL
 from core.utils import empty_queue
 from server.handlers.api.addspot import APISpotHandler
+from server.handlers.api.v1_compatability import V1RedirectHandler, V1GoneHandler
 from server.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
 from server.handlers.api.dxstats import APIDxStatsHandler
 from server.handlers.api.lookups import APILookupCallHandler, APILookupSIGRefHandler, APILookupGridHandler
@@ -15,6 +16,7 @@ from server.handlers.api.options import APIOptionsHandler
 from server.handlers.api.solar_conditions import APISolarConditionsHandler
 from server.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
 from server.handlers.api.status import APIStatusHandler
+from server.handlers.manifesthandler import ManifestHandler
 from server.handlers.metrics import PrometheusMetricsHandler
 from server.handlers.pagetemplate import PageTemplateHandler
 
@@ -24,7 +26,7 @@ _HERE = os.path.dirname(__file__ or "")
 class WebServer:
     """Provides the public-facing web server."""
 
-    def __init__(self, spots, alerts, solar_conditions, status_data):
+    def __init__(self, spots, alerts, solar_conditions, status_data, spot_providers=None):
         """Constructor"""
 
         self._spots = spots
@@ -33,6 +35,7 @@ class WebServer:
         self._sse_spot_queues = []
         self._sse_alert_queues = []
         self._status_data = status_data
+        self._spot_providers = spot_providers or []
         self._port = WEB_SERVER_PORT
         self._api_only_mode = API_ONLY_MODE
         self._shutdown_event = asyncio.Event()
@@ -63,20 +66,29 @@ class WebServer:
 
         # API endpoints are always enabled
         api_routes = [
-            (r"/api/v1/spots", APISpotsHandler, {"spots": self._spots, **handler_opts}),
-            (r"/api/v1/alerts", APIAlertsHandler, {"alerts": self._alerts, **handler_opts}),
-            (r"/api/v1/spots/stream", APISpotsStreamHandler,
+            (r"/api/v2/spots", APISpotsHandler, {"spots": self._spots, **handler_opts}),
+            (r"/api/v2/alerts", APIAlertsHandler, {"alerts": self._alerts, **handler_opts}),
+            (r"/api/v2/spots/stream", APISpotsStreamHandler,
              {"sse_spot_queues": self._sse_spot_queues, **handler_opts}),
-            (r"/api/v1/alerts/stream", APIAlertsStreamHandler,
+            (r"/api/v2/alerts/stream", APIAlertsStreamHandler,
              {"sse_alert_queues": self._sse_alert_queues, **handler_opts}),
-            (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._solar_conditions, **handler_opts}),
-            (r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._spots, **handler_opts}),
-            (r"/api/v1/options", APIOptionsHandler, {"status_data": self._status_data, **handler_opts}),
-            (r"/api/v1/status", APIStatusHandler, {"status_data": self._status_data, **handler_opts}),
-            (r"/api/v1/lookup/call", APILookupCallHandler, {**handler_opts}),
-            (r"/api/v1/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
-            (r"/api/v1/lookup/grid", APILookupGridHandler, {**handler_opts}),
-            (r"/api/v1/spot", APISpotHandler, {"spots": self._spots, **handler_opts}),
+            (r"/api/v2/solar", APISolarConditionsHandler, {"solar_conditions": self._solar_conditions, **handler_opts}),
+            (r"/api/v2/dxstats", APIDxStatsHandler, {"spots": self._spots, **handler_opts}),
+            (r"/api/v2/options", APIOptionsHandler,
+             {"status_data": self._status_data, "spot_providers": self._spot_providers, **handler_opts}),
+            (r"/api/v2/status", APIStatusHandler, {"status_data": self._status_data, **handler_opts}),
+            (r"/api/v2/lookup/call", APILookupCallHandler, {**handler_opts}),
+            (r"/api/v2/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
+            (r"/api/v2/lookup/grid", APILookupGridHandler, {**handler_opts}),
+            (r"/api/v2/spot", APISpotHandler,
+             {"spots": self._spots, "spot_providers": self._spot_providers, **handler_opts}),
+        ]
+
+        # v1 API redirects. Most v1 enpoints are unchanged in v2, and get an HTTP 308 redirect to the v2 API. The ones
+        # that have the actual breaking changes get a bespoke handler.
+        v1_compat_routes = [
+            (r"/api/v1/spot", V1GoneHandler),
+            (r"/api/v1/(.*)", V1RedirectHandler),
         ]
 
         # If in API-only mode, serve a basic homepage; in normal mode, serve the usual UI routes
@@ -99,19 +111,25 @@ class WebServer:
             if ALLOW_SPOTTING:
                 ui_routes += [(r"/add-spot", PageTemplateHandler, {"template_name": "add_spot", **handler_opts})]
 
-        # API docs, Prometheus metrics, and finally static assets are always available regardless of API-only mode.
+        # API docs, Prometheus metrics, webapp manifest and static assets are always available regardless of API-only
+        # mode.
         misc_routes = [
             (r"/apidocs", PageTemplateHandler, {"template_name": "apidocs", **handler_opts}),
             (r"/metrics", PrometheusMetricsHandler),
-            (r"/(.*)", StaticFileHandler, {"path": os.path.join(_HERE, "../webassets")})
+            (r"/manifest.webmanifest", ManifestHandler),
+            # If e.g. nginx is configured as a reverse proxy with a hard-coded path to static files, as per the README,
+            # this will never have to handle anything, but having it here allows Spothole to work without nginx for
+            # testing.
+            (r"/static/(.*)", StaticFileHandler, {"path": os.path.join(_HERE, "../static")})
         ]
 
-        app = tornado.web.Application(api_routes + ui_routes + misc_routes,
+        app = tornado.web.Application(api_routes + v1_compat_routes + ui_routes + misc_routes,
                                       template_path=os.path.join(_HERE, "../templates"),
                                       log_function=request_log,
                                       debug=False)
         app.listen(self._port, xheaders=True)
         logging.info("Web server running on port " + str(WEB_SERVER_PORT))
+        logging.info("You can access your copy of Spothole at " + BASE_URL)
         await self._shutdown_event.wait()
 
     def notify_new_spot(self, spot):

solarconditionsproviders/giroionosonde.py

@@ -5,6 +5,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.ionosonde_utils import compute_band_states
@@ -127,10 +128,18 @@ class GIROIonosonde(SolarConditionsProvider):
         from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S")
         to_str = to_time.strftime("%Y.%m.%d+%H:%M:%S")
         url = f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUFD,fmin&DMUF=3000&fromDate={from_str}&toDate={to_str}"
-        response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15))
-        if response.status_code != 200:
+        try:
+            http_response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15))
+            if not http_response.ok:
+                logging.warning(f"HTTP {http_response.status_code} when calling Giro ionosonde API.")
+                return None, None, None
+            return self._parse_all(http_response.text)
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing Giro ionosonde API.")
+            return None, None, None
+        except ConnectionError:
+            logging.warning("Connection error when accessing Giro ionosonde API.")
             return None, None, None
-        return self._parse_all(response.text)
 
     @staticmethod
     def _parse_all(text):

solarconditionsproviders/hamqsl.py

@@ -18,10 +18,6 @@ class HamQSL(HTTPSolarConditionsProvider):
         super().__init__(provider_config, URL, POLL_INTERVAL)
 
     def _http_response_to_solar_conditions(self, http_response):
-        if http_response.status_code != 200:
-            logging.warning("HamQSL solar conditions API returned HTTP " + str(http_response.status_code))
-            return None
-
         root = ElementTree.fromstring(http_response.text)
         sd = root.find("solardata")
         if sd is None:

solarconditionsproviders/http_solar_conditions_provider.py

@@ -4,6 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.solar_conditions_provider import SolarConditionsProvider
@@ -39,13 +40,22 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
         try:
             logging.debug("Polling " + self.name + " solar conditions API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            # Check response code was good
+            if http_response.ok:
                 new_data = self._http_response_to_solar_conditions(http_response)
                 self.update_data(new_data)
 
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
                 logging.debug("Received data from " + self.name + " solar conditions API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} solar conditions API.")
 
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} solar conditions API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} solar conditions API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP Solar Conditions Provider (" + self.name + ")")

solarconditionsproviders/kc2gprop.py

@@ -4,6 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.ionosonde_utils import compute_band_states
@@ -44,9 +45,9 @@ class KC2GProp(SolarConditionsProvider):
     def _poll(self):
         try:
             logging.debug("Polling KC2G ionosonde data...")
-            response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30))
-            if response.status_code != 200:
-                logging.warning(f"KC2G ionosonde API returned HTTP {response.status_code}")
+            http_response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30))
+            if not http_response.ok:
+                logging.warning(f"HTTP {http_response.status_code} when calling KG2G ionosonde API.")
                 return
 
             now = datetime.now(timezone.utc)
@@ -57,7 +58,7 @@ class KC2GProp(SolarConditionsProvider):
             ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})
             updated_count = 0
 
-            for reading in response.json():
+            for reading in http_response.json():
                 station = reading.get("station", {})
                 ursi = station.get("code")
                 name = station.get("name")
@@ -115,6 +116,10 @@ class KC2GProp(SolarConditionsProvider):
             self.last_update_time = datetime.now(pytz.UTC)
             logging.debug(f"Updated KC2G ionosonde data for {updated_count} stations.")
 
+        except ConnectionError:
+            logging.warning("Connection error when accessing KC2G ionosonde API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing KC2G ionosonde API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in KC2G ionosonde data provider")

solarconditionsproviders/noaa3dayforecast.py

@@ -80,10 +80,6 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
         return result if result else None
 
     def _http_response_to_solar_conditions(self, http_response):
-        if http_response.status_code != 200:
-            logging.warning("NOAA K-index forecast API returned HTTP " + str(http_response.status_code))
-            return None
-
         lines = http_response.text.splitlines()
 
         # Find the "NOAA Kp index breakdown" section header

spothole.py

@@ -8,7 +8,7 @@ import sys
 from diskcache import Cache
 
 from core.cleanup import CleanupTimer
-from core.config import config, SERVER_OWNER_CALLSIGN
+from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL
 from core.constants import SOFTWARE_VERSION
 from core.lookup_helper import lookup_helper
 from core.status_reporter import StatusReporter
@@ -84,9 +84,9 @@ def get_solar_conditions_provider_from_config(config_providers_entry):
 if __name__ == '__main__':
     # Set up logging
     root = logging.getLogger()
-    root.setLevel(logging.INFO)
+    root.setLevel(LOG_LEVEL)
     handler = logging.StreamHandler(sys.stdout)
-    handler.setLevel(logging.INFO)
+    handler.setLevel(LOG_LEVEL)
     formatter = logging.Formatter("%(levelname)s : %(message)s")
     handler.setFormatter(formatter)
     root.handlers.clear()
@@ -102,18 +102,21 @@ if __name__ == '__main__':
     # Set up lookup helper
     lookup_helper.start()
 
-    # Set up web server
-    web_server = WebServer(spots=spots, alerts=alerts, solar_conditions=solar_conditions, status_data=status_data)
-
-    # Fetch, set up and start spot providers
+    # Create spot providers
     for entry in config["spot-providers"]:
         spot_providers.append(get_spot_provider_from_config(entry))
+
+    # Set up web server
+    web_server = WebServer(spots=spots, alerts=alerts, solar_conditions=solar_conditions, status_data=status_data,
+                           spot_providers=spot_providers)
+
+    # Set up and start spot providers
     for p in spot_providers:
         p.setup(spots=spots, web_server=web_server)
         if p.enabled:
             p.start()
 
-    # Fetch, set up and start alert providers
+    # Create, set up and start alert providers
     for entry in config["alert-providers"]:
         alert_providers.append(get_alert_provider_from_config(entry))
     for p in alert_providers:
@@ -121,7 +124,7 @@ if __name__ == '__main__':
         if p.enabled:
             p.start()
 
-    # Fetch, set up and start solar conditions providers
+    # Create, set up and start solar conditions providers
     for entry in config.get("solar-condition-providers", []):
         solar_condition_providers.append(get_solar_conditions_provider_from_config(entry))
     for p in solar_condition_providers:

spotproviders/gma.py

@@ -58,8 +58,8 @@ class GMA(HTTPSpotProvider):
                     try:
                         ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
                                                                       headers=HTTP_HEADERS)
-                        # Sometimes this is blank, so handle that
-                        if ref_response.text is not None and ref_response.text != "":
+                        # Sometimes this is blank even if it's a 200 response, so handle that
+                        if ref_response.ok and ref_response.text is not None and ref_response.text != "":
                             ref_info = ref_response.json()
                             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
                             # spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
@@ -94,9 +94,17 @@ class GMA(HTTPSpotProvider):
                                         spot.sig_refs[0].sig = ref_info["reftype"]
                                         spot.sig = ref_info["reftype"]
 
-                                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
-                                # that for us.
+                                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point;
+                                # other code will do that for us.
                                 new_spots.append(spot)
+
+                        elif not ref_response.from_cache:
+                            if not ref_response.ok:
+                                logging.warning(
+                                    f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot["REF"]}")
+                            else:
+                                logging.warning(
+                                    f"GMA API returned a malformed response when looking up ref {source_spot["REF"]}")
                     except:
                         logging.warning("Exception when looking up " + self.REF_INFO_URL_ROOT + source_spot[
                             "REF"] + ", ignoring this spot for now")
@@ -104,3 +112,11 @@ class GMA(HTTPSpotProvider):
             logging.warning(f"The GMA API returned an unexpected response (HTTP {http_response.status_code}).")
 
         return new_spots
+
+    def can_submit_spot(self, sig):
+        return sig == "GMA"
+
+    def submit_spot(self, spot, credentials):
+        # TODO: Implement.
+        # Spotting to GMA is documented: https://www.cqgma.org/api/doc/apigma_spot.pdf We (or the user) need a GMA account, and to send the password in plaintext(!!)
+        raise NotImplementedError("GMA upstream spot submission is not yet implemented")

spotproviders/hema.py

@@ -1,8 +1,10 @@
+import logging
 import re
 from datetime import datetime
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
@@ -35,6 +37,7 @@ class HEMA(HTTPSpotProvider):
         new_spots = []
         # OK, if the spot seed actually changed, now we make the real request for data.
         if spot_seed_changed:
+            try:
                 source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30))
                 source_data_items = source_data.text.split("=")
                 # Iterate through source data items.
@@ -62,7 +65,19 @@ class HEMA(HTTPSpotProvider):
                                     dx_latitude=float(spot_items[7]),
                                     dx_longitude=float(spot_items[8]))
 
-                    # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
-                    # that for us.
+                        # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other
+                        # code will do that for us.
                         new_spots.append(spot)
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when accessing HEMA spots API.")
+            except ConnectionError:
+                logging.warning("Connection error when accessing HEMA spots API.")
         return new_spots
+
+    def can_submit_spot(self, sig):
+        return sig == "HEMA"
+
+    def submit_spot(self, spot, credentials):
+        # TODO: Implement. Currently blocked awaiting their API team to make a change to allow us to spot with a
+        # reference and not a reference *number*.
+        raise NotImplementedError("HEMA upstream spot submission is not yet implemented")

spotproviders/http_spot_provider.py

@@ -4,6 +4,8 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from spotproviders.spot_provider import SpotProvider
@@ -19,6 +21,7 @@ class HTTPSpotProvider(SpotProvider):
         self._poll_interval = poll_interval
         self._thread = None
         self._stop_event = Event()
+        self._wakeup_event = Event()
 
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
@@ -29,11 +32,19 @@ class HTTPSpotProvider(SpotProvider):
 
     def stop(self):
         self._stop_event.set()
+        self._wakeup_event.set()
+
+    def force_poll(self):
+        """Trigger an immediate poll without waiting for the normal interval."""
+
+        self._wakeup_event.set()
 
     def _run(self):
         while True:
+            self._wakeup_event.clear()
             self._poll()
-            if self._stop_event.wait(timeout=self._poll_interval):
+            self._wakeup_event.wait(timeout=self._poll_interval)
+            if self._stop_event.is_set():
                 break
 
     def _poll(self):
@@ -41,6 +52,8 @@ class HTTPSpotProvider(SpotProvider):
             # Request data from API
             logging.debug("Polling " + self.name + " spot API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            # Check response code was good
+            if http_response.ok:
                 # Pass off to the subclass for processing
                 new_spots = self._http_response_to_spots(http_response)
                 # Submit the new spots for processing. There might not be any spots for the less popular programs.
@@ -50,7 +63,14 @@ class HTTPSpotProvider(SpotProvider):
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
                 logging.debug("Received data from " + self.name + " spot API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} spot API.")
 
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} spots API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} spots API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP JSON Spot Provider (" + self.name + ")")

spotproviders/parksnpeaks.py

@@ -3,7 +3,9 @@ import re
 from datetime import datetime
 
 import pytz
+import requests
 
+from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -14,7 +16,9 @@ class ParksNPeaks(HTTPSpotProvider):
 
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
+    SUBMIT_URL = "https://www.parksnpeaks.org/api/SPOT/"
     SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
+    SUBMITTABLE_SIGS = ["POTA", "SOTA", "WWFF", "HEMA", "WOTA", "ZLOTA", "SIOTA", "KRMNPA"]
 
     def __init__(self, provider_config):
         super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -63,3 +67,28 @@ class ParksNPeaks(HTTPSpotProvider):
                 # Add new spot to the list
                 new_spots.append(spot)
         return new_spots
+
+    def can_submit_spot(self, sig):
+        return sig in self.SUBMITTABLE_SIGS
+
+    def submit_spot(self, spot, credentials):
+        # TODO test this works
+        user_id = credentials.get("user_id", "")
+        api_key = credentials.get("api_key", "")
+        if not user_id or not api_key:
+            raise ValueError(
+                "Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account.")
+        sig_ref = spot.sig_refs[0].id if spot.sig_refs else ""
+        body = {
+            "actClass": spot.sig or "",
+            "actCallsign": spot.dx_call,
+            "actSite": sig_ref,
+            "mode": spot.mode or "",
+            "freq": str(spot.freq / 1000000.0),
+            "comments": spot.comment or "",
+            "userID": user_id,
+            "APIKey": api_key,
+        }
+        response = requests.post(self.SUBMIT_URL, json=body, headers=HTTP_HEADERS, timeout=(5, 30))
+        if not response.ok:
+            raise RuntimeError("Parks N Peaks API returned " + str(response.status_code) + ": " + response.text)

spotproviders/pota.py

@@ -1,7 +1,9 @@
 from datetime import datetime
 
 import pytz
+import requests
 
+from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -12,6 +14,7 @@ class POTA(HTTPSpotProvider):
 
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://api.pota.app/spot/activator"
+    SUBMIT_URL = "https://api.pota.app/spot"
 
     def __init__(self, provider_config):
         super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -40,3 +43,25 @@ class POTA(HTTPSpotProvider):
             # that for us.
             new_spots.append(spot)
         return new_spots
+
+    def can_submit_spot(self, sig):
+        return sig == "POTA"
+
+    def submit_spot(self, spot, credentials):
+        sig_ref = spot.sig_refs[0].id if spot.sig_refs else None
+        if sig_ref:
+            body = {
+                "activator": spot.dx_call,
+                "spotter": spot.de_call,
+                "frequency": str(spot.freq / 1000.0),
+                "mode": spot.mode or "",
+                "reference": sig_ref,
+                "comments": spot.comment or "",
+                "source": "Spothole",
+            }
+            headers = {**HTTP_HEADERS, "Content-Type": "application/json"}
+            response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
+            if not response.ok:
+                raise RuntimeError("POTA API returned " + str(response.status_code) + ": " + response.text)
+        else:
+            raise RuntimeError("Park reference is required for submitting POTA spots.")

spotproviders/sota.py

@@ -1,8 +1,10 @@
+import logging
 from datetime import datetime
 
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
-from core.constants import HTTP_HEADERS
+from core.constants import HTTP_HEADERS, SSB_SUB_MODES, DV_SUB_MODES
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -20,6 +22,9 @@ class SOTA(HTTPSpotProvider):
     # SOTA spots don't contain lat/lon, we need a separate lookup for that
     SUMMIT_URL_ROOT = "https://api-db2.sota.org.uk/api/summits/"
 
+    SUBMIT_URL = "https://api-db2.sota.org.uk/api/spots"
+    VALID_MODES = ["AM", "CW", "Data", "DV", "FM", "SSB"]
+
     def __init__(self, provider_config):
         super().__init__(provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC)
         self._api_epoch = ""
@@ -33,6 +38,7 @@ class SOTA(HTTPSpotProvider):
         new_spots = []
         # OK, if the epoch actually changed, now we make the real request for data.
         if epoch_changed:
+            try:
                 source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30)).json()
                 # Iterate through source data
                 for source_spot in source_data:
@@ -55,4 +61,51 @@ class SOTA(HTTPSpotProvider):
                     # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
                     # that for us.
                     new_spots.append(spot)
+            except ConnectionError:
+                logging.warning("Connection error when accessing SOTA spots API")
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when accessing SOTA spots API.")
         return new_spots
+
+    def can_submit_spot(self, sig):
+        return sig == "SOTA"
+
+    def submit_spot(self, spot, credentials):
+        # TODO test this method works
+        access_token = credentials.get("access_token", "")
+        id_token = credentials.get("id_token", "")
+        if not access_token or not id_token:
+            raise ValueError("SOTA API tokens are required. Please log into SOTA in order to spot to it.")
+        sig_ref = spot.sig_refs[0].id if spot.sig_refs else ""
+        if sig_ref:
+            # Split reference into association and summit codes
+            ref_split = sig_ref.split("/")
+
+            # Figure out a valid mode. Borrowed this from PoLo :)
+            # https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js
+            mode = spot.mode
+            if mode and mode not in self.VALID_MODES:
+                if mode in SSB_SUB_MODES:
+                    mode = "SSB"
+                elif mode in DV_SUB_MODES:
+                    mode = "DV"
+                else:
+                    mode = "Data"
+
+            body = {
+                "activatorCallsign": spot.dx_call,
+                "associationCode": ref_split[0],
+                "summitCode": ref_split[1],
+                "frequency": spot.freq / 1000000.0,
+                "mode": mode or "",
+                "callsign": spot.de_call,
+                "comments": spot.comment or "",
+                "type": "TEST"  # todo replatce with NORMAL/QRT once testing complete
+            }
+            headers = {**HTTP_HEADERS, "Authorization": "bearer " + access_token, "id_token": id_token,
+                       "Content-Type": "application/json"}
+            response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
+            if not response.ok:
+                raise RuntimeError("SOTA API returned " + str(response.status_code) + ": " + response.text)
+        else:
+            raise RuntimeError("Summit reference is required for submitting SOTA spots.")

spotproviders/spot_provider.py

@@ -68,3 +68,20 @@ class SpotProvider:
         """Stop any threads and prepare for application shutdown"""
 
         raise NotImplementedError("Subclasses must implement this method")
+
+    def can_submit_spot(self, sig):
+        """Return True if this provider supports submitting spots upstream for the given SIG."""
+
+        return False
+
+    def submit_spot(self, spot, credentials):
+        """Submit a spot upstream to this provider's API. credentials is a dict with provider-specific keys.
+        Raises an exception with a descriptive message on failure."""
+
+        raise NotImplementedError("This provider does not support spot submission")
+
+    def force_poll(self):
+        """Trigger an immediate poll without waiting for the normal interval. Default implementation here does nothing
+        because not all spot providers have a polling mechanism. Providers that do should override this method."""
+
+        return

spotproviders/tiles.py

@@ -1,5 +1,8 @@
 from datetime import datetime
 
+import requests
+
+from core.constants import HTTP_HEADERS, SSB_SUB_MODES
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -10,6 +13,9 @@ class Tiles(HTTPSpotProvider):
 
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/spots?active_hours=24"
+    SUBMIT_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/self-spot"
+    VALID_MODES = ["SSB", "CW", "FT8", "FT4", "FM", "DMR", "D-STAR", "M17", "AX.25", "JS8Call", "PSK31", "Olivia",
+                   "VarAC", "Other"]
 
     def __init__(self, provider_config):
         super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -42,6 +48,49 @@ class Tiles(HTTPSpotProvider):
             new_spots.append(spot)
         return new_spots
 
+    def can_submit_spot(self, sig):
+        return sig == "Tiles"
+
+    def submit_spot(self, spot, credentials):
+        # Tiles on the air currently only supports *self* spots
+        if spot.dx_call == spot.de_call:
+
+            # Figure out a valid mode. Borrowed this from PoLo :)
+            # https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js
+            if spot.mode:
+                mode = spot.mode
+                if mode not in self.VALID_MODES:
+                    if mode in SSB_SUB_MODES:
+                        mode = "SSB"
+                    elif mode == "OLIVIA":
+                        mode = "Olivia"
+                    elif mode == "JS8":
+                        mode = "JS8Call"
+                    else:
+                        mode = "Other"
+
+                body = {
+                    "call_sign": spot.dx_call,
+                    "frequency": str(spot.freq / 1000000.0),
+                    "mode": mode or "",
+                    "grid": spot.dx_grid or "",
+                    "comment": spot.comment or "",
+                    "lat": spot.dx_latitude or None,
+                    "lon": spot.dx_longitude or None,
+                    "qrt": spot.qrt or False,
+                    "pin": credentials.get("offline_spot_gateway_pin", "")
+                }
+                headers = {**HTTP_HEADERS, "Content-Type": "application/json"}
+                response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
+                if not response.ok:
+                    raise RuntimeError(
+                        "Tiles on the Air API returned " + str(response.status_code) + ": " + response.text)
+            else:
+                raise RuntimeError("The Tiles on the Air API requires a mode to be set.")
+        else:
+            raise RuntimeError(
+                "The Tiles on the Air API only supports self-spots, the DX call and spotter call must match.")
+
 
 # Utility function to keep the first decimal point in a given string but remove any others. Used to parse Tiles'
 # strange frequency format where we can sometimes have e.g. "14.123.5".

spotproviders/wota.py

@@ -79,3 +79,10 @@ class WOTA(HTTPSpotProvider):
             except Exception as e:
                 logging.error("Exception parsing WOTA spot", e)
         return new_spots
+
+    def can_submit_spot(self, sig):
+        return sig == "WOTA"
+
+    def submit_spot(self, spot, credentials):
+        # TODO Ask M5TEA if he's happy to share how this is done from his app
+        raise NotImplementedError("WOTA upstream spot submission is not yet implemented")

spotproviders/wwbota.py

@@ -41,3 +41,10 @@ class WWBOTA(SSESpotProvider):
 
         # WWBOTA does support a special "Test" spot type, we need to avoid adding that.
         return spot if source_spot["type"] != "Test" else None
+
+    def can_submit_spot(self, sig):
+        return sig == "WWBOTA"
+
+    def submit_spot(self, spot, credentials):
+        # TODO: Implement. WWBOTA API docs cover this: https://api.wwbota.org/#tag/Spots/operation/create_spot_spots__post
+        raise NotImplementedError("WWBOTA upstream spot submission is not yet implemented")

spotproviders/wwff.py

@@ -38,3 +38,11 @@ class WWFF(HTTPSpotProvider):
             # that for us.
             new_spots.append(spot)
         return new_spots
+
+    def can_submit_spot(self, sig):
+        return sig == "WWFF"
+
+    def submit_spot(self, spot, credentials):
+        # TODO: Implement. Spotting to WWFF should be possible, need to look up the Spotline docs or copy approach from
+        # PoLo. Either way I think we need an API key for the app (but maybe not for the user?)
+        raise NotImplementedError("WWFF upstream spot submission is not yet implemented")

spotproviders/zlota.py

@@ -41,3 +41,10 @@ class ZLOTA(HTTPSpotProvider):
 
             new_spots.append(spot)
         return new_spots
+
+    def can_submit_spot(self, sig):
+        return sig == "ZLOTA"
+
+    def submit_spot(self, spot, credentials):
+        # TODO: Implement. Spotting to ZLOTA is supported via POST, see https://ontheair.nz/api
+        raise NotImplementedError("ZLOTA upstream spot submission is not yet implemented")

static/apidocs/openapi.yml

@@ -15,6 +15,14 @@ info:
 
     ## Changelog
     
+    ### 2.0
+
+    * POST `/spot` now supports upstream submission to the spotting services associated with various SIGs.
+    * **Breaking change:** The "add spot" API has changed to enable this: instead of just posting the spot object itself as the JSON content of the POST, this has moved into a `spot` object within the structure. A new `handling` object alongside it contains the `submit_upstream`, `upstream_provider`, `upstream_credentials`, and `captcha_token` fields which control the server handling of the spot.
+    * POST `/spot` now supports Google reCaptcha and (if the site owner has set it up) now requires `captcha_token` in order to successfully submit. (This is used to lock down the submit function and prevent submission via Spothole by bots or third-party clients.)
+    * GET `/options` now returns `spot_submit_providers`, a map of SIG names to the names of providers that support upstream spot submission for that SIG. (This allows clients to present the user with options of where a new spot can be sent to.)
+    * **Breaking change:** A user's QRZ.com and HamQTH credentials are now supplied as request headers (`X-QRZ-Username`, `X-QRZ-Password`, `X-QRZ-Session-Key`, `X-HamQTH-Username`, `X-HamQTH-Password`, `X-HamQTH-Session-ID`) rather than query parameters, to keep credentials out of server logs.
+
     ### 1.4
     
     * Spots can now include a "propagation_mode" field, and the `/options` call enumerates the options that can have.
@@ -45,10 +53,10 @@ info:
   license:
     name: The Unlicense
     url: https://unlicense.org/#the-unlicense
-  version: v1.4
+  version: 2.0
 
 servers:
-  - url: https://spothole.app/api/v1
+  - url: https://spothole.app/api/v2
 
 tags:
   - name: Spots
@@ -329,7 +337,8 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/ErrorResponse'
+                type: string
+                example: "Failed"
 
 
   /lookup/sigref:
@@ -357,7 +366,8 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/ErrorResponse'
+                type: string
+                example: "Failed"
 
 
 
@@ -383,7 +393,8 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/ErrorResponse'
+                type: string
+                example: "Failed"
 
 
   /spot:
@@ -392,50 +403,53 @@ paths:
         - Spots
       summary: Add a spot
       description: >
-        Supply a new spot object, which will be added to the system. Currently, this will not be
-        reported up the chain to a cluster, POTA, SOTA etc. This may be introduced in a future version.
-        cURL example: `curl --request POST --header "Content-Type: application/json" --data
-        '{"dx_call":"M0TRT","time":1760019539, "freq":14200000, "comment":"Test spot please ignore",
-        "de_call":"M0TRT"}' https://spothole.app/api/v1/spot`
+        Supply a JSON object containing a `spot` sub-object (the spot data) and an optional `handling` sub-object
+        containing server-side instructions such as upstream submission). Check `spot_submit_providers` in the
+        `/options` response to see which SIGs and providers support upstream submission. cURL example:
+        `curl --request POST --header \"Content-Type: application/json\" --data '{\"spot\":{\"dx_call\":\"M0TRT\",\"time\":1760019539,\"freq\":14200000,\"comment\":\"Test spot please ignore\",\"de_call\":\"M0TRT\"}}' https://spothole.app/api/v2/spot`"
       operationId: spot
       requestBody:
-        description: The JSON spot object
+        description: Object containing a "spot" sub-object with the spot data, and an optional "handling" sub-object with server-side instructions of what to do with it.
         required: true
         content:
           application/json:
             schema:
-              $ref: '#/components/schemas/Spot'
+              $ref: '#/components/schemas/SpotSubmission'
       responses:
-        '200':
+        '201':
           description: Success
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/OkResponse'
+                type: string
+                example: "OK"
         '415':
           description: Incorrect Content-Type
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/ErrorResponse'
+                type: string
+                example: "Failed"
         '422':
           description: Validation error
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/ErrorResponse'
+                type: string
+                example: "Failed"
         '500':
           description: Internal server error
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/ErrorResponse'
+                type: string
+                example: "Failed"
 
 components:
   parameters:
     QrzUsername:
-      name: qrz_username
-      in: query
+      name: X-QRZ-Username
+      in: header
       description: >
         QRZ.com username for online callsign lookup, which will enrich the returned spots and alerts
         with extra data. Requires a QRZ.com XML Subscriber (paid) account. Supply together with
@@ -443,14 +457,14 @@ components:
       schema:
         type: string
     QrzPassword:
-      name: qrz_password
-      in: query
+      name: X-QRZ-Password
+      in: header
       description: QRZ.com password. Supply together with `qrz_username`.
       schema:
         type: string
     QrzSessionKey:
-      name: qrz_session_key
-      in: query
+      name: X-QRZ-Session-Key
+      in: header
       description: >
         A pre-obtained QRZ.com XML session key, as an alternative to supplying `qrz_username` and
         `qrz_password`. See https://www.qrz.com/docs/xml/current_spec.html for details on how to
@@ -458,22 +472,22 @@ components:
       schema:
         type: string
     HamqthUsername:
-      name: hamqth_username
-      in: query
+      name: X-HamQTH-Username
+      in: header
       description: >
         HamQTH username for online callsign lookup, which will enrich the returned spots and alerts
         with extra data. Supply together with `hamqth_password`, or supply `hamqth_session_id` instead.
       schema:
         type: string
     HamqthPassword:
-      name: hamqth_password
-      in: query
+      name: X-HamQTH-Password
+      in: header
       description: HamQTH password. Supply together with `hamqth_username`.
       schema:
         type: string
     HamqthSessionId:
-      name: hamqth_session_id
-      in: query
+      name: X-HamQTH-Session-ID
+      in: header
       description: >
         A pre-obtained HamQTH session ID, as an alternative to supplying `hamqth_username` and
         `hamqth_password`. See https://www.hamqth.com/developers.php for details on how to retrieve
@@ -1184,6 +1198,54 @@ components:
           $ref: "#/components/schemas/PropagationMode"
 
 
+    SpotSubmission:
+      description: >
+        Request body for POST /spot. Contains a "spot" sub-object with the spot data, and an optional
+        "handling" sub-object with server-side instructions consumed by Spothole.
+      type: object
+      required:
+        - spot
+      properties:
+        spot:
+          $ref: '#/components/schemas/Spot'
+        handling:
+          type: object
+          description: >
+            Optional server-side instructions for how to process this spot submission.
+          properties:
+            submit_upstream:
+              type: boolean
+              description: >
+                If true, forward the spot to an external upstream provider (e.g. POTA, SOTA) rather
+                than only adding it to this Spothole server. Requires `sig`, at least one `sig_refs`
+                entry, and `upstream_provider` to be set. Check `spot_submit_providers` in the
+                /options response to see which SIGs and providers support this.
+              default: false
+            upstream_provider:
+              type: string
+              description: >
+                Name of the upstream provider to submit the spot to, e.g. "POTA" or "SOTA". Must
+                match one of the provider names returned in `spot_submit_providers` for the chosen SIG.
+              example: POTA
+            upstream_credentials:
+              type: object
+              description: >
+                Provider-specific credentials required to authenticate the upstream submission.
+                The required keys depend on the provider. Credentials are used only for the upstream
+                call and are never stored by Spothole.
+              additionalProperties:
+                type: string
+              example:
+                user_id: "12345"
+                api_key: "abc123"
+            captcha_token:
+              type: string
+              description: >
+                A Google reCAPTCHA v2 response token. Required when submitting upstream if the
+                server has reCAPTCHA configured. Obtain the token by completing the reCAPTCHA
+                widget rendered on the Add Spot page.
+              example: "03AFY_a8Xq..."
+
     SpotStream:
       type: object
       description: A server-sent event containing a spot
@@ -1730,14 +1792,6 @@ components:
       items:
         $ref: '#/components/schemas/Alert'
 
-    OkResponse:
-      type: string
-      example: "OK"
-
-    ErrorResponse:
-      type: string
-      example: "Failed"
-
     DxStats:
       type: object
       description: Spot counts keyed by DE continent
@@ -1875,7 +1929,7 @@ components:
           type: integer
           description: >
             The maximum age, in seconds, of any spot before it will be deleted by the system. When
-            querying the /api/v1/spots endpoint and providing a "max_age" or "since" parameter, there
+            querying the /api/v2/spots endpoint and providing a "max_age" or "since" parameter, there
             is no point providing a number larger than this, because the system drops all spots older
             than this.
           example: 3600
@@ -1885,6 +1939,20 @@ components:
             Whether the POST /spot call, to add spots to the server directly via its API, is permitted
             on this server.
           example: true
+        spot_submit_providers:
+          type: object
+          description: >
+            A map of SIG name to a list of provider names that support upstream spot submission for that SIG.
+            If a SIG appears as a key here, the POST /spot endpoint accepts `submit_upstream: true` for
+            spots with that SIG, and will forward the spot to one of the listed providers. Omitted if no
+            providers support upstream submission.
+          additionalProperties:
+            type: array
+            items:
+              type: string
+          example:
+            POTA: [ POTA ]
+            SOTA: [ SOTA, GMA, ParksNPeaks ]
 
     CallLookup:
       type: object