Merge branch 'main' into 95-send-spots-to-xota

# Conflicts:
#	README.md
#	config-example.yml
#	core/config.py
#	server/handlers/api/addspot.py
#	server/handlers/api/options.py
#	static/js/add-spot.js
#	static/js/bands.js
#	static/js/map.js

.idea/spothole.iml

@@ -2,8 +2,9 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
       <excludeFolder url="file://$MODULE_DIR$/.venv" />
-      <excludeFolder url="file://$MODULE_DIR$/webassets/vendor" />
+      <excludeFolder url="file://$MODULE_DIR$/static/vendor" />
     </content>
     <orderEntry type="jdk" jdkName="Python 3.13 virtualenv at ~/code/spothole/.venv" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />

README.md

@@ -1,4 +1,4 @@
-# ![Spothole](/webassets/img/logo.png)
+# ![Spothole](/static/img/logo.png)
 
 Spothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open
 JSON API as well as a website to browse the data.
@@ -89,15 +89,15 @@ callsign and park data.
 
 Various approaches exist to writing your own client, but in general:
 
-* Refer to the API docs. These are built on an OpenAPI definition file (`/webassets/apidocs/openapi.yml`), which you can
+* Refer to the API docs. These are built on an OpenAPI definition file (`/static/apidocs/openapi.yml`), which you can
   automatically use to generate a client skeleton using various software.
 * Call the main "spots" or "alerts" API endpoints to get the data you want. For example, your app could call
   `https://spothole.app/api/v2/spots` once every few minutes. Apply filters if necessary.
 * Call the "options" API to get an idea of which bands, modes etc. the server knows about. You might want to do that
   first before calling the spots/alerts APIs, to allow you to populate your filters correctly.
-* Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "map" and "bands"
-  pages simply query the main spot API on a timer, whereas the main/spots page combines this approach with using the
-  Server-Sent Events (SSE) endpoint to update live.
+* Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "alerts"/"upcoming"
+  page simply query the main spot API on a timer, whereas the spots, map and bands pages combine this approach with
+  using the Server-Sent Events (SSE) endpoint to update live.
 * Let me know if you get stuck, I'm happy to help.
 
 Please don't hammer the API with an unnecessarily high request rate. For example, Spothole only queries the POTA API
@@ -318,93 +318,78 @@ To set up nginx as a reverse proxy that sits in front of Spothole, first ensure
 
 Create a file at `/etc/nginx/sites-available/` called `spothole`. Give it the following contents, replacing
 `spothole.app` with the domain name on which you want to run Spothole. If you changed the port on which Spothole runs,
-update that on the "proxy_pass" line too.
+update that on the "proxy_pass" line, and if you installed Spothole somewhere other than `/home/spothole/spothole`,
+adjust the alias location for serving static files.
+
+(The latter section, configuring the nginx server to serve static files directly, improves efficiency because it saves
+Spothole itself from serving JS, CSS etc. files. If you can't do this for some reason, e.g. your nginx and spothole are
+on different computers, you can omit the `location /static/ {}` block.)
 
 ```nginx
 server {
     server_name spothole.app;
 
+    # Global proxy settings
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+    proxy_connect_timeout 10s;
+    proxy_buffering on;
+    
+    # Pass on IP address and host information to Spothole, in case logging this information is required
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Forwarded-Proto $scheme;
+
     # Wellknown area for Lets Encrypt
     location /.well-known/ {
         alias /var/www/html/.well-known/;
     }
+
+    # Load static assets directly from the Spothole static directory
+    location /static/ {
+        alias /home/spothole/spothole/static/;
+        expires 1h;
+        add_header Cache-Control "public, max-age=3600, must-revalidate";
+    }
     
     # SSE endpoints
-    location ~ ^/api/v2/(spots|alerts)/stream {
+    location ~ ^/api/v2/(spots|alerts)/stream/? {
         proxy_pass http://127.0.0.1:8080;
         
-        # Allow keep-alive
-        proxy_http_version 1.1;
-        proxy_set_header Connection "";
-        
-        # Set correct content type for SSE API calls
-        add_header Content-Type text/event-stream always;
-        
-        # Set remove buffering, remove caching, add suitable timeouts for SSE API calls
+        # Remove buffering, remove caching, add suitable timeouts for SSE API calls
         proxy_buffering off;
         proxy_cache off;
         proxy_read_timeout 24h;
-        proxy_connect_timeout 10s;
         proxy_send_timeout 24h;
         proxy_set_header X-Accel-Buffering no;
         add_header Cache-Control no-store always;
         
         # Allow cross-origin requests to API
         proxy_hide_header Access-Control-Allow-Origin;
-        add_header Access-Control-Allow-Origin * always;
-        
-        # Pass on IP address and host information to Spothole, in case logging this information is required
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto $scheme;
+        add_header Access-Control-Allow-Origin * always; 
     }
 
     # Other API endpoints
     location /api/ {
         proxy_pass http://127.0.0.1:8080;
         
-        # Allow keep-alive
-        proxy_http_version 1.1;
-        proxy_set_header Connection "";
-        
-        # Set up buffering, remove caching, add suitable timeouts for API calls
-        proxy_buffering on;
+        # Remove buffering, remove caching, add suitable timeouts for API calls
+        proxy_buffering off;
         proxy_cache off;
         proxy_read_timeout 30s;
-        proxy_connect_timeout 10s;
         add_header Cache-Control no-store always;
         
         # Allow cross-origin requests to API
         proxy_hide_header Access-Control-Allow-Origin;
         add_header Access-Control-Allow-Origin * always;
-        
-        # Pass on IP address and host information to Spothole, in case logging this information is required
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto $scheme;
     }
 
-    # Static assets
+    # Templated pages
     location / {
         proxy_pass http://127.0.0.1:8080;
-        
-        # Allow keep-alive
-        proxy_http_version 1.1;
-        proxy_set_header Connection "";
-        
-        # Set up buffering and caching, add suitable timeouts for static asset requests
-        proxy_buffering on;
         proxy_read_timeout 30s;
-        proxy_connect_timeout 10s;
-        add_header Cache-Control "public, max-age=3600, must-revalidate" always;
-        
-        # Pass on IP address and host information to Spothole, in case logging this information is required
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto $scheme;
+        add_header Cache-Control "no-cache, must-revalidate" always;
     }
 }
 ```
@@ -467,12 +452,12 @@ To navigate your way around the source code, this list may help.
 
 *HTML/JS/CSS front-end code*
 
-* `/webassets` - Root for static files served by the web server
-* `/webassets/apidocs` - Contains the OpenAPI spec (`openapi.yml`)
-* `/webassets/css` - CSS files used by the web front-end
-* `/webassets/img` - image files used by the web front-end
-* `/webassets/js` - JavaScript used by the web front-end
-* `/webassets/vendor` - Third-party libraries (CSS, JS, fonts and images)
+* `/static` - Root for static files served by the web server. These are all served from a path starting `/static/`.
+* `/static/apidocs` - Contains the OpenAPI spec (`openapi.yml`)
+* `/static/css` - CSS files used by the web front-end
+* `/static/img` - image files used by the web front-end
+* `/static/js` - JavaScript used by the web front-end
+* `/static/vendor` - Third-party libraries (CSS, JS, fonts and images)
 
 *Miscellaneous*
 
@@ -524,14 +509,14 @@ my knowledge, created by HA8TKS for his CQ and ITU zone layers for Leaflet. `/da
 maps of conference centres.
 
 The project contains a set of flag icons generated using the "Noto Color Emoji" font on a Debian system, in the
-`/webassets/img/flags/` directory.
+`/static/img/flags/` directory.
 
 The software uses a number of Python libraries as listed in `requirements.txt`, and a number of JavaScript libraries.
 This project would not have been possible without these libraries, so many thanks to their developers.
 
 ### Third Party Libraries
 
-A number of third-party libraries are self-hosted in the `/webassets/vendor/` directory. These files are subject to
+A number of third-party libraries are self-hosted in the `/static/vendor/` directory. These files are subject to
 their own licences and are not covered by the overall licence declared in the `LICENSE` file.
 
 A number of third-party libraries are self-hosted in the `/webassets/vendor/` directory. These files are subject to

alertproviders/http_alert_provider.py

@@ -4,6 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from alertproviders.alert_provider import AlertProvider
 from core.constants import HTTP_HEADERS
@@ -41,16 +42,25 @@ class HTTPAlertProvider(AlertProvider):
             # Request data from API
             logging.debug("Polling " + self.name + " alert API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
-            # Pass off to the subclass for processing
-            new_alerts = self._http_response_to_alerts(http_response)
-            # Submit the new alerts for processing. There might not be any alerts for the less popular programs.
-            if new_alerts:
-                self._submit_batch(new_alerts)
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                new_alerts = self._http_response_to_alerts(http_response)
+                # Submit the new alerts for processing. There might not be any alerts for the less popular programs.
+                if new_alerts:
+                    self._submit_batch(new_alerts)
 
-            self.status = "OK"
-            self.last_update_time = datetime.now(pytz.UTC)
-            logging.debug("Received data from " + self.name + " alert API.")
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.debug("Received data from " + self.name + " alert API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} alerts API.")
 
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} alerts API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} alerts API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP JSON Alert Provider (" + self.name + ")")

config-example.yml

@@ -225,6 +225,14 @@ allow-upstream-spotting: true
 recaptcha-site-key: ""
 recaptcha-secret-key: ""
 
+# Log web requests? Useful to see what your users are requesting in terms of pages and API endpoints, but will fill up
+# your log quickly on a popular server.
+log-web-requests: false
+
+# Minimum severity of log statements to print to the system log. Defaults to INFO, change to DEBUG if you need more
+# information.
+log-level: INFO
+
 # Options for the web UI.
 web-ui-options:
   spot-count: [ 10, 25, 50, 100 ]

core/cache_utils.py

@@ -8,8 +8,11 @@ from requests_cache import CachedSession
 # of time has passed. This is used throughout Spothole to cache data that does not change
 # rapidly. The ThreadSafeSession construct here protects it against some multithreading
 # contention weirdness we sometimes used to see on startup where the cache was hammered
-# pretty hard.
-_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30))
+# pretty hard. The expanded list of allowable_codes ensures we also cache and return 400-type
+# responses, e.g "this SOTA summit ref doesn't actually exist", to avoid hammering remote
+# servers for data they've told us they can't provide.
+_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30),
+                         allowable_codes=(200, 400, 401, 403, 404))
 _lock = threading.Lock()
 
 

core/config.py

@@ -25,6 +25,8 @@ WEB_UI_OPTIONS = config.get("web-ui-options", {})
 API_ONLY_MODE = config.get("api-only-mode", False)
 RECAPTCHA_SECRET_KEY = config.get("recaptcha-secret-key", "")
 RECAPTCHA_SITE_KEY = config.get("recaptcha-site-key", "")
+LOG_LEVEL = config.get("log-level", "INFO")
+LOG_WEB_REQUESTS = config.get("log-web-requests", False)
 
 # For ease of config, each spot provider owns its own config about whether it should be enabled by default in the web UI
 # but for consistency we provide this to the front-end in web-ui-options because it has no impact outside of the web UI.

core/constants.py

@@ -27,8 +27,8 @@ SIGS = [
     SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"),
     SIG(name="BOTA", comment_names=[], description="Beaches on the Air"),
     SIG(name="KRMNPA", comment_names=["KRMNPA"], description="Keith Roget Memorial National Parks Award"),
-    SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"[A-Z]{2}\-\d{4}"),
-    SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2}R\-\d{4}"),
+    SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"LL[A-Z]{2}\-\d{4}"),
+    SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2,3}R\-\d{4}"),
     SIG(name="Tiles", comment_names=[], description="Tiles on the Air", ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}"),
     SIG(name="WAB", comment_names=["WAB"], description="Worked All Britain", ref_regex=r"[A-Z]{1,2}[0-9]{2}"),
     SIG(name="WAI", comment_names=["WAI"], description="Worked All Ireland", ref_regex=r"[A-Z][0-9]{2}"),
@@ -104,5 +104,6 @@ PROPAGATION_MODES = {
     "AU": "Aurora",
     "MS": "Meteor scatter",
     "RS": "Rain scatter",
-    "AS": "Aircraft scatter"
+    "AS": "Aircraft scatter",
+    "SAT": "Satellite"
 }

core/lookup_helper.py

@@ -5,13 +5,13 @@ import re
 import urllib.parse
 from datetime import timedelta
 
-import requests
 import xmltodict
 from diskcache import Cache
 from pyhamtools import LookupLib, Callinfo, callinfo
 from pyhamtools.exceptions import APIKeyMissingError
 from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 from requests_cache import CachedSession
 
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
@@ -143,13 +143,21 @@ class LookupHelper:
         try:
             logging.info("Downloading Country-files.com cty.plist...")
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
-                                                      headers=HTTP_HEADERS).text
+                                                      headers=HTTP_HEADERS)
 
-            with open(self._country_files_cty_plist_download_location, "w") as f:
-                f.write(response)
-                f.flush()
-                return True
+            if response.ok:
+                with open(self._country_files_cty_plist_download_location, "w") as f:
+                    f.write(response.text)
+                    f.flush()
+                    return True
+            else:
+                logging.warning(f"HTTP {response.status_code} when downloading Country-files.com cty.plist.")
+                return False
 
+        except ConnectionError:
+            logging.warning(f"Connection error when downloading Clublog cty.xml.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when downloading Clublog cty.xml.")
         except Exception as e:
             logging.error("Exception when downloading Clublog cty.xml", e)
             return False
@@ -161,13 +169,21 @@ class LookupHelper:
             logging.info("Downloading dxcc.json...")
             response = SEMI_STATIC_URL_DATA_CACHE.get(
                 "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
-                headers=HTTP_HEADERS).text
+                headers=HTTP_HEADERS)
 
-            with open(self._dxcc_json_download_location, "w") as f:
-                f.write(response)
-                f.flush()
-                return True
+            if response.ok:
+                with open(self._dxcc_json_download_location, "w") as f:
+                    f.write(response.text)
+                    f.flush()
+                    return True
+            else:
+                logging.warning(f"HTTP {response.status_code} when downloading dxcc.json.")
+                return False
 
+        except ConnectionError:
+            logging.warning(f"Connection error when downloading dxcc.json.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when downloading dxcc.json.")
         except Exception as e:
             logging.error("Exception when downloading dxcc.json", e)
             return False
@@ -490,25 +506,47 @@ class LookupHelper:
 
         # Try the call as given, then fall back to the base call (strips /P, /M etc.)
         calls_to_try = [call]
-        home_call = callinfo.Callinfo.get_homecall(call)
-        if home_call != call:
-            calls_to_try.append(home_call)
+        try:
+            home_call = callinfo.Callinfo.get_homecall(call)
+            if home_call != call:
+                calls_to_try.append(home_call)
+        except ValueError:
+            logging.debug("Could not look up home call for callsign %s", call)
 
         for lookup_call in calls_to_try:
             try:
-                lookup_response = requests.get(
+                response = SEMI_STATIC_URL_DATA_CACHE.get(
                     self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
-                    headers=HTTP_HEADERS, timeout=10).content
-                raw = xmltodict.parse(lookup_response).get("QRZDatabase", {}).get("Callsign")
-                if raw:
-                    data = _normalize_qrz_data(raw)
-                    self._qrz_callsign_data_cache.add(call, data, expire=604800)  # 1 week in seconds
-                    return data
+                    headers=HTTP_HEADERS, timeout=10)
+                if response.ok:
+                    qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
+                    if qrz_response:
+                        if "Callsign" in qrz_response:
+                            data = _normalize_qrz_data(qrz_response.get("Callsign"))
+                            self._qrz_callsign_data_cache.add(call, data, expire=604800)  # 1 week in seconds
+                            return data
+                        elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
+                            # Errors here are normally just "callsign not in database", no need to log that ourselves
+                            # above debug level.
+                            logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call,
+                                          qrz_response.get("Session").get("Error"))
+
+                    elif not response.from_cache:
+                        logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call)
+                elif not response.from_cache:
+                    logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call)
+
             except (KeyError, ValueError):
                 continue
+            except ConnectionError:
+                logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call)
+                continue
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call)
+                continue
             except Exception:
-                logging.error("Exception when looking up QRZ data")
-                return None
+                logging.error("Exception when looking up callsign %s using QRZ", lookup_call, exc_info=True)
+                continue
 
         # Not found in QRZ; cache None so we don't keep retrying
         self._qrz_callsign_data_cache.add(call, None, expire=604800)  # 1 week in seconds
@@ -546,23 +584,36 @@ class LookupHelper:
 
         # Try the call as given, then fall back to the base call (strips /P, /M etc.)
         calls_to_try = [call]
-        home_call = callinfo.Callinfo.get_homecall(call)
-        if home_call != call:
-            calls_to_try.append(home_call)
+        try:
+            home_call = callinfo.Callinfo.get_homecall(call)
+            if home_call != call:
+                calls_to_try.append(home_call)
+        except ValueError:
+            logging.debug("Could not look up home call for callsign %s", call)
 
         for lookup_call in calls_to_try:
             try:
-                lookup_data = SEMI_STATIC_URL_DATA_CACHE.get(
+                response = SEMI_STATIC_URL_DATA_CACHE.get(
                     self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
-                        lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS).content
-                data = xmltodict.parse(lookup_data)["HamQTH"]["search"]
-                self._hamqth_callsign_data_cache.add(call, data, expire=604800)  # 1 week in seconds
-                return data
+                        lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
+                if response.ok:
+                    data = xmltodict.parse(response.content)["HamQTH"]["search"]
+                    self._hamqth_callsign_data_cache.add(call, data, expire=604800)  # 1 week in seconds
+                    return data
+                elif not response.from_cache:
+                    logging.warning("HTTP %d looking up callsign %s using HamQTH", response.status_code, lookup_call)
+
             except (KeyError, ValueError):
                 continue
+            except ConnectionError:
+                logging.warning(f"Connection error when looking up callsign %s using HamQTH", lookup_call)
+                continue
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when looking up callsign %s using HamQTH", lookup_call)
+                continue
             except Exception:
-                logging.error("Exception when looking up HamQTH data")
-                return None
+                logging.error("Exception when looking up callsign %s using HamQTH", lookup_call, exc_info=True)
+                continue
 
         # Not found in HamQTH; cache None so we don't keep retrying
         self._hamqth_callsign_data_cache.add(call, None, expire=604800)  # 1 week in seconds

core/sig_utils.py

@@ -2,6 +2,7 @@ import csv
 import logging
 
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import SIGS, HTTP_HEADERS
@@ -11,6 +12,14 @@ from core.geo_utils import wab_wai_square_to_lat_lon
 # file in Spothole and load it on startup.
 with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
     _DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
+# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
+# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
+# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
+_WWFF_INDEX_CACHE = {}
+_SIOTA_INDEX_CACHE = {}
+_WOTA_INDEX_CACHE = {}
+_ZLOTA_INDEX_CACHE = {}
+_LLOTA_INDEX_CACHE = {}
 
 
 def get_ref_regex_for_sig(sig):
@@ -45,92 +54,136 @@ def populate_sig_ref_info(sig_ref):
     try:
         if sig.upper() == "POTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
-            if not response.ok:
+            if response.ok:
+                data = response.json()
+                if data:
+                    fullname = str(data["name"]) if "name" in data else None
+                    if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
+                        fullname = fullname + " " + data["parktypeDesc"]
+                    sig_ref.name = fullname
+                    sig_ref.url = "https://pota.app/#/park/" + ref_id
+                    sig_ref.grid = data["grid6"] if "grid6" in data else None
+                    sig_ref.latitude = data["latitude"] if "latitude" in data else None
+                    sig_ref.longitude = data["longitude"] if "longitude" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
-            if data:
-                fullname = str(data["name"]) if "name" in data else None
-                if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
-                    fullname = fullname + " " + data["parktypeDesc"]
-                sig_ref.name = fullname
-                sig_ref.url = "https://pota.app/#/park/" + ref_id
-                sig_ref.grid = data["grid6"] if "grid6" in data else None
-                sig_ref.latitude = data["latitude"] if "latitude" in data else None
-                sig_ref.longitude = data["longitude"] if "longitude" in data else None
+
         elif sig.upper() == "SOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
+            if response.ok:
+                data = response.json()
+                if data:
+                    sig_ref.name = data["name"] if "name" in data else None
+                    sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
+                    sig_ref.grid = data["locator"] if "locator" in data else None
+                    sig_ref.latitude = data["latitude"] if "latitude" in data else None
+                    sig_ref.longitude = data["longitude"] if "longitude" in data else None
+                    sig_ref.activation_score = data["points"] if "points" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
-            if data:
-                sig_ref.name = data["name"] if "name" in data else None
-                sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
-                sig_ref.grid = data["locator"] if "locator" in data else None
-                sig_ref.latitude = data["latitude"] if "latitude" in data else None
-                sig_ref.longitude = data["longitude"] if "longitude" in data else None
-                sig_ref.activation_score = data["points"] if "points" in data else None
+
         elif sig.upper() == "WWBOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
+            if response.ok:
+                data = response.json()
+                if data:
+                    sig_ref.name = data["name"] if "name" in data else None
+                    sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
+                    sig_ref.grid = data["locator"] if "locator" in data else None
+                    sig_ref.latitude = data["lat"] if "lat" in data else None
+                    sig_ref.longitude = data["long"] if "long" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
-            if data:
-                sig_ref.name = data["name"] if "name" in data else None
-                sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
-                sig_ref.grid = data["locator"] if "locator" in data else None
-                sig_ref.latitude = data["lat"] if "lat" in data else None
-                sig_ref.longitude = data["long"] if "long" in data else None
+
         elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
+            if response.ok:
+                data = response.json()
+                if data:
+                    sig_ref.name = data["name"] if "name" in data else None
+                    sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
+                    sig_ref.grid = data["locator"] if "locator" in data else None
+
+                    # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
+                    # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
+                    # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
+                    # just use the single point we have.
+                    if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
+                            "lat2") is not None and data.get("lng2") is not None:
+                        sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
+                        sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
+                    else:
+                        sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
+                        sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
+            elif not response.from_cache:
                 logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
-            if data:
-                sig_ref.name = data["name"] if "name" in data else None
-                sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
-                sig_ref.grid = data["locator"] if "locator" in data else None
-                sig_ref.latitude = data["latitude"] if "latitude" in data else None
-                sig_ref.longitude = data["longitude"] if "longitude" in data else None
+
         elif sig.upper() == "WWFF":
-            wwff_response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
-                                                           headers=HTTP_HEADERS)
-            if not wwff_response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", wwff_response.status_code, sig, ref_id)
-                return sig_ref
-            wwff_index = {row["reference"]: row for row in csv.DictReader(wwff_response.content.decode().splitlines())}
-            row = wwff_index.get(ref_id)
-            if row:
-                sig_ref.name = row["name"] if "name" in row else None
-                sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
-                sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
-                sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
-                sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row["longitude"] != "-" else None
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
+                                                      headers=HTTP_HEADERS)
+            if response.ok:
+                global _WWFF_INDEX_CACHE
+                if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
+                    # New data from WWFF, update our internal map
+                    _WWFF_INDEX_CACHE = {row["reference"]: row for row in
+                                         csv.DictReader(response.content.decode().splitlines())}
+                row = _WWFF_INDEX_CACHE.get(ref_id)
+                if row:
+                    sig_ref.name = row["name"] if "name" in row else None
+                    sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
+                    sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
+                    sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
+                    sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row[
+                        "longitude"] != "-" else None
+                elif not response.from_cache:
+                    logging.warning("WWFF database did not contain data for ref %s", ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "SIOTA":
-            siota_response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
-                                                            headers=HTTP_HEADERS)
-            if not siota_response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", siota_response.status_code, sig, ref_id)
-                return sig_ref
-            siota_index = {row["SILO_CODE"]: row for row in
-                           csv.DictReader(siota_response.content.decode().splitlines())}
-            row = siota_index.get(ref_id)
-            if row:
-                sig_ref.name = row["NAME"] if "NAME" in row else None
-                sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
-                sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
-                sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
+                                                      headers=HTTP_HEADERS)
+            if response.ok:
+                global _SIOTA_INDEX_CACHE
+                if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
+                    # New data from SIOTA, update our internal map
+                    _SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
+                                          csv.DictReader(response.content.decode().splitlines())}
+                row = _SIOTA_INDEX_CACHE.get(ref_id)
+                if row:
+                    sig_ref.name = row["NAME"] if "NAME" in row else None
+                    sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
+                    sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
+                    sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
+                elif not response.from_cache:
+                    logging.warning("SIOTA database did not contain data for ref %s", ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "WOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
-            if data:
-                for feature in data.get("features", []):
-                    if feature["properties"]["wotaId"] == ref_id:
+            if response.ok:
+                data = response.json()
+                if data:
+                    global _WOTA_INDEX_CACHE
+                    if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from WOTA, update our internal map
+                        _WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
+                                             data.get("features", [])}
+                    feature = _WOTA_INDEX_CACHE.get(ref_id)
+                    if feature:
                         sig_ref.name = feature["properties"]["title"]
                         # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
                         # added to them
@@ -141,48 +194,68 @@ def populate_sig_ref_info(sig_ref):
                         sig_ref.grid = feature["properties"]["qthLocator"]
                         sig_ref.latitude = feature["geometry"]["coordinates"][1]
                         sig_ref.longitude = feature["geometry"]["coordinates"][0]
-                        break
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "ZLOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
-            if isinstance(data, list):
-                for asset in data:
-                    if asset["code"] == ref_id:
+            if response.ok:
+                data = response.json()
+                if isinstance(data, list):
+                    global _ZLOTA_INDEX_CACHE
+                    if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from ZLOTA, update our internal map
+                        _ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
+                    asset = _ZLOTA_INDEX_CACHE.get(ref_id)
+                    if asset:
                         sig_ref.name = asset["name"]
-                        sig_ref.url = "https://ontheair.nz/assets/ZLI_OT-030" + ref_id.replace("/", "_")
+                        sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
                         try:
                             sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
                         except:
                             logging.debug("Invalid lat/lon received for reference")
                             sig_ref.latitude = asset["y"]
                             sig_ref.longitude = asset["x"]
-                        break
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "BOTA":
             if not sig_ref.name:
                 sig_ref.name = sig_ref.id
             sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
+
         elif sig.upper() == "LLOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
-            if isinstance(data, list):
-                for ref in data:
-                    if ref["reference_code"] == ref_id:
+            if response.ok:
+                data = response.json()
+                if isinstance(data, list):
+                    global _LLOTA_INDEX_CACHE
+                    if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from LLOTA, update our internal map
+                        _LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
+                    ref = _LLOTA_INDEX_CACHE.get(ref_id)
+                    if ref:
                         sig_ref.name = str(ref["name"])
                         sig_ref.url = "https://llota.app/list/ref/" + ref_id
                         sig_ref.grid = str(ref["grid_locator"])
                         ll = locator_to_latlong(sig_ref.grid)
                         sig_ref.latitude = ll[0]
                         sig_ref.longitude = ll[1]
-                        break
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "WWTOTA":
             if not sig_ref.name:
                 sig_ref.name = sig_ref.id
             sig_ref.url = "https://wwtota.com/seznam/karta_rozhledny.php?ref=" + str(sig_ref.name)
+
         elif sig.upper() == "TILES":
             # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
             if not sig_ref.name:
@@ -193,6 +266,7 @@ def populate_sig_ref_info(sig_ref):
                 ll = locator_to_latlong(str(sig_ref.grid))
                 sig_ref.latitude = ll[0]
                 sig_ref.longitude = ll[1]
+
         elif sig.upper() == "WAB" or sig.upper() == "WAI":
             ll = wab_wai_square_to_lat_lon(ref_id)
             if ll:
@@ -202,21 +276,31 @@ def populate_sig_ref_info(sig_ref):
                     sig_ref.latitude = ll[0]
                     sig_ref.longitude = ll[1]
                 except:
-                    logging.debug("Invalid lat/lon received for reference")
+                    logging.warning("Invalid lat/lon received for WAB/WAI reference")
+
         elif sig.upper() == "DME":
             # Zero-pad to 5 digits to match our source data
             row = _DME_INDEX.get(ref_id.zfill(5))
             if row:
                 sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
-                sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get("LATITUD_ETRS89_REGCAN95") else None
-                sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get("LONGITUD_ETRS89_REGCAN95") else None
+                sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LATITUD_ETRS89_REGCAN95") else None
+                sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LONGITUD_ETRS89_REGCAN95") else None
                 if sig_ref.latitude and sig_ref.longitude:
                     try:
                         sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6)
                     except Exception:
-                        logging.debug("Invalid lat/lon received for reference")
+                        logging.warning("Invalid lat/lon received for DME reference")
+            else:
+                logging.warning("DME database did not contain data for ref %s", ref_id)
+
+    except ConnectionError:
+        logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
+    except (ConnectTimeout, ReadTimeout):
+        logging.warning(f"Timeout when looking up sig_ref info for " + sig + " ref " + ref_id)
     except Exception:
-        logging.warning("Failed to look up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
+        logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
     return sig_ref
 
 

core/utils.py

@@ -1,8 +1,11 @@
-def serialize_everything(obj):
-    """Convert objects to serialisable things. Used by JSON serialiser as a default when it encounters unserializable things.
-    Just converts objects to dict. Try to avoid doing anything clever here when serialising spots, because we also need
-    to receive spots without complex handling."""
-    return obj.__dict__
+import simplejson
+
+
+def safe_json_dumps(obj):
+    """Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats
+    which are invalid in JSON."""
+
+    return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
 
 
 def empty_queue(q):

data/spot.py

@@ -133,6 +133,17 @@ class Spot:
     # The ID the source gave it, if any.
     source_id: str | None = None
 
+    def __post_init__(self):
+        """Normalise fields that don't survive a plain dict to Spot conversion. This is used in the "add spot" API
+        endpoint where the client is submitting JSON, and we want to recreate a full Spot object, including nested
+        objects such as the sig_refs list.."""
+
+        if self.sig_refs:
+            self.sig_refs = [
+                sig_ref if isinstance(sig_ref, SIGRef) else SIGRef(**sig_ref)
+                for sig_ref in self.sig_refs
+            ]
+
     def infer_missing(self, credentials=None):
         """Infer missing parameters where possible"""
 
@@ -312,7 +323,7 @@ class Spot:
                         self.propagation_mode = PROPAGATION_MODES[mode_tag]
                     else:
                         self.propagation_mode = mode_tag
-                        logging.info("Seen a new propagation mode tag not yet in the system: {}", mode_tag)
+                        logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag)
 
         # DX Grid to lat/lon and vice versa in case one is missing
         if self.dx_grid and not self.dx_latitude:

requirements.txt

@@ -16,4 +16,5 @@ beautifulsoup4~=4.14.2
 websocket-client~=1.8.0
 tornado~=6.4.2
 tornado_eventsource~=3.0.0
-geopandas~=0.13.2
+geopandas~=0.13.2
+simplejson~=4.1.1

server/handlers/api/addspot.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import re
 import threading
@@ -16,8 +15,7 @@ from core.constants import UNKNOWN_BAND
 from core.lookup_helper import infer_band_from_freq
 from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
-from core.utils import serialize_everything
-from data.sig_ref import SIGRef
+from core.utils import safe_json_dumps
 from data.spot import Spot
 from spotproviders.spot_provider import SpotProvider
 
@@ -49,8 +47,7 @@ class APISpotHandler(tornado.web.RequestHandler):
             # Reject if not allowed
             if not ALLOW_SPOTTING:
                 self.set_status(401)
-                self.write(json.dumps("Error - this server does not allow new spots to be added via the API.",
-                                      default=serialize_everything))
+                self.write(safe_json_dumps("Error - this server does not allow new spots to be added via the API."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -58,8 +55,7 @@ class APISpotHandler(tornado.web.RequestHandler):
             # Reject if format not json
             if not self.request.headers.get('Content-Type', '').startswith("application/json"):
                 self.set_status(415)
-                self.write(
-                    json.dumps("Error - request Content-Type must be application/json", default=serialize_everything))
+                self.write(safe_json_dumps("Error - request Content-Type must be application/json"))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -68,7 +64,7 @@ class APISpotHandler(tornado.web.RequestHandler):
             post_data = self.request.body
             if not post_data:
                 self.set_status(422)
-                self.write(json.dumps("Error - request body is empty", default=serialize_everything))
+                self.write(safe_json_dumps("Error - request body is empty"))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -106,20 +102,10 @@ class APISpotHandler(tornado.web.RequestHandler):
             # Convert spot field to a Spot object
             spot = Spot(**spot_data)
 
-            # Converting to a spot object this way won't have coped with sig_ref objects, so fix that. (Would be nice to
-            # redo this in a functional style)
-            if spot.sig and spot.sig_refs:
-                real_sig_refs = []
-                for dict_obj in spot.sig_refs:
-                    dict_obj = {**dict_obj, "sig": spot.sig}
-                    real_sig_refs.append(json.loads(json.dumps(dict_obj), object_hook=lambda d: SIGRef(**d)))
-                spot.sig_refs = real_sig_refs
-
             # Reject if no timestamp, frequency, dx_call or de_call
             if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
                 self.set_status(422)
-                self.write(json.dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.",
-                                      default=serialize_everything))
+                self.write(safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -127,15 +113,13 @@ class APISpotHandler(tornado.web.RequestHandler):
             # Reject invalid-looking callsigns
             if not re.match(r"^[A-Za-z0-9/\-]*$", spot.dx_call):
                 self.set_status(422)
-                self.write(json.dumps("Error - '" + spot.dx_call + "' does not look like a valid callsign.",
-                                      default=serialize_everything))
+                self.write(safe_json_dumps("Error - '" + spot.dx_call + "' does not look like a valid callsign."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
             if not re.match(r"^[A-Za-z0-9/\-]*$", spot.de_call):
                 self.set_status(422)
-                self.write(json.dumps("Error - '" + spot.de_call + "' does not look like a valid callsign.",
-                                      default=serialize_everything))
+                self.write(safe_json_dumps("Error - '" + spot.de_call + "' does not look like a valid callsign."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -143,8 +127,7 @@ class APISpotHandler(tornado.web.RequestHandler):
             # Reject if frequency not in a known band
             if infer_band_from_freq(spot.freq) == UNKNOWN_BAND:
                 self.set_status(422)
-                self.write(json.dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band.",
-                                      default=serialize_everything))
+                self.write(safe_json_dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -154,8 +137,7 @@ class APISpotHandler(tornado.web.RequestHandler):
                     r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$",
                     spot.dx_grid.upper()):
                 self.set_status(422)
-                self.write(json.dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid.",
-                                      default=serialize_everything))
+                self.write(safe_json_dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -164,9 +146,8 @@ class APISpotHandler(tornado.web.RequestHandler):
             if spot.sig and spot.sig_refs and len(spot.sig_refs) > 0 and spot.sig_refs[0].id and get_ref_regex_for_sig(
                     spot.sig) and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id):
                 self.set_status(422)
-                self.write(json.dumps(
-                    "Error - '" + spot.sig_refs[0].id + "' does not look like a valid reference for " + spot.sig + ".",
-                    default=serialize_everything))
+                self.write(safe_json_dumps(
+                    "Error - '" + spot.sig_refs[0].id + "' does not look like a valid reference for " + spot.sig + "."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -241,14 +222,14 @@ class APISpotHandler(tornado.web.RequestHandler):
                 self.write(json.dumps("Warning - " + upstream_warning, default=serialize_everything))
                 self.set_status(201)
             else:
-                self.write(json.dumps("OK", default=serialize_everything))
+                self.write(safe_json_dumps("OK"))
                 self.set_status(201)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
 
         except Exception as e:
-            logging.error(e)
-            self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
+            logging.error("Exception when handling client request to add spot API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")

server/handlers/api/alerts.py

@@ -1,5 +1,4 @@
 import copy
-import json
 import logging
 from datetime import datetime
 from queue import Queue
@@ -12,7 +11,7 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import serialize_everything, empty_queue
+from core.utils import safe_json_dumps, empty_queue
 from data.lookup_credentials import extract_credentials
 
 SSE_HANDLER_MAX_QUEUE_SIZE = 100
@@ -57,15 +56,14 @@ class APIAlertsHandler(tornado.web.RequestHandler):
             data = get_alert_list_with_filters(self._alerts, query_params)
             if credentials:
                 data = self._enrich(data, credentials)
-            self.write(json.dumps(data, default=serialize_everything))
+            self.write(safe_json_dumps(data))
             self.set_status(200)
         except ValueError as e:
-            logging.error(e)
-            self.write(json.dumps("Bad request - " + str(e), default=serialize_everything))
+            self.write(safe_json_dumps("Bad request - " + str(e)))
             self.set_status(400)
         except Exception as e:
-            logging.error(e)
-            self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
+            logging.error("Exception when handling client request to alerts API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
         self.set_header("Cache-Control", "no-store")
         self.set_header("Content-Type", "application/json")
@@ -150,7 +148,7 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
                             if self._credentials:
                                 alert = copy.deepcopy(alert)
                                 alert.infer_missing(self._credentials)
-                            self.write_message(msg=json.dumps(alert, default=serialize_everything))
+                            self.write_message(msg=safe_json_dumps(alert))
 
                 else:
                     # Send a keepalive comment if the queue was empty

server/handlers/api/dxstats.py

@@ -1,4 +1,5 @@
 import json
+import logging
 from collections import Counter
 from datetime import datetime, timedelta
 from typing import Any
@@ -9,6 +10,7 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
 
 CONTINENTS = ["EU", "NA", "SA", "AS", "AF", "OC", "AN"]
 BANDS = ["160m", "80m", "60m", "40m", "30m", "20m", "17m", "15m", "12m", "10m", "6m"]
@@ -29,29 +31,35 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
         self._web_server_metrics = web_server_metrics
 
     def get(self):
-        self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-        self._web_server_metrics["api_access_counter"] += 1
-        self._web_server_metrics["status"] = "OK"
-        api_requests_counter.inc()
+        try:
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
 
-        one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp()
-        counts = Counter()
+            one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp()
+            counts = Counter()
 
-        for key in self._spots.iterkeys():
-            spot = self._spots.get(key)
-            if spot is None:
-                continue
-            if not spot.time or spot.time < one_hour_ago:
-                continue
-            if spot.de_continent in CONTINENTS_SET and spot.dx_continent in CONTINENTS_SET and spot.band in BANDS_SET:
-                counts[spot.de_continent, spot.dx_continent, spot.band] += 1
+            for key in self._spots.iterkeys():
+                spot = self._spots.get(key)
+                if spot is None:
+                    continue
+                if not spot.time or spot.time < one_hour_ago:
+                    continue
+                if spot.de_continent in CONTINENTS_SET and spot.dx_continent in CONTINENTS_SET and spot.band in BANDS_SET:
+                    counts[spot.de_continent, spot.dx_continent, spot.band] += 1
 
-        result = {
-            de: {dx: {band: counts[de, dx, band] for band in BANDS} for dx in CONTINENTS}
-            for de in CONTINENTS
-        }
+            result = {
+                de: {dx: {band: counts[de, dx, band] for band in BANDS} for dx in CONTINENTS}
+                for de in CONTINENTS
+            }
 
-        self.write(json.dumps(result))
-        self.set_status(200)
-        self.set_header("Cache-Control", "no-store")
-        self.set_header("Content-Type", "application/json")
+            self.write(json.dumps(result))
+            self.set_status(200)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to dx stats API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/api/lookups.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import re
 from datetime import datetime
@@ -13,7 +12,7 @@ from core.constants import SIGS
 from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone
 from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig, populate_sig_ref_info
-from core.utils import serialize_everything
+from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 from data.sig_ref import SIGRef
 from data.spot import Spot
@@ -65,19 +64,18 @@ class APILookupCallHandler(tornado.web.RequestHandler):
                         "longitude": fake_spot.dx_longitude,
                         "location_source": fake_spot.dx_location_source
                     }
-                    self.write(json.dumps(data, default=serialize_everything))
+                    self.write(safe_json_dumps(data))
 
                 else:
-                    self.write(json.dumps("Error - '" + call + "' does not look like a valid callsign.",
-                                          default=serialize_everything))
+                    self.write(safe_json_dumps("Error - '" + call + "' does not look like a valid callsign."))
                     self.set_status(422)
             else:
-                self.write(json.dumps("Error - call must be provided", default=serialize_everything))
+                self.write(safe_json_dumps("Error - call must be provided"))
                 self.set_status(422)
 
         except Exception as e:
-            logging.error(e)
-            self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
+            logging.error("Exception when handling client request to call lookup API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
 
         self.set_header("Cache-Control", "no-store")
@@ -114,24 +112,22 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
                 if sig in list(map(lambda p: p.name.upper(), SIGS)):
                     if not get_ref_regex_for_sig(sig) or re.match(get_ref_regex_for_sig(sig), ref_id):
                         data = populate_sig_ref_info(SIGRef(id=ref_id, sig=sig))
-                        self.write(json.dumps(data, default=serialize_everything))
+                        self.write(safe_json_dumps(data))
 
                     else:
-                        self.write(
-                            json.dumps(
-                                "Error - '" + ref_id + "' does not look like a valid reference ID for " + sig + ".",
-                                default=serialize_everything))
+                        self.write(safe_json_dumps(
+                                "Error - '" + ref_id + "' does not look like a valid reference ID for " + sig + "."))
                         self.set_status(422)
                 else:
-                    self.write(json.dumps("Error - sig '" + sig + "' is not known.", default=serialize_everything))
+                    self.write(safe_json_dumps("Error - sig '" + sig + "' is not known."))
                     self.set_status(422)
             else:
-                self.write(json.dumps("Error - sig and id must be provided", default=serialize_everything))
+                self.write(safe_json_dumps("Error - sig and id must be provided"))
                 self.set_status(422)
 
         except Exception as e:
-            logging.error(e)
-            self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
+            logging.error("Exception when handling client request to sig ref lookup API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
 
         self.set_header("Cache-Control", "no-store")
@@ -185,15 +181,15 @@ class APILookupGridHandler(tornado.web.RequestHandler):
                             "latitude": lat + lat_cell_size,
                             "longitude": lon + lon_cell_size,
                         }}
-                    self.write(json.dumps(response, default=serialize_everything))
+                    self.write(safe_json_dumps(response))
 
             else:
-                self.write(json.dumps("Error - grid must be provided", default=serialize_everything))
+                self.write(safe_json_dumps("Error - grid must be provided"))
                 self.set_status(422)
 
         except Exception as e:
-            logging.error(e)
-            self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
+            logging.error("Exception when handling client request to grid ref lookup API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
 
         self.set_header("Cache-Control", "no-store")

server/handlers/api/options.py

@@ -1,4 +1,4 @@
-import json
+import logging
 from datetime import datetime
 from typing import Any
 
@@ -10,7 +10,7 @@ from tornado.web import Application
 from core.config import MAX_SPOT_AGE, ALLOW_SPOTTING
 from core.constants import BANDS, ALL_MODES, MODE_TYPES, SIGS, CONTINENTS, PROPAGATION_MODES
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import serialize_everything
+from core.utils import safe_json_dumps
 
 
 class APIOptionsHandler(tornado.web.RequestHandler):
@@ -28,19 +28,20 @@ class APIOptionsHandler(tornado.web.RequestHandler):
         self._spot_providers = spot_providers or []
 
     def get(self):
-        # Metrics
-        self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-        self._web_server_metrics["api_access_counter"] += 1
-        self._web_server_metrics["status"] = "OK"
-        api_requests_counter.inc()
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
 
-        # Build a map of SIG name -> list of provider names that can submit spots for that SIG
+            # Build a map of SIG name -> list of provider names that can submit spots for that SIG
         spot_submit_providers = {}
         for provider in self._spot_providers:
             if not provider.enabled:
                 continue
             for sig in SIGS:
-                if provider.can_submit_spot(sig.name):
+                    if provider.can_submit_spot(sig.name):
                     spot_submit_providers.setdefault(sig.name, []).append(provider.name)
 
         # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle
@@ -51,8 +52,8 @@ class APIOptionsHandler(tornado.web.RequestHandler):
             map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["alert_providers"])))
         # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from
         # one of our providers.
-        if ALLOW_SPOTTING:
-            spot_sources.append("API")
+            if ALLOW_SPOTTING:
+                spot_sources.append("API")
 
         options = {"bands": BANDS,
                    "modes": ALL_MODES,
@@ -66,7 +67,12 @@ class APIOptionsHandler(tornado.web.RequestHandler):
                    "spot_allowed": ALLOW_SPOTTING,
                    "spot_submit_providers": spot_submit_providers}
 
-        self.write(json.dumps(options, default=serialize_everything))
-        self.set_status(200)
-        self.set_header("Cache-Control", "no-store")
-        self.set_header("Content-Type", "application/json")
+            self.write(safe_json_dumps(options))
+            self.set_status(200)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to options API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/api/solar_conditions.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 from typing import Any
 
@@ -7,6 +8,7 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
 
 
 class APISolarConditionsHandler(tornado.web.RequestHandler):
@@ -22,13 +24,19 @@ class APISolarConditionsHandler(tornado.web.RequestHandler):
         self._web_server_metrics = web_server_metrics
 
     def get(self):
-        # Metrics
-        self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-        self._web_server_metrics["api_access_counter"] += 1
-        self._web_server_metrics["status"] = "OK"
-        api_requests_counter.inc()
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
 
-        self.write(self._solar_conditions.to_json())
-        self.set_status(200)
-        self.set_header("Cache-Control", "no-store")
-        self.set_header("Content-Type", "application/json")
+            self.write(self._solar_conditions.to_json())
+            self.set_status(200)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to solar conditions API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/api/spots.py

@@ -1,5 +1,4 @@
 import copy
-import json
 import logging
 from datetime import datetime, timedelta
 from queue import Queue
@@ -12,7 +11,7 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import serialize_everything, empty_queue
+from core.utils import safe_json_dumps, empty_queue
 from data.lookup_credentials import extract_credentials
 
 SSE_HANDLER_MAX_QUEUE_SIZE = 1000
@@ -57,15 +56,14 @@ class APISpotsHandler(tornado.web.RequestHandler):
             data = get_spot_list_with_filters(self._spots, query_params)
             if credentials:
                 data = self._enrich(data, credentials)
-            self.write(json.dumps(data, default=serialize_everything))
+            self.write(safe_json_dumps(data))
             self.set_status(200)
         except ValueError as e:
-            logging.error(e)
-            self.write(json.dumps("Bad request - " + str(e), default=serialize_everything))
+            self.write(safe_json_dumps("Bad request - " + str(e)))
             self.set_status(400)
         except Exception as e:
-            logging.error(e)
-            self.write(json.dumps("Error - an internal server error occurred.", default=serialize_everything))
+            logging.error("Excedption when handling client request to spots API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
         self.set_header("Cache-Control", "no-store")
         self.set_header("Content-Type", "application/json")
@@ -152,7 +150,7 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
                             if self._credentials:
                                 spot = copy.deepcopy(spot)
                                 spot.infer_missing(self._credentials)
-                            self.write_message(msg=json.dumps(spot, default=serialize_everything))
+                            self.write_message(msg=safe_json_dumps(spot))
 
                 else:
                     # Send a keepalive comment if the queue was empty

server/handlers/api/status.py

@@ -1,4 +1,4 @@
-import json
+import logging
 from datetime import datetime
 from typing import Any
 
@@ -8,7 +8,7 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import serialize_everything
+from core.utils import safe_json_dumps
 
 
 class APIStatusHandler(tornado.web.RequestHandler):
@@ -24,13 +24,19 @@ class APIStatusHandler(tornado.web.RequestHandler):
         self._web_server_metrics = web_server_metrics
 
     def get(self):
-        # Metrics
-        self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
-        self._web_server_metrics["api_access_counter"] += 1
-        self._web_server_metrics["status"] = "OK"
-        api_requests_counter.inc()
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
 
-        self.write(json.dumps(self._status_data, default=serialize_everything))
-        self.set_status(200)
-        self.set_header("Cache-Control", "no-store")
-        self.set_header("Content-Type", "application/json")
+            self.write(safe_json_dumps(self._status_data))
+            self.set_status(200)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to status API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/manifesthandler.py

@@ -0,0 +1,11 @@
+import tornado.web
+
+from core.config import BASE_URL
+
+
+class ManifestHandler(tornado.web.RequestHandler):
+    """Handler for manifest.webmanifest, which needs BASE_URL inserted and a custom content-type"""
+
+    def get(self):
+        self.set_header("Content-Type", "application/manifest+json; charset=UTF-8")
+        self.render("manifest.webmanifest", baseurl=BASE_URL)

server/webserver.py

@@ -5,7 +5,7 @@ import os
 import tornado
 from tornado.web import StaticFileHandler
 
-from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE
+from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL
 from core.utils import empty_queue
 from server.handlers.api.addspot import APISpotHandler
 from server.handlers.api.v1_compatability import V1RedirectHandler, V1GoneHandler
@@ -16,6 +16,7 @@ from server.handlers.api.options import APIOptionsHandler
 from server.handlers.api.solar_conditions import APISolarConditionsHandler
 from server.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
 from server.handlers.api.status import APIStatusHandler
+from server.handlers.manifesthandler import ManifestHandler
 from server.handlers.metrics import PrometheusMetricsHandler
 from server.handlers.pagetemplate import PageTemplateHandler
 
@@ -110,11 +111,16 @@ class WebServer:
             if ALLOW_SPOTTING:
                 ui_routes += [(r"/add-spot", PageTemplateHandler, {"template_name": "add_spot", **handler_opts})]
 
-        # API docs, Prometheus metrics, and finally static assets are always available regardless of API-only mode.
+        # API docs, Prometheus metrics, webapp manifest and static assets are always available regardless of API-only
+        # mode.
         misc_routes = [
             (r"/apidocs", PageTemplateHandler, {"template_name": "apidocs", **handler_opts}),
             (r"/metrics", PrometheusMetricsHandler),
-            (r"/(.*)", StaticFileHandler, {"path": os.path.join(_HERE, "../webassets")})
+            (r"/manifest.webmanifest", ManifestHandler),
+            # If e.g. nginx is configured as a reverse proxy with a hard-coded path to static files, as per the README,
+            # this will never have to handle anything, but having it here allows Spothole to work without nginx for
+            # testing.
+            (r"/static/(.*)", StaticFileHandler, {"path": os.path.join(_HERE, "../static")})
         ]
 
         app = tornado.web.Application(api_routes + v1_compat_routes + ui_routes + misc_routes,
@@ -123,6 +129,7 @@ class WebServer:
                                       debug=False)
         app.listen(self._port, xheaders=True)
         logging.info("Web server running on port " + str(WEB_SERVER_PORT))
+        logging.info("You can access your copy of Spothole at " + BASE_URL)
         await self._shutdown_event.wait()
 
     def notify_new_spot(self, spot):
@@ -176,20 +183,22 @@ class WebServer:
         pass
 
 def request_log(handler):
-    """Custom log function to provide more data about requests."""
+    """Custom log function to provide more data about requests when enabled, and to provide the ability to turn off
+    web request logging altogetether."""
 
-    if handler.get_status() < 500:
-        log_method = logging.info
-    else:
-        log_method = logging.warning
+    if LOG_WEB_REQUESTS:
+        if handler.get_status() < 500:
+            log_method = logging.info
+        else:
+            log_method = logging.warning
 
-    request = handler.request
-    client_ip = request.remote_ip
-    referrer = request.headers.get("Referer", "-")
-    user_agent = request.headers.get("User-Agent", "-")
+        request = handler.request
+        client_ip = request.remote_ip
+        referrer = request.headers.get("Referer", "-")
+        user_agent = request.headers.get("User-Agent", "-")
 
-    log_method(
-        f'{client_ip} - "{request.method} {request.uri}" '
-        f'{handler.get_status()} {request.request_time():.2f}ms | '
-        f'Ref: {referrer} | UA: {user_agent}'
-    )
+        log_method(
+            f'{client_ip} - "{request.method} {request.uri}" '
+            f'{handler.get_status()} {request.request_time():.2f}ms | '
+            f'Ref: {referrer} | UA: {user_agent}'
+        )

solarconditionsproviders/giroionosonde.py

@@ -5,6 +5,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.ionosonde_utils import compute_band_states
@@ -127,10 +128,18 @@ class GIROIonosonde(SolarConditionsProvider):
         from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S")
         to_str = to_time.strftime("%Y.%m.%d+%H:%M:%S")
         url = f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUFD,fmin&DMUF=3000&fromDate={from_str}&toDate={to_str}"
-        response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15))
-        if response.status_code != 200:
+        try:
+            http_response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15))
+            if not http_response.ok:
+                logging.warning(f"HTTP {http_response.status_code} when calling Giro ionosonde API.")
+                return None, None, None
+            return self._parse_all(http_response.text)
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing Giro ionosonde API.")
+            return None, None, None
+        except ConnectionError:
+            logging.warning("Connection error when accessing Giro ionosonde API.")
             return None, None, None
-        return self._parse_all(response.text)
 
     @staticmethod
     def _parse_all(text):

solarconditionsproviders/hamqsl.py

@@ -18,10 +18,6 @@ class HamQSL(HTTPSolarConditionsProvider):
         super().__init__(provider_config, URL, POLL_INTERVAL)
 
     def _http_response_to_solar_conditions(self, http_response):
-        if http_response.status_code != 200:
-            logging.warning("HamQSL solar conditions API returned HTTP " + str(http_response.status_code))
-            return None
-
         root = ElementTree.fromstring(http_response.text)
         sd = root.find("solardata")
         if sd is None:

solarconditionsproviders/http_solar_conditions_provider.py

@@ -4,6 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.solar_conditions_provider import SolarConditionsProvider
@@ -39,13 +40,22 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
         try:
             logging.debug("Polling " + self.name + " solar conditions API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
-            new_data = self._http_response_to_solar_conditions(http_response)
-            self.update_data(new_data)
+            # Check response code was good
+            if http_response.ok:
+                new_data = self._http_response_to_solar_conditions(http_response)
+                self.update_data(new_data)
 
-            self.status = "OK"
-            self.last_update_time = datetime.now(pytz.UTC)
-            logging.debug("Received data from " + self.name + " solar conditions API.")
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.debug("Received data from " + self.name + " solar conditions API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} solar conditions API.")
 
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} solar conditions API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} solar conditions API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP Solar Conditions Provider (" + self.name + ")")

solarconditionsproviders/kc2gprop.py

@@ -4,6 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.ionosonde_utils import compute_band_states
@@ -44,9 +45,9 @@ class KC2GProp(SolarConditionsProvider):
     def _poll(self):
         try:
             logging.debug("Polling KC2G ionosonde data...")
-            response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30))
-            if response.status_code != 200:
-                logging.warning(f"KC2G ionosonde API returned HTTP {response.status_code}")
+            http_response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30))
+            if not http_response.ok:
+                logging.warning(f"HTTP {http_response.status_code} when calling KG2G ionosonde API.")
                 return
 
             now = datetime.now(timezone.utc)
@@ -57,7 +58,7 @@ class KC2GProp(SolarConditionsProvider):
             ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})
             updated_count = 0
 
-            for reading in response.json():
+            for reading in http_response.json():
                 station = reading.get("station", {})
                 ursi = station.get("code")
                 name = station.get("name")
@@ -115,6 +116,10 @@ class KC2GProp(SolarConditionsProvider):
             self.last_update_time = datetime.now(pytz.UTC)
             logging.debug(f"Updated KC2G ionosonde data for {updated_count} stations.")
 
+        except ConnectionError:
+            logging.warning("Connection error when accessing KC2G ionosonde API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing KC2G ionosonde API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in KC2G ionosonde data provider")

solarconditionsproviders/noaa3dayforecast.py

@@ -80,10 +80,6 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
         return result if result else None
 
     def _http_response_to_solar_conditions(self, http_response):
-        if http_response.status_code != 200:
-            logging.warning("NOAA K-index forecast API returned HTTP " + str(http_response.status_code))
-            return None
-
         lines = http_response.text.splitlines()
 
         # Find the "NOAA Kp index breakdown" section header

spothole.py

@@ -8,7 +8,7 @@ import sys
 from diskcache import Cache
 
 from core.cleanup import CleanupTimer
-from core.config import config, SERVER_OWNER_CALLSIGN
+from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL
 from core.constants import SOFTWARE_VERSION
 from core.lookup_helper import lookup_helper
 from core.status_reporter import StatusReporter
@@ -84,9 +84,9 @@ def get_solar_conditions_provider_from_config(config_providers_entry):
 if __name__ == '__main__':
     # Set up logging
     root = logging.getLogger()
-    root.setLevel(logging.INFO)
+    root.setLevel(LOG_LEVEL)
     handler = logging.StreamHandler(sys.stdout)
-    handler.setLevel(logging.INFO)
+    handler.setLevel(LOG_LEVEL)
     formatter = logging.Formatter("%(levelname)s : %(message)s")
     handler.setFormatter(formatter)
     root.handlers.clear()

spotproviders/gma.py

@@ -14,9 +14,9 @@ class GMA(HTTPSpotProvider):
     """Spot provider for General Mountain Activity"""
 
     POLL_INTERVAL_SEC = 120
-    SPOTS_URL = "https://www.cqgma.org/api/spots/25/"
+    SPOTS_URL = "https://www.gma.rocks/api/spots/25/"
     # GMA spots don't contain the details of the programme they are for, we need a separate lookup for that
-    REF_INFO_URL_ROOT = "https://www.cqgma.org/api/ref/?"
+    REF_INFO_URL_ROOT = "https://www.gma.rocks/api/ref/?"
 
     def __init__(self, provider_config):
         # Ensure there is an API key in our config, and set up the query URL using it. If no key is provided,
@@ -39,7 +39,7 @@ class GMA(HTTPSpotProvider):
                             de_call=source_spot["SPOTTER"].upper(),
                             # Seen GMA spots with no frequency or with "QRT" in this field
                             freq=float(source_spot["QRG"]) * 1000 if (
-                                        source_spot["QRG"] != "" and source_spot["QRG"] != "QRT") else None,
+                                    source_spot["QRG"] != "" and source_spot["QRG"] != "QRT") else None,
                             # Filter out some weird mode strings
                             mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
                             comment=source_spot["TEXT"],
@@ -58,8 +58,8 @@ class GMA(HTTPSpotProvider):
                     try:
                         ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
                                                                       headers=HTTP_HEADERS)
-                        # Sometimes this is blank, so handle that
-                        if ref_response.text is not None and ref_response.text != "":
+                        # Sometimes this is blank even if it's a 200 response, so handle that
+                        if ref_response.ok and ref_response.text is not None and ref_response.text != "":
                             ref_info = ref_response.json()
                             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
                             # spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
@@ -94,9 +94,17 @@ class GMA(HTTPSpotProvider):
                                         spot.sig_refs[0].sig = ref_info["reftype"]
                                         spot.sig = ref_info["reftype"]
 
-                                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
-                                # that for us.
+                                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point;
+                                # other code will do that for us.
                                 new_spots.append(spot)
+
+                        elif not ref_response.from_cache:
+                            if not ref_response.ok:
+                                logging.warning(
+                                    f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot["REF"]}")
+                            else:
+                                logging.warning(
+                                    f"GMA API returned a malformed response when looking up ref {source_spot["REF"]}")
                     except:
                         logging.warning("Exception when looking up " + self.REF_INFO_URL_ROOT + source_spot[
                             "REF"] + ", ignoring this spot for now")

spotproviders/hema.py

@@ -1,8 +1,10 @@
+import logging
 import re
 from datetime import datetime
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
@@ -35,36 +37,41 @@ class HEMA(HTTPSpotProvider):
         new_spots = []
         # OK, if the spot seed actually changed, now we make the real request for data.
         if spot_seed_changed:
-            source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30))
-            source_data_items = source_data.text.split("=")
-            # Iterate through source data items.
-            for source_spot in source_data_items:
-                spot_items = source_spot.split(";")
-                # Any line with less than 9 items is not a proper spot line
-                if len(spot_items) >= 9:
-                    # Fiddle with some data to extract bits we need. Freq/mode and spotter/comment come in combined fields.
-                    freq_mode_match = re.search(self.FREQ_MODE_PATTERN, spot_items[5])
-                    spotter_comment_match = re.search(self.SPOTTER_COMMENT_PATTERN, spot_items[6])
-                    if not freq_mode_match or not spotter_comment_match:
-                        continue
+            try:
+                source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30))
+                source_data_items = source_data.text.split("=")
+                # Iterate through source data items.
+                for source_spot in source_data_items:
+                    spot_items = source_spot.split(";")
+                    # Any line with less than 9 items is not a proper spot line
+                    if len(spot_items) >= 9:
+                        # Fiddle with some data to extract bits we need. Freq/mode and spotter/comment come in combined fields.
+                        freq_mode_match = re.search(self.FREQ_MODE_PATTERN, spot_items[5])
+                        spotter_comment_match = re.search(self.SPOTTER_COMMENT_PATTERN, spot_items[6])
+                        if not freq_mode_match or not spotter_comment_match:
+                            continue
 
-                    # Convert to our spot format
-                    spot = Spot(source=self.name,
-                                dx_call=spot_items[2].upper(),
-                                de_call=spotter_comment_match.group(1).upper(),
-                                freq=float(freq_mode_match.group(1)) * 1000000,
-                                mode=freq_mode_match.group(2).upper(),
-                                comment=spotter_comment_match.group(2),
-                                sig="HEMA",
-                                sig_refs=[SIGRef(id=spot_items[3].upper(), sig="HEMA", name=spot_items[4])],
-                                time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace(
-                                    tzinfo=pytz.UTC).timestamp(),
-                                dx_latitude=float(spot_items[7]),
-                                dx_longitude=float(spot_items[8]))
+                        # Convert to our spot format
+                        spot = Spot(source=self.name,
+                                    dx_call=spot_items[2].upper(),
+                                    de_call=spotter_comment_match.group(1).upper(),
+                                    freq=float(freq_mode_match.group(1)) * 1000000,
+                                    mode=freq_mode_match.group(2).upper(),
+                                    comment=spotter_comment_match.group(2),
+                                    sig="HEMA",
+                                    sig_refs=[SIGRef(id=spot_items[3].upper(), sig="HEMA", name=spot_items[4])],
+                                    time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace(
+                                        tzinfo=pytz.UTC).timestamp(),
+                                    dx_latitude=float(spot_items[7]),
+                                    dx_longitude=float(spot_items[8]))
 
-                    # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
-                    # that for us.
-                    new_spots.append(spot)
+                        # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other
+                        # code will do that for us.
+                        new_spots.append(spot)
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when accessing HEMA spots API.")
+            except ConnectionError:
+                logging.warning("Connection error when accessing HEMA spots API.")
         return new_spots
 
     def can_submit_spot(self, sig):

spotproviders/http_spot_provider.py

@@ -4,6 +4,8 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from spotproviders.spot_provider import SpotProvider
@@ -50,16 +52,25 @@ class HTTPSpotProvider(SpotProvider):
             # Request data from API
             logging.debug("Polling " + self.name + " spot API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
-            # Pass off to the subclass for processing
-            new_spots = self._http_response_to_spots(http_response)
-            # Submit the new spots for processing. There might not be any spots for the less popular programs.
-            if new_spots:
-                self._submit_batch(new_spots)
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                new_spots = self._http_response_to_spots(http_response)
+                # Submit the new spots for processing. There might not be any spots for the less popular programs.
+                if new_spots:
+                    self._submit_batch(new_spots)
 
-            self.status = "OK"
-            self.last_update_time = datetime.now(pytz.UTC)
-            logging.debug("Received data from " + self.name + " spot API.")
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.debug("Received data from " + self.name + " spot API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} spot API.")
 
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} spots API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} spots API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP JSON Spot Provider (" + self.name + ")")

spotproviders/parksnpeaks.py

@@ -34,8 +34,8 @@ class ParksNPeaks(HTTPSpotProvider):
                             dx_call=source_spot["actCallsign"].upper(),
                             de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,
                             # typo exists in API
-                            freq=float(source_spot["actFreq"].replace(",", "")) * 1000000 if (
-                                    source_spot["actFreq"] != "") else None,
+                            freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "")
+                                       .replace("+/-", "").strip()) * 1000000 if (source_spot["actFreq"] != "") else None,
                             # Seen PNP spots with empty frequency, and with comma-separated thousands digits
                             mode=source_spot["actMode"].upper(),
                             comment=source_spot["actComments"],

spotproviders/sota.py

@@ -1,6 +1,8 @@
+import logging
 from datetime import datetime
 
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS, SSB_SUB_MODES, DV_SUB_MODES
 from data.sig_ref import SIGRef
@@ -36,28 +38,33 @@ class SOTA(HTTPSpotProvider):
         new_spots = []
         # OK, if the epoch actually changed, now we make the real request for data.
         if epoch_changed:
-            source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30)).json()
-            # Iterate through source data
-            for source_spot in source_data:
-                # Convert to our spot format
-                spot = Spot(source=self.name,
-                            source_id=source_spot["id"],
-                            dx_call=source_spot["activatorCallsign"].upper(),
-                            dx_name=source_spot["activatorName"],
-                            de_call=source_spot["callsign"].upper(),
-                            freq=(float(source_spot["frequency"]) * 1000000) if (
-                                    source_spot["frequency"] is not None) else None,
-                            # Seen SOTA spots with no frequency!
-                            mode=source_spot["mode"].upper(),
-                            comment=source_spot["comments"],
-                            sig="SOTA",
-                            sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA", name=source_spot["summitName"],
-                                             activation_score=source_spot["points"])],
-                            time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp())
+            try:
+                source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30)).json()
+                # Iterate through source data
+                for source_spot in source_data:
+                    # Convert to our spot format
+                    spot = Spot(source=self.name,
+                                source_id=source_spot["id"],
+                                dx_call=source_spot["activatorCallsign"].upper(),
+                                dx_name=source_spot["activatorName"],
+                                de_call=source_spot["callsign"].upper(),
+                                freq=(float(source_spot["frequency"]) * 1000000) if (
+                                        source_spot["frequency"] is not None) else None,
+                                # Seen SOTA spots with no frequency!
+                                mode=source_spot["mode"].upper(),
+                                comment=source_spot["comments"],
+                                sig="SOTA",
+                                sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA", name=source_spot["summitName"],
+                                                 activation_score=source_spot["points"])],
+                                time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp())
 
-                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
-                # that for us.
-                new_spots.append(spot)
+                    # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
+                    # that for us.
+                    new_spots.append(spot)
+            except ConnectionError:
+                logging.warning("Connection error when accessing SOTA spots API")
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when accessing SOTA spots API.")
         return new_spots
 
     def can_submit_spot(self, sig):