Update README.md

.dockerignore

@@ -0,0 +1,7 @@
+/.venv
+__pycache__
+*.pyc
+/config.yml
+/cache/
+.git
+.gitignore

.idea/spothole.iml

@@ -4,7 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
       <excludeFolder url="file://$MODULE_DIR$/.venv" />
-      <excludeFolder url="file://$MODULE_DIR$/webassets/vendor" />
+      <excludeFolder url="file://$MODULE_DIR$/static/vendor" />
     </content>
     <orderEntry type="jdk" jdkName="Python 3.13 virtualenv at ~/code/spothole/.venv" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />

Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.13-slim
+
+WORKDIR /app
+COPY . .
+RUN pip install --no-cache-dir -r requirements.txt
+EXPOSE 8080
+
+CMD ["python3", "spothole.py"]

README.md

@@ -1,4 +1,4 @@
-# ![Spothole](/webassets/img/logo.png)
+# ![Spothole](/static/img/logo.png)
 
 Spothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open
 JSON API as well as a website to browse the data.
@@ -89,7 +89,7 @@ callsign and park data.
 
 Various approaches exist to writing your own client, but in general:
 
-* Refer to the API docs. These are built on an OpenAPI definition file (`/webassets/apidocs/openapi.yml`), which you can
+* Refer to the API docs. These are built on an OpenAPI definition file (`/static/apidocs/openapi.yml`), which you can
   automatically use to generate a client skeleton using various software.
 * Call the main "spots" or "alerts" API endpoints to get the data you want. For example, your app could call
   `https://spothole.app/api/v1/spots` once every few minutes. Apply filters if necessary.
@@ -172,6 +172,33 @@ countries. This is normal, don't panic!
 If you see some errors on startup, check your configuration, e.g. in case you have specified a port for the web server
 that is already in use by something else.
 
+### Running using Docker
+
+Spothole comes with a Docker configuration to make it easy to run it in a containerised environment. To set it up using
+Docker, the easiest way is to use a Docker Compose file. Create a new directory such as `/opt/docker/spothole` and
+create a `compose.yaml` file inside it with the following contents:
+
+```yaml
+services:
+  spothole:
+    container_name: spothole
+    build:
+      context: https://git.ianrenton.com/ian/spothole.git#latest
+    restart: unless-stopped
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./config.yml:/app/config.yml
+      - ./cache:/app/cache
+```
+
+Save the file. You will still need to create a copy of `config-example.yml` and name it `config.yml`, though with the
+Docker setup nothing has actually been downloaded yet, so you will have to copy the example from the repository some
+other way, e.g. [from the repo in a web browser](https://git.ianrenton.com/ian/spothole/src/branch/main/config-example.yml).
+
+With that in place, run `docker compose up` and you should be good to go. To detach, press `d` or run the command with
+the `-d` flag.
+
 ### Multiple cluster nodes with different settings
 
 Dan, S50U has written in with his Spothole cluster settings. He is using a cluster node which provides RBN spots, and
@@ -318,33 +345,49 @@ To set up nginx as a reverse proxy that sits in front of Spothole, first ensure
 
 Create a file at `/etc/nginx/sites-available/` called `spothole`. Give it the following contents, replacing
 `spothole.app` with the domain name on which you want to run Spothole. If you changed the port on which Spothole runs,
-update that on the "proxy_pass" line too.
+update that on the "proxy_pass" line, and if you installed Spothole somewhere other than `/home/spothole/spothole`,
+adjust the alias location for serving static files.
+
+(The latter section, configuring the nginx server to serve static files directly, improves efficiency because it saves
+Spothole itself from serving JS, CSS etc. files. If you can't do this for some reason, e.g. your nginx and spothole are
+on different computers, you can omit the `location /static/ {}` block.)
 
 ```nginx
 server {
     server_name spothole.app;
 
+    # Global proxy settings
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+    proxy_connect_timeout 10s;
+    proxy_buffering on;
+    
+    # Pass on IP address and host information to Spothole, in case logging this information is required
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Forwarded-Proto $scheme;
+
     # Wellknown area for Lets Encrypt
     location /.well-known/ {
         alias /var/www/html/.well-known/;
     }
 
+    # Load static assets directly from the Spothole static directory
+    location /static/ {
+        alias /home/spothole/spothole/static/;
+        expires 1h;
+        add_header Cache-Control "public, max-age=3600, must-revalidate";
+    }
+    
     # SSE endpoints
-    location ~ ^/api/v1/(spots|alerts)/stream {
+    location ~ ^/api/v1/(spots|alerts)/stream/? {
         proxy_pass http://127.0.0.1:8080;
         
-        # Allow keep-alive
-        proxy_http_version 1.1;
-        proxy_set_header Connection "";
-        
-        # Set correct content type for SSE API calls
-        add_header Content-Type text/event-stream always;
-        
-        # Set remove buffering, remove caching, add suitable timeouts for SSE API calls
+        # Remove buffering, remove caching, add suitable timeouts for SSE API calls
         proxy_buffering off;
         proxy_cache off;
         proxy_read_timeout 24h;
-        proxy_connect_timeout 10s;
         proxy_send_timeout 24h;
         proxy_set_header X-Accel-Buffering no;
         add_header Cache-Control no-store always;
@@ -352,59 +395,28 @@ server {
         # Allow cross-origin requests to API
         proxy_hide_header Access-Control-Allow-Origin;
         add_header Access-Control-Allow-Origin * always; 
-        
-        # Pass on IP address and host information to Spothole, in case logging this information is required
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto $scheme;
     }
 
     # Other API endpoints
     location /api/ {
         proxy_pass http://127.0.0.1:8080;
         
-        # Allow keep-alive
-        proxy_http_version 1.1;
-        proxy_set_header Connection "";
-        
-        # Set up buffering, remove caching, add suitable timeouts for API calls
-        proxy_buffering on;
+        # Remove buffering, remove caching, add suitable timeouts for API calls
+        proxy_buffering off;
         proxy_cache off;
         proxy_read_timeout 30s;
-        proxy_connect_timeout 10s;
         add_header Cache-Control no-store always;
         
         # Allow cross-origin requests to API
         proxy_hide_header Access-Control-Allow-Origin;
         add_header Access-Control-Allow-Origin * always;
-        
-        # Pass on IP address and host information to Spothole, in case logging this information is required
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto $scheme;
     }
 
-    # Static assets
+    # Templated pages
     location / {
         proxy_pass http://127.0.0.1:8080;
-        
-        # Allow keep-alive
-        proxy_http_version 1.1;
-        proxy_set_header Connection "";
-        
-        # Set up buffering and caching, add suitable timeouts for static asset requests
-        proxy_buffering on;
         proxy_read_timeout 30s;
-        proxy_connect_timeout 10s;
-        add_header Cache-Control "public, max-age=3600, must-revalidate" always;
-        
-        # Pass on IP address and host information to Spothole, in case logging this information is required
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto $scheme;
+        add_header Cache-Control "no-cache, must-revalidate" always;
     }
 }
 ```
@@ -467,12 +479,12 @@ To navigate your way around the source code, this list may help.
 
 *HTML/JS/CSS front-end code*
 
-* `/webassets` - Root for static files served by the web server
-* `/webassets/apidocs` - Contains the OpenAPI spec (`openapi.yml`)
-* `/webassets/css` - CSS files used by the web front-end
-* `/webassets/img` - image files used by the web front-end
-* `/webassets/js` - JavaScript used by the web front-end
-* `/webassets/vendor` - Third-party libraries (CSS, JS, fonts and images)
+* `/static` - Root for static files served by the web server. These are all served from a path starting `/static/`.
+* `/static/apidocs` - Contains the OpenAPI spec (`openapi.yml`)
+* `/static/css` - CSS files used by the web front-end
+* `/static/img` - image files used by the web front-end
+* `/static/js` - JavaScript used by the web front-end
+* `/static/vendor` - Third-party libraries (CSS, JS, fonts and images)
 
 *Miscellaneous*
 
@@ -524,14 +536,14 @@ my knowledge, created by HA8TKS for his CQ and ITU zone layers for Leaflet. `/da
 maps of conference centres.
 
 The project contains a set of flag icons generated using the "Noto Color Emoji" font on a Debian system, in the
-`/webassets/img/flags/` directory.
+`/static/img/flags/` directory.
 
 The software uses a number of Python libraries as listed in `requirements.txt`, and a number of JavaScript libraries.
 This project would not have been possible without these libraries, so many thanks to their developers.
 
 ### Third Party Libraries
 
-A number of third-party libraries are self-hosted in the `/webassets/vendor/` directory. These files are subject to
+A number of third-party libraries are self-hosted in the `/static/vendor/` directory. These files are subject to
 their own licences and are not covered by the overall licence declared in the `LICENSE` file.
 
 Particular thanks go to country-files.com for providing country lookup data for amateur radio, to K0SWE

alertproviders/alert_provider.py

@@ -8,10 +8,10 @@ from core.config import MAX_ALERT_AGE
 class AlertProvider:
     """Generic alert provider class. Subclasses of this query the individual APIs for alerts."""
 
-    def __init__(self, provider_config):
+    def __init__(self, name, provider_config):
         """Constructor"""
 
-        self.name = provider_config["name"]
+        self.name = name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"

alertproviders/bota.py

@@ -15,7 +15,7 @@ class BOTA(HTTPAlertProvider):
     ALERTS_URL = "https://www.beachesontheair.com/"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("BOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []

alertproviders/http_alert_provider.py

@@ -4,6 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from alertproviders.alert_provider import AlertProvider
 from core.constants import HTTP_HEADERS
@@ -13,8 +14,8 @@ class HTTPAlertProvider(AlertProvider):
     """Generic alert provider class for providers that request data via HTTP(S). Just for convenience to avoid code
     duplication. Subclasses of this query the individual APIs for data."""
 
-    def __init__(self, provider_config, url, poll_interval):
-        super().__init__(provider_config)
+    def __init__(self, name, provider_config, url, poll_interval):
+        super().__init__(name, provider_config)
         self._url = url
         self._poll_interval = poll_interval
         self._thread = None
@@ -41,6 +42,8 @@ class HTTPAlertProvider(AlertProvider):
             # Request data from API
             logging.debug("Polling " + self.name + " alert API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            # Check response code was good
+            if http_response.ok:
                 # Pass off to the subclass for processing
                 new_alerts = self._http_response_to_alerts(http_response)
                 # Submit the new alerts for processing. There might not be any alerts for the less popular programs.
@@ -50,7 +53,14 @@ class HTTPAlertProvider(AlertProvider):
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
                 logging.debug("Received data from " + self.name + " alert API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} alerts API.")
 
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} alerts API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} alerts API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP JSON Alert Provider (" + self.name + ")")

alertproviders/ng3k.py

@@ -18,7 +18,7 @@ class NG3K(HTTPAlertProvider):
     AS_CALL_PATTERN = re.compile("as ([a-z0-9/]+)", re.IGNORECASE)
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("NG3K", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []

alertproviders/parksnpeaks.py

@@ -15,7 +15,7 @@ class ParksNPeaks(HTTPAlertProvider):
     ALERTS_URL = "https://parksnpeaks.org/api/ALERTS/"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("ParksNPeaks", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []

alertproviders/pota.py

@@ -14,7 +14,7 @@ class POTA(HTTPAlertProvider):
     ALERTS_URL = "https://api.pota.app/activation"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("POTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []

alertproviders/sota.py

@@ -14,7 +14,7 @@ class SOTA(HTTPAlertProvider):
     ALERTS_URL = "https://api-db2.sota.org.uk/api/alerts/365/all/all"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("SOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []

alertproviders/wota.py

@@ -18,7 +18,7 @@ class WOTA(HTTPAlertProvider):
     RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("WOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []

alertproviders/wwff.py

@@ -14,7 +14,7 @@ class WWFF(HTTPAlertProvider):
     ALERTS_URL = "https://spots.wwff.co/static/agendas.json"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("WWFF", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []

config-example.yml

@@ -19,66 +19,55 @@ base-url: "http://localhost:8080"
 
 # Spot providers to use. This is an example set, tailor it to your liking by commenting and uncommenting.
 # RBN and APRS-IS are supported but have such a high data rate, you probably don't want them enabled.
-# Each provider needs a class, a name, and an enabled/disabled state. Some require more config such as hostnames/IP
+# Each provider needs a class and an enabled/disabled state. Some require more config such as hostnames/IP
 # addresses and ports. You can duplicate them if you like, e.g. to support several DX clusters. RBN uses two ports, 7000
 # for CW/RTTY and 7001 for FT8, so if you want both, you need two entries, as shown below.
 # Feel free to write your own provider classes! There are details in the README.
 spot-providers:
   - class: "POTA"
-    name: "POTA"
     enabled: true
 
   - class: "SOTA"
-    name: "SOTA"
     enabled: true
 
   - class: "WWFF"
-    name: "WWFF"
     enabled: true
 
   - class: "WWBOTA"
-    name: "WWBOTA"
     enabled: true
 
   - class: "GMA"
-    name: "GMA"
     enabled: true
     # GMA requires an API key to fetch spots. After creating an account on cqgma.org, email support and request one.
     api-key: ""
 
   - class: "HEMA"
-    name: "HEMA"
     enabled: true
 
   - class: "ParksNPeaks"
-    name: "ParksNPeaks"
     enabled: true
 
   - class: "ZLOTA"
-    name: "ZLOTA"
     enabled: true
 
   - class: "WOTA"
-    name: "WOTA"
     enabled: true
 
   - class: "LLOTA"
-    name: "LLOTA"
     enabled: true
 
   - class: "Towers"
-    name: "Towers"
     enabled: true
 
   - class: "Tiles"
-    name: "Tiles"
     enabled: true
 
   - class: "APRSIS"
-    name: "APRS-IS"
     enabled: false
 
   - class: "DXCluster"
+    # Clusters have a "name" property in case you want to say which cluster node the data is from, or connect to
+    # several clusters and name them separately.
     name: "HRD Cluster"
     enabled: true
     host: "hrd.wa9pie.net"
@@ -99,18 +88,12 @@ spot-providers:
     enabled: false
     host: "w3lpl.net"
     port: 7373
-    # Prompt the cluster node gives when asking for a callsign to log in. Varies between cluster node software.
     login_prompt: "Please enter your call:"
-    # Callsign Spothole will use to log into this cluster. Ensure the SSID (e.g. -99) is different to any personal
-    # connection you might make to this cluster node.
     login_callsign: "N0CALL-99"
-    # Whether to allow RBN spots that come via this cluster. If you don't want RBN spots or you are making a separate
-    # connection to RBN directly, leave this as False. If you want RBN spots from this cluster, set this to True. (Make
-    # sure you aren't also separately connecting to RBN directly, otherwise you may get duplicate spots.) Note that not
-    # all clusters sent RBN spots anyway.
     allow_rbn_spots: false
 
   - class: "RBN"
+  - # RBN sources have a "name" property in case you want to differentiate between the CW/RTTY and FT8 sources.
     name: "RBN CW/RTTY"
     enabled: false
     port: 7000
@@ -127,11 +110,12 @@ spot-providers:
     enabled-by-default-in-web-ui: false
 
   - class: "UKPacketNet"
-    name: "UK Packet Radio Net"
     enabled: false
     enabled-by-default-in-web-ui: false
 
   - class: "XOTA"
+    # xOTA sources have a "name" property so you can define what programme it is for, since xOTA is generic software
+    # for running "something-on-the-air" programmes.
     name: "39C3 TOTA"
     enabled: false
     url: "wss://39c3.totawatch.de/api/spot/live"
@@ -152,31 +136,24 @@ spot-providers:
 # Alert providers to use. Same setup as the spot providers list above.
 alert-providers:
   - class: "POTA"
-    name: "POTA"
     enabled: true
 
   - class: "SOTA"
-    name: "SOTA"
     enabled: true
 
   - class: "WWFF"
-    name: "WWFF"
     enabled: true
 
   - class: "ParksNPeaks"
-    name: "ParksNPeaks"
     enabled: true
 
   - class: "WOTA"
-    name: "WOTA"
     enabled: true
 
   - class: "BOTA"
-    name: "BOTA"
     enabled: true
 
   - class: "NG3K"
-    name: "NG3K"
     enabled: true
 
 
@@ -184,19 +161,15 @@ alert-providers:
 # conditions, etc.) and make it available via the /api/v1/solar endpoint.
 solar-condition-providers:
   - class: "HamQSL"
-    name: "HamQSL"
     enabled: true
 
   - class: "NOAA3dayForecast"
-    name: "NOAA 3-day Forecast"
     enabled: true
 
   - class: "GIROIonosonde"
-    name: "GIRO Ionosonde Data"
     enabled: true
 
   - class: "KC2GProp"
-    name: "KC2G Propagation Data"
     enabled: true
 
 # Maximum time to keep spots and alerts in the system before deleting them. By default, one hour for spots and one week
@@ -211,6 +184,14 @@ clublog-api-key: ""
 # Allow submitting spots to the Spothole API?
 allow-spotting: true
 
+# Log web requests? Useful to see what your users are requesting in terms of pages and API endpoints, but will fill up
+# your log quickly on a popular server.
+log-web-requests: false
+
+# Minimum severity of log statements to print to the system log. Defaults to INFO, change to DEBUG if you need more
+# information.
+log-level: INFO
+
 # Options for the web UI.
 web-ui-options:
   spot-count: [ 10, 25, 50, 100 ]

core/cache_utils.py

@@ -8,8 +8,11 @@ from requests_cache import CachedSession
 # of time has passed. This is used throughout Spothole to cache data that does not change
 # rapidly. The ThreadSafeSession construct here protects it against some multithreading
 # contention weirdness we sometimes used to see on startup where the cache was hammered
-# pretty hard.
-_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30))
+# pretty hard. The expanded list of allowable_codes ensures we also cache and return 400-type
+# responses, e.g "this SOTA summit ref doesn't actually exist", to avoid hammering remote
+# servers for data they've told us they can't provide.
+_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30),
+                         allowable_codes=(200, 400, 401, 403, 404))
 _lock = threading.Lock()
 
 

core/config.py

@@ -22,6 +22,8 @@ WEB_SERVER_PORT = config["web-server-port"]
 ALLOW_SPOTTING = config["allow-spotting"]
 WEB_UI_OPTIONS = config["web-ui-options"]
 API_ONLY_MODE = config.get("api-only-mode", False)
+LOG_LEVEL = config.get("log-level", "INFO")
+LOG_WEB_REQUESTS = config.get("log-web-requests", False)
 
 # For ease of config, each spot provider owns its own config about whether it should be enabled by default in the web UI
 # but for consistency we provide this to the front-end in web-ui-options because it has no impact outside of the web UI.

core/constants.py

@@ -3,7 +3,7 @@ from data.band import Band
 from data.sig import SIG
 
 # General software
-SOFTWARE_VERSION = "1.4-pre"
+SOFTWARE_VERSION = "1.4"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": "Spothole v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
@@ -27,8 +27,8 @@ SIGS = [
     SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"),
     SIG(name="BOTA", comment_names=[], description="Beaches on the Air"),
     SIG(name="KRMNPA", comment_names=["KRMNPA"], description="Keith Roget Memorial National Parks Award"),
-    SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"[A-Z]{2}\-\d{4}"),
-    SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2}R\-\d{4}"),
+    SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"LL[A-Z]{2}\-\d{4}"),
+    SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2,3}R\-\d{4}"),
     SIG(name="Tiles", comment_names=[], description="Tiles on the Air", ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}"),
     SIG(name="WAB", comment_names=["WAB"], description="Worked All Britain", ref_regex=r"[A-Z]{1,2}[0-9]{2}"),
     SIG(name="WAI", comment_names=["WAI"], description="Worked All Ireland", ref_regex=r"[A-Z][0-9]{2}"),
@@ -102,5 +102,6 @@ PROPAGATION_MODES = {
     "AU": "Aurora",
     "MS": "Meteor scatter",
     "RS": "Rain scatter",
-    "AS": "Aircraft scatter"
+    "AS": "Aircraft scatter",
+    "SAT": "Satellite"
 }

core/lookup_helper.py

@@ -5,13 +5,13 @@ import re
 import urllib.parse
 from datetime import timedelta
 
-import requests
 import xmltodict
 from diskcache import Cache
 from pyhamtools import LookupLib, Callinfo, callinfo
 from pyhamtools.exceptions import APIKeyMissingError
 from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 from requests_cache import CachedSession
 
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
@@ -143,13 +143,21 @@ class LookupHelper:
         try:
             logging.info("Downloading Country-files.com cty.plist...")
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
-                                                      headers=HTTP_HEADERS).text
+                                                      headers=HTTP_HEADERS)
 
+            if response.ok:
                 with open(self._country_files_cty_plist_download_location, "w") as f:
-                f.write(response)
+                    f.write(response.text)
                     f.flush()
                     return True
+            else:
+                logging.warning(f"HTTP {response.status_code} when downloading Country-files.com cty.plist.")
+                return False
 
+        except ConnectionError:
+            logging.warning(f"Connection error when downloading Clublog cty.xml.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when downloading Clublog cty.xml.")
         except Exception as e:
             logging.error("Exception when downloading Clublog cty.xml", e)
             return False
@@ -161,13 +169,21 @@ class LookupHelper:
             logging.info("Downloading dxcc.json...")
             response = SEMI_STATIC_URL_DATA_CACHE.get(
                 "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
-                headers=HTTP_HEADERS).text
+                headers=HTTP_HEADERS)
 
+            if response.ok:
                 with open(self._dxcc_json_download_location, "w") as f:
-                f.write(response)
+                    f.write(response.text)
                     f.flush()
                     return True
+            else:
+                logging.warning(f"HTTP {response.status_code} when downloading dxcc.json.")
+                return False
 
+        except ConnectionError:
+            logging.warning(f"Connection error when downloading dxcc.json.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when downloading dxcc.json.")
         except Exception as e:
             logging.error("Exception when downloading dxcc.json", e)
             return False
@@ -490,25 +506,47 @@ class LookupHelper:
 
         # Try the call as given, then fall back to the base call (strips /P, /M etc.)
         calls_to_try = [call]
+        try:
             home_call = callinfo.Callinfo.get_homecall(call)
             if home_call != call:
                 calls_to_try.append(home_call)
+        except ValueError:
+            logging.debug("Could not look up home call for callsign %s", call)
 
         for lookup_call in calls_to_try:
             try:
-                lookup_response = requests.get(
+                response = SEMI_STATIC_URL_DATA_CACHE.get(
                     self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
-                    headers=HTTP_HEADERS, timeout=10).content
-                raw = xmltodict.parse(lookup_response).get("QRZDatabase", {}).get("Callsign")
-                if raw:
-                    data = _normalize_qrz_data(raw)
+                    headers=HTTP_HEADERS, timeout=10)
+                if response.ok:
+                    qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
+                    if qrz_response:
+                        if "Callsign" in qrz_response:
+                            data = _normalize_qrz_data(qrz_response.get("Callsign"))
                             self._qrz_callsign_data_cache.add(call, data, expire=604800)  # 1 week in seconds
                             return data
+                        elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
+                            # Errors here are normally just "callsign not in database", no need to log that ourselves
+                            # above debug level.
+                            logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call,
+                                          qrz_response.get("Session").get("Error"))
+
+                    elif not response.from_cache:
+                        logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call)
+                elif not response.from_cache:
+                    logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call)
+
             except (KeyError, ValueError):
                 continue
+            except ConnectionError:
+                logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call)
+                continue
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call)
+                continue
             except Exception:
-                logging.error("Exception when looking up QRZ data")
-                return None
+                logging.error("Exception when looking up callsign %s using QRZ", lookup_call, exc_info=True)
+                continue
 
         # Not found in QRZ; cache None so we don't keep retrying
         self._qrz_callsign_data_cache.add(call, None, expire=604800)  # 1 week in seconds
@@ -546,23 +584,36 @@ class LookupHelper:
 
         # Try the call as given, then fall back to the base call (strips /P, /M etc.)
         calls_to_try = [call]
+        try:
             home_call = callinfo.Callinfo.get_homecall(call)
             if home_call != call:
                 calls_to_try.append(home_call)
+        except ValueError:
+            logging.debug("Could not look up home call for callsign %s", call)
 
         for lookup_call in calls_to_try:
             try:
-                lookup_data = SEMI_STATIC_URL_DATA_CACHE.get(
+                response = SEMI_STATIC_URL_DATA_CACHE.get(
                     self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
-                        lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS).content
-                data = xmltodict.parse(lookup_data)["HamQTH"]["search"]
+                        lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
+                if response.ok:
+                    data = xmltodict.parse(response.content)["HamQTH"]["search"]
                     self._hamqth_callsign_data_cache.add(call, data, expire=604800)  # 1 week in seconds
                     return data
+                elif not response.from_cache:
+                    logging.warning("HTTP %d looking up callsign %s using HamQTH", response.status_code, lookup_call)
+
             except (KeyError, ValueError):
                 continue
+            except ConnectionError:
+                logging.warning(f"Connection error when looking up callsign %s using HamQTH", lookup_call)
+                continue
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when looking up callsign %s using HamQTH", lookup_call)
+                continue
             except Exception:
-                logging.error("Exception when looking up HamQTH data")
-                return None
+                logging.error("Exception when looking up callsign %s using HamQTH", lookup_call, exc_info=True)
+                continue
 
         # Not found in HamQTH; cache None so we don't keep retrying
         self._hamqth_callsign_data_cache.add(call, None, expire=604800)  # 1 week in seconds

core/sig_utils.py

@@ -2,6 +2,7 @@ import csv
 import logging
 
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import SIGS, HTTP_HEADERS
@@ -11,6 +12,14 @@ from core.geo_utils import wab_wai_square_to_lat_lon
 # file in Spothole and load it on startup.
 with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
     _DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
+# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
+# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
+# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
+_WWFF_INDEX_CACHE = {}
+_SIOTA_INDEX_CACHE = {}
+_WOTA_INDEX_CACHE = {}
+_ZLOTA_INDEX_CACHE = {}
+_LLOTA_INDEX_CACHE = {}
 
 
 def get_ref_regex_for_sig(sig):
@@ -45,9 +54,8 @@ def populate_sig_ref_info(sig_ref):
     try:
         if sig.upper() == "POTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if data:
                     fullname = str(data["name"]) if "name" in data else None
                     if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
@@ -57,12 +65,16 @@ def populate_sig_ref_info(sig_ref):
                     sig_ref.grid = data["grid6"] if "grid6" in data else None
                     sig_ref.latitude = data["latitude"] if "latitude" in data else None
                     sig_ref.longitude = data["longitude"] if "longitude" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "SOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if data:
                     sig_ref.name = data["name"] if "name" in data else None
                     sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
@@ -70,67 +82,108 @@ def populate_sig_ref_info(sig_ref):
                     sig_ref.latitude = data["latitude"] if "latitude" in data else None
                     sig_ref.longitude = data["longitude"] if "longitude" in data else None
                     sig_ref.activation_score = data["points"] if "points" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "WWBOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if data:
                     sig_ref.name = data["name"] if "name" in data else None
                     sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
                     sig_ref.grid = data["locator"] if "locator" in data else None
                     sig_ref.latitude = data["lat"] if "lat" in data else None
                     sig_ref.longitude = data["long"] if "long" in data else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if data:
                     sig_ref.name = data["name"] if "name" in data else None
                     sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
                     sig_ref.grid = data["locator"] if "locator" in data else None
-                sig_ref.latitude = data["latitude"] if "latitude" in data else None
-                sig_ref.longitude = data["longitude"] if "longitude" in data else None
+
+                    # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
+                    # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
+                    # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
+                    # just use the single point we have.
+                    if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
+                            "lat2") is not None and data.get("lng2") is not None:
+                        sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
+                        sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
+                    else:
+                        sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
+                        sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "WWFF":
-            wwff_response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
                                                       headers=HTTP_HEADERS)
-            if not wwff_response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", wwff_response.status_code, sig, ref_id)
-                return sig_ref
-            wwff_index = {row["reference"]: row for row in csv.DictReader(wwff_response.content.decode().splitlines())}
-            row = wwff_index.get(ref_id)
+            if response.ok:
+                global _WWFF_INDEX_CACHE
+                if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
+                    # New data from WWFF, update our internal map
+                    _WWFF_INDEX_CACHE = {row["reference"]: row for row in
+                                         csv.DictReader(response.content.decode().splitlines())}
+                row = _WWFF_INDEX_CACHE.get(ref_id)
                 if row:
                     sig_ref.name = row["name"] if "name" in row else None
                     sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
                     sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
                     sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
-                sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row["longitude"] != "-" else None
+                    sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row[
+                        "longitude"] != "-" else None
+                elif not response.from_cache:
+                    logging.warning("WWFF database did not contain data for ref %s", ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "SIOTA":
-            siota_response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
+            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
                                                       headers=HTTP_HEADERS)
-            if not siota_response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", siota_response.status_code, sig, ref_id)
-                return sig_ref
-            siota_index = {row["SILO_CODE"]: row for row in
-                           csv.DictReader(siota_response.content.decode().splitlines())}
-            row = siota_index.get(ref_id)
+            if response.ok:
+                global _SIOTA_INDEX_CACHE
+                if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
+                    # New data from SIOTA, update our internal map
+                    _SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
+                                          csv.DictReader(response.content.decode().splitlines())}
+                row = _SIOTA_INDEX_CACHE.get(ref_id)
                 if row:
                     sig_ref.name = row["NAME"] if "NAME" in row else None
                     sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
                     sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
                     sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
+                elif not response.from_cache:
+                    logging.warning("SIOTA database did not contain data for ref %s", ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "WOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if data:
-                for feature in data.get("features", []):
-                    if feature["properties"]["wotaId"] == ref_id:
+                    global _WOTA_INDEX_CACHE
+                    if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from WOTA, update our internal map
+                        _WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
+                                             data.get("features", [])}
+                    feature = _WOTA_INDEX_CACHE.get(ref_id)
+                    if feature:
                         sig_ref.name = feature["properties"]["title"]
                         # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
                         # added to them
@@ -141,48 +194,68 @@ def populate_sig_ref_info(sig_ref):
                         sig_ref.grid = feature["properties"]["qthLocator"]
                         sig_ref.latitude = feature["geometry"]["coordinates"][1]
                         sig_ref.longitude = feature["geometry"]["coordinates"][0]
-                        break
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "ZLOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if isinstance(data, list):
-                for asset in data:
-                    if asset["code"] == ref_id:
+                    global _ZLOTA_INDEX_CACHE
+                    if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from ZLOTA, update our internal map
+                        _ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
+                    asset = _ZLOTA_INDEX_CACHE.get(ref_id)
+                    if asset:
                         sig_ref.name = asset["name"]
-                        sig_ref.url = "https://ontheair.nz/assets/ZLI_OT-030" + ref_id.replace("/", "_")
+                        sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
                         try:
                             sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
                         except:
                             logging.debug("Invalid lat/lon received for reference")
                             sig_ref.latitude = asset["y"]
                             sig_ref.longitude = asset["x"]
-                        break
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "BOTA":
             if not sig_ref.name:
                 sig_ref.name = sig_ref.id
             sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
+
         elif sig.upper() == "LLOTA":
             response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
                                                       headers=HTTP_HEADERS)
-            if not response.ok:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-            data = response.json() if response.ok else None
+            if response.ok:
+                data = response.json()
                 if isinstance(data, list):
-                for ref in data:
-                    if ref["reference_code"] == ref_id:
+                    global _LLOTA_INDEX_CACHE
+                    if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
+                        # New data from LLOTA, update our internal map
+                        _LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
+                    ref = _LLOTA_INDEX_CACHE.get(ref_id)
+                    if ref:
                         sig_ref.name = str(ref["name"])
                         sig_ref.url = "https://llota.app/list/ref/" + ref_id
                         sig_ref.grid = str(ref["grid_locator"])
                         ll = locator_to_latlong(sig_ref.grid)
                         sig_ref.latitude = ll[0]
                         sig_ref.longitude = ll[1]
-                        break
+                elif not response.from_cache:
+                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
+            elif not response.from_cache:
+                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
         elif sig.upper() == "WWTOTA":
             if not sig_ref.name:
                 sig_ref.name = sig_ref.id
             sig_ref.url = "https://wwtota.com/seznam/karta_rozhledny.php?ref=" + str(sig_ref.name)
+
         elif sig.upper() == "TILES":
             # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
             if not sig_ref.name:
@@ -193,6 +266,7 @@ def populate_sig_ref_info(sig_ref):
                 ll = locator_to_latlong(str(sig_ref.grid))
                 sig_ref.latitude = ll[0]
                 sig_ref.longitude = ll[1]
+
         elif sig.upper() == "WAB" or sig.upper() == "WAI":
             ll = wab_wai_square_to_lat_lon(ref_id)
             if ll:
@@ -202,21 +276,31 @@ def populate_sig_ref_info(sig_ref):
                     sig_ref.latitude = ll[0]
                     sig_ref.longitude = ll[1]
                 except:
-                    logging.debug("Invalid lat/lon received for reference")
+                    logging.warning("Invalid lat/lon received for WAB/WAI reference")
+
         elif sig.upper() == "DME":
             # Zero-pad to 5 digits to match our source data
             row = _DME_INDEX.get(ref_id.zfill(5))
             if row:
                 sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
-                sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get("LATITUD_ETRS89_REGCAN95") else None
-                sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get("LONGITUD_ETRS89_REGCAN95") else None
+                sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LATITUD_ETRS89_REGCAN95") else None
+                sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LONGITUD_ETRS89_REGCAN95") else None
                 if sig_ref.latitude and sig_ref.longitude:
                     try:
                         sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6)
                     except Exception:
-                        logging.debug("Invalid lat/lon received for reference")
+                        logging.warning("Invalid lat/lon received for DME reference")
+            else:
+                logging.warning("DME database did not contain data for ref %s", ref_id)
+
+    except ConnectionError:
+        logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
+    except (ConnectTimeout, ReadTimeout):
+        logging.warning(f"Timeout when looking up sig_ref info for " + sig + " ref " + ref_id)
     except Exception:
-        logging.warning("Failed to look up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
+        logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
     return sig_ref
 
 

data/spot.py

@@ -133,6 +133,17 @@ class Spot:
     # The ID the source gave it, if any.
     source_id: str | None = None
 
+    def __post_init__(self):
+        """Normalise fields that don't survive a plain dict to Spot conversion. This is used in the "add spot" API
+        endpoint where the client is submitting JSON, and we want to recreate a full Spot object, including nested
+        objects such as the sig_refs list.."""
+
+        if self.sig_refs:
+            self.sig_refs = [
+                sig_ref if isinstance(sig_ref, SIGRef) else SIGRef(**sig_ref)
+                for sig_ref in self.sig_refs
+            ]
+
     def infer_missing(self, credentials=None):
         """Infer missing parameters where possible"""
 
@@ -312,7 +323,7 @@ class Spot:
                         self.propagation_mode = PROPAGATION_MODES[mode_tag]
                     else:
                         self.propagation_mode = mode_tag
-                        logging.info("Seen a new propagation mode tag not yet in the system: {}", mode_tag)
+                        logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag)
 
         # DX Grid to lat/lon and vice versa in case one is missing
         if self.dx_grid and not self.dx_latitude:

server/handlers/api/addspot.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import re
 from datetime import datetime
@@ -15,7 +14,6 @@ from core.lookup_helper import infer_band_from_freq
 from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import safe_json_dumps
-from data.sig_ref import SIGRef
 from data.spot import Spot
 
 
@@ -68,14 +66,6 @@ class APISpotHandler(tornado.web.RequestHandler):
             json_spot = tornado.escape.json_decode(post_data)
             spot = Spot(**json_spot)
 
-            # Converting to a spot object this way won't have coped with sig_ref objects, so fix that. (Would be nice to
-            # redo this in a functional style)
-            if spot.sig_refs:
-                real_sig_refs = []
-                for dict_obj in spot.sig_refs:
-                    real_sig_refs.append(json.loads(json.dumps(dict_obj), object_hook=lambda d: SIGRef(**d)))
-                spot.sig_refs = real_sig_refs
-
             # Reject if no timestamp, frequency, dx_call or de_call
             if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
                 self.set_status(422)
@@ -137,7 +127,7 @@ class APISpotHandler(tornado.web.RequestHandler):
             self.set_header("Content-Type", "application/json")
 
         except Exception as e:
-            logging.error(e)
+            logging.error("Exception when handling client request to add spot API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
             self.set_header("Cache-Control", "no-store")

server/handlers/api/alerts.py

@@ -59,11 +59,10 @@ class APIAlertsHandler(tornado.web.RequestHandler):
             self.write(safe_json_dumps(data))
             self.set_status(200)
         except ValueError as e:
-            logging.error(e)
             self.write(safe_json_dumps("Bad request - " + str(e)))
             self.set_status(400)
         except Exception as e:
-            logging.error(e)
+            logging.error("Exception when handling client request to alerts API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
         self.set_header("Cache-Control", "no-store")

server/handlers/api/dxstats.py

@@ -1,4 +1,5 @@
 import json
+import logging
 from collections import Counter
 from datetime import datetime, timedelta
 from typing import Any
@@ -9,6 +10,7 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
 
 CONTINENTS = ["EU", "NA", "SA", "AS", "AF", "OC", "AN"]
 BANDS = ["160m", "80m", "60m", "40m", "30m", "20m", "17m", "15m", "12m", "10m", "6m"]
@@ -29,6 +31,7 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
         self._web_server_metrics = web_server_metrics
 
     def get(self):
+        try:
             self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
             self._web_server_metrics["api_access_counter"] += 1
             self._web_server_metrics["status"] = "OK"
@@ -55,3 +58,8 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to dx stats API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/api/lookups.py

@@ -74,7 +74,7 @@ class APILookupCallHandler(tornado.web.RequestHandler):
                 self.set_status(422)
 
         except Exception as e:
-            logging.error(e)
+            logging.error("Exception when handling client request to call lookup API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
 
@@ -126,7 +126,7 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
                 self.set_status(422)
 
         except Exception as e:
-            logging.error(e)
+            logging.error("Exception when handling client request to sig ref lookup API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
 
@@ -188,7 +188,7 @@ class APILookupGridHandler(tornado.web.RequestHandler):
                 self.set_status(422)
 
         except Exception as e:
-            logging.error(e)
+            logging.error("Exception when handling client request to grid ref lookup API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
 

server/handlers/api/options.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 from typing import Any
 
@@ -25,6 +26,7 @@ class APIOptionsHandler(tornado.web.RequestHandler):
         self._web_server_metrics = web_server_metrics
 
     def get(self):
+        try:
             # Metrics
             self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
             self._web_server_metrics["api_access_counter"] += 1
@@ -53,3 +55,8 @@ class APIOptionsHandler(tornado.web.RequestHandler):
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to options API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/api/solar_conditions.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 from typing import Any
 
@@ -7,6 +8,7 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
 
 
 class APISolarConditionsHandler(tornado.web.RequestHandler):
@@ -22,6 +24,7 @@ class APISolarConditionsHandler(tornado.web.RequestHandler):
         self._web_server_metrics = web_server_metrics
 
     def get(self):
+        try:
             # Metrics
             self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
             self._web_server_metrics["api_access_counter"] += 1
@@ -32,3 +35,8 @@ class APISolarConditionsHandler(tornado.web.RequestHandler):
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to solar conditions API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/api/spots.py

@@ -59,11 +59,10 @@ class APISpotsHandler(tornado.web.RequestHandler):
             self.write(safe_json_dumps(data))
             self.set_status(200)
         except ValueError as e:
-            logging.error(e)
             self.write(safe_json_dumps("Bad request - " + str(e)))
             self.set_status(400)
         except Exception as e:
-            logging.error(e)
+            logging.error("Excedption when handling client request to spots API: %s", e, exc_info=True)
             self.write(safe_json_dumps("Error - an internal server error occurred."))
             self.set_status(500)
         self.set_header("Cache-Control", "no-store")

server/handlers/api/status.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 from typing import Any
 
@@ -23,6 +24,7 @@ class APIStatusHandler(tornado.web.RequestHandler):
         self._web_server_metrics = web_server_metrics
 
     def get(self):
+        try:
             # Metrics
             self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
             self._web_server_metrics["api_access_counter"] += 1
@@ -33,3 +35,8 @@ class APIStatusHandler(tornado.web.RequestHandler):
             self.set_status(200)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to status API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

server/handlers/manifesthandler.py

@@ -0,0 +1,11 @@
+import tornado.web
+
+from core.config import BASE_URL
+
+
+class ManifestHandler(tornado.web.RequestHandler):
+    """Handler for manifest.webmanifest, which needs BASE_URL inserted and a custom content-type"""
+
+    def get(self):
+        self.set_header("Content-Type", "application/manifest+json; charset=UTF-8")
+        self.render("manifest.webmanifest", baseurl=BASE_URL)

server/webserver.py

@@ -5,7 +5,7 @@ import os
 import tornado
 from tornado.web import StaticFileHandler
 
-from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE
+from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL
 from core.utils import empty_queue
 from server.handlers.api.addspot import APISpotHandler
 from server.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
@@ -15,6 +15,7 @@ from server.handlers.api.options import APIOptionsHandler
 from server.handlers.api.solar_conditions import APISolarConditionsHandler
 from server.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
 from server.handlers.api.status import APIStatusHandler
+from server.handlers.manifesthandler import ManifestHandler
 from server.handlers.metrics import PrometheusMetricsHandler
 from server.handlers.pagetemplate import PageTemplateHandler
 
@@ -99,11 +100,16 @@ class WebServer:
             if ALLOW_SPOTTING:
                 ui_routes += [(r"/add-spot", PageTemplateHandler, {"template_name": "add_spot", **handler_opts})]
 
-        # API docs, Prometheus metrics, and finally static assets are always available regardless of API-only mode.
+        # API docs, Prometheus metrics, webapp manifest and static assets are always available regardless of API-only
+        # mode.
         misc_routes = [
             (r"/apidocs", PageTemplateHandler, {"template_name": "apidocs", **handler_opts}),
             (r"/metrics", PrometheusMetricsHandler),
-            (r"/(.*)", StaticFileHandler, {"path": os.path.join(_HERE, "../webassets")})
+            (r"/manifest.webmanifest", ManifestHandler),
+            # If e.g. nginx is configured as a reverse proxy with a hard-coded path to static files, as per the README,
+            # this will never have to handle anything, but having it here allows Spothole to work without nginx for
+            # testing.
+            (r"/static/(.*)", StaticFileHandler, {"path": os.path.join(_HERE, "../static")})
         ]
 
         app = tornado.web.Application(api_routes + ui_routes + misc_routes,
@@ -112,6 +118,7 @@ class WebServer:
                                       debug=False)
         app.listen(self._port, xheaders=True)
         logging.info("Web server running on port " + str(WEB_SERVER_PORT))
+        logging.info("You can access your copy of Spothole at " + BASE_URL)
         await self._shutdown_event.wait()
 
     def notify_new_spot(self, spot):
@@ -165,8 +172,10 @@ class WebServer:
         pass
 
 def request_log(handler):
-    """Custom log function to provide more data about requests."""
+    """Custom log function to provide more data about requests when enabled, and to provide the ability to turn off
+    web request logging altogetether."""
 
+    if LOG_WEB_REQUESTS:
         if handler.get_status() < 500:
             log_method = logging.info
         else:

solarconditionsproviders/giroionosonde.py

@@ -5,6 +5,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.ionosonde_utils import compute_band_states
@@ -29,7 +30,7 @@ class GIROIonosonde(SolarConditionsProvider):
     data, but is less reliable and often offline."""
 
     def __init__(self, provider_config):
-        super().__init__(provider_config)
+        super().__init__("GIRO Ionosonde Data", provider_config)
         self._stations = self._load_stations()
         self._thread = None
         self._stop_event = Event()
@@ -127,10 +128,18 @@ class GIROIonosonde(SolarConditionsProvider):
         from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S")
         to_str = to_time.strftime("%Y.%m.%d+%H:%M:%S")
         url = f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUFD,fmin&DMUF=3000&fromDate={from_str}&toDate={to_str}"
-        response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15))
-        if response.status_code != 200:
+        try:
+            http_response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15))
+            if not http_response.ok:
+                logging.warning(f"HTTP {http_response.status_code} when calling Giro ionosonde API.")
+                return None, None, None
+            return self._parse_all(http_response.text)
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing Giro ionosonde API.")
+            return None, None, None
+        except ConnectionError:
+            logging.warning("Connection error when accessing Giro ionosonde API.")
             return None, None, None
-        return self._parse_all(response.text)
 
     @staticmethod
     def _parse_all(text):

solarconditionsproviders/hamqsl.py

@@ -15,13 +15,9 @@ class HamQSL(HTTPSolarConditionsProvider):
     Provides solar flux index, geomagnetic indices, and HF/VHF propagation condition summaries."""
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, URL, POLL_INTERVAL)
+        super().__init__("HamQSL", provider_config, URL, POLL_INTERVAL)
 
     def _http_response_to_solar_conditions(self, http_response):
-        if http_response.status_code != 200:
-            logging.warning("HamQSL solar conditions API returned HTTP " + str(http_response.status_code))
-            return None
-
         root = ElementTree.fromstring(http_response.text)
         sd = root.find("solardata")
         if sd is None:

solarconditionsproviders/http_solar_conditions_provider.py

@@ -4,6 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.solar_conditions_provider import SolarConditionsProvider
@@ -13,8 +14,8 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
     """Generic solar conditions provider for providers that request data via HTTP(S). Subclasses implement
     _http_response_to_solar_conditions() to parse the specific API response format."""
 
-    def __init__(self, provider_config, url, poll_interval):
-        super().__init__(provider_config)
+    def __init__(self, name, provider_config, url, poll_interval):
+        super().__init__(name, provider_config)
         self._url = url
         self._poll_interval = poll_interval
         self._thread = None
@@ -39,13 +40,22 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
         try:
             logging.debug("Polling " + self.name + " solar conditions API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            # Check response code was good
+            if http_response.ok:
                 new_data = self._http_response_to_solar_conditions(http_response)
                 self.update_data(new_data)
 
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
                 logging.debug("Received data from " + self.name + " solar conditions API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} solar conditions API.")
 
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} solar conditions API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} solar conditions API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP Solar Conditions Provider (" + self.name + ")")

solarconditionsproviders/kc2gprop.py

@@ -4,6 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.ionosonde_utils import compute_band_states
@@ -23,7 +24,7 @@ class KC2GProp(SolarConditionsProvider):
     online, but has fewer stations and does not provide LUF data."""
 
     def __init__(self, provider_config):
-        super().__init__(provider_config)
+        super().__init__("KC2G Propagation Data", provider_config)
         self._thread = None
         self._stop_event = Event()
 
@@ -44,9 +45,9 @@ class KC2GProp(SolarConditionsProvider):
     def _poll(self):
         try:
             logging.debug("Polling KC2G ionosonde data...")
-            response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30))
-            if response.status_code != 200:
-                logging.warning(f"KC2G ionosonde API returned HTTP {response.status_code}")
+            http_response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30))
+            if not http_response.ok:
+                logging.warning(f"HTTP {http_response.status_code} when calling KG2G ionosonde API.")
                 return
 
             now = datetime.now(timezone.utc)
@@ -57,7 +58,7 @@ class KC2GProp(SolarConditionsProvider):
             ionosonde_data = dict(self._solar_conditions.ionosonde_data or {})
             updated_count = 0
 
-            for reading in response.json():
+            for reading in http_response.json():
                 station = reading.get("station", {})
                 ursi = station.get("code")
                 name = station.get("name")
@@ -115,6 +116,10 @@ class KC2GProp(SolarConditionsProvider):
             self.last_update_time = datetime.now(pytz.UTC)
             logging.debug(f"Updated KC2G ionosonde data for {updated_count} stations.")
 
+        except ConnectionError:
+            logging.warning("Connection error when accessing KC2G ionosonde API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing KC2G ionosonde API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in KC2G ionosonde data provider")

solarconditionsproviders/noaa3dayforecast.py

@@ -13,7 +13,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
     corresponding fields in the solar conditions object.."""
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, URL, POLL_INTERVAL)
+        super().__init__("NOAA 3-day Forecast", provider_config, URL, POLL_INTERVAL)
 
     @staticmethod
     def _parse_percentage_table(lines, section_header, year):
@@ -80,10 +80,6 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
         return result if result else None
 
     def _http_response_to_solar_conditions(self, http_response):
-        if http_response.status_code != 200:
-            logging.warning("NOAA K-index forecast API returned HTTP " + str(http_response.status_code))
-            return None
-
         lines = http_response.text.splitlines()
 
         # Find the "NOAA Kp index breakdown" section header

solarconditionsproviders/solar_conditions_provider.py

@@ -7,11 +7,11 @@ class SolarConditionsProvider:
     """Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
      propagation data."""
 
-    def __init__(self, provider_config):
+    def __init__(self, name, provider_config):
         """Constructor"""
 
         self._solar_conditions_cache = None
-        self.name = provider_config["name"]
+        self.name = name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"

spothole.py

@@ -8,7 +8,7 @@ import sys
 from diskcache import Cache
 
 from core.cleanup import CleanupTimer
-from core.config import config, SERVER_OWNER_CALLSIGN
+from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL
 from core.constants import SOFTWARE_VERSION
 from core.lookup_helper import lookup_helper
 from core.status_reporter import StatusReporter
@@ -84,9 +84,9 @@ def get_solar_conditions_provider_from_config(config_providers_entry):
 if __name__ == '__main__':
     # Set up logging
     root = logging.getLogger()
-    root.setLevel(logging.INFO)
+    root.setLevel(LOG_LEVEL)
     handler = logging.StreamHandler(sys.stdout)
-    handler.setLevel(logging.INFO)
+    handler.setLevel(LOG_LEVEL)
     formatter = logging.Formatter("%(levelname)s : %(message)s")
     handler.setFormatter(formatter)
     root.handlers.clear()

spotproviders/aprsis.py

@@ -14,7 +14,7 @@ class APRSIS(SpotProvider):
     """Spot provider for the APRS-IS."""
 
     def __init__(self, provider_config):
-        super().__init__(provider_config)
+        super().__init__("APRS-IS", provider_config)
         self._thread = Thread(target=self._connect)
         self._thread.daemon = True
         self._aprsis = None

spotproviders/dxcluster.py

@@ -26,7 +26,8 @@ class DXCluster(SpotProvider):
     def __init__(self, provider_config):
         """Constructor requires hostname and port"""
 
-        super().__init__(provider_config)
+        name = provider_config["name"] if "name" in provider_config else "Cluster"
+        super().__init__(name, provider_config)
         self._hostname = provider_config["host"]
         self._port = provider_config["port"]
         self._login_prompt = provider_config["login_prompt"] if "login_prompt" in provider_config else "login:"
@@ -59,6 +60,10 @@ class DXCluster(SpotProvider):
                     self._telnet.write((self._login_callsign + "\n").encode("latin-1"))
                     connected = True
                     logging.info("DX Cluster " + self._hostname + " connected.")
+                except ConnectionRefusedError:
+                    self.status = "Error"
+                    logging.warning("Connection refused to DX cluster " + self._hostname)
+                    sleep(300)
                 except Exception:
                     self.status = "Error"
                     logging.exception("Exception while connecting to DX Cluster Provider (" + self._hostname + ").")

spotproviders/gma.py

@@ -14,9 +14,9 @@ class GMA(HTTPSpotProvider):
     """Spot provider for General Mountain Activity"""
 
     POLL_INTERVAL_SEC = 120
-    SPOTS_URL = "https://www.cqgma.org/api/spots/25/"
+    SPOTS_URL = "https://www.gma.rocks/api/spots/25/"
     # GMA spots don't contain the details of the programme they are for, we need a separate lookup for that
-    REF_INFO_URL_ROOT = "https://www.cqgma.org/api/ref/?"
+    REF_INFO_URL_ROOT = "https://www.gma.rocks/api/ref/?"
 
     def __init__(self, provider_config):
         # Ensure there is an API key in our config, and set up the query URL using it. If no key is provided,
@@ -26,7 +26,7 @@ class GMA(HTTPSpotProvider):
             provider_config["enabled"] = False
             logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
 
-        super().__init__(provider_config, self.SPOTS_URL + "?key=" + self.api_key, self.POLL_INTERVAL_SEC)
+        super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self.api_key, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -58,8 +58,8 @@ class GMA(HTTPSpotProvider):
                     try:
                         ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
                                                                       headers=HTTP_HEADERS)
-                        # Sometimes this is blank, so handle that
-                        if ref_response.text is not None and ref_response.text != "":
+                        # Sometimes this is blank even if it's a 200 response, so handle that
+                        if ref_response.ok and ref_response.text is not None and ref_response.text != "":
                             ref_info = ref_response.json()
                             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
                             # spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
@@ -94,9 +94,17 @@ class GMA(HTTPSpotProvider):
                                         spot.sig_refs[0].sig = ref_info["reftype"]
                                         spot.sig = ref_info["reftype"]
 
-                                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
-                                # that for us.
+                                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point;
+                                # other code will do that for us.
                                 new_spots.append(spot)
+
+                        elif not ref_response.from_cache:
+                            if not ref_response.ok:
+                                logging.warning(
+                                    f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot["REF"]}")
+                            else:
+                                logging.warning(
+                                    f"GMA API returned a malformed response when looking up ref {source_spot["REF"]}")
                     except:
                         logging.warning("Exception when looking up " + self.REF_INFO_URL_ROOT + source_spot[
                             "REF"] + ", ignoring this spot for now")

spotproviders/hema.py

@@ -1,8 +1,10 @@
+import logging
 import re
 from datetime import datetime
 
 import pytz
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
@@ -23,7 +25,7 @@ class HEMA(HTTPSpotProvider):
     SPOTTER_COMMENT_PATTERN = re.compile("^\\((.*)\\) (.*)$")
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOT_SEED_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("HEMA", provider_config, self.SPOT_SEED_URL, self.POLL_INTERVAL_SEC)
         self._spot_seed = ""
 
     def _http_response_to_spots(self, http_response):
@@ -35,6 +37,7 @@ class HEMA(HTTPSpotProvider):
         new_spots = []
         # OK, if the spot seed actually changed, now we make the real request for data.
         if spot_seed_changed:
+            try:
                 source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30))
                 source_data_items = source_data.text.split("=")
                 # Iterate through source data items.
@@ -62,7 +65,11 @@ class HEMA(HTTPSpotProvider):
                                     dx_latitude=float(spot_items[7]),
                                     dx_longitude=float(spot_items[8]))
 
-                    # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
-                    # that for us.
+                        # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other
+                        # code will do that for us.
                         new_spots.append(spot)
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when accessing HEMA spots API.")
+            except ConnectionError:
+                logging.warning("Connection error when accessing HEMA spots API.")
         return new_spots

spotproviders/http_spot_provider.py

@@ -4,6 +4,8 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from spotproviders.spot_provider import SpotProvider
@@ -13,8 +15,8 @@ class HTTPSpotProvider(SpotProvider):
     """Generic spot provider class for providers that request data via HTTP(S). Just for convenience to avoid code
     duplication. Subclasses of this query the individual APIs for data."""
 
-    def __init__(self, provider_config, url, poll_interval):
-        super().__init__(provider_config)
+    def __init__(self, name, provider_config, url, poll_interval):
+        super().__init__(name, provider_config)
         self._url = url
         self._poll_interval = poll_interval
         self._thread = None
@@ -41,6 +43,8 @@ class HTTPSpotProvider(SpotProvider):
             # Request data from API
             logging.debug("Polling " + self.name + " spot API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            # Check response code was good
+            if http_response.ok:
                 # Pass off to the subclass for processing
                 new_spots = self._http_response_to_spots(http_response)
                 # Submit the new spots for processing. There might not be any spots for the less popular programs.
@@ -50,7 +54,14 @@ class HTTPSpotProvider(SpotProvider):
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
                 logging.debug("Received data from " + self.name + " spot API.")
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} spot API.")
 
+        except ConnectionError:
+            logging.warning(f"Connection error when accessing {self.name} spots API.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when accessing {self.name} spots API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP JSON Spot Provider (" + self.name + ")")

spotproviders/llota.py

@@ -12,7 +12,7 @@ class LLOTA(HTTPSpotProvider):
     SPOTS_URL = "https://llota.app/api/public/spots"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("LLOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []

spotproviders/parksnpeaks.py

@@ -17,7 +17,7 @@ class ParksNPeaks(HTTPSpotProvider):
     SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("ParksNPeaks", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -30,8 +30,8 @@ class ParksNPeaks(HTTPSpotProvider):
                             dx_call=source_spot["actCallsign"].upper(),
                             de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,
                             # typo exists in API
-                            freq=float(source_spot["actFreq"].replace(",", "")) * 1000000 if (
-                                    source_spot["actFreq"] != "") else None,
+                            freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "")
+                                       .replace("+/-", "").strip()) * 1000000 if (source_spot["actFreq"] != "") else None,
                             # Seen PNP spots with empty frequency, and with comma-separated thousands digits
                             mode=source_spot["actMode"].upper(),
                             comment=source_spot["actComments"],

spotproviders/pota.py

@@ -14,7 +14,7 @@ class POTA(HTTPSpotProvider):
     SPOTS_URL = "https://api.pota.app/spot/activator"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("POTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []

spotproviders/rbn.py

@@ -23,7 +23,8 @@ class RBN(SpotProvider):
     def __init__(self, provider_config):
         """Constructor requires port number."""
 
-        super().__init__(provider_config)
+        name = provider_config["name"] if "name" in provider_config else "RBN"
+        super().__init__(name, provider_config)
         self._port = provider_config["port"]
         self._telnet = None
         self._thread = Thread(target=self._handle)

spotproviders/sota.py

@@ -1,6 +1,8 @@
+import logging
 from datetime import datetime
 
 import requests
+from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
@@ -21,7 +23,7 @@ class SOTA(HTTPSpotProvider):
     SUMMIT_URL_ROOT = "https://api-db2.sota.org.uk/api/summits/"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("SOTA", provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC)
         self._api_epoch = ""
 
     def _http_response_to_spots(self, http_response):
@@ -33,6 +35,7 @@ class SOTA(HTTPSpotProvider):
         new_spots = []
         # OK, if the epoch actually changed, now we make the real request for data.
         if epoch_changed:
+            try:
                 source_data = requests.get(self.SPOTS_URL, headers=HTTP_HEADERS, timeout=(5, 30)).json()
                 # Iterate through source data
                 for source_spot in source_data:
@@ -55,4 +58,8 @@ class SOTA(HTTPSpotProvider):
                     # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
                     # that for us.
                     new_spots.append(spot)
+            except ConnectionError:
+                logging.warning("Connection error when accessing SOTA spots API")
+            except (ConnectTimeout, ReadTimeout):
+                logging.warning(f"Timeout when accessing SOTA spots API.")
         return new_spots

spotproviders/spot_provider.py

@@ -8,10 +8,10 @@ from core.config import MAX_SPOT_AGE
 class SpotProvider:
     """Generic spot provider class. Subclasses of this query the individual APIs for data."""
 
-    def __init__(self, provider_config):
+    def __init__(self, name, provider_config):
         """Constructor"""
 
-        self.name = provider_config["name"]
+        self.name = name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)

spotproviders/sse_spot_provider.py

@@ -13,8 +13,8 @@ from spotproviders.spot_provider import SpotProvider
 class SSESpotProvider(SpotProvider):
     """Spot provider using Server-Sent Events."""
 
-    def __init__(self, provider_config, url):
-        super().__init__(provider_config)
+    def __init__(self, name, provider_config, url):
+        super().__init__(name, provider_config)
         self._url = url
         self._event_source = None
         self._thread = None

spotproviders/tiles.py

@@ -12,7 +12,7 @@ class Tiles(HTTPSpotProvider):
     SPOTS_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/spots?active_hours=24"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("Tiles", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []

spotproviders/towers.py

@@ -13,7 +13,7 @@ class Towers(HTTPSpotProvider):
     SPOTS_URL = "https://wwtota.com/api/cluster_live.php"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("Towers", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []

spotproviders/ukpacketnet.py

@@ -14,7 +14,7 @@ class UKPacketNet(HTTPSpotProvider):
     SPOTS_URL = "https://nodes.ukpacketradio.network/api/nodedata"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("UK Packet Net", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []

spotproviders/websocket_spot_provider.py

@@ -13,8 +13,8 @@ from spotproviders.spot_provider import SpotProvider
 class WebsocketSpotProvider(SpotProvider):
     """Spot provider using websockets."""
 
-    def __init__(self, provider_config, url):
-        super().__init__(provider_config)
+    def __init__(self, name, provider_config, url):
+        super().__init__(name, provider_config)
         self._url = url
         self._ws = None
         self._thread = None

spotproviders/wota.py

@@ -21,7 +21,7 @@ class WOTA(HTTPSpotProvider):
     RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("WOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []

spotproviders/wwbota.py

@@ -12,7 +12,7 @@ class WWBOTA(SSESpotProvider):
     SPOTS_URL = "https://api.wwbota.net/spots/"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL)
+        super().__init__("WWBOTA", provider_config, self.SPOTS_URL)
 
     def _sse_message_to_spot(self, message):
         source_spot = json.loads(message)

spotproviders/wwff.py

@@ -14,7 +14,7 @@ class WWFF(HTTPSpotProvider):
     SPOTS_URL = "https://spots.wwff.co/static/spots.json"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("WWFF", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []

spotproviders/xota.py

@@ -21,7 +21,8 @@ class XOTA(WebsocketSpotProvider):
     SIG = None
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, provider_config["url"])
+        name = provider_config["name"] if "name" in provider_config else "xOTA"
+        super().__init__(name, provider_config, provider_config["url"])
         locations_csv = str(provider_config["locations-csv"]) if "locations-csv" in provider_config else None
         self.SIG = str(provider_config["sig"]) if "sig" in provider_config else None
 

spotproviders/zlota.py

@@ -15,7 +15,7 @@ class ZLOTA(HTTPSpotProvider):
     LIST_URL = "https://ontheair.nz/assets/assets.json"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("ZLOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []