Improve use of URL cache #118

README.md

@@ -92,7 +92,7 @@ Various approaches exist to writing your own client, but in general:
 * Refer to the API docs. These are built on an OpenAPI definition file (`/static/apidocs/openapi.yml`), which you can
   automatically use to generate a client skeleton using various software.
 * Call the main "spots" or "alerts" API endpoints to get the data you want. For example, your app could call
-  `https://spothole.app/api/v2/spots` once every few minutes. Apply filters if necessary.
+  `https://spothole.app/api/v1/spots` once every few minutes. Apply filters if necessary.
 * Call the "options" API to get an idea of which bands, modes etc. the server knows about. You might want to do that
   first before calling the spots/alerts APIs, to allow you to populate your filters correctly.
 * Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "alerts"/"upcoming"
@@ -105,12 +105,12 @@ once every two minutes, so if your client is interested in POTA data there's no
 than that.
 
 If you absolutely must be informed within seconds of a spot arriving in Spothole, please use the SSE endpoints instead,
-e.g. `https://spothole.app/api/v2/spots/stream`.
+e.g. `https://spothole.app/api/v1/spots/stream`.
 
 If you want to handle different types of spot or alert differently within your client, please consider making a single
 request to the Spothole API to retrieve all the data, then filtering on your side. For example, call
-`https://spothole.app/api/v2/spots?sig=POTA,SOTA` rather than making two separate calls to
+`https://spothole.app/api/v1/spots?sig=POTA,SOTA` rather than making two separate calls to
-`https://spothole.app/api/v2/spots?sig=POTA` and `https://spothole.app/api/v2/spots?sig=SOTA`.
+`https://spothole.app/api/v1/spots?sig=POTA` and `https://spothole.app/api/v1/spots?sig=SOTA`.
 
 Remember, here at Spothole Inc. we offer an industry-standard "five nines" uptime on our server, with our own unique
 twist: we don't tell you which side of the decimal point the nines start! (Translation: This is a hobby project.
@@ -354,7 +354,7 @@ server {
     }
     
     # SSE endpoints
-    location ~ ^/api/v2/(spots|alerts)/stream/? {
+    location ~ ^/api/v1/(spots|alerts)/stream/? {
         proxy_pass http://127.0.0.1:8080;
         
         # Remove buffering, remove caching, add suitable timeouts for SSE API calls
@@ -519,9 +519,6 @@ This project would not have been possible without these libraries, so many thank
 A number of third-party libraries are self-hosted in the `/static/vendor/` directory. These files are subject to
 their own licences and are not covered by the overall licence declared in the `LICENSE` file.
 
-A number of third-party libraries are self-hosted in the `/webassets/vendor/` directory. These files are subject to
-their own licences and are not covered by the overall licence declared in the `LICENSE` file.
-
 Particular thanks go to country-files.com for providing country lookup data for amateur radio, to K0SWE
 for [this JSON-formatted DXCC data](https://github.com/k0swe/dxcc-json/), and to the developers of `pyhamtools` for
 making it easy to use country-files.com data as well as QRZ.com and Clublog lookup.

alertproviders/alert_provider.py

@@ -2,27 +2,20 @@ from datetime import datetime
 
 import pytz
 
-from core.config import MAX_ALERT_AGE
+from core.data_store import DATA_STORE
 
 
 class AlertProvider:
     """Generic alert provider class. Subclasses of this query the individual APIs for alerts."""
 
-    def __init__(self, provider_config):
+    def __init__(self, name, provider_config):
         """Constructor"""
 
-        self.name = provider_config["name"]
+        self.name = name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._alerts = None
+        self._alerts = DATA_STORE.alerts
-        self._web_server = None
-
-    def setup(self, alerts, web_server):
-        """Set up the provider, e.g. giving it the alert list to work from"""
-
-        self._alerts = alerts
-        self._web_server = web_server
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -44,10 +37,7 @@ class AlertProvider:
 
     def _add_alert(self, alert):
         if not alert.expired():
-            self._alerts.add(alert.id, alert, expire=MAX_ALERT_AGE)
+            self._alerts.set(alert.id, alert)
-            # Ping the web server in case we have any SSE connections that need to see this immediately
-            if self._web_server:
-                self._web_server.notify_new_alert(alert)
 
     def stop(self):
         """Stop any threads and prepare for application shutdown"""

alertproviders/bota.py

@@ -15,12 +15,12 @@ class BOTA(HTTPAlertProvider):
     ALERTS_URL = "https://www.beachesontheair.com/"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("BOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
         # Find the table of upcoming alerts
-        bs = BeautifulSoup(http_response.content.decode(), features="lxml")
+        bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml")
         if not bs.body:
             return new_alerts
         div = bs.body.find('div', attrs={'class': 'view-activations-public'})

alertproviders/http_alert_provider.py

@@ -14,8 +14,8 @@ class HTTPAlertProvider(AlertProvider):
     """Generic alert provider class for providers that request data via HTTP(S). Just for convenience to avoid code
     duplication. Subclasses of this query the individual APIs for data."""
 
-    def __init__(self, provider_config, url, poll_interval):
+    def __init__(self, name, provider_config, url, poll_interval):
-        super().__init__(provider_config)
+        super().__init__(name, provider_config)
         self._url = url
         self._poll_interval = poll_interval
         self._thread = None

alertproviders/ng3k.py

@@ -18,11 +18,11 @@ class NG3K(HTTPAlertProvider):
     AS_CALL_PATTERN = re.compile("as ([a-z0-9/]+)", re.IGNORECASE)
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("NG3K", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
-        rss = cast(RSS, Parser.parse(http_response.content.decode()))
+        rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
         # Iterate through source data
         for source_alert in rss.channel.items:
             # Deal with "the format"...

alertproviders/parksnpeaks.py

@@ -15,14 +15,14 @@ class ParksNPeaks(HTTPAlertProvider):
     ALERTS_URL = "https://parksnpeaks.org/api/ALERTS/"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("ParksNPeaks", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
         # Iterate through source data
         for source_alert in http_response.json():
             # Calculate some things
-            sig = source_alert["Class"]
+            sig = source_alert["Class"].upper()
             if " - " in source_alert["Location"]:
                 split = source_alert["Location"].split(" - ")
                 sig_ref = split[0]
@@ -50,7 +50,7 @@ class ParksNPeaks(HTTPAlertProvider):
                           is_dxpedition=False)
 
             # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
-            if sig and sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]:
+            if sig and sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]:
                 logging.warning("PNP alert found with sig " + sig + ", developer needs to add support for this!")
 
             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to

alertproviders/pota.py

@@ -14,7 +14,7 @@ class POTA(HTTPAlertProvider):
     ALERTS_URL = "https://api.pota.app/activation"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("POTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []

alertproviders/sota.py

@@ -14,7 +14,7 @@ class SOTA(HTTPAlertProvider):
     ALERTS_URL = "https://api-db2.sota.org.uk/api/alerts/365/all/all"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("SOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []

alertproviders/wota.py

@@ -18,11 +18,11 @@ class WOTA(HTTPAlertProvider):
     RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("WOTA", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
-        rss = cast(RSS, RSSParser.parse(http_response.content.decode()))
+        rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig")))
         # Iterate through source data
         for source_alert in rss.channel.items:
 

alertproviders/wwff.py

@@ -14,7 +14,7 @@ class WWFF(HTTPAlertProvider):
     ALERTS_URL = "https://spots.wwff.co/static/agendas.json"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("WWFF", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []

config-example.yml

@@ -19,66 +19,55 @@ base-url: "http://localhost:8080"
 
 # Spot providers to use. This is an example set, tailor it to your liking by commenting and uncommenting.
 # RBN and APRS-IS are supported but have such a high data rate, you probably don't want them enabled.
-# Each provider needs a class, a name, and an enabled/disabled state. Some require more config such as hostnames/IP
+# Each provider needs a class and an enabled/disabled state. Some require more config such as hostnames/IP
 # addresses and ports. You can duplicate them if you like, e.g. to support several DX clusters. RBN uses two ports, 7000
 # for CW/RTTY and 7001 for FT8, so if you want both, you need two entries, as shown below.
 # Feel free to write your own provider classes! There are details in the README.
 spot-providers:
   - class: "POTA"
-    name: "POTA"
     enabled: true
 
   - class: "SOTA"
-    name: "SOTA"
     enabled: true
 
   - class: "WWFF"
-    name: "WWFF"
     enabled: true
 
   - class: "WWBOTA"
-    name: "WWBOTA"
     enabled: true
 
   - class: "GMA"
-    name: "GMA"
     enabled: true
     # GMA requires an API key to fetch spots. After creating an account on cqgma.org, email support and request one.
     api-key: ""
 
   - class: "HEMA"
-    name: "HEMA"
     enabled: true
 
   - class: "ParksNPeaks"
-    name: "ParksNPeaks"
     enabled: true
 
   - class: "ZLOTA"
-    name: "ZLOTA"
     enabled: true
 
   - class: "WOTA"
-    name: "WOTA"
     enabled: true
 
   - class: "LLOTA"
-    name: "LLOTA"
     enabled: true
 
   - class: "Towers"
-    name: "Towers"
     enabled: true
 
   - class: "Tiles"
-    name: "Tiles"
     enabled: true
 
   - class: "APRSIS"
-    name: "APRS-IS"
     enabled: false
 
   - class: "DXCluster"
+    # Clusters have a "name" property in case you want to say which cluster node the data is from, or connect to
+    # several clusters and name them separately.
     name: "HRD Cluster"
     enabled: true
     host: "hrd.wa9pie.net"
@@ -99,18 +88,12 @@ spot-providers:
     enabled: false
     host: "w3lpl.net"
     port: 7373
-    # Prompt the cluster node gives when asking for a callsign to log in. Varies between cluster node software.
     login_prompt: "Please enter your call:"
-    # Callsign Spothole will use to log into this cluster. Ensure the SSID (e.g. -99) is different to any personal
-    # connection you might make to this cluster node.
     login_callsign: "N0CALL-99"
-    # Whether to allow RBN spots that come via this cluster. If you don't want RBN spots or you are making a separate
-    # connection to RBN directly, leave this as False. If you want RBN spots from this cluster, set this to True. (Make
-    # sure you aren't also separately connecting to RBN directly, otherwise you may get duplicate spots.) Note that not
-    # all clusters sent RBN spots anyway.
     allow_rbn_spots: false
 
   - class: "RBN"
+  - # RBN sources have a "name" property in case you want to differentiate between the CW/RTTY and FT8 sources.
     name: "RBN CW/RTTY"
     enabled: false
     port: 7000
@@ -127,76 +110,129 @@ spot-providers:
     enabled-by-default-in-web-ui: false
 
   - class: "UKPacketNet"
-    name: "UK Packet Radio Net"
     enabled: false
     enabled-by-default-in-web-ui: false
 
   - class: "XOTA"
-    name: "39C3 TOTA"
+    # xOTA sources have a "name" property so you can define what programme it is for, since xOTA is generic software
+    # for running "something-on-the-air" programmes.
+    name: "C3 TOTA"
     enabled: false
     url: "wss://39c3.totawatch.de/api/spot/live"
-    # Fixed SIG for all spots from a provider & location CSV are currently only a feature for the "XOTA" provider,
+    # For the "XOTA" provider, a SIG must be set menually here because xOTA is a generic backend for xOTA
-    # the software found at https://github.com/nischu/xOTA/. This is because this is a generic backend for xOTA
+    # programmes and so different URLs potentially provide different programmes.
-    # programmes and so different URLs provide different programmes.
+    sig: "Toilets"
-    sig: "TOTA"
+    # For Toilets on the Air, we prefix the SIG references (T-01 etc) with some characters that define the conference:
-    locations-csv: "datafiles/39c3-tota.csv"
+    # C3, EH or HOPE - so we can look up the correct locations in our database, because each conference starts from T-01
+    # but refers to a toilet in a different building (or continent!)
+    sig-ref-prefix: "C3"
 
   - class: "XOTA"
     name: "EH23 TOTA"
     enabled: false
     url: "wss://eh23.totawatch.de/api/spot/live"
-    sig: "TOTA"
+    sig: "Toilets"
-    locations-csv: "datafiles/eh23-tota.csv"
+    sig-ref-prefix: "EH"
+
+  - class: "XOTA"
+    name: "HOPE26 TOTA"
+    enabled: false
+    url: "wss://hope-26.totawatch.de/api/spot/live"
+    sig: "Toilets"
+    sig-ref-prefix: "HOPE"
 
 
 # Alert providers to use. Same setup as the spot providers list above.
 alert-providers:
   - class: "POTA"
-    name: "POTA"
     enabled: true
 
   - class: "SOTA"
-    name: "SOTA"
     enabled: true
 
   - class: "WWFF"
-    name: "WWFF"
     enabled: true
 
   - class: "ParksNPeaks"
-    name: "ParksNPeaks"
     enabled: true
 
   - class: "WOTA"
-    name: "WOTA"
     enabled: true
 
   - class: "BOTA"
-    name: "BOTA"
     enabled: true
 
   - class: "NG3K"
-    name: "NG3K"
     enabled: true
 
 
+# SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF
+# park IDs to their name and location.
+sig-ref-data-providers:
+  - class: "POTA"
+    enabled: true
+
+  - class: "SOTA"
+    enabled: true
+
+  - class: "WWFF"
+    enabled: true
+
+  - class: "WWBOTA"
+    enabled: true
+
+  - class: "GMA"
+    enabled: true
+
+  - class: "MOTA"
+    enabled: true
+
+  - class: "ILLW"
+    enabled: true
+
+  - class: "ARLHS"
+    enabled: true
+
+  - class: "WCA"
+    enabled: true
+
+  - class: "IOTA"
+    enabled: true
+
+  - class: "WOTA"
+    enabled: true
+
+  - class: "SIOTA"
+    enabled: true
+
+  - class: "ZLOTA"
+    enabled: true
+
+  - class: "LLOTA"
+    enabled: true
+
+  - class: "Towers"
+    enabled: true
+
+  - class: "DME"
+    enabled: true
+
+  - class: "Toilets"
+    enabled: true
+
 # Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band
-# conditions, etc.) and make it available via the /api/v2/solar endpoint.
+# conditions, etc.) and make it available via the /api/v1/solar endpoint.
 solar-condition-providers:
   - class: "HamQSL"
-    name: "HamQSL"
     enabled: true
 
   - class: "NOAA3dayForecast"
-    name: "NOAA 3-day Forecast"
     enabled: true
 
   - class: "GIROIonosonde"
-    name: "GIRO Ionosonde Data"
     enabled: true
 
   - class: "KC2GProp"
-    name: "KC2G Propagation Data"
     enabled: true
 
 # Maximum time to keep spots and alerts in the system before deleting them. By default, one hour for spots and one week
@@ -211,20 +247,6 @@ clublog-api-key: ""
 # Allow submitting spots to the Spothole API?
 allow-spotting: true
 
-# Allow upstream submission of spots to external providers (POTA, SOTA, etc.) via the API?
-# Requires allow-spotting to also be true. Set to false to only accept spots into the local
-# Spothole database, without forwarding them to any external service.
-allow-upstream-spotting: true
-
-# Google reCAPTCHA v2 keys for CAPTCHA protection on upstream spot submission. Both keys must be set to enable CAPTCHA.
-# Leave both empty to disable CAPTCHA (e.g. for a private/trusted server) or if allow-spotting is false, in which case
-# they will do nothing. Note that with CAPTCHA enabled, this will prevent third-party clients submitting spots through
-# Spothole unless the clients are web-based, use the same site key, have their domains enabled in your reCAPTCHA config,
-# and of course their user solves the CAPTCHA.
-# You can sign up for reCAPTCHA at https://www.google.com/recaptcha/
-recaptcha-site-key: ""
-recaptcha-secret-key: ""
-
 # Log web requests? Useful to see what your users are requesting in terms of pages and API endpoints, but will fill up
 # your log quickly on a popular server.
 log-web-requests: false

core/cache_utils.py

@@ -1,27 +0,0 @@
-import threading
-from datetime import timedelta
-
-from requests_cache import CachedSession
-
-# Cache for "semi-static" data such as the locations of parks, CSVs of reference lists, etc.
-# This has an expiry time of 30 days, so will re-request from the source after that amount
-# of time has passed. This is used throughout Spothole to cache data that does not change
-# rapidly. The ThreadSafeSession construct here protects it against some multithreading
-# contention weirdness we sometimes used to see on startup where the cache was hammered
-# pretty hard. The expanded list of allowable_codes ensures we also cache and return 400-type
-# responses, e.g "this SOTA summit ref doesn't actually exist", to avoid hammering remote
-# servers for data they've told us they can't provide.
-_session = CachedSession("cache/semi_static_url_data_cache", expire_after=timedelta(days=30),
-                         allowable_codes=(200, 400, 401, 403, 404))
-_lock = threading.Lock()
-
-
-class _ThreadSafeSession:
-    """Wraps CachedSession with a lock to prevent concurrent SQLite access across threads."""
-
-    def get(self, *args, **kwargs):
-        with _lock:
-            return _session.get(*args, **kwargs)
-
-
-SEMI_STATIC_URL_DATA_CACHE = _ThreadSafeSession()

core/cleanup.py

@@ -1,73 +0,0 @@
-import logging
-from datetime import datetime
-from threading import Event, Thread
-
-import pytz
-
-
-class CleanupTimer:
-    """Provides a timed cleanup of the spot list."""
-
-    def __init__(self, spots, alerts, web_server, cleanup_interval):
-        """Constructor"""
-
-        self._spots = spots
-        self._alerts = alerts
-        self._web_server = web_server
-        self._cleanup_interval = cleanup_interval
-        self.last_cleanup_time = datetime.min.replace(tzinfo=pytz.UTC)
-        self.status = "Starting"
-        self._thread = None
-        self._stop_event = Event()
-
-    def start(self):
-        """Start the cleanup timer"""
-
-        self._thread = Thread(target=self._run, daemon=True)
-        self._thread.start()
-
-    def stop(self):
-        """Stop any threads and prepare for application shutdown"""
-
-        self._stop_event.set()
-
-    def _run(self):
-        while not self._stop_event.wait(timeout=self._cleanup_interval):
-            self._cleanup()
-
-    def _cleanup(self):
-        """Perform cleanup and reschedule next timer"""
-
-        try:
-            # Perform cleanup via letting the data expire
-            self._spots.expire()
-            self._alerts.expire()
-
-            # Explicitly clean up any spots and alerts that have expired
-            for i in list(self._spots.iterkeys()):
-                try:
-                    spot = self._spots[i]
-                    if spot.expired():
-                        self._spots.delete(i)
-                except KeyError:
-                    # Must have already been deleted, OK with that
-                    pass
-            for i in list(self._alerts.iterkeys()):
-                try:
-                    alert = self._alerts[i]
-                    if alert.expired():
-                        self._alerts.delete(i)
-                except KeyError:
-                    # Must have already been deleted, OK with that
-                    pass
-
-            # Clean up web server SSE spot/alert queues
-            self._web_server.clean_up_sse_queues()
-
-            self.status = "OK"
-            self.last_cleanup_time = datetime.now(pytz.UTC)
-
-        except Exception:
-            self.status = "Error"
-            logging.exception("Exception in Cleanup thread")
-            self._stop_event.wait(timeout=1)

core/config.py

@@ -1,3 +1,4 @@
+import importlib
 import logging
 import os
 
@@ -14,17 +15,14 @@ with open("config.yml") as f:
     config = yaml.safe_load(f)
 logging.info("Loaded config.")
 
-BASE_URL = config.get("base-url", "http://localhost:8080")
+BASE_URL = config["base-url"]
-MAX_SPOT_AGE = config.get("max-spot-age-sec", 3600)
+MAX_SPOT_AGE = config["max-spot-age-sec"]
-MAX_ALERT_AGE = config.get("max-alert-age-sec", 604800)
+MAX_ALERT_AGE = config["max-alert-age-sec"]
-SERVER_OWNER_CALLSIGN = config.get("server-owner-callsign", "N0CALL")
+SERVER_OWNER_CALLSIGN = config["server-owner-callsign"]
-WEB_SERVER_PORT = config.get("web-server-port", 8080)
+WEB_SERVER_PORT = config["web-server-port"]
-ALLOW_SPOTTING = config.get("allow-spotting", True)
+ALLOW_SPOTTING = config["allow-spotting"]
-ALLOW_UPSTREAM_SPOTTING = config.get("allow-upstream-spotting", True)
+WEB_UI_OPTIONS = config["web-ui-options"]
-WEB_UI_OPTIONS = config.get("web-ui-options", {})
 API_ONLY_MODE = config.get("api-only-mode", False)
-RECAPTCHA_SECRET_KEY = config.get("recaptcha-secret-key", "")
-RECAPTCHA_SITE_KEY = config.get("recaptcha-site-key", "")
 LOG_LEVEL = config.get("log-level", "INFO")
 LOG_WEB_REQUESTS = config.get("log-web-requests", False)
 
@@ -33,9 +31,38 @@ LOG_WEB_REQUESTS = config.get("log-web-requests", False)
 WEB_UI_OPTIONS["spot-providers-enabled-by-default"] = [p["name"] for p in config["spot-providers"] if p["enabled"] and (
         "enabled-by-default-in-web-ui" not in p or p["enabled-by-default-in-web-ui"])]
 # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from
-# one of our proviers. We set that to also be enabled by default. We can also include the reCaptcha site key so the UI
+# one of our proviers. We set that to also be enabled by default.
-# can access it.
 if ALLOW_SPOTTING:
     WEB_UI_OPTIONS["spot-providers-enabled-by-default"].append("API")
-    WEB_UI_OPTIONS["recaptcha-site-key"] = RECAPTCHA_SITE_KEY
+
-WEB_UI_OPTIONS["allow-upstream-spotting"] = ALLOW_SPOTTING and ALLOW_UPSTREAM_SPOTTING
+
+def get_spot_provider_from_config(config_providers_entry):
+    """Utility method to get a spot provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('spotproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
+def get_alert_provider_from_config(config_providers_entry):
+    """Utility method to get an alert provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('alertproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
+def get_solar_conditions_provider_from_config(config_providers_entry):
+    """Utility method to get a solar conditions provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('solarconditionsproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
+def get_sig_ref_data_provider_from_config(config_providers_entry):
+    """Utility method to get a SIG reference data provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('sigrefdataproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)

core/constants.py

@@ -11,7 +11,7 @@ HAMQTH_PRG = ("Spothole v" + SOFTWARE_VERSION + " operated by " + SERVER_OWNER_C
 
 # Special Interest Groups
 SIGS = [
-    SIG(name="POTA", comment_names=["POTA"], description="Parks on the Air", ref_regex=r"[A-Z]{2}\-\d{4,5}|K\-TEST"),
+    SIG(name="POTA", comment_names=["POTA"], description="Parks on the Air", ref_regex=r"[A-Z]{2}\-\d{4,5}"),
     SIG(name="SOTA", comment_names=["SOTA"], description="Summits on the Air", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}"),
     SIG(name="WWFF", comment_names=["WWFF"], description="World Wide Flora & Fauna", ref_regex=r"[A-Z0-9]{1,3}FF\-\d{4}"),
     SIG(name="GMA", comment_names=["GMA"], description="Global Mountain Activity", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}"),
@@ -19,9 +19,9 @@ SIGS = [
     SIG(name="HEMA", comment_names=["HEMA"], description="HuMPs Excluding Marilyns Award", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}"),
     SIG(name="IOTA", comment_names=["IOTA"], description="Islands on the Air", ref_regex=r"[A-Z]{2}\-\d{3}"),
     SIG(name="MOTA", comment_names=["MOTA"], description="Mills on the Air", ref_regex=r"X\d{4,6}"),
-    SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}\-\d{3,4}"),
+    SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}[\- ]\d{3,4}"),
     SIG(name="ILLW", comment_names=["ILLW"], description="International Lighthouse & Lightship Weekend", ref_regex=r"[A-Z]{2}\d{4}"),
-    SIG(name="SiOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
+    SIG(name="SIOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
     SIG(name="WCA", comment_names=["WCA"], description="World Castles Award", ref_regex=r"[A-Z0-9]{1,3}\-\d{5}"),
     SIG(name="ZLOTA", comment_names=["ZLOTA"], description="New Zealand on the Air", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}"),
     SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"),
@@ -38,12 +38,10 @@ SIGS = [
 
 # Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
 CW_MODES = ["CW"]
-PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "FUSION", "M17"]
+PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "M17"]
 DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "PSK", "OLIVIA", "PKT", "MSK144"]
 ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
 MODE_TYPES = ["CW", "PHONE", "DATA"]
-SSB_SUB_MODES = ["USB", "LSB"]
-DV_SUB_MODES = ["DMR", "DSTAR", "C4FM", "FUSION", "M17"]
 
 # Mode aliases. Sometimes we get spots with a mode described in a different way that is effectively the same as a mode
 # we already know, or we want to normalise things for consistency. The lookup table for this is here. Incoming spots

core/data_store.py

@@ -0,0 +1,80 @@
+import logging
+from pathlib import Path
+
+import diskcache
+
+from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
+from core.live_data_cache import LiveDataCache
+from data.solar_conditions import SolarConditions
+
+CACHE_DIR = "./cache/"
+
+
+class DataStore:
+    """Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign
+    lookup data using different caching strategies for each."""
+
+    def __init__(self):
+        self._MAX_SPOT_COUNT = 100000
+        self._MAX_ALERT_COUNT = 100000
+        self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
+        self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
+        self.alerts = None
+        self.spots = None
+        self.callsigns = None
+        self.sigrefs = None
+        self.status_data = None
+        self._status = None
+        self.solar_conditions = None
+        self._solar = None
+
+    def setup(self):
+        Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
+
+        # Standard disk cache for solar data and status data, but each cache contains only a single object which we
+        # expose to the wider application
+        self._solar = diskcache.Cache(CACHE_DIR + "solar")
+        if "solar_conditions" not in self._solar:
+            self._solar.add("solar_conditions", SolarConditions())
+        self.solar_conditions = self._solar.get("solar_conditions")
+        self._status = diskcache.Cache(CACHE_DIR + "status")
+        if "status_data" not in self._status:
+            self._status.add("status_data", {})
+        self.status_data = self._status.get("status_data")
+
+        # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
+        # but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
+        # and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
+        # so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
+        self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs")
+        logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
+
+        # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
+        # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
+        # we've seen, so we rely on them timing out and this triggering another lookup.
+        self.callsigns = diskcache.Cache(CACHE_DIR + "callsigns")
+        logging.info(f"Loaded data for %d callsigns.", len(self.callsigns))
+
+        # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
+        # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
+        # before the spots and alerts are live in the system.
+        self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
+                                   snapshot_dir=CACHE_DIR + "spots",
+                                   snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
+        logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys()))
+
+        self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
+                                    snapshot_dir=CACHE_DIR + "alerts",
+                                    snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
+        logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))
+
+    def close(self):
+        self.spots.close()
+        self.alerts.close()
+        self._solar.close()
+        self._status.close()
+        self.sigrefs.close()
+        self.callsigns.close()
+
+# Global object
+DATA_STORE = DataStore()

core/live_data_cache.py

@@ -0,0 +1,100 @@
+import logging
+import threading
+import time
+
+import diskcache
+from cachetools import TTLCache
+
+
+class LiveDataCache:
+    """Cache for spots and alerts. Uses the fast in-memory TTLCache for normal data I/O, including the TTL to enforce
+    maximum lifetime, and adds a separate diskcache to which we can save and load the TTLCache to provide persistence.
+    Also adds thread safety so spots and alerts can come from any thread, and a listener mechanism so the web server
+    can get a callback when new spots/alerts are added, and send them to any SSE clients."""
+
+    def __init__(self, maxsize, ttl, snapshot_dir, snapshot_interval_sec):
+        self._cache = TTLCache(maxsize=maxsize, ttl=ttl)
+        self._lock = threading.Lock()
+        self._ttl = ttl
+        self._listeners = []
+        self._listeners_lock = threading.Lock()
+        self._snapshot_dir = snapshot_dir
+        self._disk_cache = diskcache.Cache(str(snapshot_dir))
+        self._load_snapshot()
+        self._start_periodic_snapshot(snapshot_interval_sec)
+
+    def set(self, key, value):
+        with self._lock:
+            self._cache[key] = value
+
+        # Notify listeners
+        with self._listeners_lock:
+            listeners = list(self._listeners)
+        for callback in listeners:
+            try:
+                callback(value)
+            except Exception:
+                logging.error("Listener raised an exception for key %s", key, exc_info=True)
+
+
+    def get(self, key, default=None):
+        with self._lock:
+            return self._cache.get(key, default)
+
+    def delete(self, key):
+        with self._lock:
+            self._cache.pop(key, None)
+
+    def keys(self):
+        with self._lock:
+            return list(self._cache.keys())
+
+    def values(self):
+        with self._lock:
+            return list(self._cache.values())
+
+    def add_listener(self, callback):
+        """Register callback(value) which will be called whenever a new spot/alert item is added via set(). Used by the
+        web server (via SSEBroadcaster) to send SSE clients an update on every new spot."""
+
+        with self._listeners_lock:
+            self._listeners.append(callback)
+
+    def remove_listener(self, callback):
+        with self._listeners_lock:
+            self._listeners.remove(callback)
+
+    def save_snapshot(self):
+        with self._lock:
+            # Store the time with the data so we can avoid loading anything nxt time that's older than TTL
+            data = [(k, v, time.time()) for k, v in self._cache.items()]
+        try:
+            self._disk_cache.set("snapshot", data)
+        except Exception as e:
+            logging.error("Failed to write snapshot to %s", self._snapshot_dir, e, exc_info=True)
+
+    def _load_snapshot(self):
+        data = self._disk_cache.get("snapshot")
+        if not data:
+            return
+
+        now = time.time()
+        with self._lock:
+            for key, value, saved_at in data:
+                # Only restore entries that would still be within TTL
+                if now - saved_at < self._ttl:
+                    self._cache[key] = value
+        logging.info("Loaded snapshot from %s", self._snapshot_dir)
+
+    def _start_periodic_snapshot(self, interval):
+        def loop():
+            while True:
+                time.sleep(interval)
+                self.save_snapshot()
+
+        t = threading.Thread(target=loop, daemon=True, name=f"snapshot-{self._snapshot_dir}")
+        t.start()
+
+    def close(self):
+        self.save_snapshot()
+        self._disk_cache.close()

core/lookup_helper.py

@@ -14,10 +14,10 @@ from pyhamtools.locator import latlong_to_locator
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 from requests_cache import CachedSession
 
-from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.config import config
 from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
     HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
+from core.url_data_cache import URLDataCache
 
 # QRZ XML field names differ from pyhamtools' normalised names; map them here.
 _QRZ_FIELD_MAP = {
@@ -29,6 +29,7 @@ _QRZ_FIELD_MAP = {
 }
 _QRZ_INT_FIELDS = {"adif", "cqz", "ituz"}
 _QRZ_FLOAT_FIELDS = {"latitude", "longitude"}
+_URL_DATA_CACHE = URLDataCache("callsign_lookup")
 
 
 def _normalize_qrz_data(raw):
@@ -142,7 +143,7 @@ class LookupHelper:
 
         try:
             logging.info("Downloading Country-files.com cty.plist...")
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
+            response = _URL_DATA_CACHE.get("https://www.country-files.com/cty/cty.plist",
                                           headers=HTTP_HEADERS)
 
             if response.ok:
@@ -167,7 +168,7 @@ class LookupHelper:
 
         try:
             logging.info("Downloading dxcc.json...")
-            response = SEMI_STATIC_URL_DATA_CACHE.get(
+            response = _URL_DATA_CACHE.get(
                 "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
                 headers=HTTP_HEADERS)
 
@@ -515,7 +516,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = SEMI_STATIC_URL_DATA_CACHE.get(
+                response = _URL_DATA_CACHE.get(
                     self._qrz_base_url + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
                     headers=HTTP_HEADERS, timeout=10)
                 if response.ok:
@@ -593,7 +594,7 @@ class LookupHelper:
 
         for lookup_call in calls_to_try:
             try:
-                response = SEMI_STATIC_URL_DATA_CACHE.get(
+                response = _URL_DATA_CACHE.get(
                     self._hamqth_base_url + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
                         lookup_call) + "&prg=" + HAMQTH_PRG, headers=HTTP_HEADERS)
                 if response.ok:

core/sig_utils.py

@@ -1,26 +1,11 @@
-import csv
 import logging
 
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
-from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
-from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
+from core.constants import SIGS
-from core.constants import SIGS, HTTP_HEADERS
+from core.data_store import DATA_STORE
 from core.geo_utils import wab_wai_square_to_lat_lon
 
-# Load Spanish municipality data for the DME programme. There's no convenient lookup API for this, so we embed the data
-# file in Spothole and load it on startup.
-with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
-    _DME_INDEX = {row["COD_INE"][:5]: row for row in csv.DictReader(_f, delimiter=";")}
-# Caches for data for the SIGs where we have to download a whole global reference list, rather than looking up a single
-# reference. These get populated from the SEMI_STATIC_URL_DATA_CACHE only if the data actually came
-# live from the internet, to avoid repopulating them every time we pull the same data from the cache.
-_WWFF_INDEX_CACHE = {}
-_SIOTA_INDEX_CACHE = {}
-_WOTA_INDEX_CACHE = {}
-_ZLOTA_INDEX_CACHE = {}
-_LLOTA_INDEX_CACHE = {}
-
 
 def get_ref_regex_for_sig(sig):
     """Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
@@ -42,220 +27,31 @@ def get_sig_name_from_comment_name(sig):
 
 
 def populate_sig_ref_info(sig_ref):
-    """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object which
+    """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
-    must at minimum have a "sig" and an "id". The rest of the object will be populated and returned.
+    which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes
+    use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
+    SIG we are getting data for.
     Note there is currently no support for KRMNPA location lookup, see issue #61."""
 
-    if sig_ref.sig is None or sig_ref.id is None:
+    if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "":
-        logging.warning("Failed to look up sig_ref info, sig or id were not set.")
+        logging.debug("Failed to look up sig_ref info, sig or id were not set.")
+        return sig_ref
 
-    sig = sig_ref.sig or ""
+    sig = sig_ref.sig
     ref_id = sig_ref.id
+
+    # DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all activators
+    # add leading zeros.
+    if sig.upper() == "DME":
+        ref_id = ref_id.zfill(5)
+
     try:
-        if sig.upper() == "POTA":
+        # If the SIG is HEMA or KRMNPA, we have no current lookup for this so just skip it.
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
+        if sig.upper() == "HEMA" or sig.upper() == "KRMNPA":
-            if response.ok:
+            return sig_ref
-                data = response.json()
-                if data:
-                    fullname = str(data["name"]) if "name" in data else None
-                    if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
-                        fullname = fullname + " " + data["parktypeDesc"]
-                    sig_ref.name = fullname
-                    sig_ref.url = "https://pota.app/#/park/" + ref_id
-                    sig_ref.grid = data["grid6"] if "grid6" in data else None
-                    sig_ref.latitude = data["latitude"] if "latitude" in data else None
-                    sig_ref.longitude = data["longitude"] if "longitude" in data else None
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "SOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if data:
-                    sig_ref.name = data["name"] if "name" in data else None
-                    sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
-                    sig_ref.grid = data["locator"] if "locator" in data else None
-                    sig_ref.latitude = data["latitude"] if "latitude" in data else None
-                    sig_ref.longitude = data["longitude"] if "longitude" in data else None
-                    sig_ref.activation_score = data["points"] if "points" in data else None
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "WWBOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if data:
-                    sig_ref.name = data["name"] if "name" in data else None
-                    sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
-                    sig_ref.grid = data["locator"] if "locator" in data else None
-                    sig_ref.latitude = data["lat"] if "lat" in data else None
-                    sig_ref.longitude = data["long"] if "long" in data else None
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if data:
-                    sig_ref.name = data["name"] if "name" in data else None
-                    sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
-                    sig_ref.grid = data["locator"] if "locator" in data else None
-
-                    # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
-                    # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
-                    # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
-                    # just use the single point we have.
-                    if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
-                            "lat2") is not None and data.get("lng2") is not None:
-                        sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
-                        sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
-                    else:
-                        sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
-                        sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "WWFF":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                global _WWFF_INDEX_CACHE
-                if not bool(_WWFF_INDEX_CACHE) or not response.from_cache:
-                    # New data from WWFF, update our internal map
-                    _WWFF_INDEX_CACHE = {row["reference"]: row for row in
-                                         csv.DictReader(response.content.decode().splitlines())}
-                row = _WWFF_INDEX_CACHE.get(ref_id)
-                if row:
-                    sig_ref.name = row["name"] if "name" in row else None
-                    sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
-                    sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
-                    sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
-                    sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row[
-                        "longitude"] != "-" else None
-                elif not response.from_cache:
-                    logging.warning("WWFF database did not contain data for ref %s", ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "SIOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                global _SIOTA_INDEX_CACHE
-                if not bool(_SIOTA_INDEX_CACHE) or not response.from_cache:
-                    # New data from SIOTA, update our internal map
-                    _SIOTA_INDEX_CACHE = {row["SILO_CODE"]: row for row in
-                                          csv.DictReader(response.content.decode().splitlines())}
-                row = _SIOTA_INDEX_CACHE.get(ref_id)
-                if row:
-                    sig_ref.name = row["NAME"] if "NAME" in row else None
-                    sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
-                    sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
-                    sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
-                elif not response.from_cache:
-                    logging.warning("SIOTA database did not contain data for ref %s", ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "WOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if data:
-                    global _WOTA_INDEX_CACHE
-                    if not bool(_WOTA_INDEX_CACHE) or not response.from_cache:
-                        # New data from WOTA, update our internal map
-                        _WOTA_INDEX_CACHE = {feature["properties"]["wotaId"]: feature for feature in
-                                             data.get("features", [])}
-                    feature = _WOTA_INDEX_CACHE.get(ref_id)
-                    if feature:
-                        sig_ref.name = feature["properties"]["title"]
-                        # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
-                        # added to them
-                        sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id
-                        if ref_id.upper().startswith("LDO-"):
-                            number = int(ref_id.upper().replace("LDO-", ""))
-                            sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
-                        sig_ref.grid = feature["properties"]["qthLocator"]
-                        sig_ref.latitude = feature["geometry"]["coordinates"][1]
-                        sig_ref.longitude = feature["geometry"]["coordinates"][0]
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "ZLOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if isinstance(data, list):
-                    global _ZLOTA_INDEX_CACHE
-                    if not bool(_ZLOTA_INDEX_CACHE) or not response.from_cache:
-                        # New data from ZLOTA, update our internal map
-                        _ZLOTA_INDEX_CACHE = {asset["code"]: asset for asset in data}
-                    asset = _ZLOTA_INDEX_CACHE.get(ref_id)
-                    if asset:
-                        sig_ref.name = asset["name"]
-                        sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
-                        try:
-                            sig_ref.grid = latlong_to_locator(asset["y"], asset["x"], 6)
-                        except:
-                            logging.debug("Invalid lat/lon received for reference")
-                            sig_ref.latitude = asset["y"]
-                            sig_ref.longitude = asset["x"]
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "BOTA":
-            if not sig_ref.name:
-                sig_ref.name = sig_ref.id
-            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
-
-        elif sig.upper() == "LLOTA":
-            response = SEMI_STATIC_URL_DATA_CACHE.get("https://llota.app/api/public/references",
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if isinstance(data, list):
-                    global _LLOTA_INDEX_CACHE
-                    if not bool(_LLOTA_INDEX_CACHE) or not response.from_cache:
-                        # New data from LLOTA, update our internal map
-                        _LLOTA_INDEX_CACHE = {ref["reference_code"]: ref for ref in data}
-                    ref = _LLOTA_INDEX_CACHE.get(ref_id)
-                    if ref:
-                        sig_ref.name = str(ref["name"])
-                        sig_ref.url = "https://llota.app/list/ref/" + ref_id
-                        sig_ref.grid = str(ref["grid_locator"])
-                        ll = locator_to_latlong(sig_ref.grid)
-                        sig_ref.latitude = ll[0]
-                        sig_ref.longitude = ll[1]
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "WWTOTA":
-            if not sig_ref.name:
-                sig_ref.name = sig_ref.id
-            sig_ref.url = "https://wwtota.com/seznam/karta_rozhledny.php?ref=" + str(sig_ref.name)
 
+        # If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
+        # calculate all the information we are going to get directly. So handle those cases first
         elif sig.upper() == "TILES":
             # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
             if not sig_ref.name:
@@ -278,27 +74,26 @@ def populate_sig_ref_info(sig_ref):
                 except:
                     logging.warning("Invalid lat/lon received for WAB/WAI reference")
 
-        elif sig.upper() == "DME":
+        elif sig.upper() == "BOTA":
-            # Zero-pad to 5 digits to match our source data
+            # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
-            row = _DME_INDEX.get(ref_id.zfill(5))
+            if not sig_ref.name:
-            if row:
+                sig_ref.name = sig_ref.id
-                sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
+            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
-                sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+
-                    "LATITUD_ETRS89_REGCAN95") else None
+        # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
-                sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+        # this SIG. If so, check for the reference data and use that.
-                    "LONGITUD_ETRS89_REGCAN95") else None
+        elif sig in DATA_STORE.sigrefs:
-                if sig_ref.latitude and sig_ref.longitude:
+            key = sig + ":" + ref_id
-                    try:
+            lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
-                        sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6)
+            if lookup_data:
-                    except Exception:
+                # Copy new sig ref data into existing object
-                        logging.warning("Invalid lat/lon received for DME reference")
+                sig_ref.__dict__.update(lookup_data.__dict__)
-            else:
+            else:
-                logging.warning("DME database did not contain data for ref %s", ref_id)
+                logging.warning("%s database did not contain data for ref %s", sig, ref_id)
+
+        else:
+            logging.warning(f"Tried to look up a SIG called %s but Spothole does not know what that is.", sig)
 
-    except ConnectionError:
-        logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
-    except (ConnectTimeout, ReadTimeout):
-        logging.warning(f"Timeout when looking up sig_ref info for " + sig + " ref " + ref_id)
     except Exception:
         logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
     return sig_ref

core/status_reporter.py

@@ -7,31 +7,29 @@ import pytz
 
 from core.config import SERVER_OWNER_CALLSIGN
 from core.constants import SOFTWARE_VERSION
+from core.data_store import DATA_STORE
 from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge
 
 
 class StatusReporter:
     """Provides a timed update of the application's status data."""
 
-    def __init__(self, status_data, run_interval, web_server, cleanup_timer, spots, spot_providers, alerts,
+    def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers,
-                 alert_providers, solar_condition_providers):
+                 sig_ref_data_providers):
         """Constructor"""
 
-        self._status_data = status_data
         self._run_interval = run_interval
         self._web_server = web_server
-        self._cleanup_timer = cleanup_timer
-        self._spots = spots
         self._spot_providers = spot_providers
-        self._alerts = alerts
         self._alert_providers = alert_providers
         self._solar_condition_providers = solar_condition_providers
+        self._sig_ref_data_providers = sig_ref_data_providers
         self._thread = None
         self._stop_event = Event()
         self._startup_time = datetime.now(pytz.UTC)
 
-        self._status_data["software-version"] = SOFTWARE_VERSION
+        DATA_STORE.status_data["software-version"] = SOFTWARE_VERSION
-        self._status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
+        DATA_STORE.status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
 
     def start(self):
         """Start the reporter thread"""
@@ -55,31 +53,34 @@ class StatusReporter:
     def _report(self):
         """Write status information"""
 
-        self._status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
+        DATA_STORE.status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
-        self._status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
+        DATA_STORE.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
-        self._status_data["num_spots"] = len(self._spots)
+        DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
-        self._status_data["num_alerts"] = len(self._alerts)
+        DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
-        self._status_data["spot_providers"] = list(
+        DATA_STORE.status_data["spot_providers"] = list(
         map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                        "last_updated": p.last_update_time.replace(
                            tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
                        "last_spot": p.last_spot_time.replace(
                            tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
             self._spot_providers))
-        self._status_data["alert_providers"] = list(
+        DATA_STORE.status_data["alert_providers"] = list(
         map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                        "last_updated": p.last_update_time.replace(
                            tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
             self._alert_providers))
-        self._status_data["solar_condition_providers"] = list(
+        DATA_STORE.status_data["solar_condition_providers"] = list(
         map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
                        "last_updated": p.last_update_time.replace(
                            tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
             self._solar_condition_providers))
-        self._status_data["cleanup"] = {"status": self._cleanup_timer.status,
+        DATA_STORE.status_data["sig_ref_data_providers"] = list(
-                                        "last_ran": self._cleanup_timer.last_cleanup_time.replace(
+            map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
-                                            tzinfo=pytz.UTC).timestamp() if self._cleanup_timer.last_cleanup_time else 0}
+                           "last_updated": p.last_update_time.replace(
-        self._status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
+                           "reference_count": p.reference_count},
+                self._sig_ref_data_providers))
+        DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
                                       "last_api_access": self._web_server.web_server_metrics[
                                           "last_api_access_time"].replace(
                                           tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
@@ -94,5 +95,5 @@ class StatusReporter:
 
         # Update Prometheus metrics
         memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)
-        spots_gauge.set(len(self._spots))
+        spots_gauge.set(len(DATA_STORE.spots.values()))
-        alerts_gauge.set(len(self._alerts))
+        alerts_gauge.set(len(DATA_STORE.alerts.values()))

core/url_data_cache.py

@@ -0,0 +1,25 @@
+import threading
+from datetime import timedelta
+
+from requests_cache import CachedSession
+
+from core.data_store import CACHE_DIR
+
+
+class URLDataCache(CachedSession):
+    """Cache for "semi-static" data retrieved from a URL. This is an extra layer of caching in addition to the normal
+    caching of spots, alerts and other data in the DataStore class. Its purpose is to avoid hitting remote endpoints
+    frequently when e.g. restarting Spothole many times during testing. It should only be used for data that isn't
+    expected to change on a sub-daily basis, so never for spots & alerts. It also implements a lock to allow it to be
+    used across multiple threads, though note that URL lookups will block each other this way, so it is still better to
+    create one of these objects per thread if possible."""
+
+    _lock = threading.Lock()
+
+    def __init__(self, name):
+        super().__init__(CACHE_DIR + "urls/" + name, expire_after=timedelta(days=1),
+                         allowable_codes=(200, 400, 401, 403, 404))
+
+    def get(self, *args, **kwargs):
+        with self._lock:
+            return super().get(*args, **kwargs)

core/utils.py

@@ -6,13 +6,3 @@ def safe_json_dumps(obj):
     which are invalid in JSON."""
 
     return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
-
-
-def empty_queue(q):
-    """Empty a queue"""
-
-    while not q.empty():
-        try:
-            q.get_nowait()
-        except:
-            break

data/alert.py

@@ -1,6 +1,7 @@
 import copy
 import hashlib
 import json
+import logging
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 
@@ -64,6 +65,7 @@ class Alert:
     def infer_missing(self, credentials=None):
         """Infer missing parameters where possible"""
 
+        try:
             # If we somehow don't have a start time, set it to zero so it sorts off the bottom of any list but
             # clients can still reliably parse it as a number.
             if not self.start_time:
@@ -128,6 +130,9 @@ class Alert:
                 self.dx_names = list(
                     map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c, credentials), self.dx_calls))
 
+        except Exception as e:
+            logging.error("Exception while inferring missing data from spot", e, exc_info=True)
+
     def to_json(self):
         """JSON serialise"""
 

data/lookup_credentials.py

@@ -12,15 +12,15 @@ class LookupCredentials:
     hamqth_session_id: str = ""  # alternative to username/password
 
 
-def extract_credentials(headers):
+def extract_credentials(query_params):
-    """Build a LookupCredentials from HTTP request headers; returns None if no usable credentials are present."""
+    """Build a LookupCredentials from HTTP query params; returns None if no usable credentials are present."""
     creds = LookupCredentials(
-        qrz_username=headers.get("X-QRZ-Username", ""),
+        qrz_username=query_params.get("qrz_username", ""),
-        qrz_password=headers.get("X-QRZ-Password", ""),
+        qrz_password=query_params.get("qrz_password", ""),
-        qrz_session_key=headers.get("X-QRZ-Session-Key", ""),
+        qrz_session_key=query_params.get("qrz_session_key", ""),
-        hamqth_username=headers.get("X-HamQTH-Username", ""),
+        hamqth_username=query_params.get("hamqth_username", ""),
-        hamqth_password=headers.get("X-HamQTH-Password", ""),
+        hamqth_password=query_params.get("hamqth_password", ""),
-        hamqth_session_id=headers.get("X-HamQTH-Session-ID", ""),
+        hamqth_session_id=query_params.get("hamqth_session_id", ""),
     )
     has_qrz = creds.qrz_session_key or (creds.qrz_username and creds.qrz_password)
     has_hamqth = creds.hamqth_session_id or (creds.hamqth_username and creds.hamqth_password)

data/spot.py

@@ -147,6 +147,7 @@ class Spot:
     def infer_missing(self, credentials=None):
         """Infer missing parameters where possible"""
 
+        try:
             # If we somehow don't have a spot time, set it to zero so it sorts off the bottom of any list but
             # clients can still reliably parse it as a number.
             if not self.time:
@@ -437,6 +438,9 @@ class Spot:
                         self.de_longitude = latlon[1]
                         self.de_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.de_call)
 
+        except Exception as e:
+            logging.error("Exception while inferring missing data from spot", e, exc_info=True)
+
     def to_json(self):
         """JSON serialise"""
 

datafiles/39c3-tota.csv

@@ -1,18 +0,0 @@
-ref,lat,lon
-T-01,53.56278090617755,9.984341869295505
-T-02,53.562383404176416,9.98551893027115
-T-03,53.56170184391514,9.985416035619778
-T-04,53.562026534393176,9.986372919078974
-T-11,53.56284641242506,9.98475590239655
-T-12,53.562431705517035,9.98551675702443
-T-13,53.56223704898424,9.985774520335664
-T-14,53.5617893512591,9.986344302837976
-T-21,53.56284641242506,9.98475590239655
-T-22,53.56245816412497,9.985456089490567
-T-23,53.56199560857136,9.985636761412673
-T-24,53.5617893512591,9.986344302837976
-T-31,53.56247470064887,9.985611427551902
-T-32,53.5617893512591,9.986344302837976
-T-41,53.56245039134992,9.985486136112701
-T-91,53.56147934973529,9.984626806439744
-T-92,53.561396810300735,9.987553052152899

datafiles/eh23-tota.csv

@@ -1,13 +0,0 @@
-ref,lat,lon
-T-01,50.3636495,7.5584857
-T-02,50.3636495,7.5584857
-T-03,50.3636495,7.5584857
-T-11,50.3636495,7.5584857
-T-13,50.3636495,7.5584857
-T-14,50.3636495,7.5584857
-T-21,50.3636495,7.5584857
-T-31,50.3636495,7.5584857
-T-33,50.3636495,7.5584857
-T-34,50.3636495,7.5584857
-T-41,50.3636495,7.5584857
-T-51,50.3636495,7.5584857

datafiles/toilets.csv

@@ -0,0 +1,30 @@
+ref,lat,lon
+C3 T-01,53.56278090617755,9.984341869295505
+C3 T-02,53.562383404176416,9.98551893027115
+C3 T-03,53.56170184391514,9.985416035619778
+C3 T-04,53.562026534393176,9.986372919078974
+C3 T-11,53.56284641242506,9.98475590239655
+C3 T-12,53.562431705517035,9.98551675702443
+C3 T-13,53.56223704898424,9.985774520335664
+C3 T-14,53.5617893512591,9.986344302837976
+C3 T-21,53.56284641242506,9.98475590239655
+C3 T-22,53.56245816412497,9.985456089490567
+C3 T-23,53.56199560857136,9.985636761412673
+C3 T-24,53.5617893512591,9.986344302837976
+C3 T-31,53.56247470064887,9.985611427551902
+C3 T-32,53.5617893512591,9.986344302837976
+C3 T-41,53.56245039134992,9.985486136112701
+C3 T-91,53.56147934973529,9.984626806439744
+C3 T-92,53.561396810300735,9.987553052152899
+EH T-01,50.3636495,7.5584857
+EH T-02,50.3636495,7.5584857
+EH T-03,50.3636495,7.5584857
+EH T-11,50.3636495,7.5584857
+EH T-13,50.3636495,7.5584857
+EH T-14,50.3636495,7.5584857
+EH T-21,50.3636495,7.5584857
+EH T-31,50.3636495,7.5584857
+EH T-33,50.3636495,7.5584857
+EH T-34,50.3636495,7.5584857
+EH T-41,50.3636495,7.5584857
+EH T-51,50.3636495,7.5584857

requirements.txt

@@ -18,3 +18,4 @@ tornado~=6.4.2
 tornado_eventsource~=3.0.0
 geopandas~=0.13.2
 simplejson~=4.1.1
+cachetools~=7.1.6

server/handlers/api/addspot.py

@@ -1,40 +1,33 @@
 import logging
 import re
-import threading
 from datetime import datetime
 from typing import Any
 
 import pytz
-import requests
 import tornado
 from tornado import httputil
 from tornado.web import Application
 
-from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, MAX_SPOT_AGE, RECAPTCHA_SECRET_KEY
+from core.config import ALLOW_SPOTTING
 from core.constants import UNKNOWN_BAND
 from core.lookup_helper import infer_band_from_freq
 from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import safe_json_dumps
 from data.spot import Spot
-from spotproviders.spot_provider import SpotProvider
-
-RECAPTCHA_VERIFY_URL = "https://www.google.com/recaptcha/api/siteverify"
 
 
 class APISpotHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v2/spot (POST)"""
+    """API request handler for /api/v1/spot (POST)"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._spots = None
         self._web_server_metrics = None
-        self._spot_providers = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, spots, web_server_metrics, spot_providers=None):
+    def initialize(self, spots, web_server_metrics):
         self._spots = spots
         self._web_server_metrics = web_server_metrics
-        self._spot_providers = spot_providers or []
 
     def post(self):
         try:
@@ -69,38 +62,9 @@ class APISpotHandler(tornado.web.RequestHandler):
                 self.set_header("Content-Type", "application/json")
                 return
 
-            # Read in the request body as JSON
+            # Read in the request body as JSON then convert to a Spot object
-            json_body = tornado.escape.json_decode(post_data)
+            json_spot = tornado.escape.json_decode(post_data)
-
+            spot = Spot(**json_spot)
-            # Extract the "spot" and "handling" sub-objects from the request body
-            spot_data = json_body.get("spot", {})
-            handling = json_body.get("handling", {})
-
-            # Extract individual parameters that say how this spot should be handled by the server
-            submit_upstream = handling.get("submit_upstream", False)
-            upstream_provider_name = handling.get("upstream_provider", None)
-            upstream_credentials = handling.get("upstream_credentials", {})
-            captcha_token = handling.get("captcha_token", None)
-
-            # Verify CAPTCHA if required
-            if RECAPTCHA_SECRET_KEY:
-                if not captcha_token:
-                    self.set_status(422)
-                    self.write(json.dumps("Error - CAPTCHA token is required for spot submission.",
-                                          default=serialize_everything))
-                    self.set_header("Cache-Control", "no-store")
-                    self.set_header("Content-Type", "application/json")
-                    return
-                if not self._verify_recaptcha(captcha_token):
-                    self.set_status(422)
-                    self.write(json.dumps("Error - CAPTCHA verification failed.",
-                                          default=serialize_everything))
-                    self.set_header("Cache-Control", "no-store")
-                    self.set_header("Content-Type", "application/json")
-                    return
-
-            # Convert spot field to a Spot object
-            spot = Spot(**spot_data)
 
             # Reject if no timestamp, frequency, dx_call or de_call
             if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
@@ -152,76 +116,11 @@ class APISpotHandler(tornado.web.RequestHandler):
                 self.set_header("Content-Type", "application/json")
                 return
 
-            # Reject upstream submission if not permitted
+            # infer missing data, and add it to our database.
-            if submit_upstream and not ALLOW_UPSTREAM_SPOTTING:
+            spot.source = "API"
-                self.set_status(403)
-                self.write(json.dumps("Error - this server does not allow upstream spot submission.",
-                                      default=serialize_everything))
-                self.set_header("Cache-Control", "no-store")
-                self.set_header("Content-Type", "application/json")
-                return
-
-            # Validate upstream submission requirements
-            if submit_upstream and upstream_provider_name:
-                if not spot.sig:
-                    self.set_status(422)
-                    self.write(json.dumps("Error - a SIG must be selected to submit upstream.",
-                                          default=serialize_everything))
-                    self.set_header("Cache-Control", "no-store")
-                    self.set_header("Content-Type", "application/json")
-                    return
-                if not spot.sig_refs and upstream_provider_name != "Tiles":
-                    self.set_status(422)
-                    self.write(json.dumps("Error - a SIG reference is required to submit upstream.",
-                                          default=serialize_everything))
-                    self.set_header("Cache-Control", "no-store")
-                    self.set_header("Content-Type", "application/json")
-                    return
-                if not spot.dx_grid and upstream_provider_name == "Tiles":
-                    self.set_status(422)
-                    self.write(json.dumps("Error - a grid reference is required to submit upstream to Tiles on the Air.",
-                                          default=serialize_everything))
-                    self.set_header("Cache-Control", "no-store")
-                    self.set_header("Content-Type", "application/json")
-                    return
-                if not spot.mode and upstream_provider_name == "Tiles":
-                    self.set_status(422)
-                    self.write(json.dumps("Error - a mode is required to submit upstream to Tiles on the Air.",
-                                          default=serialize_everything))
-                    self.set_header("Cache-Control", "no-store")
-                    self.set_header("Content-Type", "application/json")
-                    return
-
-            # Submit upstream if requested
-            upstream_warning = None
-            if submit_upstream and upstream_provider_name:
-                provider = self._find_provider(upstream_provider_name, spot.sig)
-                if provider:
-                    try:
-                        # Submit spot to the upstream provider
-                        provider.submit_spot(spot, upstream_credentials)
-                        # Trigger a re-poll after 1 second so the spot appears quickly
-                        threading.Timer(1.0, provider.force_poll).start()
-                    except NotImplementedError as e:
-                        upstream_warning = str(e)
-                    except Exception as e:
-                        logging.warning("Failed to submit spot upstream to " + upstream_provider_name + ": " + str(e))
-                        upstream_warning = "Spot was saved locally but upstream submission to " + upstream_provider_name + " failed: " + str(
-                            e)
-                else:
-                    upstream_warning = "No enabled provider named '" + upstream_provider_name + "' supports upstream submission for " + spot.sig + " spots."
-
-            # If we successfully submitted the spot upstream, don't add it direct to Spothole, otherwise it will be a
-            # duplicate with what immediately comes back from the API. But if we weren't asked to send it upstream, or
-            # we were but it failed, we should still add it to our database anyway.
-            if not submit_upstream or upstream_warning:
             spot.infer_missing()
-                self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
+            self._spots.set(spot.id, spot)
 
-            if upstream_warning:
-                self.write(json.dumps("Warning - " + upstream_warning, default=serialize_everything))
-                self.set_status(201)
-            else:
             self.write(safe_json_dumps("OK"))
             self.set_status(201)
             self.set_header("Cache-Control", "no-store")
@@ -233,24 +132,3 @@ class APISpotHandler(tornado.web.RequestHandler):
             self.set_status(500)
             self.set_header("Cache-Control", "no-store")
             self.set_header("Content-Type", "application/json")
-
-    def _find_provider(self, provider_name, sig) -> SpotProvider | None:
-        """Find an enabled provider by name that can submit spots for the given SIG."""
-
-        for p in self._spot_providers:
-            if p.enabled and p.name == provider_name and p.can_submit_spot(sig):
-                return p
-        return None
-
-    @staticmethod
-    def _verify_recaptcha(token):
-        """Verify a Google reCAPTCHA v2 token. Returns True if valid."""
-
-        try:
-            response = requests.post(RECAPTCHA_VERIFY_URL,
-                                     data={"secret": RECAPTCHA_SECRET_KEY, "response": token},
-                                     timeout=(5, 10))
-            return response.ok and response.json().get("success", False)
-        except Exception as e:
-            logging.warning("reCAPTCHA verification request failed: " + str(e))
-            return False

server/handlers/api/alerts.py

@@ -1,7 +1,6 @@
 import copy
 import logging
 from datetime import datetime
-from queue import Queue
 from typing import Any
 
 import pytz
@@ -11,15 +10,12 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import safe_json_dumps, empty_queue
+from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 
-SSE_HANDLER_MAX_QUEUE_SIZE = 100
-SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
-
 
 class APIAlertsHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v2/alerts"""
+    """API request handler for /api/v1/alerts"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._alerts = None
@@ -52,7 +48,7 @@ class APIAlertsHandler(tornado.web.RequestHandler):
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
 
             # Fetch all alerts matching the query, then optionally enrich with online data
-            credentials = extract_credentials(self.request.headers)
+            credentials = extract_credentials(query_params)
             data = get_alert_list_with_filters(self._alerts, query_params)
             if credentials:
                 data = self._enrich(data, credentials)
@@ -70,19 +66,17 @@ class APIAlertsHandler(tornado.web.RequestHandler):
 
 
 class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
-    """API request handler for /api/v2/alerts/stream"""
+    """API request handler for /api/v1/alerts/stream"""
 
     def __init__(self, application, request, **kwargs: Any):
-        self._sse_alert_queues = None
+        self._sse_alert_broadcaster = None
         self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
-        self._alert_queue = None
-        self._heartbeat = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, sse_alert_queues, web_server_metrics):
+    def initialize(self, _sse_alert_broadcaster, web_server_metrics):
-        self._sse_alert_queues = sse_alert_queues
+        self._sse_alert_broadcaster = _sse_alert_broadcaster
         self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
@@ -102,61 +96,34 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
-            self._credentials = extract_credentials(self.request.headers)
+            self._credentials = extract_credentials(self._query_params)
-
-            # Create a alert queue and add it to the web server's list. The web server will fill this when alerts arrive
-            self._alert_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
-            self._sse_alert_queues.append(self._alert_queue)
-
-            # Set up a timed callback to check if anything is in the queue
-            self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
-            self._heartbeat.start()
 
             # Flush headers immediately so nginx doesn't time out waiting for a response
             self.write_message("keepalive", "")
 
+            # Register to handle new alerts arriving. The callback() method will get called with the new alert as an
+            # argument.
+            self._sse_alert_broadcaster.register(self)
+
         except Exception as e:
             logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
             self.close()
 
     def close(self):
-        """When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
+        """When the user closes the socket, deregister ourselves from the alert broadcaster"""
 
-        try:
+        self._sse_alert_broadcaster.unregister(self)
-            if self._alert_queue in self._sse_alert_queues:
-                self._sse_alert_queues.remove(self._alert_queue)
-            empty_queue(self._alert_queue)
-        except:
-            pass
-        try:
-            self._heartbeat.stop()
-        except:
-            pass
-        self._alert_queue = None
         super().close()
 
-    def _callback(self):
+    def callback(self, alert):
-        """Callback to check if anything has arrived in the queue, and if so send it to the client"""
+        """Callback when a new alert arrives"""
 
         try:
-            if self._alert_queue:
-                if not self._alert_queue.empty():
-                    while not self._alert_queue.empty():
-                        alert = self._alert_queue.get()
-                        # If the new alert matches our param filters, send it to the client. If not, ignore it.
             if alert_allowed_by_query(alert, self._query_params):
                 if self._credentials:
                     alert = copy.deepcopy(alert)
                     alert.infer_missing(self._credentials)
                 self.write_message(msg=safe_json_dumps(alert))
-
-                else:
-                    # Send a keepalive comment if the queue was empty
-                    self.write_message("keepalive", "")
-
-                if self._alert_queue not in self._sse_alert_queues:
-                    logging.error("Web server cleared up a queue of an active connection!")
-                    self.close()
         except Exception as e:
             logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
             self.close()
@@ -169,7 +136,7 @@ def get_alert_list_with_filters(all_alerts, query):
     # Create a shallow copy of the alert list ordered by start time, then filter the list to reduce it only to alerts
     # that match the filter parameters in the query string. Finally, apply a limit to the number of alerts returned.
     # The list of query string filters is defined in the API docs.
-    alert_ids = list(all_alerts.iterkeys())
+    alert_ids = all_alerts.keys()
     alerts = []
     for k in alert_ids:
         a = all_alerts.get(k)

server/handlers/api/dxstats.py

@@ -19,7 +19,7 @@ BANDS_SET = frozenset(BANDS)
 
 
 class APIDxStatsHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v2/dxstats"""
+    """API request handler for /api/v1/dxstats"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._spots = None
@@ -40,7 +40,7 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
             one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp()
             counts = Counter()
 
-            for key in self._spots.iterkeys():
+            for key in self._spots.keys():
                 spot = self._spots.get(key)
                 if spot is None:
                     continue

server/handlers/api/lookups.py

@@ -19,7 +19,7 @@ from data.spot import Spot
 
 
 class APILookupCallHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v2/lookup/call"""
+    """API request handler for /api/v1/lookup/call"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._web_server_metrics = None
@@ -46,7 +46,7 @@ class APILookupCallHandler(tornado.web.RequestHandler):
                 if re.match(r"^[A-Z0-9/\-]*$", call):
                     # Take the callsign, make a "fake spot" so we can run infer_missing() on it, then repack the
                     # resulting data in the correct way for the API response.
-                    credentials = extract_credentials(self.request.headers)
+                    credentials = extract_credentials(query_params)
                     fake_spot = Spot(dx_call=call)
                     fake_spot.infer_missing(credentials)
                     data = {
@@ -83,7 +83,7 @@ class APILookupCallHandler(tornado.web.RequestHandler):
 
 
 class APILookupSIGRefHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v2/lookup/sigref"""
+    """API request handler for /api/v1/lookup/sigref"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._web_server_metrics = None
@@ -135,7 +135,7 @@ class APILookupSIGRefHandler(tornado.web.RequestHandler):
 
 
 class APILookupGridHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v2/lookup/grid"""
+    """API request handler for /api/v1/lookup/grid"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._web_server_metrics = None

server/handlers/api/options.py

@@ -14,18 +14,16 @@ from core.utils import safe_json_dumps
 
 
 class APIOptionsHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v2/options"""
+    """API request handler for /api/v1/options"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._status_data = None
         self._web_server_metrics = None
-        self._spot_providers = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, status_data, web_server_metrics, spot_providers=None):
+    def initialize(self, status_data, web_server_metrics):
         self._status_data = status_data
         self._web_server_metrics = web_server_metrics
-        self._spot_providers = spot_providers or []
 
     def get(self):
         try:
@@ -35,37 +33,23 @@ class APIOptionsHandler(tornado.web.RequestHandler):
             self._web_server_metrics["status"] = "OK"
             api_requests_counter.inc()
 
-            # Build a map of SIG name -> list of provider names that can submit spots for that SIG
-        spot_submit_providers = {}
-        for provider in self._spot_providers:
-            if not provider.enabled:
-                continue
-            for sig in SIGS:
-                    if provider.can_submit_spot(sig.name):
-                    spot_submit_providers.setdefault(sig.name, []).append(provider.name)
-
-        # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle
-        # things that aren't even available.
-        spot_sources: list = list(
-            map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["spot_providers"])))
-        alert_sources = list(
-            map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["alert_providers"])))
-        # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from
-        # one of our providers.
-            if ALLOW_SPOTTING:
-                spot_sources.append("API")
-
             options = {"bands": BANDS,
                        "modes": ALL_MODES,
                        "mode_types": MODE_TYPES,
                        "sigs": SIGS,
-                   "spot_sources": spot_sources,
+                       # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle things that aren't even available.
-                   "alert_sources": alert_sources,
+                       "spot_sources": list(
+                           map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["spot_providers"]))),
+                       "alert_sources": list(
+                           map(lambda p: p["name"], filter(lambda p: p["enabled"], self._status_data["alert_providers"]))),
                        "continents": CONTINENTS,
                        "propagation_modes": list(PROPAGATION_MODES.values()),
                        "max_spot_age": MAX_SPOT_AGE,
-                   "spot_allowed": ALLOW_SPOTTING,
+                       "spot_allowed": ALLOW_SPOTTING}
-                   "spot_submit_providers": spot_submit_providers}
+            # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from
+            # one of our proviers.
+            if ALLOW_SPOTTING:
+                options["spot_sources"].append("API")
 
             self.write(safe_json_dumps(options))
             self.set_status(200)

server/handlers/api/solar_conditions.py

@@ -12,7 +12,7 @@ from core.utils import safe_json_dumps
 
 
 class APISolarConditionsHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v2/solar"""
+    """API request handler for /api/v1/solar"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._solar_conditions = None

server/handlers/api/spots.py

@@ -1,7 +1,6 @@
 import copy
 import logging
 from datetime import datetime, timedelta
-from queue import Queue
 from typing import Any
 
 import pytz
@@ -11,15 +10,12 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.prometheus_metrics_handler import api_requests_counter
-from core.utils import safe_json_dumps, empty_queue
+from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 
-SSE_HANDLER_MAX_QUEUE_SIZE = 1000
-SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
-
 
 class APISpotsHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v2/spots"""
+    """API request handler for /api/v1/spots"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._spots = None
@@ -52,7 +48,7 @@ class APISpotsHandler(tornado.web.RequestHandler):
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
 
             # Fetch all spots matching the query, then optionally enrich with online data
-            credentials = extract_credentials(self.request.headers)
+            credentials = extract_credentials(query_params)
             data = get_spot_list_with_filters(self._spots, query_params)
             if credentials:
                 data = self._enrich(data, credentials)
@@ -70,19 +66,17 @@ class APISpotsHandler(tornado.web.RequestHandler):
 
 
 class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
-    """API request handler for /api/v2/spots/stream"""
+    """API request handler for /api/v1/spots/stream"""
 
     def __init__(self, application, request, **kwargs: Any):
-        self._sse_spot_queues = None
+        self._sse_spot_broadcaster = None
         self._web_server_metrics = None
         self._query_params = None
         self._credentials = None
-        self._spot_queue = None
-        self._heartbeat = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, sse_spot_queues, web_server_metrics):
+    def initialize(self, sse_spot_broadcaster, web_server_metrics):
-        self._sse_spot_queues = sse_spot_queues
+        self._sse_spot_broadcaster = sse_spot_broadcaster
         self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):
@@ -104,61 +98,35 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
-            self._credentials = extract_credentials(self.request.headers)
+            self._credentials = extract_credentials(self._query_params)
-
-            # Create a spot queue and add it to the web server's list. The web server will fill this when spots arrive
-            self._spot_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
-            self._sse_spot_queues.append(self._spot_queue)
-
-            # Set up a timed callback to check if anything is in the queue
-            self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
-            self._heartbeat.start()
 
             # Flush headers immediately so nginx doesn't time out waiting for a response
             self.write_message("keepalive", "")
 
+            # Register to handle new spots arriving. The callback() method will get called with the new spot as an
+            # argument.
+            self._sse_spot_broadcaster.register(self)
+
         except Exception as e:
             logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
             self.close()
 
     def close(self):
-        """When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
+        """When the user closes the socket, deregister ourselves from the spot broadcaster"""
 
-        try:
+        self._sse_spot_broadcaster.unregister(self)
-            if self._spot_queue in self._sse_spot_queues:
-                self._sse_spot_queues.remove(self._spot_queue)
-            empty_queue(self._spot_queue)
-        except:
-            pass
-        try:
-            self._heartbeat.stop()
-        except:
-            pass
-        self._spot_queue = None
         super().close()
 
-    def _callback(self):
+    def callback(self, spot):
-        """Callback to check if anything has arrived in the queue, and if so send it to the client"""
+        """Callback when a new spot arrives"""
 
         try:
-            if self._spot_queue:
-                if not self._spot_queue.empty():
-                    while not self._spot_queue.empty():
-                        spot = self._spot_queue.get()
             # If the new spot matches our param filters, send it to the client. If not, ignore it.
             if spot_allowed_by_query(spot, self._query_params):
                 if self._credentials:
                     spot = copy.deepcopy(spot)
                     spot.infer_missing(self._credentials)
                 self.write_message(msg=safe_json_dumps(spot))
-
-                else:
-                    # Send a keepalive comment if the queue was empty
-                    self.write_message("keepalive", "")
-
-                if self._spot_queue not in self._sse_spot_queues:
-                    logging.error("Web server cleared up a queue of an active connection!")
-                    self.close()
         except Exception as e:
             logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
             self.close()
@@ -171,7 +139,7 @@ def get_spot_list_with_filters(all_spots, query):
     # Create a shallow copy of the spot list, ordered by spot time, then filter the list to reduce it only to spots
     # that match the filter parameters in the query string. Finally, apply a limit to the number of spots returned.
     # The list of query string filters is defined in the API docs.
-    spot_ids = list(all_spots.iterkeys())
+    spot_ids = all_spots.keys()
     spots = []
     for k in spot_ids:
         s = all_spots.get(k)

server/handlers/api/status.py

@@ -12,7 +12,7 @@ from core.utils import safe_json_dumps
 
 
 class APIStatusHandler(tornado.web.RequestHandler):
-    """API request handler for /api/v2/status"""
+    """API request handler for /api/v1/status"""
 
     def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
         self._status_data = None

server/handlers/api/v1_compatability.py

@@ -1,31 +0,0 @@
-import json
-
-import tornado
-
-from core.utils import serialize_everything
-
-
-class V1GoneHandler(tornado.web.RequestHandler):
-    """Returns 410 Gone with a message for any endpoints in the old API that have breaking changes in the new one or
-    have been retired."""
-
-    def post(self):
-        self.set_status(410)
-        self.write(json.dumps(
-            "This API endpoint has a breaking change or has been removed in the current version of the Spothole API. Please see /apidocs for details of the current API version and the endpoints available.",
-            default=serialize_everything
-        ))
-        self.set_header("Cache-Control", "no-store")
-        self.set_header("Content-Type", "application/json")
-
-
-class V1RedirectHandler(tornado.web.RequestHandler):
-    """Returns 308 Permanent Redirect from any path in the old API to the new one, where there were no breaking changes."""
-
-    def get(self, path):
-        new_url = "/api/v2/" + path
-        if self.request.query:
-            new_url += "?" + self.request.query
-        self.set_status(308)
-        self.set_header("Location", new_url)
-        self.finish()

server/sse_broadcaster.py

@@ -0,0 +1,36 @@
+import logging
+import threading
+
+from tornado.ioloop import IOLoop
+
+
+class SSEBroadcaster:
+    """Bridge between DataStore listener callbacks (which fire on provider threads) to Tornado's async SSE handlers
+    (which live on the IOLoop thread) to avoid any interdependency between them."""
+
+    def __init__(self):
+        self._handlers = set()
+        self._lock = threading.Lock()
+        self._loop = IOLoop.current()
+
+    def register(self, handler):
+        with self._lock:
+            self._handlers.add(handler)
+
+    def unregister(self, handler):
+        with self._lock:
+            self._handlers.discard(handler)
+
+    def publish(self, value):
+        self._loop.add_callback(self._fan_out, value)
+
+    def _fan_out(self, value):
+        with self._lock:
+            handlers = list(self._handlers)
+        for handler in handlers:
+            try:
+                handler.callback(value)
+            except Exception:
+                # Connection probably dropped, ignore and de-register the handler to stop getting future items.
+                logging.debug("Failed to push to an SSE client; dropping it")
+                self.unregister(handler)

server/webserver.py

@@ -6,9 +6,8 @@ import tornado
 from tornado.web import StaticFileHandler
 
 from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL
-from core.utils import empty_queue
+from core.data_store import DATA_STORE
 from server.handlers.api.addspot import APISpotHandler
-from server.handlers.api.v1_compatability import V1RedirectHandler, V1GoneHandler
 from server.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
 from server.handlers.api.dxstats import APIDxStatsHandler
 from server.handlers.api.lookups import APILookupCallHandler, APILookupSIGRefHandler, APILookupGridHandler
@@ -19,6 +18,7 @@ from server.handlers.api.status import APIStatusHandler
 from server.handlers.manifesthandler import ManifestHandler
 from server.handlers.metrics import PrometheusMetricsHandler
 from server.handlers.pagetemplate import PageTemplateHandler
+from server.sse_broadcaster import SSEBroadcaster
 
 _HERE = os.path.dirname(__file__ or "")
 
@@ -26,16 +26,12 @@ _HERE = os.path.dirname(__file__ or "")
 class WebServer:
     """Provides the public-facing web server."""
 
-    def __init__(self, spots, alerts, solar_conditions, status_data, spot_providers=None):
+    def __init__(self):
         """Constructor"""
 
-        self._spots = spots
+        self._data_store = DATA_STORE
-        self._alerts = alerts
+        self._spot_broadcaster = SSEBroadcaster()
-        self._solar_conditions = solar_conditions
+        self._alert_broadcaster = SSEBroadcaster()
-        self._sse_spot_queues = []
-        self._sse_alert_queues = []
-        self._status_data = status_data
-        self._spot_providers = spot_providers or []
         self._port = WEB_SERVER_PORT
         self._api_only_mode = API_ONLY_MODE
         self._shutdown_event = asyncio.Event()
@@ -47,6 +43,10 @@ class WebServer:
             "status": "Starting"
         }
 
+        # Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately
+        DATA_STORE.spots.add_listener(self._spot_broadcaster.publish)
+        DATA_STORE.alerts.add_listener(self._alert_broadcaster.publish)
+
     def start(self):
         """Start the web server"""
 
@@ -66,29 +66,21 @@ class WebServer:
 
         # API endpoints are always enabled
         api_routes = [
-            (r"/api/v2/spots", APISpotsHandler, {"spots": self._spots, **handler_opts}),
+            (r"/api/v1/spots", APISpotsHandler, {"spots": self._data_store.spots, **handler_opts}),
-            (r"/api/v2/alerts", APIAlertsHandler, {"alerts": self._alerts, **handler_opts}),
+            (r"/api/v1/alerts", APIAlertsHandler, {"alerts": self._data_store.alerts, **handler_opts}),
-            (r"/api/v2/spots/stream", APISpotsStreamHandler,
+            (r"/api/v1/spots/stream", APISpotsStreamHandler,
-             {"sse_spot_queues": self._sse_spot_queues, **handler_opts}),
+             {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts}),
-            (r"/api/v2/alerts/stream", APIAlertsStreamHandler,
+            (r"/api/v1/alerts/stream", APIAlertsStreamHandler,
-             {"sse_alert_queues": self._sse_alert_queues, **handler_opts}),
+             {"sse_alert_broadcaster": self._alert_broadcaster, **handler_opts}),
-            (r"/api/v2/solar", APISolarConditionsHandler, {"solar_conditions": self._solar_conditions, **handler_opts}),
+            (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._data_store.solar_conditions,
-            (r"/api/v2/dxstats", APIDxStatsHandler, {"spots": self._spots, **handler_opts}),
+                                                           **handler_opts}),
-            (r"/api/v2/options", APIOptionsHandler,
+            (r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._data_store.spots, **handler_opts}),
-             {"status_data": self._status_data, "spot_providers": self._spot_providers, **handler_opts}),
+            (r"/api/v1/options", APIOptionsHandler, {"status_data": self._data_store.status_data, **handler_opts}),
-            (r"/api/v2/status", APIStatusHandler, {"status_data": self._status_data, **handler_opts}),
+            (r"/api/v1/status", APIStatusHandler, {"status_data": self._data_store.status_data, **handler_opts}),
-            (r"/api/v2/lookup/call", APILookupCallHandler, {**handler_opts}),
+            (r"/api/v1/lookup/call", APILookupCallHandler, {**handler_opts}),
-            (r"/api/v2/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
+            (r"/api/v1/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
-            (r"/api/v2/lookup/grid", APILookupGridHandler, {**handler_opts}),
+            (r"/api/v1/lookup/grid", APILookupGridHandler, {**handler_opts}),
-            (r"/api/v2/spot", APISpotHandler,
+            (r"/api/v1/spot", APISpotHandler, {"spots": self._data_store.spots, **handler_opts}),
-             {"spots": self._spots, "spot_providers": self._spot_providers, **handler_opts}),
-        ]
-
-        # v1 API redirects. Most v1 enpoints are unchanged in v2, and get an HTTP 308 redirect to the v2 API. The ones
-        # that have the actual breaking changes get a bespoke handler.
-        v1_compat_routes = [
-            (r"/api/v1/spot", V1GoneHandler),
-            (r"/api/v1/(.*)", V1RedirectHandler),
         ]
 
         # If in API-only mode, serve a basic homepage; in normal mode, serve the usual UI routes
@@ -123,7 +115,7 @@ class WebServer:
             (r"/static/(.*)", StaticFileHandler, {"path": os.path.join(_HERE, "../static")})
         ]
 
-        app = tornado.web.Application(api_routes + v1_compat_routes + ui_routes + misc_routes,
+        app = tornado.web.Application(api_routes + ui_routes + misc_routes,
                                       template_path=os.path.join(_HERE, "../templates"),
                                       log_function=request_log,
                                       debug=False)
@@ -132,55 +124,6 @@ class WebServer:
         logging.info("You can access your copy of Spothole at " + BASE_URL)
         await self._shutdown_event.wait()
 
-    def notify_new_spot(self, spot):
-        """Internal method called when a new spot is added to the system. This is used to ping any SSE clients that are
-        awaiting a server-sent message with new spots."""
-
-        for queue in self._sse_spot_queues:
-            try:
-                queue.put(spot)
-            except:
-                # Cleanup thread was probably deleting the queue, that's fine
-                pass
-        pass
-
-    def notify_new_alert(self, alert):
-        """Internal method called when a new alert is added to the system. This is used to ping any SSE clients that are
-        awaiting a server-sent message with new spots."""
-
-        for queue in self._sse_alert_queues:
-            try:
-                queue.put(alert)
-            except:
-                # Cleanup thread was probably deleting the queue, that's fine
-                pass
-        pass
-
-    def clean_up_sse_queues(self):
-        """Clean up any SSE queues that are growing too large; probably their client disconnected and we didn't catch it
-        properly for some reason."""
-
-        for q in self._sse_spot_queues:
-            try:
-                if q.full():
-                    logging.warning(
-                        "A full SSE spot queue was found, presumably because the client disconnected strangely. It has been removed.")
-                    self._sse_spot_queues.remove(q)
-                    empty_queue(q)
-            except:
-                # Probably got deleted already on another thread
-                pass
-        for q in self._sse_alert_queues:
-            try:
-                if q.full():
-                    logging.warning(
-                        "A full SSE alert queue was found, presumably because the client disconnected strangely. It has been removed.")
-                    self._sse_alert_queues.remove(q)
-                    empty_queue(q)
-            except:
-                # Probably got deleted already on another thread
-                pass
-        pass
 
 def request_log(handler):
     """Custom log function to provide more data about requests when enabled, and to provide the ability to turn off

sigrefdataproviders/arlhs.py

@@ -0,0 +1,30 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class ARLHS(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Amateur Radio Light House Society"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "ARLHS"
+    DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            if "ARLHS" in row and row["ARLHS"] != "":
+                ref_id = row["ARLHS"]
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                              "Latitude"] != "" else None,
+                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                              "Longitude"] != "" else None,
+                                          grid=row["Maidenhead Locator"]))
+
+        return new_data

sigrefdataproviders/dme.py

@@ -0,0 +1,35 @@
+import csv
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
+
+
+class DME(LocalFileSIGRefDataProvider):
+    """SIG ref data provider for Diploma Municipios de Espana"""
+
+    SIG = "DME"
+    PATH = "datafiles/MUNICIPIOS.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.PATH)
+
+    def _file_to_data(self, path):
+        new_data = []
+        with open(path, encoding="latin-1") as _f:
+            for row in csv.DictReader(_f, delimiter=";"):
+                ref_id = row["COD_INE"][:5]
+                latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LATITUD_ETRS89_REGCAN95") else None
+                longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
+                    "LONGITUD_ETRS89_REGCAN95") else None
+
+                ref = SIGRef(sig=self.SIG, id=ref_id,
+                                          name=row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"],
+                                          latitude=latitude,
+                                          longitude=longitude)
+                if latitude and longitude:
+                    ref.grid = latlong_to_locator(latitude, longitude, 6)
+                new_data.append(ref)
+        return new_data

sigrefdataproviders/file_download_sig_ref_data_provider.py

@@ -0,0 +1,78 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
+from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider
+
+
+class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
+    """Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file."""
+
+    def __init__(self, sig_name, provider_config, url, poll_interval):
+        """ Set up the provider, note poll_interval is in *days*."""
+        super().__init__(sig_name, provider_config)
+        self._url = url
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+        self._url_data_cache = URLDataCache("sigrefdata-" + sig_name)
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info(
+            "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " days.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
+                break
+
+    def _poll(self):
+        try:
+            # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+            # caching, but just so continual restarts of the software during testing don't hammer the servers.
+            logging.debug("Downloading " + self.sig_name + " SIG ref data...")
+            http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                new_data = self._http_response_to_data(http_response)
+                # Submit the new spots for processing. There might not be any spots for the less popular programs.
+                if new_data:
+                    self._add_data(new_data)
+
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.debug("Received SIG ref data for " + self.sig_name)
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when downloading SIG ref data for {self.sig_name}.")
+
+        except ConnectionError:
+            logging.warning(f"Connection error when downloading SIG ref data for {self.sig_name}.")
+        except (ConnectTimeout, ReadTimeout):
+            logging.warning(f"Timeout when downloading SIG ref data for {self.sig_name}.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in HTTP SIG Ref Data Provider (" + self.sig_name + ")")
+            self._stop_event.wait(timeout=1)
+
+    def _http_response_to_data(self, http_response):
+        """Convert an HTTP response returned by the server into SIG Ref data. The whole response is provided here so the
+        subclass implementations can check for HTTP status codes if necessary, and handle the response as JSON, CSV,
+        whatever the remote file actually is."""
+
+        raise NotImplementedError("Subclasses must implement this method")

sigrefdataproviders/gma.py

@@ -0,0 +1,29 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class GMA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Global Mountain Activity"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "GMA"
+    DATA_URL = "https://www.gma.rocks/download/summits.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["Reference"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                          "Latitude"] != "" else None,
+                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                          "Longitude"] != "" else None,
+                                      grid=row["Maidenhead Locator"]))
+
+        return new_data

sigrefdataproviders/illw.py

@@ -0,0 +1,30 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class ILLW(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for International Lighthouse & Lightship Weekend"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "ILLW"
+    DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            if "ILLW" in row and row["ILLW"] != "":
+                ref_id = row["ILLW"]
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                              "Latitude"] != "" else None,
+                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                              "Longitude"] != "" else None,
+                                          grid=row["Maidenhead Locator"]))
+
+        return new_data

sigrefdataproviders/iota.py

@@ -0,0 +1,36 @@
+import logging
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class IOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Islands on the Air"""
+
+    POLL_INTERVAL_DAYS = 365
+    SIG = "IOTA"
+    DATA_URL = "https://www.iota-world.org/islands-on-the-air/downloads/download-file.html?path=groups.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data:
+                ref_id = ref["refno"]
+                latitude = float(ref["latitude_min"]) + float(ref["latitude_max"]) / 2.0
+                longitude = float(ref["longitude_min"]) + float(ref["longitude_max"]) / 2.0
+                grid = None
+                try:
+                    grid = latlong_to_locator(latitude, longitude, 6)
+                except ValueError:
+                    logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude)
+
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
+                                          grid=grid, latitude=latitude, longitude=longitude))
+
+        return new_data

sigrefdataproviders/llota.py

@@ -0,0 +1,32 @@
+from pyhamtools.locator import locator_to_latlong
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class LLOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Lagos y Lagunas on the Air"""
+
+    POLL_INTERVAL_DAYS = 7
+    SIG = "LLOTA"
+    DATA_URL = "https://llota.app/api/public/references"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data:
+                ref_id = ref["reference_code"]
+                grid = str(ref["grid_locator"])
+                ll = locator_to_latlong(grid)
+
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
+                                          url="https://llota.app/list/ref/" + ref_id,
+                                          grid=grid,
+                                          latitude=ll[0],
+                                          longitude=ll[1]))
+
+        return new_data

sigrefdataproviders/local_file_sig_ref_data_provider.py

@@ -0,0 +1,36 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider
+
+
+class LocalFileSIGRefDataProvider(SIGRefDataProvider):
+    """Generic SIG ref data provider class for providers that fetch their data from a local file on startup."""
+
+    def __init__(self, sig, provider_config, path):
+        super().__init__(sig, provider_config)
+        self._path = path
+
+    def start(self):
+        logging.debug("Loading " + self.sig_name + " SIG ref data from file.")
+        try:
+            new_data = self._file_to_data(self._path)
+            if new_data:
+                self._add_data(new_data)
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+            else:
+                logging.info("No new SIG ref data found for " + self.sig_name)
+        except Exception as e:
+            self.status = "Error"
+            logging.exception("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")")
+
+    def stop(self):
+        pass
+
+    def _file_to_data(self, path):
+        """Load a file on the given path and turn it into SIG Ref data."""
+
+        raise NotImplementedError("Subclasses must implement this method")

sigrefdataproviders/mota.py

@@ -0,0 +1,29 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class MOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Mills on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "MOTA"
+    DATA_URL = "https://www.gma.rocks/download/mills.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["Reference"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                          "Latitude"] != "" else None,
+                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                          "Longitude"] != "" else None,
+                                      grid=row["Maidenhead Locator"]))
+
+        return new_data

sigrefdataproviders/pota.py

@@ -0,0 +1,29 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class POTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Parks on the Air"""
+
+    POLL_INTERVAL_DAYS = 7
+    SIG = "POTA"
+    DATA_URL = "https://pota.app/all_parks_ext.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["reference"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
+                                      url="https://pota.app/#/park/" + ref_id,
+                                      grid=row["grid"] if "grid" in row else None,
+                                      latitude=float(row["latitude"]) if "latitude" in row and row[
+                                          "latitude"] != "" else None,
+                                      longitude=float(row["longitude"]) if "longitude" in row and row[
+                                          "longitude"] != "" else None))
+
+        return new_data

sigrefdataproviders/sig_ref_data_provider.py

@@ -0,0 +1,40 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from core.data_store import DATA_STORE
+
+
+class SIGRefDataProvider:
+    """Generic SIG reference data provider class. Subclasses of this query the individual URLs or files for data."""
+
+    def __init__(self, sig_name, provider_config):
+        """Constructor"""
+
+        self.sig_name = sig_name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+        self.reference_count = 0
+
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def _add_data(self, new_data):
+        """Add all the provided reference data objects to the data store."""
+
+        for d in new_data:
+            DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d
+        self.reference_count = len(new_data)
+        logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)

sigrefdataproviders/siota.py

@@ -0,0 +1,26 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class SIOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Silos on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "SIOTA"
+    DATA_URL = "https://www.silosontheair.com/data/silos.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["SILO_CODE"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
+                                      grid=row["LOCATOR"] if "LOCATOR" in row else None,
+                                      latitude=float(row["LAT"]) if "LAT" in row else None,
+                                      longitude=float(row["LNG"]) if "LNG" in row else None))
+
+        return new_data

sigrefdataproviders/sota.py

@@ -0,0 +1,34 @@
+import csv
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class SOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Summits on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "SOTA"
+    DATA_URL = "https://storage.sota.org.uk/summitslist.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["SummitCode"]
+            latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
+            longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
+            ref = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
+                                      url="https://www.sotadata.org.uk/en/summit/" + ref_id,
+                                      latitude=latitude,
+                                      longitude=longitude,
+                                      activation_score=int(row["Points"]) if "Points" in row else None)
+            if latitude and longitude:
+                ref.grid = latlong_to_locator(latitude, longitude, 6)
+            new_data.append(ref)
+
+        return new_data

sigrefdataproviders/toilets.py

@@ -0,0 +1,24 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
+
+
+class Toilets(LocalFileSIGRefDataProvider):
+    """SIG ref data provider for Toilets on the Air"""
+
+    SIG = "Toilets"
+    PATH = "datafiles/toilets.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.PATH)
+
+    def _file_to_data(self, path):
+        new_data = []
+        f = open(path)
+        csv_data = f.read()
+        dr = csv.DictReader(csv_data.splitlines())
+        for row in dr:
+            new_data.append(SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]),
+                                          longitude=float(row["lon"])))
+        return new_data

sigrefdataproviders/towers.py

@@ -0,0 +1,27 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class Towers(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Towers on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "Towers"
+    DATA_URL = "https://wwtota.com/servis/generate_csv.php?ref=&filter=all"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
+            ref_id = row["Ref"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
+                                      url="https://wwtota.com/seznam/karta_rozhledny.php?ref=" + ref_id,
+                                      grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
+                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
+                                      longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None))
+
+        return new_data

sigrefdataproviders/wca.py

@@ -0,0 +1,44 @@
+import csv
+import logging
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WCA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for World Castles Award"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "WCA"
+    DATA_URL = "https://polo.ham2k.com/data/activities/wca/all-castles.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["REF"]
+
+            coordsStr = row["COORDINATES"]
+            latitude = None
+            longitude = None
+            grid = None
+            try:
+                if coordsStr:
+                    split = coordsStr.split(", ")
+                    latitude = float(split[0])
+                    longitude = float(split[1])
+                    grid = latlong_to_locator(latitude, longitude)
+            except ValueError:
+                logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id)
+
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=latitude,
+                                      longitude=longitude,
+                                      grid=grid))
+
+        return new_data

sigrefdataproviders/wota.py

@@ -0,0 +1,31 @@
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Wainwrights on the Air"""
+
+    POLL_INTERVAL_DAYS = 365
+    SIG = "WOTA"
+    DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for feature in http_response.json().get("features", []):
+            ref_id = feature["properties"]["wotaId"]
+            # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
+            # added to them
+            url = "https://www.wota.org.uk/MM_" + ref_id
+            if ref_id.upper().startswith("LDO-"):
+                number = int(ref_id.upper().replace("LDO-", ""))
+                url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
+
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
+                                      grid=feature["properties"]["qthLocator"],
+                                      latitude=feature["geometry"]["coordinates"][1],
+                                      longitude=feature["geometry"]["coordinates"][0]))
+
+        return new_data

sigrefdataproviders/wwbota.py

@@ -0,0 +1,27 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WWBOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Worldwide Bunkers on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "WWBOTA"
+    DATA_URL = "https://api.wwbota.org/bunkers/?format=CSV"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["Reference"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None,
+                                      grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
+                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
+                                      longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None))
+
+        return new_data

sigrefdataproviders/wwff.py

@@ -0,0 +1,30 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WWFF(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Worldwide Flora & Fauna"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "WWFF"
+    DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["reference"]
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
+                                      url="https://wwff.co/directory/?showRef=" + ref_id,
+                                      grid=row["iaruLocator"] if "iaruLocator" in row and row[
+                                          "iaruLocator"] != "-" else None,
+                                      latitude=float(row["latitude"]) if "latitude" in row and row[
+                                          "latitude"] != "" and row["latitude"] != "-" else None,
+                                      longitude=float(row["longitude"]) if "longitude" in row and row[
+                                          "longitude"] != "" and row["longitude"] != "-" else None))
+
+        return new_data

sigrefdataproviders/zlota.py

@@ -0,0 +1,34 @@
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class ZLOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for New Zealand on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "ZLOTA"
+    DATA_URL = "https://ontheair.nz/assets/assets.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = []
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data:
+                ref_id = ref["code"]
+                latitude = ref["y"]
+                longitude = ref["x"]
+
+                ref = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
+                                          url="https://ontheair.nz/assets/" + ref_id.replace("/", "_"),
+                                          latitude=latitude,
+                                          longitude=longitude)
+                if latitude and longitude:
+                    ref.grid = latlong_to_locator(latitude, longitude, 6)
+                new_data.append(ref)
+
+        return new_data

solarconditionsproviders/giroionosonde.py

@@ -30,11 +30,23 @@ class GIROIonosonde(SolarConditionsProvider):
     data, but is less reliable and often offline."""
 
     def __init__(self, provider_config):
-        super().__init__(provider_config)
+        super().__init__("GIRO Ionosonde Data", provider_config)
         self._stations = self._load_stations()
         self._thread = None
         self._stop_event = Event()
 
+        # Pre-populate ionosonde_data with known station names for stations not already present,
+        # so the station dropdown is available before the first poll. Does not overwrite existing
+        # entries so KC2G cache data is preserved.
+        existing = self._solar_conditions.ionosonde_data or {}
+        new_entries = {
+            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
+                        "luf": None, "band_states": None}
+            for s in self._stations if s["ursi"] not in existing
+        }
+        if new_entries:
+            self.update_data({"ionosonde_data": {**existing, **new_entries}})
+
     @staticmethod
     def _load_stations():
         stations = []
@@ -44,21 +56,6 @@ class GIROIonosonde(SolarConditionsProvider):
                     stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
         return stations
 
-    def setup(self, solar_conditions, solar_conditions_cache):
-        """Pre-populate ionosonde_data with known station names for stations not already present,
-        so the station dropdown is available before the first poll. Does not overwrite existing
-        entries so KC2G cache data is preserved."""
-
-        super().setup(solar_conditions, solar_conditions_cache)
-        existing = solar_conditions.ionosonde_data or {}
-        new_entries = {
-            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
-                        "luf": None, "band_states": None}
-            for s in self._stations if s["ursi"] not in existing
-        }
-        if new_entries:
-            self.update_data({"ionosonde_data": {**existing, **new_entries}})
-
     def start(self):
         logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
         self._thread = Thread(target=self._run, daemon=True)

solarconditionsproviders/hamqsl.py

@@ -15,7 +15,7 @@ class HamQSL(HTTPSolarConditionsProvider):
     Provides solar flux index, geomagnetic indices, and HF/VHF propagation condition summaries."""
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, URL, POLL_INTERVAL)
+        super().__init__("HamQSL", provider_config, URL, POLL_INTERVAL)
 
     def _http_response_to_solar_conditions(self, http_response):
         root = ElementTree.fromstring(http_response.text)

solarconditionsproviders/http_solar_conditions_provider.py

@@ -14,8 +14,8 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
     """Generic solar conditions provider for providers that request data via HTTP(S). Subclasses implement
     _http_response_to_solar_conditions() to parse the specific API response format."""
 
-    def __init__(self, provider_config, url, poll_interval):
+    def __init__(self, name, provider_config, url, poll_interval):
-        super().__init__(provider_config)
+        super().__init__(name, provider_config)
         self._url = url
         self._poll_interval = poll_interval
         self._thread = None

solarconditionsproviders/kc2gprop.py

@@ -24,7 +24,7 @@ class KC2GProp(SolarConditionsProvider):
     online, but has fewer stations and does not provide LUF data."""
 
     def __init__(self, provider_config):
-        super().__init__(provider_config)
+        super().__init__("KC2G Propagation Data", provider_config)
         self._thread = None
         self._stop_event = Event()
 

solarconditionsproviders/noaa3dayforecast.py

@@ -13,7 +13,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
     corresponding fields in the solar conditions object.."""
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, URL, POLL_INTERVAL)
+        super().__init__("NOAA 3-day Forecast", provider_config, URL, POLL_INTERVAL)
 
     @staticmethod
     def _parse_percentage_table(lines, section_header, year):

solarconditionsproviders/solar_conditions_provider.py

@@ -2,26 +2,21 @@ from datetime import datetime
 
 import pytz
 
+from core.data_store import DATA_STORE
+
 
 class SolarConditionsProvider:
     """Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
      propagation data."""
 
-    def __init__(self, provider_config):
+    def __init__(self, name, provider_config):
         """Constructor"""
 
-        self._solar_conditions_cache = None
+        self.name = name
-        self.name = provider_config["name"]
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._solar_conditions = None
+        self._solar_conditions = DATA_STORE.solar_conditions
-
-    def setup(self, solar_conditions, solar_conditions_cache):
-        """Set up the provider, giving it the solar conditions object and its backing cache"""
-
-        self._solar_conditions = solar_conditions
-        self._solar_conditions_cache = solar_conditions_cache
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -41,4 +36,3 @@ class SolarConditionsProvider:
                 if hasattr(self._solar_conditions, key):
                     setattr(self._solar_conditions, key, value)
             self._solar_conditions.infer_descriptions()
-            self._solar_conditions_cache['solar_conditions'] = self._solar_conditions

spothole.py

@@ -1,30 +1,23 @@
 # Main script
-import importlib
 import logging
 import os
 import signal
 import sys
 
-from diskcache import Cache
+from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL, get_sig_ref_data_provider_from_config, \
-
+    get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config
-from core.cleanup import CleanupTimer
-from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL
 from core.constants import SOFTWARE_VERSION
+from core.data_store import DATA_STORE
 from core.lookup_helper import lookup_helper
 from core.status_reporter import StatusReporter
-from data.solar_conditions import SolarConditions
 from server.webserver import WebServer
 
 # Globals
-spots = Cache('cache/spots_cache')
-alerts = Cache('cache/alerts_cache')
-solar_conditions_cache = Cache('cache/solar_conditions_cache')
-solar_conditions = solar_conditions_cache.get('solar_conditions', SolarConditions())
 web_server = None
-status_data = {}
 spot_providers = []
 alert_providers = []
 solar_condition_providers = []
+sig_ref_data_providers = []
 cleanup_timer = None
 run = True
 
@@ -46,40 +39,13 @@ def shutdown(_signum=None, _frame=None):
     for scp in solar_condition_providers:
         if scp.enabled:
             scp.stop()
-    if cleanup_timer:
+    for srdp in sig_ref_data_providers:
-        cleanup_timer.stop()
+        if srdp.enabled:
-    if lookup_helper:
+            srdp.stop()
-        lookup_helper.stop()
+    DATA_STORE.close()
-    spots.close()
-    alerts.close()
-    solar_conditions_cache.close()
     os._exit(0)
 
 
-def get_spot_provider_from_config(config_providers_entry):
-    """Utility method to get a spot provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('spotproviders.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
-def get_alert_provider_from_config(config_providers_entry):
-    """Utility method to get an alert provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('alertproviders.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
-def get_solar_conditions_provider_from_config(config_providers_entry):
-    """Utility method to get a solar conditions provider based on the class specified in its config entry."""
-
-    module = importlib.import_module('solarconditionsproviders.' + config_providers_entry["class"].lower())
-    provider_class = getattr(module, config_providers_entry["class"])
-    return provider_class(config_providers_entry)
-
-
 # Main function
 if __name__ == '__main__':
     # Set up logging
@@ -99,47 +65,47 @@ if __name__ == '__main__':
     # Shut down gracefully on SIGINT
     signal.signal(signal.SIGINT, shutdown)
 
+    # Set up data store
+    DATA_STORE.setup()
+
     # Set up lookup helper
     lookup_helper.start()
 
-    # Create spot providers
+    # Set up web server
+    web_server = WebServer()
+
+    # Fetch, set up and start spot providers
     for entry in config["spot-providers"]:
         spot_providers.append(get_spot_provider_from_config(entry))
-
-    # Set up web server
-    web_server = WebServer(spots=spots, alerts=alerts, solar_conditions=solar_conditions, status_data=status_data,
-                           spot_providers=spot_providers)
-
-    # Set up and start spot providers
     for p in spot_providers:
-        p.setup(spots=spots, web_server=web_server)
         if p.enabled:
             p.start()
 
-    # Create, set up and start alert providers
+    # Fetch, set up and start alert providers
     for entry in config["alert-providers"]:
         alert_providers.append(get_alert_provider_from_config(entry))
     for p in alert_providers:
-        p.setup(alerts=alerts, web_server=web_server)
         if p.enabled:
             p.start()
 
-    # Create, set up and start solar conditions providers
+    # Fetch, set up and start solar conditions providers
     for entry in config.get("solar-condition-providers", []):
         solar_condition_providers.append(get_solar_conditions_provider_from_config(entry))
     for p in solar_condition_providers:
-        p.setup(solar_conditions=solar_conditions, solar_conditions_cache=solar_conditions_cache)
         if p.enabled:
             p.start()
 
-    # Set up timer to clear spot list of old data
+    # Fetch, set up and start SIG reference data providers
-    cleanup_timer = CleanupTimer(spots=spots, alerts=alerts, web_server=web_server, cleanup_interval=60)
+    for entry in config.get("sig-ref-data-providers", []):
-    cleanup_timer.start()
+        sig_ref_data_providers.append(get_sig_ref_data_provider_from_config(entry))
+    for p in sig_ref_data_providers:
+        if p.enabled:
+            p.start()
 
     # Set up status reporter
-    status_reporter = StatusReporter(status_data=status_data, spots=spots, alerts=alerts, web_server=web_server,
+    status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers,
-                                     cleanup_timer=cleanup_timer, spot_providers=spot_providers,
                                      alert_providers=alert_providers,
+                                     sig_ref_data_providers=sig_ref_data_providers,
                                      solar_condition_providers=solar_condition_providers, run_interval=5)
     status_reporter.start()
 

spotproviders/aprsis.py

@@ -14,7 +14,7 @@ class APRSIS(SpotProvider):
     """Spot provider for the APRS-IS."""
 
     def __init__(self, provider_config):
-        super().__init__(provider_config)
+        super().__init__("APRS-IS", provider_config)
         self._thread = Thread(target=self._connect)
         self._thread.daemon = True
         self._aprsis = None

spotproviders/dxcluster.py

@@ -26,7 +26,8 @@ class DXCluster(SpotProvider):
     def __init__(self, provider_config):
         """Constructor requires hostname and port"""
 
-        super().__init__(provider_config)
+        name = provider_config["name"] if "name" in provider_config else "Cluster"
+        super().__init__(name, provider_config)
         self._hostname = provider_config["host"]
         self._port = provider_config["port"]
         self._login_prompt = provider_config["login_prompt"] if "login_prompt" in provider_config else "login:"
@@ -59,6 +60,10 @@ class DXCluster(SpotProvider):
                     self._telnet.write((self._login_callsign + "\n").encode("latin-1"))
                     connected = True
                     logging.info("DX Cluster " + self._hostname + " connected.")
+                except ConnectionRefusedError:
+                    self.status = "Error"
+                    logging.warning("Connection refused to DX cluster " + self._hostname)
+                    sleep(300)
                 except Exception:
                     self.status = "Error"
                     logging.exception("Exception while connecting to DX Cluster Provider (" + self._hostname + ").")

spotproviders/gma.py

@@ -3,8 +3,8 @@ from datetime import datetime
 
 import pytz
 
-from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -21,12 +21,13 @@ class GMA(HTTPSpotProvider):
     def __init__(self, provider_config):
         # Ensure there is an API key in our config, and set up the query URL using it. If no key is provided,
         # disable this spot provider.
-        self.api_key = provider_config.get("api-key", "")
+        self._api_key = provider_config.get("api-key", "")
-        if self.api_key == "":
+        if self._api_key == "":
             provider_config["enabled"] = False
             logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
+        self._url_data_cache = URLDataCache("GMA")
 
-        super().__init__(provider_config, self.SPOTS_URL + "?key=" + self.api_key, self.POLL_INTERVAL_SEC)
+        super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self._api_key, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -56,7 +57,7 @@ class GMA(HTTPSpotProvider):
                 # GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
                 if "REF" in source_spot:
                     try:
-                        ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
+                        ref_response = self._url_data_cache.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
                                                           headers=HTTP_HEADERS)
                         # Sometimes this is blank even if it's a 200 response, so handle that
                         if ref_response.ok and ref_response.text is not None and ref_response.text != "":
@@ -112,11 +113,3 @@ class GMA(HTTPSpotProvider):
             logging.warning(f"The GMA API returned an unexpected response (HTTP {http_response.status_code}).")
 
         return new_spots
-
-    def can_submit_spot(self, sig):
-        return sig == "GMA"
-
-    def submit_spot(self, spot, credentials):
-        # TODO: Implement.
-        # Spotting to GMA is documented: https://www.cqgma.org/api/doc/apigma_spot.pdf We (or the user) need a GMA account, and to send the password in plaintext(!!)
-        raise NotImplementedError("GMA upstream spot submission is not yet implemented")

spotproviders/hema.py

@@ -25,7 +25,7 @@ class HEMA(HTTPSpotProvider):
     SPOTTER_COMMENT_PATTERN = re.compile("^\\((.*)\\) (.*)$")
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOT_SEED_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("HEMA", provider_config, self.SPOT_SEED_URL, self.POLL_INTERVAL_SEC)
         self._spot_seed = ""
 
     def _http_response_to_spots(self, http_response):
@@ -73,11 +73,3 @@ class HEMA(HTTPSpotProvider):
             except ConnectionError:
                 logging.warning("Connection error when accessing HEMA spots API.")
         return new_spots
-
-    def can_submit_spot(self, sig):
-        return sig == "HEMA"
-
-    def submit_spot(self, spot, credentials):
-        # TODO: Implement. Currently blocked awaiting their API team to make a change to allow us to spot with a
-        # reference and not a reference *number*.
-        raise NotImplementedError("HEMA upstream spot submission is not yet implemented")

spotproviders/http_spot_provider.py

@@ -15,13 +15,12 @@ class HTTPSpotProvider(SpotProvider):
     """Generic spot provider class for providers that request data via HTTP(S). Just for convenience to avoid code
     duplication. Subclasses of this query the individual APIs for data."""
 
-    def __init__(self, provider_config, url, poll_interval):
+    def __init__(self, name, provider_config, url, poll_interval):
-        super().__init__(provider_config)
+        super().__init__(name, provider_config)
         self._url = url
         self._poll_interval = poll_interval
         self._thread = None
         self._stop_event = Event()
-        self._wakeup_event = Event()
 
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
@@ -32,19 +31,11 @@ class HTTPSpotProvider(SpotProvider):
 
     def stop(self):
         self._stop_event.set()
-        self._wakeup_event.set()
-
-    def force_poll(self):
-        """Trigger an immediate poll without waiting for the normal interval."""
-
-        self._wakeup_event.set()
 
     def _run(self):
         while True:
-            self._wakeup_event.clear()
             self._poll()
-            self._wakeup_event.wait(timeout=self._poll_interval)
+            if self._stop_event.wait(timeout=self._poll_interval):
-            if self._stop_event.is_set():
                 break
 
     def _poll(self):
@@ -73,7 +64,7 @@ class HTTPSpotProvider(SpotProvider):
             logging.warning(f"Timeout when accessing {self.name} spots API.")
         except Exception:
             self.status = "Error"
-            logging.exception("Exception in HTTP JSON Spot Provider (" + self.name + ")")
+            logging.exception("Exception in HTTP Spot Provider (" + self.name + ")")
             self._stop_event.wait(timeout=1)
 
     def _http_response_to_spots(self, http_response):

spotproviders/llota.py

@@ -12,7 +12,7 @@ class LLOTA(HTTPSpotProvider):
     SPOTS_URL = "https://llota.app/api/public/spots"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("LLOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []

spotproviders/parksnpeaks.py

@@ -3,9 +3,7 @@ import re
 from datetime import datetime
 
 import pytz
-import requests
 
-from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -16,12 +14,10 @@ class ParksNPeaks(HTTPSpotProvider):
 
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
-    SUBMIT_URL = "https://www.parksnpeaks.org/api/SPOT/"
     SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
-    SUBMITTABLE_SIGS = ["POTA", "SOTA", "WWFF", "HEMA", "WOTA", "ZLOTA", "SIOTA", "KRMNPA"]
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("ParksNPeaks", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -67,28 +63,3 @@ class ParksNPeaks(HTTPSpotProvider):
                 # Add new spot to the list
                 new_spots.append(spot)
         return new_spots
-
-    def can_submit_spot(self, sig):
-        return sig in self.SUBMITTABLE_SIGS
-
-    def submit_spot(self, spot, credentials):
-        # TODO test this works
-        user_id = credentials.get("user_id", "")
-        api_key = credentials.get("api_key", "")
-        if not user_id or not api_key:
-            raise ValueError(
-                "Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account.")
-        sig_ref = spot.sig_refs[0].id if spot.sig_refs else ""
-        body = {
-            "actClass": spot.sig or "",
-            "actCallsign": spot.dx_call,
-            "actSite": sig_ref,
-            "mode": spot.mode or "",
-            "freq": str(spot.freq / 1000000.0),
-            "comments": spot.comment or "",
-            "userID": user_id,
-            "APIKey": api_key,
-        }
-        response = requests.post(self.SUBMIT_URL, json=body, headers=HTTP_HEADERS, timeout=(5, 30))
-        if not response.ok:
-            raise RuntimeError("Parks N Peaks API returned " + str(response.status_code) + ": " + response.text)

spotproviders/pota.py

@@ -1,9 +1,7 @@
 from datetime import datetime
 
 import pytz
-import requests
 
-from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -14,10 +12,9 @@ class POTA(HTTPSpotProvider):
 
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://api.pota.app/spot/activator"
-    SUBMIT_URL = "https://api.pota.app/spot"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("POTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -43,25 +40,3 @@ class POTA(HTTPSpotProvider):
             # that for us.
             new_spots.append(spot)
         return new_spots
-
-    def can_submit_spot(self, sig):
-        return sig == "POTA"
-
-    def submit_spot(self, spot, credentials):
-        sig_ref = spot.sig_refs[0].id if spot.sig_refs else None
-        if sig_ref:
-            body = {
-                "activator": spot.dx_call,
-                "spotter": spot.de_call,
-                "frequency": str(spot.freq / 1000.0),
-                "mode": spot.mode or "",
-                "reference": sig_ref,
-                "comments": spot.comment or "",
-                "source": "Spothole",
-            }
-            headers = {**HTTP_HEADERS, "Content-Type": "application/json"}
-            response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
-            if not response.ok:
-                raise RuntimeError("POTA API returned " + str(response.status_code) + ": " + response.text)
-        else:
-            raise RuntimeError("Park reference is required for submitting POTA spots.")

spotproviders/rbn.py

@@ -23,7 +23,8 @@ class RBN(SpotProvider):
     def __init__(self, provider_config):
         """Constructor requires port number."""
 
-        super().__init__(provider_config)
+        name = provider_config["name"] if "name" in provider_config else "RBN"
+        super().__init__(name, provider_config)
         self._port = provider_config["port"]
         self._telnet = None
         self._thread = Thread(target=self._handle)

spotproviders/sota.py

@@ -4,7 +4,7 @@ from datetime import datetime
 import requests
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
-from core.constants import HTTP_HEADERS, SSB_SUB_MODES, DV_SUB_MODES
+from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -19,14 +19,9 @@ class SOTA(HTTPSpotProvider):
     # The actual data lookup all happens after parsing and checking the epoch.
     EPOCH_URL = "https://api-db2.sota.org.uk/api/spots/epoch"
     SPOTS_URL = "https://api-db2.sota.org.uk/api/spots/60/all/all"
-    # SOTA spots don't contain lat/lon, we need a separate lookup for that
-    SUMMIT_URL_ROOT = "https://api-db2.sota.org.uk/api/summits/"
-
-    SUBMIT_URL = "https://api-db2.sota.org.uk/api/spots"
-    VALID_MODES = ["AM", "CW", "Data", "DV", "FM", "SSB"]
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("SOTA", provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC)
         self._api_epoch = ""
 
     def _http_response_to_spots(self, http_response):
@@ -66,46 +61,3 @@ class SOTA(HTTPSpotProvider):
             except (ConnectTimeout, ReadTimeout):
                 logging.warning(f"Timeout when accessing SOTA spots API.")
         return new_spots
-
-    def can_submit_spot(self, sig):
-        return sig == "SOTA"
-
-    def submit_spot(self, spot, credentials):
-        # TODO test this method works
-        access_token = credentials.get("access_token", "")
-        id_token = credentials.get("id_token", "")
-        if not access_token or not id_token:
-            raise ValueError("SOTA API tokens are required. Please log into SOTA in order to spot to it.")
-        sig_ref = spot.sig_refs[0].id if spot.sig_refs else ""
-        if sig_ref:
-            # Split reference into association and summit codes
-            ref_split = sig_ref.split("/")
-
-            # Figure out a valid mode. Borrowed this from PoLo :)
-            # https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js
-            mode = spot.mode
-            if mode and mode not in self.VALID_MODES:
-                if mode in SSB_SUB_MODES:
-                    mode = "SSB"
-                elif mode in DV_SUB_MODES:
-                    mode = "DV"
-                else:
-                    mode = "Data"
-
-            body = {
-                "activatorCallsign": spot.dx_call,
-                "associationCode": ref_split[0],
-                "summitCode": ref_split[1],
-                "frequency": spot.freq / 1000000.0,
-                "mode": mode or "",
-                "callsign": spot.de_call,
-                "comments": spot.comment or "",
-                "type": "TEST"  # todo replatce with NORMAL/QRT once testing complete
-            }
-            headers = {**HTTP_HEADERS, "Authorization": "bearer " + access_token, "id_token": id_token,
-                       "Content-Type": "application/json"}
-            response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
-            if not response.ok:
-                raise RuntimeError("SOTA API returned " + str(response.status_code) + ": " + response.text)
-        else:
-            raise RuntimeError("Summit reference is required for submitting SOTA spots.")

spotproviders/spot_provider.py

@@ -2,28 +2,21 @@ from datetime import datetime
 
 import pytz
 
-from core.config import MAX_SPOT_AGE
+from core.data_store import DATA_STORE
 
 
 class SpotProvider:
     """Generic spot provider class. Subclasses of this query the individual APIs for data."""
 
-    def __init__(self, provider_config):
+    def __init__(self, name, provider_config):
         """Constructor"""
 
-        self.name = provider_config["name"]
+        self.name = name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._spots = None
+        self._spots = DATA_STORE.spots
-        self._web_server = None
-
-    def setup(self, spots, web_server):
-        """Set up the provider, e.g. giving it the spot list to work from"""
-
-        self._spots = spots
-        self._web_server = web_server
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -59,29 +52,9 @@ class SpotProvider:
 
     def _add_spot(self, spot):
         if not spot.expired():
-            self._spots.add(spot.id, spot, expire=MAX_SPOT_AGE)
+            self._spots.set(spot.id, spot)
-            # Ping the web server in case we have any SSE connections that need to see this immediately
-            if self._web_server:
-                self._web_server.notify_new_spot(spot)
 
     def stop(self):
         """Stop any threads and prepare for application shutdown"""
 
         raise NotImplementedError("Subclasses must implement this method")
-
-    def can_submit_spot(self, sig):
-        """Return True if this provider supports submitting spots upstream for the given SIG."""
-
-        return False
-
-    def submit_spot(self, spot, credentials):
-        """Submit a spot upstream to this provider's API. credentials is a dict with provider-specific keys.
-        Raises an exception with a descriptive message on failure."""
-
-        raise NotImplementedError("This provider does not support spot submission")
-
-    def force_poll(self):
-        """Trigger an immediate poll without waiting for the normal interval. Default implementation here does nothing
-        because not all spot providers have a polling mechanism. Providers that do should override this method."""
-
-        return

spotproviders/sse_spot_provider.py

@@ -13,8 +13,8 @@ from spotproviders.spot_provider import SpotProvider
 class SSESpotProvider(SpotProvider):
     """Spot provider using Server-Sent Events."""
 
-    def __init__(self, provider_config, url):
+    def __init__(self, name, provider_config, url):
-        super().__init__(provider_config)
+        super().__init__(name, provider_config)
         self._url = url
         self._event_source = None
         self._thread = None

spotproviders/tiles.py

@@ -1,8 +1,5 @@
 from datetime import datetime
 
-import requests
-
-from core.constants import HTTP_HEADERS, SSB_SUB_MODES
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from spotproviders.http_spot_provider import HTTPSpotProvider
@@ -13,12 +10,9 @@ class Tiles(HTTPSpotProvider):
 
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/spots?active_hours=24"
-    SUBMIT_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/self-spot"
-    VALID_MODES = ["SSB", "CW", "FT8", "FT4", "FM", "DMR", "D-STAR", "M17", "AX.25", "JS8Call", "PSK31", "Olivia",
-                   "VarAC", "Other"]
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("Tiles", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -48,49 +42,6 @@ class Tiles(HTTPSpotProvider):
             new_spots.append(spot)
         return new_spots
 
-    def can_submit_spot(self, sig):
-        return sig == "Tiles"
-
-    def submit_spot(self, spot, credentials):
-        # Tiles on the air currently only supports *self* spots
-        if spot.dx_call == spot.de_call:
-
-            # Figure out a valid mode. Borrowed this from PoLo :)
-            # https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js
-            if spot.mode:
-                mode = spot.mode
-                if mode not in self.VALID_MODES:
-                    if mode in SSB_SUB_MODES:
-                        mode = "SSB"
-                    elif mode == "OLIVIA":
-                        mode = "Olivia"
-                    elif mode == "JS8":
-                        mode = "JS8Call"
-                    else:
-                        mode = "Other"
-
-                body = {
-                    "call_sign": spot.dx_call,
-                    "frequency": str(spot.freq / 1000000.0),
-                    "mode": mode or "",
-                    "grid": spot.dx_grid or "",
-                    "comment": spot.comment or "",
-                    "lat": spot.dx_latitude or None,
-                    "lon": spot.dx_longitude or None,
-                    "qrt": spot.qrt or False,
-                    "pin": credentials.get("offline_spot_gateway_pin", "")
-                }
-                headers = {**HTTP_HEADERS, "Content-Type": "application/json"}
-                response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
-                if not response.ok:
-                    raise RuntimeError(
-                        "Tiles on the Air API returned " + str(response.status_code) + ": " + response.text)
-            else:
-                raise RuntimeError("The Tiles on the Air API requires a mode to be set.")
-        else:
-            raise RuntimeError(
-                "The Tiles on the Air API only supports self-spots, the DX call and spotter call must match.")
-
 
 # Utility function to keep the first decimal point in a given string but remove any others. Used to parse Tiles'
 # strange frequency format where we can sometimes have e.g. "14.123.5".

spotproviders/towers.py

@@ -13,7 +13,7 @@ class Towers(HTTPSpotProvider):
     SPOTS_URL = "https://wwtota.com/api/cluster_live.php"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("Towers", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -30,8 +30,8 @@ class Towers(HTTPSpotProvider):
                         dx_call=source_spot["call"].upper(),
                         freq=likely_freq,
                         comment=source_spot["comment"],
-                        sig="WWTOTA",
+                        sig="Towers",
-                        sig_refs=[SIGRef(id=source_spot["ref"], sig="WWTOTA")],
+                        sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
                         time=datetime.strptime(response_json["updated"][:10] + source_spot["time"],
                                                "%Y-%m-%d%H:%M").timestamp())
 

spotproviders/ukpacketnet.py

@@ -14,7 +14,7 @@ class UKPacketNet(HTTPSpotProvider):
     SPOTS_URL = "https://nodes.ukpacketradio.network/api/nodedata"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("UK Packet Net", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []

spotproviders/websocket_spot_provider.py

@@ -13,8 +13,8 @@ from spotproviders.spot_provider import SpotProvider
 class WebsocketSpotProvider(SpotProvider):
     """Spot provider using websockets."""
 
-    def __init__(self, provider_config, url):
+    def __init__(self, name, provider_config, url):
-        super().__init__(provider_config)
+        super().__init__(name, provider_config)
         self._url = url
         self._ws = None
         self._thread = None

spotproviders/wota.py

@@ -21,11 +21,11 @@ class WOTA(HTTPSpotProvider):
     RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("WOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
-        rss = cast(RSS, Parser.parse(http_response.content.decode()))
+        rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
         # Iterate through source data
         for source_spot in rss.channel.items:
 
@@ -79,10 +79,3 @@ class WOTA(HTTPSpotProvider):
             except Exception as e:
                 logging.error("Exception parsing WOTA spot", e)
         return new_spots
-
-    def can_submit_spot(self, sig):
-        return sig == "WOTA"
-
-    def submit_spot(self, spot, credentials):
-        # TODO Ask M5TEA if he's happy to share how this is done from his app
-        raise NotImplementedError("WOTA upstream spot submission is not yet implemented")

spotproviders/wwbota.py

@@ -12,7 +12,7 @@ class WWBOTA(SSESpotProvider):
     SPOTS_URL = "https://api.wwbota.net/spots/"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL)
+        super().__init__("WWBOTA", provider_config, self.SPOTS_URL)
 
     def _sse_message_to_spot(self, message):
         source_spot = json.loads(message)
@@ -41,10 +41,3 @@ class WWBOTA(SSESpotProvider):
 
         # WWBOTA does support a special "Test" spot type, we need to avoid adding that.
         return spot if source_spot["type"] != "Test" else None
-
-    def can_submit_spot(self, sig):
-        return sig == "WWBOTA"
-
-    def submit_spot(self, spot, credentials):
-        # TODO: Implement. WWBOTA API docs cover this: https://api.wwbota.org/#tag/Spots/operation/create_spot_spots__post
-        raise NotImplementedError("WWBOTA upstream spot submission is not yet implemented")

spotproviders/wwff.py

@@ -14,7 +14,7 @@ class WWFF(HTTPSpotProvider):
     SPOTS_URL = "https://spots.wwff.co/static/spots.json"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("WWFF", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -38,11 +38,3 @@ class WWFF(HTTPSpotProvider):
             # that for us.
             new_spots.append(spot)
         return new_spots
-
-    def can_submit_spot(self, sig):
-        return sig == "WWFF"
-
-    def submit_spot(self, spot, credentials):
-        # TODO: Implement. Spotting to WWFF should be possible, need to look up the Spotline docs or copy approach from
-        # PoLo. Either way I think we need an API key for the app (but maybe not for the user?)
-        raise NotImplementedError("WWFF upstream spot submission is not yet implemented")

spotproviders/xota.py

@@ -1,6 +1,4 @@
-import csv
 import json
-import logging
 from datetime import datetime
 
 import pytz
@@ -12,36 +10,25 @@ from spotproviders.websocket_spot_provider import WebsocketSpotProvider
 
 class XOTA(WebsocketSpotProvider):
     """Spot provider for servers based on the "xOTA" software at https://github.com/nischu/xOTA/
-    The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG and a reference
+    The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG which we can
-    to a local CSV file with location information. This functionality is implemented for TOTA events, of which there are
+    then use for lookups. This functionality is implemented for Toilets on the Air events, of which there are
-    several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA and hence which server
+    several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA, which is why we also
-    supplied the data, which is why the CSV location lookup is here and not in sig_utils."""
+    provide a sig_ref_prefix in our config. This is applied to the reference ID, so e.g. "T-01" at C3 might become
+    "C3 T-01". This allows us to provide location lookups for TOTA at several conferences."""
 
     LOCATION_DATA = {}
     SIG = None
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, provider_config["url"])
+        name = provider_config["name"] if "name" in provider_config else "xOTA"
-        locations_csv = str(provider_config["locations-csv"]) if "locations-csv" in provider_config else None
+        super().__init__(name, provider_config, provider_config["url"])
         self.SIG = str(provider_config["sig"]) if "sig" in provider_config else None
-
+        self._sig_ref_prefix = str(provider_config["sig-ref-prefix"]) if "sig-ref-prefix" in provider_config else ""
-        # Load location data
-        if locations_csv:
-            try:
-                f = open(locations_csv)
-                csv_data = f.read()
-                dr = csv.DictReader(csv_data.splitlines())
-                for row in dr:
-                    self.LOCATION_DATA[row["ref"]] = {"lat": row["lat"], "lon": row["lon"]}
-            except:
-                logging.exception("Could not look up location data for XOTA source.")
 
     def _ws_message_to_spot(self, b):
         string = b.decode("utf-8")
         source_spot = json.loads(string)
-        ref_id = source_spot["reference"]["title"]
+        ref_id = self._sig_ref_prefix + " " + source_spot["reference"]["title"]
-        lat = float(self.LOCATION_DATA[ref_id]["lat"]) if ref_id in self.LOCATION_DATA else None
-        lon = float(self.LOCATION_DATA[ref_id]["lon"]) if ref_id in self.LOCATION_DATA else None
         spot = Spot(source=self.name,
                     source_id=source_spot["id"],
                     dx_call=source_spot["stationCallSign"].upper(),
@@ -49,10 +36,7 @@ class XOTA(WebsocketSpotProvider):
                     mode=source_spot["mode"].upper(),
                     sig=self.SIG,
                     sig_refs=[
-                        SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"], latitude=lat,
+                        SIGRef(id=ref_id, sig=self.SIG or "", url=source_spot["reference"]["website"])],
-                               longitude=lon)],
                     time=datetime.now(pytz.UTC).timestamp(),
-                    dx_latitude=lat,
-                    dx_longitude=lon,
                     qrt=source_spot["state"] != "active")
         return spot

spotproviders/zlota.py

@@ -15,7 +15,7 @@ class ZLOTA(HTTPSpotProvider):
     LIST_URL = "https://ontheair.nz/assets/assets.json"
 
     def __init__(self, provider_config):
-        super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
+        super().__init__("ZLOTA", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -41,10 +41,3 @@ class ZLOTA(HTTPSpotProvider):
 
             new_spots.append(spot)
         return new_spots
-
-    def can_submit_spot(self, sig):
-        return sig == "ZLOTA"
-
-    def submit_spot(self, spot, credentials):
-        # TODO: Implement. Spotting to ZLOTA is supported via POST, see https://ontheair.nz/api
-        raise NotImplementedError("ZLOTA upstream spot submission is not yet implemented")

static/apidocs/openapi.yml

@@ -15,14 +15,6 @@ info:
 
     ## Changelog
     
-    ### 2.0
-
-    * POST `/spot` now supports upstream submission to the spotting services associated with various SIGs.
-    * **Breaking change:** The "add spot" API has changed to enable this: instead of just posting the spot object itself as the JSON content of the POST, this has moved into a `spot` object within the structure. A new `handling` object alongside it contains the `submit_upstream`, `upstream_provider`, `upstream_credentials`, and `captcha_token` fields which control the server handling of the spot.
-    * POST `/spot` now supports Google reCaptcha and (if the site owner has set it up) now requires `captcha_token` in order to successfully submit. (This is used to lock down the submit function and prevent submission via Spothole by bots or third-party clients.)
-    * GET `/options` now returns `spot_submit_providers`, a map of SIG names to the names of providers that support upstream spot submission for that SIG. (This allows clients to present the user with options of where a new spot can be sent to.)
-    * **Breaking change:** A user's QRZ.com and HamQTH credentials are now supplied as request headers (`X-QRZ-Username`, `X-QRZ-Password`, `X-QRZ-Session-Key`, `X-HamQTH-Username`, `X-HamQTH-Password`, `X-HamQTH-Session-ID`) rather than query parameters, to keep credentials out of server logs.
-
     ### 1.4
     
     * Spots can now include a "propagation_mode" field, and the `/options` call enumerates the options that can have.
@@ -31,6 +23,7 @@ info:
     * Added `comment_names` to SIGs in the `/options`, to reflect how they might be referred to in spot comments where
       it differs from their `name`.
     * Added `propagation_mode` field to spots
+    * Added `sig_ref_data_providers` to status and removed `cleanup`
 
     ### 1.3
 
@@ -53,10 +46,10 @@ info:
   license:
     name: The Unlicense
     url: https://unlicense.org/#the-unlicense
-  version: 2.0
+  version: v1.4
 
 servers:
-  - url: https://spothole.app/api/v2
+  - url: https://spothole.app/api/v1
 
 tags:
   - name: Spots
@@ -337,8 +330,7 @@ paths:
           content:
             application/json:
               schema:
-                type: string
+                $ref: '#/components/schemas/ErrorResponse'
-                example: "Failed"
 
 
   /lookup/sigref:
@@ -366,8 +358,7 @@ paths:
           content:
             application/json:
               schema:
-                type: string
+                $ref: '#/components/schemas/ErrorResponse'
-                example: "Failed"
 
 
 
@@ -393,8 +384,7 @@ paths:
           content:
             application/json:
               schema:
-                type: string
+                $ref: '#/components/schemas/ErrorResponse'
-                example: "Failed"
 
 
   /spot:
@@ -403,53 +393,50 @@ paths:
         - Spots
       summary: Add a spot
       description: >
-        Supply a JSON object containing a `spot` sub-object (the spot data) and an optional `handling` sub-object
+        Supply a new spot object, which will be added to the system. Currently, this will not be
-        containing server-side instructions such as upstream submission). Check `spot_submit_providers` in the
+        reported up the chain to a cluster, POTA, SOTA etc. This may be introduced in a future version.
-        `/options` response to see which SIGs and providers support upstream submission. cURL example:
+        cURL example: `curl --request POST --header "Content-Type: application/json" --data
-        `curl --request POST --header \"Content-Type: application/json\" --data '{\"spot\":{\"dx_call\":\"M0TRT\",\"time\":1760019539,\"freq\":14200000,\"comment\":\"Test spot please ignore\",\"de_call\":\"M0TRT\"}}' https://spothole.app/api/v2/spot`"
+        '{"dx_call":"M0TRT","time":1760019539, "freq":14200000, "comment":"Test spot please ignore",
+        "de_call":"M0TRT"}' https://spothole.app/api/v1/spot`
       operationId: spot
       requestBody:
-        description: Object containing a "spot" sub-object with the spot data, and an optional "handling" sub-object with server-side instructions of what to do with it.
+        description: The JSON spot object
         required: true
         content:
           application/json:
             schema:
-              $ref: '#/components/schemas/SpotSubmission'
+              $ref: '#/components/schemas/Spot'
       responses:
-        '201':
+        '200':
           description: Success
           content:
             application/json:
               schema:
-                type: string
+                $ref: '#/components/schemas/OkResponse'
-                example: "OK"
         '415':
           description: Incorrect Content-Type
           content:
             application/json:
               schema:
-                type: string
+                $ref: '#/components/schemas/ErrorResponse'
-                example: "Failed"
         '422':
           description: Validation error
           content:
             application/json:
               schema:
-                type: string
+                $ref: '#/components/schemas/ErrorResponse'
-                example: "Failed"
         '500':
           description: Internal server error
           content:
             application/json:
               schema:
-                type: string
+                $ref: '#/components/schemas/ErrorResponse'
-                example: "Failed"
 
 components:
   parameters:
     QrzUsername:
-      name: X-QRZ-Username
+      name: qrz_username
-      in: header
+      in: query
       description: >
         QRZ.com username for online callsign lookup, which will enrich the returned spots and alerts
         with extra data. Requires a QRZ.com XML Subscriber (paid) account. Supply together with
@@ -457,14 +444,14 @@ components:
       schema:
         type: string
     QrzPassword:
-      name: X-QRZ-Password
+      name: qrz_password
-      in: header
+      in: query
       description: QRZ.com password. Supply together with `qrz_username`.
       schema:
         type: string
     QrzSessionKey:
-      name: X-QRZ-Session-Key
+      name: qrz_session_key
-      in: header
+      in: query
       description: >
         A pre-obtained QRZ.com XML session key, as an alternative to supplying `qrz_username` and
         `qrz_password`. See https://www.qrz.com/docs/xml/current_spec.html for details on how to
@@ -472,22 +459,22 @@ components:
       schema:
         type: string
     HamqthUsername:
-      name: X-HamQTH-Username
+      name: hamqth_username
-      in: header
+      in: query
       description: >
         HamQTH username for online callsign lookup, which will enrich the returned spots and alerts
         with extra data. Supply together with `hamqth_password`, or supply `hamqth_session_id` instead.
       schema:
         type: string
     HamqthPassword:
-      name: X-HamQTH-Password
+      name: hamqth_password
-      in: header
+      in: query
       description: HamQTH password. Supply together with `hamqth_username`.
       schema:
         type: string
     HamqthSessionId:
-      name: X-HamQTH-Session-ID
+      name: hamqth_session_id
-      in: header
+      in: query
       description: >
         A pre-obtained HamQTH session ID, as an alternative to supplying `hamqth_username` and
         `hamqth_password`. See https://www.hamqth.com/developers.php for details on how to retrieve
@@ -809,7 +796,7 @@ components:
         - HEMA
         - WCA
         - MOTA
-        - SiOTA
+        - SIOTA
         - ARLHS
         - ILLW
         - ZLOTA
@@ -1198,54 +1185,6 @@ components:
           $ref: "#/components/schemas/PropagationMode"
 
 
-    SpotSubmission:
-      description: >
-        Request body for POST /spot. Contains a "spot" sub-object with the spot data, and an optional
-        "handling" sub-object with server-side instructions consumed by Spothole.
-      type: object
-      required:
-        - spot
-      properties:
-        spot:
-          $ref: '#/components/schemas/Spot'
-        handling:
-          type: object
-          description: >
-            Optional server-side instructions for how to process this spot submission.
-          properties:
-            submit_upstream:
-              type: boolean
-              description: >
-                If true, forward the spot to an external upstream provider (e.g. POTA, SOTA) rather
-                than only adding it to this Spothole server. Requires `sig`, at least one `sig_refs`
-                entry, and `upstream_provider` to be set. Check `spot_submit_providers` in the
-                /options response to see which SIGs and providers support this.
-              default: false
-            upstream_provider:
-              type: string
-              description: >
-                Name of the upstream provider to submit the spot to, e.g. "POTA" or "SOTA". Must
-                match one of the provider names returned in `spot_submit_providers` for the chosen SIG.
-              example: POTA
-            upstream_credentials:
-              type: object
-              description: >
-                Provider-specific credentials required to authenticate the upstream submission.
-                The required keys depend on the provider. Credentials are used only for the upstream
-                call and are never stored by Spothole.
-              additionalProperties:
-                type: string
-              example:
-                user_id: "12345"
-                api_key: "abc123"
-            captcha_token:
-              type: string
-              description: >
-                A Google reCAPTCHA v2 response token. Required when submitting upstream if the
-                server has reCAPTCHA configured. Obtain the token by completing the reCAPTCHA
-                widget rendered on the Add Spot page.
-              example: "03AFY_a8Xq..."
-
     SpotStream:
       type: object
       description: A server-sent event containing a spot
@@ -1782,6 +1721,32 @@ components:
             is zero, the provider has never updated.
           example: 1759579508
 
+    SIGRefDataProviderStatus:
+      type: object
+      properties:
+        sig_name:
+          type: string
+          description: The name of the SIG.
+          example: WWFF
+        enabled:
+          type: boolean
+          description: Whether the provider is enabled or not.
+          example: true
+        status:
+          type: string
+          description: The status of the provider.
+          example: OK
+        last_updated:
+          type: number
+          description: >
+            The last time at which this provider received data, UTC seconds since UNIX epoch. If this
+            is zero, the provider has never updated.
+          example: 1759579508
+        reference_count:
+          type: number
+          description: The number of references fetched using this provider.
+          example: 1234
+
     SpotList:
       type: array
       items:
@@ -1792,6 +1757,14 @@ components:
       items:
         $ref: '#/components/schemas/Alert'
 
+    OkResponse:
+      type: string
+      example: "OK"
+
+    ErrorResponse:
+      type: string
+      example: "Failed"
+
     DxStats:
       type: object
       description: Spot counts keyed by DE continent
@@ -1841,17 +1814,6 @@ components:
           type: integer
           description: Number of alerts currently in the system.
           example: 123
-        "cleanup":
-          type: object
-          properties:
-            status:
-              type: string
-              description: The status of the cleanup thread
-              example: OK
-            last_ran:
-              type: number
-              description: The last time the cleanup operation ran, UTC seconds since UNIX epoch.
-              example: 1759579508
         "webserver":
           type: object
           properties:
@@ -1882,6 +1844,11 @@ components:
           description: An array of all the solar conditions providers.
           items:
             $ref: '#/components/schemas/SolarConditionsProviderStatus'
+        sig_ref_data_providers:
+          type: array
+          description: An array of all the SIG reference data providers.
+          items:
+            $ref: '#/components/schemas/SIGRefDataProviderStatus'
 
     Options:
       type: object
@@ -1929,7 +1896,7 @@ components:
           type: integer
           description: >
             The maximum age, in seconds, of any spot before it will be deleted by the system. When
-            querying the /api/v2/spots endpoint and providing a "max_age" or "since" parameter, there
+            querying the /api/v1/spots endpoint and providing a "max_age" or "since" parameter, there
             is no point providing a number larger than this, because the system drops all spots older
             than this.
           example: 3600
@@ -1939,20 +1906,6 @@ components:
             Whether the POST /spot call, to add spots to the server directly via its API, is permitted
             on this server.
           example: true
-        spot_submit_providers:
-          type: object
-          description: >
-            A map of SIG name to a list of provider names that support upstream spot submission for that SIG.
-            If a SIG appears as a key here, the POST /spot endpoint accepts `submit_upstream: true` for
-            spots with that SIG, and will forward the spot to one of the listed providers. Omitted if no
-            providers support upstream submission.
-          additionalProperties:
-            type: array
-            items:
-              type: string
-          example:
-            POTA: [ POTA ]
-            SOTA: [ SOTA, GMA, ParksNPeaks ]
 
     CallLookup:
       type: object

static/js/add-spot.js

@@ -1,35 +1,7 @@
-// Credentials schema per provider name. Defines the fields to collect and how to label them.
-const PROVIDER_CREDENTIAL_SCHEMAS = {
-    // todo Figure out SOTA authentication
-    // see e.g. https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAAccount.jsx
-    //          https://github.com/ham2k/app-polo/blob/main/src/store/apis/apiSOTA/apiSOTA.js
-    // Refresh token? Way to show user that they need to log in again because cached credentials aren't valid?
-    // todo type: text/password distinction on text boxes so API keys can be obscured
-    "SOTA": [
-        {key: "access_token", label: "SOTA Access Token", help: ""},
-        {key: "id_token", label: "SOTA ID Token", help: "TODO SOTA authentication to provide this..."}
-    ],
-    "ParksNPeaks": [
-        {key: "user_id", label: "Parks N Peaks User ID", help: ""},
-        {key: "api_key", label: "Parks N Peaks API Key", help: "Get your API key from your Parks N Peaks account."}
-    ],
-    "ZLOTA": [
-        {key: "user_id", label: "ZLOTA User ID", help: ""},
-        {key: "api_key", label: "ZLOTA User PIN", help: "Get your PIN from your ZLOTA account."}
-    ],
-    "Tiles": [
-        {
-            key: "offline_spot_gateway_pin",
-            label: "Offline Spot Gateway PIN",
-            help: "Get your PIN from your Tiles on the Air account profile."
-        }
-    ]
-};
-
 // Load server options. Once a successful callback is made from this, we can populate the choice boxes in the form and load
 // any saved values from local storage.
 function loadOptions() {
-    $.getJSON('/api/v2/options', function (jsonData) {
+    $.getJSON('/api/v1/options', function (jsonData) {
         // Store options
         options = jsonData;
 
@@ -49,144 +21,11 @@ function loadOptions() {
             }));
         });
 
-        // Load reCAPTCHA if a site key is configured (key is inlined into page by server)
-        if (window._recaptchaSiteKey) {
-            loadRecaptcha(window._recaptchaSiteKey);
-        }
-
         // Load settings from settings storage now all the controls are available
         loadSettings();
-
-        // Update the upstream area for any pre-selected SIG
-        updateUpstreamArea();
     });
 }
 
-// Load and inject the reCAPTCHA script
-function loadRecaptcha(siteKey) {
-    window._recaptchaSiteKey = siteKey;
-    if (!document.getElementById('recaptcha-script')) {
-        const script = document.createElement('script');
-        script.id = 'recaptcha-script';
-        script.src = 'https://www.google.com/recaptcha/api.js?render=explicit&onload=renderRecaptcha';
-        script.async = true;
-        script.defer = true;
-        document.head.appendChild(script);
-    }
-    $("#recaptcha-area").show();
-}
-
-// Called by reCAPTCHA after its script loads
-function renderRecaptcha() {
-    window._recaptchaWidgetId = grecaptcha.render('recaptcha-widget', {
-        sitekey: window._recaptchaSiteKey,
-        size: 'normal'
-    });
-}
-
-// Update the "Send spot to..." area based on the currently selected SIG
-function updateUpstreamArea() {
-    if (!window._allowUpstreamSpotting || !options || !options["spot_submit_providers"]) {
-        $("#upstream-area").hide();
-        return;
-    }
-
-    const sig = $("#sig").val();
-    const providers = (sig && options["spot_submit_providers"][sig]) ? options["spot_submit_providers"][sig] : [];
-
-    if (providers.length === 0) {
-        $("#upstream-area").hide();
-        return;
-    }
-
-    $("#upstream-area").show();
-
-    // Update the provider selector
-    $("#upstream-provider-select").empty();
-    $.each(providers, function (i, name) {
-        $("#upstream-provider-select").append($('<option>', {value: name, text: name}));
-    });
-
-    if (providers.length > 1) {
-        $("#upstream-provider-label").text("upstream spot sources:");
-        $("#upstream-provider-select-col").show();
-    } else {
-        $("#upstream-provider-label").text(providers[0]);
-        $("#upstream-provider-select-col").hide();
-    }
-
-    // Show the credentials button if this provider has an authentication mechanism and we need input from the user
-    updateCredentialsButton();
-}
-
-// Update the credentials button visibility based on selected provider
-function updateCredentialsButton() {
-    const providerName = getSelectedUpstreamProvider();
-    if (providerName && PROVIDER_CREDENTIAL_SCHEMAS[providerName]) {
-        $("#upstream-credentials-btn").show();
-    } else {
-        $("#upstream-credentials-btn").hide();
-    }
-}
-
-// Get the currently selected upstream provider name
-function getSelectedUpstreamProvider() {
-    const providers = (options && options["spot_submit_providers"] && $("#sig").val())
-        ? (options["spot_submit_providers"][$("#sig").val()] || [])
-        : [];
-    if (providers.length === 0) return null;
-    if (providers.length === 1) return providers[0];
-    return $("#upstream-provider-select").val();
-}
-
-// Show the credentials modal for the currently selected upstream provider
-function showCredentialsModal() {
-    const providerName = getSelectedUpstreamProvider();
-    if (!providerName || !PROVIDER_CREDENTIAL_SCHEMAS[providerName]) return;
-
-    const schema = PROVIDER_CREDENTIAL_SCHEMAS[providerName];
-    const stored = loadCredentials(providerName);
-
-    $("#credentials-provider-name").text(providerName);
-    $("#credentials-fields").empty();
-
-    $.each(schema, function (i, field) {
-        const val = stored[field.key] || "";
-        let html = '<div class="mb-3">';
-        html += '<label for="cred-' + field.key + '" class="form-label">' + field.label + '</label>';
-        html += '<input type="text" class="form-control" id="cred-' + field.key + '" value="' + $('<div>').text(val).html() + '">';
-        if (field.help) {
-            html += '<div class="form-text">' + field.help + '</div>';
-        }
-        html += '</div>';
-        $("#credentials-fields").append(html);
-    });
-
-    // Store provider name for saveCredentials()
-    $("#credentials-modal").data("provider", providerName);
-    new bootstrap.Modal(document.getElementById('credentials-modal')).show();
-}
-
-// Save credentials from the modal to local storage
-function saveCredentials() {
-    const providerName = $("#credentials-modal").data("provider");
-    if (!providerName || !PROVIDER_CREDENTIAL_SCHEMAS[providerName]) return;
-
-    const schema = PROVIDER_CREDENTIAL_SCHEMAS[providerName];
-    const creds = {};
-    $.each(schema, function (i, field) {
-        creds[field.key] = $("#cred-" + field.key).val();
-    });
-    localStorage.setItem("upstream-credentials-" + providerName, JSON.stringify(creds));
-    bootstrap.Modal.getInstance(document.getElementById('credentials-modal')).hide();
-}
-
-// Load credentials for a provider from local storage
-function loadCredentials(providerName) {
-    const stored = localStorage.getItem("upstream-credentials-" + providerName);
-    return stored ? JSON.parse(stored) : {};
-}
-
 // Method called to add a spot to the server
 function addSpot() {
     try {
@@ -203,65 +42,57 @@ function addSpot() {
         const comment = $("#comment").val();
         const de = $("#de-call").val().toUpperCase();
 
-        // Prepare the spot object for the server
         const spot = {};
+        if (dx !== "") {
             spot["dx_call"] = dx;
+        } else {
+            showAddSpotError("A DX callsign is required in order to spot.");
+            return;
+        }
+        if (freqStr !== "") {
             spot["freq"] = parseFloat(freqStr) * 1000;
-        if (mode !== "") spot["mode"] = mode;
+        } else {
-        if (sig !== "") spot["sig"] = sig;
+            showAddSpotError("A frequency is required in order to spot.");
-        if (sigRef !== "") spot["sig_refs"] = [{sig: sig, id: sigRef}];
+            return;
-        if (dxGrid !== "") spot["dx_grid"] = dxGrid;
+        }
-        if (comment !== "") spot["comment"] = comment;
+        if (mode !== "") {
+            spot["mode"] = mode;
+        }
+        if (sig !== "") {
+            spot["sig"] = sig;
+        }
+        if (sigRef !== "") {
+            spot["sig_refs"] = [{sig: sig, id: sigRef}];
+        }
+        if (dxGrid !== "") {
+            spot["dx_grid"] = dxGrid;
+        }
+        if (comment !== "") {
+            spot["comment"] = comment;
+        }
+        if (de !== "") {
             spot["de_call"] = de;
+        } else {
+            showAddSpotError("A spotter callsign is required in order to spot.");
+            return;
+        }
         spot["time"] = moment.utc().valueOf() / 1000.0;
 
-        // Prepare "handling" structure to tell the server what to do with this spot
+        $.ajax("/api/v1/spot", {
-        const handling = {};
+            data: JSON.stringify(spot),
-
-        // Add CAPTCHA token if reCAPTCHA is loaded
-        if (window._recaptchaWidgetId !== undefined) {
-            handling["captcha_token"] = grecaptcha.getResponse(window._recaptchaWidgetId);
-        }
-
-        // Upstream submission
-        const submitUpstream = $("#submit-upstream").is(":checked");
-        const upstreamProviderName = getSelectedUpstreamProvider();
-        if (submitUpstream && upstreamProviderName) {
-            handling["submit_upstream"] = true;
-            handling["upstream_provider"] = upstreamProviderName;
-            handling["upstream_credentials"] = loadCredentials(upstreamProviderName);
-        }
-
-        $.ajax("/api/v2/spot", {
-            data: JSON.stringify({spot, handling}),
             contentType: 'application/json',
             type: 'POST',
             timeout: 10000,
-            success: async function (result) {
+            success: async function () {
-                // Reset CAPTCHA for next use
-                if (window._recaptchaWidgetId !== undefined) {
-                    grecaptcha.reset(window._recaptchaWidgetId);
-                }
-                if (result && result.startsWith && result.startsWith("Warning")) {
-                    $("#result-good").html("<div class='alert alert-warning fade show mb-0 mt-4' role='alert'><i class='fa-solid fa-triangle-exclamation'></i> " + result + " Returning you to the spots list...</div>");
-                } else {
                 $("#result-good").html("<div class='alert alert-success fade show mb-0 mt-4' role='alert'><i class='fa-solid fa-check'></i> Spot submitted. Returning you to the spots list...</div>");
-                }
                 $("#result-bad").html("");
                 setTimeout(() => {
                     $("#result-good").hide();
                     window.location.replace("/");
-                }, 2000);
+                }, 1000);
             },
             error: function (result) {
-                if (window._recaptchaWidgetId !== undefined) {
-                    grecaptcha.reset(window._recaptchaWidgetId);
-                }
-                if (result.responseText) {
                 showAddSpotError(result.responseText.slice(1, -1));
-                } else {
-                    showAddSpotError("The server did not return a response.");
-                }
             }
         });
     } catch (error) {
@@ -290,18 +121,20 @@ $("#mode").change(function () {
     $(this).val($(this).val().trim().toUpperCase());
 });
 
-// Update upstream area and credentials button when SIG changes
+// Display the intro box, unless the user has already dismissed it once.
-$("#sig").change(function () {
+function displayIntroBox() {
-    updateUpstreamArea();
+    if (localStorage.getItem("add-spot-intro-box-dismissed") == null) {
-});
+        $("#add-spot-intro-box").show();
-
+    }
-// Update credentials button when provider selector changes
+    $("#add-spot-intro-box-dismiss").click(function () {
-$("#upstream-provider-select").change(function () {
+        localStorage.setItem("add-spot-intro-box-dismissed", true);
-    updateCredentialsButton();
     });
+}
 
 // Startup
 $(document).ready(function () {
     // Load options
     loadOptions();
+    // Display intro box
+    displayIntroBox();
 });

static/js/alerts.js

@@ -8,7 +8,7 @@ let alerts = [];
 
 // Load alerts and populate the table.
 function loadAlerts() {
-    $.ajax({url: '/api/v2/alerts' + buildQueryString(), dataType: 'json', headers: getCredentialHeaders(), success: function (jsonData) {
+    $.getJSON('/api/v1/alerts' + buildQueryString(false), function (jsonData) {
         // Store last updated time
         lastUpdateTime = moment.utc();
         updateRefreshDisplay();
@@ -16,11 +16,11 @@ function loadAlerts() {
         alerts = jsonData;
         // Update table
         updateTable();
-    }});
+    });
 }
 
 // Build a query string for the API, based on the filters that the user has selected.
-function buildQueryString() {
+function buildQueryString(includeCredentials) {
     let str = "?";
     ["dx_continent", "source"].forEach(fn => {
         if (!allFilterOptionsSelected(fn)) {
@@ -35,6 +35,9 @@ function buildQueryString() {
     if ($("#dxpeditions_skip_max_duration_check")[0].checked) {
         str = str + "&dxpeditions_skip_max_duration_check=true";
     }
+    if (includeCredentials) {
+        str = str + getCredentialQueryString();
+    }
     return str;
 }
 
@@ -279,7 +282,7 @@ function addAlertRowsToTable(tbody, alerts) {
 
 // Load server options. Once a successful callback is made from this, we then query alerts.
 function loadOptions() {
-    $.getJSON('/api/v2/options', function (jsonData) {
+    $.getJSON('/api/v1/options', function (jsonData) {
         // Store options
         options = jsonData;
 

static/js/bands.js

@@ -20,7 +20,7 @@ function loadSpots() {
     if (evtSource != null) {
         evtSource.close();
     }
-    $.ajax({url: '/api/v2/spots' + buildQueryString(), dataType: 'json', headers: getCredentialHeaders(), success: function (jsonData) {
+    $.getJSON('/api/v1/spots' + buildQueryString(), function (jsonData) {
         // Store data
         spots = jsonData;
         // Update bands display
@@ -35,7 +35,7 @@ function startSSEConnection() {
     if (evtSource != null) {
         evtSource.close();
     }
-    evtSource = new EventSource('/api/v2/spots/stream' + buildQueryString());
+    evtSource = new EventSource('/api/v1/spots/stream' + buildQueryString());
 
     evtSource.onmessage = function (event) {
         const newSpot = JSON.parse(event.data);
@@ -273,7 +273,7 @@ function removeDuplicatesForBandPanel(spotList) {
 // Load server options. Once a successful callback is made from this, we then query spots and set up the timer to query
 // spots repeatedly.
 function loadOptions() {
-    $.getJSON('/api/v2/options', function (jsonData) {
+    $.getJSON('/api/v1/options', function (jsonData) {
         // Store options
         options = jsonData;
 

static/js/common.js

@@ -250,23 +250,23 @@ function closeDataPanel() {
     closePanel("#data-area");
 }
 
-// Build a headers object containing any QRZ.com / HamQTH credentials the user has supplied,
+// Build a query string fragment containing any QRZ.com / HamQTH credentials the user has supplied,
 // provided the corresponding "enabled" checkbox is ticked.
-function getCredentialHeaders() {
+function getCredentialQueryString() {
-    const headers = {};
+    let str = "";
     if ($("#qrz-enabled")[0] && $("#qrz-enabled")[0].checked) {
         const qrzUsername = $("#qrz-username").val();
         const qrzPassword = $("#qrz-password").val();
-        if (qrzUsername) headers["X-QRZ-Username"] = qrzUsername;
+        if (qrzUsername) str += "&qrz_username=" + encodeURIComponent(qrzUsername);
-        if (qrzPassword) headers["X-QRZ-Password"] = qrzPassword;
+        if (qrzPassword) str += "&qrz_password=" + encodeURIComponent(qrzPassword);
     }
     if ($("#hamqth-enabled")[0] && $("#hamqth-enabled")[0].checked) {
         const hamqthUsername = $("#hamqth-username").val();
         const hamqthPassword = $("#hamqth-password").val();
-        if (hamqthUsername) headers["X-HamQTH-Username"] = hamqthUsername;
+        if (hamqthUsername) str += "&hamqth_username=" + encodeURIComponent(hamqthUsername);
-        if (hamqthPassword) headers["X-HamQTH-Password"] = hamqthPassword;
+        if (hamqthPassword) str += "&hamqth_password=" + encodeURIComponent(hamqthPassword);
     }
-    return headers;
+    return str;
 }
 
 

static/js/conditions.js

@@ -10,7 +10,7 @@ let ionosondeChart = null;
 
 // Load solar conditions
 function loadSolarConditions() {
-    $.getJSON('/api/v2/solar', function (jsonData) {
+    $.getJSON('/api/v1/solar', function (jsonData) {
 
         // HF
 
@@ -539,7 +539,7 @@ function renderIonosondeData() {
             ctx.strokeStyle = gridColor;
             ctx.lineWidth = 1;
             ctx.setLineDash([]);
-            // Add an extra horizontal line for 30MHz, which should correspond to the top of the chart and avoid having
+            // Add an extra vertical line for 30MHz, which should correspond to the top of the chart and avoid having
             // no top "border" gridline
             const y30 = scales.y.getPixelForValue(30);
             if (y30 >= chartArea.top && y30 <= chartArea.bottom) {
@@ -660,7 +660,7 @@ function dxStatsContientChanged() {
 
 // Fetch DX stats from the API and render
 function loadDxStats() {
-    $.getJSON('/api/v2/dxstats', function (jsonData) {
+    $.getJSON('/api/v1/dxstats', function (jsonData) {
         dxStatsData = jsonData;
         renderDxStats();
     });

static/js/map.js

@@ -45,7 +45,7 @@ function loadSpots() {
     // 2) (If we have credentials) reload with them, replacing what's already there,
     // 3) Subscribe to the SSE endpoint (with credentials if we have them) so that updates come with augmented
     // data if they can.
-    $.ajax({url: '/api/v2/spots' + buildQueryString(), dataType: 'json', headers: getCredentialHeaders(), success: function (jsonData) {
+    $.getJSON('/api/v1/spots' + buildQueryString(false), function (jsonData) {
         // Store data
         spots = jsonData;
         // Update map
@@ -68,7 +68,7 @@ function loadSpots() {
             // now
             startSSEConnection();
         }
-    }});
+    });
 }
 
 // Start the SSE connection to receive new spots as they arrive
@@ -163,7 +163,7 @@ function removeSpotFromMap(key) {
 }
 
 // Build a query string for the API, based on the filters that the user has selected.
-function buildQueryString() {
+function buildQueryString(includeCredentials) {
     let str = "?";
     ["dx_continent", "de_continent", "mode", "source", "band", "sig"].forEach(fn => {
         if (!allFilterOptionsSelected(fn)) {
@@ -173,6 +173,9 @@ function buildQueryString() {
     str = str + "max_age=" + $("#max-spot-age option:selected").val();
     // Additional filters for the map view: No dupes, no QRT, only spots with good locations
     str = str + "&dedupe=true&allow_qrt=false";
+    if (includeCredentials) {
+        str = str + getCredentialQueryString();
+    }
     return str;
 }
 
@@ -301,7 +304,7 @@ function getTooltipText(s) {
 // Load server options. Once a successful callback is made from this, we then query spots and set up the timer to query
 // spots repeatedly.
 function loadOptions() {
-    $.getJSON('/api/v2/options', function (jsonData) {
+    $.getJSON('/api/v1/options', function (jsonData) {
         // Store options
         options = jsonData;
 

static/js/spots.js

@@ -20,7 +20,7 @@ function loadSpots() {
     }
 
     // Make the new query
-    $.ajax({url: '/api/v2/spots' + buildQueryString(), dataType: 'json', headers: getCredentialHeaders(), success: function (jsonData) {
+    $.getJSON('/api/v1/spots' + buildQueryString(false), function (jsonData) {
         // Store data
         spots = jsonData;
         // Update table
@@ -30,7 +30,7 @@ function loadSpots() {
         if (run) {
             startSSEConnection();
         }
-    }});
+    });
 }
 
 // Start an SSE connection (closing an existing one if it exists). This will then be used to add to the table on the
@@ -39,7 +39,7 @@ function startSSEConnection() {
     if (evtSource != null) {
         evtSource.close();
     }
-    evtSource = new EventSource('/api/v2/spots/stream' + buildQueryString());
+    evtSource = new EventSource('/api/v1/spots/stream' + buildQueryString(true));
 
     evtSource.onmessage = function (event) {
         // Get the new spot
@@ -86,7 +86,7 @@ function startSSEConnection() {
 }
 
 // Build a query string for the API, based on the filters that the user has selected.
-function buildQueryString() {
+function buildQueryString(includeCredentials) {
     let str = "?";
     ["dx_continent", "de_continent", "mode", "source", "band", "sig"].forEach(fn => {
         if (!allFilterOptionsSelected(fn)) {
@@ -97,6 +97,9 @@ function buildQueryString() {
     if ($("#search").val() !== "") {
         str = str + "&text_includes=" + encodeURIComponent($("#search").val());
     }
+    if (includeCredentials) {
+        str = str + getCredentialQueryString();
+    }
     return str;
 }
 
@@ -415,7 +418,7 @@ function createNewTableRowsForSpot(s, highlightNew) {
 // Load server options. Once a successful callback is made from this, we then query spots and set up the timer to query
 // spots repeatedly.
 function loadOptions() {
-    $.getJSON('/api/v2/options', function (jsonData) {
+    $.getJSON('/api/v1/options', function (jsonData) {
         // Store options
         options = jsonData;
 

static/js/status.js

@@ -1,6 +1,6 @@
 // Load server status
 function loadStatus() {
-    $.getJSON('/api/v2/status', function (jsonData) {
+    $.getJSON('/api/v1/status', function (jsonData) {
         $("#software-version").text(jsonData["software-version"]);
         $("#server-owner-callsign").text(jsonData["server-owner-callsign"]);
         $("#up-since").text(moment().subtract(jsonData["uptime"], 'seconds').fromNow());
@@ -12,9 +12,6 @@ function loadStatus() {
         $("#web-server-last-api").text(moment.unix(jsonData["webserver"]["last_api_access"]).utc().fromNow());
         $("#web-server-last-page").text(moment.unix(jsonData["webserver"]["last_page_access"]).utc().fromNow());
 
-        $("#cleanup-status").text(jsonData["cleanup"]["status"]);
-        $("#cleanup-last-ran").text(moment.unix(jsonData["cleanup"]["last_ran"]).utc().fromNow());
-
         jsonData["spot_providers"].forEach(p => {
             $("#spot-providers-status-container").append(`
                 <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
@@ -42,6 +39,16 @@ function loadStatus() {
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
                 </div>`);
         });
+
+        jsonData["sig_ref_data_providers"].forEach(p => {
+            $("#sig-ref-data-providers-status-container").append(`
+                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+                    <div class="col"><strong>${p["sig_name"]}</strong></div>
+                    <div class="col">Status: ${p["status"]}</div>
+                    <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
+                    <div class="col">References: ${(p["enabled"] && p["reference_count"] > 0) ? p["reference_count"] : "N/A"}</div>
+                </div>`);
+        });
     });
 }
 

static/js/ui-ham.js

@@ -296,16 +296,6 @@ function setBandColorScheme(scheme) {
     return ret;
 }
 
-// Get the list of known bands
-function getKnownBands() {
-    return Array.from(Object.keys(BAND_COLOR_SCHEMES[bandColorScheme]));
-}
-
-// Get the list of available band colour schemes
-function getAvailableBandColorSchemes() {
-    return Array.from(Object.keys(BAND_COLOR_SCHEMES));
-}
-
 // Band name to colour (in the current colour scheme). If the band is unknown, black will be returned.
 function bandToColor(band) {
     let col = (band != null) ? BAND_COLOR_SCHEMES[bandColorScheme][band] : null;
@@ -324,22 +314,6 @@ function bandToContrastColor(band) {
     return (lum > 128) ? "#000000" : "#ffffff";
 }
 
-const MODE_TYPE_COLOR_SCHEMES = {
-    "CW": "red",
-    "PHONE": "green",
-    "DATA": "blue"
-}
-
-// Mode type (CW, PHONE, DATA) to colour. If the mode type is unknown, black will be returned.
-function modeTypeToColor(modeType) {
-    let col = (modeType != null) ? MODE_TYPE_COLOR_SCHEMES[modeType.toUpperCase()] : null;
-    if (col) {
-        return col;
-    } else {
-        return "#000000";
-    }
-}
-
 const SIG_ICONS = {
     "POTA": "fa-tree",
     "SOTA": "fa-mountain-sun",
@@ -366,31 +340,6 @@ const SIG_ICONS = {
     "Toilets": "fa-toilet"
 }
 
-const SIG_NAMES = {
-    "POTA": "Parks on the Air",
-    "SOTA": "Summits on the Air",
-    "WWFF": "Worldwide Flora & Fauna",
-    "GMA": "Global Mountain Activity",
-    "WWBOTA": "Bunkers on the Air",
-    "HEMA": "Humps Excluding Marilyns Award",
-    "IOTA": "Islands on the Air",
-    "MOTA": "Mills on the Air",
-    "ARLHS": "Amateur Radio Lighthouse Society",
-    "ILLW": "International Lighthouse Lightship Weekend",
-    "SIOTA": "Silos on the Air",
-    "WCA": "World Castles Award",
-    "ZLOTA": "New Zealand on the Air",
-    "WOTA": "Wainwrights on the Air",
-    "BOTA": "Beaches on the Air",
-    "KRMNPA": "Keith Roget Memorial National Parks Award",
-    "LLOTA": "Lagos y Lagunas on the Air",
-    "WWTOTA": "Towers on the Air",
-    "WAB": "Worked All Britain",
-    "WAI": "Worked All Ireland",
-    "Tiles": "Tiles on the Air",
-    "TOTA": "Toilets on the Air"
-}
-
 // Get the Font Awesome icon for a given SIG. If the SIG is unknown, the provided default symbol will be returned
 function sigToIcon(sig, defaultIcon) {
     let col = (sig != null) ? SIG_ICONS[sig] : null;
@@ -405,35 +354,3 @@ function sigToIcon(sig, defaultIcon) {
         }
     }
 }
-
-// Get the full name for a given SIG abbreviation. If the SIG is unknown, an empty string will be returned.
-function sigToName(sig) {
-    let col = (sig != null) ? SIG_NAMES[sig] : null;
-    if (col) {
-        return col;
-    } else {
-        let col = (sig != null) ? SIG_NAMES[sig.toUpperCase()] : null;
-        if (col) {
-            return col;
-        } else {
-            return "";
-        }
-    }
-}
-
-// Get the list of known SIGs
-function getKnownSIGs() {
-    return Array.from(Object.keys(SIG_ICONS));
-}
-
-// Format a Maidenhead grid with alternating alphabetic blocks in lower case
-function formatGrid(grid) {
-    grid = grid.toUpperCase();
-    if (grid.length >= 6) {
-        grid = grid.substring(0, 4) + grid.substring(4, 6).toLowerCase() + grid.substring(6);
-    }
-    if (grid.length >= 12) {
-        grid = grid.substring(0, 10) + grid.substring(10, 12).toLowerCase() + grid.substring(14);
-    }
-    return grid;
-}

templates/about.html

@@ -117,7 +117,6 @@
         <li>Some SIGs, such as Worked all Britain (WAB), don't have their own spotting site and can <em>only</em> be
             identified through comments on spots retrieved from other sources.
         </li>
-        <li>SIGs have well-defined names, whereas the server owner may name the sources as they see fit.</li>
     </ol>
     <p>Spothole's web interface exists not just for the end user, but also as a reference implementation for the API, so
         I have chosen to demonstrate both methods of filtering.</p>

templates/add_spot.html

@@ -24,14 +24,14 @@
             </div>
         </div>
         <div class="card-body">
-            <form class="row g-3" onsubmit="return addSpot();">
+            <form class="row g-3">
                 <div class="col-auto">
                     <label for="dx-call" class="form-label">DX Call *</label>
-                    <input type="text" class="form-control input-narrow" id="dx-call" placeholder="N0CALL" required>
+                    <input type="text" class="form-control input-narrow" id="dx-call" placeholder="N0CALL">
                 </div>
                 <div class="col-auto">
                     <label for="freq" class="form-label">Frequency (kHz) *</label>
-                    <input type="text" class="form-control input-narrow" id="freq" placeholder="e.g. 14100" required>
+                    <input type="text" class="form-control input-narrow" id="freq" placeholder="e.g. 14100">
                 </div>
                 <div class="col-auto">
                     <label for="mode" class="form-label">Mode</label>
@@ -60,10 +60,10 @@
                 <div class="col-auto">
                     <label for="de-call" class="form-label">Your Call *</label>
                     <input type="text" class="form-control storeable-text input-narrow" id="de-call"
-                           placeholder="N0CALL" required>
+                           placeholder="N0CALL">
                 </div>
                 <div class="col-auto">
-                    <button type="submit" class="btn btn-primary mt-2em">Spot</button>
+                    <button type="button" class="btn btn-primary mt-2em" onclick="addSpot();">Spot</button>
                 </div>
             </form>
 
@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/static/js/add-spot.js?v=1785082221"></script>
+<script src="/static/js/add-spot.js?v=1785434214"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>