Fix a bug where solar conditions and status data were not properly persisted to disk between restarts. Release v2.0.5

# Conflicts:
#	core/constants.py
#	pyproject.toml
#	templates/add_spot.html
#	templates/alerts.html
#	templates/bands.html
#	templates/base.html
#	templates/conditions.html
#	templates/map.html
#	templates/spots.html
#	templates/status.html

.idea/modules.xml

@@ -2,7 +2,7 @@
 <project version="4">
   <component name="ProjectModuleManager">
     <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/Spothole.iml" filepath="$PROJECT_DIR$/.idea/Spothole.iml" />
+      <module fileurl="file://$PROJECT_DIR$/.idea/spothole.iml" filepath="$PROJECT_DIR$/.idea/spothole.iml" />
     </modules>
   </component>
 </project>

.idea/runConfigurations/Run.xml

@@ -1,6 +1,6 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="Run" type="PythonConfigurationType" factoryName="Python">
-    <module name="Spothole" />
+    <module name="spothole" />
     <option name="ENV_FILES" value="" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />

config-example.yml

@@ -67,8 +67,8 @@ spot_providers:
 
   - class: "DXCluster"
     # Clusters have a "name" property in case you want to say which cluster node the data is from, or connect to
-    # several clusters and name them separately.
-    name: "HRD Cluster"
+    # several clusters and name them separately. The name will be used as the "source" field for the spots received.
+    name: "Cluster"
     enabled: true
     host: "hrd.wa9pie.net"
     port: 8000
@@ -84,11 +84,11 @@ spot_providers:
     allow_rbn_spots: false
 
   - class: "DXCluster"
-    name: "W3LPL Cluster"
+    name: "Cluster"
     enabled: false
     host: "w3lpl.net"
     port: 7373
-    login_prompt: "Please enter your call:"
+    login_prompt: "login:"
     login_callsign: "N0CALL-99"
     allow_rbn_spots: false
 
@@ -345,4 +345,6 @@ web_ui_options:
   # (home) page, although being arbitrary HTML you can also use a div with absolute, relative, float placement etc. This
   # is designed for a "donate/support the server" type button, though you are free to do whatever you want with it.
   # As the server owner you are responsible for the safe usage of this option!
-  support_button_html: ""
+  support_button_html: ""
+  # CartoDB API key. Required to use the CartoDB layers without watermark. Get one from https://carto.com/basemaps/apikey/.
+  cartodb_api_key: ""

core/call_lookup_helper.py

@@ -1,4 +1,5 @@
-from core.constants import CALL_ONLY_PATTERN
+import re
+
 from core.data_providers import DATA_PROVIDERS
 from data.callsign import Callsign
 
@@ -11,7 +12,7 @@ def get_call_info(callsign, lookup_credentials):
     callsign_data = Callsign(call=callsign)
 
     # First check our input looks like a real callsign
-    if callsign and CALL_ONLY_PATTERN.match(callsign):
+    if callsign and re.match(r"^[A-Za-z0-9/\-]*$", callsign):
         # Sort callsign providers by priority order, so we query the highest priority (lowest numbers) first, and only
         # query other providers for data we are missing as we go along.
         for p in sorted(DATA_PROVIDERS.callsign_data_providers, key=lambda p2: p2.priority):

core/constants.py

@@ -1,21 +1,14 @@
-import re
-
 from core.config import SERVER_OWNER_CALLSIGN
 from data.band import Band
 from data.sig import SIG
 
 # General software
-SOFTWARE_VERSION = "2.0-pre"
+SOFTWARE_VERSION = "2.0.5"
 
 # HTTP headers used for spot providers that use HTTP
 HTTP_HEADERS = {"User-Agent": f"Spothole v{SOFTWARE_VERSION} (operated by {SERVER_OWNER_CALLSIGN})"}
 HAMQTH_PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}".replace(" ", "_")
 
-# Generally useful regexes
-CALL_REGEX = r"[A-Za-z0-9/\-]+"
-CALL_PATTERN = re.compile(CALL_REGEX)
-CALL_ONLY_PATTERN = re.compile(rf"^{CALL_REGEX}$")
-
 # Special Interest Groups
 SIGS = [
     SIG(
@@ -60,6 +53,12 @@ SIGS = [
         description="Islands on the Air",
         ref_regex=r"[A-Z]{2}\-\d{3}",
     ),
+    SIG(
+        name="GMA Islands",
+        comment_names=[],
+        description="Global Mountain Activity - Islands",
+        ref_regex=r"(([A-Z]{2}\-\d{3})|([A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}))",
+    ),
     SIG(
         name="MOTA",
         comment_names=["MOTA"],

core/data_store.py

@@ -6,6 +6,7 @@ import diskcache
 
 from core.config import MAX_ALERT_AGE, MAX_SPOT_AGE
 from core.live_data_cache import LiveDataCache
+from core.single_object_data_cache import SingleObjectDataCache
 from data.solar_conditions import SolarConditions
 
 logger = logging.getLogger(__name__)
@@ -34,10 +35,8 @@ class DataStore:
         self.dxcc_data = None
         self.dxcc_lookup_by_call_regex = []
         self.sigrefs = None
-        self.status_data = {}
-        self._status = None
-        self.solar_conditions = {}
-        self._solar = None
+        self.status = None
+        self.solar_conditions = None
         # ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be
         # caches, they can just be straight objects
         self.cq_zone_data = None
@@ -46,16 +45,11 @@ class DataStore:
     def setup(self):
         Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
 
-        # Standard disk cache for solar data and status data, but each cache contains only a single object which we
-        # expose to the wider application
-        self._solar = diskcache.Cache(f"{CACHE_DIR}solar")
-        if "solar_conditions" not in self._solar:
-            self._solar.add("solar_conditions", SolarConditions())
-        self.solar_conditions = self._solar.get("solar_conditions")
-        self._status = diskcache.Cache(f"{CACHE_DIR}status")
-        if "status_data" not in self._status:
-            self._status.add("status_data", {})
-        self.status_data = self._status.get("status_data")
+        # For solar data and status data, we use a wrapper around disk cache where each cache contains only a single
+        # object exposed to the wider application, and provides a store() method for callers to notify diskcache that
+        # the object has changed and needs to be re-cached.
+        self.solar_conditions = SingleObjectDataCache(f"{CACHE_DIR}solar", SolarConditions())
+        self.status = SingleObjectDataCache(f"{CACHE_DIR}status", {})
 
         # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
         # a regular basis but there's no need for a TTL since old data is better than no data.
@@ -116,13 +110,10 @@ class DataStore:
             self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))
 
     def close(self):
-        self.spots.save_snapshot()
-        self.alerts.save_snapshot()
-
         self.spots.close()
         self.alerts.close()
-        self._solar.close()
-        self._status.close()
+        self.solar.close()
+        self.status.close()
         self.dxcc_data.close()
         self.sigrefs.close()
         self.callsign_data_countryfiles.close()

core/sig_lookup_helper.py

@@ -73,6 +73,20 @@ def get_sig_ref_info(sig, ref_id):
                 sig_ref.url = f"https://www.beachesontheair.com/beaches/{sig_ref.name.lower().replace(' ', '-')}"
             return sig_ref
 
+        elif sig.upper() == "GMA Islands":
+            # GMA Islands is a bit of a mess of GMA and IOTA references. Try looking them both up and see what returns
+            # the best result.
+            iota_lookup = get_sig_ref_info("IOTA", ref_id)
+            gma_lookup = get_sig_ref_info("GMA", ref_id)
+            for key, value in iota_lookup.__dict__.items():
+                if value is not None and sig_ref.__dict__.get(key) is None:
+                    sig_ref.__dict__[key] = value
+            for key, value in gma_lookup.__dict__.items():
+                if value is not None and sig_ref.__dict__.get(key) is None:
+                    sig_ref.__dict__[key] = value
+            sig_ref.ref_type = "Island"
+            return sig_ref
+
         ### ACTUAL LOOKUP ###
         #
         # OK, this is something we have to look up. Now check to see if our data store contains reference data and use

core/single_object_data_cache.py

@@ -0,0 +1,39 @@
+import logging
+import threading
+
+import diskcache
+
+logger = logging.getLogger(__name__)
+
+
+class SingleObjectDataCache:
+    """Cache for status and solar conditions. This uses DiskCache, but unlike the standard DiskCache users like SIG and
+    callsign lookup handlers, status and solar conditions are persisted as a single object. If we just load the object
+    from DiskCache and modify it, DiskCache doesn't know that it's been updated and needs re-caching, so we provide a
+    store() method that any functions updating the object can call afterwards."""
+
+    def __init__(self, cache_dir, object_if_empty):
+        """Initialize a SingleObjectDataCache. Provide the directory to load the cache from and save it to. If the cache
+        is empty, the provided object_if_empty parameter will be used to initialise it."""
+
+        self._lock = threading.Lock()
+        self._cache = diskcache.Cache(cache_dir)
+        # This cache stores a single object, doesn't matter what it's called so "object" will do
+        if "object" not in self._cache:
+            self._cache.add("object", object_if_empty)
+        self._obj = self._cache.get("object")
+
+    def get(self):
+        """Get the data object. This can then be manipulated as necessary across multiple threads. Any function
+        modifying the object must remember to call store() afterwards."""
+        return self._obj
+
+    def store(self):
+        """Store the updated object in the cache. Any function modifying the object must remember to call this
+        afterwards."""
+        with self._lock:
+            self._cache.set("object", self._obj)
+
+    def close(self):
+        self.store()
+        self._cache.close()

core/status_reporter.py

@@ -25,8 +25,9 @@ class StatusReporter:
         self._stop_event = Event()
         self._startup_time = datetime.now(pytz.UTC)
 
-        DATA_STORE.status_data["software_version"] = SOFTWARE_VERSION
-        DATA_STORE.status_data["server_owner_callsign"] = SERVER_OWNER_CALLSIGN
+        DATA_STORE.status.get()["software_version"] = SOFTWARE_VERSION
+        DATA_STORE.status.get()["server_owner_callsign"] = SERVER_OWNER_CALLSIGN
+        DATA_STORE.status.store()
 
     def start(self):
         """Start the reporter thread"""
@@ -50,11 +51,11 @@ class StatusReporter:
     def _report(self):
         """Write status information"""
 
-        DATA_STORE.status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
-        DATA_STORE.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
-        DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
-        DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
-        DATA_STORE.status_data["spot_providers"] = [
+        DATA_STORE.status.get()["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
+        DATA_STORE.status.get()["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
+        DATA_STORE.status.get()["num_spots"] = len(DATA_STORE.spots.values())
+        DATA_STORE.status.get()["num_alerts"] = len(DATA_STORE.alerts.values())
+        DATA_STORE.status.get()["spot_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -69,7 +70,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.spot_providers
         ]
-        DATA_STORE.status_data["alert_providers"] = [
+        DATA_STORE.status.get()["alert_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -80,7 +81,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.alert_providers
         ]
-        DATA_STORE.status_data["solar_condition_providers"] = [
+        DATA_STORE.status.get()["solar_condition_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -91,7 +92,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.solar_condition_providers
         ]
-        DATA_STORE.status_data["static_data_providers"] = [
+        DATA_STORE.status.get()["static_data_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -102,7 +103,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.static_data_providers
         ]
-        DATA_STORE.status_data["sig_ref_data_providers"] = [
+        DATA_STORE.status.get()["sig_ref_data_providers"] = [
             {
                 "sig_name": p.sig_name,
                 "enabled": p.enabled,
@@ -114,7 +115,7 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.sig_ref_data_providers
         ]
-        DATA_STORE.status_data["callsign_data_providers"] = [
+        DATA_STORE.status.get()["callsign_data_providers"] = [
             {
                 "name": p.name,
                 "enabled": p.enabled,
@@ -126,13 +127,13 @@ class StatusReporter:
             }
             for p in DATA_PROVIDERS.callsign_data_providers
         ]
-        DATA_STORE.status_data["cleanup"] = {
+        DATA_STORE.status.get()["cleanup"] = {
             "status": CLEANUP_TIMER.status,
             "last_ran": CLEANUP_TIMER.last_cleanup_time.replace(tzinfo=pytz.UTC).timestamp()
             if CLEANUP_TIMER.last_cleanup_time
             else 0,
         }
-        DATA_STORE.status_data["webserver"] = {
+        DATA_STORE.status.get()["webserver"] = {
             "status": WEB_SERVER.web_server_metrics["status"],
             "last_api_access": WEB_SERVER.web_server_metrics["last_api_access_time"]
             .replace(tzinfo=pytz.UTC)
@@ -147,6 +148,7 @@ class StatusReporter:
             else 0,
             "page_access_count": WEB_SERVER.web_server_metrics["page_access_counter"],
         }
+        DATA_STORE.status.store()
 
         # Update Prometheus metrics
         memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)

data/spot.py

@@ -11,7 +11,7 @@ from pyhamtools.locator import latlong_to_locator, locator_to_latlong
 
 from core.call_lookup_helper import get_call_info
 from core.config import MAX_SPOT_AGE
-from core.constants import CALL_REGEX, MODE_ALIASES, PROPAGATION_MODES
+from core.constants import MODE_ALIASES, PROPAGATION_MODES
 from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
 from core.sig_lookup_helper import populate_missing_sig_ref_info
 from core.sig_utils import (
@@ -201,14 +201,14 @@ class Spot:
             # If we have a spotter of "RBNHOLE", we should have the actual spotter callsign in the comment, so extract it.
             # RBNHole posts come from a number of providers, so it's dealt with here in the generic spot handling code.
             if self.de_call == "RBNHOLE" and self.comment:
-                rbnhole_call_match = re.search(rf"\Wat ({CALL_REGEX})\W", self.comment, re.IGNORECASE)
+                rbnhole_call_match = re.search(r"\Wat ([a-z0-9/]+)\W", self.comment, re.IGNORECASE)
                 if rbnhole_call_match:
                     self.de_call = rbnhole_call_match.group(1).upper()
 
             # If we have a spotter of "SOTAMAT", we might have the actual spotter callsign in the comment, if so extract it.
             # SOTAMAT can do POTA as well as SOTA, so it's dealt with here in the generic spot handling code.
             if self.de_call == "SOTAMAT" and self.comment:
-                sotamat_call_match = re.search(rf"\Wfrom ({CALL_REGEX})]", self.comment, re.IGNORECASE)
+                sotamat_call_match = re.search(r"\Wfrom ([a-z0-9/]+)]", self.comment, re.IGNORECASE)
                 if sotamat_call_match:
                     self.de_call = sotamat_call_match.group(1).upper()
 

providers/alert/ng3k.py

@@ -6,7 +6,6 @@ import pytz
 from rss_parser import Parser
 from rss_parser.models.rss import RSS
 
-from core.constants import CALL_REGEX
 from data.alert import Alert
 from providers.alert.http_alert_provider import HTTPAlertProvider
 
@@ -16,7 +15,7 @@ class NG3K(HTTPAlertProvider):
 
     POLL_INTERVAL_SEC = 1800
     ALERTS_URL = "https://www.ng3k.com/adxo.xml"
-    AS_CALL_PATTERN = re.compile(rf"as ({CALL_REGEX})", re.IGNORECASE)
+    AS_CALL_PATTERN = re.compile("as ([a-z0-9/]+)", re.IGNORECASE)
 
     def __init__(self, provider_config):
         super().__init__("NG3K", provider_config, self.ALERTS_URL, self.POLL_INTERVAL_SEC)

providers/solarconditions/giroionosonde.py

@@ -20,7 +20,7 @@ POLL_INTERVAL = 7200
 # To avoid looking up all stations in the GIRO system and working out which ones are providing live data, this has been
 # manually determined and a CSV provided of all the stations that we can query for live data.
 STATIONS_INDEX = "datafiles/didbase-stations.csv"
-LGDC_URL = "https://lgdc.uml.edu/common/DIDBGetValues"
+LGDC_URL = "https://lgdc.uml.edu/fastchar/getbest"
 HISTORY_HOURS = 24
 
 
@@ -140,7 +140,7 @@ class GIROIonosonde(SolarConditionsProvider):
 
         from_str = from_time.strftime("%Y.%m.%d+%H:%M:%S")
         to_str = to_time.strftime("%Y.%m.%d+%H:%M:%S")
-        url = f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUFD,fmin&DMUF=3000&fromDate={from_str}&toDate={to_str}"
+        url = f"{LGDC_URL}?ursiCode={ursi}&charName=foF2,MUF%28D%29,fmin&DMUF=3000&fromDate={from_str}&toDate={to_str}"
         try:
             http_response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15))
             if not http_response.ok:

providers/solarconditions/solar_conditions_provider.py

@@ -16,7 +16,7 @@ class SolarConditionsProvider:
         self.enabled = provider_config.get("enabled", True)
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
-        self._solar_conditions = DATA_STORE.solar_conditions
+        self._solar_conditions = DATA_STORE.solar_conditions.get()
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -36,3 +36,4 @@ class SolarConditionsProvider:
                 if hasattr(self._solar_conditions, key):
                     setattr(self._solar_conditions, key, value)
             self._solar_conditions.infer_descriptions()
+            DATA_STORE.solar_conditions.store()

providers/spot/dxcluster.py

@@ -8,7 +8,6 @@ import pytz
 import telnetlib3
 
 from core.config import SERVER_OWNER_CALLSIGN
-from core.constants import CALL_REGEX
 from data.spot import Spot
 from providers.spot.spot_provider import SpotProvider
 
@@ -20,11 +19,11 @@ class DXCluster(SpotProvider):
     See config-example.yml for examples."""
 
     _LINE_PATTERN_EXCLUDE_RBN = re.compile(
-        rf"^DX de ({CALL_REGEX}):\s+([0-9.]+)\s+({CALL_REGEX})\s+(.*)\s+(\d{4}Z)",
+        r"^DX de ([a-z0-9/]+):\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
         re.IGNORECASE,
     )
     _LINE_PATTERN_ALLOW_RBN = re.compile(
-        rf"^DX de ({CALL_REGEX})-?#?:\s+([0-9.]+)\s+({CALL_REGEX})\s+(.*)\s+(\d{4}Z)",
+        r"^DX de ([a-z0-9/]+)-?#?:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
         re.IGNORECASE,
     )
 
@@ -51,7 +50,8 @@ class DXCluster(SpotProvider):
 
     def stop(self):
         self._running = False
-        self._telnet.close()
+        if self._telnet:
+            self._telnet.close()
         self._thread.join()
 
     def _handle(self):

providers/spot/gma.py

@@ -117,6 +117,9 @@ class GMA(HTTPSpotProvider):
                                     case "IOTA Island":
                                         spot.sig_refs[0].sig = "IOTA"
                                         spot.sig = "IOTA"
+                                    case "GMA Island":
+                                        spot.sig_refs[0].sig = "GMA Islands"
+                                        spot.sig = "GMA Islands"
                                     case "Lighthouse (ILLW)":
                                         spot.sig_refs[0].sig = "ILLW"
                                         spot.sig = "ILLW"

providers/spot/parksnpeaks.py

@@ -5,7 +5,7 @@ from datetime import datetime
 import pytz
 import requests
 
-from core.constants import HTTP_HEADERS, CALL_REGEX
+from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -60,7 +60,7 @@ class ParksNPeaks(HTTPSpotProvider):
                 )
 
                 # Extract a de_call if it's in the comment but not in the "actSpoter" field
-                m = re.search(rf"\(de ({CALL_REGEX})\)", spot.comment or "")
+                m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment or "")
                 if not spot.de_call and m:
                     spot.de_call = str(m.group(1))
 

providers/spot/rbn.py

@@ -8,7 +8,6 @@ import pytz
 import telnetlib3
 
 from core.config import SERVER_OWNER_CALLSIGN
-from core.constants import CALL_REGEX
 from data.spot import Spot
 from providers.spot.spot_provider import SpotProvider
 
@@ -20,7 +19,7 @@ class RBN(SpotProvider):
     (port 7001) you need to instantiate two copies of this. The port is provided as an argument to the constructor."""
 
     _LINE_PATTERN = re.compile(
-        rf"^DX de ({CALL_REGEX})-.*:\s+([0-9.]+)\s+({CALL_REGEX})\s+(.*)\s+(\d{4}Z)",
+        r"^DX de ([a-z0-9/]+)-.*:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
         re.IGNORECASE,
     )
 
@@ -40,7 +39,8 @@ class RBN(SpotProvider):
 
     def stop(self):
         self._running = False
-        self._telnet.close()
+        if self._telnet:
+            self._telnet.close()
         self._thread.join()
 
     def _handle(self):

pyproject.toml

@@ -1,8 +1,8 @@
 [project]
-name = "Spothole"
-version = "2.0"
+name = "spothole"
+version = "2.0.5"
 authors = [
-  { name="Ian Renton", email="ian@ianrenton.com" },
+    { name = "Ian Renton", email = "ian@ianrenton.com" },
 ]
 description = "Spothole is a utility to aggregate \"spots\" from amateur radio DX clusters and xOTA spotting sites, and provide an open JSON API as well as a website to browse the data."
 readme = "README.md"

server/handlers/api/addspot.py

@@ -11,7 +11,7 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRET_KEY
-from core.constants import CALL_ONLY_PATTERN, UNKNOWN_BAND
+from core.constants import UNKNOWN_BAND
 from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import infer_band_from_freq, safe_json_dumps
@@ -121,13 +121,13 @@ class APISpotHandler(tornado.web.RequestHandler):
                 return
 
             # Reject invalid-looking callsigns
-            if not CALL_ONLY_PATTERN.match(spot.dx_call):
+            if not re.match(r"^[A-Za-z0-9/\-]*$", spot.dx_call):
                 self.set_status(422)
                 self.write(safe_json_dumps(f"Error - '{spot.dx_call}' does not look like a valid callsign."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
-            if not CALL_ONLY_PATTERN.match(spot.de_call):
+            if not re.match(r"^[A-Za-z0-9/\-]*$", spot.de_call):
                 self.set_status(422)
                 self.write(safe_json_dumps(f"Error - '{spot.de_call}' does not look like a valid callsign."))
                 self.set_header("Cache-Control", "no-store")

server/handlers/api/alerts.py

@@ -82,8 +82,8 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
         self._credentials = None
         super().__init__(application, request, **kwargs)
 
-    def initialize(self, _sse_alert_broadcaster, web_server_metrics):
-        self._sse_alert_broadcaster = _sse_alert_broadcaster
+    def initialize(self, sse_alert_broadcaster, web_server_metrics):
+        self._sse_alert_broadcaster = sse_alert_broadcaster
         self._web_server_metrics = web_server_metrics
 
     def custom_headers(self):

server/handlers/api/lookups.py

@@ -9,7 +9,7 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.call_lookup_helper import get_call_info
-from core.constants import CALL_ONLY_PATTERN, SIGS
+from core.constants import SIGS
 from core.geo_utils import (
     lat_lon_for_grid_sw_corner_plus_size,
     lat_lon_to_cq_zone,
@@ -55,7 +55,7 @@ class APILookupCallHandler(tornado.web.RequestHandler):
             # The "call" query param must exist and look like a callsign
             if "call" in query_params:
                 call = str(query_params.get("call")).upper()
-                if CALL_ONLY_PATTERN.match(call):
+                if re.match(r"^[A-Z0-9/\-]*$", call):
                     credentials = extract_credentials(self.request.headers)
                     callsign_data = get_call_info(call, credentials)
                     self.write(safe_json_dumps(callsign_data))

server/handlers/api/v1_addspot.py

@@ -9,7 +9,7 @@ from tornado import httputil
 from tornado.web import Application
 
 from core.config import ALLOW_SPOTTING
-from core.constants import CALL_ONLY_PATTERN, UNKNOWN_BAND
+from core.constants import UNKNOWN_BAND
 from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import infer_band_from_freq, safe_json_dumps
@@ -83,13 +83,13 @@ class V1APISpotHandler(tornado.web.RequestHandler):
                 return
 
             # Reject invalid-looking callsigns
-            if not CALL_ONLY_PATTERN.match(spot.dx_call):
+            if not re.match(r"^[A-Za-z0-9/\-]*$", spot.dx_call):
                 self.set_status(422)
                 self.write(safe_json_dumps(f"Error - '{spot.dx_call}' does not look like a valid callsign."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
-            if not CALL_ONLY_PATTERN.match(spot.de_call):
+            if not re.match(r"^[A-Za-z0-9/\-]*$", spot.de_call):
                 self.set_status(422)
                 self.write(safe_json_dumps(f"Error - '{spot.de_call}' does not look like a valid callsign."))
                 self.set_header("Cache-Control", "no-store")

server/handlers/api/v1_compatability.py

@@ -2,6 +2,7 @@ import tornado
 from tornado.httpclient import AsyncHTTPClient
 from tornado.httputil import HTTPHeaders
 
+
 _LEGACY_PARAM_TO_HEADER_MAP = {
     "qrz_username": "X-QRZ-Username",
     "qrz_password": "X-QRZ-Password",
@@ -11,7 +12,6 @@ _LEGACY_PARAM_TO_HEADER_MAP = {
     "hamqth_session_id": "X-HamQTH-Session-ID",
 }
 
-
 class V1RedirectHandler(tornado.web.RequestHandler):
     """Transparently proxies requests from the old API to the new one,
     returning whatever the v2 endpoint returns, for endpoints with no breaking changes."""
@@ -34,7 +34,7 @@ class V1RedirectHandler(tornado.web.RequestHandler):
             response = await client.fetch(
                 new_url,
                 method=self.request.method,
-                headers=headers,
+                headers=self.request.headers,
                 body=None if self.request.method == "GET" else (self.request.body or b""),
                 raise_error=False,
                 follow_redirects=False,
@@ -60,4 +60,4 @@ class V1RedirectHandler(tornado.web.RequestHandler):
         await self._proxy(path)
 
     async def post(self, path):
-        await self._proxy(path)
+        await self._proxy(path)

server/webserver.py

@@ -111,7 +111,7 @@ class WebServer:
             (
                 r"/api/v2/solar",
                 APISolarConditionsHandler,
-                {"solar_conditions": self._data_store.solar_conditions, **handler_opts},
+                {"solar_conditions": self._data_store.solar_conditions.get(), **handler_opts},
             ),
             (
                 r"/api/v2/dxstats",
@@ -121,12 +121,12 @@ class WebServer:
             (
                 r"/api/v2/options",
                 APIOptionsHandler,
-                {"status_data": self._data_store.status_data, **handler_opts},
+                {"status_data": self._data_store.status.get(), **handler_opts},
             ),
             (
                 r"/api/v2/status",
                 APIStatusHandler,
-                {"status_data": self._data_store.status_data, **handler_opts},
+                {"status_data": self._data_store.status.get(), **handler_opts},
             ),
             (r"/api/v2/lookup/call", APILookupCallHandler, {**handler_opts}),
             (r"/api/v2/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
@@ -160,7 +160,6 @@ class WebServer:
                 V1APISpotHandler,
                 {
                     "spots": self._data_store.spots,
-                    "spot_providers": self._data_providers,
                     **handler_opts,
                 },
             ),

spothole.py

@@ -33,7 +33,7 @@ if __name__ == "__main__":
     root.setLevel(LOG_LEVEL)
     handler = logging.StreamHandler(sys.stdout)
     handler.setLevel(LOG_LEVEL)
-    formatter = logging.Formatter("%(levelname)s : %(message)s")
+    formatter = logging.Formatter("%(asctime)s | %(levelname)7s | %(message)s", "%Y-%m-%d %H:%M:%S")
     handler.setFormatter(formatter)
     root.handlers.clear()
     root.addHandler(handler)

static/apidocs/openapi.yml

@@ -527,7 +527,8 @@ components:
       in: query
       description: >
         Limit the spots to only ones from one or more sources. To select more than one source, supply a
-        comma-separated list.
+        comma-separated list. Note that this enumeration is not exhaustive; in some cases the server owner may have
+        named some sources like clusters differently, and provided multiple options.
       schema:
         $ref: "#/components/schemas/Source"
     SpotSig:

static/js/map.js

@@ -394,9 +394,11 @@ function setBasemap(basemapname) {
         }
         // OpenStreetMap.Mapnik.Dark is a synthetic variant that uses Mapnik tiles with a CSS filter applied
         const providerName = basemapname === "OpenStreetMap.Mapnik.Dark" ? "OpenStreetMap.Mapnik" : basemapname;
+        const isCartoDB = basemapname === "CartoDB.Voyager" || basemapname === "CartoDB.DarkMatter";
         backgroundTileLayer = L.tileLayer.provider(providerName, {
             opacity: parseFloat($("#basemapOpacity").val()),
-            edgeBufferTiles: 1
+            edgeBufferTiles: 1,
+            key: isCartoDB ? CARTODB_API_KEY : null
         });
         backgroundTileLayer.addTo(map);
         backgroundTileLayer.bringToBack();
@@ -529,9 +531,11 @@ function setUpMap() {
     // Add basemap
     loadedBasemap = $("#basemap").val();
     const initialProviderName = loadedBasemap === "OpenStreetMap.Mapnik.Dark" ? "OpenStreetMap.Mapnik" : loadedBasemap;
+    const isCartoDB = loadedBasemap === "CartoDB.Voyager" || loadedBasemap === "CartoDB.DarkMatter";
     backgroundTileLayer = L.tileLayer.provider(initialProviderName, {
         opacity: parseFloat($("#basemapOpacity").val()),
-        edgeBufferTiles: 1
+        edgeBufferTiles: 1,
+        key: isCartoDB ? CARTODB_API_KEY : null
     });
     backgroundTileLayer.addTo(map);
     backgroundTileLayer.bringToBack();

static/js/spotsbandsandmap.js

@@ -40,7 +40,7 @@ function setHamHFBandToggles() {
 
 // Generate SIGs filter card. This one is also a special case.
 function generateSIGsMultiToggleFilterCard(sig_options) {
-    const $grid = $('<div class="row row-cols-2 row-cols-xxl-3 g-1 mb-1">');
+    const $grid = $('<div class="row row-cols-2 g-1 mb-1">');
     sig_options.forEach(o => {
         const domSafeName = o["name"].replace(/^[^A-Za-z0-9]+|[^\w]+/gi, "");
         $grid.append(`<div class="col"><div class="form-check"><input type="checkbox" class="form-check-input filter-button-sig storeable-checkbox" id="filter-button-sig-${domSafeName}" value="${o['name']}" autocomplete="off" onClick="filtersUpdated()" checked><label class="form-check-label" id="filter-button-label-sig-${domSafeName}" for="filter-button-sig-${domSafeName}" title="${o['description']}"><i class="fa-solid ${sigToIcon(o['name'], 'fa-tower-cell')}"></i> ${o['name']}</label></div></div>`);

static/js/ui-ham.js

@@ -345,6 +345,7 @@ const SIG_ICONS = {
     "SOTA": "fa-mountain-sun",
     "WWFF": "fa-seedling",
     "GMA": "fa-person-hiking",
+    "GMA Islands": "fa-person-hiking",
     "WWBOTA": "fa-radiation",
     "HEMA": "fa-mound",
     "IOTA": "fa-book-atlas",
@@ -372,6 +373,7 @@ const SIG_NAMES = {
     "SOTA": "Summits on the Air",
     "WWFF": "Worldwide Flora & Fauna",
     "GMA": "Global Mountain Activity",
+    "GMA Islands": "Global Mountain Activity - Islands",
     "WWBOTA": "Bunkers on the Air",
     "HEMA": "Humps Excluding Marilyns Award",
     "IOTA": "Islands on the Air",

static/vendor/js/leaflet-providers-2.0.0.js

@@ -801,12 +801,13 @@
 			}
 		},
 		CartoDB: {
-			url: 'https://{s}.basemaps.cartocdn.com/{variant}/{z}/{x}/{y}{r}.png',
+			url: 'https://{s}.basemaps.cartocdn.com/{variant}/{z}/{x}/{y}{r}.png?key={key}',
 			options: {
 				attribution: '{attribution.OpenStreetMap} &copy; <a href="https://carto.com/attributions">CARTO</a>',
 				subdomains: 'abcd',
 				maxZoom: 20,
-				variant: 'light_all'
+				variant: 'light_all',
+				key: '<insert your API key here>'
 			},
 			variants: {
 				Positron: 'light_all',

templates/about.html

@@ -98,7 +98,7 @@
     <p>Between the various data sources, the following Special Interest Groups (SIGs) are supported: Parks on the Air
         (POTA), Summits on the Air (SOTA), Worldwide Flora & Fauna (WWFF), Global Mountain Activity (GMA), Worldwide
         Bunkers on the Air (WWBOTA), HuMPs Excluding Marilyns Award (HEMA), Islands on the Air (IOTA), Mills on the Air
-        (MOTA), the Amateur Radio Lighthouse Socirty (ARLHS), International Lighthouse Lightship Weekend (ILLW), Silos
+        (MOTA), the Amateur Radio Lighthouse Society (ARLHS), International Lighthouse Lightship Weekend (ILLW), Silos
         on the Air (SIOTA), World Castles Award (WCA), New Zealand on the Air (ZLOTA), Keith Roget Memorial National
         Parks Award (KRMNPA), South Australia National Parks and Conservation Parks Award (SANPCPA), Wainwrights on the
         Air (WOTA), Beaches on the Air (BOTA), Lagos y Lagunas On the Air (LLOTA), Towers on the Air (WWTOTA), Tiles on
@@ -171,10 +171,6 @@
         section, Privacy, for details of how these are handled. If you are looking at the map and see lots of spots in
         the geographic centre of countries, allowing these lookups will help. For QRZ.com a paid account is required to
         look up operator locations.</p>
-    <p>IOTA in particular causes a mapping problem, firstly that there is no official set of geodata beyond the names of
-        islands. We also have no way of telling whether priority should be given to IOTA reference or e.g. QRZ lookup,
-        for example for a G callsign in EU-005 Great Britain the QRZ home address is probably more accurate, but in
-        EU-002 Aaland islands, it's more likely a DXpedition and the IOTA would be more accurate.</p>
     <h2 id="privacy" class="mt-4">Privacy</h2>
     <p>Spothole collects no data about you on a permanent basis. All spots and alerts are "timed out" and deleted from
         the system after a set interval, which by default is one hour for spots and one week for alerts.</p>
@@ -182,11 +178,13 @@
         filters. They are also stored in your browser's local storage, so that your preferences are remembered between
         sessions.</p>
     <p>The data you provide can optionally include your login credentials for QRZ.com and HamQTH. You can provide these
-        in the "Data" menu of most pages. If you do, Spothole will augment the data it produces with lookups from these
+        in the "Your Data" menu of most pages. If you do, Spothole will augment the data it produces with lookups from these
         services, which can for example provide more accurate markers on the map tab, and operator names when you mouse
         over a DX callsign. Spothole will still work fine if you don't provide these. The values you enter are sent to
         Spothole via HTTPS so are protected in transit, though of course you do have to trust Spothole with this
         sensitive data in order to use this feature.</p>
+    <p>Any data you send as part of a query, such as your QRZ or HamQTH credentials, is used only for the lifetime of that
+    query and is not saved anywhere apart from your own device.</p>
     <p>Spothole uses no trackers, no ads, and no cookies.</p>
     {% if len(web_ui_options["support_button_html"]) > 0 %}
     <p><strong>Caveat: </strong> The owner of this server has chosen to inject their own content into the "spots" page.

templates/add_spot.html

@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/static/js/add-spot.js?v=1787072396"></script>
+<script src="/static/js/add-spot.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -84,7 +84,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1787072395"></script>
+<script src="/static/js/alerts.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -76,8 +76,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1787072395"></script>
-<script src="/static/js/bands.js?v=1787072395"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1787991444"></script>
+<script src="/static/js/bands.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1787072395" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1787991444" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -15,10 +15,10 @@
   window.fetchEventSource = fetchEventSource;
 </script>
 
-<script src="/static/js/utils.js?v=1787072395"></script>
-<script src="/static/js/ui-ham.js?v=1787072395"></script>
-<script src="/static/js/geo.js?v=1787072395"></script>
-<script src="/static/js/common.js?v=1787072395"></script>
+<script src="/static/js/utils.js?v=1787991444"></script>
+<script src="/static/js/ui-ham.js?v=1787991444"></script>
+<script src="/static/js/geo.js?v=1787991444"></script>
+<script src="/static/js/common.js?v=1787991444"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/cards/spot_age.html

@@ -6,7 +6,7 @@
                     oninput="filtersUpdated();" style="width: 5em; display: inline-block;">
                 {% for a in web_ui_options["max_spot_age"] %}
                 <option value="{{a*60}}" {% if web_ui_options[
-                "max_spot_age_default"] == a*60 %}selected{% end %}>{{a}}</option>
+                "max_spot_age_default"] == a %}selected{% end %}>{{a}}</option>
                 {% end %}
             </select>
             minutes

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1787072395"></script>
+<script src="/static/js/conditions.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -109,8 +109,12 @@
 <script src="/static/vendor/js/leaflet-cqzones.js"></script>
 <script src="/static/vendor/js/leaflet-workedallbritainireland.js" type="module"></script>
 
-<script src="/static/js/spotsbandsandmap.js?v=1787072394"></script>
-<script src="/static/js/map.js?v=1787072394"></script>
+<script>
+    const CARTODB_API_KEY = "{{ web_ui_options.get('cartodb_api_key', '') }}";
+</script>
+
+<script src="/static/js/spotsbandsandmap.js?v=1787991444"></script>
+<script src="/static/js/map.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -113,8 +113,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1787072394"></script>
-<script src="/static/js/spots.js?v=1787072394"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1787991444"></script>
+<script src="/static/js/spots.js?v=1787991444"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -86,7 +86,7 @@
     </div>
 </div>
 
-<script src="/static/js/status.js?v=1787072395"></script>
+<script src="/static/js/status.js?v=1787991444"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");