Refactor of caching & data storage part 4 #118

alertproviders/bota.py

@@ -20,7 +20,7 @@ class BOTA(HTTPAlertProvider):
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
         # Find the table of upcoming alerts
-        bs = BeautifulSoup(http_response.content.decode(), features="lxml")
+        bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml")
         if not bs.body:
             return new_alerts
         div = bs.body.find('div', attrs={'class': 'view-activations-public'})

alertproviders/ng3k.py

@@ -22,7 +22,7 @@ class NG3K(HTTPAlertProvider):
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
-        rss = cast(RSS, Parser.parse(http_response.content.decode()))
+        rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
         # Iterate through source data
         for source_alert in rss.channel.items:
             # Deal with "the format"...

alertproviders/parksnpeaks.py

@@ -22,7 +22,7 @@ class ParksNPeaks(HTTPAlertProvider):
         # Iterate through source data
         for source_alert in http_response.json():
             # Calculate some things
-            sig = source_alert["Class"]
+            sig = source_alert["Class"].upper()
             if " - " in source_alert["Location"]:
                 split = source_alert["Location"].split(" - ")
                 sig_ref = split[0]
@@ -50,7 +50,7 @@ class ParksNPeaks(HTTPAlertProvider):
                           is_dxpedition=False)
 
             # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
-            if sig and sig not in ["POTA", "SOTA", "WWFF", "SiOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]:
+            if sig and sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "LLOTA", "QRP"]:
                 logging.warning("PNP alert found with sig " + sig + ", developer needs to add support for this!")
 
             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to

alertproviders/wota.py

@@ -22,7 +22,7 @@ class WOTA(HTTPAlertProvider):
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
-        rss = cast(RSS, RSSParser.parse(http_response.content.decode()))
+        rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig")))
         # Iterate through source data
         for source_alert in rss.channel.items:
 

config-example.yml

@@ -121,7 +121,7 @@ spot-providers:
     url: "wss://39c3.totawatch.de/api/spot/live"
     # For the "XOTA" provider, a SIG must be set menually here because xOTA is a generic backend for xOTA
     # programmes and so different URLs potentially provide different programmes.
-    sig: "TOTA"
+    sig: "Toilets"
     # For Toilets on the Air, we prefix the SIG references (T-01 etc) with some characters that define the conference:
     # C3, EH or HOPE - so we can look up the correct locations in our database, because each conference starts from T-01
     # but refers to a toilet in a different building (or continent!)
@@ -131,14 +131,14 @@ spot-providers:
     name: "EH23 TOTA"
     enabled: false
     url: "wss://eh23.totawatch.de/api/spot/live"
-    sig: "TOTA"
+    sig: "Toilets"
     sig-ref-prefix: "EH"
 
   - class: "XOTA"
     name: "HOPE26 TOTA"
     enabled: false
     url: "wss://hope-26.totawatch.de/api/spot/live"
-    sig: "TOTA"
+    sig: "Toilets"
     sig-ref-prefix: "HOPE"
 
 
@@ -169,9 +169,36 @@ alert-providers:
 # SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF
 # park IDs to their name and location.
 sig-ref-data-providers:
+  - class: "POTA"
+    enabled: true
+
+  - class: "SOTA"
+    enabled: true
+
   - class: "WWFF"
     enabled: true
 
+  - class: "WWBOTA"
+    enabled: true
+
+  - class: "GMA"
+    enabled: true
+
+  - class: "MOTA"
+    enabled: true
+
+  - class: "ILLW"
+    enabled: true
+
+  - class: "ARLHS"
+    enabled: true
+
+  - class: "WCA"
+    enabled: true
+
+  - class: "IOTA"
+    enabled: true
+
   - class: "WOTA"
     enabled: true
 
@@ -184,10 +211,13 @@ sig-ref-data-providers:
   - class: "LLOTA"
     enabled: true
 
+  - class: "Towers"
+    enabled: true
+
   - class: "DME"
     enabled: true
 
-  - class: "TOTA"
+  - class: "Toilets"
     enabled: true
 
 # Solar condition providers to use. These poll external APIs for solar propagation data (SFI, A/K indices, band

core/constants.py

@@ -19,9 +19,9 @@ SIGS = [
     SIG(name="HEMA", comment_names=["HEMA"], description="HuMPs Excluding Marilyns Award", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}"),
     SIG(name="IOTA", comment_names=["IOTA"], description="Islands on the Air", ref_regex=r"[A-Z]{2}\-\d{3}"),
     SIG(name="MOTA", comment_names=["MOTA"], description="Mills on the Air", ref_regex=r"X\d{4,6}"),
-    SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}\-\d{3,4}"),
+    SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}[\- ]\d{3,4}"),
     SIG(name="ILLW", comment_names=["ILLW"], description="International Lighthouse & Lightship Weekend", ref_regex=r"[A-Z]{2}\d{4}"),
-    SIG(name="SiOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
+    SIG(name="SIOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
     SIG(name="WCA", comment_names=["WCA"], description="World Castles Award", ref_regex=r"[A-Z0-9]{1,3}\-\d{5}"),
     SIG(name="ZLOTA", comment_names=["ZLOTA"], description="New Zealand on the Air", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}"),
     SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"),

core/data_store.py

@@ -1,8 +1,10 @@
+import logging
 from pathlib import Path
 
 import diskcache
 
 from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
+from core.constants import SIGS
 from core.live_data_cache import LiveDataCache
 from data.solar_conditions import SolarConditions
 
@@ -12,51 +14,65 @@ class DataStore:
     lookup data using different caching strategies for each."""
 
     def __init__(self):
-        self.CACHE_DIR = "./cache"
-        self.MAX_SPOT_COUNT = 10000
-        self.MAX_ALERT_COUNT = 10000
-        self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
-        self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
+        self._CACHE_DIR = "./cache"
+        self._MAX_SPOT_COUNT = 100000
+        self._MAX_ALERT_COUNT = 100000
+        self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
+        self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
+        self.alerts = None
+        self.spots = None
+        self.callsigns = None
+        self.sigrefs = None
+        self.status_data = None
+        self._status = None
+        self.solar_conditions = None
+        self._solar = None
 
     def setup(self):
-        Path(self.CACHE_DIR).mkdir(parents=True, exist_ok=True)
+        Path(self._CACHE_DIR).mkdir(parents=True, exist_ok=True)
 
         # Standard disk cache for solar data and status data, but each cache contains only a single object which we
         # expose to the wider application
-        self.solar = diskcache.Cache(self.CACHE_DIR + "/solar")
-        if "solar_conditions" not in self.solar:
-            self.solar.add("solar_conditions", SolarConditions())
-        self.solar_conditions = self.solar.get("solar_conditions")
-        self.status = diskcache.Cache(self.CACHE_DIR + "/status")
-        if "status_data" not in self.status:
-            self.status.add("status_data", {})
-        self.status_data = self.status.get("status_data")
+        self._solar = diskcache.Cache(self._CACHE_DIR + "/solar")
+        if "solar_conditions" not in self._solar:
+            self._solar.add("solar_conditions", SolarConditions())
+        self.solar_conditions = self._solar.get("solar_conditions")
+        self._status = diskcache.Cache(self._CACHE_DIR + "/status")
+        if "status_data" not in self._status:
+            self._status.add("status_data", {})
+        self.status_data = self._status.get("status_data")
 
         # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
         # but there's no need for a TTL since old data is better than no data. This is a two-layer dict, keys are SIG
         # name and then reference ID, with the final value being a SIGRef object.
-        self.sigrefs = diskcache.Cache(self.CACHE_DIR + "/sigrefs")
+        self.sigrefs = diskcache.Cache(self._CACHE_DIR + "/sigrefs")
+        for k in list(self.sigrefs.iterkeys()):
+            logging.info(f"Loaded data for %d references in %s SIG.", len(self.sigrefs[k]), k)
 
         # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
         # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
         # we've seen, so we rely on them timing out and this triggering another lookup.
-        self.callsigns = diskcache.Cache(self.CACHE_DIR + "/callsigns")
+        self.callsigns = diskcache.Cache(self._CACHE_DIR + "/callsigns")
+        logging.info(f"Loaded data for %d callsigns.", len(self.callsigns))
 
         # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
         # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
         # before the spots and alerts are live in the system.
-        self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
-                                   snapshot_dir=self.CACHE_DIR + "/spots",
-                                   snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
-        self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
-                                   snapshot_dir=self.CACHE_DIR + "/alerts",
-                                    snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
+        self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
+                                   snapshot_dir=self._CACHE_DIR + "/spots",
+                                   snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
+        logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys()))
+
+        self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
+                                    snapshot_dir=self._CACHE_DIR + "/alerts",
+                                    snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
+        logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))
 
     def close(self):
         self.spots.close()
         self.alerts.close()
-        self.solar.close()
-        self.status.close()
+        self._solar.close()
+        self._status.close()
         self.sigrefs.close()
         self.callsigns.close()
 

core/sig_utils.py

@@ -35,157 +35,25 @@ def populate_sig_ref_info(sig_ref):
     SIG we are getting data for.
     Note there is currently no support for KRMNPA location lookup, see issue #61."""
 
-    if sig_ref.sig is None or sig_ref.id is None or sig_ref.id == "":
+    if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "":
         logging.debug("Failed to look up sig_ref info, sig or id were not set.")
         return sig_ref
 
-    sig = sig_ref.sig or ""
+    sig = sig_ref.sig
     ref_id = sig_ref.id
+
+    # DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all activators
+    # add leading zeros.
+    if sig.upper() == "DME":
+        ref_id = ref_id.zfill(5)
+
     try:
-        if sig.upper() == "POTA":
-            response = URL_DATA_CACHE.get("https://api.pota.app/park/" + ref_id, headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if data:
-                    fullname = str(data["name"]) if "name" in data else None
-                    if fullname and "parktypeDesc" in data and data["parktypeDesc"] != "":
-                        fullname = fullname + " " + data["parktypeDesc"]
-                    sig_ref.name = fullname
-                    sig_ref.url = "https://pota.app/#/park/" + ref_id
-                    sig_ref.grid = data["grid6"] if "grid6" in data else None
-                    sig_ref.latitude = data["latitude"] if "latitude" in data else None
-                    sig_ref.longitude = data["longitude"] if "longitude" in data else None
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "SOTA":
-            response = URL_DATA_CACHE.get("https://api-db2.sota.org.uk/api/summits/" + ref_id,
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if data:
-                    sig_ref.name = data["name"] if "name" in data else None
-                    sig_ref.url = "https://www.sotadata.org.uk/en/summit/" + ref_id
-                    sig_ref.grid = data["locator"] if "locator" in data else None
-                    sig_ref.latitude = data["latitude"] if "latitude" in data else None
-                    sig_ref.longitude = data["longitude"] if "longitude" in data else None
-                    sig_ref.activation_score = data["points"] if "points" in data else None
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "WWBOTA":
-            response = URL_DATA_CACHE.get("https://api.wwbota.org/bunkers/" + ref_id,
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if data:
-                    sig_ref.name = data["name"] if "name" in data else None
-                    sig_ref.url = "https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None
-                    sig_ref.grid = data["locator"] if "locator" in data else None
-                    sig_ref.latitude = data["lat"] if "lat" in data else None
-                    sig_ref.longitude = data["long"] if "long" in data else None
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "GMA" or sig.upper() == "ARLHS" or sig.upper() == "ILLW" or sig.upper() == "WCA" or sig.upper() == "MOTA" or sig.upper() == "IOTA":
-            response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
-                                                      headers=HTTP_HEADERS)
-            if response.ok:
-                data = response.json()
-                if data:
-                    sig_ref.name = data["name"] if "name" in data else None
-                    sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
-                    sig_ref.grid = data["locator"] if "locator" in data else None
-
-                    # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
-                    # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
-                    # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
-                    # just use the single point we have.
-                    if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
-                            "lat2") is not None and data.get("lng2") is not None:
-                        sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
-                        sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
-                    else:
-                        sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
-                        sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
-                elif not response.from_cache:
-                    logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
-            elif not response.from_cache:
-                logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
-        elif sig.upper() == "WWFF":
-            lookup_data = DATA_STORE.sigrefs["WWFF"][ref_id] if ref_id in DATA_STORE.sigrefs["WWFF"] else None
-            if lookup_data:
-                # Copy new sig ref data into existing object
-                sig_ref.__dict__.update(lookup_data.__dict__)
-            else:
-                logging.warning("WWFF database did not contain data for ref %s", ref_id)
-
-        elif sig.upper() == "SIOTA":
-            lookup_data = DATA_STORE.sigrefs["SIOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["SIOTA"] else None
-            if lookup_data:
-                # Copy new sig ref data into existing object
-                sig_ref.__dict__.update(lookup_data.__dict__)
-            else:
-                logging.warning("SIOTA database did not contain data for ref %s", ref_id)
-
-        elif sig.upper() == "WOTA":
-            lookup_data = DATA_STORE.sigrefs["WOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["WOTA"] else None
-            if lookup_data:
-                # Copy new sig ref data into existing object
-                sig_ref.__dict__.update(lookup_data.__dict__)
-            else:
-                logging.warning("WOTA database did not contain data for ref %s", ref_id)
-
-        elif sig.upper() == "ZLOTA":
-            lookup_data = DATA_STORE.sigrefs["ZLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["ZLOTA"] else None
-            if lookup_data:
-                # Copy new sig ref data into existing object
-                sig_ref.__dict__.update(lookup_data.__dict__)
-            else:
-                logging.warning("ZLOTA database did not contain data for ref %s", ref_id)
-
-        elif sig.upper() == "BOTA":
-            if not sig_ref.name:
-                sig_ref.name = sig_ref.id
-            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
-
-        elif sig.upper() == "LLOTA":
-            lookup_data = DATA_STORE.sigrefs["LLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["LLOTA"] else None
-            if lookup_data:
-                # Copy new sig ref data into existing object
-                sig_ref.__dict__.update(lookup_data.__dict__)
-            else:
-                logging.warning("LLOTA database did not contain data for ref %s", ref_id)
-
-        elif sig.upper() == "DME":
-            # Zero-pad to 5 digits to match our source data
-            lookup_data = DATA_STORE.sigrefs["DME"][ref_id.zfill(5)] if ref_id.zfill(5) in DATA_STORE.sigrefs["DME"] else None
-            if lookup_data:
-                # Copy new sig ref data into existing object
-                sig_ref.__dict__.update(lookup_data.__dict__)
-            else:
-                logging.warning("DME database did not contain data for ref %s", ref_id)
-
-        elif sig.upper() == "TOTA":
-            lookup_data = DATA_STORE.sigrefs["TOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["TOTA"] else None
-            if lookup_data:
-                # Copy new sig ref data into existing object
-                sig_ref.__dict__.update(lookup_data.__dict__)
-            else:
-                logging.warning("TOTA database did not contain data for ref %s", ref_id)
-
-        elif sig.upper() == "WWTOTA":
-            if not sig_ref.name:
-                sig_ref.name = sig_ref.id
-            sig_ref.url = "https://wwtota.com/seznam/karta_rozhledny.php?ref=" + str(sig_ref.name)
+        # If the SIG is HEMA or KRMNPA, we have no current lookup for this so just skip it.
+        if sig.upper() == "HEMA" or sig.upper() == "KRMNPA":
+            return sig_ref
 
+        # If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
+        # calculate all the information we are going to get directly. So handle those cases first
         elif sig.upper() == "TILES":
             # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
             if not sig_ref.name:
@@ -208,10 +76,54 @@ def populate_sig_ref_info(sig_ref):
                 except:
                     logging.warning("Invalid lat/lon received for WAB/WAI reference")
 
-    except ConnectionError:
-        logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
-    except (ConnectTimeout, ReadTimeout):
-        logging.warning(f"Timeout when looking up sig_ref info for " + sig + " ref " + ref_id)
+        elif sig.upper() == "BOTA":
+            # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
+            if not sig_ref.name:
+                sig_ref.name = sig_ref.id
+            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
+
+        # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
+        # this SIG. If so, check for the reference data and use that.
+        elif sig in DATA_STORE.sigrefs:
+            lookup_data = DATA_STORE.sigrefs[sig][ref_id] if ref_id in DATA_STORE.sigrefs[sig] else None
+            if lookup_data:
+                # Copy new sig ref data into existing object
+                sig_ref.__dict__.update(lookup_data.__dict__)
+            else:
+                logging.warning("%s database did not contain data for ref %s", sig, ref_id)
+
+        elif False:
+            # TODO remove
+            # OK, this is not a SIG we have stored data for. Maybe it's of a type we can query information for live.
+            if sig.upper() == "IOTA":
+                response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
+                                                          headers=HTTP_HEADERS)
+                if response.ok:
+                    data = response.json()
+                    if data:
+                        sig_ref.name = data["name"] if "name" in data else None
+                        sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
+                        sig_ref.grid = data["locator"] if "locator" in data else None
+
+                        # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
+                        # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
+                        # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
+                        # just use the single point we have.
+                        if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
+                                "lat2") is not None and data.get("lng2") is not None:
+                            sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
+                            sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
+                        else:
+                            sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
+                            sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
+                    elif not response.from_cache:
+                        logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
+                elif not response.from_cache:
+                    logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
+
+        else:
+            logging.warning(f"Tried to look up a SIG called %s but Spothole does not know what that is.", sig)
+
     except Exception:
         logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
     return sig_ref

core/status_reporter.py

@@ -77,7 +77,8 @@ class StatusReporter:
         DATA_STORE.status_data["sig_ref_data_providers"] = list(
             map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
+                           "reference_count": p.reference_count},
                 self._sig_ref_data_providers))
         DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
                                       "last_api_access": self._web_server.web_server_metrics[

data/alert.py

@@ -1,6 +1,7 @@
 import copy
 import hashlib
 import json
+import logging
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 
@@ -64,69 +65,73 @@ class Alert:
     def infer_missing(self, credentials=None):
         """Infer missing parameters where possible"""
 
-        # If we somehow don't have a start time, set it to zero so it sorts off the bottom of any list but
-        # clients can still reliably parse it as a number.
-        if not self.start_time:
-            self.start_time = 0
+        try:
+            # If we somehow don't have a start time, set it to zero so it sorts off the bottom of any list but
+            # clients can still reliably parse it as a number.
+            if not self.start_time:
+                self.start_time = 0
 
-        # If we don't have a received time, this has just been received so set that to "now"
-        if not self.received_time:
-            self.received_time = datetime.now(pytz.UTC).timestamp()
+            # If we don't have a received time, this has just been received so set that to "now"
+            if not self.received_time:
+                self.received_time = datetime.now(pytz.UTC).timestamp()
 
-        # Fill in ISO versions of times, in case the client prefers that
-        if self.start_time and not self.start_time_iso:
-            self.start_time_iso = datetime.fromtimestamp(self.start_time, pytz.UTC).isoformat()
-        if self.end_time and not self.end_time_iso:
-            self.end_time_iso = datetime.fromtimestamp(self.end_time, pytz.UTC).isoformat()
-        if self.received_time and not self.received_time_iso:
-            self.received_time_iso = datetime.fromtimestamp(self.received_time, pytz.UTC).isoformat()
+            # Fill in ISO versions of times, in case the client prefers that
+            if self.start_time and not self.start_time_iso:
+                self.start_time_iso = datetime.fromtimestamp(self.start_time, pytz.UTC).isoformat()
+            if self.end_time and not self.end_time_iso:
+                self.end_time_iso = datetime.fromtimestamp(self.end_time, pytz.UTC).isoformat()
+            if self.received_time and not self.received_time_iso:
+                self.received_time_iso = datetime.fromtimestamp(self.received_time, pytz.UTC).isoformat()
 
-        # DX country, continent, zones etc. from callsign. CQ/ITU zone are better looked up with a location but we don't
-        # have a real location for alerts.
-        if self.dx_calls and self.dx_calls[0] and not self.dx_country:
-            self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_calls[0], credentials)
-        if self.dx_calls and self.dx_calls[0] and not self.dx_continent:
-            self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_calls[0], credentials)
-        if self.dx_calls and self.dx_calls[0] and not self.dx_cq_zone:
-            self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_calls[0], credentials)
-        if self.dx_calls and self.dx_calls[0] and not self.dx_itu_zone:
-            self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_calls[0], credentials)
-        if self.dx_calls and self.dx_calls[0] and not self.dx_dxcc_id:
-            self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_calls[0], credentials)
-        if self.dx_dxcc_id and not self.dx_flag:
-            self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id)
+            # DX country, continent, zones etc. from callsign. CQ/ITU zone are better looked up with a location but we don't
+            # have a real location for alerts.
+            if self.dx_calls and self.dx_calls[0] and not self.dx_country:
+                self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_calls[0], credentials)
+            if self.dx_calls and self.dx_calls[0] and not self.dx_continent:
+                self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_calls[0], credentials)
+            if self.dx_calls and self.dx_calls[0] and not self.dx_cq_zone:
+                self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_calls[0], credentials)
+            if self.dx_calls and self.dx_calls[0] and not self.dx_itu_zone:
+                self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_calls[0], credentials)
+            if self.dx_calls and self.dx_calls[0] and not self.dx_dxcc_id:
+                self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_calls[0], credentials)
+            if self.dx_dxcc_id and not self.dx_flag:
+                self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id)
 
-        # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference
-        # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs
-        # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths
-        if self.sig_refs and len(self.sig_refs) > 0:
-            for sig_ref in self.sig_refs:
-                populate_sig_ref_info(sig_ref)
+            # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference
+            # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs
+            # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths
+            if self.sig_refs and len(self.sig_refs) > 0:
+                for sig_ref in self.sig_refs:
+                    populate_sig_ref_info(sig_ref)
 
-        # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG
-        # and apply it to the whole spot.
-        if self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0] and not self.sig:
-            self.sig = self.sig_refs[0].sig
+            # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG
+            # and apply it to the whole spot.
+            if self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0] and not self.sig:
+                self.sig = self.sig_refs[0].sig
 
-        # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
-        # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
-        # apart from that they were retrieved from the API at different times. Note that the simple Python hash()
-        # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
-        # use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
-        # The ID is computed before the online lookups below so that it is stable regardless of whether credentials
-        # are provided, allowing the enriched API response to be matched to the stored alert by ID.
-        if not self.id:
-            self_copy = copy.deepcopy(self)
-            self_copy.received_time = 0
-            self_copy.received_time_iso = ""
-            self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
+            # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
+            # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
+            # apart from that they were retrieved from the API at different times. Note that the simple Python hash()
+            # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
+            # use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
+            # The ID is computed before the online lookups below so that it is stable regardless of whether credentials
+            # are provided, allowing the enriched API response to be matched to the stored alert by ID.
+            if not self.id:
+                self_copy = copy.deepcopy(self)
+                self_copy.received_time = 0
+                self_copy.received_time_iso = ""
+                self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
 
-        # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data
-        # from the actual alerting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon
-        # instead of the one from the park reference they're at.
-        if self.dx_calls and not self.dx_names:
-            self.dx_names = list(
-                map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c, credentials), self.dx_calls))
+            # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data
+            # from the actual alerting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon
+            # instead of the one from the park reference they're at.
+            if self.dx_calls and not self.dx_names:
+                self.dx_names = list(
+                    map(lambda c: lookup_helper.infer_name_from_callsign_online_lookup(c, credentials), self.dx_calls))
+
+        except Exception as e:
+            logging.error("Exception while inferring missing data from spot", e, exc_info=True)
 
     def to_json(self):
         """JSON serialise"""

data/spot.py

@@ -147,295 +147,299 @@ class Spot:
     def infer_missing(self, credentials=None):
         """Infer missing parameters where possible"""
 
-        # If we somehow don't have a spot time, set it to zero so it sorts off the bottom of any list but
-        # clients can still reliably parse it as a number.
-        if not self.time:
-            self.time = 0
+        try:
+            # If we somehow don't have a spot time, set it to zero so it sorts off the bottom of any list but
+            # clients can still reliably parse it as a number.
+            if not self.time:
+                self.time = 0
 
-        # If we don't have a received time, this has just been received so set that to "now"
-        if not self.received_time:
-            self.received_time = datetime.now(pytz.UTC).timestamp()
+            # If we don't have a received time, this has just been received so set that to "now"
+            if not self.received_time:
+                self.received_time = datetime.now(pytz.UTC).timestamp()
 
-        # Fill in ISO versions of times, in case the client prefers that
-        if self.time and not self.time_iso:
-            self.time_iso = datetime.fromtimestamp(self.time, pytz.UTC).isoformat()
-        if self.received_time and not self.received_time_iso:
-            self.received_time_iso = datetime.fromtimestamp(self.received_time, pytz.UTC).isoformat()
+            # Fill in ISO versions of times, in case the client prefers that
+            if self.time and not self.time_iso:
+                self.time_iso = datetime.fromtimestamp(self.time, pytz.UTC).isoformat()
+            if self.received_time and not self.received_time_iso:
+                self.received_time_iso = datetime.fromtimestamp(self.received_time, pytz.UTC).isoformat()
 
-        # Clean up DX call if it has an SSID or -# from RBN
-        if self.dx_call and "-" in self.dx_call:
-            split = self.dx_call.split("-")
-            self.dx_call = split[0]
-            if len(split) > 1 and split[1] != "#":
-                self.dx_ssid = split[1]
+            # Clean up DX call if it has an SSID or -# from RBN
+            if self.dx_call and "-" in self.dx_call:
+                split = self.dx_call.split("-")
+                self.dx_call = split[0]
+                if len(split) > 1 and split[1] != "#":
+                    self.dx_ssid = split[1]
 
-        # DX country, continent etc. from callsign
-        if self.dx_call and not self.dx_country:
-            self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_call, credentials)
-        if self.dx_call and not self.dx_continent:
-            self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_call, credentials)
-        if self.dx_call and not self.dx_dxcc_id:
-            self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_call, credentials)
-        if self.dx_dxcc_id and not self.dx_flag:
-            self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id)
+            # DX country, continent etc. from callsign
+            if self.dx_call and not self.dx_country:
+                self.dx_country = lookup_helper.infer_country_from_callsign(self.dx_call, credentials)
+            if self.dx_call and not self.dx_continent:
+                self.dx_continent = lookup_helper.infer_continent_from_callsign(self.dx_call, credentials)
+            if self.dx_call and not self.dx_dxcc_id:
+                self.dx_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.dx_call, credentials)
+            if self.dx_dxcc_id and not self.dx_flag:
+                self.dx_flag = lookup_helper.get_flag_for_dxcc(self.dx_dxcc_id)
 
-        # Clean up spotter call if it has an SSID or -# from RBN
-        if self.de_call and "-" in self.de_call:
-            split = self.de_call.split("-")
-            self.de_call = split[0]
-            if len(split) > 1 and split[1] != "#":
-                self.de_ssid = split[1]
+            # Clean up spotter call if it has an SSID or -# from RBN
+            if self.de_call and "-" in self.de_call:
+                split = self.de_call.split("-")
+                self.de_call = split[0]
+                if len(split) > 1 and split[1] != "#":
+                    self.de_ssid = split[1]
 
-        # If we have a spotter of "RBNHOLE", we should have the actual spotter callsign in the comment, so extract it.
-        # RBNHole posts come from a number of providers, so it's dealt with here in the generic spot handling code.
-        if self.de_call == "RBNHOLE" and self.comment:
-            rbnhole_call_match = re.search(r"\Wat ([a-z0-9/]+)\W", self.comment, re.IGNORECASE)
-            if rbnhole_call_match:
-                self.de_call = rbnhole_call_match.group(1).upper()
+            # If we have a spotter of "RBNHOLE", we should have the actual spotter callsign in the comment, so extract it.
+            # RBNHole posts come from a number of providers, so it's dealt with here in the generic spot handling code.
+            if self.de_call == "RBNHOLE" and self.comment:
+                rbnhole_call_match = re.search(r"\Wat ([a-z0-9/]+)\W", self.comment, re.IGNORECASE)
+                if rbnhole_call_match:
+                    self.de_call = rbnhole_call_match.group(1).upper()
 
-        # If we have a spotter of "SOTAMAT", we might have the actual spotter callsign in the comment, if so extract it.
-        # SOTAMAT can do POTA as well as SOTA, so it's dealt with here in the generic spot handling code.
-        if self.de_call == "SOTAMAT" and self.comment:
-            sotamat_call_match = re.search(r"\Wfrom ([a-z0-9/]+)]", self.comment, re.IGNORECASE)
-            if sotamat_call_match:
-                self.de_call = sotamat_call_match.group(1).upper()
+            # If we have a spotter of "SOTAMAT", we might have the actual spotter callsign in the comment, if so extract it.
+            # SOTAMAT can do POTA as well as SOTA, so it's dealt with here in the generic spot handling code.
+            if self.de_call == "SOTAMAT" and self.comment:
+                sotamat_call_match = re.search(r"\Wfrom ([a-z0-9/]+)]", self.comment, re.IGNORECASE)
+                if sotamat_call_match:
+                    self.de_call = sotamat_call_match.group(1).upper()
 
-        # Spotter country, continent, zones etc. from callsign.
-        # DE call with no digits, or APRS servers starting "T2" are not things we can look up location for
-        if self.de_call and any(char.isdigit() for char in self.de_call) and not (
-                self.de_call.startswith("T2") and self.source == "APRS-IS"):
-            if not self.de_country:
-                self.de_country = lookup_helper.infer_country_from_callsign(self.de_call, credentials)
-            if not self.de_continent:
-                self.de_continent = lookup_helper.infer_continent_from_callsign(self.de_call, credentials)
-            if not self.de_dxcc_id:
-                self.de_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.de_call, credentials)
-            if self.de_dxcc_id and not self.de_flag:
-                self.de_flag = lookup_helper.get_flag_for_dxcc(self.de_dxcc_id)
+            # Spotter country, continent, zones etc. from callsign.
+            # DE call with no digits, or APRS servers starting "T2" are not things we can look up location for
+            if self.de_call and any(char.isdigit() for char in self.de_call) and not (
+                    self.de_call.startswith("T2") and self.source == "APRS-IS"):
+                if not self.de_country:
+                    self.de_country = lookup_helper.infer_country_from_callsign(self.de_call, credentials)
+                if not self.de_continent:
+                    self.de_continent = lookup_helper.infer_continent_from_callsign(self.de_call, credentials)
+                if not self.de_dxcc_id:
+                    self.de_dxcc_id = lookup_helper.infer_dxcc_id_from_callsign(self.de_call, credentials)
+                if self.de_dxcc_id and not self.de_flag:
+                    self.de_flag = lookup_helper.get_flag_for_dxcc(self.de_dxcc_id)
 
-        # Remove NaNs in frequency
-        if self.freq and self.freq == float("nan"):
-            self.freq = None
+            # Remove NaNs in frequency
+            if self.freq and self.freq == float("nan"):
+                self.freq = None
 
-        # Band from frequency
-        if self.freq and not self.band:
-            band = infer_band_from_freq(self.freq)
-            self.band = band.name
+            # Band from frequency
+            if self.freq and not self.band:
+                band = infer_band_from_freq(self.freq)
+                self.band = band.name
 
-        # Mode from comments or bandplan
-        if self.mode:
-            self.mode_source = "SPOT"
-        if self.comment and not self.mode:
-            self.mode = infer_mode_from_comment(self.comment)
-            self.mode_source = "COMMENT"
-        if self.freq and not self.mode:
-            self.mode = infer_mode_from_frequency(self.freq)
-            self.mode_source = "BANDPLAN"
+            # Mode from comments or bandplan
+            if self.mode:
+                self.mode_source = "SPOT"
+            if self.comment and not self.mode:
+                self.mode = infer_mode_from_comment(self.comment)
+                self.mode_source = "COMMENT"
+            if self.freq and not self.mode:
+                self.mode = infer_mode_from_frequency(self.freq)
+                self.mode_source = "BANDPLAN"
 
-        # Normalise mode if necessary.
-        if self.mode in MODE_ALIASES:
-            self.mode = MODE_ALIASES[self.mode]
+            # Normalise mode if necessary.
+            if self.mode in MODE_ALIASES:
+                self.mode = MODE_ALIASES[self.mode]
 
-        # Mode type from mode
-        if self.mode and not self.mode_type:
-            self.mode_type = infer_mode_type_from_mode(self.mode)
+            # Mode type from mode
+            if self.mode and not self.mode_type:
+                self.mode_type = infer_mode_type_from_mode(self.mode)
 
-        # If we have a latitude or grid at this point, it can only have been provided by the spot itself
-        if self.dx_latitude or self.dx_grid:
-            self.dx_location_source = "SPOT"
+            # If we have a latitude or grid at this point, it can only have been provided by the spot itself
+            if self.dx_latitude or self.dx_grid:
+                self.dx_location_source = "SPOT"
 
-        # Set the top-level "SIG" if it is missing but we have at least one SIG ref.
-        if not self.sig and self.sig_refs and len(self.sig_refs) > 0:
-            self.sig = self.sig_refs[0].sig.upper()
+            # Set the top-level "SIG" if it is missing but we have at least one SIG ref.
+            if not self.sig and self.sig_refs and len(self.sig_refs) > 0:
+                self.sig = self.sig_refs[0].sig.upper()
 
-        # See if we already have a SIG reference, but the comment looks like it contains more for the same SIG. This
-        # should catch e.g. POTA comments like "2-fer: GB-0001 GB-0002".
-        if self.comment and self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0].sig:
-            sig = self.sig_refs[0].sig.upper()
-            regex = get_ref_regex_for_sig(sig)
-            if regex:
-                all_comment_ref_matches = re.finditer(r"(^|\W)(" + regex + r")(^|\W)", self.comment, re.IGNORECASE)
-                for ref_match in all_comment_ref_matches:
-                    self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(2).upper(), sig=sig))
+            # See if we already have a SIG reference, but the comment looks like it contains more for the same SIG. This
+            # should catch e.g. POTA comments like "2-fer: GB-0001 GB-0002".
+            if self.comment and self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0].sig:
+                sig = self.sig_refs[0].sig.upper()
+                regex = get_ref_regex_for_sig(sig)
+                if regex:
+                    all_comment_ref_matches = re.finditer(r"(^|\W)(" + regex + r")(^|\W)", self.comment, re.IGNORECASE)
+                    for ref_match in all_comment_ref_matches:
+                        self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(2).upper(), sig=sig))
 
-        # See if the comment looks like it contains any SIGs (and optionally SIG references) that we can
-        # add to the spot. This should catch cluster spot comments like "POTA GB-0001 WWFF GFF-0001" and e.g. POTA
-        # comments like "also WWFF GFF-0001".
-        if self.comment:
-            sig_matches = re.finditer(r"(^|\W)" + ANY_SIG_REGEX + r"($|\W)", self.comment, re.IGNORECASE)
-            for sig_match in sig_matches:
-                # First of all, if we haven't got a SIG for this spot set yet, now we have. This covers things like cluster
-                # spots where the comment is just "POTA".
-                found_sig = get_sig_name_from_comment_name(sig_match.group(2))
-                if not self.sig:
-                    self.sig = found_sig
+            # See if the comment looks like it contains any SIGs (and optionally SIG references) that we can
+            # add to the spot. This should catch cluster spot comments like "POTA GB-0001 WWFF GFF-0001" and e.g. POTA
+            # comments like "also WWFF GFF-0001".
+            if self.comment:
+                sig_matches = re.finditer(r"(^|\W)" + ANY_SIG_REGEX + r"($|\W)", self.comment, re.IGNORECASE)
+                for sig_match in sig_matches:
+                    # First of all, if we haven't got a SIG for this spot set yet, now we have. This covers things like cluster
+                    # spots where the comment is just "POTA".
+                    found_sig = get_sig_name_from_comment_name(sig_match.group(2))
+                    if not self.sig:
+                        self.sig = found_sig
 
-                # Now look to see if that SIG name was followed by something that looks like a reference ID for that SIG.
-                # If so, add that to the sig_refs list for this spot.
-                ref_regex = get_ref_regex_for_sig(found_sig)
-                if ref_regex:
-                    ref_matches = re.finditer(r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)", self.comment,
-                                              re.IGNORECASE)
-                    for ref_match in ref_matches:
-                        self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig))
+                    # Now look to see if that SIG name was followed by something that looks like a reference ID for that SIG.
+                    # If so, add that to the sig_refs list for this spot.
+                    ref_regex = get_ref_regex_for_sig(found_sig)
+                    if ref_regex:
+                        ref_matches = re.finditer(r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)", self.comment,
+                                                  re.IGNORECASE)
+                        for ref_match in ref_matches:
+                            self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig))
 
-        # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference
-        # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs
-        # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths
-        if self.sig_refs and len(self.sig_refs) > 0:
-            for sig_ref in self.sig_refs:
-                sig_ref = populate_sig_ref_info(sig_ref)
-                # If the spot itself doesn't have location yet, but the SIG ref does, extract it
-                if sig_ref.grid and not self.dx_grid:
-                    self.dx_grid = sig_ref.grid
-                if sig_ref.latitude and not self.dx_latitude:
-                    self.dx_latitude = sig_ref.latitude
-                    self.dx_longitude = sig_ref.longitude
-                    if self.sig == "WAB" or self.sig == "WAI":
-                        self.dx_location_source = "WAB/WAI GRID"
-                    else:
-                        self.dx_location_source = "SIG REF LOOKUP"
+            # Fetch SIG data. In case a particular API doesn't provide a full set of name, lat, lon & grid for a reference
+            # in its initial call, we use this code to populate the rest of the data. This includes working out grid refs
+            # from WAB and WAI, which count as a SIG even though there's no real lookup, just maths
+            if self.sig_refs and len(self.sig_refs) > 0:
+                for sig_ref in self.sig_refs:
+                    sig_ref = populate_sig_ref_info(sig_ref)
+                    # If the spot itself doesn't have location yet, but the SIG ref does, extract it
+                    if sig_ref.grid and not self.dx_grid:
+                        self.dx_grid = sig_ref.grid
+                    if sig_ref.latitude and not self.dx_latitude:
+                        self.dx_latitude = sig_ref.latitude
+                        self.dx_longitude = sig_ref.longitude
+                        if self.sig == "WAB" or self.sig == "WAI":
+                            self.dx_location_source = "WAB/WAI GRID"
+                        else:
+                            self.dx_location_source = "SIG REF LOOKUP"
 
-        # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG
-        # and apply it to the whole spot.
-        if self.sig_refs and len(self.sig_refs) > 0 and not self.sig:
-            self.sig = self.sig_refs[0].sig
+            # If the spot itself doesn't have a SIG yet, but we have at least one SIG reference, take that reference's SIG
+            # and apply it to the whole spot.
+            if self.sig_refs and len(self.sig_refs) > 0 and not self.sig:
+                self.sig = self.sig_refs[0].sig
 
-        # Parse "de_grid<prop_mode>dx_grid" structures from the comment, e.g. "JN61ES(ES)JM56XT" or "JO02GQ<>KN17LG".
-        # These are common on cluster spots and can provide grid references in preference to e.g. QRZ lookup, as well as
-        # being the only source we have for propagation mode. Brace for nightmare regex from hell.
-        if self.comment:
-            grid_mode_grid_match = re.search(
-                r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
-                self.comment)
-            if grid_mode_grid_match:
-                # regex matches, so extract grids:
-                if not self.de_grid:
-                    self.de_grid = grid_mode_grid_match.group(1).upper()
-                if not self.dx_grid:
-                    self.dx_grid = grid_mode_grid_match.group(4).upper()
-                    self.dx_location_source = "SPOT"
+            # Parse "de_grid<prop_mode>dx_grid" structures from the comment, e.g. "JN61ES(ES)JM56XT" or "JO02GQ<>KN17LG".
+            # These are common on cluster spots and can provide grid references in preference to e.g. QRZ lookup, as well as
+            # being the only source we have for propagation mode. Brace for nightmare regex from hell.
+            if self.comment:
+                grid_mode_grid_match = re.search(
+                    r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
+                    self.comment)
+                if grid_mode_grid_match:
+                    # regex matches, so extract grids:
+                    if not self.de_grid:
+                        self.de_grid = grid_mode_grid_match.group(1).upper()
+                    if not self.dx_grid:
+                        self.dx_grid = grid_mode_grid_match.group(4).upper()
+                        self.dx_location_source = "SPOT"
 
-                # And extract propagation mode (group 2 for <...>, group 3 for (...)):
-                mode_tag = (grid_mode_grid_match.group(2) or grid_mode_grid_match.group(3) or "").upper()
-                if mode_tag and not self.propagation_mode:
-                    if mode_tag in PROPAGATION_MODES:
-                        self.propagation_mode = PROPAGATION_MODES[mode_tag]
-                    else:
-                        self.propagation_mode = mode_tag
-                        logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag)
+                    # And extract propagation mode (group 2 for <...>, group 3 for (...)):
+                    mode_tag = (grid_mode_grid_match.group(2) or grid_mode_grid_match.group(3) or "").upper()
+                    if mode_tag and not self.propagation_mode:
+                        if mode_tag in PROPAGATION_MODES:
+                            self.propagation_mode = PROPAGATION_MODES[mode_tag]
+                        else:
+                            self.propagation_mode = mode_tag
+                            logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag)
 
-        # DX Grid to lat/lon and vice versa in case one is missing
-        if self.dx_grid and not self.dx_latitude:
-            try:
-                ll = locator_to_latlong(self.dx_grid)
-                self.dx_latitude = ll[0]
-                self.dx_longitude = ll[1]
-            except:
-                logging.debug("Invalid grid received for spot")
-        if self.dx_latitude and self.dx_longitude and not self.dx_grid:
-            try:
-                self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8)
-            except:
-                logging.debug("Invalid lat/lon received for spot")
+            # DX Grid to lat/lon and vice versa in case one is missing
+            if self.dx_grid and not self.dx_latitude:
+                try:
+                    ll = locator_to_latlong(self.dx_grid)
+                    self.dx_latitude = ll[0]
+                    self.dx_longitude = ll[1]
+                except:
+                    logging.debug("Invalid grid received for spot")
+            if self.dx_latitude and self.dx_longitude and not self.dx_grid:
+                try:
+                    self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8)
+                except:
+                    logging.debug("Invalid lat/lon received for spot")
 
-        # QRT comment detection
-        if self.comment and not self.qrt:
-            self.qrt = "QRT" in self.comment.upper()
+            # QRT comment detection
+            if self.comment and not self.qrt:
+                self.qrt = "QRT" in self.comment.upper()
 
-        # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
-        # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
-        # apart from that they were retrieved from the API at different times. Note that the simple Python hash()
-        # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
-        # use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
-        # The ID is computed before the online lookups below so that it is stable regardless of whether credentials
-        # are provided, allowing the enriched API response to be matched to the stored spot by ID.
-        if not self.id:
-            self_copy = copy.deepcopy(self)
-            self_copy.received_time = 0
-            self_copy.received_time_iso = ""
-            self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
+            # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
+            # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
+            # apart from that they were retrieved from the API at different times. Note that the simple Python hash()
+            # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
+            # use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
+            # The ID is computed before the online lookups below so that it is stable regardless of whether credentials
+            # are provided, allowing the enriched API response to be matched to the stored spot by ID.
+            if not self.id:
+                self_copy = copy.deepcopy(self)
+                self_copy.received_time = 0
+                self_copy.received_time_iso = ""
+                self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
 
-        # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data
-        # from the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon
-        # instead of the one from the park reference they're at.
-        if self.dx_call and not self.dx_name:
-            self.dx_name = lookup_helper.infer_name_from_callsign_online_lookup(self.dx_call, credentials)
-        if self.dx_call and not self.dx_latitude:
-            latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.dx_call, credentials)
-            if latlon:
-                self.dx_latitude = latlon[0]
-                self.dx_longitude = latlon[1]
-            self.dx_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.dx_call, credentials)
-            self.dx_location_source = "HOME QTH"
-
-        # Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable string,
-        # otherwise see what they have set on an online lookup service.
-        if self.sig_refs and len(self.sig_refs) > 0:
-            qth = self.sig_refs[0].id
-            if self.sig_refs[0].name:
-                qth += " " + self.sig_refs[0].name
-            self.dx_qth = qth
-        else:
-            self.dx_qth = lookup_helper.infer_qth_from_callsign_online_lookup(self.dx_call, credentials)
-
-        # Last resort for getting a DX position, use the DXCC entity.
-        if self.dx_call and not self.dx_latitude:
-            latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.dx_call)
-            if latlon:
-                self.dx_latitude = latlon[0]
-                self.dx_longitude = latlon[1]
-                self.dx_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.dx_call)
-                self.dx_location_source = "DXCC"
-
-        # It looks like we can sometimes get a string into lat/lon, so try to parse as float, reject if not valid
-        if isinstance(self.dx_latitude, str) or isinstance(self.dx_longitude, str):
-            try:
-                self.dx_latitude = float(str(self.dx_latitude))
-                self.dx_longitude = float(str(self.dx_longitude))
-            except (TypeError, ValueError):
-                logging.warning("Received non-numeric strings in lat/lon (" + str(self.dx_latitude) + ", " + str(
-                    self.dx_longitude) + ") for call " + str(self.dx_call) + ", rejecting it")
-                self.dx_latitude = None
-                self.dx_longitude = None
-
-        # CQ and ITU zone lookup, preferably from location but failing that, from callsign
-        if not self.dx_cq_zone:
-            if self.dx_latitude:
-                self.dx_cq_zone = lat_lon_to_cq_zone(self.dx_latitude, self.dx_longitude)
-            elif self.dx_call:
-                self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_call, credentials)
-        if not self.dx_itu_zone:
-            if self.dx_latitude:
-                self.dx_itu_zone = lat_lon_to_itu_zone(self.dx_latitude, self.dx_longitude)
-            elif self.dx_call:
-                self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_call, credentials)
-
-        # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
-        # is likely at home.
-        self.dx_location_good = bool(self.dx_latitude and self.dx_longitude and (
-                self.dx_location_source == "SPOT" or self.dx_location_source == "SIG REF LOOKUP"
-                or self.dx_location_source == "WAB/WAI GRID"
-                or (self.dx_location_source == "HOME QTH" and "/" not in (self.dx_call or ""))))
-
-        # DE with no digits and APRS servers starting "T2" are not things we can look up location for
-        if self.de_call and any(char.isdigit() for char in self.de_call) and not (
-                self.de_call.startswith("T2") and self.source == "APRS-IS"):
-            # DE operator position lookup, using QRZ.com/HamQTH.
-            if not self.de_latitude:
-                latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.de_call, credentials)
+            # DX operator details lookup, using QRZ.com/HamQTH. This should be the last resort compared to taking the data
+            # from the actual spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon
+            # instead of the one from the park reference they're at.
+            if self.dx_call and not self.dx_name:
+                self.dx_name = lookup_helper.infer_name_from_callsign_online_lookup(self.dx_call, credentials)
+            if self.dx_call and not self.dx_latitude:
+                latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.dx_call, credentials)
                 if latlon:
-                    self.de_latitude = latlon[0]
-                    self.de_longitude = latlon[1]
-                self.de_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.de_call, credentials)
+                    self.dx_latitude = latlon[0]
+                    self.dx_longitude = latlon[1]
+                self.dx_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.dx_call, credentials)
+                self.dx_location_source = "HOME QTH"
 
-            # Last resort for getting a DE position, use the DXCC entity.
-            if not self.de_latitude:
-                latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.de_call)
+            # Determine a "QTH" string. If we have a SIG ref, pick the first one and turn it into a suitable string,
+            # otherwise see what they have set on an online lookup service.
+            if self.sig_refs and len(self.sig_refs) > 0:
+                qth = self.sig_refs[0].id
+                if self.sig_refs[0].name:
+                    qth += " " + self.sig_refs[0].name
+                self.dx_qth = qth
+            else:
+                self.dx_qth = lookup_helper.infer_qth_from_callsign_online_lookup(self.dx_call, credentials)
+
+            # Last resort for getting a DX position, use the DXCC entity.
+            if self.dx_call and not self.dx_latitude:
+                latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.dx_call)
                 if latlon:
-                    self.de_latitude = latlon[0]
-                    self.de_longitude = latlon[1]
-                    self.de_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.de_call)
+                    self.dx_latitude = latlon[0]
+                    self.dx_longitude = latlon[1]
+                    self.dx_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.dx_call)
+                    self.dx_location_source = "DXCC"
+
+            # It looks like we can sometimes get a string into lat/lon, so try to parse as float, reject if not valid
+            if isinstance(self.dx_latitude, str) or isinstance(self.dx_longitude, str):
+                try:
+                    self.dx_latitude = float(str(self.dx_latitude))
+                    self.dx_longitude = float(str(self.dx_longitude))
+                except (TypeError, ValueError):
+                    logging.warning("Received non-numeric strings in lat/lon (" + str(self.dx_latitude) + ", " + str(
+                        self.dx_longitude) + ") for call " + str(self.dx_call) + ", rejecting it")
+                    self.dx_latitude = None
+                    self.dx_longitude = None
+
+            # CQ and ITU zone lookup, preferably from location but failing that, from callsign
+            if not self.dx_cq_zone:
+                if self.dx_latitude:
+                    self.dx_cq_zone = lat_lon_to_cq_zone(self.dx_latitude, self.dx_longitude)
+                elif self.dx_call:
+                    self.dx_cq_zone = lookup_helper.infer_cq_zone_from_callsign(self.dx_call, credentials)
+            if not self.dx_itu_zone:
+                if self.dx_latitude:
+                    self.dx_itu_zone = lat_lon_to_itu_zone(self.dx_latitude, self.dx_longitude)
+                elif self.dx_call:
+                    self.dx_itu_zone = lookup_helper.infer_itu_zone_from_callsign(self.dx_call, credentials)
+
+            # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
+            # is likely at home.
+            self.dx_location_good = bool(self.dx_latitude and self.dx_longitude and (
+                    self.dx_location_source == "SPOT" or self.dx_location_source == "SIG REF LOOKUP"
+                    or self.dx_location_source == "WAB/WAI GRID"
+                    or (self.dx_location_source == "HOME QTH" and "/" not in (self.dx_call or ""))))
+
+            # DE with no digits and APRS servers starting "T2" are not things we can look up location for
+            if self.de_call and any(char.isdigit() for char in self.de_call) and not (
+                    self.de_call.startswith("T2") and self.source == "APRS-IS"):
+                # DE operator position lookup, using QRZ.com/HamQTH.
+                if not self.de_latitude:
+                    latlon = lookup_helper.infer_latlon_from_callsign_online_lookup(self.de_call, credentials)
+                    if latlon:
+                        self.de_latitude = latlon[0]
+                        self.de_longitude = latlon[1]
+                    self.de_grid = lookup_helper.infer_grid_from_callsign_online_lookup(self.de_call, credentials)
+
+                # Last resort for getting a DE position, use the DXCC entity.
+                if not self.de_latitude:
+                    latlon = lookup_helper.infer_latlon_from_callsign_dxcc(self.de_call)
+                    if latlon:
+                        self.de_latitude = latlon[0]
+                        self.de_longitude = latlon[1]
+                        self.de_grid = lookup_helper.infer_grid_from_callsign_dxcc(self.de_call)
+
+        except Exception as e:
+            logging.error("Exception while inferring missing data from spot", e, exc_info=True)
 
     def to_json(self):
         """JSON serialise"""

server/webserver.py

@@ -72,7 +72,8 @@ class WebServer:
              {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts}),
             (r"/api/v1/alerts/stream", APIAlertsStreamHandler,
              {"sse_alert_broadcaster": self._alert_broadcaster, **handler_opts}),
-            (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._data_store.solar, **handler_opts}),
+            (r"/api/v1/solar", APISolarConditionsHandler, {"solar_conditions": self._data_store.solar_conditions,
+                                                           **handler_opts}),
             (r"/api/v1/dxstats", APIDxStatsHandler, {"spots": self._data_store.spots, **handler_opts}),
             (r"/api/v1/options", APIOptionsHandler, {"status_data": self._data_store.status_data, **handler_opts}),
             (r"/api/v1/status", APIStatusHandler, {"status_data": self._data_store.status_data, **handler_opts}),

sigrefdataproviders/arlhs.py

@@ -0,0 +1,30 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class ARLHS(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Amateur Radio Light House Society"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "ARLHS"
+    DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            if "ARLHS" in row:
+                ref_id = row["ARLHS"]
+                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                              "Latitude"] != "" else None,
+                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                              "Longitude"] != "" else None,
+                                          grid=row["Maidenhead Locator"])
+
+        return new_data

sigrefdataproviders/file_download_sig_ref_data_provider.py

@@ -3,11 +3,11 @@ from datetime import datetime
 from threading import Thread, Event
 
 import pytz
-import requests
 from requests import ReadTimeout
 from requests.exceptions import ConnectionError, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
+from core.url_data_cache import URL_DATA_CACHE
 from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider
 
 
@@ -15,6 +15,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
     """Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file."""
 
     def __init__(self, sig_name, provider_config, url, poll_interval):
+        """ Set up the provider, note poll_interval is in *days*."""
         super().__init__(sig_name, provider_config)
         self._url = url
         self._poll_interval = poll_interval
@@ -25,7 +26,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info(
-            "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " seconds.")
+            "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " days.")
         self._thread = Thread(target=self._run, daemon=True)
         self._thread.start()
 
@@ -35,14 +36,15 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
     def _run(self):
         while True:
             self._poll()
-            if self._stop_event.wait(timeout=self._poll_interval):
+            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
                 break
 
     def _poll(self):
         try:
-            # Request data from API
+            # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+            # caching, but just so continual restarts of the software during testing don't hammer the servers.
             logging.debug("Downloading " + self.sig_name + " SIG ref data...")
-            http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            http_response = URL_DATA_CACHE.get(self._url, headers=HTTP_HEADERS)
             # Check response code was good
             if http_response.ok:
                 # Pass off to the subclass for processing

sigrefdataproviders/gma.py

@@ -0,0 +1,29 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class GMA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Global Mountain Activity"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "GMA"
+    DATA_URL = "https://www.gma.rocks/download/summits.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["Reference"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                          "Latitude"] != "" else None,
+                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                          "Longitude"] != "" else None,
+                                      grid=row["Maidenhead Locator"])
+
+        return new_data

sigrefdataproviders/illw.py

@@ -0,0 +1,30 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class ILLW(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for International Lighthouse & Lightship Weekend"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "ILLW"
+    DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            if "ILLW" in row:
+                ref_id = row["ILLW"]
+                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                              "Latitude"] != "" else None,
+                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                              "Longitude"] != "" else None,
+                                          grid=row["Maidenhead Locator"])
+
+        return new_data

sigrefdataproviders/iota.py

@@ -0,0 +1,36 @@
+import logging
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class IOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Islands on the Air"""
+
+    POLL_INTERVAL_DAYS = 365
+    SIG = "IOTA"
+    DATA_URL = "https://www.iota-world.org/islands-on-the-air/downloads/download-file.html?path=groups.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data:
+                ref_id = ref["refno"]
+                latitude = float(ref["latitude_min"]) + float(ref["latitude_max"]) / 2.0
+                longitude = float(ref["longitude_min"]) + float(ref["longitude_max"]) / 2.0
+                grid = None
+                try:
+                    grid = latlong_to_locator(latitude, longitude, 6)
+                except ValueError:
+                    logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude)
+
+                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
+                                          grid=grid, latitude=latitude, longitude=longitude)
+
+        return new_data

sigrefdataproviders/llota.py

@@ -7,12 +7,12 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload
 class LLOTA(FileDownloadSIGRefDataProvider):
     """SIG ref data provider for Lagos y Lagunas on the Air"""
 
-    POLL_INTERVAL_SEC = 7 * 24 * 60 * 60  # 7 days
+    POLL_INTERVAL_DAYS = 7
     SIG = "LLOTA"
     DATA_URL = "https://llota.app/api/public/references"
 
     def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
         new_data = {}

sigrefdataproviders/local_file_sig_ref_data_provider.py

@@ -14,7 +14,7 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
         self._path = path
 
     def start(self):
-        logging.info("Loading " + self.sig_name + " SIG ref data from file.")
+        logging.debug("Loading " + self.sig_name + " SIG ref data from file.")
         try:
             new_data = self._file_to_data(self._path)
             if new_data:

sigrefdataproviders/mota.py

@@ -0,0 +1,29 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class MOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Mills on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "MOTA"
+    DATA_URL = "https://www.gma.rocks/download/mills.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["Reference"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                          "Latitude"] != "" else None,
+                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                          "Longitude"] != "" else None,
+                                      grid=row["Maidenhead Locator"])
+
+        return new_data

sigrefdataproviders/pota.py

@@ -0,0 +1,29 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class POTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Parks on the Air"""
+
+    POLL_INTERVAL_DAYS = 7
+    SIG = "POTA"
+    DATA_URL = "https://pota.app/all_parks_ext.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["reference"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
+                                      url="https://pota.app/#/park/" + ref_id,
+                                      grid=row["grid"] if "grid" in row else None,
+                                      latitude=float(row["latitude"]) if "latitude" in row and row[
+                                          "latitude"] != "" else None,
+                                      longitude=float(row["longitude"]) if "longitude" in row and row[
+                                          "longitude"] != "" else None)
+
+        return new_data

sigrefdataproviders/sig_ref_data_provider.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 
 import pytz
@@ -16,6 +17,7 @@ class SIGRefDataProvider:
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
+        self.reference_count = 0
 
         # Create an empty dict to store data if one doesn't already exist
         if not sig_name in DATA_STORE.sigrefs:
@@ -38,3 +40,5 @@ class SIGRefDataProvider:
         objects."""
 
         DATA_STORE.sigrefs[self.sig_name] = new_data
+        self.reference_count = len(new_data)
+        logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)

sigrefdataproviders/siota.py

@@ -7,16 +7,16 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload
 class SIOTA(FileDownloadSIGRefDataProvider):
     """SIG ref data provider for Silos on the Air"""
 
-    POLL_INTERVAL_SEC = 30 * 24 * 60 * 60  # 30 days
+    POLL_INTERVAL_DAYS = 30
     SIG = "SIOTA"
     DATA_URL = "https://www.silosontheair.com/data/silos.csv"
 
     def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
         new_data = {}
-        for row in csv.DictReader(http_response.content.decode().splitlines()):
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["SILO_CODE"]
             new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
                                       grid=row["LOCATOR"] if "LOCATOR" in row else None,

sigrefdataproviders/sota.py

@@ -0,0 +1,33 @@
+import csv
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class SOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Summits on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "SOTA"
+    DATA_URL = "https://storage.sota.org.uk/summitslist.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["SummitCode"]
+            latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
+            longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
+                                      url="https://www.sotadata.org.uk/en/summit/" + ref_id,
+                                      latitude=latitude,
+                                      longitude=longitude,
+                                      activation_score=int(row["Points"]) if "Points" in row else None)
+            if latitude and longitude:
+                new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
+
+        return new_data

sigrefdataproviders/toilets.py

@@ -4,11 +4,11 @@ from data.sig_ref import SIGRef
 from sigrefdataproviders.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
 
 
-class TOTA(LocalFileSIGRefDataProvider):
+class Toilets(LocalFileSIGRefDataProvider):
     """SIG ref data provider for Toilets on the Air"""
 
-    SIG = "TOTA"
-    PATH = "datafiles/tota.csv"
+    SIG = "Toilets"
+    PATH = "datafiles/toilets.csv"
 
     def __init__(self, provider_config):
         super().__init__(self.SIG, provider_config, self.PATH)

sigrefdataproviders/towers.py

@@ -0,0 +1,27 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class Towers(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Towers on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "Towers"
+    DATA_URL = "https://wwtota.com/servis/generate_csv.php?ref=&filter=all"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
+            ref_id = row["Ref"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
+                                      url="https://wwtota.com/seznam/karta_rozhledny.php?ref=" + ref_id,
+                                      grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
+                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
+                                      longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None)
+
+        return new_data

sigrefdataproviders/wca.py

@@ -0,0 +1,44 @@
+import csv
+import logging
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WCA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for World Castles Award"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "WCA"
+    DATA_URL = "https://polo.ham2k.com/data/activities/wca/all-castles.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["REF"]
+
+            coordsStr = row["COORDINATES"]
+            latitude = None
+            longitude = None
+            grid = None
+            try:
+                if coordsStr:
+                    split = coordsStr.split(", ")
+                    latitude = float(split[0])
+                    longitude = float(split[1])
+                    grid = latlong_to_locator(latitude, longitude)
+            except ValueError:
+                logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id)
+
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=latitude,
+                                      longitude=longitude,
+                                      grid=grid)
+
+        return new_data

sigrefdataproviders/wota.py

@@ -5,12 +5,12 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload
 class WOTA(FileDownloadSIGRefDataProvider):
     """SIG ref data provider for Wainwrights on the Air"""
 
-    POLL_INTERVAL_SEC = 365 * 24 * 60 * 60  # 365 days
+    POLL_INTERVAL_DAYS = 365
     SIG = "WOTA"
     DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json"
 
     def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
         new_data = {}

sigrefdataproviders/wwbota.py

@@ -0,0 +1,27 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WWBOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Worldwide Bunkers on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "WWBOTA"
+    DATA_URL = "https://api.wwbota.org/bunkers/?format=CSV"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["Reference"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None,
+                                      grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
+                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
+                                      longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None)
+
+        return new_data

sigrefdataproviders/wwff.py

@@ -7,16 +7,16 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload
 class WWFF(FileDownloadSIGRefDataProvider):
     """SIG ref data provider for Worldwide Flora & Fauna"""
 
-    POLL_INTERVAL_SEC = 30 * 24 * 60 * 60  # 30 days
+    POLL_INTERVAL_DAYS = 30
     SIG = "WWFF"
     DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv"
 
     def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
         new_data = {}
-        for row in csv.DictReader(http_response.content.decode().splitlines()):
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["reference"]
             new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
                                       url="https://wwff.co/directory/?showRef=" + ref_id,

sigrefdataproviders/zlota.py

@@ -7,12 +7,12 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload
 class ZLOTA(FileDownloadSIGRefDataProvider):
     """SIG ref data provider for New Zealand on the Air"""
 
-    POLL_INTERVAL_SEC = 30 * 24 * 60 * 60  # 30 days
+    POLL_INTERVAL_DAYS = 30
     SIG = "ZLOTA"
     DATA_URL = "https://ontheair.nz/assets/assets.json"
 
     def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
         new_data = {}

spotproviders/towers.py

@@ -30,8 +30,8 @@ class Towers(HTTPSpotProvider):
                         dx_call=source_spot["call"].upper(),
                         freq=likely_freq,
                         comment=source_spot["comment"],
-                        sig="WWTOTA",
-                        sig_refs=[SIGRef(id=source_spot["ref"], sig="WWTOTA")],
+                        sig="Towers",
+                        sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
                         time=datetime.strptime(response_json["updated"][:10] + source_spot["time"],
                                                "%Y-%m-%d%H:%M").timestamp())
 

spotproviders/wota.py

@@ -25,7 +25,7 @@ class WOTA(HTTPSpotProvider):
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
-        rss = cast(RSS, Parser.parse(http_response.content.decode()))
+        rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
         # Iterate through source data
         for source_spot in rss.channel.items:
 

spotproviders/xota.py

@@ -11,7 +11,7 @@ from spotproviders.websocket_spot_provider import WebsocketSpotProvider
 class XOTA(WebsocketSpotProvider):
     """Spot provider for servers based on the "xOTA" software at https://github.com/nischu/xOTA/
     The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG which we can
-    then use for lookups. This functionality is implemented for TOTA events, of which there are
+    then use for lookups. This functionality is implemented for Toilets on the Air events, of which there are
     several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA, which is why we also
     provide a sig_ref_prefix in our config. This is applied to the reference ID, so e.g. "T-01" at C3 might become
     "C3 T-01". This allows us to provide location lookups for TOTA at several conferences."""

static/apidocs/openapi.yml

@@ -796,7 +796,7 @@ components:
         - HEMA
         - WCA
         - MOTA
-        - SiOTA
+        - SIOTA
         - ARLHS
         - ILLW
         - ZLOTA
@@ -1742,6 +1742,10 @@ components:
             The last time at which this provider received data, UTC seconds since UNIX epoch. If this
             is zero, the provider has never updated.
           example: 1759579508
+        reference_count:
+          type: number
+          description: The number of references fetched using this provider.
+          example: 1234
 
     SpotList:
       type: array

static/js/status.js

@@ -46,6 +46,7 @@ function loadStatus() {
                     <div class="col"><strong>${p["sig_name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
+                    <div class="col">Reference count: ${(p["enabled"] && p["reference_count"] > 0) ? p["reference_count"] : "N/A"}</div>
                 </div>`);
         });
     });