Refactor of caching & data storage part 8 #118

config-example.yml

@@ -166,6 +166,19 @@ alert-providers:
     enabled: true
 
 
+# Static reference data providers to use. This allows Spothole to download data such as mapping between callsign
+# prefixes and DXCC entities.
+static-data-providers:
+  - class: "K0SWE"
+    enabled: true
+
+  - class: "CQZoneData"
+    enabled: true
+
+  - class: "ITUZoneData"
+    enabled: true
+
+
 # SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF
 # park IDs to their name and location.
 sig-ref-data-providers:

core/call_lookup_helper.py

@@ -1,7 +1,5 @@
 import gzip
-import json
 import logging
-import re
 import urllib.parse
 from datetime import timedelta
 
@@ -9,14 +7,13 @@ import xmltodict
 from diskcache import Cache
 from pyhamtools import LookupLib, Callinfo, callinfo
 from pyhamtools.exceptions import APIKeyMissingError
-from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 from requests_cache import CachedSession
 
 from core.config import config
-from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
-    HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
+from core.constants import HTTP_HEADERS, HAMQTH_PRG
+from core.data_store import DATA_STORE
 from core.url_data_cache import URLDataCache
 
 # QRZ XML field names differ from pyhamtools' normalised names; map them here.
@@ -84,7 +81,6 @@ class LookupHelper:
         self._lookup_lib_basic = None
         self._country_files_cty_plist_download_location = None
         self._dxcc_json_download_location = None
-        self._dxcc_data = None
 
     def start(self):
         # Lookup helpers from pyhamtools. We use five (!) of these. The simplest is country-files.com, which downloads
@@ -118,22 +114,6 @@ class LookupHelper:
                                                          filename=self._clublog_xml_download_location)
         self._clublog_callsign_data_cache = Cache('cache/clublog_callsign_lookup_cache')
 
-        # We also get a lookup of DXCC data from K0SWE to use for additional lookups of e.g. flags.
-        self._dxcc_json_download_location = "cache/dxcc.json"
-        success = self._download_dxcc_json()
-        if success:
-            with open(self._dxcc_json_download_location) as f:
-                tmp_dxcc_data = json.load(f)["dxcc"]
-                # Reformat as a map for faster lookup
-                self._dxcc_data = {}
-                for dxcc in tmp_dxcc_data:
-                    self._dxcc_data[dxcc["entityCode"]] = dxcc
-        else:
-            logging.error("Could not download DXCC data, flags and similar data may be missing!")
-
-        # Precompile regex matches for DXCCs to improve efficiency when iterating through them
-        for dxcc in (self._dxcc_data.values() if self._dxcc_data else []):
-            dxcc["_prefixRegexCompiled"] = re.compile(dxcc["prefixRegex"])
 
     def _download_country_files_cty_plist(self):
         """Download the cty.plist file from country-files.com on first startup. The pyhamtools lib can actually download and use
@@ -163,31 +143,6 @@ class LookupHelper:
             logging.error("Exception when downloading Clublog cty.xml", e)
             return False
 
-    def _download_dxcc_json(self):
-        """Download the dxcc.json file on first startup."""
-
-        try:
-            logging.info("Downloading dxcc.json...")
-            response = _URL_DATA_CACHE.get(
-                "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
-                headers=HTTP_HEADERS)
-
-            if response.ok:
-                with open(self._dxcc_json_download_location, "w") as f:
-                    f.write(response.text)
-                    f.flush()
-                    return True
-            else:
-                logging.warning(f"HTTP {response.status_code} when downloading dxcc.json.")
-                return False
-
-        except ConnectionError:
-            logging.warning(f"Connection error when downloading dxcc.json.")
-        except (ConnectTimeout, ReadTimeout):
-            logging.warning(f"Timeout when downloading dxcc.json.")
-        except Exception as e:
-            logging.error("Exception when downloading dxcc.json", e)
-            return False
 
     def _download_clublog_ctyxml(self):
         """Download the cty.xml (gzipped) file from Clublog on first startup, so we can use it in preference to querying the
@@ -374,7 +329,8 @@ class LookupHelper:
     def get_flag_for_dxcc(self, dxcc):
         """Get an emoji flag for a given DXCC entity ID"""
 
-        return self._dxcc_data[dxcc]["flag"] if dxcc in self._dxcc_data else None
+        dxcc_data = DATA_STORE.dxcc_data[dxcc] if dxcc in DATA_STORE.dxcc_data else None
+        return dxcc_data["flag"] if dxcc_data else None
 
     def infer_name_from_callsign_online_lookup(self, call, credentials=None):
         """Infer an operator name from a callsign (requires QRZ.com/HamQTH)"""
@@ -665,9 +621,9 @@ class LookupHelper:
     def _get_dxcc_data_for_callsign(self, call) -> dict | None:
         """Utility method to get generic DXCC data from our lookup table, if we can find it"""
 
-        for entry in self._dxcc_data.values():
-            if entry["_prefixRegexCompiled"].match(call):
-                return entry
+        for pattern, entity_code in DATA_STORE.dxcc_lookup_by_call_regex:
+            if pattern.match(call):
+                return DATA_STORE.dxcc_data[entity_code]
         return None
 
     def stop(self):
@@ -680,61 +636,3 @@ class LookupHelper:
 
 # Singleton object
 lookup_helper = LookupHelper()
-
-
-def infer_mode_from_comment(comment):
-    """Infer a mode from the comment"""
-
-    for mode in ALL_MODES:
-        if mode in comment.upper():
-            return mode
-    for mode in MODE_ALIASES.keys():
-        if mode in comment.upper():
-            return MODE_ALIASES[mode]
-    return None
-
-
-def infer_mode_type_from_mode(mode):
-    """Infer a "mode family" from a mode."""
-
-    if mode.upper() in CW_MODES:
-        return "CW"
-    elif mode.upper() in PHONE_MODES:
-        return "PHONE"
-    elif mode.upper() in DATA_MODES:
-        return "DATA"
-    else:
-        if mode.upper() != "OTHER":
-            logging.warning("Found an unrecognised mode: " + mode + ". Developer should categorise this.")
-        return None
-
-
-def infer_band_from_freq(freq):
-    """Infer a band from a frequency in Hz"""
-
-    for b in BANDS:
-        if b.start_freq <= freq <= b.end_freq:
-            return b
-    return UNKNOWN_BAND
-
-
-def infer_mode_from_frequency(freq):
-    """Infer a mode from the frequency (in Hz) according to the band plan. Just a guess really."""
-
-    try:
-        khz = freq / 1000.0
-        mode = freq_to_band(khz)["mode"]
-        # Some additional common digimode ranges in addition to what the 3rd-party freq_to_band function returns.
-        # This is mostly here just because freq_to_band is very specific about things like FT8 frequencies, and e.g.
-        # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
-        # might include other common digimode centres of activity here, but this achieves the main goal of keeping
-        # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
-        if (7074 <= khz < 7077) or (10136 <= khz < 10139) or (14074 <= khz < 14077) or (18100 <= khz < 18103) or (
-                21074 <= khz < 21077) or (24915 <= khz < 24918) or (28074 <= khz < 28077):
-            mode = "FT8"
-        if (7047.5 <= khz < 7050.5) or (10140 <= khz < 10143) or (14080 <= khz < 14083) or (
-                18104 <= khz < 18107) or (21140 <= khz < 21143) or (24919 <= khz < 24922) or (28180 <= khz < 28183):
-            mode = "FT4"
-        return mode
-    except KeyError:
-        return None

core/config.py

@@ -60,6 +60,14 @@ def get_solar_conditions_provider_from_config(config_providers_entry):
     return provider_class(config_providers_entry)
 
 
+def get_static_data_provider_from_config(config_providers_entry):
+    """Utility method to get a static reference data provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('staticdataproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
 def get_sig_ref_data_provider_from_config(config_providers_entry):
     """Utility method to get a SIG reference data provider based on the class specified in its config entry."""
 

core/constants.py

@@ -26,7 +26,8 @@ SIGS = [
     SIG(name="ZLOTA", comment_names=["ZLOTA"], description="New Zealand on the Air", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}"),
     SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"),
     SIG(name="BOTA", comment_names=[], description="Beaches on the Air"),
-    SIG(name="KRMNPA", comment_names=["KRMNPA"], description="Keith Roget Memorial National Parks Award"),
+    SIG(name="KRMNPA", comment_names=["KRMNPA"], description="Keith Roget Memorial National Parks Award", ref_regex=r"VKFF\-\d{4}"),
+    SIG(name="SANPCPA", comment_names=["SANPCPA"], description="South Australian National Parks and Conservation Parks Award", ref_regex=r"VKFF\-\d{4}"),
     SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"LL[A-Z]{2}\-\d{4}"),
     SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2,3}R\-\d{4}"),
     SIG(name="Tiles", comment_names=[], description="Tiles on the Air", ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}"),

core/data_store.py

@@ -1,4 +1,5 @@
 import logging
+import re
 from pathlib import Path
 
 import diskcache
@@ -15,18 +16,26 @@ class DataStore:
     lookup data using different caching strategies for each."""
 
     def __init__(self):
+        # Constants
         self._MAX_SPOT_COUNT = 100000
         self._MAX_ALERT_COUNT = 100000
         self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
         self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
+        # Caches
         self.alerts = None
         self.spots = None
         self.callsigns = None
+        self.dxcc_data = None
+        self.dxcc_lookup_by_call_regex = []
         self.sigrefs = None
         self.status_data = None
         self._status = None
         self.solar_conditions = None
         self._solar = None
+        # ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be
+        # caches, they can just be straight objects
+        self.cq_zone_data = None
+        self.itu_zone_data = None
 
     def setup(self):
         Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
@@ -42,10 +51,14 @@ class DataStore:
             self._status.add("status_data", {})
         self.status_data = self._status.get("status_data")
 
-        # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
-        # but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
-        # and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
-        # so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
+        # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
+        # a regular basis but there's no need for a TTL since old data is better than no data.
+        self.dxcc_data = diskcache.Cache(CACHE_DIR + "dxcc_data")
+        self.regenerateCallRegexToDXCCEntityMap()
+
+        # For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers
+        # of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref"
+        # syntax for keys to keep it a single level.
         self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs")
         logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
 
@@ -68,11 +81,21 @@ class DataStore:
                                     snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
         logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))
 
+    def regenerateCallRegexToDXCCEntityMap(self):
+        """DXCC entity data from K0SWE includes a regex which we can use to match a callsign, and determine which DXCC
+        entity it belongs to. But getting every DXCC entity data object out of DiskCache, iterating, compiling its regex
+        and testing the callsign every time is expensive. So instead we build a separate in-memory lookup of compiled
+        regex against DXCC entity code, as a list of tuples we can iterate through."""
+
+        for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]:
+            self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))
+
     def close(self):
         self.spots.close()
         self.alerts.close()
         self._solar.close()
         self._status.close()
+        self.dxcc_data.close()
         self.sigrefs.close()
         self.callsigns.close()
 

core/geo_utils.py

@@ -1,66 +1,58 @@
-import json
 import logging
 import re
 from math import floor
 
-import geopandas
 from pyproj import Transformer
-from shapely import prepare
 from shapely.geometry import Point, Polygon
 
+from core.data_store import DATA_STORE
+
 TRANSFORMER_OS_GRID_TO_WGS84 = Transformer.from_crs("EPSG:27700", "EPSG:4326")
 TRANSFORMER_IRISH_GRID_TO_WGS84 = Transformer.from_crs("EPSG:29903", "EPSG:4326")
 TRANSFORMER_CI_UTM_GRID_TO_WGS84 = Transformer.from_crs("+proj=utm +zone=30 +ellps=WGS84", "EPSG:4326")
 
-with open("datafiles/cqzones.geojson") as f:
-    cq_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
-with open("datafiles/ituzones.geojson") as f:
-    itu_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
-for idx in cq_zone_data.index:
-    prepare(cq_zone_data.at[idx, 'geometry'])
-for idx in itu_zone_data.index:
-    prepare(itu_zone_data.at[idx, 'geometry'])
-
 
 def lat_lon_to_cq_zone(lat, lon):
     """Finds out which CQ zone a lat/lon point is in."""
 
-    lon = ((lon + 180) % 360) - 180
-    for index, row in cq_zone_data.iterrows():
-        polygon = Polygon(row["geometry"])
-        test_point = Point(lon, lat)
-        if polygon.contains(test_point):
-            return int(row["name"])
+    if DATA_STORE.cq_zone_data is not None:
+        lon = ((lon + 180) % 360) - 180
+        for index, row in DATA_STORE.cq_zone_data.iterrows():
+            polygon = Polygon(row["geometry"])
+            test_point = Point(lon, lat)
+            if polygon.contains(test_point):
+                return int(row["name"])
 
-        # Might have problems around the antemeridian, so if we didn't find a match, try offsetting the point by + or -
-        # 360 degrees longitude to try the other side of the Earth
-        if lon < 0:
-            test_point = Point(lon + 360, lat)
-        else:
-            test_point = Point(lon - 360, lat)
-        if polygon.contains(test_point):
-            return int(row["name"])
+            # Might have problems around the antemeridian, so if we didn't find a match, try offsetting the point by + or -
+            # 360 degrees longitude to try the other side of the Earth
+            if lon < 0:
+                test_point = Point(lon + 360, lat)
+            else:
+                test_point = Point(lon - 360, lat)
+            if polygon.contains(test_point):
+                return int(row["name"])
     return None
 
 
 def lat_lon_to_itu_zone(lat, lon):
     """Finds out which ITU zone a lat/lon point is in."""
 
-    lon = ((lon + 180) % 360) - 180
-    for index, row in itu_zone_data.iterrows():
-        polygon = Polygon(row["geometry"])
-        test_point = Point(lon, lat)
-        if polygon.contains(test_point):
-            return int(row["name"])
+    if DATA_STORE.itu_zone_data is not None:
+        lon = ((lon + 180) % 360) - 180
+        for index, row in DATA_STORE.itu_zone_data.iterrows():
+            polygon = Polygon(row["geometry"])
+            test_point = Point(lon, lat)
+            if polygon.contains(test_point):
+                return int(row["name"])
 
-        # Might have problems around the antemeridian, so if we didn't find a match, try offsetting the point by + or -
-        # 360 degrees longitude to try the other side of the Earth
-        if lon < 0:
-            test_point = Point(lon + 360, lat)
-        else:
-            test_point = Point(lon - 360, lat)
-        if polygon.contains(test_point):
-            return int(row["name"])
+            # Might have problems around the antemeridian, so if we didn't find a match, try offsetting the point by + or -
+            # 360 degrees longitude to try the other side of the Earth
+            if lon < 0:
+                test_point = Point(lon + 360, lat)
+            else:
+                test_point = Point(lon - 360, lat)
+            if polygon.contains(test_point):
+                return int(row["name"])
     return None
 
 

core/sig_lookup_helper.py

@@ -0,0 +1,93 @@
+import logging
+
+from pyhamtools.locator import locator_to_latlong, latlong_to_locator
+
+from core.data_store import DATA_STORE
+from core.geo_utils import wab_wai_square_to_lat_lon
+
+
+def populate_sig_ref_info(sig_ref):
+    """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
+    which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes
+    use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
+    SIG we are getting data for.
+    Note there is currently no support for KRMNPA location lookup, see issue #61."""
+
+    if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "":
+        logging.debug("Failed to look up sig_ref info, sig or id were not set.")
+        return sig_ref
+
+    sig = sig_ref.sig
+    ref_id = sig_ref.id
+
+    try:
+        ### FUDGES ###
+        #
+        # DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all
+        # activators add leading zeros.
+        if sig.upper() == "DME":
+            ref_id = ref_id.zfill(5)
+
+        # KRMNPA & SANPCPA fudge. These don't have their own reference system, they just use VKFF references, so pretend
+        # the sig is WWFF and carry on
+        if sig.upper() == "KRMNPA" or sig.upper() == "SANPCPA":
+            sig = "WWFF"
+
+        ### SKIPS ###
+        #
+        # If the SIG is HEMA, we have no current lookup for this so just skip the lookup here.
+        if sig.upper() == "HEMA":
+            return sig_ref
+
+        ### PROGRAMMATIC DATA GENERATION INSTEAD OF LOOKUPS ###
+        #
+        # If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
+        # calculate all the information we are going to get directly.
+        if sig.upper() == "TILES":
+            # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
+            if not sig_ref.name:
+                sig_ref.name = sig_ref.id
+            if not sig_ref.grid:
+                sig_ref.grid = sig_ref.id
+            if sig_ref.grid and not sig_ref.latitude:
+                ll = locator_to_latlong(str(sig_ref.grid))
+                sig_ref.latitude = ll[0]
+                sig_ref.longitude = ll[1]
+            return sig_ref
+
+        elif sig.upper() == "WAB" or sig.upper() == "WAI":
+            ll = wab_wai_square_to_lat_lon(ref_id)
+            if ll:
+                sig_ref.name = ref_id
+                try:
+                    sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
+                    sig_ref.latitude = ll[0]
+                    sig_ref.longitude = ll[1]
+                except:
+                    logging.warning("Invalid lat/lon received for WAB/WAI reference")
+            return sig_ref
+
+        elif sig.upper() == "BOTA":
+            # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
+            if not sig_ref.name:
+                sig_ref.name = sig_ref.id
+            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
+            return sig_ref
+
+        ### ACTUAL LOOKUP ###
+        #
+        # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
+        # this SIG. If so, check for the reference data and use that.
+        key = sig + ":" + ref_id
+        lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
+        if lookup_data:
+            # Copy new sig ref data into existing object where data was previously missing
+            for key, value in lookup_data.__dict__.items():
+                if value is not None and sig_ref.__dict__.get(key) is None:
+                    sig_ref.__dict__[key] = value
+        else:
+            logging.warning("%s database did not contain data for ref %s", sig, ref_id)
+
+    except Exception:
+        logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
+    return sig_ref

core/sig_utils.py

@@ -1,10 +1,4 @@
-import logging
-
-from pyhamtools.locator import latlong_to_locator, locator_to_latlong
-
 from core.constants import SIGS
-from core.data_store import DATA_STORE
-from core.geo_utils import wab_wai_square_to_lat_lon
 
 
 def get_ref_regex_for_sig(sig):
@@ -26,78 +20,5 @@ def get_sig_name_from_comment_name(sig):
     return None
 
 
-def populate_sig_ref_info(sig_ref):
-    """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
-    which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes
-    use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
-    SIG we are getting data for.
-    Note there is currently no support for KRMNPA location lookup, see issue #61."""
-
-    if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "":
-        logging.debug("Failed to look up sig_ref info, sig or id were not set.")
-        return sig_ref
-
-    sig = sig_ref.sig
-    ref_id = sig_ref.id
-
-    # DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all activators
-    # add leading zeros.
-    if sig.upper() == "DME":
-        ref_id = ref_id.zfill(5)
-
-    try:
-        # If the SIG is HEMA or KRMNPA, we have no current lookup for this so just skip it.
-        if sig.upper() == "HEMA" or sig.upper() == "KRMNPA":
-            return sig_ref
-
-        # If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
-        # calculate all the information we are going to get directly. So handle those cases first
-        elif sig.upper() == "TILES":
-            # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
-            if not sig_ref.name:
-                sig_ref.name = sig_ref.id
-            if not sig_ref.grid:
-                sig_ref.grid = sig_ref.id
-            if sig_ref.grid and not sig_ref.latitude:
-                ll = locator_to_latlong(str(sig_ref.grid))
-                sig_ref.latitude = ll[0]
-                sig_ref.longitude = ll[1]
-
-        elif sig.upper() == "WAB" or sig.upper() == "WAI":
-            ll = wab_wai_square_to_lat_lon(ref_id)
-            if ll:
-                sig_ref.name = ref_id
-                try:
-                    sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
-                    sig_ref.latitude = ll[0]
-                    sig_ref.longitude = ll[1]
-                except:
-                    logging.warning("Invalid lat/lon received for WAB/WAI reference")
-
-        elif sig.upper() == "BOTA":
-            # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
-            if not sig_ref.name:
-                sig_ref.name = sig_ref.id
-            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
-
-        # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
-        # this SIG. If so, check for the reference data and use that.
-        elif sig in DATA_STORE.sigrefs:
-            key = sig + ":" + ref_id
-            lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
-            if lookup_data:
-                # Copy new sig ref data into existing object
-                sig_ref.__dict__.update(lookup_data.__dict__)
-            else:
-                logging.warning("%s database did not contain data for ref %s", sig, ref_id)
-
-        else:
-            logging.warning(f"Tried to look up a SIG called %s but Spothole does not know what that is.", sig)
-
-    except Exception:
-        logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
-    return sig_ref
-
-
 # Regex matching any SIG's "comment name", i.e. how it may be referred to in spot comments
 ANY_SIG_REGEX = r"(" + r"|".join(n for s in SIGS for n in s.comment_names) + r")"

core/status_reporter.py

@@ -15,7 +15,7 @@ class StatusReporter:
     """Provides a timed update of the application's status data."""
 
     def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers,
-                 sig_ref_data_providers):
+                 static_data_providers, sig_ref_data_providers):
         """Constructor"""
 
         self._run_interval = run_interval
@@ -23,6 +23,7 @@ class StatusReporter:
         self._spot_providers = spot_providers
         self._alert_providers = alert_providers
         self._solar_condition_providers = solar_condition_providers
+        self._static_data_providers = static_data_providers
         self._sig_ref_data_providers = sig_ref_data_providers
         self._thread = None
         self._stop_event = Event()
@@ -74,6 +75,11 @@ class StatusReporter:
                        "last_updated": p.last_update_time.replace(
                            tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
             self._solar_condition_providers))
+        DATA_STORE.status_data["static_data_providers"] = list(
+            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+                           "last_updated": p.last_update_time.replace(
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                self._static_data_providers))
         DATA_STORE.status_data["sig_ref_data_providers"] = list(
             map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(

core/utils.py

@@ -1,8 +1,71 @@
+import logging
+
 import simplejson
+from pyhamtools.frequency import freq_to_band
+
+from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES
 
 
 def safe_json_dumps(obj):
     """Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats
     which are invalid in JSON."""
 
-    return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
+    return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
+
+
+def infer_mode_from_comment(comment):
+    """Infer a mode from the comment"""
+
+    for mode in ALL_MODES:
+        if mode in comment.upper():
+            return mode
+    for mode in MODE_ALIASES.keys():
+        if mode in comment.upper():
+            return MODE_ALIASES[mode]
+    return None
+
+
+def infer_mode_type_from_mode(mode):
+    """Infer a "mode family" from a mode."""
+
+    if mode.upper() in CW_MODES:
+        return "CW"
+    elif mode.upper() in PHONE_MODES:
+        return "PHONE"
+    elif mode.upper() in DATA_MODES:
+        return "DATA"
+    else:
+        if mode.upper() != "OTHER":
+            logging.warning("Found an unrecognised mode: " + mode + ". Developer should categorise this.")
+        return None
+
+
+def infer_band_from_freq(freq):
+    """Infer a band from a frequency in Hz"""
+
+    for b in BANDS:
+        if b.start_freq <= freq <= b.end_freq:
+            return b
+    return UNKNOWN_BAND
+
+
+def infer_mode_from_frequency(freq):
+    """Infer a mode from the frequency (in Hz) according to the band plan. Just a guess really."""
+
+    try:
+        khz = freq / 1000.0
+        mode = freq_to_band(khz)["mode"]
+        # Some additional common digimode ranges in addition to what the 3rd-party freq_to_band function returns.
+        # This is mostly here just because freq_to_band is very specific about things like FT8 frequencies, and e.g.
+        # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
+        # might include other common digimode centres of activity here, but this achieves the main goal of keeping
+        # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
+        if (7074 <= khz < 7077) or (10136 <= khz < 10139) or (14074 <= khz < 14077) or (18100 <= khz < 18103) or (
+                21074 <= khz < 21077) or (24915 <= khz < 24918) or (28074 <= khz < 28077):
+            mode = "FT8"
+        if (7047.5 <= khz < 7050.5) or (10140 <= khz < 10143) or (14080 <= khz < 14083) or (
+                18104 <= khz < 18107) or (21140 <= khz < 21143) or (24919 <= khz < 24922) or (28180 <= khz < 28183):
+            mode = "FT4"
+        return mode
+    except KeyError:
+        return None

data/alert.py

@@ -7,8 +7,8 @@ from datetime import datetime, timedelta
 
 import pytz
 
-from core.lookup_helper import lookup_helper
-from core.sig_utils import populate_sig_ref_info
+from core.call_lookup_helper import lookup_helper
+from core.sig_lookup_helper import populate_sig_ref_info
 
 
 @dataclass

data/spot.py

@@ -12,9 +12,11 @@ from pyhamtools.locator import locator_to_latlong, latlong_to_locator
 from core.config import MAX_SPOT_AGE
 from core.constants import MODE_ALIASES, PROPAGATION_MODES
 from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
-from core.lookup_helper import lookup_helper, infer_band_from_freq, infer_mode_from_comment, \
+from core.call_lookup_helper import lookup_helper
+from core.sig_utils import ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name
+from core.sig_lookup_helper import populate_sig_ref_info
+from core.utils import infer_band_from_freq, infer_mode_from_comment, \
     infer_mode_from_frequency, infer_mode_type_from_mode
-from core.sig_utils import populate_sig_ref_info, ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name
 from data.sig_ref import SIGRef
 
 

server/handlers/api/addspot.py

@@ -10,7 +10,7 @@ from tornado.web import Application
 
 from core.config import ALLOW_SPOTTING
 from core.constants import UNKNOWN_BAND
-from core.lookup_helper import infer_band_from_freq
+from core.utils import infer_band_from_freq
 from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import safe_json_dumps

server/handlers/api/lookups.py

@@ -11,7 +11,8 @@ from tornado.web import Application
 from core.constants import SIGS
 from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone
 from core.prometheus_metrics_handler import api_requests_counter
-from core.sig_utils import get_ref_regex_for_sig, populate_sig_ref_info
+from core.sig_utils import get_ref_regex_for_sig
+from core.sig_lookup_helper import populate_sig_ref_info
 from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 from data.sig_ref import SIGRef

sigrefdataproviders/file_download_sig_ref_data_provider.py

@@ -21,7 +21,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         self._poll_interval = poll_interval
         self._thread = None
         self._stop_event = Event()
-        self._url_data_cache = URLDataCache("sigrefdata-" + sig_name)
+        self._url_data_cache = URLDataCache("sigrefdata_" + sig_name)
 
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
@@ -50,7 +50,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
             if http_response.ok:
                 # Pass off to the subclass for processing
                 new_data = self._http_response_to_data(http_response)
-                # Submit the new spots for processing. There might not be any spots for the less popular programs.
+                # Add the new data to the SIG Ref data store
                 if new_data:
                     self._add_data(new_data)
 
@@ -62,8 +62,10 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
                 logging.warning(f"HTTP {http_response.status_code} when downloading SIG ref data for {self.sig_name}.")
 
         except ConnectionError:
+            self.status = "Error"
             logging.warning(f"Connection error when downloading SIG ref data for {self.sig_name}.")
         except (ConnectTimeout, ReadTimeout):
+            self.status = "Error"
             logging.warning(f"Timeout when downloading SIG ref data for {self.sig_name}.")
         except Exception:
             self.status = "Error"

sigrefdataproviders/local_file_sig_ref_data_provider.py

@@ -22,10 +22,11 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
             else:
-                logging.info("No new SIG ref data found for " + self.sig_name)
+                self.status = "Error"
+                logging.info("Failed to load SIG ref data for " + self.sig_name)
         except Exception as e:
             self.status = "Error"
-            logging.exception("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")")
+            logging.error("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")", e, exc_info=True)
 
     def stop(self):
         pass

sigrefdataproviders/sig_ref_data_provider.py

@@ -15,7 +15,6 @@ class SIGRefDataProvider:
         self.sig_name = sig_name
         self.enabled = provider_config["enabled"]
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
-        self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
         self.reference_count = 0
 

spothole.py

@@ -5,10 +5,11 @@ import signal
 import sys
 
 from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL, get_sig_ref_data_provider_from_config, \
-    get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config
+    get_spot_provider_from_config, get_alert_provider_from_config, get_solar_conditions_provider_from_config, \
+    get_static_data_provider_from_config
 from core.constants import SOFTWARE_VERSION
 from core.data_store import DATA_STORE
-from core.lookup_helper import lookup_helper
+from core.call_lookup_helper import lookup_helper
 from core.status_reporter import StatusReporter
 from server.webserver import WebServer
 
@@ -17,6 +18,7 @@ web_server = None
 spot_providers = []
 alert_providers = []
 solar_condition_providers = []
+static_data_providers = []
 sig_ref_data_providers = []
 cleanup_timer = None
 run = True
@@ -42,6 +44,9 @@ def shutdown(_signum=None, _frame=None):
     for srdp in sig_ref_data_providers:
         if srdp.enabled:
             srdp.stop()
+    for srdp in static_data_providers:
+        if srdp.enabled:
+            srdp.stop()
     DATA_STORE.close()
     os._exit(0)
 
@@ -95,6 +100,13 @@ if __name__ == '__main__':
         if p.enabled:
             p.start()
 
+    # Fetch, set up and start static reference data providers
+    for entry in config.get("static-data-providers", []):
+        static_data_providers.append(get_static_data_provider_from_config(entry))
+    for p in static_data_providers:
+        if p.enabled:
+            p.start()
+
     # Fetch, set up and start SIG reference data providers
     for entry in config.get("sig-ref-data-providers", []):
         sig_ref_data_providers.append(get_sig_ref_data_provider_from_config(entry))
@@ -104,7 +116,7 @@ if __name__ == '__main__':
 
     # Set up status reporter
     status_reporter = StatusReporter(web_server=web_server, spot_providers=spot_providers,
-                                     alert_providers=alert_providers,
+                                     alert_providers=alert_providers, static_data_providers=static_data_providers,
                                      sig_ref_data_providers=sig_ref_data_providers,
                                      solar_condition_providers=solar_condition_providers, run_interval=5)
     status_reporter.start()

static/apidocs/openapi.yml

@@ -23,7 +23,7 @@ info:
     * Added `comment_names` to SIGs in the `/options`, to reflect how they might be referred to in spot comments where
       it differs from their `name`.
     * Added `propagation_mode` field to spots
-    * Added `sig_ref_data_providers` to status and removed `cleanup`
+    * Added `sig_ref_data_providers` and `static_data_providers` to status and removed `cleanup`
 
     ### 1.3
 
@@ -1721,6 +1721,28 @@ components:
             is zero, the provider has never updated.
           example: 1759579508
 
+    StaticDataProviderStatus:
+      type: object
+      properties:
+        sig_name:
+          type: string
+          description: The name of the provider.
+          example: K0SWE
+        enabled:
+          type: boolean
+          description: Whether the provider is enabled or not.
+          example: true
+        status:
+          type: string
+          description: The status of the provider.
+          example: OK
+        last_updated:
+          type: number
+          description: >
+            The last time at which this provider received data, UTC seconds since UNIX epoch. If this
+            is zero, the provider has never updated.
+          example: 1759579508
+
     SIGRefDataProviderStatus:
       type: object
       properties:
@@ -1844,6 +1866,11 @@ components:
           description: An array of all the solar conditions providers.
           items:
             $ref: '#/components/schemas/SolarConditionsProviderStatus'
+        static_data_providers:
+          type: array
+          description: An array of all the static reference data providers.
+          items:
+            $ref: '#/components/schemas/StaticDataProviderStatus'
         sig_ref_data_providers:
           type: array
           description: An array of all the SIG reference data providers.

static/js/status.js

@@ -40,6 +40,15 @@ function loadStatus() {
                 </div>`);
         });
 
+        jsonData["static_data_providers"].forEach(p => {
+            $("#static-data-providers-status-container").append(`
+                <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
+                    <div class="col"><strong>${p["name"]}</strong></div>
+                    <div class="col">Status: ${p["status"]}</div>
+                    <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
+                </div>`);
+        });
+
         jsonData["sig_ref_data_providers"].forEach(p => {
             $("#sig-ref-data-providers-status-container").append(`
                 <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">

staticdataproviders/cqzonedata.py

@@ -0,0 +1,30 @@
+import json
+import logging
+
+import geopandas
+from shapely import prepare
+
+from core.data_store import DATA_STORE
+from staticdataproviders.local_file_static_data_provider import LocalFileStaticDataProvider
+
+
+class CQZoneData(LocalFileStaticDataProvider):
+    """Static data provider for CQ zone geodata."""
+
+    PATH = "datafiles/cqzones.geojson"
+
+    def __init__(self, provider_config):
+        super().__init__("CQ Zone Data", provider_config, self.PATH)
+
+    def _load_data(self, path):
+        try:
+            with open(path) as f:
+                cq_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
+            for idx in cq_zone_data.index:
+                prepare(cq_zone_data.at[idx, 'geometry'])
+            DATA_STORE.cq_zone_data = cq_zone_data
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading CQ zone data.", e, exc_info=True)
+            return False

staticdataproviders/file_download_static_data_provider.py

@@ -0,0 +1,77 @@
+import logging
+from datetime import datetime
+from threading import Thread, Event
+
+import pytz
+from requests import ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout
+
+from core.constants import HTTP_HEADERS
+from core.url_data_cache import URLDataCache
+from staticdataproviders.static_data_provider import StaticDataProvider
+
+
+class FileDownloadStaticDataProvider(StaticDataProvider):
+    """Generic static reference data provider class for providers that fetch their data from the web by downloading a
+     file."""
+
+    def __init__(self, name, provider_config, url, poll_interval):
+        """ Set up the provider, note poll_interval is in *days*."""
+        super().__init__(name, provider_config)
+        self._url = url
+        self._poll_interval = poll_interval
+        self._thread = None
+        self._stop_event = Event()
+        self._url_data_cache = URLDataCache("staticdata_" + name)
+
+    def start(self):
+        # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
+        # subsequent polls, so start() returns immediately and the application can continue starting.
+        logging.info(
+            "Set up query of " + self.name + " static reference data every " + str(self._poll_interval) + " days.")
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _run(self):
+        while True:
+            self._poll()
+            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
+                break
+
+    def _poll(self):
+        try:
+            # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+            # caching, but just so continual restarts of the software during testing don't hammer the servers.
+            logging.debug("Downloading " + self.name + " static reference data...")
+            http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
+            # Check response code was good
+            if http_response.ok:
+                # Pass off to the subclass for processing
+                ok = self._handle_http_response(http_response)
+                if ok:
+                    self.status = "OK"
+                    self.last_update_time = datetime.now(pytz.UTC)
+                    logging.info("Updated static reference data for " + self.name)
+            else:
+                self.status = "Error"
+                logging.warning(f"HTTP {http_response.status_code} when downloading static reference data for {self.name}.")
+
+        except ConnectionError:
+            self.status = "Error"
+            logging.warning(f"Connection error when downloading static reference data for {self.name}.")
+        except (ConnectTimeout, ReadTimeout):
+            self.status = "Error"
+            logging.warning(f"Timeout when downloading static reference data for {self.name}.")
+        except Exception:
+            self.status = "Error"
+            logging.exception("Exception in HTTP static reference data provider (" + self.name + ")")
+            self._stop_event.wait(timeout=1)
+
+    def _handle_http_response(self, http_response):
+        """Handle an HTTP response returned by the server and load the data from it. Return true if successful,
+         false otherwise."""
+
+        raise NotImplementedError("Subclasses must implement this method")

staticdataproviders/ituzonedata.py

@@ -0,0 +1,30 @@
+import json
+import logging
+
+import geopandas
+from shapely import prepare
+
+from core.data_store import DATA_STORE
+from staticdataproviders.local_file_static_data_provider import LocalFileStaticDataProvider
+
+
+class ITUZoneData(LocalFileStaticDataProvider):
+    """Static data provider for ITU zone geodata."""
+
+    PATH = "datafiles/ituzones.geojson"
+
+    def __init__(self, provider_config):
+        super().__init__("ITU Zone Data", provider_config, self.PATH)
+
+    def _load_data(self, path):
+        try:
+            with open(path) as f:
+                itu_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
+            for idx in itu_zone_data.index:
+                prepare(itu_zone_data.at[idx, 'geometry'])
+            DATA_STORE.itu_zone_data = itu_zone_data
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading ITU zone data.", e, exc_info=True)
+            return False

staticdataproviders/k0swe.py

@@ -0,0 +1,39 @@
+import logging
+import re
+
+from core.data_store import DATA_STORE
+from staticdataproviders.file_download_static_data_provider import FileDownloadStaticDataProvider
+
+
+class K0SWE(FileDownloadStaticDataProvider):
+    """Static data provider for K0SWE's dxcc.json, which provides callsign regex to DXCC entity mapping, plus DXCC to
+    continent, flag emoji etc."""
+
+    POLL_INTERVAL_DAYS = 7
+    DATA_URL = "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json"
+
+    def __init__(self, provider_config):
+        super().__init__("K0SWE DXCC JSON", provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _handle_http_response(self, http_response):
+        try:
+            dxcc_list = http_response.json()["dxcc"]
+            # Reformat as a map for to place in the data store
+            dxcc_map = {}
+            for dxcc in dxcc_list:
+                dxcc_map[dxcc["entityCode"]] = dxcc
+
+            # Add to data store
+            for k, v in dxcc_map.items():
+                DATA_STORE.dxcc_data[k] = v
+
+            # Regenerate in-memory regex-to-DXCC-entity-code map.
+            DATA_STORE.regenerateCallRegexToDXCCEntityMap()
+
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading K0SWE dxcc.json.", e, exc_info=True)
+            return False
+
+

staticdataproviders/local_file_static_data_provider.py

@@ -0,0 +1,37 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from staticdataproviders.static_data_provider import StaticDataProvider
+
+
+class LocalFileStaticDataProvider(StaticDataProvider):
+    """Generic static reference data provider class for providers that fetch their data from a local file on startup."""
+
+    def __init__(self, name, provider_config, path):
+        super().__init__(name, provider_config)
+        self._path = path
+
+    def start(self):
+        logging.debug("Loading " + self.name + " static reference data from file.")
+        try:
+            ok = self._load_data(self._path)
+            if ok:
+                self.status = "OK"
+                self.last_update_time = datetime.now(pytz.UTC)
+                logging.info("Updated static reference data for " + self.name)
+            else:
+                self.status = "Error"
+                logging.error("Failed to load data for " + self.name)
+        except Exception as e:
+            self.status = "Error"
+            logging.error("Exception in local file Static Data Provider (" + self.name + ")", e, exc_info=True)
+
+    def stop(self):
+        pass
+
+    def _load_data(self, path):
+        """Load data from the given file path. Return true if successful, false otherwise."""
+
+        raise NotImplementedError("Subclasses must implement this method")

staticdataproviders/static_data_provider.py

@@ -0,0 +1,31 @@
+import logging
+from datetime import datetime
+
+import pytz
+
+from core.data_store import DATA_STORE
+
+
+class StaticDataProvider:
+    """Generic static reference data provider class. Subclasses of this query the individual URLs or files for data."""
+
+    def __init__(self, name, provider_config):
+        """Constructor"""
+
+        self.name = name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+        self.reference_count = 0
+
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        raise NotImplementedError("Subclasses must implement this method")

templates/status.html

@@ -54,6 +54,15 @@
     </div>
 </div>
 
+<div class="card mt-3">
+    <div class="card-header">
+        Static Reference Data Providers
+    </div>
+    <div class="card-body" id="static-data-providers-status-container">
+
+    </div>
+</div>
+
 <div class="card mt-3">
     <div class="card-header">
         SIG Reference Data Providers