Refactor of caching & data storage part 7 #118

core/call_lookup_helper.py

@@ -329,7 +329,7 @@ class LookupHelper:
     def get_flag_for_dxcc(self, dxcc):
         """Get an emoji flag for a given DXCC entity ID"""
 
-        dxcc_data = DATA_STORE.dxcc_data[dxcc]
+        dxcc_data = DATA_STORE.dxcc_data[dxcc] if dxcc in DATA_STORE.dxcc_data else None
         return dxcc_data["flag"] if dxcc_data else None
 
     def infer_name_from_callsign_online_lookup(self, call, credentials=None):
@@ -621,9 +621,9 @@ class LookupHelper:
     def _get_dxcc_data_for_callsign(self, call) -> dict | None:
         """Utility method to get generic DXCC data from our lookup table, if we can find it"""
 
-        for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]:
-            if entry["_prefixRegexCompiled"].match(call):
-                return entry
+        for pattern, entity_code in DATA_STORE.dxcc_lookup_by_call_regex:
+            if pattern.match(call):
+                return DATA_STORE.dxcc_data[entity_code]
         return None
 
     def stop(self):

core/data_store.py

@@ -1,4 +1,5 @@
 import logging
+import re
 from pathlib import Path
 
 import diskcache
@@ -23,6 +24,7 @@ class DataStore:
         self.spots = None
         self.callsigns = None
         self.dxcc_data = None
+        self.dxcc_lookup_by_call_regex = []
         self.sigrefs = None
         self.status_data = None
         self._status = None
@@ -46,6 +48,7 @@ class DataStore:
         # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
         # a regular basis but there's no need for a TTL since old data is better than no data.
         self.dxcc_data = diskcache.Cache(CACHE_DIR + "dxcc_data")
+        self.regenerateCallRegexToDXCCEntityMap()
 
         # For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers
         # of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref"
@@ -72,6 +75,15 @@ class DataStore:
                                     snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
         logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))
 
+    def regenerateCallRegexToDXCCEntityMap(self):
+        """DXCC entity data from K0SWE includes a regex which we can use to match a callsign, and determine which DXCC
+        entity it belongs to. But getting every DXCC entity data object out of DiskCache, iterating, compiling its regex
+        and testing the callsign every time is expensive. So instead we build a separate in-memory lookup of compiled
+        regex against DXCC entity code, as a list of tuples we can iterate through."""
+
+        for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]:
+            self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))
+
     def close(self):
         self.spots.close()
         self.alerts.close()

core/sig_lookup_helper.py

@@ -0,0 +1,93 @@
+import logging
+
+from pyhamtools.locator import locator_to_latlong, latlong_to_locator
+
+from core.data_store import DATA_STORE
+from core.geo_utils import wab_wai_square_to_lat_lon
+
+
+def populate_sig_ref_info(sig_ref):
+    """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
+    which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes
+    use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
+    SIG we are getting data for.
+    Note there is currently no support for KRMNPA location lookup, see issue #61."""
+
+    if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "":
+        logging.debug("Failed to look up sig_ref info, sig or id were not set.")
+        return sig_ref
+
+    sig = sig_ref.sig
+    ref_id = sig_ref.id
+
+    try:
+        ### FUDGES ###
+        #
+        # DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all
+        # activators add leading zeros.
+        if sig.upper() == "DME":
+            ref_id = ref_id.zfill(5)
+
+        # KRMNPA & SANPCPA fudge. These don't have their own reference system, they just use VKFF references, so pretend
+        # the sig is WWFF and carry on
+        if sig.upper() == "KRMNPA" or sig.upper() == "SANPCPA":
+            sig = "WWFF"
+
+        ### SKIPS ###
+        #
+        # If the SIG is HEMA, we have no current lookup for this so just skip the lookup here.
+        if sig.upper() == "HEMA":
+            return sig_ref
+
+        ### PROGRAMMATIC DATA GENERATION INSTEAD OF LOOKUPS ###
+        #
+        # If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
+        # calculate all the information we are going to get directly.
+        if sig.upper() == "TILES":
+            # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
+            if not sig_ref.name:
+                sig_ref.name = sig_ref.id
+            if not sig_ref.grid:
+                sig_ref.grid = sig_ref.id
+            if sig_ref.grid and not sig_ref.latitude:
+                ll = locator_to_latlong(str(sig_ref.grid))
+                sig_ref.latitude = ll[0]
+                sig_ref.longitude = ll[1]
+            return sig_ref
+
+        elif sig.upper() == "WAB" or sig.upper() == "WAI":
+            ll = wab_wai_square_to_lat_lon(ref_id)
+            if ll:
+                sig_ref.name = ref_id
+                try:
+                    sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
+                    sig_ref.latitude = ll[0]
+                    sig_ref.longitude = ll[1]
+                except:
+                    logging.warning("Invalid lat/lon received for WAB/WAI reference")
+            return sig_ref
+
+        elif sig.upper() == "BOTA":
+            # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
+            if not sig_ref.name:
+                sig_ref.name = sig_ref.id
+            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
+            return sig_ref
+
+        ### ACTUAL LOOKUP ###
+        #
+        # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
+        # this SIG. If so, check for the reference data and use that.
+        key = sig + ":" + ref_id
+        lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
+        if lookup_data:
+            # Copy new sig ref data into existing object where data was previously missing
+            for key, value in lookup_data.__dict__.items():
+                if value is not None and sig_ref.__dict__.get(key) is None:
+                    sig_ref.__dict__[key] = value
+        else:
+            logging.warning("%s database did not contain data for ref %s", sig, ref_id)
+
+    except Exception:
+        logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
+    return sig_ref

core/sig_utils.py

@@ -1,10 +1,4 @@
-import logging
-
-from pyhamtools.locator import latlong_to_locator, locator_to_latlong
-
 from core.constants import SIGS
-from core.data_store import DATA_STORE
-from core.geo_utils import wab_wai_square_to_lat_lon
 
 
 def get_ref_regex_for_sig(sig):
@@ -26,89 +20,5 @@ def get_sig_name_from_comment_name(sig):
     return None
 
 
-def populate_sig_ref_info(sig_ref):
-    """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
-    which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes
-    use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
-    SIG we are getting data for.
-    Note there is currently no support for KRMNPA location lookup, see issue #61."""
-
-    if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "":
-        logging.debug("Failed to look up sig_ref info, sig or id were not set.")
-        return sig_ref
-
-    sig = sig_ref.sig
-    ref_id = sig_ref.id
-
-    try:
-        ### FUDGES ###
-        #
-        # DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all
-        # activators add leading zeros.
-        if sig.upper() == "DME":
-            ref_id = ref_id.zfill(5)
-
-        # KRMNPA & SANPCPA fudge. These don't have their own reference system, they just use VKFF references, so pretend
-        # the sig is WWFF and carry on
-        if sig.upper() == "KRMNPA" or sig.upper() == "SANPCPA":
-            sig = "WWFF"
-
-        ### SKIPS ###
-        #
-        # If the SIG is HEMA, we have no current lookup for this so just skip the lookup here.
-        if sig.upper() == "HEMA":
-            return sig_ref
-
-        ### PROGRAMMATIC DATA GENERATION INSTEAD OF LOOKUPS ###
-        #
-        # If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
-        # calculate all the information we are going to get directly.
-        if sig.upper() == "TILES":
-            # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
-            if not sig_ref.name:
-                sig_ref.name = sig_ref.id
-            if not sig_ref.grid:
-                sig_ref.grid = sig_ref.id
-            if sig_ref.grid and not sig_ref.latitude:
-                ll = locator_to_latlong(str(sig_ref.grid))
-                sig_ref.latitude = ll[0]
-                sig_ref.longitude = ll[1]
-
-        elif sig.upper() == "WAB" or sig.upper() == "WAI":
-            ll = wab_wai_square_to_lat_lon(ref_id)
-            if ll:
-                sig_ref.name = ref_id
-                try:
-                    sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
-                    sig_ref.latitude = ll[0]
-                    sig_ref.longitude = ll[1]
-                except:
-                    logging.warning("Invalid lat/lon received for WAB/WAI reference")
-
-        elif sig.upper() == "BOTA":
-            # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
-            if not sig_ref.name:
-                sig_ref.name = sig_ref.id
-            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
-
-        ### ACTUAL LOOKUP ###
-        #
-        # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
-        # this SIG. If so, check for the reference data and use that.
-        key = sig + ":" + ref_id
-        lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
-        if lookup_data:
-            # Copy new sig ref data into existing object where data was previously missing
-            for key, value in lookup_data.__dict__.items():
-                if value is not None and sig_ref.__dict__.get(key) is None:
-                    sig_ref.__dict__[key] = value
-        else:
-            logging.warning("%s database did not contain data for ref %s", sig, ref_id)
-
-    except Exception:
-        logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
-    return sig_ref
-
-
 # Regex matching any SIG's "comment name", i.e. how it may be referred to in spot comments
 ANY_SIG_REGEX = r"(" + r"|".join(n for s in SIGS for n in s.comment_names) + r")"

data/alert.py

@@ -7,8 +7,8 @@ from datetime import datetime, timedelta
 
 import pytz
 
-from core.lookup_helper import lookup_helper
-from core.sig_utils import populate_sig_ref_info
+from core.call_lookup_helper import lookup_helper
+from core.sig_lookup_helper import populate_sig_ref_info
 
 
 @dataclass

data/spot.py

@@ -12,8 +12,9 @@ from pyhamtools.locator import locator_to_latlong, latlong_to_locator
 from core.config import MAX_SPOT_AGE
 from core.constants import MODE_ALIASES, PROPAGATION_MODES
 from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
-from core.lookup_helper import lookup_helper
-from core.sig_utils import populate_sig_ref_info, ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name
+from core.call_lookup_helper import lookup_helper
+from core.sig_utils import ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name
+from core.sig_lookup_helper import populate_sig_ref_info
 from core.utils import infer_band_from_freq, infer_mode_from_comment, \
     infer_mode_from_frequency, infer_mode_type_from_mode
 from data.sig_ref import SIGRef

server/handlers/api/lookups.py

@@ -11,7 +11,8 @@ from tornado.web import Application
 from core.constants import SIGS
 from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone
 from core.prometheus_metrics_handler import api_requests_counter
-from core.sig_utils import get_ref_regex_for_sig, populate_sig_ref_info
+from core.sig_utils import get_ref_regex_for_sig
+from core.sig_lookup_helper import populate_sig_ref_info
 from core.utils import safe_json_dumps
 from data.lookup_credentials import extract_credentials
 from data.sig_ref import SIGRef

spothole.py

@@ -9,7 +9,7 @@ from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL, get_sig_ref_da
     get_static_data_provider_from_config
 from core.constants import SOFTWARE_VERSION
 from core.data_store import DATA_STORE
-from core.lookup_helper import lookup_helper
+from core.call_lookup_helper import lookup_helper
 from core.status_reporter import StatusReporter
 from server.webserver import WebServer
 

static/js/status.js

@@ -43,7 +43,7 @@ function loadStatus() {
         jsonData["static_data_providers"].forEach(p => {
             $("#static-data-providers-status-container").append(`
                 <div class="row row-cols-1 row-cols-md-4 g-4 mb-2">
-                    <div class="col"><strong>${p["sig_name"]}</strong></div>
+                    <div class="col"><strong>${p["name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
                 </div>`);

staticdataproviders/k0swe.py

@@ -23,14 +23,13 @@ class K0SWE(FileDownloadStaticDataProvider):
             for dxcc in dxcc_list:
                 dxcc_map[dxcc["entityCode"]] = dxcc
 
-            # Precompile regex matches for DXCCs to improve efficiency when iterating through them
-            for dxcc in dxcc_map.values():
-                dxcc["_prefixRegexCompiled"] = re.compile(dxcc["prefixRegex"])
-
             # Add to data store
             for k, v in dxcc_map.items():
                 DATA_STORE.dxcc_data[k] = v
 
+            # Regenerate in-memory regex-to-DXCC-entity-code map.
+            DATA_STORE.regenerateCallRegexToDXCCEntityMap()
+
             return True
 
         except Exception as e: