Refactor of caching & data storage part 6 #118

core/config.py

@@ -60,6 +60,14 @@ def get_solar_conditions_provider_from_config(config_providers_entry):
     return provider_class(config_providers_entry)
 
 
+def get_static_data_provider_from_config(config_providers_entry):
+    """Utility method to get a static reference data provider based on the class specified in its config entry."""
+
+    module = importlib.import_module('staticdataproviders.' + config_providers_entry["class"].lower())
+    provider_class = getattr(module, config_providers_entry["class"])
+    return provider_class(config_providers_entry)
+
+
 def get_sig_ref_data_provider_from_config(config_providers_entry):
     """Utility method to get a SIG reference data provider based on the class specified in its config entry."""
 

core/data_store.py

@@ -22,6 +22,7 @@ class DataStore:
         self.alerts = None
         self.spots = None
         self.callsigns = None
+        self.dxcc_data = None
         self.sigrefs = None
         self.status_data = None
         self._status = None
@@ -42,10 +43,13 @@ class DataStore:
             self._status.add("status_data", {})
         self.status_data = self._status.get("status_data")
 
-        # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
-        # but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
-        # and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
-        # so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
+        # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
+        # a regular basis but there's no need for a TTL since old data is better than no data.
+        self.dxcc_data = diskcache.Cache(CACHE_DIR + "dxcc_data")
+
+        # For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers
+        # of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref"
+        # syntax for keys to keep it a single level.
         self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs")
         logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
 
@@ -73,6 +77,7 @@ class DataStore:
         self.alerts.close()
         self._solar.close()
         self._status.close()
+        self.dxcc_data.close()
         self.sigrefs.close()
         self.callsigns.close()
 

core/lookup_helper.py

@@ -1,7 +1,5 @@
 import gzip
-import json
 import logging
-import re
 import urllib.parse
 from datetime import timedelta
 
@@ -9,14 +7,13 @@ import xmltodict
 from diskcache import Cache
 from pyhamtools import LookupLib, Callinfo, callinfo
 from pyhamtools.exceptions import APIKeyMissingError
-from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
 from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 from requests_cache import CachedSession
 
 from core.config import config
-from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
-    HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
+from core.constants import HTTP_HEADERS, HAMQTH_PRG
+from core.data_store import DATA_STORE
 from core.url_data_cache import URLDataCache
 
 # QRZ XML field names differ from pyhamtools' normalised names; map them here.
@@ -84,7 +81,6 @@ class LookupHelper:
         self._lookup_lib_basic = None
         self._country_files_cty_plist_download_location = None
         self._dxcc_json_download_location = None
-        self._dxcc_data = None
 
     def start(self):
         # Lookup helpers from pyhamtools. We use five (!) of these. The simplest is country-files.com, which downloads
@@ -118,22 +114,6 @@ class LookupHelper:
                                                          filename=self._clublog_xml_download_location)
         self._clublog_callsign_data_cache = Cache('cache/clublog_callsign_lookup_cache')
 
-        # We also get a lookup of DXCC data from K0SWE to use for additional lookups of e.g. flags.
-        self._dxcc_json_download_location = "cache/dxcc.json"
-        success = self._download_dxcc_json()
-        if success:
-            with open(self._dxcc_json_download_location) as f:
-                tmp_dxcc_data = json.load(f)["dxcc"]
-                # Reformat as a map for faster lookup
-                self._dxcc_data = {}
-                for dxcc in tmp_dxcc_data:
-                    self._dxcc_data[dxcc["entityCode"]] = dxcc
-        else:
-            logging.error("Could not download DXCC data, flags and similar data may be missing!")
-
-        # Precompile regex matches for DXCCs to improve efficiency when iterating through them
-        for dxcc in (self._dxcc_data.values() if self._dxcc_data else []):
-            dxcc["_prefixRegexCompiled"] = re.compile(dxcc["prefixRegex"])
 
     def _download_country_files_cty_plist(self):
         """Download the cty.plist file from country-files.com on first startup. The pyhamtools lib can actually download and use
@@ -163,31 +143,6 @@ class LookupHelper:
             logging.error("Exception when downloading Clublog cty.xml", e)
             return False
 
-    def _download_dxcc_json(self):
-        """Download the dxcc.json file on first startup."""
-
-        try:
-            logging.info("Downloading dxcc.json...")
-            response = _URL_DATA_CACHE.get(
-                "https://raw.githubusercontent.com/k0swe/dxcc-json/refs/heads/main/dxcc.json",
-                headers=HTTP_HEADERS)
-
-            if response.ok:
-                with open(self._dxcc_json_download_location, "w") as f:
-                    f.write(response.text)
-                    f.flush()
-                    return True
-            else:
-                logging.warning(f"HTTP {response.status_code} when downloading dxcc.json.")
-                return False
-
-        except ConnectionError:
-            logging.warning(f"Connection error when downloading dxcc.json.")
-        except (ConnectTimeout, ReadTimeout):
-            logging.warning(f"Timeout when downloading dxcc.json.")
-        except Exception as e:
-            logging.error("Exception when downloading dxcc.json", e)
-            return False
 
     def _download_clublog_ctyxml(self):
         """Download the cty.xml (gzipped) file from Clublog on first startup, so we can use it in preference to querying the
@@ -374,7 +329,8 @@ class LookupHelper:
     def get_flag_for_dxcc(self, dxcc):
         """Get an emoji flag for a given DXCC entity ID"""
 
-        return self._dxcc_data[dxcc]["flag"] if dxcc in self._dxcc_data else None
+        dxcc_data = DATA_STORE.dxcc_data[dxcc]
+        return dxcc_data["flag"] if dxcc_data else None
 
     def infer_name_from_callsign_online_lookup(self, call, credentials=None):
         """Infer an operator name from a callsign (requires QRZ.com/HamQTH)"""
@@ -680,61 +636,3 @@ class LookupHelper:
 
 # Singleton object
 lookup_helper = LookupHelper()
-
-
-def infer_mode_from_comment(comment):
-    """Infer a mode from the comment"""
-
-    for mode in ALL_MODES:
-        if mode in comment.upper():
-            return mode
-    for mode in MODE_ALIASES.keys():
-        if mode in comment.upper():
-            return MODE_ALIASES[mode]
-    return None
-
-
-def infer_mode_type_from_mode(mode):
-    """Infer a "mode family" from a mode."""
-
-    if mode.upper() in CW_MODES:
-        return "CW"
-    elif mode.upper() in PHONE_MODES:
-        return "PHONE"
-    elif mode.upper() in DATA_MODES:
-        return "DATA"
-    else:
-        if mode.upper() != "OTHER":
-            logging.warning("Found an unrecognised mode: " + mode + ". Developer should categorise this.")
-        return None
-
-
-def infer_band_from_freq(freq):
-    """Infer a band from a frequency in Hz"""
-
-    for b in BANDS:
-        if b.start_freq <= freq <= b.end_freq:
-            return b
-    return UNKNOWN_BAND
-
-
-def infer_mode_from_frequency(freq):
-    """Infer a mode from the frequency (in Hz) according to the band plan. Just a guess really."""
-
-    try:
-        khz = freq / 1000.0
-        mode = freq_to_band(khz)["mode"]
-        # Some additional common digimode ranges in addition to what the 3rd-party freq_to_band function returns.
-        # This is mostly here just because freq_to_band is very specific about things like FT8 frequencies, and e.g.
-        # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
-        # might include other common digimode centres of activity here, but this achieves the main goal of keeping
-        # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
-        if (7074 <= khz < 7077) or (10136 <= khz < 10139) or (14074 <= khz < 14077) or (18100 <= khz < 18103) or (
-                21074 <= khz < 21077) or (24915 <= khz < 24918) or (28074 <= khz < 28077):
-            mode = "FT8"
-        if (7047.5 <= khz < 7050.5) or (10140 <= khz < 10143) or (14080 <= khz < 14083) or (
-                18104 <= khz < 18107) or (21140 <= khz < 21143) or (24919 <= khz < 24922) or (28180 <= khz < 28183):
-            mode = "FT4"
-        return mode
-    except KeyError:
-        return None

core/status_reporter.py

@@ -15,7 +15,7 @@ class StatusReporter:
     """Provides a timed update of the application's status data."""
 
     def __init__(self, run_interval, web_server, spot_providers, alert_providers, solar_condition_providers,
-                 sig_ref_data_providers):
+                 static_data_providers, sig_ref_data_providers):
         """Constructor"""
 
         self._run_interval = run_interval
@@ -23,6 +23,7 @@ class StatusReporter:
         self._spot_providers = spot_providers
         self._alert_providers = alert_providers
         self._solar_condition_providers = solar_condition_providers
+        self._static_data_providers = static_data_providers
         self._sig_ref_data_providers = sig_ref_data_providers
         self._thread = None
         self._stop_event = Event()
@@ -74,6 +75,11 @@ class StatusReporter:
                        "last_updated": p.last_update_time.replace(
                            tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
             self._solar_condition_providers))
+        DATA_STORE.status_data["static_data_providers"] = list(
+            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
+                           "last_updated": p.last_update_time.replace(
+                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
+                self._static_data_providers))
         DATA_STORE.status_data["sig_ref_data_providers"] = list(
             map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
                            "last_updated": p.last_update_time.replace(

core/utils.py

@@ -1,8 +1,71 @@
+import logging
+
 import simplejson
+from pyhamtools.frequency import freq_to_band
+
+from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES
 
 
 def safe_json_dumps(obj):
     """Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats
     which are invalid in JSON."""
 
-    return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
+    return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
+
+
+def infer_mode_from_comment(comment):
+    """Infer a mode from the comment"""
+
+    for mode in ALL_MODES:
+        if mode in comment.upper():
+            return mode
+    for mode in MODE_ALIASES.keys():
+        if mode in comment.upper():
+            return MODE_ALIASES[mode]
+    return None
+
+
+def infer_mode_type_from_mode(mode):
+    """Infer a "mode family" from a mode."""
+
+    if mode.upper() in CW_MODES:
+        return "CW"
+    elif mode.upper() in PHONE_MODES:
+        return "PHONE"
+    elif mode.upper() in DATA_MODES:
+        return "DATA"
+    else:
+        if mode.upper() != "OTHER":
+            logging.warning("Found an unrecognised mode: " + mode + ". Developer should categorise this.")
+        return None
+
+
+def infer_band_from_freq(freq):
+    """Infer a band from a frequency in Hz"""
+
+    for b in BANDS:
+        if b.start_freq <= freq <= b.end_freq:
+            return b
+    return UNKNOWN_BAND
+
+
+def infer_mode_from_frequency(freq):
+    """Infer a mode from the frequency (in Hz) according to the band plan. Just a guess really."""
+
+    try:
+        khz = freq / 1000.0
+        mode = freq_to_band(khz)["mode"]
+        # Some additional common digimode ranges in addition to what the 3rd-party freq_to_band function returns.
+        # This is mostly here just because freq_to_band is very specific about things like FT8 frequencies, and e.g.
+        # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
+        # might include other common digimode centres of activity here, but this achieves the main goal of keeping
+        # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
+        if (7074 <= khz < 7077) or (10136 <= khz < 10139) or (14074 <= khz < 14077) or (18100 <= khz < 18103) or (
+                21074 <= khz < 21077) or (24915 <= khz < 24918) or (28074 <= khz < 28077):
+            mode = "FT8"
+        if (7047.5 <= khz < 7050.5) or (10140 <= khz < 10143) or (14080 <= khz < 14083) or (
+                18104 <= khz < 18107) or (21140 <= khz < 21143) or (24919 <= khz < 24922) or (28180 <= khz < 28183):
+            mode = "FT4"
+        return mode
+    except KeyError:
+        return None