Refactor of caching & data storage part 5 #118

core/data_store.py

@@ -43,11 +43,11 @@ class DataStore:
         self.status_data = self._status.get("status_data")
 
         # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
-        # but there's no need for a TTL since old data is better than no data. This is a two-layer dict, keys are SIG
-        # name and then reference ID, with the final value being a SIGRef object.
+        # but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
+        # and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
+        # so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
         self.sigrefs = diskcache.Cache(self._CACHE_DIR + "/sigrefs")
-        for k in list(self.sigrefs.iterkeys()):
-            logging.info(f"Loaded data for %d references in %s SIG.", len(self.sigrefs[k]), k)
+        logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
 
         # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
         # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign

core/sig_utils.py

@@ -1,12 +1,10 @@
 import logging
 
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
-from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
-from core.constants import SIGS, HTTP_HEADERS
+from core.constants import SIGS
 from core.data_store import DATA_STORE
 from core.geo_utils import wab_wai_square_to_lat_lon
-from core.url_data_cache import URL_DATA_CACHE
 
 
 def get_ref_regex_for_sig(sig):
@@ -85,42 +83,14 @@ def populate_sig_ref_info(sig_ref):
         # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
         # this SIG. If so, check for the reference data and use that.
         elif sig in DATA_STORE.sigrefs:
-            lookup_data = DATA_STORE.sigrefs[sig][ref_id] if ref_id in DATA_STORE.sigrefs[sig] else None
+            key = sig + ":" + ref_id
+            lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
             if lookup_data:
                 # Copy new sig ref data into existing object
                 sig_ref.__dict__.update(lookup_data.__dict__)
             else:
                 logging.warning("%s database did not contain data for ref %s", sig, ref_id)
 
-        elif False:
-            # TODO remove
-            # OK, this is not a SIG we have stored data for. Maybe it's of a type we can query information for live.
-            if sig.upper() == "IOTA":
-                response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
-                                                          headers=HTTP_HEADERS)
-                if response.ok:
-                    data = response.json()
-                    if data:
-                        sig_ref.name = data["name"] if "name" in data else None
-                        sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
-                        sig_ref.grid = data["locator"] if "locator" in data else None
-
-                        # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
-                        # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
-                        # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
-                        # just use the single point we have.
-                        if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
-                                "lat2") is not None and data.get("lng2") is not None:
-                            sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
-                            sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
-                        else:
-                            sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
-                            sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
-                    elif not response.from_cache:
-                        logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
-                elif not response.from_cache:
-                    logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
         else:
             logging.warning(f"Tried to look up a SIG called %s but Spothole does not know what that is.", sig)