import logging import re from pathlib import Path import diskcache from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE from core.live_data_cache import LiveDataCache from data.solar_conditions import SolarConditions CACHE_DIR = "./cache/" class DataStore: """Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign lookup data using different caching strategies for each.""" def __init__(self): # Constants self._MAX_SPOT_COUNT = 100000 self._MAX_ALERT_COUNT = 100000 self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300 self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60 # Caches self.alerts = None self.spots = None self.callsigns = None self.dxcc_data = None self.dxcc_lookup_by_call_regex = [] self.sigrefs = None self.status_data = None self._status = None self.solar_conditions = None self._solar = None # ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be # caches, they can just be straight objects self.cq_zone_data = None self.itu_zone_data = None def setup(self): Path(CACHE_DIR).mkdir(parents=True, exist_ok=True) # Standard disk cache for solar data and status data, but each cache contains only a single object which we # expose to the wider application self._solar = diskcache.Cache(CACHE_DIR + "solar") if "solar_conditions" not in self._solar: self._solar.add("solar_conditions", SolarConditions()) self.solar_conditions = self._solar.get("solar_conditions") self._status = diskcache.Cache(CACHE_DIR + "status") if "status_data" not in self._status: self._status.add("status_data", {}) self.status_data = self._status.get("status_data") # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on # a regular basis but there's no need for a TTL since old data is better than no data. self.dxcc_data = diskcache.Cache(CACHE_DIR + "dxcc_data") self.regenerateCallRegexToDXCCEntityMap() # For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers # of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref" # syntax for keys to keep it a single level. self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs") logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs)) # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup. # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign # we've seen, so we rely on them timing out and this triggering another lookup. self.callsigns = diskcache.Cache(CACHE_DIR + "callsigns") logging.info(f"Loaded data for %d callsigns.", len(self.callsigns)) # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache # before the spots and alerts are live in the system. self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE, snapshot_dir=CACHE_DIR + "spots", snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys())) self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE, snapshot_dir=CACHE_DIR + "alerts", snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys())) def regenerateCallRegexToDXCCEntityMap(self): """DXCC entity data from K0SWE includes a regex which we can use to match a callsign, and determine which DXCC entity it belongs to. But getting every DXCC entity data object out of DiskCache, iterating, compiling its regex and testing the callsign every time is expensive. So instead we build a separate in-memory lookup of compiled regex against DXCC entity code, as a list of tuples we can iterate through.""" for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]: self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"])) def close(self): self.spots.close() self.alerts.close() self._solar.close() self._status.close() self.dxcc_data.close() self.sigrefs.close() self.callsigns.close() # Global object DATA_STORE = DataStore()