import logging from pathlib import Path import diskcache from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE from core.live_data_cache import LiveDataCache from data.solar_conditions import SolarConditions CACHE_DIR = "./cache/" class DataStore: """Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign lookup data using different caching strategies for each.""" def __init__(self): self._MAX_SPOT_COUNT = 100000 self._MAX_ALERT_COUNT = 100000 self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300 self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60 self.alerts = None self.spots = None self.callsigns = None self.dxcc_data = None self.sigrefs = None self.status_data = None self._status = None self.solar_conditions = None self._solar = None def setup(self): Path(CACHE_DIR).mkdir(parents=True, exist_ok=True) # Standard disk cache for solar data and status data, but each cache contains only a single object which we # expose to the wider application self._solar = diskcache.Cache(CACHE_DIR + "solar") if "solar_conditions" not in self._solar: self._solar.add("solar_conditions", SolarConditions()) self.solar_conditions = self._solar.get("solar_conditions") self._status = diskcache.Cache(CACHE_DIR + "status") if "status_data" not in self._status: self._status.add("status_data", {}) self.status_data = self._status.get("status_data") # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on # a regular basis but there's no need for a TTL since old data is better than no data. self.dxcc_data = diskcache.Cache(CACHE_DIR + "dxcc_data") # For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers # of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref" # syntax for keys to keep it a single level. self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs") logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs)) # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup. # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign # we've seen, so we rely on them timing out and this triggering another lookup. self.callsigns = diskcache.Cache(CACHE_DIR + "callsigns") logging.info(f"Loaded data for %d callsigns.", len(self.callsigns)) # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache # before the spots and alerts are live in the system. self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE, snapshot_dir=CACHE_DIR + "spots", snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys())) self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE, snapshot_dir=CACHE_DIR + "alerts", snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys())) def close(self): self.spots.close() self.alerts.close() self._solar.close() self._status.close() self.dxcc_data.close() self.sigrefs.close() self.callsigns.close() # Global object DATA_STORE = DataStore()