spothole/core/data_store.py

import logging
import re
from pathlib import Path

import diskcache

from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
from core.live_data_cache import LiveDataCache
from data.solar_conditions import SolarConditions

CACHE_DIR = "./cache/"


class DataStore:
    """Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign
    lookup data using different caching strategies for each."""

    def __init__(self):
        # Constants
        self._MAX_SPOT_COUNT = 100000
        self._MAX_ALERT_COUNT = 100000
        self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
        self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
        # Caches
        self.alerts = None
        self.spots = None
        self.callsign_data_countryfiles = None
        self.callsign_data_clublogxml = None
        self.callsign_data_clublogapi = None
        self.callsign_data_qrz = None
        self.callsign_data_hamqth = None
        self.dxcc_data = None
        self.dxcc_lookup_by_call_regex = []
        self.sigrefs = None
        self.status_data = {}
        self._status = None
        self.solar_conditions = {}
        self._solar = None
        # ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be
        # caches, they can just be straight objects
        self.cq_zone_data = None
        self.itu_zone_data = None

    def setup(self):
        Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)

        # Standard disk cache for solar data and status data, but each cache contains only a single object which we
        # expose to the wider application
        self._solar = diskcache.Cache(CACHE_DIR + "solar")
        if "solar_conditions" not in self._solar:
            self._solar.add("solar_conditions", SolarConditions())
        self.solar_conditions = self._solar.get("solar_conditions")
        self._status = diskcache.Cache(CACHE_DIR + "status")
        if "status_data" not in self._status:
            self._status.add("status_data", {})
        self.status_data = self._status.get("status_data")

        # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
        # a regular basis but there's no need for a TTL since old data is better than no data.
        self.dxcc_data = diskcache.Cache(CACHE_DIR + "dxcc_data")
        self.regenerate_call_regex_to_dxcc_entity_map()

        # For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers
        # of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref"
        # syntax for keys to keep it a single level.
        self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs")
        logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))

        # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
        # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
        # we've seen, so we rely on them timing out and this triggering another lookup.
        self.callsign_data_countryfiles = diskcache.Cache(CACHE_DIR + "callsign_data_countryfiles")
        self.callsign_data_clublogxml = diskcache.Cache(CACHE_DIR + "callsign_data_clublogxml")
        self.callsign_data_clublogapi = diskcache.Cache(CACHE_DIR + "callsign_data_clublogapi")
        self.callsign_data_qrz = diskcache.Cache(CACHE_DIR + "callsign_data_qrz")
        self.callsign_data_hamqth = diskcache.Cache(CACHE_DIR + "callsign_data_hamqth")
        unique_keys = set()
        for c in [self.callsign_data_countryfiles, self.callsign_data_clublogxml, self.callsign_data_clublogapi,
                  self.callsign_data_qrz, self.callsign_data_hamqth]:
            unique_keys.update(c)
        logging.info(f"Loaded data for %d callsigns.", len(unique_keys))

        # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
        # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
        # before the spots and alerts are live in the system.
        self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
                                   snapshot_dir=CACHE_DIR + "spots",
                                   snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
        logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys()))

        self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
                                    snapshot_dir=CACHE_DIR + "alerts",
                                    snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
        logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))

    def regenerate_call_regex_to_dxcc_entity_map(self):
        """DXCC entity data from K0SWE includes a regex which we can use to match a callsign, and determine which DXCC
        entity it belongs to. But getting every DXCC entity data object out of DiskCache, iterating, compiling its regex
        and testing the callsign every time is expensive. So instead we build a separate in-memory lookup of compiled
        regex against DXCC entity code, as a list of tuples we can iterate through."""

        for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]:
            self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))

    def close(self):
        self.spots.close()
        self.alerts.close()
        self._solar.close()
        self._status.close()
        self.dxcc_data.close()
        self.sigrefs.close()
        self.callsign_data_countryfiles.close()
        self.callsign_data_clublogxml.close()
        self.callsign_data_clublogapi.close()
        self.callsign_data_qrz.close()
        self.callsign_data_hamqth.close()

# Global object
DATA_STORE = DataStore()