Partial fix for mypy issues

This commit is contained in:
Ian Renton
2026-09-20 20:42:16 +01:00
parent 93ea27510f
commit 203758fa2d
25 changed files with 244 additions and 147 deletions
+83 -25
View File
@@ -14,7 +14,8 @@ from core.single_object_data_cache import SingleObjectDataCache
from data.solar_conditions import SolarConditions
if TYPE_CHECKING:
# Deferred to avoid a circular import: data.alert and data.spot both import core.data_store at module level.
# Can't find a way to resolve the circular dependency on types but apparently this is a way of managing that
# while still having type safety in method definitions.
from data.alert import Alert
from data.spot import Spot
@@ -33,52 +34,109 @@ class DataStore:
self._MAX_ALERT_COUNT = 100000
self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
# Caches
self.alerts: LiveDataCache[Alert] | None = None
self.spots: LiveDataCache[Spot] | None = None
self.callsign_data_countryfiles: diskcache.Cache | None = None
self.callsign_data_clublogxml: diskcache.Cache | None = None
self.callsign_data_clublogapi: diskcache.Cache | None = None
self.callsign_data_qrz: diskcache.Cache | None = None
self.callsign_data_hamqth: diskcache.Cache | None = None
self.dxcc_data: diskcache.Cache | None = None
# Caches. These are all created by setup(), which must be called before use; they are None only in the window
# between construction of this (global, single-instance) object and that call.
self._alerts: LiveDataCache[Alert] | None = None
self._spots: LiveDataCache[Spot] | None = None
self._callsign_data_countryfiles: diskcache.Cache | None = None
self._callsign_data_clublogxml: diskcache.Cache | None = None
self._callsign_data_clublogapi: diskcache.Cache | None = None
self._callsign_data_qrz: diskcache.Cache | None = None
self._callsign_data_hamqth: diskcache.Cache | None = None
self._dxcc_data: diskcache.Cache | None = None
self.dxcc_lookup_by_call_regex: list[tuple[re.Pattern[str], Any]] = []
self.activity_refs: diskcache.Cache | None = None
self.status: SingleObjectDataCache[dict[str, Any]] | None = None
self.solar_conditions: SingleObjectDataCache[SolarConditions] | None = None
self._activity_refs: diskcache.Cache | None = None
self._status: SingleObjectDataCache[dict[str, Any]] | None = None
self._solar_conditions: SingleObjectDataCache[SolarConditions] | None = None
# ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be
# caches, they can just be straight objects
# caches, they can just be straight objects. Unlike the caches above, these genuinely may never be populated,
# if the corresponding static data provider isn't configured, so callers must handle None.
self.cq_zone_data: geopandas.GeoDataFrame | None = None
self.itu_zone_data: geopandas.GeoDataFrame | None = None
@property
def alerts(self) -> LiveDataCache[Alert]:
assert self._alerts is not None, "DataStore.setup() must be called before use"
return self._alerts
@property
def spots(self) -> LiveDataCache[Spot]:
assert self._spots is not None, "DataStore.setup() must be called before use"
return self._spots
@property
def callsign_data_countryfiles(self) -> diskcache.Cache:
assert self._callsign_data_countryfiles is not None, "DataStore.setup() must be called before use"
return self._callsign_data_countryfiles
@property
def callsign_data_clublogxml(self) -> diskcache.Cache:
assert self._callsign_data_clublogxml is not None, "DataStore.setup() must be called before use"
return self._callsign_data_clublogxml
@property
def callsign_data_clublogapi(self) -> diskcache.Cache:
assert self._callsign_data_clublogapi is not None, "DataStore.setup() must be called before use"
return self._callsign_data_clublogapi
@property
def callsign_data_qrz(self) -> diskcache.Cache:
assert self._callsign_data_qrz is not None, "DataStore.setup() must be called before use"
return self._callsign_data_qrz
@property
def callsign_data_hamqth(self) -> diskcache.Cache:
assert self._callsign_data_hamqth is not None, "DataStore.setup() must be called before use"
return self._callsign_data_hamqth
@property
def dxcc_data(self) -> diskcache.Cache:
assert self._dxcc_data is not None, "DataStore.setup() must be called before use"
return self._dxcc_data
@property
def activity_refs(self) -> diskcache.Cache:
assert self._activity_refs is not None, "DataStore.setup() must be called before use"
return self._activity_refs
@property
def status(self) -> SingleObjectDataCache[dict[str, Any]]:
assert self._status is not None, "DataStore.setup() must be called before use"
return self._status
@property
def solar_conditions(self) -> SingleObjectDataCache[SolarConditions]:
assert self._solar_conditions is not None, "DataStore.setup() must be called before use"
return self._solar_conditions
def setup(self) -> None:
Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
# For solar data and status data, we use a wrapper around disk cache where each cache contains only a single
# object exposed to the wider application, and provides a store() method for callers to notify diskcache that
# the object has changed and needs to be re-cached.
self.solar_conditions = SingleObjectDataCache(f"{CACHE_DIR}solar", SolarConditions())
self.status = SingleObjectDataCache(f"{CACHE_DIR}status", {})
self._solar_conditions = SingleObjectDataCache(f"{CACHE_DIR}solar", SolarConditions())
self._status = SingleObjectDataCache(f"{CACHE_DIR}status", {})
# Standard disk cache for static reference and activity ref data. Separate provider threads will repopulate
# these on a regular basis but there's no need for a TTL since old data is better than no data.
self.dxcc_data = diskcache.Cache(f"{CACHE_DIR}dxcc_data")
self._dxcc_data = diskcache.Cache(f"{CACHE_DIR}dxcc_data")
self.regenerate_call_regex_to_dxcc_entity_map()
# For activity reference data specifically, we need to key on both activity *and* reference, and trying to do
# two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an
# ugly "activity:ref" syntax for keys to keep it a single level.
self.activity_refs = diskcache.Cache(f"{CACHE_DIR}activity_refs")
self._activity_refs = diskcache.Cache(f"{CACHE_DIR}activity_refs")
logger.info(f"Loaded data for {len(self.activity_refs)} activity references.")
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
# we've seen, so we rely on them timing out and this triggering another lookup.
self.callsign_data_countryfiles = diskcache.Cache(f"{CACHE_DIR}callsign_data_countryfiles")
self.callsign_data_clublogxml = diskcache.Cache(f"{CACHE_DIR}callsign_data_clublogxml")
self.callsign_data_clublogapi = diskcache.Cache(f"{CACHE_DIR}callsign_data_clublogapi")
self.callsign_data_qrz = diskcache.Cache(f"{CACHE_DIR}callsign_data_qrz")
self.callsign_data_hamqth = diskcache.Cache(f"{CACHE_DIR}callsign_data_hamqth")
self._callsign_data_countryfiles = diskcache.Cache(f"{CACHE_DIR}callsign_data_countryfiles")
self._callsign_data_clublogxml = diskcache.Cache(f"{CACHE_DIR}callsign_data_clublogxml")
self._callsign_data_clublogapi = diskcache.Cache(f"{CACHE_DIR}callsign_data_clublogapi")
self._callsign_data_qrz = diskcache.Cache(f"{CACHE_DIR}callsign_data_qrz")
self._callsign_data_hamqth = diskcache.Cache(f"{CACHE_DIR}callsign_data_hamqth")
unique_keys = set()
for c in [
self.callsign_data_countryfiles,
@@ -93,7 +151,7 @@ class DataStore:
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
# specifically load these caches *last* so that any activity ref and callsign data is already loaded from disk
# cache before the spots and alerts are live in the system.
self.spots = LiveDataCache(
self._spots = LiveDataCache(
maxsize=self._MAX_SPOT_COUNT,
ttl=MAX_SPOT_AGE,
snapshot_dir=f"{CACHE_DIR}spots",
@@ -101,7 +159,7 @@ class DataStore:
)
logger.info(f"Loaded {len(self.spots.keys())} spots from a previous run.")
self.alerts = LiveDataCache(
self._alerts = LiveDataCache(
maxsize=self._MAX_ALERT_COUNT,
ttl=MAX_ALERT_AGE,
snapshot_dir=f"{CACHE_DIR}alerts",