w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 02:21:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 3 #118

Browse Source
This commit is contained in:
Ian Renton
2026-07-31 17:45:09 +01:00
parent 818fd2d504
commit 0f59af6f9e
24 changed files with 592 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/data_store.py
+21 -26
View File
@@ -12,57 +12,52 @@ class DataStore:
lookup data using different caching strategies for each."""
def __init__(self):
cache_dir = "./cache"
self.CACHE_DIR = "./cache"
self.MAX_SPOT_COUNT = 10000
self.MAX_ALERT_COUNT = 10000
self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
Path(cache_dir).mkdir(parents=True, exist_ok=True)
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval
self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
snapshot_dir=cache_dir + "/spots",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
snapshot_dir=cache_dir + "/alerts",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
def setup(self):
Path(self.CACHE_DIR).mkdir(parents=True, exist_ok=True)
# Standard disk cache for solar data and status data, but each cache contains only a single object which we
# expose to the wider application
self.solar = diskcache.Cache(cache_dir + "/solar")
self.solar = diskcache.Cache(self.CACHE_DIR + "/solar")
if "solar_conditions" not in self.solar:
self.solar.add("solar_conditions", SolarConditions())
self.solar_conditions = self.solar.get("solar_conditions")
self.status = diskcache.Cache(cache_dir + "/status")
self.status = diskcache.Cache(self.CACHE_DIR + "/status")
if "status_data" not in self.status:
self.status.add("status_data", {})
self.status_data = self.status.get("status_data")
# Standard disk caches for SIG ref data. Separate provider threads will repopulate these on a regular basis
# but there's no need for a TTL since old data is better than no data.
self.sigrefs_wwff = diskcache.Cache(cache_dir + "/sigrefs/wwff")
self.sigrefs_siota = diskcache.Cache(cache_dir + "/sigrefs/siota")
self.sigrefs_wota = diskcache.Cache(cache_dir + "/sigrefs/wota")
self.sigrefs_zlota = diskcache.Cache(cache_dir + "/sigrefs/zlota")
self.sigrefs_llota = diskcache.Cache(cache_dir + "/sigrefs/llota")
self.sigrefs_dme = diskcache.Cache(cache_dir + "/sigrefs/dme")
# Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
# but there's no need for a TTL since old data is better than no data. This is a two-layer dict, keys are SIG
# name and then reference ID, with the final value being a SIGRef object.
self.sigrefs = diskcache.Cache(self.CACHE_DIR + "/sigrefs")
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
# we've seen, so we rely on them timing out and this triggering another lookup.
self.callsigns = diskcache.Cache(cache_dir + "/callsigns")
self.callsigns = diskcache.Cache(self.CACHE_DIR + "/callsigns")
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
# specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
# before the spots and alerts are live in the system.
self.spots = LiveDataCache(maxsize=self.MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
snapshot_dir=self.CACHE_DIR + "/spots",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
self.alerts = LiveDataCache(maxsize=self.MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
snapshot_dir=self.CACHE_DIR + "/alerts",
snapshot_interval_sec=self.SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
def close(self):
self.spots.close()
self.alerts.close()
self.solar.close()
self.status.close()
self.sigrefs_wwff.close()
self.sigrefs_siota.close()
self.sigrefs_wota.close()
self.sigrefs_zlota.close()
self.sigrefs_llota.close()
self.sigrefs.close()
self.callsigns.close()
# Global object