w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 02:21:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Improve use of URL cache #118

Browse Source
This commit is contained in:
Ian Renton
2026-08-01 08:06:44 +01:00
parent fe10943ecc
commit c472872e10
5 changed files with 41 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/data_store.py
+9 -9
View File
@@ -4,17 +4,17 @@ from pathlib import Path
import diskcache
from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
from core.constants import SIGS
from core.live_data_cache import LiveDataCache
from data.solar_conditions import SolarConditions
CACHE_DIR = "./cache/"
class DataStore:
"""Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign
lookup data using different caching strategies for each."""
def __init__(self):
self._CACHE_DIR = "./cache"
self._MAX_SPOT_COUNT = 100000
self._MAX_ALERT_COUNT = 100000
self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
@@ -29,15 +29,15 @@ class DataStore:
self._solar = None
def setup(self):
Path(self._CACHE_DIR).mkdir(parents=True, exist_ok=True)
Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
# Standard disk cache for solar data and status data, but each cache contains only a single object which we
# expose to the wider application
self._solar = diskcache.Cache(self._CACHE_DIR + "/solar")
self._solar = diskcache.Cache(CACHE_DIR + "solar")
if "solar_conditions" not in self._solar:
self._solar.add("solar_conditions", SolarConditions())
self.solar_conditions = self._solar.get("solar_conditions")
self._status = diskcache.Cache(self._CACHE_DIR + "/status")
self._status = diskcache.Cache(CACHE_DIR + "status")
if "status_data" not in self._status:
self._status.add("status_data", {})
self.status_data = self._status.get("status_data")
@@ -46,25 +46,25 @@ class DataStore:
# but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
# and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
# so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
self.sigrefs = diskcache.Cache(self._CACHE_DIR + "/sigrefs")
self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs")
logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
# we've seen, so we rely on them timing out and this triggering another lookup.
self.callsigns = diskcache.Cache(self._CACHE_DIR + "/callsigns")
self.callsigns = diskcache.Cache(CACHE_DIR + "callsigns")
logging.info(f"Loaded data for %d callsigns.", len(self.callsigns))
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
# specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
# before the spots and alerts are live in the system.
self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
snapshot_dir=self._CACHE_DIR + "/spots",
snapshot_dir=CACHE_DIR + "spots",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys()))
self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
snapshot_dir=self._CACHE_DIR + "/alerts",
snapshot_dir=CACHE_DIR + "alerts",
snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))