w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 02:21:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 12 #118

Browse Source
This commit is contained in:
Ian Renton
2026-08-02 21:57:22 +01:00
parent cee9188538
commit e31b145b8e
11 changed files with 227 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/data_store.py
+21 -5
View File
@@ -20,11 +20,15 @@ class DataStore:
self._MAX_SPOT_COUNT = 100000
self._MAX_ALERT_COUNT = 100000
self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
self.CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
# Caches
self.alerts = None
self.spots = None
self.callsigns = None
self.callsign_data_countryfiles = None
self.callsign_data_clublogxml = None
self.callsign_data_clublogapi = None
self.callsign_data_qrz = None
self.callsign_data_hamqth = None
self.dxcc_data = None
self.dxcc_lookup_by_call_regex = []
self.sigrefs = None
@@ -65,8 +69,16 @@ class DataStore:
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
# we've seen, so we rely on them timing out and this triggering another lookup.
self.callsigns = diskcache.Cache(CACHE_DIR + "callsigns")
logging.info(f"Loaded data for %d callsigns.", len(self.callsigns))
self.callsign_data_countryfiles = diskcache.Cache(CACHE_DIR + "callsign_data_countryfiles")
self.callsign_data_clublogxml = diskcache.Cache(CACHE_DIR + "callsign_data_clublogxml")
self.callsign_data_clublogapi = diskcache.Cache(CACHE_DIR + "callsign_data_clublogapi")
self.callsign_data_qrz = diskcache.Cache(CACHE_DIR + "callsign_data_qrz")
self.callsign_data_hamqth = diskcache.Cache(CACHE_DIR + "callsign_data_hamqth")
unique_keys = set()
for c in [self.callsign_data_countryfiles, self.callsign_data_clublogxml, self.callsign_data_clublogapi,
self.callsign_data_qrz, self.callsign_data_hamqth]:
unique_keys.update(c)
logging.info(f"Loaded data for %d callsigns.", len(unique_keys))
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
# specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
@@ -97,7 +109,11 @@ class DataStore:
self._status.close()
self.dxcc_data.close()
self.sigrefs.close()
self.callsigns.close()
self.callsign_data_countryfiles.close()
self.callsign_data_clublogxml.close()
self.callsign_data_clublogapi.close()
self.callsign_data_qrz.close()
self.callsign_data_hamqth.close()
# Global object
DATA_STORE = DataStore()