w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 02:21:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 2 #118

Browse Source
This commit is contained in:
Ian Renton
2026-07-31 15:18:23 +01:00
parent d26ddff7d1
commit 818fd2d504
17 changed files with 186 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/live_data_cache.py
+30 -2
View File
@@ -7,14 +7,17 @@ from cachetools import TTLCache
class LiveDataCache:
"""Cache for spots and alerts. Uses the faster in-memory TTLCache for normal data I/O, including the TTL to enforce
"""Cache for spots and alerts. Uses the fast in-memory TTLCache for normal data I/O, including the TTL to enforce
maximum lifetime, and adds a separate diskcache to which we can save and load the TTLCache to provide persistence.
Also adds thread safety which TTLCache doesn't do."""
Also adds thread safety so spots and alerts can come from any thread, and a listener mechanism so the web server
can get a callback when new spots/alerts are added, and send them to any SSE clients."""
def __init__(self, maxsize, ttl, snapshot_dir, snapshot_interval_sec):
self._cache = TTLCache(maxsize=maxsize, ttl=ttl)
self._lock = threading.Lock()
self._ttl = ttl
self._listeners = []
self._listeners_lock = threading.Lock()
self._snapshot_dir = snapshot_dir
self._disk_cache = diskcache.Cache(str(snapshot_dir))
self._load_snapshot()
@@ -24,6 +27,16 @@ class LiveDataCache:
with self._lock:
self._cache[key] = value
# Notify listeners
with self._listeners_lock:
listeners = list(self._listeners)
for callback in listeners:
try:
callback(value)
except Exception:
logging.error("Listener raised an exception for key %s", key, exc_info=True)
def get(self, key, default=None):
with self._lock:
return self._cache.get(key, default)
@@ -32,10 +45,25 @@ class LiveDataCache:
with self._lock:
self._cache.pop(key, None)
def keys(self):
with self._lock:
return list(self._cache.keys())
def values(self):
with self._lock:
return list(self._cache.values())
def add_listener(self, callback):
"""Register callback(value) which will be called whenever a new spot/alert item is added via set(). Used by the
web server (via SSEBroadcaster) to send SSE clients an update on every new spot."""
with self._listeners_lock:
self._listeners.append(callback)
def remove_listener(self, callback):
with self._listeners_lock:
self._listeners.remove(callback)
def save_snapshot(self):
with self._lock:
# Store the time with the data so we can avoid loading anything nxt time that's older than TTL
core/lookup_helper.py
+1 -1
View File
@@ -14,10 +14,10 @@ from pyhamtools.locator import latlong_to_locator
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from requests_cache import CachedSession
from core.url_data_cache import URL_DATA_CACHE
from core.config import config
from core.constants import BANDS, UNKNOWN_BAND, CW_MODES, PHONE_MODES, DATA_MODES, ALL_MODES, \
HTTP_HEADERS, HAMQTH_PRG, MODE_ALIASES
from core.url_data_cache import URL_DATA_CACHE
# QRZ XML field names differ from pyhamtools' normalised names; map them here.
_QRZ_FIELD_MAP = {
core/status_reporter.py
+14 -14
View File
@@ -7,16 +7,16 @@ import pytz
from core.config import SERVER_OWNER_CALLSIGN
from core.constants import SOFTWARE_VERSION
from core.data_store import DATA_STORE
from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge
class StatusReporter:
"""Provides a timed update of the application's status data."""
def __init__(self, data_store, run_interval, web_server,spot_providers, alert_providers, solar_condition_providers):
def __init__(self, run_interval, web_server,spot_providers, alert_providers, solar_condition_providers):
"""Constructor"""
self._data_store = data_store
self._run_interval = run_interval
self._web_server = web_server
self._spot_providers = spot_providers
@@ -26,8 +26,8 @@ class StatusReporter:
self._stop_event = Event()
self._startup_time = datetime.now(pytz.UTC)
self._data_store.status_data["software-version"] = SOFTWARE_VERSION
self._data_store.status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
DATA_STORE.status_data["software-version"] = SOFTWARE_VERSION
DATA_STORE.status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
def start(self):
"""Start the reporter thread"""
@@ -51,28 +51,28 @@ class StatusReporter:
def _report(self):
"""Write status information"""
self._data_store.status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
self._data_store.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
self._data_store.status_data["num_spots"] = len(self._data_store.spots.values())
self._data_store.status_data["num_alerts"] = len(self._data_store.alerts.values())
self._data_store.status_data["spot_providers"] = list(
DATA_STORE.status_data["uptime"] = (datetime.now(pytz.UTC) - self._startup_time).total_seconds()
DATA_STORE.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
DATA_STORE.status_data["spot_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
"last_spot": p.last_spot_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
self._spot_providers))
self._data_store.status_data["alert_providers"] = list(
DATA_STORE.status_data["alert_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
self._alert_providers))
self._data_store.status_data["solar_condition_providers"] = list(
DATA_STORE.status_data["solar_condition_providers"] = list(
map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
"last_updated": p.last_update_time.replace(
tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
self._solar_condition_providers))
self._data_store.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
DATA_STORE.status_data["webserver"] = {"status": self._web_server.web_server_metrics["status"],
"last_api_access": self._web_server.web_server_metrics[
"last_api_access_time"].replace(
tzinfo=pytz.UTC).timestamp() if self._web_server.web_server_metrics[
@@ -87,5 +87,5 @@ class StatusReporter:
# Update Prometheus metrics
memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)
spots_gauge.set(len(self._data_store.spots.values()))
alerts_gauge.set(len(self._data_store.alerts.values()))
spots_gauge.set(len(DATA_STORE.spots.values()))
alerts_gauge.set(len(DATA_STORE.alerts.values()))
core/utils.py
+1 -11
View File
@@ -5,14 +5,4 @@ def safe_json_dumps(obj):
"""Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats
which are invalid in JSON."""
return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)
def empty_queue(q):
"""Empty a queue"""
while not q.empty():
try:
q.get_nowait()
except:
break
return simplejson.dumps(obj, ensure_ascii=False, ignore_nan=True, default=lambda o: o.__dict__)