mirror of
https://git.ianrenton.com/ian/spothole.git
synced 2026-09-21 06:47:42 +00:00
119 lines
4.7 KiB
Python
119 lines
4.7 KiB
Python
import logging
|
|
import threading
|
|
import time
|
|
from collections.abc import Callable
|
|
from typing import Generic, TypeVar
|
|
|
|
import diskcache
|
|
from cachetools import TTLCache
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
VT = TypeVar("VT")
|
|
|
|
|
|
class LiveDataCache(Generic[VT]):
|
|
"""Cache for spots and alerts. Uses the fast in-memory TTLCache for normal data I/O, including the TTL to enforce
|
|
maximum lifetime, and adds a separate diskcache to which we can save and load the TTLCache to provide persistence.
|
|
Also adds thread safety so spots and alerts can come from any thread, and a listener mechanism so the web server
|
|
can get a callback when new spots/alerts are added, and send them to any SSE clients."""
|
|
|
|
def __init__(self, maxsize: int, ttl: int, snapshot_dir: str, snapshot_interval_sec: int) -> None:
|
|
self._cache: TTLCache = TTLCache(maxsize=maxsize, ttl=ttl)
|
|
self._lock = threading.Lock()
|
|
self._ttl = ttl
|
|
self._listeners: list[Callable[[VT], None]] = []
|
|
self._listeners_lock = threading.Lock()
|
|
self._snapshot_dir = snapshot_dir
|
|
self._disk_cache = diskcache.Cache(str(snapshot_dir))
|
|
self._stop_event = threading.Event()
|
|
self._snapshot_thread: threading.Thread | None = None
|
|
self._load_snapshot()
|
|
self._start_periodic_snapshot(snapshot_interval_sec)
|
|
|
|
def set(self, key: str, value: VT) -> None:
|
|
with self._lock:
|
|
self._cache[key] = value
|
|
|
|
# Notify listeners
|
|
with self._listeners_lock:
|
|
listeners = list(self._listeners)
|
|
for callback in listeners:
|
|
try:
|
|
callback(value)
|
|
except Exception:
|
|
logger.exception(f"Listener raised an exception for key {key}")
|
|
|
|
def get(self, key: str, default: VT | None = None) -> VT | None:
|
|
with self._lock:
|
|
return self._cache.get(key, default)
|
|
|
|
def delete(self, key: str) -> None:
|
|
with self._lock:
|
|
self._cache.pop(key, None)
|
|
|
|
def keys(self) -> list[str]:
|
|
with self._lock:
|
|
return list(self._cache.keys())
|
|
|
|
def values(self) -> list[VT]:
|
|
with self._lock:
|
|
return list(self._cache.values())
|
|
|
|
def add_listener(self, callback: Callable[[VT], None]) -> None:
|
|
"""Register callback(value) which will be called whenever a new spot/alert item is added via set(). Used by the
|
|
web server (via SSEBroadcaster) to send SSE clients an update on every new spot."""
|
|
|
|
with self._listeners_lock:
|
|
self._listeners.append(callback)
|
|
|
|
def remove_listener(self, callback: Callable[[VT], None]) -> None:
|
|
with self._listeners_lock:
|
|
self._listeners.remove(callback)
|
|
|
|
def save_snapshot(self) -> None:
|
|
with self._lock:
|
|
# Store the time with the data so we can avoid loading anything nxt time that's older than TTL
|
|
data = [(k, v, time.time()) for k, v in self._cache.items()]
|
|
try:
|
|
self._disk_cache.set("snapshot", data)
|
|
except Exception:
|
|
logger.exception(f"Failed to write snapshot to {self._snapshot_dir}")
|
|
|
|
def _load_snapshot(self) -> None:
|
|
try:
|
|
data: list[tuple[str, VT, float]] | None = self._disk_cache.get("snapshot")
|
|
except Exception: # noqa: BLE001 (If any exceptions we treat the data as junk and start from scratch, it's probably due to a version upgrade having incompatible data structures, fine to not log the exception in this case)
|
|
logger.warning(f"Failed to load snapshot from {self._snapshot_dir}, clearing it.")
|
|
self._disk_cache.clear()
|
|
return
|
|
if not data:
|
|
return
|
|
|
|
now = time.time()
|
|
with self._lock:
|
|
for key, value, saved_at in data:
|
|
# Only restore entries that would still be within TTL
|
|
if now - saved_at < self._ttl:
|
|
self._cache[key] = value
|
|
logger.info(f"Loaded snapshot from {self._snapshot_dir}")
|
|
|
|
def _start_periodic_snapshot(self, interval: int) -> None:
|
|
def loop() -> None:
|
|
while not self._stop_event.wait(timeout=interval):
|
|
self.save_snapshot()
|
|
|
|
self._snapshot_thread = threading.Thread(
|
|
target=loop, name=f"LiveDataCache-Snapshot-{self._snapshot_dir}", daemon=True
|
|
)
|
|
self._snapshot_thread.start()
|
|
|
|
def close(self) -> None:
|
|
self._stop_event.set()
|
|
if self._snapshot_thread:
|
|
self._snapshot_thread.join(timeout=15)
|
|
if self._snapshot_thread.is_alive():
|
|
logger.warning(f"LiveDataCache snapshot thread for {self._snapshot_dir} did not exit on time.")
|
|
self.save_snapshot()
|
|
self._disk_cache.close()
|