w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 10:31:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 2 #118

Browse Source
This commit is contained in:
Ian Renton
2026-07-31 15:18:23 +01:00
parent d26ddff7d1
commit 818fd2d504
17 changed files with 186 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/handlers/api/addspot.py
+1 -1
View File
@@ -8,7 +8,7 @@ import tornado
from tornado import httputil
from tornado.web import Application
from core.config import ALLOW_SPOTTING, MAX_SPOT_AGE
from core.config import ALLOW_SPOTTING
from core.constants import UNKNOWN_BAND
from core.lookup_helper import infer_band_from_freq
from core.prometheus_metrics_handler import api_requests_counter
server/handlers/api/alerts.py
+18 -51
View File
@@ -1,7 +1,6 @@
import copy
import logging
from datetime import datetime
from queue import Queue
from typing import Any
import pytz
@@ -11,12 +10,9 @@ from tornado import httputil
from tornado.web import Application
from core.prometheus_metrics_handler import api_requests_counter
from core.utils import safe_json_dumps, empty_queue
from core.utils import safe_json_dumps
from data.lookup_credentials import extract_credentials
SSE_HANDLER_MAX_QUEUE_SIZE = 100
SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
class APIAlertsHandler(tornado.web.RequestHandler):
"""API request handler for /api/v1/alerts"""
@@ -73,16 +69,14 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
"""API request handler for /api/v1/alerts/stream"""
def __init__(self, application, request, **kwargs: Any):
self._sse_alert_queues = None
self._sse_alert_broadcaster = None
self._web_server_metrics = None
self._query_params = None
self._credentials = None
self._alert_queue = None
self._heartbeat = None
super().__init__(application, request, **kwargs)
def initialize(self, sse_alert_queues, web_server_metrics):
self._sse_alert_queues = sse_alert_queues
def initialize(self, _sse_alert_broadcaster, web_server_metrics):
self._sse_alert_broadcaster = _sse_alert_broadcaster
self._web_server_metrics = web_server_metrics
def custom_headers(self):
@@ -104,59 +98,32 @@ class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
self._credentials = extract_credentials(self._query_params)
# Create a alert queue and add it to the web server's list. The web server will fill this when alerts arrive
self._alert_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
self._sse_alert_queues.append(self._alert_queue)
# Set up a timed callback to check if anything is in the queue
self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
self._heartbeat.start()
# Flush headers immediately so nginx doesn't time out waiting for a response
self.write_message("keepalive", "")
# Register to handle new alerts arriving. The callback() method will get called with the new alert as an
# argument.
self._sse_alert_broadcaster.register(self)
except Exception as e:
logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
self.close()
def close(self):
"""When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
"""When the user closes the socket, deregister ourselves from the alert broadcaster"""
try:
if self._alert_queue in self._sse_alert_queues:
self._sse_alert_queues.remove(self._alert_queue)
empty_queue(self._alert_queue)
except:
pass
try:
self._heartbeat.stop()
except:
pass
self._alert_queue = None
self._sse_alert_broadcaster.unregister(self)
super().close()
def _callback(self):
"""Callback to check if anything has arrived in the queue, and if so send it to the client"""
def callback(self, alert):
"""Callback when a new alert arrives"""
try:
if self._alert_queue:
if not self._alert_queue.empty():
while not self._alert_queue.empty():
alert = self._alert_queue.get()
# If the new alert matches our param filters, send it to the client. If not, ignore it.
if alert_allowed_by_query(alert, self._query_params):
if self._credentials:
alert = copy.deepcopy(alert)
alert.infer_missing(self._credentials)
self.write_message(msg=safe_json_dumps(alert))
else:
# Send a keepalive comment if the queue was empty
self.write_message("keepalive", "")
if self._alert_queue not in self._sse_alert_queues:
logging.error("Web server cleared up a queue of an active connection!")
self.close()
if alert_allowed_by_query(alert, self._query_params):
if self._credentials:
alert = copy.deepcopy(alert)
alert.infer_missing(self._credentials)
self.write_message(msg=safe_json_dumps(alert))
except Exception as e:
logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
self.close()
@@ -169,7 +136,7 @@ def get_alert_list_with_filters(all_alerts, query):
# Create a shallow copy of the alert list ordered by start time, then filter the list to reduce it only to alerts
# that match the filter parameters in the query string. Finally, apply a limit to the number of alerts returned.
# The list of query string filters is defined in the API docs.
alert_ids = list(all_alerts.iterkeys())
alert_ids = all_alerts.keys()
alerts = []
for k in alert_ids:
a = all_alerts.get(k)
server/handlers/api/dxstats.py
+1 -1
View File
@@ -40,7 +40,7 @@ class APIDxStatsHandler(tornado.web.RequestHandler):
one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp()
counts = Counter()
for key in self._spots.iterkeys():
for key in self._spots.keys():
spot = self._spots.get(key)
if spot is None:
continue
server/handlers/api/spots.py
+19 -51
View File
@@ -1,7 +1,6 @@
import copy
import logging
from datetime import datetime, timedelta
from queue import Queue
from typing import Any
import pytz
@@ -11,12 +10,9 @@ from tornado import httputil
from tornado.web import Application
from core.prometheus_metrics_handler import api_requests_counter
from core.utils import safe_json_dumps, empty_queue
from core.utils import safe_json_dumps
from data.lookup_credentials import extract_credentials
SSE_HANDLER_MAX_QUEUE_SIZE = 1000
SSE_HANDLER_QUEUE_CHECK_INTERVAL = 5000
class APISpotsHandler(tornado.web.RequestHandler):
"""API request handler for /api/v1/spots"""
@@ -73,16 +69,14 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
"""API request handler for /api/v1/spots/stream"""
def __init__(self, application, request, **kwargs: Any):
self._sse_spot_queues = None
self._sse_spot_broadcaster = None
self._web_server_metrics = None
self._query_params = None
self._credentials = None
self._spot_queue = None
self._heartbeat = None
super().__init__(application, request, **kwargs)
def initialize(self, sse_spot_queues, web_server_metrics):
self._sse_spot_queues = sse_spot_queues
def initialize(self, sse_spot_broadcaster, web_server_metrics):
self._sse_spot_broadcaster = sse_spot_broadcaster
self._web_server_metrics = web_server_metrics
def custom_headers(self):
@@ -106,59 +100,33 @@ class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
self._credentials = extract_credentials(self._query_params)
# Create a spot queue and add it to the web server's list. The web server will fill this when spots arrive
self._spot_queue = Queue(maxsize=SSE_HANDLER_MAX_QUEUE_SIZE)
self._sse_spot_queues.append(self._spot_queue)
# Set up a timed callback to check if anything is in the queue
self._heartbeat = tornado.ioloop.PeriodicCallback(self._callback, SSE_HANDLER_QUEUE_CHECK_INTERVAL)
self._heartbeat.start()
# Flush headers immediately so nginx doesn't time out waiting for a response
self.write_message("keepalive", "")
# Register to handle new spots arriving. The callback() method will get called with the new spot as an
# argument.
self._sse_spot_broadcaster.register(self)
except Exception as e:
logging.warning("Exception when serving SSE socket: %s", e, exc_info=True)
self.close()
def close(self):
"""When the user closes the socket, empty our queue and remove it from the list so the server no longer fills it"""
"""When the user closes the socket, deregister ourselves from the spot broadcaster"""
try:
if self._spot_queue in self._sse_spot_queues:
self._sse_spot_queues.remove(self._spot_queue)
empty_queue(self._spot_queue)
except:
pass
try:
self._heartbeat.stop()
except:
pass
self._spot_queue = None
self._sse_spot_broadcaster.unregister(self)
super().close()
def _callback(self):
"""Callback to check if anything has arrived in the queue, and if so send it to the client"""
def callback(self, spot):
"""Callback when a new spot arrives"""
try:
if self._spot_queue:
if not self._spot_queue.empty():
while not self._spot_queue.empty():
spot = self._spot_queue.get()
# If the new spot matches our param filters, send it to the client. If not, ignore it.
if spot_allowed_by_query(spot, self._query_params):
if self._credentials:
spot = copy.deepcopy(spot)
spot.infer_missing(self._credentials)
self.write_message(msg=safe_json_dumps(spot))
else:
# Send a keepalive comment if the queue was empty
self.write_message("keepalive", "")
if self._spot_queue not in self._sse_spot_queues:
logging.error("Web server cleared up a queue of an active connection!")
self.close()
# If the new spot matches our param filters, send it to the client. If not, ignore it.
if spot_allowed_by_query(spot, self._query_params):
if self._credentials:
spot = copy.deepcopy(spot)
spot.infer_missing(self._credentials)
self.write_message(msg=safe_json_dumps(spot))
except Exception as e:
logging.warning("Exception in SSE callback, connection will be closed: %s", e, exc_info=True)
self.close()
@@ -171,7 +139,7 @@ def get_spot_list_with_filters(all_spots, query):
# Create a shallow copy of the spot list, ordered by spot time, then filter the list to reduce it only to spots
# that match the filter parameters in the query string. Finally, apply a limit to the number of spots returned.
# The list of query string filters is defined in the API docs.
spot_ids = list(all_spots.iterkeys())
spot_ids = all_spots.keys()
spots = []
for k in spot_ids:
s = all_spots.get(k)