From bab14bd7ac1b72dd97d0188986aae2299e4354f2 Mon Sep 17 00:00:00 2001 From: Ian Renton Date: Thu, 13 Aug 2026 07:07:15 +0100 Subject: [PATCH] Bring back cleanup thread --- core/cleanup.py | 71 ++++++++++++++++++++++++++++++++++++++ core/status_reporter.py | 4 +++ spothole.py | 4 +++ static/apidocs/openapi.yml | 12 ++++++- static/js/status.js | 3 ++ templates/add_spot.html | 2 +- templates/alerts.html | 2 +- templates/bands.html | 4 +-- templates/base.html | 10 +++--- templates/conditions.html | 2 +- templates/map.html | 4 +-- templates/spots.html | 4 +-- templates/status.html | 7 +++- 13 files changed, 113 insertions(+), 16 deletions(-) create mode 100644 core/cleanup.py diff --git a/core/cleanup.py b/core/cleanup.py new file mode 100644 index 0000000..6ba076a --- /dev/null +++ b/core/cleanup.py @@ -0,0 +1,71 @@ +import logging +from datetime import datetime +from threading import Event, Thread + +import pytz + +from core.data_store import DATA_STORE + + +class CleanupTimer: + """Provides a timed cleanup of the spot list.""" + + def __init__(self): + """Constructor""" + + self._cleanup_interval = None + self.last_cleanup_time = datetime.min.replace(tzinfo=pytz.UTC) + self.status = "Starting" + self._thread = None + self._stop_event = Event() + + def setup(self, cleanup_interval): + self._cleanup_interval = cleanup_interval + + def start(self): + """Start the cleanup timer""" + + self._thread = Thread(target=self._run, daemon=True) + self._thread.start() + + def stop(self): + """Stop any threads and prepare for application shutdown""" + + self._stop_event.set() + + def _run(self): + while not self._stop_event.wait(timeout=self._cleanup_interval): + self._cleanup() + + def _cleanup(self): + """Perform cleanup and reschedule next timer""" + + try: + for i in list(DATA_STORE.spots.iterkeys()): + try: + spot = DATA_STORE.spots[i] + if spot.expired(): + DATA_STORE.spots.delete(i) + except KeyError: + # Must have already been deleted, OK with that + pass + for i in list(DATA_STORE.alerts.iterkeys()): + try: + alert = DATA_STORE.alerts[i] + if alert.expired(): + DATA_STORE.alerts.delete(i) + except KeyError: + # Must have already been deleted, OK with that + pass + + self.status = "OK" + self.last_cleanup_time = datetime.now(pytz.UTC) + + except Exception: + self.status = "Error" + logging.exception("Exception in Cleanup thread") + self._stop_event.wait(timeout=1) + + +# Global object +CLEANUP_TIMER = CleanupTimer() \ No newline at end of file diff --git a/core/status_reporter.py b/core/status_reporter.py index 37917c7..60c18f2 100644 --- a/core/status_reporter.py +++ b/core/status_reporter.py @@ -5,6 +5,7 @@ from threading import Thread, Event import psutil import pytz +from core.cleanup import CLEANUP_TIMER from core.config import SERVER_OWNER_CALLSIGN from core.constants import SOFTWARE_VERSION from core.data_providers import DATA_PROVIDERS @@ -88,6 +89,9 @@ class StatusReporter: tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0, "lookup_count": p.lookup_count}, DATA_PROVIDERS.callsign_data_providers)) + DATA_STORE.status_data["cleanup"] = {"status": CLEANUP_TIMER.status, + "last_ran": CLEANUP_TIMER.last_cleanup_time.replace( + tzinfo=pytz.UTC).timestamp() if CLEANUP_TIMER.last_cleanup_time else 0} DATA_STORE.status_data["webserver"] = {"status": WEB_SERVER.web_server_metrics["status"], "last_api_access": WEB_SERVER.web_server_metrics[ "last_api_access_time"].replace( diff --git a/spothole.py b/spothole.py index fac892c..c01a421 100644 --- a/spothole.py +++ b/spothole.py @@ -4,6 +4,7 @@ import os import signal import sys +from core.cleanup import CLEANUP_TIMER from core.config import SERVER_OWNER_CALLSIGN, LOG_LEVEL from core.constants import SOFTWARE_VERSION from core.data_providers import DATA_PROVIDERS @@ -22,6 +23,7 @@ def shutdown(_signum=None, _frame=None): logging.info("Stopping program...") WEB_SERVER.stop() DATA_PROVIDERS.stop() + CLEANUP_TIMER.stop() DATA_STORE.close() os._exit(0) @@ -47,6 +49,8 @@ if __name__ == '__main__': # Set up data store DATA_STORE.setup() + CLEANUP_TIMER.setup(cleanup_interval=60) + CLEANUP_TIMER.start() # Set up and start data providers DATA_PROVIDERS.setup() diff --git a/static/apidocs/openapi.yml b/static/apidocs/openapi.yml index 9474e48..6c452bf 100644 --- a/static/apidocs/openapi.yml +++ b/static/apidocs/openapi.yml @@ -18,7 +18,6 @@ info: ### 2.0 * **Breaking change:** The "add spot" API has changed to enable future support for upstream submission to the spotting services associated with various SIGs. Instead of just posting the spot object itself as the JSON content of the POST, this has moved into a `spot` object within the structure. A new `handling` object alongside it contains the `submit_upstream`, `upstream_provider`, `upstream_credentials`, and `captcha_token` fields which control the server handling of the spot. - * **Breaking change:** Removed `cleanup` from `/status` response * **Breaking change:** In the `/options` response, renamed `spot_sources` and `alert_sources` to `spot_providers` and `alert_providers` * **Breaking change:** A user's QRZ.com and HamQTH credentials are now supplied as request headers (`X-QRZ-Username`, `X-QRZ-Password`, `X-QRZ-Session-Key`, `X-HamQTH-Username`, `X-HamQTH-Password`, `X-HamQTH-Session-ID`) rather than query parameters, to keep credentials out of server logs. * **Breaking change:** "WAB/WAI GRID" as a location source has been renamed to just "GRID", and also applies to Tiles on the Air spots and grid references from cluster spot comments. @@ -1951,6 +1950,17 @@ components: type: integer description: Number of alerts currently in the system. example: 123 + "cleanup": + type: object + properties: + status: + type: string + description: The status of the cleanup thread + example: OK + last_ran: + type: number + description: The last time the cleanup operation ran, UTC seconds since UNIX epoch. + example: 1759579508 "webserver": type: object properties: diff --git a/static/js/status.js b/static/js/status.js index 90f9c5e..e42dd2e 100644 --- a/static/js/status.js +++ b/static/js/status.js @@ -12,6 +12,9 @@ function loadStatus() { $("#web-server-last-api").text(moment.unix(jsonData["webserver"]["last_api_access"]).utc().fromNow()); $("#web-server-last-page").text(moment.unix(jsonData["webserver"]["last_page_access"]).utc().fromNow()); + $("#cleanup-status").text(jsonData["cleanup"]["status"]); + $("#cleanup-last-ran").text((jsonData["cleanup"]["last_ran"] > 0) ? moment.unix(jsonData["cleanup"]["last_ran"]).utc().fromNow() : "N/A"); + jsonData["spot_providers"].forEach(p => { $("#spot_providers-status-container").append(`
diff --git a/templates/add_spot.html b/templates/add_spot.html index 48575e8..f6abd52 100644 --- a/templates/add_spot.html +++ b/templates/add_spot.html @@ -76,7 +76,7 @@
- + diff --git a/templates/alerts.html b/templates/alerts.html index 4da5b57..d7647c3 100644 --- a/templates/alerts.html +++ b/templates/alerts.html @@ -82,7 +82,7 @@ - + diff --git a/templates/bands.html b/templates/bands.html index 49fc35a..715404e 100644 --- a/templates/bands.html +++ b/templates/bands.html @@ -79,8 +79,8 @@ - - + + diff --git a/templates/base.html b/templates/base.html index b132f47..defc5df 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,6 +1,6 @@ {% extends "skeleton.html" %} {% block head_extra %} - + @@ -10,10 +10,10 @@ - - - - + + + + {% end %} {% block body %}
diff --git a/templates/conditions.html b/templates/conditions.html index 8f08c4d..9c17940 100644 --- a/templates/conditions.html +++ b/templates/conditions.html @@ -284,7 +284,7 @@
- + diff --git a/templates/map.html b/templates/map.html index 6b5fa11..5710acf 100644 --- a/templates/map.html +++ b/templates/map.html @@ -112,8 +112,8 @@ - - + + diff --git a/templates/spots.html b/templates/spots.html index ebe6f3b..c3fc5ee 100644 --- a/templates/spots.html +++ b/templates/spots.html @@ -118,8 +118,8 @@ - - + + diff --git a/templates/status.html b/templates/status.html index 3a6a8ad..06e7990 100644 --- a/templates/status.html +++ b/templates/status.html @@ -24,6 +24,11 @@
Last API call:
Last page req:
+
+
Cleanup Service
+
Status:
+
Last ran:
+
@@ -81,7 +86,7 @@ - +