From 1a9dc0b634d35efcbd3f4f662ae762b1f9635ac7 Mon Sep 17 00:00:00 2001 From: Ian Renton Date: Thu, 2 Oct 2025 09:57:25 +0100 Subject: [PATCH] Implement a max spot age filter. Closes #18 --- core/config.py | 6 +++++- main.py | 10 +++++++--- providers/aprsis.py | 4 ++-- providers/dxcluster.py | 4 ++-- providers/provider.py | 4 ++-- providers/rbn.py | 4 ++-- server/webserver.py | 10 ++++++++-- webassets/apidocs/openapi.yml | 24 +++++++++++++++++++++--- 8 files changed, 49 insertions(+), 17 deletions(-) diff --git a/core/config.py b/core/config.py index 52c04f2..0c2d630 100644 --- a/core/config.py +++ b/core/config.py @@ -10,4 +10,8 @@ if not os.path.isfile("config.yml"): # Load config config = yaml.safe_load(open("config.yml")) -logging.info("Loaded config.") \ No newline at end of file +logging.info("Loaded config.") + +MAX_SPOT_AGE = config["max-spot-age-sec"] +SERVER_OWNER_CALLSIGN = config["server-owner-callsign"] +WEB_SERVER_PORT = config["web-server-port"] \ No newline at end of file diff --git a/main.py b/main.py index 70c0796..df45cf8 100644 --- a/main.py +++ b/main.py @@ -10,7 +10,8 @@ import psutil import pytz from core.cleanup import CleanupTimer -from core.config import config +from core.config import config, MAX_SPOT_AGE, WEB_SERVER_PORT, SERVER_OWNER_CALLSIGN +from core.constants import SOFTWARE_VERSION from providers.aprsis import APRSIS from providers.dxcluster import DXCluster from providers.gma import GMA @@ -76,8 +77,11 @@ if __name__ == '__main__': formatter = logging.Formatter("%(message)s") handler.setFormatter(formatter) root.addHandler(handler) + logging.info("Starting...") startup_time = datetime.now(pytz.UTC) + status_data["software-version"] = SOFTWARE_VERSION + status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN # Shut down gracefully on SIGINT signal.signal(signal.SIGINT, shutdown) @@ -92,11 +96,11 @@ if __name__ == '__main__': p.start() # Set up timer to clear spot list of old data - cleanup_timer = CleanupTimer(spot_list=spot_list, cleanup_interval=60, max_spot_age=config["max-spot-age-sec"]) + cleanup_timer = CleanupTimer(spot_list=spot_list, cleanup_interval=60, max_spot_age=MAX_SPOT_AGE) cleanup_timer.start() # Set up web server - web_server = WebServer(spot_list=spot_list, status_data=status_data, port=config["web-server-port"]) + web_server = WebServer(spot_list=spot_list, status_data=status_data, port=WEB_SERVER_PORT) web_server.start() logging.info("Startup complete.") diff --git a/providers/aprsis.py b/providers/aprsis.py index 64bb299..4ece117 100644 --- a/providers/aprsis.py +++ b/providers/aprsis.py @@ -5,7 +5,7 @@ from threading import Thread import aprslib import pytz -from core.config import config +from core.config import SERVER_OWNER_CALLSIGN from data.spot import Spot from providers.provider import Provider @@ -23,7 +23,7 @@ class APRSIS(Provider): self.thread.start() def connect(self): - self.aprsis = aprslib.IS(config["server-owner-callsign"]) + self.aprsis = aprslib.IS(SERVER_OWNER_CALLSIGN) self.status = "Connecting" logging.info("APRS-IS connecting...") self.aprsis.connect() diff --git a/providers/dxcluster.py b/providers/dxcluster.py index 93da002..0647b92 100644 --- a/providers/dxcluster.py +++ b/providers/dxcluster.py @@ -8,7 +8,7 @@ import pytz import telnetlib3 from data.spot import Spot -from core.config import config +from core.config import SERVER_OWNER_CALLSIGN from providers.provider import Provider @@ -47,7 +47,7 @@ class DXCluster(Provider): logging.info("DX Cluster " + self.hostname + " connecting...") self.telnet = telnetlib3.Telnet(self.hostname, self.port) self.telnet.read_until("login: ".encode("latin-1")) - self.telnet.write((config["server-owner-callsign"] + "\n").encode("latin-1")) + self.telnet.write((SERVER_OWNER_CALLSIGN + "\n").encode("latin-1")) connected = True logging.info("DX Cluster " + self.hostname + " connected.") except Exception as e: diff --git a/providers/provider.py b/providers/provider.py index 4984115..13d8c7f 100644 --- a/providers/provider.py +++ b/providers/provider.py @@ -3,14 +3,14 @@ from datetime import datetime import pytz from core.constants import SOFTWARE_NAME, SOFTWARE_VERSION -from core.config import config +from core.config import config, SERVER_OWNER_CALLSIGN # Generic data provider class. Subclasses of this query the individual APIs for data. class Provider: # HTTP headers used for providers that use HTTP - HTTP_HEADERS = { "User-Agent": SOFTWARE_NAME + " " + SOFTWARE_VERSION + " (operated by " + config["server-owner-callsign"] + ")" } + HTTP_HEADERS = { "User-Agent": SOFTWARE_NAME + " " + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")" } # Constructor def __init__(self, provider_config): diff --git a/providers/rbn.py b/providers/rbn.py index 85a8819..f66f910 100644 --- a/providers/rbn.py +++ b/providers/rbn.py @@ -8,7 +8,7 @@ import pytz import telnetlib3 from data.spot import Spot -from core.config import config +from core.config import SERVER_OWNER_CALLSIGN from providers.provider import Provider @@ -48,7 +48,7 @@ class RBN(Provider): logging.info("RBN port " + str(self.port) + " connecting...") self.telnet = telnetlib3.Telnet("telnet.reversebeacon.net", self.port) telnet_output = self.telnet.read_until("Please enter your call: ".encode("latin-1")) - self.telnet.write((config["server-owner-callsign"] + "\n").encode("latin-1")) + self.telnet.write((SERVER_OWNER_CALLSIGN + "\n").encode("latin-1")) connected = True logging.info("RBN port " + str(self.port) + " connected.") except Exception as e: diff --git a/server/webserver.py b/server/webserver.py index c2afd25..61ebc79 100644 --- a/server/webserver.py +++ b/server/webserver.py @@ -1,12 +1,13 @@ import json import logging -from datetime import datetime +from datetime import datetime, timedelta from threading import Thread import bottle import pytz from bottle import run, response +from core.config import MAX_SPOT_AGE from core.constants import BANDS, ALL_MODES, MODE_TYPES, SIGS, SOURCES, CONTINENTS from core.utils import serialize_everything @@ -100,6 +101,10 @@ class WebServer: case "since": since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC) spots = [s for s in spots if s.time > since] + case "max_age": + max_age = int(query.get(k)) + since = datetime.now(pytz.UTC) - timedelta(seconds=max_age) + spots = [s for s in spots if s.time > since] case "received_since": since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC) spots = [s for s in spots if s.received_time > since] @@ -138,4 +143,5 @@ class WebServer: "mode_types": MODE_TYPES, "sigs": SIGS, "sources": SOURCES, - "continents": CONTINENTS} + "continents": CONTINENTS, + "max_spot_age": MAX_SPOT_AGE} diff --git a/webassets/apidocs/openapi.yml b/webassets/apidocs/openapi.yml index d58b9ae..762a052 100644 --- a/webassets/apidocs/openapi.yml +++ b/webassets/apidocs/openapi.yml @@ -4,7 +4,7 @@ info: description: |- (S)pothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open JSON API as well as a website to browse the data. - While there are other web-based interfaces to DX clusters, and sites that aggregate spots from various outfoor activity programmes for amateur radio, (S)pothole differentiates itself by supporting a large number of data sources, and by being "API first" rather than just providing a web front-end. This allows other software to be built on top of it. Spothole itself is also open source, Public Domain licenced code that anyone can take and modify. + While there are other web-based interfaces to DX clusters, and sites that aggregate spots from various outfoor activity programmes for amateur radio, (S)pothole differentiates itself by supporting a large number of data sources, and by being "API first" rather than just providing a web front-end. This allows other software to be built on top of it. (S)pothole itself is also open source, Public Domain licenced code that anyone can take and modify. contact: email: ian@ianrenton.com license: @@ -30,13 +30,19 @@ paths: type: integer - name: since in: query - description: Limit the spots to only ones at this time or later. Time in UTC seconds since UNIX epoch. + description: Limit the spots to only ones at this time or later. Time in UTC seconds since UNIX epoch. Equivalent to "max_age" but saves the client having to work out how many seconds ago "midnight" was. + required: false + schema: + type: integer + - name: max_age + in: query + description: Limit the spots to only ones received in the last 'n' seconds. Equivalent to "since" but saves the client having to work out what time was 'n' seconds ago on every call. Refer to the "max_spot_age" in the /options call to figure out what the maximum useful value you can provide is. Larger values will still be accepted, there just won't be any spots in the system older than max_spot_age. required: false schema: type: integer - name: received_since in: query - description: Limit the spots to only ones that the system found out about at this time or later. Time in UTC seconds since UNIX epoch. If you are using a front-end that tracks the last time it queried the API and requests spots since then, you want *this* version of the query parameter, not "since", because otherwise it may miss things. + description: Limit the spots to only ones that the system found out about at this time or later. Time in UTC seconds since UNIX epoch. If you are using a front-end that tracks the last time it queried the API and requests spots since then, you want *this* version of the query parameter, not "since", because otherwise it may miss things. The logic is "greater than" rather than "greater than or equal to", so you can submit the time of the last received item back to this call and you will get all the more recent spots back, without duplicating the previous latest spot. required: false schema: type: integer @@ -188,6 +194,14 @@ paths: schema: type: object properties: + "software-version": + type: string + description: The version number of the software. + example: "1.0.1" + "server-owner-callsign": + type: string + description: The callsign of this server's operator. + example: "M0TRT" "uptime": type: string description: The amount of time the software has been running for. @@ -283,6 +297,10 @@ paths: items: type: string example: "EU" + max_spot_age: + type: integer + description: The maximum age, in seconds, of any spot before it will be deleted by the system. When querying the /api/spots endpoint and providing a "max_age" or "since" parameter, there is no point providing a number larger than this, because the system drops all spots older than this. + example: 3600 components: schemas: