Implement a max spot age filter. Closes #18

core/config.py

@@ -10,4 +10,8 @@ if not os.path.isfile("config.yml"):
 
 # Load config
 config = yaml.safe_load(open("config.yml"))
-logging.info("Loaded config.")
+logging.info("Loaded config.")
+
+MAX_SPOT_AGE = config["max-spot-age-sec"]
+SERVER_OWNER_CALLSIGN = config["server-owner-callsign"]
+WEB_SERVER_PORT = config["web-server-port"]

main.py

@@ -10,7 +10,8 @@ import psutil
 import pytz
 
 from core.cleanup import CleanupTimer
-from core.config import config
+from core.config import config, MAX_SPOT_AGE, WEB_SERVER_PORT, SERVER_OWNER_CALLSIGN
+from core.constants import SOFTWARE_VERSION
 from providers.aprsis import APRSIS
 from providers.dxcluster import DXCluster
 from providers.gma import GMA
@@ -76,8 +77,11 @@ if __name__ == '__main__':
     formatter = logging.Formatter("%(message)s")
     handler.setFormatter(formatter)
     root.addHandler(handler)
+
     logging.info("Starting...")
     startup_time = datetime.now(pytz.UTC)
+    status_data["software-version"] = SOFTWARE_VERSION
+    status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
 
     # Shut down gracefully on SIGINT
     signal.signal(signal.SIGINT, shutdown)
@@ -92,11 +96,11 @@ if __name__ == '__main__':
             p.start()
 
     # Set up timer to clear spot list of old data
-    cleanup_timer = CleanupTimer(spot_list=spot_list, cleanup_interval=60, max_spot_age=config["max-spot-age-sec"])
+    cleanup_timer = CleanupTimer(spot_list=spot_list, cleanup_interval=60, max_spot_age=MAX_SPOT_AGE)
     cleanup_timer.start()
 
     # Set up web server
-    web_server = WebServer(spot_list=spot_list, status_data=status_data, port=config["web-server-port"])
+    web_server = WebServer(spot_list=spot_list, status_data=status_data, port=WEB_SERVER_PORT)
     web_server.start()
 
     logging.info("Startup complete.")

providers/aprsis.py

@@ -5,7 +5,7 @@ from threading import Thread
 import aprslib
 import pytz
 
-from core.config import config
+from core.config import SERVER_OWNER_CALLSIGN
 from data.spot import Spot
 from providers.provider import Provider
 
@@ -23,7 +23,7 @@ class APRSIS(Provider):
         self.thread.start()
 
     def connect(self):
-        self.aprsis = aprslib.IS(config["server-owner-callsign"])
+        self.aprsis = aprslib.IS(SERVER_OWNER_CALLSIGN)
         self.status = "Connecting"
         logging.info("APRS-IS connecting...")
         self.aprsis.connect()

providers/dxcluster.py

@@ -8,7 +8,7 @@ import pytz
 import telnetlib3
 
 from data.spot import Spot
-from core.config import config
+from core.config import SERVER_OWNER_CALLSIGN
 from providers.provider import Provider
 
 
@@ -47,7 +47,7 @@ class DXCluster(Provider):
                     logging.info("DX Cluster " + self.hostname + " connecting...")
                     self.telnet = telnetlib3.Telnet(self.hostname, self.port)
                     self.telnet.read_until("login: ".encode("latin-1"))
-                    self.telnet.write((config["server-owner-callsign"] + "\n").encode("latin-1"))
+                    self.telnet.write((SERVER_OWNER_CALLSIGN + "\n").encode("latin-1"))
                     connected = True
                     logging.info("DX Cluster " + self.hostname + " connected.")
                 except Exception as e:

providers/provider.py

@@ -3,14 +3,14 @@ from datetime import datetime
 import pytz
 
 from core.constants import SOFTWARE_NAME, SOFTWARE_VERSION
-from core.config import config
+from core.config import config, SERVER_OWNER_CALLSIGN
 
 
 # Generic data provider class. Subclasses of this query the individual APIs for data.
 class Provider:
 
     # HTTP headers used for providers that use HTTP
-    HTTP_HEADERS = { "User-Agent": SOFTWARE_NAME + " " + SOFTWARE_VERSION + " (operated by " + config["server-owner-callsign"] + ")" }
+    HTTP_HEADERS = { "User-Agent": SOFTWARE_NAME + " " + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")" }
 
     # Constructor
     def __init__(self, provider_config):

providers/rbn.py

@@ -8,7 +8,7 @@ import pytz
 import telnetlib3
 
 from data.spot import Spot
-from core.config import config
+from core.config import SERVER_OWNER_CALLSIGN
 from providers.provider import Provider
 
 
@@ -48,7 +48,7 @@ class RBN(Provider):
                     logging.info("RBN port " + str(self.port) + " connecting...")
                     self.telnet = telnetlib3.Telnet("telnet.reversebeacon.net", self.port)
                     telnet_output = self.telnet.read_until("Please enter your call: ".encode("latin-1"))
-                    self.telnet.write((config["server-owner-callsign"] + "\n").encode("latin-1"))
+                    self.telnet.write((SERVER_OWNER_CALLSIGN + "\n").encode("latin-1"))
                     connected = True
                     logging.info("RBN port " + str(self.port) + " connected.")
                 except Exception as e:

server/webserver.py

@@ -1,12 +1,13 @@
 import json
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 from threading import Thread
 
 import bottle
 import pytz
 from bottle import run, response
 
+from core.config import MAX_SPOT_AGE
 from core.constants import BANDS, ALL_MODES, MODE_TYPES, SIGS, SOURCES, CONTINENTS
 from core.utils import serialize_everything
 
@@ -100,6 +101,10 @@ class WebServer:
                 case "since":
                     since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC)
                     spots = [s for s in spots if s.time > since]
+                case "max_age":
+                    max_age = int(query.get(k))
+                    since = datetime.now(pytz.UTC) - timedelta(seconds=max_age)
+                    spots = [s for s in spots if s.time > since]
                 case "received_since":
                     since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC)
                     spots = [s for s in spots if s.received_time > since]
@@ -138,4 +143,5 @@ class WebServer:
                 "mode_types": MODE_TYPES,
                 "sigs": SIGS,
                 "sources": SOURCES,
-                "continents": CONTINENTS}
+                "continents": CONTINENTS,
+                "max_spot_age": MAX_SPOT_AGE}

webassets/apidocs/openapi.yml

@@ -4,7 +4,7 @@ info:
   description: |-
     (S)pothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open JSON API as well as a website to browse the data.
     
-    While there are other web-based interfaces to DX clusters, and sites that aggregate spots from various outfoor activity programmes for amateur radio, (S)pothole differentiates itself by supporting a large number of data sources, and by being "API first" rather than just providing a web front-end. This allows other software to be built on top of it. Spothole itself is also open source, Public Domain licenced code that anyone can take and modify.
+    While there are other web-based interfaces to DX clusters, and sites that aggregate spots from various outfoor activity programmes for amateur radio, (S)pothole differentiates itself by supporting a large number of data sources, and by being "API first" rather than just providing a web front-end. This allows other software to be built on top of it. (S)pothole itself is also open source, Public Domain licenced code that anyone can take and modify.
   contact:
     email: ian@ianrenton.com
   license:
@@ -30,13 +30,19 @@ paths:
             type: integer
         - name: since
           in: query
-          description: Limit the spots to only ones at this time or later. Time in UTC seconds since UNIX epoch.
+          description: Limit the spots to only ones at this time or later. Time in UTC seconds since UNIX epoch. Equivalent to "max_age" but saves the client having to work out how many seconds ago "midnight" was.
+          required: false
+          schema:
+            type: integer
+        - name: max_age
+          in: query
+          description: Limit the spots to only ones received in the last 'n' seconds. Equivalent to "since" but saves the client having to work out what time was 'n' seconds ago on every call. Refer to the "max_spot_age" in the /options call to figure out what the maximum useful value you can provide is. Larger values will still be accepted, there just won't be any spots in the system older than max_spot_age.
           required: false
           schema:
             type: integer
         - name: received_since
           in: query
-          description: Limit the spots to only ones that the system found out about at this time or later. Time in UTC seconds since UNIX epoch. If you are using a front-end that tracks the last time it queried the API and requests spots since then, you want *this* version of the query parameter, not "since", because otherwise it may miss things.
+          description: Limit the spots to only ones that the system found out about at this time or later. Time in UTC seconds since UNIX epoch. If you are using a front-end that tracks the last time it queried the API and requests spots since then, you want *this* version of the query parameter, not "since", because otherwise it may miss things. The logic is "greater than" rather than "greater than or equal to", so you can submit the time of the last received item back to this call and you will get all the more recent spots back, without duplicating the previous latest spot.
           required: false
           schema:
             type: integer
@@ -188,6 +194,14 @@ paths:
               schema:
                 type: object
                 properties:
+                  "software-version":
+                    type: string
+                    description: The version number of the software.
+                    example: "1.0.1"
+                  "server-owner-callsign":
+                    type: string
+                    description: The callsign of this server's operator.
+                    example: "M0TRT"
                   "uptime":
                     type: string
                     description: The amount of time the software has been running for.
@@ -283,6 +297,10 @@ paths:
                     items:
                       type: string
                       example: "EU"
+                  max_spot_age:
+                    type: integer
+                    description: The maximum age, in seconds, of any spot before it will be deleted by the system. When querying the /api/spots endpoint and providing a "max_age" or "since" parameter, there is no point providing a number larger than this, because the system drops all spots older than this.
+                    example: 3600
 
 components:
   schemas: