Backwards compatibility for critical v1 APIs. Closes #119

providers/spot/gma.py

@@ -39,6 +39,13 @@ class GMA(HTTPSpotProvider):
                 lat = float(source_spot["LAT"]) if (source_spot["LAT"] and source_spot["LAT"] != "") else None
                 lon = float(source_spot["LON"]) if (source_spot["LON"] and source_spot["LON"] != "") else None
 
+                # Seen some real janky times from GMA, if we don't understand it just ignore this spot
+                try:
+                    time = datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace(
+                                tzinfo=pytz.UTC).timestamp()
+                except ValueError:
+                    continue
+
                 spot = Spot(source=self.name,
                             dx_call=source_spot["ACTIVATOR"].upper(),
                             de_call=source_spot["SPOTTER"].upper(),
@@ -50,8 +57,7 @@ class GMA(HTTPSpotProvider):
                             comment=source_spot["TEXT"],
                             sig_refs=[SIGRef(id=source_spot["REF"], sig="", name=source_spot["NAME"], latitude=lat,
                                              longitude=lon)],
-                            time=datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace(
+                            time=time,
-                                tzinfo=pytz.UTC).timestamp(),
                             dx_latitude=lat,
                             dx_longitude=lon,
                             qrt=source_spot["QRG"] == "QRT")

server/handlers/api/v1_addspot.py

@@ -0,0 +1,133 @@
+import logging
+import re
+from datetime import datetime
+from typing import Any
+
+import pytz
+import tornado
+from tornado import httputil
+from tornado.web import Application
+
+from core.config import ALLOW_SPOTTING
+from core.constants import UNKNOWN_BAND
+from core.prometheus_metrics_handler import api_requests_counter
+from core.sig_utils import get_ref_regex_for_sig
+from core.utils import safe_json_dumps, infer_band_from_freq
+from data.spot import Spot
+
+
+class V1APISpotHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v1/spot (POST). Included in early Spothole v2 for backwards compatibility."""
+
+    def __init__(self, application: "Application", request: httputil.HTTPServerRequest, **kwargs: Any):
+        self._spots = None
+        self._web_server_metrics = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, spots, web_server_metrics):
+        self._spots = spots
+        self._web_server_metrics = web_server_metrics
+
+    def post(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            # Reject if not allowed
+            if not ALLOW_SPOTTING:
+                self.set_status(401)
+                self.write(safe_json_dumps("Error - this server does not allow new spots to be added via the API."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if format not json
+            if not self.request.headers.get('Content-Type', '').startswith("application/json"):
+                self.set_status(415)
+                self.write(safe_json_dumps("Error - request Content-Type must be application/json"))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if request body is empty
+            post_data = self.request.body
+            if not post_data:
+                self.set_status(422)
+                self.write(safe_json_dumps("Error - request body is empty"))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Read in the request body as JSON then convert to a Spot object
+            json_spot = tornado.escape.json_decode(post_data)
+            spot = Spot(**json_spot)
+
+            # Reject if no timestamp, frequency, dx_call or de_call
+            if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
+                self.set_status(422)
+                self.write(safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject invalid-looking callsigns
+            if not re.match(r"^[A-Za-z0-9/\-]*$", spot.dx_call):
+                self.set_status(422)
+                self.write(safe_json_dumps("Error - '" + spot.dx_call + "' does not look like a valid callsign."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+            if not re.match(r"^[A-Za-z0-9/\-]*$", spot.de_call):
+                self.set_status(422)
+                self.write(safe_json_dumps("Error - '" + spot.de_call + "' does not look like a valid callsign."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if frequency not in a known band
+            if infer_band_from_freq(spot.freq) == UNKNOWN_BAND:
+                self.set_status(422)
+                self.write(safe_json_dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if grid formatting incorrect
+            if spot.dx_grid and not re.match(
+                    r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$",
+                    spot.dx_grid.upper()):
+                self.set_status(422)
+                self.write(safe_json_dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if sig_ref format incorrect for sig
+            if spot.sig and spot.sig_refs and len(spot.sig_refs) > 0 and spot.sig_refs[0].id and get_ref_regex_for_sig(
+                    spot.sig) and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id):
+                self.set_status(422)
+                self.write(safe_json_dumps(
+                    "Error - '" + spot.sig_refs[0].id + "' does not look like a valid reference for " + spot.sig + "."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # infer missing data, and add it to our database.
+            spot.source = "API"
+            spot.infer_missing()
+            self._spots.set(spot.id, spot)
+
+            self.write(safe_json_dumps("OK"))
+            self.set_status(201)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+        except Exception as e:
+            logging.error("Exception when handling client request to add spot API: %s", e, exc_info=True)
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")

server/handlers/api/v1_spots.py

@@ -0,0 +1,27 @@
+import re
+
+from server.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
+
+_GRID_SOURCE_RE = re.compile(r'"dx_location_source":\s*"GRID"')
+
+
+class V1APISpotsHandler(APISpotsHandler):
+    """API request handler for /api/v1/spots (GET). Included in early Spothole v2 for backwards compatibility."""
+
+    def write(self, chunk):
+        if isinstance(chunk, str):
+            chunk = _GRID_SOURCE_RE.sub('"dx_location_source": "SPOT"', chunk)
+        super().write(chunk)
+
+class V1APISpotsStreamHandler(APISpotsStreamHandler):
+    """API request handler for /api/v1/spots/stream (SSE). Included in early Spothole v2 for backwards compatibility."""
+
+    def write_message(self, *args, **kwargs):
+        args = list(args)
+        for i, a in enumerate(args):
+            if isinstance(a, str) and '"dx_location_source"' in a:
+                args[i] = _GRID_SOURCE_RE.sub('"dx_location_source": "SPOT"', a)
+        for k, v in kwargs.items():
+            if isinstance(v, str) and '"dx_location_source"' in v:
+                kwargs[k] = _GRID_SOURCE_RE.sub('"dx_location_source": "SPOT"', v)
+        super().write_message(*args, **kwargs)

server/webserver.py

@@ -16,7 +16,9 @@ from server.handlers.api.options import APIOptionsHandler
 from server.handlers.api.solar_conditions import APISolarConditionsHandler
 from server.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
 from server.handlers.api.status import APIStatusHandler
-from server.handlers.api.v1_compatability import V1RedirectHandler, V1GoneHandler
+from server.handlers.api.v1_addspot import V1APISpotHandler
+from server.handlers.api.v1_compatability import V1RedirectHandler
+from server.handlers.api.v1_spots import V1APISpotsHandler, V1APISpotsStreamHandler
 from server.handlers.manifesthandler import ManifestHandler
 from server.handlers.metrics import PrometheusMetricsHandler
 from server.handlers.pagetemplate import PageTemplateHandler
@@ -95,7 +97,10 @@ class WebServer:
         # v1 API redirects. Most v1 enpoints are unchanged in v2, and get an HTTP 308 redirect to the v2 API. The ones
         # that have the major breaking changes get a bespoke handler.
         v1_compat_routes = [
-            (r"/api/v1/spot", V1GoneHandler),
+            (r"/api/v1/spots", V1APISpotsHandler, {"spots": self._data_store.spots, **handler_opts}),
+            (r"/api/v1/spots/stream", V1APISpotsStreamHandler,
+             {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts}),
+            (r"/api/v1/spot", V1APISpotHandler),
             (r"/api/v1/(.*)", V1RedirectHandler),
         ]
 

static/apidocs/openapi.yml

@@ -1491,7 +1491,7 @@ components:
           example: 7
         k_index:
           type: integer
-          description: 3-hour geomagnetic activity index, 0–9
+          description: 3-hour geomagnetic activity index, 0 to 9
           example: 2
         xray:
           type: string
@@ -1589,7 +1589,7 @@ components:
           type: object
           description: >
             NOAA Kp index 3-day forecast. Keys are UNIX timestamps (UTC seconds since epoch) for the
-            start of each 3-hour period. Values are the forecast Kp index (0–9) for that period.
+            start of each 3-hour period. Values are the forecast Kp index (0 to 9) for that period.
             Only forecast values are included; observed actuals (shown in parentheses in the source
             data) are discarded.
           additionalProperties: