Add telnet server

webserver/handlers/api/addspot.py

@@ -0,0 +1,276 @@
+import logging
+import re
+import threading
+from datetime import datetime
+from typing import Any
+
+import pytz
+import requests
+import tornado
+from tornado import httputil
+from tornado.web import Application
+
+from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRET_KEY
+from core.constants import UNKNOWN_BAND
+from core.prometheus_metrics_handler import api_requests_counter
+from core.sig_utils import get_ref_regex_for_sig
+from core.utils import infer_band_from_freq, safe_json_dumps
+from data.spot import Spot
+from providers.spot.spot_provider import SpotProvider
+
+logger = logging.getLogger(__name__)
+
+RECAPTCHA_VERIFY_URL = "https://www.google.com/recaptcha/api/siteverify"
+
+
+class APISpotHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v2/spot (POST)"""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._spots = None
+        self._web_server_metrics = None
+        self._spot_providers = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, spots, web_server_metrics, spot_providers=None):
+        self._spots = spots
+        self._web_server_metrics = web_server_metrics
+        self._spot_providers = spot_providers or []
+
+    def post(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            # Reject if not allowed
+            if not ALLOW_SPOTTING:
+                self.set_status(401)
+                self.write(safe_json_dumps("Error - this server does not allow new spots to be added via the API."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if format not json
+            if not self.request.headers.get("Content-Type", "").startswith("application/json"):
+                self.set_status(415)
+                self.write(safe_json_dumps("Error - request Content-Type must be application/json"))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if request body is empty
+            post_data = self.request.body
+            if not post_data:
+                self.set_status(422)
+                self.write(safe_json_dumps("Error - request body is empty"))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Read in the request body as JSON
+            json_body = tornado.escape.json_decode(post_data)
+
+            # Extract the "spot" and "handling" sub-objects from the request body
+            spot_data = json_body.get("spot", {})
+            handling = json_body.get("handling", {})
+
+            # Extract individual parameters that say how this spot should be handled by the server
+            submit_upstream = handling.get("submit_upstream", False)
+            upstream_provider_name = handling.get("upstream_provider", None)
+            upstream_credentials = handling.get("upstream_credentials", {})
+            captcha_token = handling.get("captcha_token", None)
+
+            # Spothole v2.0 release only: deny upstream spotting. Spothole API breaking changes were in v2.0 but
+            # functionality is not ready yet. TODO
+            submit_upstream = False
+
+            # Verify CAPTCHA if required
+            if RECAPTCHA_SECRET_KEY:
+                if not captcha_token:
+                    self.set_status(422)
+                    self.write(safe_json_dumps("Error - CAPTCHA token is required for spot submission."))
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+                if not self._verify_recaptcha(captcha_token):
+                    self.set_status(422)
+                    self.write(safe_json_dumps("Error - CAPTCHA verification failed."))
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+
+            # Convert spot field to a Spot object
+            spot = Spot(**spot_data)
+
+            # Reject if no timestamp, frequency, dx_call or de_call
+            if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
+                self.set_status(422)
+                self.write(
+                    safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.")
+                )
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject invalid-looking callsigns
+            if not re.match(r"^[A-Za-z0-9/\-]*$", spot.dx_call):
+                self.set_status(422)
+                self.write(safe_json_dumps(f"Error - '{spot.dx_call}' does not look like a valid callsign."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+            if not re.match(r"^[A-Za-z0-9/\-]*$", spot.de_call):
+                self.set_status(422)
+                self.write(safe_json_dumps(f"Error - '{spot.de_call}' does not look like a valid callsign."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if frequency not in a known band
+            if infer_band_from_freq(spot.freq) == UNKNOWN_BAND:
+                self.set_status(422)
+                self.write(safe_json_dumps(f"Error - Frequency of {spot.freq / 1000.0!s}kHz is not in a known band."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if grid formatting incorrect
+            if spot.dx_grid and not re.match(
+                r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$",
+                spot.dx_grid.upper(),
+            ):
+                self.set_status(422)
+                self.write(safe_json_dumps(f"Error - '{spot.dx_grid}' does not look like a valid Maidenhead grid."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if sig_ref format incorrect for sig
+            if (
+                spot.sig
+                and spot.sig_refs
+                and len(spot.sig_refs) > 0
+                and spot.sig_refs[0].id
+                and get_ref_regex_for_sig(spot.sig)
+                and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id)
+            ):
+                self.set_status(422)
+                self.write(
+                    safe_json_dumps(
+                        f"Error - '{spot.sig_refs[0].id}' does not look like a valid reference for {spot.sig}."
+                    )
+                )
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject upstream submission if not permitted
+            if submit_upstream and not ALLOW_UPSTREAM_SPOTTING:
+                self.set_status(403)
+                self.write(safe_json_dumps("Error - this server does not allow upstream spot submission."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Validate upstream submission requirements
+            if submit_upstream and upstream_provider_name:
+                if not spot.sig:
+                    self.set_status(422)
+                    self.write(safe_json_dumps("Error - a SIG must be selected to submit upstream."))
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+                if not spot.sig_refs and upstream_provider_name != "Tiles":
+                    self.set_status(422)
+                    self.write(safe_json_dumps("Error - a SIG reference is required to submit upstream."))
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+                if not spot.dx_grid and upstream_provider_name == "Tiles":
+                    self.set_status(422)
+                    self.write(
+                        safe_json_dumps("Error - a grid reference is required to submit upstream to Tiles on the Air.")
+                    )
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+                if not spot.mode and upstream_provider_name == "Tiles":
+                    self.set_status(422)
+                    self.write(safe_json_dumps("Error - a mode is required to submit upstream to Tiles on the Air."))
+                    self.set_header("Cache-Control", "no-store")
+                    self.set_header("Content-Type", "application/json")
+                    return
+
+            # Submit upstream if requested
+            upstream_warning = None
+            if submit_upstream and upstream_provider_name:
+                provider = self._find_provider(upstream_provider_name, spot.sig)
+                if provider:
+                    try:
+                        # Submit spot to the upstream provider
+                        provider.submit_spot(spot, upstream_credentials)
+                        # Trigger a re-poll after 1 second so the spot appears quickly
+                        threading.Timer(1.0, provider.force_poll).start()
+                    except NotImplementedError as e:
+                        upstream_warning = str(e)
+                    except Exception:
+                        logger.exception(f"Failed to submit spot upstream to {upstream_provider_name}")
+                        upstream_warning = (
+                            f"Spot was saved locally but upstream submission to {upstream_provider_name} failed."
+                        )
+                else:
+                    upstream_warning = f"No enabled provider named '{upstream_provider_name}' supports upstream submission for {spot.sig if spot.sig else ''} spots."
+
+            # If we successfully submitted the spot upstream, don't add it direct to Spothole, otherwise it will be a
+            # duplicate with what immediately comes back from the API. But if we weren't asked to send it upstream, or
+            # we were but it failed, we should still add it to our database anyway.
+            if not submit_upstream or upstream_warning:
+                spot.infer_missing()
+                self._spots.set(spot.id, spot)
+
+            if upstream_warning:
+                self.write(safe_json_dumps(f"Warning - {upstream_warning}"))
+                self.set_status(201)
+            else:
+                self.write(safe_json_dumps("OK"))
+                self.set_status(201)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+        except Exception:
+            logger.exception("Exception when handling client request to add spot API")
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+    def _find_provider(self, provider_name, sig) -> SpotProvider | None:
+        """Find an enabled provider by name that can submit spots for the given SIG."""
+
+        for p in self._spot_providers:
+            if p.enabled and p.name == provider_name and p.can_submit_spot(sig):
+                return p
+        return None
+
+    @staticmethod
+    def _verify_recaptcha(token):
+        """Verify a Google reCAPTCHA v2 token. Returns True if valid."""
+
+        try:
+            response = requests.post(
+                RECAPTCHA_VERIFY_URL,
+                data={"secret": RECAPTCHA_SECRET_KEY, "response": token},
+                timeout=(5, 10),
+            )
+            return response.ok and response.json().get("success", False)
+        except Exception:
+            logger.exception("reCAPTCHA verification request failed")
+            return False

webserver/handlers/api/alerts.py

@@ -0,0 +1,237 @@
+import copy
+import logging
+from datetime import datetime
+from typing import Any
+
+import pytz
+import tornado
+import tornado_eventsource.handler
+from tornado import httputil
+from tornado.web import Application
+
+from core.enums import AlertType
+from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
+from data.lookup_credentials import extract_credentials
+
+logger = logging.getLogger(__name__)
+
+
+class APIAlertsHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v2/alerts"""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._alerts = None
+        self._web_server_metrics = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, alerts, web_server_metrics):
+        self._alerts = alerts
+        self._web_server_metrics = web_server_metrics
+
+    @staticmethod
+    def _enrich(alerts, credentials):
+        enriched = []
+        for alert in alerts:
+            alert_copy = copy.deepcopy(alert)
+            alert_copy.infer_missing(credentials)
+            enriched.append(alert_copy)
+        return enriched
+
+    def get(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            # request.arguments contains lists for each param key because technically the client can supply multiple,
+            # reduce that to just the first entry, and convert bytes to string
+            query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
+
+            # Fetch all alerts matching the query, then optionally enrich with online data
+            credentials = extract_credentials(self.request.headers)
+            data = get_alert_list_with_filters(self._alerts, query_params)
+            fields = [f.strip() for f in query_params["fields"].split(",")] if "fields" in query_params else []
+            if credentials:
+                data = self._enrich(data, credentials)
+            # Filter for only the required fields, if necessary
+            if fields:
+                data = filter_fields(data, fields)
+            self.write(safe_json_dumps(data))
+            self.set_status(200)
+        except ValueError as e:
+            self.write(safe_json_dumps(f"Bad request - {e!s}"))
+            self.set_status(400)
+        except Exception:
+            logger.exception("Exception when handling client request to alerts API")
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)
+        self.set_header("Cache-Control", "no-store")
+        self.set_header("Content-Type", "application/json")
+
+
+class APIAlertsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
+    """API request handler for /api/v2/alerts/stream"""
+
+    def __init__(self, application, request, **kwargs: Any):
+        self._sse_alert_broadcaster = None
+        self._web_server_metrics = None
+        self._query_params = None
+        self._credentials = None
+        self._fields = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, sse_alert_broadcaster, web_server_metrics):
+        self._sse_alert_broadcaster = sse_alert_broadcaster
+        self._web_server_metrics = web_server_metrics
+
+    def custom_headers(self):
+        """Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data"""
+
+        return {"Cache-Control": "no-store", "X-Accel-Buffering": "no"}
+
+    def open(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            # request.arguments contains lists for each param key because technically the client can supply multiple,
+            # reduce that to just the first entry, and convert bytes to string
+            self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
+            self._credentials = extract_credentials(self.request.headers)
+            self._fields = (
+                [f.strip() for f in self._query_params["fields"].split(",")] if "fields" in self._query_params else []
+            )
+
+            # Flush headers immediately so nginx doesn't time out waiting for a response
+            self.write_message("keepalive", "")
+
+            # Register to handle new alerts arriving. The callback() method will get called with the new alert as an
+            # argument.
+            self._sse_alert_broadcaster.register(self)
+
+        except Exception:
+            logger.exception("Exception when serving SSE socket")
+            self.close()
+
+    def close(self):
+        """When the user closes the socket, deregister ourselves from the alert broadcaster"""
+
+        self._sse_alert_broadcaster.unregister(self)
+        super().close()
+
+    def callback(self, alert):
+        """Callback when a new alert arrives"""
+
+        try:
+            # If the new alert matches our param filters, send it to the client. If not, ignore it.
+            if alert_allowed_by_query(alert, self._query_params):
+                # Add lookup data if we have credentials
+                if self._credentials:
+                    alert = copy.deepcopy(alert)
+                    alert.infer_missing(self._credentials)
+                # Filter fields returned if necessary
+                if self._fields:
+                    alert = filter_fields([alert], self._fields)[0]
+                self.write_message(msg=safe_json_dumps(alert))
+        except Exception:
+            logger.exception("Exception in SSE callback, connection will be closed")
+            self.close()
+
+
+def get_alert_list_with_filters(all_alerts, query):
+    """Utility method to apply filters to the overall alert list and return only a subset. Enables query parameters in
+    the main "alerts" GET call."""
+
+    # Create a shallow copy of the alert list ordered by start time, then filter the list to reduce it only to alerts
+    # that match the filter parameters in the query string. Finally, apply a limit to the number of alerts returned.
+    # The list of query string filters is defined in the API docs.
+    alert_ids = all_alerts.keys()
+    alerts = []
+    for k in alert_ids:
+        a = all_alerts.get(k)
+        if a is not None:
+            alerts.append(a)
+    alerts = sorted(alerts, key=lambda alert: alert.start_time if alert and alert.start_time else 0)
+    alerts = list(filter(lambda alert: alert_allowed_by_query(alert, query), alerts))
+    if "limit" in query:
+        alerts = alerts[: int(query.get("limit"))]
+    return alerts
+
+
+def alert_allowed_by_query(alert, query):
+    """Given URL query params and an alert, figure out if the alert "passes" the requested filters or is rejected. The list
+    of query parameters and their function is defined in the API docs."""
+
+    for k in query:
+        match k:
+            case "received_since":
+                since = datetime.fromtimestamp(float(query.get(k)), pytz.UTC)
+                if not alert.received_time or alert.received_time <= since:
+                    return False
+            case "max_duration":
+                max_duration = int(query.get(k))
+                # Check the duration if end_time is provided. If end_time is not provided, assume the activation is
+                # "short", i.e. it always passes this check. If dxpeditions_skip_max_duration_check is true and
+                # the alert is a dxpedition, or contests_skip_max_duration_check and the alert is a contest, it also
+                # always passes the check.
+                if (
+                    alert.alert_type == AlertType.DXPEDITION
+                    and "dxpeditions_skip_max_duration_check" in query
+                    and query.get("dxpeditions_skip_max_duration_check").upper() == "TRUE"
+                ):
+                    continue
+                if (
+                    alert.alert_type == AlertType.CONTEST
+                    and "contests_skip_max_duration_check" in query
+                    and query.get("contests_skip_max_duration_check").upper() == "TRUE"
+                ):
+                    continue
+                if alert.end_time and alert.start_time and alert.end_time - alert.start_time > max_duration:
+                    return False
+            case "source":
+                sources = query.get(k).split(",")
+                if not alert.source or alert.source not in sources:
+                    return False
+            case "sig":
+                # If a list of sigs is provided, the alert must have a sig and it must match one of them.
+                # The special "sig" "NO_SIG", when supplied in the list, mathches alerts with no sig.
+                sigs = query.get(k).split(",")
+                include_no_sig = "NO_SIG" in sigs
+                if not alert.sig and not include_no_sig:
+                    return False
+                if alert.sig and alert.sig not in sigs:
+                    return False
+            case "dx_continent":
+                dxconts = query.get(k).split(",")
+                if not alert.dx_continent or alert.dx_continent not in dxconts:
+                    return False
+            case "dx_call_includes":
+                dx_call_includes = query.get(k).strip()
+                if not alert.dx_call or dx_call_includes.upper() not in alert.dx_call.upper():
+                    return False
+            case "text_includes":
+                text_includes = query.get(k).strip()
+                if (
+                    (not alert.dx_call or text_includes.upper() not in alert.dx_call.upper())
+                    and (not alert.comment or text_includes.upper() not in alert.comment.upper())
+                    and (not alert.freqs_modes or text_includes.upper() not in alert.freqs_modes.upper())
+                ):
+                    return False
+    return True
+
+
+def filter_fields(alerts, fields):
+    """Given a list of alert objects, return copies containing only the named fields."""
+
+    return [{k: v for k, v in alert.__dict__.items() if k in fields} for alert in alerts]

webserver/handlers/api/dxstats.py

@@ -0,0 +1,71 @@
+import json
+import logging
+from collections import Counter
+from datetime import datetime, timedelta
+from typing import Any
+
+import pytz
+import tornado
+from tornado import httputil
+from tornado.web import Application
+
+from core.constants import BANDS
+from core.enums import Continent
+from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
+
+logger = logging.getLogger(__name__)
+
+CONTINENTS = [c.value for c in Continent]
+HF_BANDS = [b.name for b in BANDS if b.is_ham_hf]
+
+
+class APIDxStatsHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v2/dxstats"""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._spots = None
+        self._web_server_metrics = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, spots, web_server_metrics):
+        self._spots = spots
+        self._web_server_metrics = web_server_metrics
+
+    def get(self):
+        try:
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            one_hour_ago = (datetime.now(pytz.UTC) - timedelta(hours=1)).timestamp()
+            counts = Counter()
+
+            for key in self._spots.keys():  # noqa: SIM118
+                spot = self._spots.get(key)
+                if spot is None:
+                    continue
+                if not spot.time or spot.time < one_hour_ago:
+                    continue
+                if spot.de_continent in CONTINENTS and spot.dx_continent in CONTINENTS and spot.band in HF_BANDS:
+                    counts[spot.de_continent, spot.dx_continent, spot.band] += 1
+
+            result = {
+                de: {dx: {band: counts[de, dx, band] for band in HF_BANDS} for dx in CONTINENTS} for de in CONTINENTS
+            }
+
+            self.write(json.dumps(result))
+            self.set_status(200)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+        except Exception:
+            logger.exception("Exception when handling client request to dx stats API")
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

webserver/handlers/api/lookups.py

@@ -0,0 +1,202 @@
+import logging
+import re
+from datetime import datetime
+from typing import Any
+
+import pytz
+import tornado
+from tornado import httputil
+from tornado.web import Application
+
+from core.call_lookup_helper import get_call_info
+from core.constants import SIGS
+from core.geo_utils import (
+    lat_lon_for_grid_sw_corner_plus_size,
+    lat_lon_to_cq_zone,
+    lat_lon_to_itu_zone,
+)
+from core.prometheus_metrics_handler import api_requests_counter
+from core.sig_lookup_helper import populate_missing_sig_ref_info
+from core.sig_utils import get_ref_regex_for_sig
+from core.utils import safe_json_dumps
+from data.lookup_credentials import extract_credentials
+from data.sig_ref import SIGRef
+
+logger = logging.getLogger(__name__)
+
+
+class APILookupCallHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v2/lookup/call"""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._web_server_metrics = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, web_server_metrics):
+        self._web_server_metrics = web_server_metrics
+
+    def get(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            # request.arguments contains lists for each param key because technically the client can supply multiple,
+            # reduce that to just the first entry, and convert bytes to string
+            query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
+
+            # The "call" query param must exist and look like a callsign
+            if "call" in query_params:
+                call = str(query_params.get("call")).upper()
+                if re.match(r"^[A-Z0-9/\-]*$", call):
+                    credentials = extract_credentials(self.request.headers)
+                    callsign_data = get_call_info(call, credentials)
+                    self.write(safe_json_dumps(callsign_data))
+
+                else:
+                    self.write(safe_json_dumps(f"Error - '{call}' does not look like a valid callsign."))
+                    self.set_status(422)
+            else:
+                self.write(safe_json_dumps("Error - call must be provided"))
+                self.set_status(422)
+
+        except Exception:
+            logger.exception("Exception when handling client request to call lookup API")
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)
+
+        self.set_header("Cache-Control", "no-store")
+        self.set_header("Content-Type", "application/json")
+
+
+class APILookupSIGRefHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v2/lookup/sigref"""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._web_server_metrics = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, web_server_metrics):
+        self._web_server_metrics = web_server_metrics
+
+    def get(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            # request.arguments contains lists for each param key because technically the client can supply multiple,
+            # reduce that to just the first entry, and convert bytes to string
+            query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
+
+            # "sig" and "id" query params must exist, SIG must be known, and if we have a reference regex for that SIG,
+            # the provided id must match it.
+            if "sig" in query_params and "id" in query_params:
+                sig = str(query_params.get("sig")).upper()
+                ref_id = str(query_params.get("id")).upper()
+                if sig in [p.name.upper() for p in SIGS]:
+                    if not get_ref_regex_for_sig(sig) or re.match(get_ref_regex_for_sig(sig), ref_id):
+                        data = populate_missing_sig_ref_info(SIGRef(id=ref_id, sig=sig))
+                        self.write(safe_json_dumps(data))
+
+                    else:
+                        self.write(
+                            safe_json_dumps(f"Error - '{ref_id}' does not look like a valid reference ID for {sig}.")
+                        )
+                        self.set_status(422)
+                else:
+                    self.write(safe_json_dumps(f"Error - sig '{sig}' is not known."))
+                    self.set_status(422)
+            else:
+                self.write(safe_json_dumps("Error - sig and id must be provided"))
+                self.set_status(422)
+
+        except Exception:
+            logger.exception("Exception when handling client request to sig ref lookup API")
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)
+
+        self.set_header("Cache-Control", "no-store")
+        self.set_header("Content-Type", "application/json")
+
+
+class APILookupGridHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v2/lookup/grid"""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._web_server_metrics = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, web_server_metrics):
+        self._web_server_metrics = web_server_metrics
+
+    def get(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            # request.arguments contains lists for each param key because technically the client can supply multiple,
+            # reduce that to just the first entry, and convert bytes to string
+            query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
+
+            # "grid" query param must exist.
+            if "grid" in query_params:
+                grid = str(query_params.get("grid")).upper()
+                lat, lon, lat_cell_size, lon_cell_size = lat_lon_for_grid_sw_corner_plus_size(grid)
+                if lat is not None and lon is not None and lat_cell_size is not None and lon_cell_size is not None:
+                    center_lat = lat + lat_cell_size / 2.0
+                    center_lon = lon + lon_cell_size / 2.0
+                    center_cq_zone = lat_lon_to_cq_zone(center_lat, center_lon)
+                    center_itu_zone = lat_lon_to_itu_zone(center_lat, center_lon)
+
+                    response = {
+                        "center": {
+                            "latitude": center_lat,
+                            "longitude": center_lon,
+                            "cq_zone": center_cq_zone,
+                            "itu_zone": center_itu_zone,
+                        },
+                        "southwest": {
+                            "latitude": lat,
+                            "longitude": lon,
+                        },
+                        "northeast": {
+                            "latitude": lat + lat_cell_size,
+                            "longitude": lon + lon_cell_size,
+                        },
+                    }
+                    self.write(safe_json_dumps(response))
+
+            else:
+                self.write(safe_json_dumps("Error - grid must be provided"))
+                self.set_status(422)
+
+        except Exception:
+            logger.exception("Exception when handling client request to grid ref lookup API")
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)
+
+        self.set_header("Cache-Control", "no-store")
+        self.set_header("Content-Type", "application/json")

webserver/handlers/api/options.py

@@ -0,0 +1,108 @@
+import logging
+from datetime import datetime
+from typing import Any
+
+import pytz
+import tornado
+from tornado import httputil
+from tornado.web import Application
+
+from core.config import ALLOW_SPOTTING, MAX_SPOT_AGE
+from core.constants import BANDS, PROPAGATION_MODES, SIGS
+from core.enums import Continent, Mode, ModeType
+from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
+
+logger = logging.getLogger(__name__)
+
+
+class APIOptionsHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v2/options"""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._status_data = None
+        self._web_server_metrics = None
+        self._spot_providers = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, status_data, web_server_metrics, spot_providers=None):
+        self._status_data = status_data
+        self._web_server_metrics = web_server_metrics
+        self._spot_providers = spot_providers or []
+
+    def get(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            # Build a map of SIG name -> list of provider names that can submit spots for that SIG
+            spot_submit_providers = {}
+
+            # Spothole v2.0 - disable this for now, API changes are in but this functionality is not ready yet. TODO
+            # for provider in self._spot_providers:
+            #     if not provider.enabled:
+            #         continue
+            #     for sig in SIGS:
+            #         if provider.can_submit_spot(sig.name):
+            #             spot_submit_providers.setdefault(sig.name, []).append(provider.name)
+
+            # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle
+            # things that aren't even available.
+            spot_providers: list = [
+                p["name"] for p in filter(lambda p: p["enabled"], self._status_data["spot_providers"])
+            ]
+            alert_providers = [p["name"] for p in filter(lambda p: p["enabled"], self._status_data["alert_providers"])]
+            callsign_data_providers = [
+                p["name"]
+                for p in filter(
+                    lambda p: p["enabled"],
+                    self._status_data["callsign_data_providers"],
+                )
+            ]
+            spot_providers_enabled_by_default = [
+                p["name"]
+                for p in filter(
+                    lambda p: p["enabled"] and p["enabled_by_default_in_web_ui"],
+                    self._status_data["spot_providers"],
+                )
+            ]
+
+            # If spotting to this server is enabled, "API" is another valid spot source even though it does not come from
+            # one of our providers.
+            if ALLOW_SPOTTING:
+                spot_providers.append("API")
+                spot_providers_enabled_by_default.append("API")
+
+            options = {
+                "bands": BANDS,
+                "modes": [m.value for m in Mode],
+                "mode_types": [t.value for t in ModeType],
+                "sigs": SIGS,
+                "spot_providers": spot_providers,
+                "spot_providers_enabled_by_default": spot_providers_enabled_by_default,
+                "alert_providers": alert_providers,
+                "callsign_data_providers": callsign_data_providers,
+                "continents": [c.value for c in Continent],
+                "propagation_modes": list(PROPAGATION_MODES.values()),
+                "max_spot_age": MAX_SPOT_AGE,
+                "spot_allowed": ALLOW_SPOTTING,
+                "spot_submit_providers": spot_submit_providers,
+            }
+
+            self.write(safe_json_dumps(options))
+            self.set_status(200)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+        except Exception:
+            logger.exception("Exception when handling client request to options API")
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

webserver/handlers/api/solar_conditions.py

@@ -0,0 +1,49 @@
+import logging
+from datetime import datetime
+from typing import Any
+
+import pytz
+import tornado
+from tornado import httputil
+from tornado.web import Application
+
+from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
+
+logger = logging.getLogger(__name__)
+
+
+class APISolarConditionsHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v2/solar"""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._solar_conditions = None
+        self._web_server_metrics = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, solar_conditions, web_server_metrics):
+        self._solar_conditions = solar_conditions
+        self._web_server_metrics = web_server_metrics
+
+    def get(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            self.write(self._solar_conditions.to_json())
+            self.set_status(200)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+        except Exception:
+            logger.exception("Exception when handling client request to solar conditions API")
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

webserver/handlers/api/spots.py

@@ -0,0 +1,286 @@
+import copy
+import logging
+from datetime import datetime, timedelta
+from typing import Any
+
+import pytz
+import tornado
+import tornado_eventsource.handler
+from tornado import httputil
+from tornado.web import Application
+
+from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
+from data.lookup_credentials import extract_credentials
+
+logger = logging.getLogger(__name__)
+
+
+class APISpotsHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v2/spots"""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._spots = None
+        self._web_server_metrics = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, spots, web_server_metrics):
+        self._spots = spots
+        self._web_server_metrics = web_server_metrics
+
+    @staticmethod
+    def _enrich(spots, credentials):
+        enriched = []
+        for spot in spots:
+            spot_copy = copy.deepcopy(spot)
+            spot_copy.infer_missing(credentials)
+            enriched.append(spot_copy)
+        return enriched
+
+    def get(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            # request.arguments contains lists for each param key because technically the client can supply multiple,
+            # reduce that to just the first entry, and convert bytes to string
+            query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
+
+            # Fetch all spots matching the query, then optionally enrich with online data
+            credentials = extract_credentials(self.request.headers)
+            fields = [f.strip() for f in query_params["fields"].split(",")] if "fields" in query_params else []
+            data = get_spot_list_with_filters(self._spots, query_params)
+            if credentials:
+                data = self._enrich(data, credentials)
+            # Filter for only the required fields, if necessary
+            if fields:
+                data = filter_fields(data, fields)
+            self.write(safe_json_dumps(data))
+            self.set_status(200)
+        except ValueError as e:
+            self.write(safe_json_dumps(f"Bad request - {e!s}"))
+            self.set_status(400)
+        except Exception:
+            logger.exception("Excedption when handling client request to spots API")
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)
+        self.set_header("Cache-Control", "no-store")
+        self.set_header("Content-Type", "application/json")
+
+
+class APISpotsStreamHandler(tornado_eventsource.handler.EventSourceHandler):
+    """API request handler for /api/v2/spots/stream"""
+
+    def __init__(self, application, request, **kwargs: Any):
+        self._sse_spot_broadcaster = None
+        self._web_server_metrics = None
+        self._query_params = None
+        self._credentials = None
+        self._fields = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, sse_spot_broadcaster, web_server_metrics):
+        self._sse_spot_broadcaster = sse_spot_broadcaster
+        self._web_server_metrics = web_server_metrics
+
+    def custom_headers(self):
+        """Custom headers to avoid e.g. nginx reverse proxy from buffering SSE data"""
+
+        return {"Cache-Control": "no-store", "X-Accel-Buffering": "no"}
+
+    def open(self):
+        """Called once on the client opening a connection, set things up"""
+
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            # request.arguments contains lists for each param key because technically the client can supply multiple,
+            # reduce that to just the first entry, and convert bytes to string
+            self._query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
+            self._credentials = extract_credentials(self.request.headers)
+            self._fields = (
+                [f.strip() for f in self._query_params["fields"].split(",")] if "fields" in self._query_params else []
+            )
+
+            # Flush headers immediately so nginx doesn't time out waiting for a response
+            self.write_message("keepalive", "")
+
+            # Register to handle new spots arriving. The callback() method will get called with the new spot as an
+            # argument.
+            self._sse_spot_broadcaster.register(self)
+
+        except Exception:
+            logger.exception("Exception when serving SSE socket")
+            self.close()
+
+    def close(self):
+        """When the user closes the socket, deregister ourselves from the spot broadcaster"""
+
+        self._sse_spot_broadcaster.unregister(self)
+        super().close()
+
+    def callback(self, spot):
+        """Callback when a new spot arrives"""
+
+        try:
+            # If the new spot matches our param filters, send it to the client. If not, ignore it.
+            if spot_allowed_by_query(spot, self._query_params):
+                # Add lookup data if we have credentials
+                if self._credentials:
+                    spot = copy.deepcopy(spot)
+                    spot.infer_missing(self._credentials)
+                # Filter fields returned if necessary
+                if self._fields:
+                    spot = filter_fields([spot], self._fields)[0]
+                self.write_message(msg=safe_json_dumps(spot))
+        except Exception:
+            logger.exception("Exception in SSE callback, connection will be closed")
+            self.close()
+
+
+def get_spot_list_with_filters(all_spots, query):
+    """Utility method to apply filters to the overall spot list and return only a subset. Enables query parameters in
+    the main "spots" GET call."""
+
+    # Create a shallow copy of the spot list, ordered by spot time, then filter the list to reduce it only to spots
+    # that match the filter parameters in the query string. Finally, apply a limit to the number of spots returned.
+    # The list of query string filters is defined in the API docs.
+    spot_ids = all_spots.keys()
+    spots = []
+    for k in spot_ids:
+        s = all_spots.get(k)
+        if s is not None:
+            spots.append(s)
+    spots = sorted(spots, key=lambda spot: spot.time if spot and spot.time else 0, reverse=True)
+    spots = list(filter(lambda spot: spot_allowed_by_query(spot, query), spots))
+    if "limit" in query:
+        spots = spots[: int(query.get("limit"))]
+
+    # Ensure only the latest spot of each callsign-SSID combo is present in the list. This relies on the
+    # list being in reverse time order, so if any future change allows re-ordering the list, that should
+    # be done *after* this. SSIDs are deliberately included here (see issue #68) because e.g. M0TRT-7
+    # and M0TRT-9 APRS transponders could well be in different locations, on different frequencies etc.
+    # This is a special consideration for the geo map and band map views (and Field Spotter) because while
+    # duplicates are fine in the main spot list (e.g. different cluster spots of the same DX) this doesn't
+    # work well for the other views.
+    if "dedupe" in query:
+        dedupe = query.get("dedupe").upper() == "TRUE"
+        if dedupe:
+            spots_temp = []
+            already_seen = []
+            for s in spots:
+                call_plus_ssid = s.dx_call + (s.dx_ssid if s.dx_ssid else "")
+                if call_plus_ssid not in already_seen:
+                    spots_temp.append(s)
+                    already_seen.append(call_plus_ssid)
+            spots = spots_temp
+
+    return spots
+
+
+def spot_allowed_by_query(spot, query):
+    """Given URL query params and a spot, figure out if the spot "passes" the requested filters or is rejected. The list
+    of query parameters and their function is defined in the API docs."""
+
+    for k in query:
+        match k:
+            case "since":
+                since = datetime.fromtimestamp(int(query.get(k)), pytz.UTC).timestamp()
+                if not spot.time or spot.time <= since:
+                    return False
+            case "max_age":
+                max_age = int(query.get(k))
+                since = (datetime.now(pytz.UTC) - timedelta(seconds=max_age)).timestamp()
+                if not spot.time or spot.time <= since:
+                    return False
+            case "received_since":
+                since = datetime.fromtimestamp(float(query.get(k)), pytz.UTC).timestamp()
+                if not spot.received_time or spot.received_time <= since:
+                    return False
+            case "source":
+                sources = query.get(k).split(",")
+                if not spot.source or spot.source not in sources:
+                    return False
+            case "sig":
+                # If a list of sigs is provided, the spot must have a sig and it must match one of them.
+                # The special "sig" "NO_SIG", when supplied in the list, mathches spots with no sig.
+                sigs = query.get(k).split(",")
+                include_no_sig = "NO_SIG" in sigs
+                if not spot.sig and not include_no_sig:
+                    return False
+                if spot.sig and spot.sig not in sigs:
+                    return False
+            case "needs_sig":
+                # If true, a sig is required, regardless of what it is, it just can't be missing. Mutually
+                # exclusive with supplying the special "NO_SIG" parameter to the "sig" query param.
+                needs_sig = query.get(k).upper() == "TRUE"
+                if needs_sig and not spot.sig:
+                    return False
+            case "needs_sig_ref":
+                # If true, at least one sig ref is required, regardless of what it is, it just can't be missing.
+                needs_sig_ref = query.get(k).upper() == "TRUE"
+                if needs_sig_ref and (not spot.sig_refs or len(spot.sig_refs) == 0):
+                    return False
+            case "band":
+                bands = query.get(k).split(",")
+                if not spot.band or spot.band not in bands:
+                    return False
+            case "mode":
+                modes = query.get(k).split(",")
+                if not spot.mode or spot.mode not in modes:
+                    return False
+            case "mode_type":
+                mode_types = query.get(k).split(",")
+                if not spot.mode_type or spot.mode_type not in mode_types:
+                    return False
+            case "dx_continent":
+                dxconts = query.get(k).split(",")
+                if not spot.dx_continent or spot.dx_continent not in dxconts:
+                    return False
+            case "de_continent":
+                deconts = query.get(k).split(",")
+                if not spot.de_continent or spot.de_continent not in deconts:
+                    return False
+            case "comment_includes":
+                comment_includes = query.get(k).strip()
+                if not spot.comment or comment_includes.upper() not in spot.comment.upper():
+                    return False
+            case "dx_call_includes":
+                dx_call_includes = query.get(k).strip()
+                if not spot.dx_call or dx_call_includes.upper() not in spot.dx_call.upper():
+                    return False
+            case "text_includes":
+                text_includes = query.get(k).strip()
+                if (not spot.dx_call or text_includes.upper() not in spot.dx_call.upper()) and (
+                    not spot.comment or text_includes.upper() not in spot.comment.upper()
+                ):
+                    return False
+            case "allow_qrt":
+                # If false, spots that are flagged as QRT are not returned.
+                prevent_qrt = query.get(k).upper() == "FALSE"
+                if prevent_qrt and spot.qrt:
+                    return False
+            case "needs_good_location":
+                # If true, spots require a "good" location to be returned
+                needs_good_location = query.get(k).upper() == "TRUE"
+                if needs_good_location and not spot.dx_location_good:
+                    return False
+    return True
+
+
+def filter_fields(spots, fields):
+    """Given a list of spot objects, return copies containing only the named fields."""
+
+    return [{k: v for k, v in spot.__dict__.items() if k in fields} for spot in spots]

webserver/handlers/api/status.py

@@ -0,0 +1,49 @@
+import logging
+from datetime import datetime
+from typing import Any
+
+import pytz
+import tornado
+from tornado import httputil
+from tornado.web import Application
+
+from core.prometheus_metrics_handler import api_requests_counter
+from core.utils import safe_json_dumps
+
+logger = logging.getLogger(__name__)
+
+
+class APIStatusHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v2/status"""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._status_data = None
+        self._web_server_metrics = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, status_data, web_server_metrics):
+        self._status_data = status_data
+        self._web_server_metrics = web_server_metrics
+
+    def get(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            self.write(safe_json_dumps(self._status_data))
+            self.set_status(200)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+        except Exception:
+            logger.exception("Exception when handling client request to status API")
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)

webserver/handlers/api/v1_addspot.py

@@ -0,0 +1,152 @@
+import logging
+import re
+from datetime import datetime
+from typing import Any
+
+import pytz
+import tornado
+from tornado import httputil
+from tornado.web import Application
+
+from core.config import ALLOW_SPOTTING
+from core.constants import UNKNOWN_BAND
+from core.prometheus_metrics_handler import api_requests_counter
+from core.sig_utils import get_ref_regex_for_sig
+from core.utils import infer_band_from_freq, safe_json_dumps
+from data.spot import Spot
+
+logger = logging.getLogger(__name__)
+
+
+class V1APISpotHandler(tornado.web.RequestHandler):
+    """API request handler for /api/v1/spot (POST). Included in early Spothole v2 for backwards compatibility."""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._spots = None
+        self._web_server_metrics = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, spots, web_server_metrics):
+        self._spots = spots
+        self._web_server_metrics = web_server_metrics
+
+    def post(self):
+        try:
+            # Metrics
+            self._web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self._web_server_metrics["api_access_counter"] += 1
+            self._web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
+            # Reject if not allowed
+            if not ALLOW_SPOTTING:
+                self.set_status(401)
+                self.write(safe_json_dumps("Error - this server does not allow new spots to be added via the API."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if format not json
+            if not self.request.headers.get("Content-Type", "").startswith("application/json"):
+                self.set_status(415)
+                self.write(safe_json_dumps("Error - request Content-Type must be application/json"))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if request body is empty
+            post_data = self.request.body
+            if not post_data:
+                self.set_status(422)
+                self.write(safe_json_dumps("Error - request body is empty"))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Read in the request body as JSON then convert to a Spot object
+            json_spot = tornado.escape.json_decode(post_data)
+            spot = Spot(**json_spot)
+
+            # Reject if no timestamp, frequency, dx_call or de_call
+            if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
+                self.set_status(422)
+                self.write(
+                    safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.")
+                )
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject invalid-looking callsigns
+            if not re.match(r"^[A-Za-z0-9/\-]*$", spot.dx_call):
+                self.set_status(422)
+                self.write(safe_json_dumps(f"Error - '{spot.dx_call}' does not look like a valid callsign."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+            if not re.match(r"^[A-Za-z0-9/\-]*$", spot.de_call):
+                self.set_status(422)
+                self.write(safe_json_dumps(f"Error - '{spot.de_call}' does not look like a valid callsign."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if frequency not in a known band
+            if infer_band_from_freq(spot.freq) == UNKNOWN_BAND:
+                self.set_status(422)
+                self.write(safe_json_dumps(f"Error - Frequency of {spot.freq / 1000.0!s}kHz is not in a known band."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if grid formatting incorrect
+            if spot.dx_grid and not re.match(
+                r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$",
+                spot.dx_grid.upper(),
+            ):
+                self.set_status(422)
+                self.write(safe_json_dumps(f"Error - '{spot.dx_grid}' does not look like a valid Maidenhead grid."))
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # Reject if sig_ref format incorrect for sig
+            if (
+                spot.sig
+                and spot.sig_refs
+                and len(spot.sig_refs) > 0
+                and spot.sig_refs[0].id
+                and get_ref_regex_for_sig(spot.sig)
+                and not re.match(get_ref_regex_for_sig(spot.sig), spot.sig_refs[0].id)
+            ):
+                self.set_status(422)
+                self.write(
+                    safe_json_dumps(
+                        f"Error - '{spot.sig_refs[0].id}' does not look like a valid reference for {spot.sig}."
+                    )
+                )
+                self.set_header("Cache-Control", "no-store")
+                self.set_header("Content-Type", "application/json")
+                return
+
+            # infer missing data, and add it to our database.
+            spot.source = "API"
+            spot.infer_missing()
+            self._spots.set(spot.id, spot)
+
+            self.write(safe_json_dumps("OK"))
+            self.set_status(201)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")
+
+        except Exception:
+            logger.exception("Exception when handling client request to add spot API")
+            self.write(safe_json_dumps("Error - an internal server error occurred."))
+            self.set_status(500)
+            self.set_header("Cache-Control", "no-store")
+            self.set_header("Content-Type", "application/json")

webserver/handlers/api/v1_compatability.py

@@ -0,0 +1,63 @@
+import tornado
+from tornado.httpclient import AsyncHTTPClient
+from tornado.httputil import HTTPHeaders
+
+_LEGACY_PARAM_TO_HEADER_MAP = {
+    "qrz_username": "X-QRZ-Username",
+    "qrz_password": "X-QRZ-Password",
+    "qrz_session_key": "X-QRZ-Session-Key",
+    "hamqth_username": "X-HamQTH-Username",
+    "hamqth_password": "X-HamQTH-Password",
+    "hamqth_session_id": "X-HamQTH-Session-ID",
+}
+
+
+class V1RedirectHandler(tornado.web.RequestHandler):
+    """Transparently proxies requests from the old API to the new one,
+    returning whatever the v2 endpoint returns, for endpoints with no breaking changes."""
+
+    async def _proxy(self, path):
+        new_url = f"{self.request.protocol}://{self.request.host}/api/v2/{path}"
+        if self.request.query:
+            new_url += f"?{self.request.query}"
+
+        # Copy the incoming headers so we can add translated legacy credentials without changing the original
+        # request.
+        headers = HTTPHeaders(self.request.headers)
+        for param, header in _LEGACY_PARAM_TO_HEADER_MAP.items():
+            value = self.get_query_argument(param, default=None)
+            if value:
+                headers[header] = value
+
+        client = AsyncHTTPClient()
+        try:
+            response = await client.fetch(
+                new_url,
+                method=self.request.method,
+                headers=self.request.headers,
+                body=None if self.request.method == "GET" else (self.request.body or b""),
+                raise_error=False,
+                follow_redirects=False,
+                request_timeout=10.0,
+            )
+        except Exception as e:
+            raise tornado.web.HTTPError(502, reason=str(e))
+
+        self.set_status(response.code, response.reason)
+        if isinstance(response.headers, HTTPHeaders):
+            for name, value in response.headers.get_all():
+                # Let Tornado recompute these for the outgoing response
+                if name.lower() not in (
+                    "content-length",
+                    "transfer-encoding",
+                    "connection",
+                ):
+                    self.add_header(name, value)
+            if response.body:
+                self.write(response.body)
+
+    async def get(self, path):
+        await self._proxy(path)
+
+    async def post(self, path):
+        await self._proxy(path)

webserver/handlers/api/v1_spots.py

@@ -0,0 +1,55 @@
+import re
+
+from webserver.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
+
+_GRID_SOURCE_RE = re.compile(r'"dx_location_source":\s*"GRID"')
+_LEGACY_PARAM_TO_HEADER_MAP = {
+    "qrz_username": "X-QRZ-Username",
+    "qrz_password": "X-QRZ-Password",
+    "qrz_session_key": "X-QRZ-Session-Key",
+    "hamqth_username": "X-HamQTH-Username",
+    "hamqth_password": "X-HamQTH-Password",
+    "hamqth_session_id": "X-HamQTH-Session-ID",
+}
+
+
+def _handle_legacy_params(handler):
+    """Copy v1 query-string QRZ/HamQTH credentials into the v2 headers, so the v2 handler can see them"""
+
+    for param, header in _LEGACY_PARAM_TO_HEADER_MAP.items():
+        if header in handler.request.headers:
+            continue
+        value = handler.get_query_argument(param, default=None)
+        if value:
+            handler.request.headers[header] = value
+
+
+class V1APISpotsHandler(APISpotsHandler):
+    """API request handler for /api/v1/spots (GET). Included in early Spothole v2 for backwards compatibility."""
+
+    def prepare(self):
+        _handle_legacy_params(self)
+        super().prepare()
+
+    def write(self, chunk):
+        if isinstance(chunk, str):
+            chunk = _GRID_SOURCE_RE.sub('"dx_location_source": "SPOT"', chunk)
+        super().write(chunk)
+
+
+class V1APISpotsStreamHandler(APISpotsStreamHandler):
+    """API request handler for /api/v1/spots/stream (SSE). Included in early Spothole v2 for backwards compatibility."""
+
+    def prepare(self):
+        _handle_legacy_params(self)
+        super().prepare()
+
+    def write_message(self, *args, **kwargs):
+        args = list(args)
+        for i, a in enumerate(args):
+            if isinstance(a, str) and '"dx_location_source"' in a:
+                args[i] = _GRID_SOURCE_RE.sub('"dx_location_source": "SPOT"', a)
+        for k, v in kwargs.items():
+            if isinstance(v, str) and '"dx_location_source"' in v:
+                kwargs[k] = _GRID_SOURCE_RE.sub('"dx_location_source": "SPOT"', v)
+        super().write_message(*args, **kwargs)

webserver/handlers/manifesthandler.py

@@ -0,0 +1,11 @@
+import tornado.web
+
+from core.config import BASE_URL
+
+
+class ManifestHandler(tornado.web.RequestHandler):
+    """Handler for manifest.webmanifest, which needs BASE_URL inserted and a custom content-type"""
+
+    def get(self):
+        self.set_header("Content-Type", "application/manifest+json; charset=UTF-8")
+        self.render("manifest.webmanifest", baseurl=BASE_URL)

webserver/handlers/metrics.py

@@ -0,0 +1,13 @@
+import tornado
+from prometheus_client import CONTENT_TYPE_LATEST
+
+from core.prometheus_metrics_handler import get_metrics
+
+
+class PrometheusMetricsHandler(tornado.web.RequestHandler):
+    """Handler for Prometheus metrics endpoint"""
+
+    def get(self):
+        self.write(get_metrics())
+        self.set_status(200)
+        self.set_header("Content-Type", CONTENT_TYPE_LATEST)

webserver/handlers/pagetemplate.py

@@ -0,0 +1,47 @@
+from datetime import datetime
+from typing import Any
+
+import pytz
+import tornado
+from tornado import httputil
+from tornado.web import Application
+
+from core.config import ALLOW_SPOTTING, BASE_URL, SERVER_OWNER_CALLSIGN, WEB_UI_OPTIONS
+from core.constants import SOFTWARE_VERSION
+from core.prometheus_metrics_handler import page_requests_counter
+
+
+class PageTemplateHandler(tornado.web.RequestHandler):
+    """Handler for all HTML pages generated from templates"""
+
+    def __init__(
+        self,
+        application: "Application",
+        request: httputil.HTTPServerRequest,
+        **kwargs: Any,
+    ):
+        self._template_name = None
+        self._web_server_metrics = None
+        super().__init__(application, request, **kwargs)
+
+    def initialize(self, template_name, web_server_metrics):
+        self._template_name = template_name
+        self._web_server_metrics = web_server_metrics
+
+    def get(self):
+        # Metrics
+        self._web_server_metrics["last_page_access_time"] = datetime.now(pytz.UTC)
+        self._web_server_metrics["page_access_counter"] += 1
+        self._web_server_metrics["status"] = "OK"
+        page_requests_counter.inc()
+
+        # Load named template, and provide variables used in templates
+        self.render(
+            f"{self._template_name}.html",
+            software_version=SOFTWARE_VERSION,
+            server_owner_callsign=SERVER_OWNER_CALLSIGN,
+            allow_spotting=ALLOW_SPOTTING,
+            web_ui_options=WEB_UI_OPTIONS,
+            baseurl=BASE_URL,
+            current_path=self.request.path,
+        )

webserver/sse_broadcaster.py

@@ -0,0 +1,41 @@
+import logging
+import threading
+
+from tornado.ioloop import IOLoop
+
+logger = logging.getLogger(__name__)
+
+
+class SSEBroadcaster:
+    """Bridge between DataStore listener callbacks (which fire on provider threads) to Tornado's async SSE handlers
+    (which live on the IOLoop thread) to avoid any interdependency between them."""
+
+    def __init__(self):
+        self._handlers = set()
+        self._lock = threading.Lock()
+        self._loop = None
+
+    def bind_to_web_server_loop(self):
+        self._loop = IOLoop.current()
+
+    def register(self, handler):
+        with self._lock:
+            self._handlers.add(handler)
+
+    def unregister(self, handler):
+        with self._lock:
+            self._handlers.discard(handler)
+
+    def publish(self, value):
+        self._loop.add_callback(self._broadcast, value)
+
+    def _broadcast(self, value):
+        with self._lock:
+            handlers = list(self._handlers)
+        for handler in handlers:
+            try:
+                handler.callback(value)
+            except Exception:
+                # Connection probably dropped, ignore and de-register the handler to stop getting future items.
+                logger.debug("Failed to push to an SSE client; dropping it")
+                self.unregister(handler)

webserver/webserver.py

@@ -0,0 +1,276 @@
+import asyncio
+import logging
+import os
+
+import tornado
+from tornado.web import StaticFileHandler
+
+from core.config import (
+    ALLOW_SPOTTING,
+    API_ONLY_MODE,
+    BASE_URL,
+    LOG_WEB_REQUESTS,
+    WEB_SERVER_PORT,
+)
+from core.data_providers import DATA_PROVIDERS
+from core.data_store import DATA_STORE
+from webserver.handlers.api.addspot import APISpotHandler
+from webserver.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
+from webserver.handlers.api.dxstats import APIDxStatsHandler
+from webserver.handlers.api.lookups import (
+    APILookupCallHandler,
+    APILookupGridHandler,
+    APILookupSIGRefHandler,
+)
+from webserver.handlers.api.options import APIOptionsHandler
+from webserver.handlers.api.solar_conditions import APISolarConditionsHandler
+from webserver.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
+from webserver.handlers.api.status import APIStatusHandler
+from webserver.handlers.api.v1_addspot import V1APISpotHandler
+from webserver.handlers.api.v1_compatability import V1RedirectHandler
+from webserver.handlers.api.v1_spots import V1APISpotsHandler, V1APISpotsStreamHandler
+from webserver.handlers.manifesthandler import ManifestHandler
+from webserver.handlers.metrics import PrometheusMetricsHandler
+from webserver.handlers.pagetemplate import PageTemplateHandler
+from webserver.sse_broadcaster import SSEBroadcaster
+
+logger = logging.getLogger(__name__)
+
+_HERE = os.path.dirname(__file__ or "")
+
+
+class WebServer:
+    """Provides the public-facing web server."""
+
+    def __init__(self):
+        """Constructor"""
+
+        self._data_store = DATA_STORE
+        self._data_providers = DATA_PROVIDERS
+        self._spot_broadcaster = SSEBroadcaster()
+        self._alert_broadcaster = SSEBroadcaster()
+        self._port = WEB_SERVER_PORT
+        self._api_only_mode = API_ONLY_MODE
+        self._shutdown_event = asyncio.Event()
+        self.web_server_metrics = {
+            "last_page_access_time": None,
+            "last_api_access_time": None,
+            "page_access_counter": 0,
+            "api_access_counter": 0,
+            "status": "Starting",
+        }
+
+    def setup(self):
+        # Listen for new spots and alerts being added to the cache, so we can notify SSE clients immediately
+        DATA_STORE.spots.add_listener(self._spot_broadcaster.publish)
+        DATA_STORE.alerts.add_listener(self._alert_broadcaster.publish)
+
+    def start(self):
+        """Start the web server"""
+
+        asyncio.run(self._start_inner())
+
+    def stop(self):
+        """Stop the web server"""
+
+        self._shutdown_event.set()
+
+    async def _start_inner(self):
+        """Start method (async). Sets up the Tornado application."""
+
+        # Bind the SSE broadcasters to the web server's loop, so they fire correctly
+        self._spot_broadcaster.bind_to_web_server_loop()
+        self._alert_broadcaster.bind_to_web_server_loop()
+
+        # Prepare a list of common arguments that are passed in to every API & page handler. This is just a basic thing
+        # to avoid copy-pasting the same thing to every route declaration below.
+        handler_opts = {"web_server_metrics": self.web_server_metrics}
+
+        # API endpoints are always enabled
+        api_routes = [
+            (
+                r"/api/v2/spots",
+                APISpotsHandler,
+                {"spots": self._data_store.spots, **handler_opts},
+            ),
+            (
+                r"/api/v2/alerts",
+                APIAlertsHandler,
+                {"alerts": self._data_store.alerts, **handler_opts},
+            ),
+            (
+                r"/api/v2/spots/stream",
+                APISpotsStreamHandler,
+                {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts},
+            ),
+            (
+                r"/api/v2/alerts/stream",
+                APIAlertsStreamHandler,
+                {"sse_alert_broadcaster": self._alert_broadcaster, **handler_opts},
+            ),
+            (
+                r"/api/v2/solar",
+                APISolarConditionsHandler,
+                {"solar_conditions": self._data_store.solar_conditions.get(), **handler_opts},
+            ),
+            (
+                r"/api/v2/dxstats",
+                APIDxStatsHandler,
+                {"spots": self._data_store.spots, **handler_opts},
+            ),
+            (
+                r"/api/v2/options",
+                APIOptionsHandler,
+                {"status_data": self._data_store.status.get(), **handler_opts},
+            ),
+            (
+                r"/api/v2/status",
+                APIStatusHandler,
+                {"status_data": self._data_store.status.get(), **handler_opts},
+            ),
+            (r"/api/v2/lookup/call", APILookupCallHandler, {**handler_opts}),
+            (r"/api/v2/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
+            (r"/api/v2/lookup/grid", APILookupGridHandler, {**handler_opts}),
+            (
+                r"/api/v2/spot",
+                APISpotHandler,
+                {
+                    "spots": self._data_store.spots,
+                    "spot_providers": self._data_providers,
+                    **handler_opts,
+                },
+            ),
+        ]
+
+        # v1 API redirects. Most v1 enpoints are unchanged in v2, and get an HTTP 308 redirect to the v2 API. The ones
+        # that have the major breaking changes get a bespoke handler.
+        v1_compat_routes = [
+            (
+                r"/api/v1/spots",
+                V1APISpotsHandler,
+                {"spots": self._data_store.spots, **handler_opts},
+            ),
+            (
+                r"/api/v1/spots/stream",
+                V1APISpotsStreamHandler,
+                {"sse_spot_broadcaster": self._spot_broadcaster, **handler_opts},
+            ),
+            (
+                r"/api/v1/spot",
+                V1APISpotHandler,
+                {
+                    "spots": self._data_store.spots,
+                    **handler_opts,
+                },
+            ),
+            (r"/api/v1/(.*)", V1RedirectHandler),
+        ]
+
+        # If in API-only mode, serve a basic homepage; in normal mode, serve the usual UI routes
+        if self._api_only_mode:
+            logger.info("API-only mode is enabled. Web UI will not be served.")
+            ui_routes = [
+                (
+                    r"/",
+                    PageTemplateHandler,
+                    {"template_name": "api_only_home", **handler_opts},
+                )
+            ]
+        else:
+            ui_routes = [
+                (r"/", PageTemplateHandler, {"template_name": "spots", **handler_opts}),
+                (
+                    r"/map",
+                    PageTemplateHandler,
+                    {"template_name": "map", **handler_opts},
+                ),
+                (
+                    r"/bands",
+                    PageTemplateHandler,
+                    {"template_name": "bands", **handler_opts},
+                ),
+                (
+                    r"/alerts",
+                    PageTemplateHandler,
+                    {"template_name": "alerts", **handler_opts},
+                ),
+                (
+                    r"/conditions",
+                    PageTemplateHandler,
+                    {"template_name": "conditions", **handler_opts},
+                ),
+                (
+                    r"/status",
+                    PageTemplateHandler,
+                    {"template_name": "status", **handler_opts},
+                ),
+                (
+                    r"/about",
+                    PageTemplateHandler,
+                    {"template_name": "about", **handler_opts},
+                ),
+            ]
+            # Only allow the Add Spot page if spotting is allowed
+            if ALLOW_SPOTTING:
+                ui_routes += [
+                    (
+                        r"/add-spot",
+                        PageTemplateHandler,
+                        {"template_name": "add_spot", **handler_opts},
+                    )
+                ]
+
+        # API docs, Prometheus metrics, webapp manifest and static assets are always available regardless of API-only
+        # mode.
+        misc_routes = [
+            (
+                r"/apidocs",
+                PageTemplateHandler,
+                {"template_name": "apidocs", **handler_opts},
+            ),
+            (r"/metrics", PrometheusMetricsHandler),
+            (r"/manifest.webmanifest", ManifestHandler),
+            # If e.g. nginx is configured as a reverse proxy with a hard-coded path to static files, as per the README,
+            # this will never have to handle anything, but having it here allows Spothole to work without nginx for
+            # testing.
+            (
+                r"/static/(.*)",
+                StaticFileHandler,
+                {"path": os.path.join(_HERE, "../static")},
+            ),
+        ]
+
+        app = tornado.web.Application(
+            api_routes + v1_compat_routes + ui_routes + misc_routes,
+            template_path=os.path.join(_HERE, "../templates"),
+            log_function=request_log,
+            debug=False,
+        )
+        app.listen(self._port, xheaders=True)
+        logger.info(f"Web server running on port {WEB_SERVER_PORT!s}")
+        logger.info(f"You can access your copy of Spothole at {BASE_URL}")
+        await self._shutdown_event.wait()
+
+
+def request_log(handler):
+    """Custom log function to provide more data about requests when enabled, and to provide the ability to turn off
+    web request logging altogetether."""
+
+    if LOG_WEB_REQUESTS:
+        if handler.get_status() < 500:
+            log_method = logger.info
+        else:
+            log_method = logger.warning
+
+        request = handler.request
+        client_ip = request.remote_ip
+        referrer = request.headers.get("Referer", "-")
+        user_agent = request.headers.get("User-Agent", "-")
+
+        log_method(
+            f'{client_ip} - "{request.method} {request.uri}" {handler.get_status()} {request.request_time():.2f}ms | Ref: {referrer} | UA: {user_agent}'
+        )
+
+
+# Global object
+WEB_SERVER = WebServer()