Create the concept of API keys to allow third party clients to skip the CAPTCHA check on spot submission

config-example.yml

@@ -354,13 +354,21 @@ allow_spotting: true
 # this, and will log in using the spotter's callsign to send the spot. Upstream spotting to POTA, SOTA etc. is a work
 # in progress. Requires allow_spotting to also be true. Set to false to only accept spots into the local Spothole
 # database, without forwarding them to any external service.
-allow_upstream_spotting: false
+allow_upstream_spotting: true
 
-# Google reCAPTCHA v2 keys for CAPTCHA protection on upstream spot submission. Both keys must be set to enable CAPTCHA.
+# Require a CAPTCHA or API key to submit spots? If false, anyone can submit spots via the web interface or the API. If
-# Leave both empty to disable CAPTCHA (e.g. for a private/trusted server) or if allow_spotting is false, in which case
+# true, users of the web interface must solve a CAPTCHA (see reCAPTCHA config below), and third-party clients must
-# they will do nothing. Note that with CAPTCHA enabled, this will prevent third-party clients submitting spots through
+# provide one of the API keys listed below. Recommended for public servers where spotting is allowed.
-# Spothole unless the clients are web-based, use the same site key, have their domains enabled in your reCAPTCHA config,
+protect_spot_submission: true
-# and of course their user solves the CAPTCHA.
+
+# API keys used by third-party client developers. Clients provide a key in the "X-API-Key" request header when calling
+# the add spot API, and this is compared against the server's list below. Generate a long random string for each
+# client you want to allow (e.g. openssl rand -hex 32), and send it to them privately. To revoke a client's access,
+# remove their key from this list and restart Spothole.
+api_keys: []
+
+# Google reCAPTCHA v2 keys for CAPTCHA protection on spot submission from the web UI. Both keys must be set to enable
+# CAPTCHA. They are only used if protect_spot_submission is true.
 # You can sign up for reCAPTCHA at https://www.google.com/recaptcha/
 recaptcha_site_key: ""
 recaptcha_secret_key: ""

core/config.py

@@ -31,8 +31,12 @@ ALLOW_SPOTTING = config.get("allow_spotting", True)
 ALLOW_UPSTREAM_SPOTTING = config.get("allow_upstream_spotting", True)
 WEB_UI_OPTIONS = config.get("web_ui_options", {})
 API_ONLY_MODE = config.get("api_only_mode", False)
+API_KEYS = [k for k in (config.get("api_keys") or []) if k]
 RECAPTCHA_SECRET_KEY = config.get("recaptcha_secret_key", "")
 RECAPTCHA_SITE_KEY = config.get("recaptcha_site_key", "")
+# If not explicitly set, protect spot submission if CAPTCHA is configured, as this was the behaviour before the option
+# existed. This avoids silently opening up spot submission on servers that upgrade without updating their config.
+PROTECT_SPOT_SUBMISSION = config.get("protect_spot_submission", bool(RECAPTCHA_SECRET_KEY))
 LOG_LEVEL = config.get("log_level", "INFO")
 LOG_WEB_REQUESTS = config.get("log_web_requests", False)
 
@@ -40,10 +44,17 @@ WEB_UI_OPTIONS["qrz_enabled"] = any(p["class"] == "QRZ" and p["enabled"] for p i
 WEB_UI_OPTIONS["hamqth_enabled"] = any(
     p["class"] == "HamQTH" and p["enabled"] for p in config["callsign_data_providers"]
 )
-WEB_UI_OPTIONS["recaptcha_site_key"] = RECAPTCHA_SITE_KEY
+# The web UI only needs to show a CAPTCHA if spot submission is protected
+WEB_UI_OPTIONS["recaptcha_site_key"] = RECAPTCHA_SITE_KEY if PROTECT_SPOT_SUBMISSION else ""
 WEB_UI_OPTIONS["allow_upstream_spotting"] = ALLOW_SPOTTING and ALLOW_UPSTREAM_SPOTTING
 
 
+if ALLOW_SPOTTING and PROTECT_SPOT_SUBMISSION and not (RECAPTCHA_SITE_KEY and RECAPTCHA_SECRET_KEY):
+    logger.warning(
+        "Spot submission is protected but reCAPTCHA keys are not set, so only clients with an API key will be able to submit spots. Users of the web interface will not be able to add spots."
+    )
+
+
 def create_provider_from_config(package, config_providers_entry):
     """Utility method to get a provider based on the class specified in its config entry. You must also provide the
     package to look for it in, as there are several types of provider. e.g. package "providers.spot", where the config

static/apidocs/openapi.yml

@@ -9,6 +9,8 @@ info:
     
     While I provide this API for free, there are some conditions of use that you must adhere to. These are not onerous, but ensure the API can remain available to everyone. These apply even if you are getting an AI to write your client software for you. See https://spothole.app/help/usage/clients#terms for details.
     
+    If you want to submit spots from your client, an API key is required. Please contact the server owner for a key if you want to use this functionality.
+    
     ## Changelog
 
     ### 3.0
@@ -24,11 +26,16 @@ info:
     * **Breaking change:** In the `/options` response, `sigs` has been renamed to `activities`, and within each activity, `sig_type` has been renamed to `activity_type`.
     * **Breaking change:** In the `/status` response, `sig_ref_data_providers` has been renamed to `activity_ref_data_providers`, and within each provider, `sig_name` has been renamed to `activity_name`.
     * **Breaking change:** In the POST `/spot` `handling` object, `submit_upstream` and `upstream_provider` have been replaced by `upstream_providers`, a list of provider names, so a spot can be sent to multiple upstream providers at once. `upstream_credentials` is now a map of provider name to that provider's credentials.
+    * POST `/spot` now accepts an `X-API-Key` request header. On servers that protect spot submission, a valid API key allows a third party client to submit spots without needing to solve a CAPTCHA.
     
     #### Upgrading a client from v2 to v3 API endpoints
 
     In v3.0 of Spothole, the `v2` (and `v1`) API endpoints will be maintained for backwards compatibility, so if you have written a client against the `v2` API, it will continue to receive `sig`, `sig_refs` etc. as before. Where a spot or alert has more than one activity, the `v2` and `v1` APIs will return only the first one as `sig`.
     
+    If your client submits spots via POST `/spot` and uses upstream submission, replace `submit_upstream` and `upstream_provider` in the `handling` object with an `upstream_providers` list, and turn `upstream_credentials` into a map where the key is the provider name, and the value is another map of credential name to value.
+
+    Some Spothole servers, including `spothole.app`, require a CAPTCHA to submit spots via the web interface, which third-party clients can't solve. If you want your client to submit spots to such a server, ask the server operator for an API key, and send it in the `X-API-Key` request header with each add spot request.
+    
     You are encouraged to move to the `v3` API endpoints as soon as possible. To upgrade, replace `v2` with `v3` in the URLs your code calls, then rename any use of the fields, query parameters and values listed above, and handle `activities` being a list rather than a single `sig` value. If you use the activity reference lookup, call `/lookup/activityref?activity=...&id=...` instead of `/lookup/sigref?sig=...&id=...`.
 
     ### 2.2
@@ -476,9 +483,11 @@ paths:
       description: >
         Supply a JSON object containing a `spot` sub-object (the spot data) and an optional `handling` sub-object
         containing server-side instructions such as upstream submission). Check `spot_submit_providers` in the
-        `/options` response to see which activities and providers support upstream submission. cURL example:
+        `/options` response to see which activities and providers support upstream submission. If the server requires
-        `curl --request POST --header \"Content-Type: application/json\" --data '{\"spot\":{\"dx_call\":\"M0TRT\",\"time\":1760019539,\"freq\":14200000,\"comment\":\"Test spot please ignore\",\"de_call\":\"M0TRT\"}}' https://spothole.app/api/v3/spot`"
+        an API key for spot submission, you must supply a valid API key in the `X-API-Key` header.
       operationId: spot
+      parameters:
+        - $ref: '#/components/parameters/ApiKey'
       requestBody:
         description: Object containing a "spot" sub-object with the spot data, and an optional "handling" sub-object with server-side instructions of what to do with it.
         required: true
@@ -494,6 +503,22 @@ paths:
               schema:
                 type: string
                 example: "OK"
+        '401':
+          description: >
+            Spot submission is not allowed on this server, or the server requires an API key or CAPTCHA token and
+            neither was provided, or the API key was not recognised
+          content:
+            application/json:
+              schema:
+                type: string
+                example: "Error - API key not recognised."
+        '403':
+          description: Upstream submission was requested but this server does not allow it
+          content:
+            application/json:
+              schema:
+                type: string
+                example: "Error - this server does not allow upstream spot submission."
         '415':
           description: Incorrect Content-Type
           content:
@@ -518,6 +543,14 @@ paths:
 
 components:
   parameters:
+    ApiKey:
+      name: X-API-Key
+      in: header
+      description: >
+        An API key issued by the server operator. On servers that protect spot submission, a valid API key allows
+        the spot to be submitted without a CAPTCHA token. Not required on servers that don't protect spot submission.
+      schema:
+        type: string
     QrzUsername:
       name: X-QRZ-Username
       in: header
@@ -1410,9 +1443,10 @@ components:
             captcha_token:
               type: string
               description: >
-                A Google reCAPTCHA v2 response token. Required when submitting upstream if the
+                A Google reCAPTCHA v2 response token. Required if the server protects spot submission,
-                server has reCAPTCHA configured. Obtain the token by completing the reCAPTCHA
+                unless a valid API key is supplied in the `X-API-Key` header instead. Only useful for the
-                widget rendered on the Add Spot page.
+                Spothole server itself, not for third-party clients, as without access to Spothole's CAPTCHA
+                secret they have no way of solving the CAPTCHA anyway. They must use `X-API-Key` instead.
               example: "03AFY_a8Xq..."
 
     SpotStream:

templates/add-spot.html

@@ -130,7 +130,7 @@
     const ALLOW_UPSTREAM_SPOTTING = {% raw safe_json_dumps(web_ui_options["allow_upstream_spotting"]) %};
 </script>
 
-<script src="/static/js/add-spot.js?v=1790517301"></script>
+<script src="/static/js/add-spot.js?v=1790521969"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -87,7 +87,7 @@
 
 </div>
 
-<script src="/static/js/alerts.js?v=1790517301"></script>
+<script src="/static/js/alerts.js?v=1790521969"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -82,8 +82,8 @@
     const BANDS = {% raw safe_json_dumps(options["bands"]) %};
 </script>
 
-<script src="/static/js/spotsbandsandmap.js?v=1790517301"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1790521969"></script>
-<script src="/static/js/bands.js?v=1790517301"></script>
+<script src="/static/js/bands.js?v=1790521969"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/static/css/style.css?v=1790517300" type="text/css">
+<link rel="stylesheet" href="/static/css/style.css?v=1790521969" type="text/css">
 <link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -16,10 +16,10 @@
     window.fetchEventSource = fetchEventSource;
 </script>
 
-<script src="/static/js/utils.js?v=1790517300"></script>
+<script src="/static/js/utils.js?v=1790521969"></script>
-<script src="/static/js/ui-ham.js?v=1790517300"></script>
+<script src="/static/js/ui-ham.js?v=1790521969"></script>
-<script src="/static/js/geo.js?v=1790517300"></script>
+<script src="/static/js/geo.js?v=1790521969"></script>
-<script src="/static/js/common.js?v=1790517300"></script>
+<script src="/static/js/common.js?v=1790521969"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/static/js/conditions.js?v=1790517300"></script>
+<script src="/static/js/conditions.js?v=1790521969"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/help/usage/clients.html

@@ -47,6 +47,25 @@
     all means base your own project on data from the main server if you like, but if you want any control over
     reliability and downtime, please run your own copy instead.)</p>
 
+<h3 class="mt-4" id="submitting-spots">Submitting Spots</h3>
+<p>As well as reading data, clients can submit new spots to Spothole using the "add spot" API endpoint, e.g.
+    <code>https://spothole.app/api/v3/spot</code>. Spots can be added to Spothole itself, and optionally sent "upstream"
+    to other services such as the DX cluster. Check the <code>spot_allowed</code> and
+    <code>spot_submit_providers</code> fields in the "options" API response to see what the server allows.</p>
+<p>To stop bots and spammers abusing the spotting function, a Spothole server can be configured to protect against this,
+    which means you will need an <strong>API key</strong>. API keys are issued by the operator of each Spothole server,
+    so get in touch with the server owner and let them know what your software is and how it will use the API. Once you
+    have a key, send it in the <code>X-API-Key</code> header of each add spot request. For example:</p>
+<pre><code>curl --request POST \
+     --header "Content-Type: application/json" \
+     --header "X-API-Key: your-api-key-here" \
+     --data '{"spot":{"dx_call":"M0TRT","time":1760019539,"freq":14200000,"de_call":"M0TRT"}}' \
+     https://spothole.app/api/v3/spot</code></pre>
+<p>Your API key identifies your software, so please keep it private. Don't commit it to a public code repository, and
+    don't embed it in JavaScript or anywhere else your users could extract it. If a key is misused, the server operator
+    can revoke it, and spot submission from your client will stop working. Servers that don't protect spot submission
+    don't need an API key, and will accept spots with or without one.</p>
+
 <h3 class="mt-4" id="terms">Conditions of Use</h3>
 <p>There are some simple, hopefully not onerous terms and conditions that you should agree to before writing a client
     for the Spothole API. These probably aren't legally binding, and I'm just a random guy on the internet, I'm not

templates/help/usage/running.html

@@ -30,6 +30,14 @@ cp config-example.yml config.yml
     helpdesk ticket and explaining what you'll use it for. The admin team are happy with the rate of requests made by my
     Spothole server, so unless you change the source code of yours to radically increase the rate of querying Clublog,
     I'm sure they will be fine with your server too.</p>
+<p>If your server is public and allows spots to be submitted, you may want to protect it from bots and spammers by
+    setting <code>protect_spot_submission</code> to <code>true</code> and setting the reCAPTCHA keys in
+    <code>config.yml</code>. Users of the web interface will then need to solve a CAPTCHA to submit a spot. Third-party
+    client software can't do that, so if you want to allow a particular client to submit spots, generate an API key for
+    it, add it to the <code>api_keys</code> list in <code>config.yml</code>, and send it privately to the client's
+    developer. They send it in the <code>X-API-Key</code> header of each request, and Spothole will accept their spots.
+    To revoke a key, remove it from the list and restart Spothole. If <code>protect_spot_submission</code> is
+    <code>false</code>, anyone can submit spots and API keys aren't needed.</p>
 <p>Once you're happy with the content of <code>config.yml</code>, you can proceed to running the software.</p>
 <p>To run the software this time and any future times you want to run it directly from the command line:</p>
 <pre><code>source .venv/bin/activate

templates/map.html

@@ -115,8 +115,8 @@
     const CARTODB_API_KEY = "{{ web_ui_options.get('cartodb_api_key', '') }}";
 </script>
 
-<script src="/static/js/spotsbandsandmap.js?v=1790517300"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1790521968"></script>
-<script src="/static/js/map.js?v=1790517300"></script>
+<script src="/static/js/map.js?v=1790521968"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -127,8 +127,8 @@
 
 </div>
 
-<script src="/static/js/spotsbandsandmap.js?v=1790517300"></script>
+<script src="/static/js/spotsbandsandmap.js?v=1790521968"></script>
-<script src="/static/js/spots.js?v=1790517300"></script>
+<script src="/static/js/spots.js?v=1790521968"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -96,7 +96,7 @@
     </div>
 </div>
 
-<script src="/static/js/status.js?v=1790517301"></script>
+<script src="/static/js/status.js?v=1790521969"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");

webserver/handlers/api/addspot.py

@@ -1,4 +1,5 @@
 import asyncio
+import hmac
 import logging
 import re
 import threading
@@ -11,7 +12,13 @@ from tornado.ioloop import IOLoop
 from tornado.web import Application
 
 from core.activity_utils import get_ref_regex_for_activity
-from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRET_KEY
+from core.config import (
+    ALLOW_SPOTTING,
+    ALLOW_UPSTREAM_SPOTTING,
+    API_KEYS,
+    PROTECT_SPOT_SUBMISSION,
+    RECAPTCHA_SECRET_KEY,
+)
 from core.constants import UNKNOWN_BAND
 from core.utils import infer_band_from_freq, safe_json_dumps
 from data.spot import Spot
@@ -84,17 +91,34 @@ class APISpotHandler(tornado.web.RequestHandler):
             submit_upstream = len(upstream_provider_names) > 0
             captcha_token = handling.get("captcha_token", None)
 
-            # Verify CAPTCHA if required
+            # If spot submission is protected, the client must either provide a valid API key in the request header, or
-            if RECAPTCHA_SECRET_KEY:
+            # a valid CAPTCHA token in the request body. API keys are how we allow trusted third-party clients to submit
-                if not captcha_token:
+            # spots, as they can't solve a CAPTCHA.
-                    self.set_status(422)
+            if PROTECT_SPOT_SUBMISSION:
-                    self.write(safe_json_dumps("Error - CAPTCHA token is required for spot submission."))
+                api_key = self.request.headers.get("X-API-Key", "")
-                    self.set_header("Cache-Control", "no-store")
+                if api_key:
-                    self.set_header("Content-Type", "application/json")
+                    if not self._is_valid_api_key(api_key):
-                    return
+                        self.set_status(401)
-                if not await IOLoop.current().run_in_executor(None, self._verify_recaptcha, captcha_token):
+                        self.write(safe_json_dumps("Error - API key not recognised."))
-                    self.set_status(422)
+                        self.set_header("Cache-Control", "no-store")
-                    self.write(safe_json_dumps("Error - CAPTCHA verification failed."))
+                        self.set_header("Content-Type", "application/json")
+                        return
+                elif captcha_token and RECAPTCHA_SECRET_KEY:
+                    if not await IOLoop.current().run_in_executor(None, self._verify_recaptcha, captcha_token):
+                        self.set_status(422)
+                        self.write(safe_json_dumps("Error - CAPTCHA verification failed."))
+                        self.set_header("Cache-Control", "no-store")
+                        self.set_header("Content-Type", "application/json")
+                        return
+                else:
+                    self.set_status(401)
+                    if RECAPTCHA_SECRET_KEY:
+                        message = (
+                            "Error - this server requires either an API key or a CAPTCHA token for spot submission."
+                        )
+                    else:
+                        message = "Error - this server requires an API key for spot submission."
+                    self.write(safe_json_dumps(message))
                     self.set_header("Cache-Control", "no-store")
                     self.set_header("Content-Type", "application/json")
                     return
@@ -246,9 +270,7 @@ class APISpotHandler(tornado.web.RequestHandler):
             # Submit spot to the upstream provider. Run in a separate thread otherwise this blocks the whole web server
             # for everyone!
             await IOLoop.current().run_in_executor(None, provider.submit_spot, spot, credentials)
-            logger.info(
+            logger.info(f"Spot of {spot.dx_call} by {spot.de_call} submitted upstream to {upstream_provider_name}.")
-                f"Spot of {spot.dx_call} by {spot.de_call} submitted upstream to {upstream_provider_name}."
-            )
             # Trigger a re-poll after 3 second so the spot appears quickly. (Submitting to a cluster node is slower than
             # this, but we get data as a live stream from cluster anyway, so force_poll does nothing in that case. This
             # is really just for the HTTP providers when we submit a spot to them)
@@ -272,6 +294,17 @@ class APISpotHandler(tornado.web.RequestHandler):
                 return p
         return None
 
+    @staticmethod
+    def _is_valid_api_key(api_key):
+        """Check whether the supplied API key is one of the ones allowed in config."""
+
+        # HMAC Compare Digest is a recommended thing for security reasons. If you just compare strings
+        # then the comparison returns at the first non-matching character, which means in theory you can
+        # use the time it takes to compare strings to figure out how much of the string you've got right.
+        # With the digest approach it's not the real strings being compared but generated digests, so
+        # it will take a constant amount of time regardless of how well the actual strings match.
+        return any(hmac.compare_digest(api_key.encode(), k.encode()) for k in API_KEYS)
+
     @staticmethod
     def _verify_recaptcha(token):
         """Verify a Google reCAPTCHA v2 token. Returns True if valid."""