w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-09-29 02:32:05 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Multi select upstream providers to send spots to #95

Browse Source
This commit is contained in:
Ian Renton
2026-09-27 14:37:48 +01:00
parent 0cfe92db01
commit e6f0f737d0
13 changed files with 178 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
static/apidocs/openapi.yml
+19 -17
View File
@@ -23,6 +23,7 @@ info:
* **Breaking change:** POST `/spot` now expects `activities` (a list) and `activity_refs` in the `spot` object, rather than `sig` and `sig_refs`.
* **Breaking change:** In the `/options` response, `sigs` has been renamed to `activities`, and within each activity, `sig_type` has been renamed to `activity_type`.
* **Breaking change:** In the `/status` response, `sig_ref_data_providers` has been renamed to `activity_ref_data_providers`, and within each provider, `sig_name` has been renamed to `activity_name`.
* **Breaking change:** In the POST `/spot` `handling` object, `submit_upstream` and `upstream_provider` have been replaced by `upstream_providers`, a list of provider names, so a spot can be sent to multiple upstream providers at once. `upstream_credentials` is now a map of provider name to that provider's credentials.
#### Upgrading a client from v2 to v3 API endpoints
@@ -940,6 +941,7 @@ components:
- BIWOTA
- COTA
- PGA
- Railways
- Toilets
example: POTA
@@ -1381,28 +1383,28 @@ components:
description: >
Optional server-side instructions for how to process this spot submission.
properties:
submit_upstream:
type: boolean
description: >
If true, forward the spot to an external upstream provider (e.g. POTA, SOTA) rather than only adding it
to this Spothole server. Requires `upstream_provider` to be set. Check `spot_submit_providers` in the
`/options` response to see which activities and providers support this.
default: false
upstream_provider:
upstream_providers:
type: array
items:
type: string
description: >
Name of the upstream provider to submit the spot to, e.g. "POTA" or "SOTA". Must
match one of the provider names returned in `spot_submit_providers` for the chosen activity.
example: POTA
Names of upstream providers to forward the spot to (e.g. "POTA", "SOTA"), in addition to or instead
of only adding it to this Spothole server. Each name must match one of the provider names returned in
`spot_submit_providers` in the `/options` response for the chosen activity. Omit or leave empty to
add the spot to this Spothole server only.
example: [ POTA, ParksNPeaks ]
upstream_credentials:
type: object
description: >
Provider-specific credentials required to authenticate the upstream submission.
The required keys depend on the provider. Credentials are used only for the upstream
call and are never stored by Spothole.
A map of provider name to the provider-specific credentials required to authenticate the upstream
submission to that provider. The required keys depend on the provider. Credentials are used only for
the upstream call and are never stored by Spothole.
additionalProperties:
type: object
additionalProperties:
type: string
example:
ParksNPeaks:
user_id: "12345"
api_key: "abc123"
captcha_token:
@@ -2294,9 +2296,9 @@ components:
type: object
description: >
A map of activity name to a list of provider names that support upstream spot submission for that
activity. If an activity appears as a key here, the POST /spot endpoint accepts `submit_upstream: true`
for spots with that activity, and will forward the spot to one of the listed providers. Omitted if no
providers support upstream submission.
activity. If an activity appears as a key here, the POST /spot endpoint accepts these provider names in
`upstream_providers` for spots with that activity, and will forward the spot to any of the listed
providers. Omitted if no providers support upstream submission.
additionalProperties:
type: array
items:
static/js/add-spot.js
+50 -42
View File
@@ -63,46 +63,46 @@ function updateUpstreamArea() {
$("#upstream-area").show();
// Update the provider selector
$("#upstream-provider-select").empty();
// Rebuild the provider checkboxes, keeping any that were already checked for the previous activity
const previouslyChecked = getSelectedUpstreamProviders();
const container = $("#upstream-providers").empty();
$.each(providers, function (i, name) {
$("#upstream-provider-select").append($('<option>', {value: name, text: name}));
const id = "submit-upstream-" + i;
const row = $("<div class='row g-2 align-items-center mb-1'></div>");
const check = $("<div class='form-check'></div>");
check.append($("<input class='form-check-input upstream-provider-check' type='checkbox'>")
.attr("id", id).val(name).prop("checked", previouslyChecked.includes(name)));
check.append($("<label class='form-check-label'></label>").attr("for", id).text(name));
row.append($("<div class='col-auto'></div>").append(check));
// Add a credentials button if this provider has an authentication mechanism and we need input from the user
if (PROVIDER_CREDENTIAL_SCHEMAS[name]) {
const button = $("<button type='button' class='btn btn-sm btn-outline-secondary'></button>")
.attr("aria-label", "Credentials for " + name)
.html("<i class='fa-solid fa-key'></i> Credentials")
.click(() => showCredentialsModal(name));
row.append($("<div class='col-auto'></div>").append(button));
}
container.append(row);
});
if (providers.length > 1) {
$("#upstream-provider-label").text("upstream spot sources:");
$("#upstream-provider-select-col").show();
} else {
$("#upstream-provider-label").text(providers[0]);
$("#upstream-provider-select-col").hide();
updateSpotholeOnlyCheck();
}
// Show the credentials button if this provider has an authentication mechanism and we need input from the user
updateCredentialsButton();
// The "Spothole only" checkbox is checked when no upstream providers are selected
function updateSpotholeOnlyCheck() {
$("#spothole-only").prop("checked", getSelectedUpstreamProviders().length === 0);
}
// Update the credentials button visibility based on selected provider
function updateCredentialsButton() {
const providerName = getSelectedUpstreamProvider();
if (providerName && PROVIDER_CREDENTIAL_SCHEMAS[providerName]) {
$("#upstream-credentials-btn").show();
} else {
$("#upstream-credentials-btn").hide();
}
// Get the names of the upstream providers the user has chosen to send the spot to
function getSelectedUpstreamProviders() {
return $(".upstream-provider-check:checked").map(function () {
return $(this).val();
}).get();
}
// Get the currently selected upstream provider name
function getSelectedUpstreamProvider() {
const providers = SPOT_SUBMIT_PROVIDERS[$("#activity").val()] || [];
if (providers.length === 0) return null;
if (providers.length === 1) return providers[0];
return $("#upstream-provider-select").val();
}
// Show the credentials modal for the currently selected upstream provider
function showCredentialsModal() {
const providerName = getSelectedUpstreamProvider();
if (!providerName || !PROVIDER_CREDENTIAL_SCHEMAS[providerName]) return;
// Show the credentials modal for the given upstream provider
function showCredentialsModal(providerName) {
if (!PROVIDER_CREDENTIAL_SCHEMAS[providerName]) return;
const schema = PROVIDER_CREDENTIAL_SCHEMAS[providerName];
const stored = loadCredentials(providerName);
@@ -187,12 +187,13 @@ function addSpot() {
}
// Upstream submission
const submitUpstream = $("#submit-upstream").is(":checked");
const upstreamProviderName = getSelectedUpstreamProvider();
if (submitUpstream && upstreamProviderName) {
handling["submit_upstream"] = true;
handling["upstream_provider"] = upstreamProviderName;
handling["upstream_credentials"] = loadCredentials(upstreamProviderName);
const upstreamProviders = getSelectedUpstreamProviders();
if (upstreamProviders.length > 0) {
handling["upstream_providers"] = upstreamProviders;
handling["upstream_credentials"] = {};
$.each(upstreamProviders, function (i, name) {
handling["upstream_credentials"][name] = loadCredentials(name);
});
}
$.ajax("/api/v3/spot", {
@@ -262,14 +263,21 @@ $("#mode").change(function () {
$(this).val($(this).val().trim().toUpperCase());
});
// Update upstream area and credentials button when activity changes
// Update upstream area when activity changes
$("#activity").change(function () {
updateUpstreamArea();
});
// Update credentials button when provider selector changes
$("#upstream-provider-select").change(function () {
updateCredentialsButton();
// Keep the "Spothole only" checkbox in sync with the upstream provider checkboxes
$("#upstream-providers").on("change", ".upstream-provider-check", function () {
updateSpotholeOnlyCheck();
});
// Clicking "Spothole only" deselects all upstream providers. It can't be unchecked directly, as that would leave
// nowhere to send the spot; the user unchecks it implicitly by choosing an upstream provider instead.
$("#spothole-only").change(function () {
$(".upstream-provider-check").prop("checked", false);
updateSpotholeOnlyCheck();
});
// Startup
templates/add-spot.html
+11 -23
View File
@@ -71,6 +71,16 @@
<input type="text" class="form-control storeable-text input-narrow" id="de-call"
placeholder="N0CALL" required>
</div>
<div id="upstream-area" class="mt-3" style="display:none;">
<div class="form-label">Spot to:</div>
<div class="form-check mb-1">
<input class="form-check-input" type="checkbox" id="spothole-only" checked>
<label class="form-check-label" for="spothole-only">Spothole only</label>
</div>
<div id="upstream-providers"></div>
</div>
<div class="col-auto">
<button type="submit" id="add-spot-submit" class="btn btn-primary mt-2em">
<span id="add-spot-spinner" class="spinner-border spinner-border-sm me-1" aria-hidden="true"
@@ -80,28 +90,6 @@
</div>
</form>
<div id="upstream-area" class="mt-3" style="display:none;">
<div class="row g-2 align-items-center">
<div class="col-auto">
<div class="form-check">
<input class="form-check-input" type="checkbox" id="submit-upstream">
<label class="form-check-label" for="submit-upstream">
Send spot to <span id="upstream-provider-label"></span>
</label>
</div>
</div>
<div class="col-auto" id="upstream-provider-select-col" style="display:none;">
<select id="upstream-provider-select" class="form-select form-select-sm"></select>
</div>
<div class="col-auto">
<button type="button" class="btn btn-sm btn-outline-secondary" id="upstream-credentials-btn"
onclick="showCredentialsModal();" style="display:none;">
<i class="fa-solid fa-key"></i> Credentials
</button>
</div>
</div>
</div>
<div id="recaptcha-area" class="mt-3" style="display:none;">
<div id="recaptcha-widget"></div>
</div>
@@ -142,7 +130,7 @@
const ALLOW_UPSTREAM_SPOTTING = {% raw safe_json_dumps(web_ui_options["allow_upstream_spotting"]) %};
</script>
<script src="/static/js/add-spot.js?v=1790514703"></script>
<script src="/static/js/add-spot.js?v=1790516268"></script>
<script>$(document).ready(function () {
$("#nav-link-add-spot").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/alerts.html
+1 -1
View File
@@ -87,7 +87,7 @@
</div>
<script src="/static/js/alerts.js?v=1790514703"></script>
<script src="/static/js/alerts.js?v=1790516268"></script>
<script>$(document).ready(function () {
$("#nav-link-alerts").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/bands.html
+2 -2
View File
@@ -82,8 +82,8 @@
const BANDS = {% raw safe_json_dumps(options["bands"]) %};
</script>
<script src="/static/js/spotsbandsandmap.js?v=1790514703"></script>
<script src="/static/js/bands.js?v=1790514703"></script>
<script src="/static/js/spotsbandsandmap.js?v=1790516268"></script>
<script src="/static/js/bands.js?v=1790516268"></script>
<script>$(document).ready(function () {
$("#nav-link-bands").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/base.html
+5 -5
View File
@@ -1,6 +1,6 @@
{% extends "skeleton.html" %}
{% block head_extra %}
<link rel="stylesheet" href="/static/css/style.css?v=1790514702" type="text/css">
<link rel="stylesheet" href="/static/css/style.css?v=1790516268" type="text/css">
<link href="/static/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
<link href="/static/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
<link href="/static/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -16,10 +16,10 @@
window.fetchEventSource = fetchEventSource;
</script>
<script src="/static/js/utils.js?v=1790514702"></script>
<script src="/static/js/ui-ham.js?v=1790514702"></script>
<script src="/static/js/geo.js?v=1790514702"></script>
<script src="/static/js/common.js?v=1790514702"></script>
<script src="/static/js/utils.js?v=1790516268"></script>
<script src="/static/js/ui-ham.js?v=1790516268"></script>
<script src="/static/js/geo.js?v=1790516268"></script>
<script src="/static/js/common.js?v=1790516268"></script>
{% end %}
{% block body %}
<div class="container">
templates/conditions.html
+1 -1
View File
@@ -284,7 +284,7 @@
</div>
<script src="/static/vendor/js/chart-4.4.9.umd.min.js"></script>
<script src="/static/js/conditions.js?v=1790514702"></script>
<script src="/static/js/conditions.js?v=1790516268"></script>
<script>$(document).ready(function () {
$("#nav-link-conditions").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/help/thanks.html
+6 -5
View File
@@ -9,11 +9,12 @@
modelling software, and other online tools on which Spothole's data is based. The vast majority of these are not
profit-seeking and are made purely for the love of the hobby and to help others in the community. Spothole is
standing on the shoulders of giants, who deserve a huge amount of thanks for all the work they put in.</p>
<p>The following people have specifically helped develop Spothole either by contributing code, requesting features,
or helping squash bugs: Steven M1SDH, Dan S50U, Matt W1CDN, Norby LX1NO, Andrew VK3ARR, Mario DL4MFM, Mark M5TEA,
Michael G7VJR, Rob G7LAS, Ed DD5LP, Raph F4LUB, Luc ON7KEC, Richard GD4OFB, Daniel PY2TDB, Alan VK1AO,
Andrew WK1AD, Ullrich DF5WC, Wayne N3CDF, Jonathan G4IVV, Erik N2EPE, Sorin YO9TSN, Matt HB9HWI, Priit ES1TEB,
Leigh KG7WED, Jouni OH3CUF, Onno VK6FLAB, and Bruce WA7BNM. (If I've forgotten you, let me know!)</p>
<p>The following people have specifically helped develop Spothole either by contributing code, allowing access to
their data, providing guidance, requesting features, or helping squash bugs: Steven M1SDH, Dan S50U, Matt W1CDN,
Norby LX1NO, Andrew VK3ARR, Mario DL4MFM, Mark M5TEA, Michael G7VJR, Rob G7LAS, Ed DD5LP, Raph F4LUB,
Luc ON7KEC, Richard GD4OFB, Daniel PY2TDB, Alan VK1AO, Andrew WK1AD, Ullrich DF5WC, Wayne N3CDF, Jonathan G4IVV,
Erik N2EPE, Sorin YO9TSN, Matt HB9HWI, Priit ES1TEB, Leigh KG7WED, Jouni OH3CUF, Onno VK6FLAB, Bruce WA7BNM, and
Larry F5PYI. (If I've forgotten you, let me know!)</p>
<p>Spothole is also dependent on a number of Python libraries, such as pyhamtools, and many JavaScript
libraries, as well as the Font Awesome icon set and flag icons from the Noto Color Emoji set, and MIT-licenced
GeoJSON files for CQ and ITU zones from HA8TKS.</p>
templates/map.html
+2 -2
View File
@@ -115,8 +115,8 @@
const CARTODB_API_KEY = "{{ web_ui_options.get('cartodb_api_key', '') }}";
</script>
<script src="/static/js/spotsbandsandmap.js?v=1790514702"></script>
<script src="/static/js/map.js?v=1790514702"></script>
<script src="/static/js/spotsbandsandmap.js?v=1790516267"></script>
<script src="/static/js/map.js?v=1790516267"></script>
<script>$(document).ready(function () {
$("#nav-link-map").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/spots.html
+2 -2
View File
@@ -127,8 +127,8 @@
</div>
<script src="/static/js/spotsbandsandmap.js?v=1790514702"></script>
<script src="/static/js/spots.js?v=1790514702"></script>
<script src="/static/js/spotsbandsandmap.js?v=1790516267"></script>
<script src="/static/js/spots.js?v=1790516267"></script>
<script>$(document).ready(function () {
$("#nav-link-spots").addClass("active");
}); <!-- highlight active page in nav --></script>
templates/status.html
+1 -1
View File
@@ -96,7 +96,7 @@
</div>
</div>
<script src="/static/js/status.js?v=1790514703"></script>
<script src="/static/js/status.js?v=1790516268"></script>
<script>
$(document).ready(function () {
$("#nav-link-status").addClass("active");
webserver/handlers/api/addspot.py
+55 -41
View File
@@ -1,3 +1,4 @@
import asyncio
import logging
import re
import threading
@@ -78,9 +79,9 @@ class APISpotHandler(tornado.web.RequestHandler):
handling = json_body.get("handling", {})
# Extract individual parameters that say how this spot should be handled by the server
submit_upstream = handling.get("submit_upstream", False)
upstream_provider_name = handling.get("upstream_provider", None)
upstream_credentials = handling.get("upstream_credentials", {})
upstream_provider_names = handling.get("upstream_providers", None) or []
upstream_credentials = handling.get("upstream_credentials", None) or {}
submit_upstream = len(upstream_provider_names) > 0
captcha_token = handling.get("captcha_token", None)
# Verify CAPTCHA if required
@@ -166,13 +167,18 @@ class APISpotHandler(tornado.web.RequestHandler):
self.set_header("Content-Type", "application/json")
return
# Validate upstream submission requirements
if submit_upstream and upstream_provider_name:
is_cluster = isinstance(self._find_provider(upstream_provider_name, spot.activities), DXCluster)
is_tiles = isinstance(self._find_provider(upstream_provider_name, spot.activities), Tiles)
# Validate upstream submission requirements for each requested provider
for upstream_provider_name in upstream_provider_names:
provider = self._find_provider(upstream_provider_name, spot.activities)
is_cluster = isinstance(provider, DXCluster)
is_tiles = isinstance(provider, Tiles)
if not spot.activity_refs and not is_tiles and not is_cluster:
self.set_status(422)
self.write(safe_json_dumps("Error - an activity reference is required to submit upstream."))
self.write(
safe_json_dumps(
f"Error - an activity reference is required to submit upstream to {upstream_provider_name}."
)
)
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -191,46 +197,27 @@ class APISpotHandler(tornado.web.RequestHandler):
self.set_header("Content-Type", "application/json")
return
# Submit upstream if requested
upstream_warning = None
if submit_upstream and upstream_provider_name:
provider = self._find_provider(upstream_provider_name, spot.activities)
if provider:
try:
# Submit spot to the upstream provider. Run in a separate thread otherwise this blocks the whole
# web server for everyone!
await IOLoop.current().run_in_executor(None, provider.submit_spot, spot, upstream_credentials)
# Trigger a re-poll after 3 second so the spot appears quickly. (Submitting to a cluster node is
# slower than this, but we get data as a live stream from cluster anyway, so force_poll does
# nothing in that case. This is really just for the HTTP providers when we submit a spot to them)
threading.Timer(3.0, provider.force_poll).start()
except NotImplementedError as e:
upstream_warning = str(e)
except SpotSubmissionError as e:
logger.warning(f"Upstream submission to {upstream_provider_name} was not accepted: {e}")
upstream_warning = (
f"Spot was saved locally but upstream submission to {upstream_provider_name} failed: {e}"
# Submit upstream to all requested providers in parallel, collecting any warnings
results = await asyncio.gather(
*(
self._submit_upstream(name, spot, upstream_credentials.get(name, {}))
for name in upstream_provider_names
)
except Exception:
logger.exception(f"Failed to submit spot upstream to {upstream_provider_name}")
upstream_warning = (
f"Spot was saved locally but upstream submission to {upstream_provider_name} failed."
)
elif spot.activities:
upstream_warning = f"No enabled provider named '{upstream_provider_name}' supports upstream submission for {', '.join(spot.activities)} spots."
else:
upstream_warning = f"No enabled provider named '{upstream_provider_name}' supports upstream submission for spots with no activity."
upstream_warnings = [w for w in results if w]
any_upstream_succeeded = len(upstream_warnings) < len(upstream_provider_names)
# If we successfully submitted the spot upstream, don't add it direct to Spothole, otherwise it will be a
# duplicate with what immediately comes back from the API. But if we weren't asked to send it upstream, or
# we were but it failed, we should still add it to our database anyway.
if not submit_upstream or upstream_warning:
# If we successfully submitted the spot to at least one upstream provider, don't add it direct to Spothole,
# otherwise it will be a duplicate with what immediately comes back from the API. But if we weren't asked to
# send it upstream, or we were but every submission failed, we should still add it to our database anyway.
if not any_upstream_succeeded:
spot.source = "API"
spot.infer_missing()
self._spots.set(spot.id, spot)
if upstream_warning:
self.write(safe_json_dumps(f"Warning - {upstream_warning}"))
if upstream_warnings:
saved_locally_note = "" if any_upstream_succeeded else " The spot was saved to Spothole only."
self.write(safe_json_dumps(f"Warning - {' '.join(upstream_warnings)}{saved_locally_note}"))
self.set_status(201)
else:
self.write(safe_json_dumps("OK"))
@@ -245,6 +232,33 @@ class APISpotHandler(tornado.web.RequestHandler):
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
async def _submit_upstream(self, upstream_provider_name, spot, credentials) -> str | None:
"""Submit a spot to the named upstream provider. Returns None on success, or a warning message on failure."""
provider = self._find_provider(upstream_provider_name, spot.activities)
if not provider:
if spot.activities:
return f"No enabled provider named '{upstream_provider_name}' supports upstream submission for {', '.join(spot.activities)} spots."
return f"No enabled provider named '{upstream_provider_name}' supports upstream submission for spots with no activity."
try:
# Submit spot to the upstream provider. Run in a separate thread otherwise this blocks the whole web server
# for everyone!
await IOLoop.current().run_in_executor(None, provider.submit_spot, spot, credentials)
# Trigger a re-poll after 3 second so the spot appears quickly. (Submitting to a cluster node is slower than
# this, but we get data as a live stream from cluster anyway, so force_poll does nothing in that case. This
# is really just for the HTTP providers when we submit a spot to them)
threading.Timer(3.0, provider.force_poll).start()
return None
except NotImplementedError as e:
return str(e)
except SpotSubmissionError as e:
logger.warning(f"Upstream submission to {upstream_provider_name} was not accepted: {e}")
return f"Upstream submission to {upstream_provider_name} failed: {e}"
except Exception:
logger.exception(f"Failed to submit spot upstream to {upstream_provider_name}")
return f"Upstream submission to {upstream_provider_name} failed."
def _find_provider(self, provider_name, activities) -> SpotProvider | None:
"""Find an enabled provider by name that can submit spots for at least one of the given activities. If there
are no activities, find one that can submit spots with no activity."""
webserver/handlers/api/compatibility/v2_compatibility.py
+17
View File
@@ -116,6 +116,8 @@ class V2APISpotHandler(V2CompatibilityWrapper, RequestCompatibilityWrapper, APIS
def translate(body):
if isinstance(body, dict) and isinstance(body.get("spot"), dict):
body["spot"] = self._translate_v2_spot(body["spot"])
if isinstance(body, dict) and isinstance(body.get("handling"), dict):
body["handling"] = self._translate_v2_handling(body["handling"])
return body
translate_json_body(self.request, translate)
@@ -141,6 +143,21 @@ class V2APISpotHandler(V2CompatibilityWrapper, RequestCompatibilityWrapper, APIS
spot_data["activity_refs"] = refs
return spot_data
@staticmethod
def _translate_v2_handling(handling):
"""Translate the "handling" object provided by a client calling the add spot method in v2 format into v3
format. v2 had a "submit_upstream" flag with a single "upstream_provider" and its credentials, whereas v3 has
a list of "upstream_providers" and a map of provider name to credentials."""
handling = dict(handling)
submit_upstream = handling.pop("submit_upstream", False)
provider_name = handling.pop("upstream_provider", None)
credentials = handling.pop("upstream_credentials", None) or {}
if submit_upstream and provider_name:
handling["upstream_providers"] = [provider_name]
handling["upstream_credentials"] = {provider_name: credentials}
return handling
def collapse_activities(obj):
"""Utility method to replace the "activities" list in a spot or alert JSON object with a single "activity" value,