Hellmerge #95 branch into 2.0-pre. #119

This commit is contained in:
Ian Renton
2026-08-08 09:52:14 +01:00
parent 4e7c5a05aa
commit ac35a920cf
13 changed files with 109 additions and 93 deletions
+25 -25
View File
@@ -330,32 +330,32 @@ class Spot:
self.propagation_mode = mode_tag
logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag)
# Parse "de_grid -> dx_grid" structures from the comment
if self.comment:
grid_mode_grid_match = re.search(
r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
self.comment)
if grid_mode_grid_match:
# regex matches, so extract grids:
if not self.dx_grid:
self.dx_grid = grid_mode_grid_match.group(1).upper()
self.dx_location_source = "SPOT"
if not self.de_grid:
self.de_grid = grid_mode_grid_match.group(2).upper()
# Parse "de_grid -> dx_grid" structures from the comment
if self.comment:
grid_mode_grid_match = re.search(
r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
self.comment)
if grid_mode_grid_match:
# regex matches, so extract grids:
if not self.dx_grid:
self.dx_grid = grid_mode_grid_match.group(1).upper()
self.dx_location_source = "SPOT"
if not self.de_grid:
self.de_grid = grid_mode_grid_match.group(2).upper()
# DX Grid to lat/lon and vice versa in case one is missing
if self.dx_grid and not self.dx_latitude:
try:
ll = locator_to_latlong(self.dx_grid)
self.dx_latitude = ll[0]
self.dx_longitude = ll[1]
except:
logging.debug("Invalid grid received for spot")
if self.dx_latitude and self.dx_longitude and not self.dx_grid:
try:
self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8)
except:
logging.debug("Invalid lat/lon received for spot")
# DX Grid to lat/lon and vice versa in case one is missing
if self.dx_grid and not self.dx_latitude:
try:
ll = locator_to_latlong(self.dx_grid)
self.dx_latitude = ll[0]
self.dx_longitude = ll[1]
except:
logging.debug("Invalid grid received for spot")
if self.dx_latitude and self.dx_longitude and not self.dx_grid:
try:
self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8)
except:
logging.debug("Invalid lat/lon received for spot")
# QRT comment detection
if self.comment and not self.qrt:
+5 -5
View File
@@ -12,7 +12,7 @@ Various approaches exist to writing your own client, but in general:
* Refer to the API docs. These are built on an OpenAPI definition file (`/static/apidocs/openapi.yml`), which you can
automatically use to generate a client skeleton using various software.
* Call the main "spots" or "alerts" API endpoints to get the data you want. For example, your app could call
`https://spothole.app/api/v1/spots` once every few minutes. Apply filters if necessary.
`https://spothole.app/api/v2/spots` once every few minutes. Apply filters if necessary.
* Call the "options" API to get an idea of which bands, modes etc. the server knows about. You might want to do that
first before calling the spots/alerts APIs, to allow you to populate your filters correctly.
* Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "alerts"/"upcoming"
@@ -25,15 +25,15 @@ once every two minutes, so if your client is interested in POTA data there's no
than that.
If you absolutely must be informed within seconds of a spot arriving in Spothole, please use the SSE endpoints instead,
e.g. `https://spothole.app/api/v1/spots/stream`.
e.g. `https://spothole.app/api/v2/spots/stream`.
If you want to handle different types of spot or alert differently within your client, please consider making a single
request to the Spothole API to retrieve all the data, then filtering on your side. For example, call
`https://spothole.app/api/v1/spots?sig=POTA,SOTA` rather than making two separate calls to
`https://spothole.app/api/v1/spots?sig=POTA` and `https://spothole.app/api/v1/spots?sig=SOTA`.
`https://spothole.app/api/v2/spots?sig=POTA,SOTA` rather than making two separate calls to
`https://spothole.app/api/v2/spots?sig=POTA` and `https://spothole.app/api/v2/spots?sig=SOTA`.
Remember, here at Spothole Inc. we offer an industry-standard "five nines" uptime on our server, with our own unique
twist: we don't tell you which side of the decimal point the nines start! (Translation: This is a hobby project.
`spothole.app` runs on the same server as my blog and other stuff. It might go down without warning. By all means base
your own project on data from the main server if you like, but if you want any control over reliability and downtime,
please run your own copy instead.)
please run your own copy instead.)
+2 -2
View File
@@ -85,7 +85,7 @@ server {
}
# SSE endpoints
location ~ ^/api/v1/(spots|alerts)/stream/? {
location ~ ^/api/v2/(spots|alerts)/stream/? {
proxy_pass http://spothole:8080;
# Remove buffering, remove caching, add suitable timeouts for SSE API calls
@@ -147,4 +147,4 @@ server {
If desired, you could even change the port on which Spothole runs from 8080 to a plain 80, in which case your
`proxy_pass` statements could drop the `:8080` suffix. Since Spothole is in a container, it can serve HTTP on port 80
if desired, because it doesn't conflict with the host system.
if desired, because it doesn't conflict with the host system.
+18 -11
View File
@@ -9,12 +9,17 @@ To navigate your way around the source code, this list may help.
*Python back-end code*
* `/core` - Core classes and scripts
* `/core` - Core classes and utilities
* `/data` - Data storage classes
* `/spotproviders` - Classes providing spots by accessing the APIs of other services
* `/alertproviders` - Classes providing alerts by accessing the APIs of other services
* `/solarconditionsproviders` - Classes providing solar and propagation by accessing the APIs of other services
* `/providers/spot` - Classes providing spots by accessing the APIs of other services
* `/providers/alert` - Classes providing alerts by accessing the APIs of other services
* `/providers/solarconditions` - Classes providing solar and propagation by accessing the APIs of other services
* `/providers/staticdata` - Classes providing static lookup data by accessing bundled data files or the APIs of other
services
* `/providers/sigrefdata` - Classes providing SIG reference lookup data by accessing bundled data files or the APIs of
other services
* `/server` - Classes for running Spothole's own web server
* `spothole.py` - Main application script
*Templates*
@@ -24,6 +29,7 @@ To navigate your way around the source code, this list may help.
* `/static` - Root for static files served by the web server. These are all served from a path starting `/static/`.
* `/static/apidocs` - Contains the OpenAPI spec (`openapi.yml`)
* `/static/audio` - Audio files used by the web front-end
* `/static/css` - CSS files used by the web front-end
* `/static/img` - image files used by the web front-end
* `/static/js` - JavaScript used by the web front-end
@@ -31,18 +37,18 @@ To navigate your way around the source code, this list may help.
*Miscellaneous*
* `/` - Main script (`spothole.py`), pip `requirements.txt`, config, README, etc.
* `/` - pip `requirements.txt`, config, README, etc.
* `/docs` - Documentation
* `/images` - Image sources
* `/datafiles` - Local data sources (differentiated from the majority of data files which are loaded from URLs and
cached in `/cache`)
* `/cache` - Directory where static-ish data downloaded from the internet is cached to avoid rapid re-requests, and
where spot/alert data is cached so that it survives a software restart. Created on first run.
* `/datafiles` - Local data files, used by some providers when the data will never change and/or is not easily available
online in a format Spothole can handle
* `/cache` - Directory where Spothole stores all the data it uses that should be persisted to disk. Created on first
run.
### Extending the server
Spothole is designed to be easily extensible. If you want to write your own spot provider, for example, simply add a
module to the `spotproviders` package containing your class. (Currently, in order to be loaded correctly, the module (
module to the `providers.spot` package containing your class. (Currently, in order to be loaded correctly, the module (
file) name should be the same as the class name, but lower case.)
Your class should extend "SpotProvider"; if it operates by polling an HTTP Server on a timer, it can instead extend "
@@ -66,4 +72,5 @@ parameters are optional, but you will at least want to provide a `time` (which m
Finally, simply add the appropriate config to the `spot_providers` section of `config.yml`, and your provider should be
instantiated on startup.
The same approach as above is also used for alert providers.
The same approach as above is also used for alerts, and other types of providers. Give me a shout if you need any
advice.
+2 -2
View File
@@ -48,7 +48,7 @@ server {
}
# SSE endpoints
location ~ ^/api/v1/(spots|alerts)/stream/? {
location ~ ^/api/v2/(spots|alerts)/stream/? {
proxy_pass http://127.0.0.1:8080;
# Remove buffering, remove caching, add suitable timeouts for SSE API calls
@@ -120,4 +120,4 @@ You should now be able to access the web interface by going to the domain from y
Once that's working, [install certbot](https://certbot.eff.org/instructions?ws=nginx&os=snap) onto your server. Run it
as root, and when prompted pick your domain name from the list. After a few seconds, it should successfully provision a
certificate and modify your nginx config files automatically. You should then be able to access the site via HTTPS.
certificate and modify your nginx config files automatically. You should then be able to access the site via HTTPS.
+3 -2
View File
@@ -47,7 +47,8 @@ python3 spothole.py
```
The software can take a few seconds to start up, mostly because it is downloading an updated file to match callsigns to
countries. This is normal, don't panic!
countries. This is normal, don't panic! Once you see `You can access your copy of Spothole at
http://localhost:8080` in the log, your server is good to go.
If you see some errors on startup, check your configuration, e.g. in case you have specified a port for the web server
that is already in use by something else.
that is already in use by something else.
+26 -21
View File
@@ -12,12 +12,12 @@ from tornado.web import Application
from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRET_KEY
from core.constants import UNKNOWN_BAND
from core.utils import infer_band_from_freq
from core.prometheus_metrics_handler import api_requests_counter
from core.sig_utils import get_ref_regex_for_sig
from core.utils import infer_band_from_freq
from core.utils import safe_json_dumps
from data.spot import Spot
from spotproviders.spot_provider import SpotProvider
from providers.spot.spot_provider import SpotProvider
RECAPTCHA_VERIFY_URL = "https://www.google.com/recaptcha/api/siteverify"
@@ -82,19 +82,24 @@ class APISpotHandler(tornado.web.RequestHandler):
upstream_credentials = handling.get("upstream_credentials", {})
captcha_token = handling.get("captcha_token", None)
# Spothole v2.0 release only: deny upstream spotting. Spothole API breaking changes were in v2.0 but
# functionality is not ready yet. TODO
submit_upstream = False
# Verify CAPTCHA if required
if RECAPTCHA_SECRET_KEY:
if not captcha_token:
self.set_status(422)
self.write(json.dumps("Error - CAPTCHA token is required for spot submission.",
default=serialize_everything))
self.write(safe_json_dumps("Error - CAPTCHA token is required for spot submission."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
if not self._verify_recaptcha(captcha_token):
self.set_status(422)
self.write(json.dumps("Error - CAPTCHA verification failed.",
default=serialize_everything))
self.write(safe_json_dumps("Error - CAPTCHA verification failed."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -105,7 +110,8 @@ class APISpotHandler(tornado.web.RequestHandler):
# Reject if no timestamp, frequency, dx_call or de_call
if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
self.set_status(422)
self.write(safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum."))
self.write(
safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -127,7 +133,8 @@ class APISpotHandler(tornado.web.RequestHandler):
# Reject if frequency not in a known band
if infer_band_from_freq(spot.freq) == UNKNOWN_BAND:
self.set_status(422)
self.write(safe_json_dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band."))
self.write(
safe_json_dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -137,7 +144,8 @@ class APISpotHandler(tornado.web.RequestHandler):
r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$",
spot.dx_grid.upper()):
self.set_status(422)
self.write(safe_json_dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid."))
self.write(
safe_json_dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -155,8 +163,7 @@ class APISpotHandler(tornado.web.RequestHandler):
# Reject upstream submission if not permitted
if submit_upstream and not ALLOW_UPSTREAM_SPOTTING:
self.set_status(403)
self.write(json.dumps("Error - this server does not allow upstream spot submission.",
default=serialize_everything))
self.write(safe_json_dumps("Error - this server does not allow upstream spot submission."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -165,29 +172,26 @@ class APISpotHandler(tornado.web.RequestHandler):
if submit_upstream and upstream_provider_name:
if not spot.sig:
self.set_status(422)
self.write(json.dumps("Error - a SIG must be selected to submit upstream.",
default=serialize_everything))
self.write(safe_json_dumps("Error - a SIG must be selected to submit upstream."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
if not spot.sig_refs and upstream_provider_name != "Tiles":
self.set_status(422)
self.write(json.dumps("Error - a SIG reference is required to submit upstream.",
default=serialize_everything))
self.write(safe_json_dumps("Error - a SIG reference is required to submit upstream."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
if not spot.dx_grid and upstream_provider_name == "Tiles":
self.set_status(422)
self.write(json.dumps("Error - a grid reference is required to submit upstream to Tiles on the Air.",
default=serialize_everything))
self.write(
safe_json_dumps("Error - a grid reference is required to submit upstream to Tiles on the Air."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
if not spot.mode and upstream_provider_name == "Tiles":
self.set_status(422)
self.write(json.dumps("Error - a mode is required to submit upstream to Tiles on the Air.",
default=serialize_everything))
self.write(safe_json_dumps("Error - a mode is required to submit upstream to Tiles on the Air."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
return
@@ -209,7 +213,8 @@ class APISpotHandler(tornado.web.RequestHandler):
upstream_warning = "Spot was saved locally but upstream submission to " + upstream_provider_name + " failed: " + str(
e)
else:
upstream_warning = "No enabled provider named '" + upstream_provider_name + "' supports upstream submission for " + spot.sig + " spots."
upstream_warning = "No enabled provider named '" + upstream_provider_name + "' supports upstream submission for " + (
spot.sig if spot.sig else "") + " spots."
# If we successfully submitted the spot upstream, don't add it direct to Spothole, otherwise it will be a
# duplicate with what immediately comes back from the API. But if we weren't asked to send it upstream, or
@@ -219,7 +224,7 @@ class APISpotHandler(tornado.web.RequestHandler):
self._spots.set(spot.id, spot)
if upstream_warning:
self.write(json.dumps("Warning - " + upstream_warning, default=serialize_everything))
self.write(safe_json_dumps("Warning - " + upstream_warning))
self.set_status(201)
else:
self.write(safe_json_dumps("OK"))
+8 -6
View File
@@ -37,12 +37,14 @@ class APIOptionsHandler(tornado.web.RequestHandler):
# Build a map of SIG name -> list of provider names that can submit spots for that SIG
spot_submit_providers = {}
for provider in self._spot_providers:
if not provider.enabled:
continue
for sig in SIGS:
if provider.can_submit_spot(sig.name):
spot_submit_providers.setdefault(sig.name, []).append(provider.name)
# Spothole v2.0 - disable this for now, API changes are in but this functionality is not ready yet. TODO
# for provider in self._spot_providers:
# if not provider.enabled:
# continue
# for sig in SIGS:
# if provider.can_submit_spot(sig.name):
# spot_submit_providers.setdefault(sig.name, []).append(provider.name)
# Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle
# things that aren't even available.
+3 -5
View File
@@ -2,7 +2,7 @@ import json
import tornado
from core.utils import serialize_everything
from core.utils import safe_json_dumps
class V1GoneHandler(tornado.web.RequestHandler):
@@ -11,10 +11,8 @@ class V1GoneHandler(tornado.web.RequestHandler):
def post(self):
self.set_status(410)
self.write(json.dumps(
"This API endpoint has a breaking change or has been removed in the current version of the Spothole API. Please see /apidocs for details of the current API version and the endpoints available.",
default=serialize_everything
))
self.write(safe_json_dumps(
"This API endpoint has a breaking change or has been removed in the current version of the Spothole API. Please see /apidocs for details of the current API version and the endpoints available."))
self.set_header("Cache-Control", "no-store")
self.set_header("Content-Type", "application/json")
+7 -5
View File
@@ -1,15 +1,14 @@
import asyncio
import logging
import os
import threading
import tornado
from tornado.web import StaticFileHandler
from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL
from core.data_providers import DATA_PROVIDERS
from core.data_store import DATA_STORE
from server.handlers.api.addspot import APISpotHandler
from server.handlers.api.v1_compatability import V1RedirectHandler, V1GoneHandler
from server.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
from server.handlers.api.dxstats import APIDxStatsHandler
from server.handlers.api.lookups import APILookupCallHandler, APILookupSIGRefHandler, APILookupGridHandler
@@ -17,6 +16,7 @@ from server.handlers.api.options import APIOptionsHandler
from server.handlers.api.solar_conditions import APISolarConditionsHandler
from server.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
from server.handlers.api.status import APIStatusHandler
from server.handlers.api.v1_compatability import V1RedirectHandler, V1GoneHandler
from server.handlers.manifesthandler import ManifestHandler
from server.handlers.metrics import PrometheusMetricsHandler
from server.handlers.pagetemplate import PageTemplateHandler
@@ -32,6 +32,7 @@ class WebServer:
"""Constructor"""
self._data_store = DATA_STORE
self._data_providers = DATA_PROVIDERS
self._spot_broadcaster = SSEBroadcaster()
self._alert_broadcaster = SSEBroadcaster()
self._port = WEB_SERVER_PORT
@@ -87,11 +88,12 @@ class WebServer:
(r"/api/v2/lookup/call", APILookupCallHandler, {**handler_opts}),
(r"/api/v2/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
(r"/api/v2/lookup/grid", APILookupGridHandler, {**handler_opts}),
(r"/api/v2/spot", APISpotHandler, {"spots": self._data_store.spots, **handler_opts}),
(r"/api/v2/spot", APISpotHandler,
{"spots": self._data_store.spots, "spot_providers": self._data_providers, **handler_opts}),
]
# v1 API redirects. Most v1 enpoints are unchanged in v2, and get an HTTP 308 redirect to the v2 API. The ones
# that have the actual breaking changes get a bespoke handler.
# that have the major breaking changes get a bespoke handler.
v1_compat_routes = [
(r"/api/v1/spot", V1GoneHandler),
(r"/api/v1/(.*)", V1RedirectHandler),
@@ -162,4 +164,4 @@ def request_log(handler):
# Global object
WEB_SERVER = WebServer()
WEB_SERVER = WebServer()
+7 -6
View File
@@ -21,12 +21,13 @@ function loadSpots() {
evtSource.close();
}
$.ajax({url: '/api/v2/spots' + buildQueryString(), dataType: 'json', headers: getCredentialHeaders(), success: function (jsonData) {
// Store data
spots = jsonData;
// Update bands display
updateBands();
// Start the ongoing SSE connection
startSSEConnection();
// Store data
spots = jsonData;
// Update bands display
updateBands();
// Start the ongoing SSE connection
startSSEConnection();
}
});
}
+2 -2
View File
@@ -55,7 +55,7 @@ function loadStatus() {
<div class="col"><strong>${p["sig_name"]}</strong></div>
<div class="col">Status: ${p["status"]}</div>
<div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
<div class="col">References: ${(p["enabled"] && p["reference_count"] > 0) ? p["reference_count"] : "N/A"}</div>
<div class="col">References: ${p["enabled"] ? p["reference_count"] : "N/A"}</div>
</div>`);
});
@@ -65,7 +65,7 @@ function loadStatus() {
<div class="col"><strong>${p["name"]}</strong></div>
<div class="col">Status: ${p["status"]}</div>
<div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
<div class="col">Lookups: ${(p["enabled"] && p["lookup_count"] > 0) ? p["lookup_count"] : "N/A"}</div>
<div class="col">Lookups: ${p["enabled"] ? p["lookup_count"] : "N/A"}</div>
</div>`);
});
});
+1 -1
View File
@@ -2,7 +2,7 @@
{% block content %}
<div id="add-spot-intro-box" class="permanently-dismissible-box mt-3">
<div class="alert alert-primary alert-dismissible fade show" role="alert">
<div class="alert alert-primary alert-dismissible fade show" role="alert"> <!-- TODO Remove when feature available -->
<i class="fa-solid fa-circle-info"></i> <strong>Adding spots to Spothole</strong><br/>This page is implemented
as a proof of concept for adding spots to the Spothole system. Currently, spots added in this way are only
visible within Spothole and are not sent "upstream" to DX clusters or xOTA spotting sites. The functionality