diff --git a/data/spot.py b/data/spot.py index 482f144..9b12452 100644 --- a/data/spot.py +++ b/data/spot.py @@ -330,32 +330,32 @@ class Spot: self.propagation_mode = mode_tag logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag) - # Parse "de_grid -> dx_grid" structures from the comment - if self.comment: - grid_mode_grid_match = re.search( - r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b', - self.comment) - if grid_mode_grid_match: - # regex matches, so extract grids: - if not self.dx_grid: - self.dx_grid = grid_mode_grid_match.group(1).upper() - self.dx_location_source = "SPOT" - if not self.de_grid: - self.de_grid = grid_mode_grid_match.group(2).upper() + # Parse "de_grid -> dx_grid" structures from the comment + if self.comment: + grid_mode_grid_match = re.search( + r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b', + self.comment) + if grid_mode_grid_match: + # regex matches, so extract grids: + if not self.dx_grid: + self.dx_grid = grid_mode_grid_match.group(1).upper() + self.dx_location_source = "SPOT" + if not self.de_grid: + self.de_grid = grid_mode_grid_match.group(2).upper() - # DX Grid to lat/lon and vice versa in case one is missing - if self.dx_grid and not self.dx_latitude: - try: - ll = locator_to_latlong(self.dx_grid) - self.dx_latitude = ll[0] - self.dx_longitude = ll[1] - except: - logging.debug("Invalid grid received for spot") - if self.dx_latitude and self.dx_longitude and not self.dx_grid: - try: - self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8) - except: - logging.debug("Invalid lat/lon received for spot") + # DX Grid to lat/lon and vice versa in case one is missing + if self.dx_grid and not self.dx_latitude: + try: + ll = locator_to_latlong(self.dx_grid) + self.dx_latitude = ll[0] + self.dx_longitude = ll[1] + except: + logging.debug("Invalid grid received for spot") + if self.dx_latitude and self.dx_longitude and not self.dx_grid: + try: + self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8) + except: + logging.debug("Invalid lat/lon received for spot") # QRT comment detection if self.comment and not self.qrt: diff --git a/docs/clients.md b/docs/clients.md index 95e0c46..d71f29b 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -12,7 +12,7 @@ Various approaches exist to writing your own client, but in general: * Refer to the API docs. These are built on an OpenAPI definition file (`/static/apidocs/openapi.yml`), which you can automatically use to generate a client skeleton using various software. * Call the main "spots" or "alerts" API endpoints to get the data you want. For example, your app could call - `https://spothole.app/api/v1/spots` once every few minutes. Apply filters if necessary. + `https://spothole.app/api/v2/spots` once every few minutes. Apply filters if necessary. * Call the "options" API to get an idea of which bands, modes etc. the server knows about. You might want to do that first before calling the spots/alerts APIs, to allow you to populate your filters correctly. * Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "alerts"/"upcoming" @@ -25,15 +25,15 @@ once every two minutes, so if your client is interested in POTA data there's no than that. If you absolutely must be informed within seconds of a spot arriving in Spothole, please use the SSE endpoints instead, -e.g. `https://spothole.app/api/v1/spots/stream`. +e.g. `https://spothole.app/api/v2/spots/stream`. If you want to handle different types of spot or alert differently within your client, please consider making a single request to the Spothole API to retrieve all the data, then filtering on your side. For example, call -`https://spothole.app/api/v1/spots?sig=POTA,SOTA` rather than making two separate calls to -`https://spothole.app/api/v1/spots?sig=POTA` and `https://spothole.app/api/v1/spots?sig=SOTA`. +`https://spothole.app/api/v2/spots?sig=POTA,SOTA` rather than making two separate calls to +`https://spothole.app/api/v2/spots?sig=POTA` and `https://spothole.app/api/v2/spots?sig=SOTA`. Remember, here at Spothole Inc. we offer an industry-standard "five nines" uptime on our server, with our own unique twist: we don't tell you which side of the decimal point the nines start! (Translation: This is a hobby project. `spothole.app` runs on the same server as my blog and other stuff. It might go down without warning. By all means base your own project on data from the main server if you like, but if you want any control over reliability and downtime, -please run your own copy instead.) \ No newline at end of file +please run your own copy instead.) diff --git a/docs/docker.md b/docs/docker.md index 2ac47f4..d623828 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -85,7 +85,7 @@ server { } # SSE endpoints - location ~ ^/api/v1/(spots|alerts)/stream/? { + location ~ ^/api/v2/(spots|alerts)/stream/? { proxy_pass http://spothole:8080; # Remove buffering, remove caching, add suitable timeouts for SSE API calls @@ -147,4 +147,4 @@ server { If desired, you could even change the port on which Spothole runs from 8080 to a plain 80, in which case your `proxy_pass` statements could drop the `:8080` suffix. Since Spothole is in a container, it can serve HTTP on port 80 -if desired, because it doesn't conflict with the host system. \ No newline at end of file +if desired, because it doesn't conflict with the host system. diff --git a/docs/modifying.md b/docs/modifying.md index e332b7d..f754970 100644 --- a/docs/modifying.md +++ b/docs/modifying.md @@ -9,12 +9,17 @@ To navigate your way around the source code, this list may help. *Python back-end code* -* `/core` - Core classes and scripts +* `/core` - Core classes and utilities * `/data` - Data storage classes -* `/spotproviders` - Classes providing spots by accessing the APIs of other services -* `/alertproviders` - Classes providing alerts by accessing the APIs of other services -* `/solarconditionsproviders` - Classes providing solar and propagation by accessing the APIs of other services +* `/providers/spot` - Classes providing spots by accessing the APIs of other services +* `/providers/alert` - Classes providing alerts by accessing the APIs of other services +* `/providers/solarconditions` - Classes providing solar and propagation by accessing the APIs of other services +* `/providers/staticdata` - Classes providing static lookup data by accessing bundled data files or the APIs of other + services +* `/providers/sigrefdata` - Classes providing SIG reference lookup data by accessing bundled data files or the APIs of + other services * `/server` - Classes for running Spothole's own web server +* `spothole.py` - Main application script *Templates* @@ -24,6 +29,7 @@ To navigate your way around the source code, this list may help. * `/static` - Root for static files served by the web server. These are all served from a path starting `/static/`. * `/static/apidocs` - Contains the OpenAPI spec (`openapi.yml`) +* `/static/audio` - Audio files used by the web front-end * `/static/css` - CSS files used by the web front-end * `/static/img` - image files used by the web front-end * `/static/js` - JavaScript used by the web front-end @@ -31,18 +37,18 @@ To navigate your way around the source code, this list may help. *Miscellaneous* -* `/` - Main script (`spothole.py`), pip `requirements.txt`, config, README, etc. +* `/` - pip `requirements.txt`, config, README, etc. * `/docs` - Documentation * `/images` - Image sources -* `/datafiles` - Local data sources (differentiated from the majority of data files which are loaded from URLs and - cached in `/cache`) -* `/cache` - Directory where static-ish data downloaded from the internet is cached to avoid rapid re-requests, and - where spot/alert data is cached so that it survives a software restart. Created on first run. +* `/datafiles` - Local data files, used by some providers when the data will never change and/or is not easily available + online in a format Spothole can handle +* `/cache` - Directory where Spothole stores all the data it uses that should be persisted to disk. Created on first + run. ### Extending the server Spothole is designed to be easily extensible. If you want to write your own spot provider, for example, simply add a -module to the `spotproviders` package containing your class. (Currently, in order to be loaded correctly, the module ( +module to the `providers.spot` package containing your class. (Currently, in order to be loaded correctly, the module ( file) name should be the same as the class name, but lower case.) Your class should extend "SpotProvider"; if it operates by polling an HTTP Server on a timer, it can instead extend " @@ -66,4 +72,5 @@ parameters are optional, but you will at least want to provide a `time` (which m Finally, simply add the appropriate config to the `spot_providers` section of `config.yml`, and your provider should be instantiated on startup. -The same approach as above is also used for alert providers. \ No newline at end of file +The same approach as above is also used for alerts, and other types of providers. Give me a shout if you need any +advice. diff --git a/docs/nginx.md b/docs/nginx.md index 8f3b753..fb9d2a7 100644 --- a/docs/nginx.md +++ b/docs/nginx.md @@ -48,7 +48,7 @@ server { } # SSE endpoints - location ~ ^/api/v1/(spots|alerts)/stream/? { + location ~ ^/api/v2/(spots|alerts)/stream/? { proxy_pass http://127.0.0.1:8080; # Remove buffering, remove caching, add suitable timeouts for SSE API calls @@ -120,4 +120,4 @@ You should now be able to access the web interface by going to the domain from y Once that's working, [install certbot](https://certbot.eff.org/instructions?ws=nginx&os=snap) onto your server. Run it as root, and when prompted pick your domain name from the list. After a few seconds, it should successfully provision a -certificate and modify your nginx config files automatically. You should then be able to access the site via HTTPS. \ No newline at end of file +certificate and modify your nginx config files automatically. You should then be able to access the site via HTTPS. diff --git a/docs/running.md b/docs/running.md index 54ae3bc..340aabd 100644 --- a/docs/running.md +++ b/docs/running.md @@ -47,7 +47,8 @@ python3 spothole.py ``` The software can take a few seconds to start up, mostly because it is downloading an updated file to match callsigns to -countries. This is normal, don't panic! +countries. This is normal, don't panic! Once you see `You can access your copy of Spothole at +http://localhost:8080` in the log, your server is good to go. If you see some errors on startup, check your configuration, e.g. in case you have specified a port for the web server -that is already in use by something else. \ No newline at end of file +that is already in use by something else. diff --git a/server/handlers/api/addspot.py b/server/handlers/api/addspot.py index 3ca83c3..34b7c92 100644 --- a/server/handlers/api/addspot.py +++ b/server/handlers/api/addspot.py @@ -12,12 +12,12 @@ from tornado.web import Application from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRET_KEY from core.constants import UNKNOWN_BAND -from core.utils import infer_band_from_freq from core.prometheus_metrics_handler import api_requests_counter from core.sig_utils import get_ref_regex_for_sig +from core.utils import infer_band_from_freq from core.utils import safe_json_dumps from data.spot import Spot -from spotproviders.spot_provider import SpotProvider +from providers.spot.spot_provider import SpotProvider RECAPTCHA_VERIFY_URL = "https://www.google.com/recaptcha/api/siteverify" @@ -82,19 +82,24 @@ class APISpotHandler(tornado.web.RequestHandler): upstream_credentials = handling.get("upstream_credentials", {}) captcha_token = handling.get("captcha_token", None) + + # Spothole v2.0 release only: deny upstream spotting. Spothole API breaking changes were in v2.0 but + # functionality is not ready yet. TODO + submit_upstream = False + + + # Verify CAPTCHA if required if RECAPTCHA_SECRET_KEY: if not captcha_token: self.set_status(422) - self.write(json.dumps("Error - CAPTCHA token is required for spot submission.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - CAPTCHA token is required for spot submission.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return if not self._verify_recaptcha(captcha_token): self.set_status(422) - self.write(json.dumps("Error - CAPTCHA verification failed.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - CAPTCHA verification failed.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -105,7 +110,8 @@ class APISpotHandler(tornado.web.RequestHandler): # Reject if no timestamp, frequency, dx_call or de_call if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call: self.set_status(422) - self.write(safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.")) + self.write( + safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -127,7 +133,8 @@ class APISpotHandler(tornado.web.RequestHandler): # Reject if frequency not in a known band if infer_band_from_freq(spot.freq) == UNKNOWN_BAND: self.set_status(422) - self.write(safe_json_dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band.")) + self.write( + safe_json_dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -137,7 +144,8 @@ class APISpotHandler(tornado.web.RequestHandler): r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$", spot.dx_grid.upper()): self.set_status(422) - self.write(safe_json_dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid.")) + self.write( + safe_json_dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -155,8 +163,7 @@ class APISpotHandler(tornado.web.RequestHandler): # Reject upstream submission if not permitted if submit_upstream and not ALLOW_UPSTREAM_SPOTTING: self.set_status(403) - self.write(json.dumps("Error - this server does not allow upstream spot submission.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - this server does not allow upstream spot submission.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -165,29 +172,26 @@ class APISpotHandler(tornado.web.RequestHandler): if submit_upstream and upstream_provider_name: if not spot.sig: self.set_status(422) - self.write(json.dumps("Error - a SIG must be selected to submit upstream.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - a SIG must be selected to submit upstream.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return if not spot.sig_refs and upstream_provider_name != "Tiles": self.set_status(422) - self.write(json.dumps("Error - a SIG reference is required to submit upstream.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - a SIG reference is required to submit upstream.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return if not spot.dx_grid and upstream_provider_name == "Tiles": self.set_status(422) - self.write(json.dumps("Error - a grid reference is required to submit upstream to Tiles on the Air.", - default=serialize_everything)) + self.write( + safe_json_dumps("Error - a grid reference is required to submit upstream to Tiles on the Air.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return if not spot.mode and upstream_provider_name == "Tiles": self.set_status(422) - self.write(json.dumps("Error - a mode is required to submit upstream to Tiles on the Air.", - default=serialize_everything)) + self.write(safe_json_dumps("Error - a mode is required to submit upstream to Tiles on the Air.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") return @@ -209,7 +213,8 @@ class APISpotHandler(tornado.web.RequestHandler): upstream_warning = "Spot was saved locally but upstream submission to " + upstream_provider_name + " failed: " + str( e) else: - upstream_warning = "No enabled provider named '" + upstream_provider_name + "' supports upstream submission for " + spot.sig + " spots." + upstream_warning = "No enabled provider named '" + upstream_provider_name + "' supports upstream submission for " + ( + spot.sig if spot.sig else "") + " spots." # If we successfully submitted the spot upstream, don't add it direct to Spothole, otherwise it will be a # duplicate with what immediately comes back from the API. But if we weren't asked to send it upstream, or @@ -219,7 +224,7 @@ class APISpotHandler(tornado.web.RequestHandler): self._spots.set(spot.id, spot) if upstream_warning: - self.write(json.dumps("Warning - " + upstream_warning, default=serialize_everything)) + self.write(safe_json_dumps("Warning - " + upstream_warning)) self.set_status(201) else: self.write(safe_json_dumps("OK")) diff --git a/server/handlers/api/options.py b/server/handlers/api/options.py index 2395dc1..d483bc1 100644 --- a/server/handlers/api/options.py +++ b/server/handlers/api/options.py @@ -37,12 +37,14 @@ class APIOptionsHandler(tornado.web.RequestHandler): # Build a map of SIG name -> list of provider names that can submit spots for that SIG spot_submit_providers = {} - for provider in self._spot_providers: - if not provider.enabled: - continue - for sig in SIGS: - if provider.can_submit_spot(sig.name): - spot_submit_providers.setdefault(sig.name, []).append(provider.name) + + # Spothole v2.0 - disable this for now, API changes are in but this functionality is not ready yet. TODO + # for provider in self._spot_providers: + # if not provider.enabled: + # continue + # for sig in SIGS: + # if provider.can_submit_spot(sig.name): + # spot_submit_providers.setdefault(sig.name, []).append(provider.name) # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle # things that aren't even available. diff --git a/server/handlers/api/v1_compatability.py b/server/handlers/api/v1_compatability.py index 9639bbc..2c1a07a 100644 --- a/server/handlers/api/v1_compatability.py +++ b/server/handlers/api/v1_compatability.py @@ -2,7 +2,7 @@ import json import tornado -from core.utils import serialize_everything +from core.utils import safe_json_dumps class V1GoneHandler(tornado.web.RequestHandler): @@ -11,10 +11,8 @@ class V1GoneHandler(tornado.web.RequestHandler): def post(self): self.set_status(410) - self.write(json.dumps( - "This API endpoint has a breaking change or has been removed in the current version of the Spothole API. Please see /apidocs for details of the current API version and the endpoints available.", - default=serialize_everything - )) + self.write(safe_json_dumps( + "This API endpoint has a breaking change or has been removed in the current version of the Spothole API. Please see /apidocs for details of the current API version and the endpoints available.")) self.set_header("Cache-Control", "no-store") self.set_header("Content-Type", "application/json") diff --git a/server/webserver.py b/server/webserver.py index 78dbf49..ed432cf 100644 --- a/server/webserver.py +++ b/server/webserver.py @@ -1,15 +1,14 @@ import asyncio import logging import os -import threading import tornado from tornado.web import StaticFileHandler from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL +from core.data_providers import DATA_PROVIDERS from core.data_store import DATA_STORE from server.handlers.api.addspot import APISpotHandler -from server.handlers.api.v1_compatability import V1RedirectHandler, V1GoneHandler from server.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler from server.handlers.api.dxstats import APIDxStatsHandler from server.handlers.api.lookups import APILookupCallHandler, APILookupSIGRefHandler, APILookupGridHandler @@ -17,6 +16,7 @@ from server.handlers.api.options import APIOptionsHandler from server.handlers.api.solar_conditions import APISolarConditionsHandler from server.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler from server.handlers.api.status import APIStatusHandler +from server.handlers.api.v1_compatability import V1RedirectHandler, V1GoneHandler from server.handlers.manifesthandler import ManifestHandler from server.handlers.metrics import PrometheusMetricsHandler from server.handlers.pagetemplate import PageTemplateHandler @@ -32,6 +32,7 @@ class WebServer: """Constructor""" self._data_store = DATA_STORE + self._data_providers = DATA_PROVIDERS self._spot_broadcaster = SSEBroadcaster() self._alert_broadcaster = SSEBroadcaster() self._port = WEB_SERVER_PORT @@ -87,11 +88,12 @@ class WebServer: (r"/api/v2/lookup/call", APILookupCallHandler, {**handler_opts}), (r"/api/v2/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}), (r"/api/v2/lookup/grid", APILookupGridHandler, {**handler_opts}), - (r"/api/v2/spot", APISpotHandler, {"spots": self._data_store.spots, **handler_opts}), + (r"/api/v2/spot", APISpotHandler, + {"spots": self._data_store.spots, "spot_providers": self._data_providers, **handler_opts}), ] # v1 API redirects. Most v1 enpoints are unchanged in v2, and get an HTTP 308 redirect to the v2 API. The ones - # that have the actual breaking changes get a bespoke handler. + # that have the major breaking changes get a bespoke handler. v1_compat_routes = [ (r"/api/v1/spot", V1GoneHandler), (r"/api/v1/(.*)", V1RedirectHandler), @@ -162,4 +164,4 @@ def request_log(handler): # Global object -WEB_SERVER = WebServer() \ No newline at end of file +WEB_SERVER = WebServer() diff --git a/static/js/bands.js b/static/js/bands.js index 70e7689..29c47f4 100644 --- a/static/js/bands.js +++ b/static/js/bands.js @@ -21,12 +21,13 @@ function loadSpots() { evtSource.close(); } $.ajax({url: '/api/v2/spots' + buildQueryString(), dataType: 'json', headers: getCredentialHeaders(), success: function (jsonData) { - // Store data - spots = jsonData; - // Update bands display - updateBands(); - // Start the ongoing SSE connection - startSSEConnection(); + // Store data + spots = jsonData; + // Update bands display + updateBands(); + // Start the ongoing SSE connection + startSSEConnection(); + } }); } diff --git a/static/js/status.js b/static/js/status.js index 3142593..87ab550 100644 --- a/static/js/status.js +++ b/static/js/status.js @@ -55,7 +55,7 @@ function loadStatus() {
${p["sig_name"]}
Status: ${p["status"]}
Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}
-
References: ${(p["enabled"] && p["reference_count"] > 0) ? p["reference_count"] : "N/A"}
+
References: ${p["enabled"] ? p["reference_count"] : "N/A"}
`); }); @@ -65,7 +65,7 @@ function loadStatus() {
${p["name"]}
Status: ${p["status"]}
Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}
-
Lookups: ${(p["enabled"] && p["lookup_count"] > 0) ? p["lookup_count"] : "N/A"}
+
Lookups: ${p["enabled"] ? p["lookup_count"] : "N/A"}
`); }); }); diff --git a/templates/add_spot.html b/templates/add_spot.html index e6ac118..da4ea25 100644 --- a/templates/add_spot.html +++ b/templates/add_spot.html @@ -2,7 +2,7 @@ {% block content %}
-