Hellmerge #95 branch into 2.0-pre. #119

data/spot.py

@@ -330,32 +330,32 @@ class Spot:
                             self.propagation_mode = mode_tag
                             logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag)
 
-        # Parse "de_grid -> dx_grid" structures from the comment
+            # Parse "de_grid -> dx_grid" structures from the comment
-        if self.comment:
+            if self.comment:
-            grid_mode_grid_match = re.search(
+                grid_mode_grid_match = re.search(
-                r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
+                    r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
-                self.comment)
+                    self.comment)
-            if grid_mode_grid_match:
+                if grid_mode_grid_match:
-                # regex matches, so extract grids:
+                    # regex matches, so extract grids:
-                if not self.dx_grid:
+                    if not self.dx_grid:
-                    self.dx_grid = grid_mode_grid_match.group(1).upper()
+                        self.dx_grid = grid_mode_grid_match.group(1).upper()
-                    self.dx_location_source = "SPOT"
+                        self.dx_location_source = "SPOT"
-                if not self.de_grid:
+                    if not self.de_grid:
-                    self.de_grid = grid_mode_grid_match.group(2).upper()
+                        self.de_grid = grid_mode_grid_match.group(2).upper()
 
-        # DX Grid to lat/lon and vice versa in case one is missing
+            # DX Grid to lat/lon and vice versa in case one is missing
-        if self.dx_grid and not self.dx_latitude:
+            if self.dx_grid and not self.dx_latitude:
-            try:
+                try:
-                ll = locator_to_latlong(self.dx_grid)
+                    ll = locator_to_latlong(self.dx_grid)
-                self.dx_latitude = ll[0]
+                    self.dx_latitude = ll[0]
-                self.dx_longitude = ll[1]
+                    self.dx_longitude = ll[1]
-            except:
+                except:
-                logging.debug("Invalid grid received for spot")
+                    logging.debug("Invalid grid received for spot")
-        if self.dx_latitude and self.dx_longitude and not self.dx_grid:
+            if self.dx_latitude and self.dx_longitude and not self.dx_grid:
-            try:
+                try:
-                self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8)
+                    self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8)
-            except:
+                except:
-                logging.debug("Invalid lat/lon received for spot")
+                    logging.debug("Invalid lat/lon received for spot")
 
             # QRT comment detection
             if self.comment and not self.qrt:

docs/clients.md

@@ -12,7 +12,7 @@ Various approaches exist to writing your own client, but in general:
 * Refer to the API docs. These are built on an OpenAPI definition file (`/static/apidocs/openapi.yml`), which you can
   automatically use to generate a client skeleton using various software.
 * Call the main "spots" or "alerts" API endpoints to get the data you want. For example, your app could call
-  `https://spothole.app/api/v1/spots` once every few minutes. Apply filters if necessary.
+  `https://spothole.app/api/v2/spots` once every few minutes. Apply filters if necessary.
 * Call the "options" API to get an idea of which bands, modes etc. the server knows about. You might want to do that
   first before calling the spots/alerts APIs, to allow you to populate your filters correctly.
 * Refer to the provided HTML/JS interface for a reference on different approaches. For example, the "alerts"/"upcoming"
@@ -25,12 +25,12 @@ once every two minutes, so if your client is interested in POTA data there's no
 than that.
 
 If you absolutely must be informed within seconds of a spot arriving in Spothole, please use the SSE endpoints instead,
-e.g. `https://spothole.app/api/v1/spots/stream`.
+e.g. `https://spothole.app/api/v2/spots/stream`.
 
 If you want to handle different types of spot or alert differently within your client, please consider making a single
 request to the Spothole API to retrieve all the data, then filtering on your side. For example, call
-`https://spothole.app/api/v1/spots?sig=POTA,SOTA` rather than making two separate calls to
+`https://spothole.app/api/v2/spots?sig=POTA,SOTA` rather than making two separate calls to
-`https://spothole.app/api/v1/spots?sig=POTA` and `https://spothole.app/api/v1/spots?sig=SOTA`.
+`https://spothole.app/api/v2/spots?sig=POTA` and `https://spothole.app/api/v2/spots?sig=SOTA`.
 
 Remember, here at Spothole Inc. we offer an industry-standard "five nines" uptime on our server, with our own unique
 twist: we don't tell you which side of the decimal point the nines start! (Translation: This is a hobby project.

docs/docker.md

@@ -85,7 +85,7 @@ server {
     }
     
     # SSE endpoints
-    location ~ ^/api/v1/(spots|alerts)/stream/? {
+    location ~ ^/api/v2/(spots|alerts)/stream/? {
         proxy_pass http://spothole:8080;
         
         # Remove buffering, remove caching, add suitable timeouts for SSE API calls

docs/modifying.md

@@ -9,12 +9,17 @@ To navigate your way around the source code, this list may help.
 
 *Python back-end code*
 
-* `/core` - Core classes and scripts
+* `/core` - Core classes and utilities
 * `/data` - Data storage classes
-* `/spotproviders` - Classes providing spots by accessing the APIs of other services
+* `/providers/spot` - Classes providing spots by accessing the APIs of other services
-* `/alertproviders` - Classes providing alerts by accessing the APIs of other services
+* `/providers/alert` - Classes providing alerts by accessing the APIs of other services
-* `/solarconditionsproviders` - Classes providing solar and propagation by accessing the APIs of other services
+* `/providers/solarconditions` - Classes providing solar and propagation by accessing the APIs of other services
+* `/providers/staticdata` - Classes providing static lookup data by accessing bundled data files or the APIs of other
+  services
+* `/providers/sigrefdata` - Classes providing SIG reference lookup data by accessing bundled data files or the APIs of
+  other services
 * `/server` - Classes for running Spothole's own web server
+* `spothole.py` - Main application script
 
 *Templates*
 
@@ -24,6 +29,7 @@ To navigate your way around the source code, this list may help.
 
 * `/static` - Root for static files served by the web server. These are all served from a path starting `/static/`.
 * `/static/apidocs` - Contains the OpenAPI spec (`openapi.yml`)
+* `/static/audio` - Audio files used by the web front-end
 * `/static/css` - CSS files used by the web front-end
 * `/static/img` - image files used by the web front-end
 * `/static/js` - JavaScript used by the web front-end
@@ -31,18 +37,18 @@ To navigate your way around the source code, this list may help.
 
 *Miscellaneous*
 
-* `/` - Main script (`spothole.py`), pip `requirements.txt`, config, README, etc.
+* `/` - pip `requirements.txt`, config, README, etc.
 * `/docs` - Documentation
 * `/images` - Image sources
-* `/datafiles` - Local data sources (differentiated from the majority of data files which are loaded from URLs and
+* `/datafiles` - Local data files, used by some providers when the data will never change and/or is not easily available
-  cached in `/cache`)
+  online in a format Spothole can handle
-* `/cache` - Directory where static-ish data downloaded from the internet is cached to avoid rapid re-requests, and
+* `/cache` - Directory where Spothole stores all the data it uses that should be persisted to disk. Created on first
-  where spot/alert data is cached so that it survives a software restart. Created on first run.
+  run.
 
 ### Extending the server
 
 Spothole is designed to be easily extensible. If you want to write your own spot provider, for example, simply add a
-module to the `spotproviders` package containing your class. (Currently, in order to be loaded correctly, the module (
+module to the `providers.spot` package containing your class. (Currently, in order to be loaded correctly, the module (
 file) name should be the same as the class name, but lower case.)
 
 Your class should extend "SpotProvider"; if it operates by polling an HTTP Server on a timer, it can instead extend "
@@ -66,4 +72,5 @@ parameters are optional, but you will at least want to provide a `time` (which m
 Finally, simply add the appropriate config to the `spot_providers` section of `config.yml`, and your provider should be
 instantiated on startup.
 
-The same approach as above is also used for alert providers.
+The same approach as above is also used for alerts, and other types of providers. Give me a shout if you need any
+advice.

docs/nginx.md

@@ -48,7 +48,7 @@ server {
     }
     
     # SSE endpoints
-    location ~ ^/api/v1/(spots|alerts)/stream/? {
+    location ~ ^/api/v2/(spots|alerts)/stream/? {
         proxy_pass http://127.0.0.1:8080;
         
         # Remove buffering, remove caching, add suitable timeouts for SSE API calls

docs/running.md

@@ -47,7 +47,8 @@ python3 spothole.py
 ```
 
 The software can take a few seconds to start up, mostly because it is downloading an updated file to match callsigns to
-countries. This is normal, don't panic!
+countries. This is normal, don't panic! Once you see `You can access your copy of Spothole at
+http://localhost:8080` in the log, your server is good to go.
 
 If you see some errors on startup, check your configuration, e.g. in case you have specified a port for the web server
 that is already in use by something else.

server/handlers/api/addspot.py

@@ -12,12 +12,12 @@ from tornado.web import Application
 
 from core.config import ALLOW_SPOTTING, ALLOW_UPSTREAM_SPOTTING, RECAPTCHA_SECRET_KEY
 from core.constants import UNKNOWN_BAND
-from core.utils import infer_band_from_freq
 from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
+from core.utils import infer_band_from_freq
 from core.utils import safe_json_dumps
 from data.spot import Spot
-from spotproviders.spot_provider import SpotProvider
+from providers.spot.spot_provider import SpotProvider
 
 RECAPTCHA_VERIFY_URL = "https://www.google.com/recaptcha/api/siteverify"
 
@@ -82,19 +82,24 @@ class APISpotHandler(tornado.web.RequestHandler):
             upstream_credentials = handling.get("upstream_credentials", {})
             captcha_token = handling.get("captcha_token", None)
 
+
+            # Spothole v2.0 release only: deny upstream spotting. Spothole API breaking changes were in v2.0 but
+            # functionality is not ready yet. TODO
+            submit_upstream = False
+
+
+
             # Verify CAPTCHA if required
             if RECAPTCHA_SECRET_KEY:
                 if not captcha_token:
                     self.set_status(422)
-                    self.write(json.dumps("Error - CAPTCHA token is required for spot submission.",
+                    self.write(safe_json_dumps("Error - CAPTCHA token is required for spot submission."))
-                                          default=serialize_everything))
                     self.set_header("Cache-Control", "no-store")
                     self.set_header("Content-Type", "application/json")
                     return
                 if not self._verify_recaptcha(captcha_token):
                     self.set_status(422)
-                    self.write(json.dumps("Error - CAPTCHA verification failed.",
+                    self.write(safe_json_dumps("Error - CAPTCHA verification failed."))
-                                          default=serialize_everything))
                     self.set_header("Cache-Control", "no-store")
                     self.set_header("Content-Type", "application/json")
                     return
@@ -105,7 +110,8 @@ class APISpotHandler(tornado.web.RequestHandler):
             # Reject if no timestamp, frequency, dx_call or de_call
             if not spot.time or not spot.dx_call or not spot.freq or not spot.de_call:
                 self.set_status(422)
-                self.write(safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum."))
+                self.write(
+                    safe_json_dumps("Error - 'time', 'dx_call', 'freq' and 'de_call' must be provided as a minimum."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -127,7 +133,8 @@ class APISpotHandler(tornado.web.RequestHandler):
             # Reject if frequency not in a known band
             if infer_band_from_freq(spot.freq) == UNKNOWN_BAND:
                 self.set_status(422)
-                self.write(safe_json_dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band."))
+                self.write(
+                    safe_json_dumps("Error - Frequency of " + str(spot.freq / 1000.0) + "kHz is not in a known band."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -137,7 +144,8 @@ class APISpotHandler(tornado.web.RequestHandler):
                     r"^([A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2}[A-X]{2}[0-9]{2}|[A-R]{2}[0-9]{2}[A-X]{2}|[A-R]{2}[0-9]{2})$",
                     spot.dx_grid.upper()):
                 self.set_status(422)
-                self.write(safe_json_dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid."))
+                self.write(
+                    safe_json_dumps("Error - '" + spot.dx_grid + "' does not look like a valid Maidenhead grid."))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -155,8 +163,7 @@ class APISpotHandler(tornado.web.RequestHandler):
             # Reject upstream submission if not permitted
             if submit_upstream and not ALLOW_UPSTREAM_SPOTTING:
                 self.set_status(403)
-                self.write(json.dumps("Error - this server does not allow upstream spot submission.",
+                self.write(safe_json_dumps("Error - this server does not allow upstream spot submission."))
-                                      default=serialize_everything))
                 self.set_header("Cache-Control", "no-store")
                 self.set_header("Content-Type", "application/json")
                 return
@@ -165,29 +172,26 @@ class APISpotHandler(tornado.web.RequestHandler):
             if submit_upstream and upstream_provider_name:
                 if not spot.sig:
                     self.set_status(422)
-                    self.write(json.dumps("Error - a SIG must be selected to submit upstream.",
+                    self.write(safe_json_dumps("Error - a SIG must be selected to submit upstream."))
-                                          default=serialize_everything))
                     self.set_header("Cache-Control", "no-store")
                     self.set_header("Content-Type", "application/json")
                     return
                 if not spot.sig_refs and upstream_provider_name != "Tiles":
                     self.set_status(422)
-                    self.write(json.dumps("Error - a SIG reference is required to submit upstream.",
+                    self.write(safe_json_dumps("Error - a SIG reference is required to submit upstream."))
-                                          default=serialize_everything))
                     self.set_header("Cache-Control", "no-store")
                     self.set_header("Content-Type", "application/json")
                     return
                 if not spot.dx_grid and upstream_provider_name == "Tiles":
                     self.set_status(422)
-                    self.write(json.dumps("Error - a grid reference is required to submit upstream to Tiles on the Air.",
+                    self.write(
-                                          default=serialize_everything))
+                        safe_json_dumps("Error - a grid reference is required to submit upstream to Tiles on the Air."))
                     self.set_header("Cache-Control", "no-store")
                     self.set_header("Content-Type", "application/json")
                     return
                 if not spot.mode and upstream_provider_name == "Tiles":
                     self.set_status(422)
-                    self.write(json.dumps("Error - a mode is required to submit upstream to Tiles on the Air.",
+                    self.write(safe_json_dumps("Error - a mode is required to submit upstream to Tiles on the Air."))
-                                          default=serialize_everything))
                     self.set_header("Cache-Control", "no-store")
                     self.set_header("Content-Type", "application/json")
                     return
@@ -209,7 +213,8 @@ class APISpotHandler(tornado.web.RequestHandler):
                         upstream_warning = "Spot was saved locally but upstream submission to " + upstream_provider_name + " failed: " + str(
                             e)
                 else:
-                    upstream_warning = "No enabled provider named '" + upstream_provider_name + "' supports upstream submission for " + spot.sig + " spots."
+                    upstream_warning = "No enabled provider named '" + upstream_provider_name + "' supports upstream submission for " + (
+                        spot.sig if spot.sig else "") + " spots."
 
             # If we successfully submitted the spot upstream, don't add it direct to Spothole, otherwise it will be a
             # duplicate with what immediately comes back from the API. But if we weren't asked to send it upstream, or
@@ -219,7 +224,7 @@ class APISpotHandler(tornado.web.RequestHandler):
                 self._spots.set(spot.id, spot)
 
             if upstream_warning:
-                self.write(json.dumps("Warning - " + upstream_warning, default=serialize_everything))
+                self.write(safe_json_dumps("Warning - " + upstream_warning))
                 self.set_status(201)
             else:
                 self.write(safe_json_dumps("OK"))

server/handlers/api/options.py

@@ -37,12 +37,14 @@ class APIOptionsHandler(tornado.web.RequestHandler):
 
             # Build a map of SIG name -> list of provider names that can submit spots for that SIG
             spot_submit_providers = {}
-            for provider in self._spot_providers:
+
-                if not provider.enabled:
+            # Spothole v2.0 - disable this for now, API changes are in but this functionality is not ready yet. TODO
-                    continue
+            # for provider in self._spot_providers:
-                for sig in SIGS:
+            #     if not provider.enabled:
-                    if provider.can_submit_spot(sig.name):
+            #         continue
-                        spot_submit_providers.setdefault(sig.name, []).append(provider.name)
+            #     for sig in SIGS:
+            #         if provider.can_submit_spot(sig.name):
+            #             spot_submit_providers.setdefault(sig.name, []).append(provider.name)
 
             # Spot/alert sources are filtered for only ones that are enabled in config, no point letting the user toggle
             # things that aren't even available.

server/handlers/api/v1_compatability.py

@@ -2,7 +2,7 @@ import json
 
 import tornado
 
-from core.utils import serialize_everything
+from core.utils import safe_json_dumps
 
 
 class V1GoneHandler(tornado.web.RequestHandler):
@@ -11,10 +11,8 @@ class V1GoneHandler(tornado.web.RequestHandler):
 
     def post(self):
         self.set_status(410)
-        self.write(json.dumps(
+        self.write(safe_json_dumps(
-            "This API endpoint has a breaking change or has been removed in the current version of the Spothole API. Please see /apidocs for details of the current API version and the endpoints available.",
+            "This API endpoint has a breaking change or has been removed in the current version of the Spothole API. Please see /apidocs for details of the current API version and the endpoints available."))
-            default=serialize_everything
-        ))
         self.set_header("Cache-Control", "no-store")
         self.set_header("Content-Type", "application/json")
 

server/webserver.py

@@ -1,15 +1,14 @@
 import asyncio
 import logging
 import os
-import threading
 
 import tornado
 from tornado.web import StaticFileHandler
 
 from core.config import ALLOW_SPOTTING, WEB_SERVER_PORT, API_ONLY_MODE, LOG_WEB_REQUESTS, BASE_URL
+from core.data_providers import DATA_PROVIDERS
 from core.data_store import DATA_STORE
 from server.handlers.api.addspot import APISpotHandler
-from server.handlers.api.v1_compatability import V1RedirectHandler, V1GoneHandler
 from server.handlers.api.alerts import APIAlertsHandler, APIAlertsStreamHandler
 from server.handlers.api.dxstats import APIDxStatsHandler
 from server.handlers.api.lookups import APILookupCallHandler, APILookupSIGRefHandler, APILookupGridHandler
@@ -17,6 +16,7 @@ from server.handlers.api.options import APIOptionsHandler
 from server.handlers.api.solar_conditions import APISolarConditionsHandler
 from server.handlers.api.spots import APISpotsHandler, APISpotsStreamHandler
 from server.handlers.api.status import APIStatusHandler
+from server.handlers.api.v1_compatability import V1RedirectHandler, V1GoneHandler
 from server.handlers.manifesthandler import ManifestHandler
 from server.handlers.metrics import PrometheusMetricsHandler
 from server.handlers.pagetemplate import PageTemplateHandler
@@ -32,6 +32,7 @@ class WebServer:
         """Constructor"""
 
         self._data_store = DATA_STORE
+        self._data_providers = DATA_PROVIDERS
         self._spot_broadcaster = SSEBroadcaster()
         self._alert_broadcaster = SSEBroadcaster()
         self._port = WEB_SERVER_PORT
@@ -87,11 +88,12 @@ class WebServer:
             (r"/api/v2/lookup/call", APILookupCallHandler, {**handler_opts}),
             (r"/api/v2/lookup/sigref", APILookupSIGRefHandler, {**handler_opts}),
             (r"/api/v2/lookup/grid", APILookupGridHandler, {**handler_opts}),
-            (r"/api/v2/spot", APISpotHandler, {"spots": self._data_store.spots, **handler_opts}),
+            (r"/api/v2/spot", APISpotHandler,
+             {"spots": self._data_store.spots, "spot_providers": self._data_providers, **handler_opts}),
         ]
 
         # v1 API redirects. Most v1 enpoints are unchanged in v2, and get an HTTP 308 redirect to the v2 API. The ones
-        # that have the actual breaking changes get a bespoke handler.
+        # that have the major breaking changes get a bespoke handler.
         v1_compat_routes = [
             (r"/api/v1/spot", V1GoneHandler),
             (r"/api/v1/(.*)", V1RedirectHandler),

static/js/bands.js

@@ -21,12 +21,13 @@ function loadSpots() {
         evtSource.close();
     }
     $.ajax({url: '/api/v2/spots' + buildQueryString(), dataType: 'json', headers: getCredentialHeaders(), success: function (jsonData) {
-        // Store data
+            // Store data
-        spots = jsonData;
+            spots = jsonData;
-        // Update bands display
+            // Update bands display
-        updateBands();
+            updateBands();
-        // Start the ongoing SSE connection
+            // Start the ongoing SSE connection
-        startSSEConnection();
+            startSSEConnection();
+        }
     });
 }
 

static/js/status.js

@@ -55,7 +55,7 @@ function loadStatus() {
                     <div class="col"><strong>${p["sig_name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
-                    <div class="col">References: ${(p["enabled"] && p["reference_count"] > 0) ? p["reference_count"] : "N/A"}</div>
+                    <div class="col">References: ${p["enabled"] ? p["reference_count"] : "N/A"}</div>
                 </div>`);
         });
 
@@ -65,7 +65,7 @@ function loadStatus() {
                     <div class="col"><strong>${p["name"]}</strong></div>
                     <div class="col">Status: ${p["status"]}</div>
                     <div class="col">Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}</div>
-                    <div class="col">Lookups: ${(p["enabled"] && p["lookup_count"] > 0) ? p["lookup_count"] : "N/A"}</div>
+                    <div class="col">Lookups: ${p["enabled"] ? p["lookup_count"] : "N/A"}</div>
                 </div>`);
         });
     });

templates/add_spot.html

@@ -2,7 +2,7 @@
 {% block content %}
 
 <div id="add-spot-intro-box" class="permanently-dismissible-box mt-3">
-    <div class="alert alert-primary alert-dismissible fade show" role="alert">
+    <div class="alert alert-primary alert-dismissible fade show" role="alert"> <!-- TODO Remove when feature available -->
         <i class="fa-solid fa-circle-info"></i> <strong>Adding spots to Spothole</strong><br/>This page is implemented
         as a proof of concept for adding spots to the Spothole system. Currently, spots added in this way are only
         visible within Spothole and are not sent "upstream" to DX clusters or xOTA spotting sites. The functionality