Check GMA response actually contains a response rather than throwing an exception

README.md

@@ -310,44 +310,80 @@ server {
     
     # SSE endpoints
     location ~ ^/api/v1/(spots|alerts)/stream {
+        proxy_pass http://127.0.0.1:8080;
+        
+        # Allow keep-alive
         proxy_http_version 1.1;
         proxy_set_header Connection "";
-        proxy_pass http://127.0.0.1:8080;
+        
+        # Set correct content type for SSE API calls
+        add_header Content-Type text/event-stream always;
+        
+        # Set remove buffering, remove caching, add suitable timeouts for SSE API calls
         proxy_buffering off;
         proxy_cache off;
         proxy_read_timeout 24h;
         proxy_connect_timeout 10s;
         proxy_send_timeout 24h;
         proxy_set_header X-Accel-Buffering no;
+        add_header Cache-Control no-store always;
+        
+        # Allow cross-origin requests to API
         proxy_hide_header Access-Control-Allow-Origin;
         add_header Access-Control-Allow-Origin * always;
-        add_header Cache-Control no-store always;
+        
-        add_header Content-Type text/event-stream always;
+        # Pass on IP address and host information to Spothole, in case logging this information is required
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-Proto $scheme;
     }
 
     # Other API endpoints
     location /api/ {
+        proxy_pass http://127.0.0.1:8080;
+        
+        # Allow keep-alive
         proxy_http_version 1.1;
         proxy_set_header Connection "";
-        proxy_pass http://127.0.0.1:8080;
+        
+        # Set up buffering, remove caching, add suitable timeouts for API calls
         proxy_buffering on;
         proxy_cache off;
         proxy_read_timeout 30s;
         proxy_connect_timeout 10s;
+        add_header Cache-Control no-store always;
+        
+        # Allow cross-origin requests to API
         proxy_hide_header Access-Control-Allow-Origin;
         add_header Access-Control-Allow-Origin * always;
-        add_header Cache-Control no-store always;
+        
+        # Pass on IP address and host information to Spothole, in case logging this information is required
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-Proto $scheme;
     }
 
     # Static assets
     location / {
+        proxy_pass http://127.0.0.1:8080;
+        
+        # Allow keep-alive
         proxy_http_version 1.1;
         proxy_set_header Connection "";
-        proxy_pass http://127.0.0.1:8080;
+        
+        # Set up buffering and caching, add suitable timeouts for static asset requests
         proxy_buffering on;
         proxy_read_timeout 30s;
         proxy_connect_timeout 10s;
         add_header Cache-Control "public, max-age=3600, must-revalidate" always;
+        
+        # Pass on IP address and host information to Spothole, in case logging this information is required
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-Proto $scheme;
     }
 }
 ```

server/webserver.py

@@ -108,8 +108,9 @@ class WebServer:
 
         app = tornado.web.Application(api_routes + ui_routes + misc_routes,
                                       template_path=os.path.join(_HERE, "../templates"),
+                                      log_function=request_log,
                                       debug=False)
-        app.listen(self._port)
+        app.listen(self._port, xheaders=True)
         logging.info("Web server running on port " + str(WEB_SERVER_PORT))
         await self._shutdown_event.wait()
 
@@ -162,3 +163,22 @@ class WebServer:
                 # Probably got deleted already on another thread
                 pass
         pass
+
+def request_log(handler):
+    """Custom log function to provide more data about requests."""
+
+    if handler.get_status() < 500:
+        log_method = logging.info
+    else:
+        log_method = logging.warning
+
+    request = handler.request
+    client_ip = request.remote_ip
+    referrer = request.headers.get("Referer", "-")
+    user_agent = request.headers.get("User-Agent", "-")
+
+    log_method(
+        f'{client_ip} - "{request.method} {request.uri}" '
+        f'{handler.get_status()} {request.request_time():.2f}ms | '
+        f'Ref: {referrer} | UA: {user_agent}'
+    )

spotproviders/gma.py

@@ -24,68 +24,69 @@ class GMA(HTTPSpotProvider):
     def _http_response_to_spots(self, http_response):
         new_spots = []
         # Iterate through source data
-        for source_spot in http_response.json()["RCD"]:
+        if "RCD" in http_response.json():
-            # Convert to our spot format
+            for source_spot in http_response.json()["RCD"]:
-            spot = Spot(source=self.name,
+                # Convert to our spot format
-                        dx_call=source_spot["ACTIVATOR"].upper(),
+                spot = Spot(source=self.name,
-                        de_call=source_spot["SPOTTER"].upper(),
+                            dx_call=source_spot["ACTIVATOR"].upper(),
-                        freq=float(source_spot["QRG"]) * 1000 if (source_spot["QRG"] != "") else None,
+                            de_call=source_spot["SPOTTER"].upper(),
-                        # Seen GMA spots with no frequency
+                            freq=float(source_spot["QRG"]) * 1000 if (source_spot["QRG"] != "") else None,
-                        mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
+                            # Seen GMA spots with no frequency
-                        # Filter out some weird mode strings
+                            mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
-                        comment=source_spot["TEXT"],
+                            # Filter out some weird mode strings
-                        sig_refs=[SIGRef(id=source_spot["REF"], sig="", name=source_spot["NAME"])],
+                            comment=source_spot["TEXT"],
-                        time=datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace(
+                            sig_refs=[SIGRef(id=source_spot["REF"], sig="", name=source_spot["NAME"])],
-                            tzinfo=pytz.UTC).timestamp(),
+                            time=datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace(
-                        dx_latitude=float(source_spot["LAT"]) if (
+                                tzinfo=pytz.UTC).timestamp(),
-                                source_spot["LAT"] and source_spot["LAT"] != "") else None,
+                            dx_latitude=float(source_spot["LAT"]) if (
-                        # Seen GMA spots with no (or empty) lat/lon
+                                    source_spot["LAT"] and source_spot["LAT"] != "") else None,
-                        dx_longitude=float(source_spot["LON"]) if (
+                            # Seen GMA spots with no (or empty) lat/lon
-                                source_spot["LON"] and source_spot["LON"] != "") else None)
+                            dx_longitude=float(source_spot["LON"]) if (
+                                    source_spot["LON"] and source_spot["LON"] != "") else None)
 
-            # GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
+                # GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
-            if "REF" in source_spot:
+                if "REF" in source_spot:
-                try:
+                    try:
-                    ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
+                        ref_response = SEMI_STATIC_URL_DATA_CACHE.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
-                                                                  headers=HTTP_HEADERS)
+                                                                      headers=HTTP_HEADERS)
-                    # Sometimes this is blank, so handle that
+                        # Sometimes this is blank, so handle that
-                    if ref_response.text is not None and ref_response.text != "":
+                        if ref_response.text is not None and ref_response.text != "":
-                        ref_info = ref_response.json()
+                            ref_info = ref_response.json()
-                        # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
+                            # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
-                        # spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
+                            # spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
-                        # and GMA summits come through with reftype=Summit, so we must check for the presence of a "sota" entry
+                            # and GMA summits come through with reftype=Summit, so we must check for the presence of a "sota" entry
-                        # to determine if it's a SOTA summit.
+                            # to determine if it's a SOTA summit.
-                        if spot.sig_refs and "reftype" in ref_info and ref_info["reftype"] not in ["POTA", "WWFF"] and (
+                            if spot.sig_refs and "reftype" in ref_info and ref_info["reftype"] not in ["POTA", "WWFF"] and (
-                                ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info["sota"] == ""):
+                                    ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info["sota"] == ""):
-                            match ref_info["reftype"]:
+                                match ref_info["reftype"]:
-                                case "Summit":
+                                    case "Summit":
-                                    spot.sig_refs[0].sig = "GMA"
+                                        spot.sig_refs[0].sig = "GMA"
-                                    spot.sig = "GMA"
+                                        spot.sig = "GMA"
-                                case "IOTA Island":
+                                    case "IOTA Island":
-                                    spot.sig_refs[0].sig = "IOTA"
+                                        spot.sig_refs[0].sig = "IOTA"
-                                    spot.sig = "IOTA"
+                                        spot.sig = "IOTA"
-                                case "Lighthouse (ILLW)":
+                                    case "Lighthouse (ILLW)":
-                                    spot.sig_refs[0].sig = "ILLW"
+                                        spot.sig_refs[0].sig = "ILLW"
-                                    spot.sig = "ILLW"
+                                        spot.sig = "ILLW"
-                                case "Lighthouse (ARLHS)":
+                                    case "Lighthouse (ARLHS)":
-                                    spot.sig_refs[0].sig = "ARLHS"
+                                        spot.sig_refs[0].sig = "ARLHS"
-                                    spot.sig = "ARLHS"
+                                        spot.sig = "ARLHS"
-                                case "Castle":
+                                    case "Castle":
-                                    spot.sig_refs[0].sig = "WCA"
+                                        spot.sig_refs[0].sig = "WCA"
-                                    spot.sig = "WCA"
+                                        spot.sig = "WCA"
-                                case "Mill":
+                                    case "Mill":
-                                    spot.sig_refs[0].sig = "MOTA"
+                                        spot.sig_refs[0].sig = "MOTA"
-                                    spot.sig = "MOTA"
+                                        spot.sig = "MOTA"
-                                case _:
+                                    case _:
-                                    logging.warning("GMA spot found with ref type " + ref_info[
+                                        logging.warning("GMA spot found with ref type " + ref_info[
-                                        "reftype"] + ", developer needs to add support for this!")
+                                            "reftype"] + ", developer needs to add support for this!")
-                                    spot.sig_refs[0].sig = ref_info["reftype"]
+                                        spot.sig_refs[0].sig = ref_info["reftype"]
-                                    spot.sig = ref_info["reftype"]
+                                        spot.sig = ref_info["reftype"]
 
-                            # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
+                                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
-                            # that for us.
+                                # that for us.
-                            new_spots.append(spot)
+                                new_spots.append(spot)
-                except:
+                    except:
-                    logging.warning("Exception when looking up " + self.REF_INFO_URL_ROOT + source_spot[
+                        logging.warning("Exception when looking up " + self.REF_INFO_URL_ROOT + source_spot[
-                        "REF"] + ", ignoring this spot for now")
+                            "REF"] + ", ignoring this spot for now")
         return new_spots

templates/about.html

@@ -13,7 +13,7 @@
             href="/apidocs">API documentation</a>. The API delivers spots in a consistent format regardless of the data
         source, freeing developers from needing to know how each individual data source presents its data.</p>
     <p>Spothole itself is also open source, Public Domain licenced code that anyone can take and modify. <a
-            href="https://git.ianrenton.com/ian/metaspot/">The source code is here</a>.</p>
+            href="https://git.ianrenton.com/ian/spothole/">The source code is here</a>.</p>
     <p>The software was written by <a href="https://ianrenton.com">Ian Renton, MØTRT</a> and other contributors. Full
         details are available in the <a href="https://git.ianrenton.com/ian/spothole/src/branch/main/README.md">README
             file</a>.</p>

templates/add_spot.html

@@ -76,7 +76,7 @@
 
 </div>
 
-<script src="/js/add-spot.js?v=1781901371"></script>
+<script src="/js/add-spot.js?v=1781955606"></script>
 <script>$(document).ready(function () {
     $("#nav-link-add-spot").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/alerts.html

@@ -75,7 +75,7 @@
 
 </div>
 
-<script src="/js/alerts.js?v=1781901371"></script>
+<script src="/js/alerts.js?v=1781955606"></script>
 <script>$(document).ready(function () {
     $("#nav-link-alerts").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/bands.html

@@ -75,13 +75,10 @@
 </div>
 
 <script>
-    let spotProvidersEnabledByDefault = { % raw
+    let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
-    json_encode(web_ui_options["spot-providers-enabled-by-default"]) %
-    }
-    ;
 </script>
-<script src="/js/spotsbandsandmap.js?v=1781901371"></script>
+<script src="/js/spotsbandsandmap.js?v=1781955606"></script>
-<script src="/js/bands.js?v=1781901371"></script>
+<script src="/js/bands.js?v=1781955606"></script>
 <script>$(document).ready(function () {
     $("#nav-link-bands").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/base.html

@@ -1,6 +1,6 @@
 {% extends "skeleton.html" %}
 {% block head_extra %}
-<link rel="stylesheet" href="/css/style.css?v=1781901371" type="text/css">
+<link rel="stylesheet" href="/css/style.css?v=1781955606" type="text/css">
 <link href="/vendor/css/bootstrap-5.3.8.min.css" rel="stylesheet">
 <link href="/vendor/css/fontawesome-6.7.2.min.css" rel="stylesheet">
 <link href="/vendor/css/solid-6.7.2.min.css" rel="stylesheet">
@@ -10,10 +10,10 @@
 <script src="/vendor/js/bootstrap-5.3.8.bundle.min.js"></script>
 <script src="/vendor/js/tinycolor2-1.6.0.min.js"></script>
 
-<script src="/js/utils.js?v=1781901371"></script>
+<script src="/js/utils.js?v=1781955606"></script>
-<script src="/js/ui-ham.js?v=1781901371"></script>
+<script src="/js/ui-ham.js?v=1781955606"></script>
-<script src="/js/geo.js?v=1781901371"></script>
+<script src="/js/geo.js?v=1781955606"></script>
-<script src="/js/common.js?v=1781901371"></script>
+<script src="/js/common.js?v=1781955606"></script>
 {% end %}
 {% block body %}
 <div class="container">

templates/conditions.html

@@ -284,7 +284,7 @@
 </div>
 
 <script src="/vendor/js/chart-4.4.9.umd.min.js"></script>
-<script src="/js/conditions.js?v=1781901371"></script>
+<script src="/js/conditions.js?v=1781955606"></script>
 <script>$(document).ready(function () {
     $("#nav-link-conditions").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/map.html

@@ -93,13 +93,10 @@
 <script src="/vendor/js/leaflet-workedallbritainireland.js"></script>
 
 <script>
-    let spotProvidersEnabledByDefault = { % raw
+    let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
-    json_encode(web_ui_options["spot-providers-enabled-by-default"]) %
-    }
-    ;
 </script>
-<script src="/js/spotsbandsandmap.js?v=1781901371"></script>
+<script src="/js/spotsbandsandmap.js?v=1781955606"></script>
-<script src="/js/map.js?v=1781901371"></script>
+<script src="/js/map.js?v=1781955606"></script>
 <script>$(document).ready(function () {
     $("#nav-link-map").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/spots.html

@@ -114,13 +114,10 @@
 </div>
 
 <script>
-    let spotProvidersEnabledByDefault = { % raw
+    let spotProvidersEnabledByDefault = {% raw json_encode(web_ui_options["spot-providers-enabled-by-default"]) %};
-    json_encode(web_ui_options["spot-providers-enabled-by-default"]) %
-    }
-    ;
 </script>
-<script src="/js/spotsbandsandmap.js?v=1781901371"></script>
+<script src="/js/spotsbandsandmap.js?v=1781955606"></script>
-<script src="/js/spots.js?v=1781901371"></script>
+<script src="/js/spots.js?v=1781955606"></script>
 <script>$(document).ready(function () {
     $("#nav-link-spots").addClass("active");
 }); <!-- highlight active page in nav --></script>

templates/status.html

@@ -59,7 +59,7 @@
     </div>
 </div>
 
-<script src="/js/status.js?v=1781901371"></script>
+<script src="/js/status.js?v=1781955606"></script>
 <script>
     $(document).ready(function () {
         $("#nav-link-status").addClass("active");