Structure changes to match other projects and minor logging improvements

.idea/spothole.iml

@@ -4,7 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
       <excludeFolder url="file://$MODULE_DIR$/.venv" />
-      <excludeFolder url="file://$MODULE_DIR$/webassets/vendor" />
+      <excludeFolder url="file://$MODULE_DIR$/static/vendor" />
     </content>
     <orderEntry type="jdk" jdkName="Python 3.13 virtualenv at ~/code/spothole/.venv" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />

README.md

@@ -1,4 +1,4 @@
-# ![Spothole](/webassets/img/logo.png)
+# ![Spothole](/static/img/logo.png)
 
 Spothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open
 JSON API as well as a website to browse the data.
@@ -89,7 +89,7 @@ callsign and park data.
 
 Various approaches exist to writing your own client, but in general:
 
-* Refer to the API docs. These are built on an OpenAPI definition file (`/webassets/apidocs/openapi.yml`), which you can
+* Refer to the API docs. These are built on an OpenAPI definition file (`/static/apidocs/openapi.yml`), which you can
   automatically use to generate a client skeleton using various software.
 * Call the main "spots" or "alerts" API endpoints to get the data you want. For example, your app could call
   `https://spothole.app/api/v1/spots` once every few minutes. Apply filters if necessary.
@@ -346,9 +346,9 @@ server {
         alias /var/www/html/.well-known/;
     }
 
-    # Load static assets directly from the Spothole webassets directory
+    # Load static assets directly from the Spothole static directory
     location /static/ {
-        alias /home/spothole/spothole/webassets/;
+        alias /home/spothole/spothole/static/;
         expires 1h;
         add_header Cache-Control "public, max-age=3600, must-revalidate";
     }
@@ -452,12 +452,12 @@ To navigate your way around the source code, this list may help.
 
 *HTML/JS/CSS front-end code*
 
-* `/webassets` - Root for static files served by the web server. These are all served from a path starting `/static/`.
-* `/webassets/apidocs` - Contains the OpenAPI spec (`openapi.yml`)
-* `/webassets/css` - CSS files used by the web front-end
-* `/webassets/img` - image files used by the web front-end
-* `/webassets/js` - JavaScript used by the web front-end
-* `/webassets/vendor` - Third-party libraries (CSS, JS, fonts and images)
+* `/static` - Root for static files served by the web server. These are all served from a path starting `/static/`.
+* `/static/apidocs` - Contains the OpenAPI spec (`openapi.yml`)
+* `/static/css` - CSS files used by the web front-end
+* `/static/img` - image files used by the web front-end
+* `/static/js` - JavaScript used by the web front-end
+* `/static/vendor` - Third-party libraries (CSS, JS, fonts and images)
 
 *Miscellaneous*
 
@@ -509,14 +509,14 @@ my knowledge, created by HA8TKS for his CQ and ITU zone layers for Leaflet. `/da
 maps of conference centres.
 
 The project contains a set of flag icons generated using the "Noto Color Emoji" font on a Debian system, in the
-`/webassets/img/flags/` directory.
+`/static/img/flags/` directory.
 
 The software uses a number of Python libraries as listed in `requirements.txt`, and a number of JavaScript libraries.
 This project would not have been possible without these libraries, so many thanks to their developers.
 
 ### Third Party Libraries
 
-A number of third-party libraries are self-hosted in the `/webassets/vendor/` directory. These files are subject to
+A number of third-party libraries are self-hosted in the `/static/vendor/` directory. These files are subject to
 their own licences and are not covered by the overall licence declared in the `LICENSE` file.
 
 Particular thanks go to country-files.com for providing country lookup data for amateur radio, to K0SWE

alertproviders/http_alert_provider.py

@@ -4,7 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError
+from requests.exceptions import ConnectionError, ReadTimeout
 
 from alertproviders.alert_provider import AlertProvider
 from core.constants import HTTP_HEADERS
@@ -59,6 +59,8 @@ class HTTPAlertProvider(AlertProvider):
 
         except ConnectionError:
             logging.warning(f"Connection error when accessing {self.name} alerts API.")
+        except ReadTimeout:
+            logging.warning(f"Read timeout when accessing {self.name} alerts API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP JSON Alert Provider (" + self.name + ")")

core/lookup_helper.py

@@ -11,7 +11,7 @@ from pyhamtools import LookupLib, Callinfo, callinfo
 from pyhamtools.exceptions import APIKeyMissingError
 from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
-from requests.exceptions import ConnectionError
+from requests.exceptions import ConnectionError, ReadTimeout
 from requests_cache import CachedSession
 
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
@@ -156,6 +156,8 @@ class LookupHelper:
 
         except ConnectionError:
             logging.warning(f"Connection error when downloading Clublog cty.xml.")
+        except ReadTimeout:
+            logging.warning(f"Read timeout when downloading Clublog cty.xml.")
         except Exception as e:
             logging.error("Exception when downloading Clublog cty.xml", e)
             return False
@@ -180,6 +182,8 @@ class LookupHelper:
 
         except ConnectionError:
             logging.warning(f"Connection error when downloading dxcc.json.")
+        except ReadTimeout:
+            logging.warning(f"Read timeout when downloading dxcc.json.")
         except Exception as e:
             logging.error("Exception when downloading dxcc.json", e)
             return False
@@ -537,6 +541,9 @@ class LookupHelper:
             except ConnectionError:
                 logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call)
                 continue
+            except ReadTimeout:
+                logging.warning(f"Read timeout when looking up callsign %s using QRZ.", lookup_call)
+                continue
             except Exception:
                 logging.error("Exception when looking up callsign %s using QRZ", lookup_call, exc_info=True)
                 continue
@@ -601,6 +608,9 @@ class LookupHelper:
             except ConnectionError:
                 logging.warning(f"Connection error when looking up callsign %s using HamQTH", lookup_call)
                 continue
+            except ReadTimeout:
+                logging.warning(f"Read timeout when looking up callsign %s using HamQTH", lookup_call)
+                continue
             except Exception:
                 logging.error("Exception when looking up callsign %s using HamQTH", lookup_call, exc_info=True)
                 continue

core/sig_utils.py

@@ -2,7 +2,7 @@ import csv
 import logging
 
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
-from requests.exceptions import ConnectionError
+from requests.exceptions import ConnectionError, ReadTimeout
 
 from core.cache_utils import SEMI_STATIC_URL_DATA_CACHE
 from core.constants import SIGS, HTTP_HEADERS
@@ -271,6 +271,8 @@ def populate_sig_ref_info(sig_ref):
 
     except ConnectionError:
         logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
+    except ReadTimeout:
+        logging.warning(f"Read timeout when looking up sig_ref info for " + sig + " ref " + ref_id)
     except Exception:
         logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
     return sig_ref

server/handlers/quietstaticfilehandler.py

@@ -1,16 +0,0 @@
-import logging
-
-from tornado.web import StaticFileHandler, HTTPError
-
-
-class QuietStaticFileHandler(StaticFileHandler):
-    """Minor override of logging in StaticFileHandler to log HTTP errors at debug level instead of their usual
-    warning level. Without this, attacks on Spothole which try to do path traversal attacks would log exceptions
-    from inside Tornado, and the server logs would contain a lot of this type of content. This effectively changes
-    the log level of these exceptions to DEBUG so they are only logged if DEBUG level logging is enabled."""
-
-    def log_exception(self, typ, value, tb):
-        if isinstance(value, HTTPError):
-            logging.debug(value)
-            return
-        super().log_exception(typ, value, tb)

server/webserver.py

@@ -18,7 +18,6 @@ from server.handlers.api.status import APIStatusHandler
 from server.handlers.manifesthandler import ManifestHandler
 from server.handlers.metrics import PrometheusMetricsHandler
 from server.handlers.pagetemplate import PageTemplateHandler
-from server.handlers.quietstaticfilehandler import QuietStaticFileHandler
 
 _HERE = os.path.dirname(__file__ or "")
 
@@ -101,12 +100,16 @@ class WebServer:
             if ALLOW_SPOTTING:
                 ui_routes += [(r"/add-spot", PageTemplateHandler, {"template_name": "add_spot", **handler_opts})]
 
-        # API docs, Prometheus metrics, webapp manifest and static assets are always available regardless of API-only mode.
+        # API docs, Prometheus metrics, webapp manifest and static assets are always available regardless of API-only
+        # mode.
         misc_routes = [
             (r"/apidocs", PageTemplateHandler, {"template_name": "apidocs", **handler_opts}),
             (r"/metrics", PrometheusMetricsHandler),
             (r"/manifest.webmanifest", ManifestHandler),
-            (r"/static/(.*)", QuietStaticFileHandler, {"path": os.path.join(_HERE, "../webassets")})
+            # If e.g. nginx is configured as a reverse proxy with a hard-coded path to static files, as per the README,
+            # this will never have to handle anything, but having it here allows Spothole to work without nginx for
+            # testing.
+            (r"/static/(.*)", StaticFileHandler, {"path": os.path.join(_HERE, "../static")})
         ]
 
         app = tornado.web.Application(api_routes + ui_routes + misc_routes,

solarconditionsproviders/giroionosonde.py

@@ -5,7 +5,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError
+from requests.exceptions import ConnectionError, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.ionosonde_utils import compute_band_states
@@ -134,6 +134,9 @@ class GIROIonosonde(SolarConditionsProvider):
                 logging.warning(f"HTTP {http_response.status_code} when calling Giro ionosonde API.")
                 return None, None, None
             return self._parse_all(http_response.text)
+        except ReadTimeout:
+            logging.warning(f"Read timeout when accessing Giro ionosonde API.")
+            return None, None, None
         except ConnectionError:
             logging.warning("Connection error when accessing Giro ionosonde API.")
             return None, None, None

solarconditionsproviders/http_solar_conditions_provider.py

@@ -4,7 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError
+from requests.exceptions import ConnectionError, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.solar_conditions_provider import SolarConditionsProvider
@@ -54,6 +54,8 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
 
         except ConnectionError:
             logging.warning(f"Connection error when accessing {self.name} solar conditions API.")
+        except ReadTimeout:
+            logging.warning(f"Read timeout when accessing {self.name} solar conditions API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP Solar Conditions Provider (" + self.name + ")")

solarconditionsproviders/kc2gprop.py

@@ -4,7 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError
+from requests.exceptions import ConnectionError, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from solarconditionsproviders.ionosonde_utils import compute_band_states
@@ -118,6 +118,8 @@ class KC2GProp(SolarConditionsProvider):
 
         except ConnectionError:
             logging.warning("Connection error when accessing KC2G ionosonde API.")
+        except ReadTimeout:
+            logging.warning(f"Read timeout when accessing KC2G ionosonde API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in KC2G ionosonde data provider")

spotproviders/hema.py

@@ -4,7 +4,7 @@ from datetime import datetime
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError
+from requests.exceptions import ConnectionError, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
@@ -68,6 +68,8 @@ class HEMA(HTTPSpotProvider):
                         # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other
                         # code will do that for us.
                         new_spots.append(spot)
+            except ReadTimeout:
+                logging.warning(f"Read timeout when accessing HEMA spots API.")
             except ConnectionError:
                 logging.warning("Connection error when accessing HEMA spots API.")
         return new_spots

spotproviders/http_spot_provider.py

@@ -4,6 +4,7 @@ from threading import Thread, Event
 
 import pytz
 import requests
+from requests import ReadTimeout
 from requests.exceptions import ConnectionError
 
 from core.constants import HTTP_HEADERS
@@ -59,6 +60,8 @@ class HTTPSpotProvider(SpotProvider):
 
         except ConnectionError:
             logging.warning(f"Connection error when accessing {self.name} spots API.")
+        except ReadTimeout:
+            logging.warning(f"Read timeout when accessing {self.name} spots API.")
         except Exception:
             self.status = "Error"
             logging.exception("Exception in HTTP JSON Spot Provider (" + self.name + ")")

spotproviders/sota.py

@@ -2,7 +2,7 @@ import logging
 from datetime import datetime
 
 import requests
-from requests.exceptions import ConnectionError
+from requests.exceptions import ConnectionError, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
@@ -60,4 +60,6 @@ class SOTA(HTTPSpotProvider):
                     new_spots.append(spot)
             except ConnectionError:
                 logging.warning("Connection error when accessing SOTA spots API")
+            except ReadTimeout:
+                logging.warning(f"Read timeout when accessing SOTA spots API.")
         return new_spots