Release v2.0.2

# Conflicts:
#	core/constants.py
#	pyproject.toml
#	templates/add_spot.html
#	templates/alerts.html
#	templates/bands.html
#	templates/base.html
#	templates/conditions.html
#	templates/map.html
#	templates/spots.html
#	templates/status.html

.idea/Spothole.iml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
+<module external.system.id="pyproject.toml" type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$">
       <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />

.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
+  </state>
+</component>

.idea/inspectionProfiles/Project_Default.xml

@@ -3,6 +3,7 @@
     <option name="myName" value="Project Default" />
     <inspection_tool class="Annotator" enabled="false" level="ERROR" enabled_by_default="false" />
     <inspection_tool class="BadExpressionStatementJS" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
+    <inspection_tool class="CheckImageSize" enabled="false" level="WARNING" enabled_by_default="false" />
     <inspection_tool class="CssOverwrittenProperties" enabled="false" level="WARNING" enabled_by_default="false" />
     <inspection_tool class="CssUnresolvedCustomProperty" enabled="false" level="ERROR" enabled_by_default="false" />
     <inspection_tool class="CssUnusedSymbol" enabled="false" level="WARNING" enabled_by_default="false" />
@@ -43,6 +44,27 @@
     <inspection_tool class="LanguageDetectionInspection" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
     <inspection_tool class="OutdatedRequirementInspection" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
     <inspection_tool class="PyBroadExceptionInspection" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
+    <inspection_tool class="PyCompatibilityInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ourVersions">
+        <value>
+          <list size="6">
+            <item index="0" class="java.lang.String" itemvalue="3.15" />
+            <item index="1" class="java.lang.String" itemvalue="3.14" />
+            <item index="2" class="java.lang.String" itemvalue="3.13" />
+            <item index="3" class="java.lang.String" itemvalue="3.12" />
+            <item index="4" class="java.lang.String" itemvalue="3.11" />
+            <item index="5" class="java.lang.String" itemvalue="3.10" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyStringConversionWithoutDunderMethodInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredTypes">
+        <list>
+          <option value="types.NoneType" />
+        </list>
+      </option>
+    </inspection_tool>
     <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
       <option name="processCode" value="true" />
       <option name="processLiterals" value="true" />

.idea/modules.xml

@@ -2,7 +2,7 @@
 <project version="4">
   <component name="ProjectModuleManager">
     <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/spothole.iml" filepath="$PROJECT_DIR$/.idea/spothole.iml" />
+      <module fileurl="file://$PROJECT_DIR$/.idea/Spothole.iml" filepath="$PROJECT_DIR$/.idea/Spothole.iml" />
     </modules>
   </component>
 </project>

.idea/pyLspTools.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PyToolsState">
+    <option name="tools">
+      <map>
+        <entry key="ruff">
+          <value>
+            <ToolEntry>
+              <option name="enabled" value="true" />
+            </ToolEntry>
+          </value>
+        </entry>
+      </map>
+    </option>
+  </component>
+</project>

.idea/runConfigurations/Run.xml

@@ -1,6 +1,6 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="Run" type="PythonConfigurationType" factoryName="Python">
-    <module name="spothole" />
+    <module name="Spothole" />
     <option name="ENV_FILES" value="" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />
@@ -12,6 +12,8 @@
     <option name="IS_MODULE_SDK" value="true" />
     <option name="ADD_CONTENT_ROOTS" value="true" />
     <option name="ADD_SOURCE_ROOTS" value="true" />
+    <option name="DEBUG_JUST_MY_CODE" value="true" />
+    <option name="RUN_TOOL" value="" />
     <option name="SCRIPT_NAME" value="$PROJECT_DIR$/spothole.py" />
     <option name="PARAMETERS" value="" />
     <option name="SHOW_COMMAND_LINE" value="false" />

README.md

@@ -34,18 +34,20 @@ see [https://spothole.app/apidocs](https://spothole.app/apidocs) for the API det
 This is a Progressive Web App, so you can also "install" it to your Android or iOS device by accessing it in Chrome or
 Safari respectively, and following the menu-driven process for installing PWAs.
 
-You are more than welcome to use the data and the API that Spothole provides to power your own software, to run your
-own Spothole server, or to modify it in any way. More details can be found in the following sections:
+You are more than welcome to use the data and the API that Spothole provides to power your own software, to run your own
+Spothole server, or to modify it in any way. More details can be found in the following sections:
 
 * [Embedding Spothole in another website](docs/embedding.md)
 * [Writing your own client](docs/clients.md)
 * [Running your own copy](docs/running.md)
-  * [nginx reverse proxy configuration](docs/nginx.md)
-  * [systemd service file](docs/systemd.md)
-  * [Running in Docker](docs/docker.md)
-  * [Use of multiple cluster logins](docs/multicluster.md)
+    * [nginx reverse proxy configuration](docs/nginx.md)
+    * [systemd service file](docs/systemd.md)
+    * [Running in Docker](docs/docker.md)
+    * [Use of multiple cluster logins](docs/multicluster.md)
 * [Modifying the source code](docs/modifying.md)
 
+![Screenshot](/images/screenshot4.png)
+
 ## Thanks
 
 As well as being my work, I have also gratefully received feature patches from Steven, M1SDH.
@@ -54,8 +56,8 @@ The project contains GeoJSON files for CQ and ITU zones, in the `/datafiles/` di
 my knowledge, created by HA8TKS for his CQ and ITU zone layers for Leaflet. `/datafiles` also contains a
 `MUNICIPIOS.csv` file, from the "Nomenclátor Geográfico de Municipios y Entidades de Población" data set sourced from
 [el Centro Nacional de Información Geográfica](https://centrodedescargas.cnig.es/CentroDescargas/home).
-`didbase-stations.csv` and the TOTA file were created by me based on publicly available data from GIRO and from
-maps of conference centres.
+`didbase-stations.csv` and the TOTA file were created by me based on publicly available data from GIRO and from maps of
+conference centres.
 
 The project contains a set of flag icons generated using the "Noto Color Emoji" font on a Debian system, in the
 `/static/img/flags/` directory.
@@ -65,12 +67,14 @@ This project would not have been possible without these libraries, so many thank
 
 ### Third Party Libraries
 
-A number of third-party libraries are self-hosted in the `/static/vendor/` directory. These files are subject to
-their own licences and are not covered by the overall licence declared in the `LICENSE` file.
+A number of third-party libraries are self-hosted in the `/static/vendor/` directory. These files are subject to their
+own licences and are not covered by the overall licence declared in the `LICENSE` file.
 
 A number of third-party libraries are self-hosted in the `/webassets/vendor/` directory. These files are subject to
 their own licences and are not covered by the overall licence declared in the `LICENSE` file.
 
+Sound effects are CC0-licenced and sourced from [Pixabay](https://pixabay.com).
+
 Particular thanks go to country-files.com for providing country lookup data for amateur radio, to K0SWE
 for [this JSON-formatted DXCC data](https://github.com/k0swe/dxcc-json/), and to the developers of `pyhamtools` for
 making it easy to use country-files.com data as well as QRZ.com and Clublog lookup.

config-example.yml

@@ -67,7 +67,7 @@ spot_providers:
 
   - class: "DXCluster"
     # Clusters have a "name" property in case you want to say which cluster node the data is from, or connect to
-    # several clusters and name them separately.
+    # several clusters and name them separately. The name will be used as the "source" field for the spots received.
     name: "HRD Cluster"
     enabled: true
     host: "hrd.wa9pie.net"
@@ -88,7 +88,7 @@ spot_providers:
     enabled: false
     host: "w3lpl.net"
     port: 7373
-    login_prompt: "Please enter your call:"
+    login_prompt: "login:"
     login_callsign: "N0CALL-99"
     allow_rbn_spots: false
 
@@ -345,4 +345,6 @@ web_ui_options:
   # (home) page, although being arbitrary HTML you can also use a div with absolute, relative, float placement etc. This
   # is designed for a "donate/support the server" type button, though you are free to do whatever you want with it.
   # As the server owner you are responsible for the safe usage of this option!
-  support_button_html: ""
+  support_button_html: ""
+  # CartoDB API key. Required to use the CartoDB layers without watermark. Get one from https://carto.com/basemaps/apikey/.
+  cartodb_api_key: ""

core/call_lookup_helper.py

@@ -1,3 +1,5 @@
+import re
+
 from core.data_providers import DATA_PROVIDERS
 from data.callsign import Callsign
 
@@ -9,10 +11,11 @@ def get_call_info(callsign, lookup_credentials):
 
     callsign_data = Callsign(call=callsign)
 
-    if callsign:
+    # First check our input looks like a real callsign
+    if callsign and re.match(r"^[A-Za-z0-9/\-]*$", callsign):
         # Sort callsign providers by priority order, so we query the highest priority (lowest numbers) first, and only
         # query other providers for data we are missing as we go along.
-        for p in sorted(DATA_PROVIDERS.callsign_data_providers, key=lambda p: p.priority):
+        for p in sorted(DATA_PROVIDERS.callsign_data_providers, key=lambda p2: p2.priority):
             if p.enabled:
                 # Get new lookup data
                 data = p.lookup(callsign, lookup_credentials)
@@ -27,4 +30,4 @@ def get_call_info(callsign, lookup_credentials):
                     if callsign_data.fully_populated():
                         break
 
-    return callsign_data
+    return callsign_data

core/cleanup.py

@@ -0,0 +1,73 @@
+import logging
+from datetime import datetime
+from threading import Event, Thread
+
+import pytz
+
+from core.data_store import DATA_STORE
+
+logger = logging.getLogger(__name__)
+
+
+class CleanupTimer:
+    """Provides a timed cleanup of the spot list."""
+
+    def __init__(self):
+        """Constructor"""
+
+        self._cleanup_interval = None
+        self.last_cleanup_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Starting"
+        self._thread = None
+        self._stop_event = Event()
+
+    def setup(self, cleanup_interval):
+        self._cleanup_interval = cleanup_interval
+
+    def start(self):
+        """Start the cleanup timer"""
+
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown"""
+
+        self._stop_event.set()
+
+    def _run(self):
+        while not self._stop_event.wait(timeout=self._cleanup_interval):
+            self._cleanup()
+
+    def _cleanup(self):
+        """Perform cleanup and reschedule next timer"""
+
+        try:
+            for i in list(DATA_STORE.spots.keys()):
+                try:
+                    spot = DATA_STORE.spots.get(i)
+                    if spot and spot.expired():
+                        DATA_STORE.spots.delete(i)
+                except KeyError:
+                    # Must have already been deleted, OK with that
+                    pass
+            for i in list(DATA_STORE.alerts.keys()):
+                try:
+                    alert = DATA_STORE.alerts.get(i)
+                    if alert and alert.expired():
+                        DATA_STORE.alerts.delete(i)
+                except KeyError:
+                    # Must have already been deleted, OK with that
+                    pass
+
+            self.status = "OK"
+            self.last_cleanup_time = datetime.now(pytz.UTC)
+
+        except Exception:
+            self.status = "Error"
+            logger.exception("Exception in Cleanup thread")
+            self._stop_event.wait(timeout=1)
+
+
+# Global object
+CLEANUP_TIMER = CleanupTimer()

core/config.py

@@ -1,19 +1,23 @@
 import importlib
 import logging
 import os
+import sys
 
 import yaml
 
+logger = logging.getLogger(__name__)
+
 # Check you have a config file
 if not os.path.isfile("config.yml"):
-    logging.error(
-        "Your config file is missing. Ensure you have copied config-example.yml to config.yml and updated it according to your needs.")
-    exit()
+    logger.error(
+        "Your config file is missing. Ensure you have copied config-example.yml to config.yml and updated it according to your needs."
+    )
+    sys.exit()
 
 # Load config
 with open("config.yml") as f:
     config = yaml.safe_load(f)
-logging.info("Loaded config.")
+logger.info("Loaded config.")
 
 BASE_URL = config.get("base_url", "http://localhost:8080")
 MAX_SPOT_AGE = config.get("max_spot_age_sec", 3600)
@@ -30,7 +34,9 @@ LOG_LEVEL = config.get("log_level", "INFO")
 LOG_WEB_REQUESTS = config.get("log_web_requests", False)
 
 WEB_UI_OPTIONS["qrz_enabled"] = any(p["class"] == "QRZ" and p["enabled"] for p in config["callsign_data_providers"])
-WEB_UI_OPTIONS["hamqth_enabled"] = any(p["class"] == "HamQTH" and p["enabled"] for p in config["callsign_data_providers"])
+WEB_UI_OPTIONS["hamqth_enabled"] = any(
+    p["class"] == "HamQTH" and p["enabled"] for p in config["callsign_data_providers"]
+)
 WEB_UI_OPTIONS["recaptcha_site_key"] = RECAPTCHA_SITE_KEY
 WEB_UI_OPTIONS["allow_upstream_spotting"] = ALLOW_SPOTTING and ALLOW_UPSTREAM_SPOTTING
 
@@ -40,6 +46,6 @@ def create_provider_from_config(package, config_providers_entry):
     package to look for it in, as there are several types of provider. e.g. package "providers.spot", where the config
     entry is for a POTA spot provider."""
 
-    module = importlib.import_module(package + "." + config_providers_entry["class"].lower())
+    module = importlib.import_module(f"{package}.{config_providers_entry['class'].lower()}")
     provider_class = getattr(module, config_providers_entry["class"])
     return provider_class(config_providers_entry)

core/constants.py

@@ -3,44 +3,191 @@ from data.band import Band
 from data.sig import SIG
 
 # General software
-SOFTWARE_VERSION = "2.0-pre"
+SOFTWARE_VERSION = "2.0.2"
 
 # HTTP headers used for spot providers that use HTTP
-HTTP_HEADERS = {"User-Agent": "Spothole v" + SOFTWARE_VERSION + " (operated by " + SERVER_OWNER_CALLSIGN + ")"}
-HAMQTH_PRG = ("Spothole v" + SOFTWARE_VERSION + " operated by " + SERVER_OWNER_CALLSIGN).replace(" ", "_")
+HTTP_HEADERS = {"User-Agent": f"Spothole v{SOFTWARE_VERSION} (operated by {SERVER_OWNER_CALLSIGN})"}
+HAMQTH_PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}".replace(" ", "_")
 
 # Special Interest Groups
 SIGS = [
-    SIG(name="POTA", comment_names=["POTA"], description="Parks on the Air", ref_regex=r"[A-Z]{2}\-\d{4,5}|K\-TEST"),
-    SIG(name="SOTA", comment_names=["SOTA"], description="Summits on the Air", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}"),
-    SIG(name="WWFF", comment_names=["WWFF"], description="World Wide Flora & Fauna", ref_regex=r"[A-Z0-9]{1,3}FF\-\d{4}"),
-    SIG(name="GMA", comment_names=["GMA"], description="Global Mountain Activity", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}"),
-    SIG(name="WWBOTA", comment_names=["WWBOTA", "BOTA"], description="Worldwide Bunkers on the Air", ref_regex=r"B\/[A-Z0-9]{1,3}\-\d{3,4}"),
-    SIG(name="HEMA", comment_names=["HEMA"], description="HuMPs Excluding Marilyns Award", ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}"),
-    SIG(name="IOTA", comment_names=["IOTA"], description="Islands on the Air", ref_regex=r"[A-Z]{2}\-\d{3}"),
-    SIG(name="MOTA", comment_names=["MOTA"], description="Mills on the Air", ref_regex=r"X\d{4,6}"),
-    SIG(name="ARLHS", comment_names=["ARLHS"], description="Amateur Radio Lighthouse Society", ref_regex=r"[A-Z]{3}[\- ]\d{3,4}"),
-    SIG(name="ILLW", comment_names=["ILLW"], description="International Lighthouse & Lightship Weekend", ref_regex=r"[A-Z]{2}\d{4}"),
-    SIG(name="SIOTA", comment_names=["SIOTA"], description="Silos on the Air", ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d"),
-    SIG(name="WCA", comment_names=["WCA"], description="World Castles Award", ref_regex=r"[A-Z0-9]{1,3}\-\d{5}"),
-    SIG(name="ZLOTA", comment_names=["ZLOTA"], description="New Zealand on the Air", ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}"),
-    SIG(name="WOTA", comment_names=["WOTA"], description="Wainwrights on the Air", ref_regex=r"[A-Z]{3}-[0-9]{2}"),
+    SIG(
+        name="POTA",
+        comment_names=["POTA"],
+        description="Parks on the Air",
+        ref_regex=r"[A-Z]{2}\-\d{4,5}|K\-TEST",
+    ),
+    SIG(
+        name="SOTA",
+        comment_names=["SOTA"],
+        description="Summits on the Air",
+        ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
+    ),
+    SIG(
+        name="WWFF",
+        comment_names=["WWFF"],
+        description="World Wide Flora & Fauna",
+        ref_regex=r"[A-Z0-9]{1,3}FF\-\d{4}",
+    ),
+    SIG(
+        name="GMA",
+        comment_names=["GMA"],
+        description="Global Mountain Activity",
+        ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
+    ),
+    SIG(
+        name="WWBOTA",
+        comment_names=["WWBOTA", "BOTA"],
+        description="Worldwide Bunkers on the Air",
+        ref_regex=r"B\/[A-Z0-9]{1,3}\-\d{3,4}",
+    ),
+    SIG(
+        name="HEMA",
+        comment_names=["HEMA"],
+        description="HuMPs Excluding Marilyns Award",
+        ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}",
+    ),
+    SIG(
+        name="IOTA",
+        comment_names=["IOTA"],
+        description="Islands on the Air",
+        ref_regex=r"[A-Z]{2}\-\d{3}",
+    ),
+    SIG(
+        name="GMA Island",
+        comment_names=[],
+        description="Global Mountain Activity - Islands",
+        ref_regex=r"(([A-Z]{2}\-\d{3})|([A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}))",
+    ),
+    SIG(
+        name="MOTA",
+        comment_names=["MOTA"],
+        description="Mills on the Air",
+        ref_regex=r"X\d{4,6}",
+    ),
+    SIG(
+        name="ARLHS",
+        comment_names=["ARLHS"],
+        description="Amateur Radio Lighthouse Society",
+        ref_regex=r"[A-Z]{3}[\- ]\d{3,4}",
+    ),
+    SIG(
+        name="ILLW",
+        comment_names=["ILLW"],
+        description="International Lighthouse & Lightship Weekend",
+        ref_regex=r"[A-Z]{2}\d{4}",
+    ),
+    SIG(
+        name="SIOTA",
+        comment_names=["SIOTA"],
+        description="Silos on the Air",
+        ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d",
+    ),
+    SIG(
+        name="WCA",
+        comment_names=["WCA"],
+        description="World Castles Award",
+        ref_regex=r"[A-Z0-9]{1,3}\-\d{5}",
+    ),
+    SIG(
+        name="ZLOTA",
+        comment_names=["ZLOTA"],
+        description="New Zealand on the Air",
+        ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}",
+    ),
+    SIG(
+        name="WOTA",
+        comment_names=["WOTA"],
+        description="Wainwrights on the Air",
+        ref_regex=r"[A-Z]{3}-[0-9]{2}",
+    ),
     SIG(name="BOTA", comment_names=[], description="Beaches on the Air"),
-    SIG(name="KRMNPA", comment_names=["KRMNPA"], description="Keith Roget Memorial National Parks Award", ref_regex=r"VKFF\-\d{4}"),
-    SIG(name="SANPCPA", comment_names=["SANPCPA"], description="South Australian National Parks and Conservation Parks Award", ref_regex=r"VKFF\-\d{4}"),
-    SIG(name="LLOTA", comment_names=["LLOTA"], description="Lagos y Lagunas on the Air", ref_regex=r"LL[A-Z]{2}\-\d{4}"),
-    SIG(name="Towers", comment_names=["TOTA"], description="Towers on the Air", ref_regex=r"[A-Z]{2,3}R\-\d{4}"),
-    SIG(name="Tiles", comment_names=[], description="Tiles on the Air", ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}"),
-    SIG(name="WAB", comment_names=["WAB"], description="Worked All Britain", ref_regex=r"[A-Z]{1,2}[0-9]{2}"),
-    SIG(name="WAI", comment_names=["WAI"], description="Worked All Ireland", ref_regex=r"[A-Z][0-9]{2}"),
-    SIG(name="DME", comment_names=["DME"], description="Diplomas de Municipios Españoles", ref_regex=r"\d{4,5}"),
-    SIG(name="Toilets", comment_names=[], description="Toilets on the Air", ref_regex=r"T\-[0-9]{2}")
+    SIG(
+        name="KRMNPA",
+        comment_names=["KRMNPA"],
+        description="Keith Roget Memorial National Parks Award",
+        ref_regex=r"VKFF\-\d{4}",
+    ),
+    SIG(
+        name="SANPCPA",
+        comment_names=["SANPCPA"],
+        description="South Australian National Parks and Conservation Parks Award",
+        ref_regex=r"VKFF\-\d{4}",
+    ),
+    SIG(
+        name="LLOTA",
+        comment_names=["LLOTA"],
+        description="Lagos y Lagunas on the Air",
+        ref_regex=r"LL[A-Z]{2}\-\d{4}",
+    ),
+    SIG(
+        name="Towers",
+        comment_names=["TOTA"],
+        description="Towers on the Air",
+        ref_regex=r"[A-Z]{2,3}R\-\d{4}",
+    ),
+    SIG(
+        name="Tiles",
+        comment_names=[],
+        description="Tiles on the Air",
+        ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}",
+    ),
+    SIG(
+        name="WAB",
+        comment_names=["WAB"],
+        description="Worked All Britain",
+        ref_regex=r"[A-Z]{1,2}[0-9]{2}",
+    ),
+    SIG(
+        name="WAI",
+        comment_names=["WAI"],
+        description="Worked All Ireland",
+        ref_regex=r"[A-Z][0-9]{2}",
+    ),
+    SIG(
+        name="DME",
+        comment_names=["DME"],
+        description="Diplomas de Municipios Españoles",
+        ref_regex=r"\d{4,5}",
+    ),
+    SIG(
+        name="Toilets",
+        comment_names=[],
+        description="Toilets on the Air",
+        ref_regex=r"T\-[0-9]{2}",
+    ),
 ]
 
 # Modes. Note "DIGI" and "DIGITAL" are also supported but are normalised into "DATA".
 CW_MODES = ["CW"]
-PHONE_MODES = ["PHONE", "SSB", "USB", "LSB", "AM", "FM", "DV", "DMR", "DSTAR", "C4FM", "FUSION", "M17"]
-DATA_MODES = ["DATA", "FT8", "FT4", "RTTY", "SSTV", "JS8", "HELL", "PSK", "OLIVIA", "PKT", "MSK144"]
+PHONE_MODES = [
+    "PHONE",
+    "SSB",
+    "USB",
+    "LSB",
+    "AM",
+    "FM",
+    "DV",
+    "DMR",
+    "DSTAR",
+    "C4FM",
+    "FUSION",
+    "M17",
+]
+DATA_MODES = [
+    "DATA",
+    "FT8",
+    "FT4",
+    "RTTY",
+    "SSTV",
+    "JS8",
+    "HELL",
+    "PSK",
+    "FSK",
+    "OLIVIA",
+    "PKT",
+    "MSK144",
+]
 ALL_MODES = CW_MODES + PHONE_MODES + DATA_MODES
 MODE_TYPES = ["CW", "PHONE", "DATA"]
 SSB_SUB_MODES = ["USB", "LSB"]
@@ -58,7 +205,7 @@ MODE_ALIASES = {
     "MFSK": "FSK",
     "MFSK32": "FSK",
     "DIGI": "DATA",
-    "DIGITAL": "DATA"
+    "DIGITAL": "DATA",
 }
 
 # Band definitions
@@ -88,7 +235,8 @@ BANDS = [
     Band(name="10GHz", start_freq=10000000000, end_freq=10500000000),
     Band(name="24GHz", start_freq=24000000000, end_freq=24050000000),
     Band(name="47GHz", start_freq=47000000000, end_freq=47200000000),
-    Band(name="76GHz", start_freq=75500000000, end_freq=81500000000)]
+    Band(name="76GHz", start_freq=75500000000, end_freq=81500000000),
+]
 UNKNOWN_BAND = Band(name="Unknown", start_freq=0, end_freq=0)
 
 # Continents
@@ -106,5 +254,5 @@ PROPAGATION_MODES = {
     "MS": "Meteor scatter",
     "RS": "Rain scatter",
     "AS": "Aircraft scatter",
-    "SAT": "Satellite"
+    "SAT": "Satellite",
 }

core/data_providers.py

@@ -3,6 +3,8 @@ import threading
 
 from core.config import config, create_provider_from_config
 
+logger = logging.getLogger(__name__)
+
 
 class DataProviders:
     """Global object for storing data providers."""
@@ -15,7 +17,6 @@ class DataProviders:
         self.sig_ref_data_providers = []
         self.callsign_data_providers = []
 
-
     def setup(self):
         for entry in config["spot_providers"]:
             self.spot_providers.append(create_provider_from_config("providers.spot", entry))
@@ -31,10 +32,10 @@ class DataProviders:
             self.callsign_data_providers.append(create_provider_from_config("providers.callsigndata", entry))
 
     @staticmethod
-    def start_providers(providers, type):
+    def start_providers(providers, provider_type):
         """Helper method to activate enabled providers in the list."""
 
-        logging.info(f"Starting %s providers...", type)
+        logger.info(f"Starting {provider_type} providers...")
         for p in providers:
             if p.enabled:
                 p.start()
@@ -43,11 +44,20 @@ class DataProviders:
         # Start data providers before spot/alert providers so the lookup data is there already for incoming spots.
         # Each category is fired off after a small delay to give the rest of Spothole chance to start up.
         threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")).start()
-        threading.Timer(10.0, lambda: self.start_providers(self.callsign_data_providers, "callsign data")).start()
+        threading.Timer(
+            10.0,
+            lambda: self.start_providers(self.callsign_data_providers, "callsign data"),
+        ).start()
         threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")).start()
         threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")).start()
-        threading.Timer(25.0, lambda: self.start_providers(self.solar_condition_providers, "solar condition")).start()
-        threading.Timer(30.0, lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data")).start()
+        threading.Timer(
+            25.0,
+            lambda: self.start_providers(self.solar_condition_providers, "solar condition"),
+        ).start()
+        threading.Timer(
+            30.0,
+            lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data"),
+        ).start()
 
     def stop(self):
         for sp in self.spot_providers:
@@ -69,5 +79,6 @@ class DataProviders:
             if cdp.enabled:
                 cdp.stop()
 
+
 # Global object
-DATA_PROVIDERS = DataProviders()
+DATA_PROVIDERS = DataProviders()

core/data_store.py

@@ -4,10 +4,12 @@ from pathlib import Path
 
 import diskcache
 
-from core.config import MAX_SPOT_AGE, MAX_ALERT_AGE
+from core.config import MAX_ALERT_AGE, MAX_SPOT_AGE
 from core.live_data_cache import LiveDataCache
 from data.solar_conditions import SolarConditions
 
+logger = logging.getLogger(__name__)
+
 CACHE_DIR = "./cache/"
 
 
@@ -46,52 +48,63 @@ class DataStore:
 
         # Standard disk cache for solar data and status data, but each cache contains only a single object which we
         # expose to the wider application
-        self._solar = diskcache.Cache(CACHE_DIR + "solar")
+        self._solar = diskcache.Cache(f"{CACHE_DIR}solar")
         if "solar_conditions" not in self._solar:
             self._solar.add("solar_conditions", SolarConditions())
         self.solar_conditions = self._solar.get("solar_conditions")
-        self._status = diskcache.Cache(CACHE_DIR + "status")
+        self._status = diskcache.Cache(f"{CACHE_DIR}status")
         if "status_data" not in self._status:
             self._status.add("status_data", {})
         self.status_data = self._status.get("status_data")
 
         # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
         # a regular basis but there's no need for a TTL since old data is better than no data.
-        self.dxcc_data = diskcache.Cache(CACHE_DIR + "dxcc_data")
+        self.dxcc_data = diskcache.Cache(f"{CACHE_DIR}dxcc_data")
         self.regenerate_call_regex_to_dxcc_entity_map()
 
         # For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers
         # of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref"
         # syntax for keys to keep it a single level.
-        self.sigrefs = diskcache.Cache(CACHE_DIR + "sigrefs")
-        logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
+        self.sigrefs = diskcache.Cache(f"{CACHE_DIR}sigrefs")
+        logger.info(f"Loaded data for {len(self.sigrefs)} SIG references.")
 
         # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
         # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
         # we've seen, so we rely on them timing out and this triggering another lookup.
-        self.callsign_data_countryfiles = diskcache.Cache(CACHE_DIR + "callsign_data_countryfiles")
-        self.callsign_data_clublogxml = diskcache.Cache(CACHE_DIR + "callsign_data_clublogxml")
-        self.callsign_data_clublogapi = diskcache.Cache(CACHE_DIR + "callsign_data_clublogapi")
-        self.callsign_data_qrz = diskcache.Cache(CACHE_DIR + "callsign_data_qrz")
-        self.callsign_data_hamqth = diskcache.Cache(CACHE_DIR + "callsign_data_hamqth")
+        self.callsign_data_countryfiles = diskcache.Cache(f"{CACHE_DIR}callsign_data_countryfiles")
+        self.callsign_data_clublogxml = diskcache.Cache(f"{CACHE_DIR}callsign_data_clublogxml")
+        self.callsign_data_clublogapi = diskcache.Cache(f"{CACHE_DIR}callsign_data_clublogapi")
+        self.callsign_data_qrz = diskcache.Cache(f"{CACHE_DIR}callsign_data_qrz")
+        self.callsign_data_hamqth = diskcache.Cache(f"{CACHE_DIR}callsign_data_hamqth")
         unique_keys = set()
-        for c in [self.callsign_data_countryfiles, self.callsign_data_clublogxml, self.callsign_data_clublogapi,
-                  self.callsign_data_qrz, self.callsign_data_hamqth]:
+        for c in [
+            self.callsign_data_countryfiles,
+            self.callsign_data_clublogxml,
+            self.callsign_data_clublogapi,
+            self.callsign_data_qrz,
+            self.callsign_data_hamqth,
+        ]:
             unique_keys.update(c)
-        logging.info(f"Loaded data for %d callsigns.", len(unique_keys))
+        logger.info(f"Loaded data for {len(unique_keys)} callsigns.")
 
         # Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
         # specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
         # before the spots and alerts are live in the system.
-        self.spots = LiveDataCache(maxsize=self._MAX_SPOT_COUNT, ttl=MAX_SPOT_AGE,
-                                   snapshot_dir=CACHE_DIR + "spots",
-                                   snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
-        logging.info(f"Loaded %d spots from a previous run.", len(self.spots.keys()))
+        self.spots = LiveDataCache(
+            maxsize=self._MAX_SPOT_COUNT,
+            ttl=MAX_SPOT_AGE,
+            snapshot_dir=f"{CACHE_DIR}spots",
+            snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC,
+        )
+        logger.info(f"Loaded {len(self.spots.keys())} spots from a previous run.")
 
-        self.alerts = LiveDataCache(maxsize=self._MAX_ALERT_COUNT, ttl=MAX_ALERT_AGE,
-                                    snapshot_dir=CACHE_DIR + "alerts",
-                                    snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC)
-        logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys()))
+        self.alerts = LiveDataCache(
+            maxsize=self._MAX_ALERT_COUNT,
+            ttl=MAX_ALERT_AGE,
+            snapshot_dir=f"{CACHE_DIR}alerts",
+            snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC,
+        )
+        logger.info(f"Loaded {len(self.alerts.keys())} alerts from a previous run.")
 
     def regenerate_call_regex_to_dxcc_entity_map(self):
         """DXCC entity data from K0SWE includes a regex which we can use to match a callsign, and determine which DXCC
@@ -103,6 +116,9 @@ class DataStore:
             self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"]))
 
     def close(self):
+        self.spots.save_snapshot()
+        self.alerts.save_snapshot()
+
         self.spots.close()
         self.alerts.close()
         self._solar.close()
@@ -115,5 +131,6 @@ class DataStore:
         self.callsign_data_qrz.close()
         self.callsign_data_hamqth.close()
 
+
 # Global object
-DATA_STORE = DataStore()
+DATA_STORE = DataStore()

core/geo_utils.py

@@ -7,6 +7,8 @@ from shapely.geometry import Point, Polygon
 
 from core.data_store import DATA_STORE
 
+logger = logging.getLogger(__name__)
+
 TRANSFORMER_OS_GRID_TO_WGS84 = Transformer.from_crs("EPSG:27700", "EPSG:4326")
 TRANSFORMER_IRISH_GRID_TO_WGS84 = Transformer.from_crs("EPSG:29903", "EPSG:4326")
 TRANSFORMER_CI_UTM_GRID_TO_WGS84 = Transformer.from_crs("+proj=utm +zone=30 +ellps=WGS84", "EPSG:4326")
@@ -71,7 +73,7 @@ def lat_lon_for_grid_sw_corner(grid):
     """Convert a Maidenhead grid reference of arbitrary precision to the lat/long of the southwest corner of the square.
     Returns None if the grid format is invalid."""
 
-    lat, lon, lat_cell_size, lon_cell_size = lat_lon_for_grid_sw_corner_plus_size(grid)
+    lat, lon, _lat_cell_size, _lon_cell_size = lat_lon_for_grid_sw_corner_plus_size(grid)
     if lat is not None and lon is not None:
         return [lat, lon]
     else:
@@ -113,8 +115,8 @@ def lat_lon_for_grid_sw_corner_plus_size(grid):
     while block * 2 < length:
         if block % 2 == 0:
             # Letters in this block
-            lon_cell_no = ord(grid[block * 2]) - ord('A')
-            lat_cell_no = ord(grid[block * 2 + 1]) - ord('A')
+            lon_cell_no = ord(grid[block * 2]) - ord("A")
+            lat_cell_no = ord(grid[block * 2 + 1]) - ord("A")
             # Bail if the values aren't in range. Allowed values are A-R (0-17) for the first letter block, or
             # A-X (0-23) thereafter.
             max_cell_no = 17 if block == 0 else 23
@@ -154,7 +156,7 @@ def lat_lon_for_grid_sw_corner_plus_size(grid):
     lat -= 90.0
 
     # Return None values on maths errors
-    if any(x != x for x in [lat, lon, lat_cell_size, lon_cell_size]):  # NaN check
+    if any(x != x for x in [lat, lon, lat_cell_size, lon_cell_size]):
         return None, None, None, None
 
     return lat, lon, lat_cell_size, lon_cell_size
@@ -172,7 +174,7 @@ def wab_wai_square_to_lat_lon(ref):
     elif re.match(r"^W[AV][0-9]{2}$", ref):
         return utm_grid_square_to_lat_lon(ref)
     else:
-        logging.warning("Invalid WAB/WAI square: " + ref)
+        logger.warning(f"Invalid WAB/WAI square: {ref}")
         return None
 
 

core/live_data_cache.py

@@ -5,6 +5,8 @@ import time
 import diskcache
 from cachetools import TTLCache
 
+logger = logging.getLogger(__name__)
+
 
 class LiveDataCache:
     """Cache for spots and alerts. Uses the fast in-memory TTLCache for normal data I/O, including the TTL to enforce
@@ -34,8 +36,7 @@ class LiveDataCache:
             try:
                 callback(value)
             except Exception:
-                logging.error("Listener raised an exception for key %s", key, exc_info=True)
-
+                logger.exception(f"Listener raised an exception for key {key}")
 
     def get(self, key, default=None):
         with self._lock:
@@ -70,8 +71,8 @@ class LiveDataCache:
             data = [(k, v, time.time()) for k, v in self._cache.items()]
         try:
             self._disk_cache.set("snapshot", data)
-        except Exception as e:
-            logging.error("Failed to write snapshot to %s", self._snapshot_dir, e, exc_info=True)
+        except Exception:
+            logger.exception(f"Failed to write snapshot to {self._snapshot_dir}")
 
     def _load_snapshot(self):
         data = self._disk_cache.get("snapshot")
@@ -84,7 +85,7 @@ class LiveDataCache:
                 # Only restore entries that would still be within TTL
                 if now - saved_at < self._ttl:
                     self._cache[key] = value
-        logging.info("Loaded snapshot from %s", self._snapshot_dir)
+        logger.info(f"Loaded snapshot from {self._snapshot_dir}")
 
     def _start_periodic_snapshot(self, interval):
         def loop():

core/prometheus_metrics_handler.py

@@ -1,4 +1,10 @@
-from prometheus_client import CollectorRegistry, generate_latest, Counter, disable_created_metrics, Gauge
+from prometheus_client import (
+    CollectorRegistry,
+    Counter,
+    Gauge,
+    disable_created_metrics,
+    generate_latest,
+)
 
 disable_created_metrics()
 # Prometheus metrics registry
@@ -9,25 +15,13 @@ page_requests_counter = Counter(
     "Total number of page requests received",
     registry=registry,
 )
-api_requests_counter = Counter(
-    "spothole_api_requests",
-    "Total number of API requests received",
-    registry=registry
-)
-spots_gauge = Gauge(
-    "spothole_spots",
-    "Number of spots currently in the software",
-    registry=registry
-)
-alerts_gauge = Gauge(
-    "spothole_alerts",
-    "Number of alerts currently in the software",
-    registry=registry
-)
+api_requests_counter = Counter("spothole_api_requests", "Total number of API requests received", registry=registry)
+spots_gauge = Gauge("spothole_spots", "Number of spots currently in the software", registry=registry)
+alerts_gauge = Gauge("spothole_alerts", "Number of alerts currently in the software", registry=registry)
 memory_use_gauge = Gauge(
     "spothole_memory_usage_bytes",
     "Current memory usage of the software in bytes",
-    registry=registry
+    registry=registry,
 )
 
 

core/sig_lookup_helper.py

@@ -1,11 +1,13 @@
 import logging
 
-from pyhamtools.locator import locator_to_latlong, latlong_to_locator
+from pyhamtools.locator import latlong_to_locator, locator_to_latlong
 
 from core.data_store import DATA_STORE
 from core.geo_utils import wab_wai_square_to_lat_lon
 from data.sig_ref import SIGRef
 
+logger = logging.getLogger(__name__)
+
 
 def get_sig_ref_info(sig, ref_id):
     """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig name and
@@ -14,7 +16,7 @@ def get_sig_ref_info(sig, ref_id):
     SIG we are getting data for."""
 
     if sig is None or sig == "" or ref_id is None or ref_id == "":
-        logging.debug("Failed to look up sig_ref info, sig or ref were not set.")
+        logger.debug("Failed to look up sig_ref info, sig or ref were not set.")
         return None
 
     sig_ref = SIGRef(sig=sig, id=ref_id)
@@ -29,6 +31,7 @@ def get_sig_ref_info(sig, ref_id):
 
         # If the SIG is HEMA, we have no current lookup for this so just skip the lookup here.
         if sig.upper() == "HEMA":
+            sig_ref.ref_type = "Summit"
             return sig_ref
 
         ### PROGRAMMATIC DATA GENERATION INSTEAD OF LOOKUPS ###
@@ -37,17 +40,19 @@ def get_sig_ref_info(sig, ref_id):
         # calculate all the information we are going to get directly.
         if sig.upper() == "TILES":
             # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
+            sig_ref.ref_type = "Grid"
             if not sig_ref.name:
                 sig_ref.name = sig_ref.id
             if not sig_ref.grid:
                 sig_ref.grid = sig_ref.id
-            if sig_ref.grid and not sig_ref.latitude:
+            if sig_ref.grid and (not sig_ref.latitude or not sig_ref.longitude):
                 ll = locator_to_latlong(str(sig_ref.grid))
                 sig_ref.latitude = ll[0]
                 sig_ref.longitude = ll[1]
             return sig_ref
 
         elif sig.upper() == "WAB" or sig.upper() == "WAI":
+            sig_ref.ref_type = "Grid"
             ll = wab_wai_square_to_lat_lon(ref_id)
             if ll:
                 sig_ref.name = ref_id
@@ -55,22 +60,38 @@ def get_sig_ref_info(sig, ref_id):
                     sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
                     sig_ref.latitude = ll[0]
                     sig_ref.longitude = ll[1]
-                except:
-                    logging.warning("Invalid lat/lon received for WAB/WAI reference")
+                except Exception:
+                    logger.warning("Invalid lat/lon received for WAB/WAI reference")
             return sig_ref
 
         elif sig.upper() == "BOTA":
             # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
+            sig_ref.ref_type = "Beach"
             if not sig_ref.name:
                 sig_ref.name = sig_ref.id
-            sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
+            if sig_ref.name:
+                sig_ref.url = f"https://www.beachesontheair.com/beaches/{sig_ref.name.lower().replace(' ', '-')}"
+            return sig_ref
+
+        elif sig.upper() == "GMA Islands":
+            # GMA Islands is a bit of a mess of GMA and IOTA references. Try looking them both up and see what returns
+            # the best result.
+            iota_lookup = get_sig_ref_info("IOTA", ref_id)
+            gma_lookup = get_sig_ref_info("GMA", ref_id)
+            for key, value in iota_lookup.__dict__.items():
+                if value is not None and sig_ref.__dict__.get(key) is None:
+                    sig_ref.__dict__[key] = value
+            for key, value in gma_lookup.__dict__.items():
+                if value is not None and sig_ref.__dict__.get(key) is None:
+                    sig_ref.__dict__[key] = value
+            sig_ref.ref_type = "Island"
             return sig_ref
 
         ### ACTUAL LOOKUP ###
         #
         # OK, this is something we have to look up. Now check to see if our data store contains reference data and use
         # that.
-        key = sig + ":" + ref_id
+        key = f"{sig}:{ref_id}"
         lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
         if lookup_data:
             return lookup_data
@@ -78,10 +99,10 @@ def get_sig_ref_info(sig, ref_id):
         else:
             # Maybe a super new reference we don't know about yet, but more likely a typo or a test reference,
             # just silently ignore it.
-            logging.debug("%s database did not contain data for ref %s", sig, ref_id)
+            logger.debug(f"{sig} database did not contain data for ref {ref_id}")
 
     except Exception:
-        logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True)
+        logger.exception(f"Exception when looking up sig_ref info for {sig} ref {ref_id}")
     return sig_ref
 
 

core/sig_utils.py

@@ -21,4 +21,4 @@ def get_sig_name_from_comment_name(sig):
 
 
 # Regex matching any SIG's "comment name", i.e. how it may be referred to in spot comments
-ANY_SIG_REGEX = r"(" + r"|".join(n for s in SIGS for n in s.comment_names) + r")"
+ANY_SIG_REGEX = rf"({'|'.join(n for s in SIGS for n in s.comment_names)})"

core/status_reporter.py

@@ -1,15 +1,16 @@
 import os
 from datetime import datetime
-from threading import Thread, Event
+from threading import Event, Thread
 
 import psutil
 import pytz
 
+from core.cleanup import CLEANUP_TIMER
 from core.config import SERVER_OWNER_CALLSIGN
 from core.constants import SOFTWARE_VERSION
 from core.data_providers import DATA_PROVIDERS
 from core.data_store import DATA_STORE
-from core.prometheus_metrics_handler import memory_use_gauge, spots_gauge, alerts_gauge
+from core.prometheus_metrics_handler import alerts_gauge, memory_use_gauge, spots_gauge
 from server.webserver import WEB_SERVER
 
 
@@ -53,53 +54,99 @@ class StatusReporter:
         DATA_STORE.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
         DATA_STORE.status_data["num_spots"] = len(DATA_STORE.spots.values())
         DATA_STORE.status_data["num_alerts"] = len(DATA_STORE.alerts.values())
-        DATA_STORE.status_data["spot_providers"] = list(
-            map(lambda p: {"name": p.name, "enabled": p.enabled,
-                           "enabled_by_default_in_web_ui": p.enabled_by_default_in_web_ui, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
-                           "last_spot": p.last_spot_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_spot_time.year > 2000 else 0},
-                DATA_PROVIDERS.spot_providers))
-        DATA_STORE.status_data["alert_providers"] = list(
-            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-                DATA_PROVIDERS.alert_providers))
-        DATA_STORE.status_data["solar_condition_providers"] = list(
-            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-                DATA_PROVIDERS.solar_condition_providers))
-        DATA_STORE.status_data["static_data_providers"] = list(
-            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0},
-                DATA_PROVIDERS.static_data_providers))
-        DATA_STORE.status_data["sig_ref_data_providers"] = list(
-            map(lambda p: {"sig_name": p.sig_name, "enabled": p.enabled, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
-                           "reference_count": p.reference_count},
-                DATA_PROVIDERS.sig_ref_data_providers))
-        DATA_STORE.status_data["callsign_data_providers"] = list(
-            map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status,
-                           "last_updated": p.last_update_time.replace(
-                               tzinfo=pytz.UTC).timestamp() if p.last_update_time.year > 2000 else 0,
-                           "lookup_count": p.lookup_count},
-                DATA_PROVIDERS.callsign_data_providers))
-        DATA_STORE.status_data["webserver"] = {"status": WEB_SERVER.web_server_metrics["status"],
-                                               "last_api_access": WEB_SERVER.web_server_metrics[
-                                                   "last_api_access_time"].replace(
-                                                   tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[
-                                                   "last_api_access_time"] else 0,
-                                               "api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"],
-                                               "last_page_access": WEB_SERVER.web_server_metrics[
-                                                   "last_page_access_time"].replace(
-                                                   tzinfo=pytz.UTC).timestamp() if WEB_SERVER.web_server_metrics[
-                                                   "last_page_access_time"] else 0,
-                                               "page_access_count": WEB_SERVER.web_server_metrics[
-                                                   "page_access_counter"]}
+        DATA_STORE.status_data["spot_providers"] = [
+            {
+                "name": p.name,
+                "enabled": p.enabled,
+                "enabled_by_default_in_web_ui": p.enabled_by_default_in_web_ui,
+                "status": p.status,
+                "last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
+                if p.last_update_time.year > 2000
+                else 0,
+                "last_spot": p.last_spot_time.replace(tzinfo=pytz.UTC).timestamp()
+                if p.last_spot_time.year > 2000
+                else 0,
+            }
+            for p in DATA_PROVIDERS.spot_providers
+        ]
+        DATA_STORE.status_data["alert_providers"] = [
+            {
+                "name": p.name,
+                "enabled": p.enabled,
+                "status": p.status,
+                "last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
+                if p.last_update_time.year > 2000
+                else 0,
+            }
+            for p in DATA_PROVIDERS.alert_providers
+        ]
+        DATA_STORE.status_data["solar_condition_providers"] = [
+            {
+                "name": p.name,
+                "enabled": p.enabled,
+                "status": p.status,
+                "last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
+                if p.last_update_time.year > 2000
+                else 0,
+            }
+            for p in DATA_PROVIDERS.solar_condition_providers
+        ]
+        DATA_STORE.status_data["static_data_providers"] = [
+            {
+                "name": p.name,
+                "enabled": p.enabled,
+                "status": p.status,
+                "last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
+                if p.last_update_time.year > 2000
+                else 0,
+            }
+            for p in DATA_PROVIDERS.static_data_providers
+        ]
+        DATA_STORE.status_data["sig_ref_data_providers"] = [
+            {
+                "sig_name": p.sig_name,
+                "enabled": p.enabled,
+                "status": p.status,
+                "last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
+                if p.last_update_time.year > 2000
+                else 0,
+                "reference_count": p.reference_count,
+            }
+            for p in DATA_PROVIDERS.sig_ref_data_providers
+        ]
+        DATA_STORE.status_data["callsign_data_providers"] = [
+            {
+                "name": p.name,
+                "enabled": p.enabled,
+                "status": p.status,
+                "last_updated": p.last_update_time.replace(tzinfo=pytz.UTC).timestamp()
+                if p.last_update_time.year > 2000
+                else 0,
+                "lookup_count": p.lookup_count,
+            }
+            for p in DATA_PROVIDERS.callsign_data_providers
+        ]
+        DATA_STORE.status_data["cleanup"] = {
+            "status": CLEANUP_TIMER.status,
+            "last_ran": CLEANUP_TIMER.last_cleanup_time.replace(tzinfo=pytz.UTC).timestamp()
+            if CLEANUP_TIMER.last_cleanup_time
+            else 0,
+        }
+        DATA_STORE.status_data["webserver"] = {
+            "status": WEB_SERVER.web_server_metrics["status"],
+            "last_api_access": WEB_SERVER.web_server_metrics["last_api_access_time"]
+            .replace(tzinfo=pytz.UTC)
+            .timestamp()
+            if WEB_SERVER.web_server_metrics["last_api_access_time"]
+            else 0,
+            "api_access_count": WEB_SERVER.web_server_metrics["api_access_counter"],
+            "last_page_access": WEB_SERVER.web_server_metrics["last_page_access_time"]
+            .replace(tzinfo=pytz.UTC)
+            .timestamp()
+            if WEB_SERVER.web_server_metrics["last_page_access_time"]
+            else 0,
+            "page_access_count": WEB_SERVER.web_server_metrics["page_access_counter"],
+        }
 
         # Update Prometheus metrics
         memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss)

core/url_data_cache.py

@@ -17,9 +17,12 @@ class URLDataCache(CachedSession):
     _lock = threading.Lock()
 
     def __init__(self, name):
-        super().__init__(CACHE_DIR + "urls/" + name, expire_after=timedelta(days=1),
-                         allowable_codes=(200, 400, 401, 403, 404))
+        super().__init__(
+            f"{CACHE_DIR}urls/{name}",
+            expire_after=timedelta(days=1),
+            allowable_codes=(200, 400, 401, 403, 404),
+        )
 
     def get(self, *args, **kwargs):
         with self._lock:
-            return super().get(*args, **kwargs)
+            return super().get(*args, **kwargs)

core/utils.py

@@ -4,10 +4,20 @@ import simplejson
 from pyhamtools.frequency import freq_to_band
 from pyhamtools.locator import latlong_to_locator
 
-from core.constants import UNKNOWN_BAND, BANDS, CW_MODES, PHONE_MODES, DATA_MODES, MODE_ALIASES, ALL_MODES
+from core.constants import (
+    ALL_MODES,
+    BANDS,
+    CW_MODES,
+    DATA_MODES,
+    MODE_ALIASES,
+    PHONE_MODES,
+    UNKNOWN_BAND,
+)
 from core.data_store import DATA_STORE
 from data.callsign import Callsign
 
+logger = logging.getLogger(__name__)
+
 
 def safe_json_dumps(obj):
     """Safe version of json.dumps that also converts objects to dicts so they can be output, and ignores NaN floats
@@ -22,7 +32,7 @@ def infer_mode_from_comment(comment):
     for mode in ALL_MODES:
         if mode in comment.upper():
             return mode
-    for mode in MODE_ALIASES.keys():
+    for mode in MODE_ALIASES:
         if mode in comment.upper():
             return MODE_ALIASES[mode]
     return None
@@ -38,8 +48,8 @@ def infer_mode_type_from_mode(mode):
     elif mode.upper() in DATA_MODES:
         return "DATA"
     else:
-        if mode.upper() != "OTHER":
-            logging.warning("Found an unrecognised mode: " + mode + ". Developer should categorise this.")
+        if mode.upper() != "OTHER" and mode != "?":
+            logger.warning(f"Found an unrecognised mode: {mode}. Developer should categorise this.")
         return None
 
 
@@ -63,11 +73,25 @@ def infer_mode_from_frequency(freq):
         # a spot at 7074.5 kHz will be indicated as LSB, even though it's clearly in the FT8 range. Future updates
         # might include other common digimode centres of activity here, but this achieves the main goal of keeping
         # large numbers of clearly-FT* spots off the list of people filtering out digimodes.
-        if (7074 <= khz < 7077) or (10136 <= khz < 10139) or (14074 <= khz < 14077) or (18100 <= khz < 18103) or (
-                21074 <= khz < 21077) or (24915 <= khz < 24918) or (28074 <= khz < 28077):
+        if (
+            (7074 <= khz < 7077)
+            or (10136 <= khz < 10139)
+            or (14074 <= khz < 14077)
+            or (18100 <= khz < 18103)
+            or (21074 <= khz < 21077)
+            or (24915 <= khz < 24918)
+            or (28074 <= khz < 28077)
+        ):
             mode = "FT8"
-        if (7047.5 <= khz < 7050.5) or (10140 <= khz < 10143) or (14080 <= khz < 14083) or (
-                18104 <= khz < 18107) or (21140 <= khz < 21143) or (24919 <= khz < 24922) or (28180 <= khz < 28183):
+        if (
+            (7047.5 <= khz < 7050.5)
+            or (10140 <= khz < 10143)
+            or (14080 <= khz < 14083)
+            or (18104 <= khz < 18107)
+            or (21140 <= khz < 21143)
+            or (24919 <= khz < 24922)
+            or (28180 <= khz < 28183)
+        ):
             mode = "FT4"
         return mode
     except KeyError:
@@ -77,7 +101,7 @@ def infer_mode_from_frequency(freq):
 def get_flag_for_dxcc(dxcc):
     """Get an emoji flag for a given DXCC entity ID"""
 
-    dxcc_data = DATA_STORE.dxcc_data[dxcc] if dxcc in DATA_STORE.dxcc_data else None
+    dxcc_data = DATA_STORE.dxcc_data.get(dxcc, None)
     return dxcc_data["flag"] if dxcc_data else None
 
 
@@ -89,28 +113,30 @@ def get_callsign_object_from_pyhamtools_callinfo(callsign, callinfo):
         home_call = callinfo.get_homecall(callsign)
         data = callinfo.get_all(callsign)
 
-        country = data["country"] if "country" in data else None
-        dxcc_id = data["adif"] if "adif" in data else None
-        continent = data["continent"] if "continent" in data else None
-        cq_zone = data["cqz"] if "cqz" in data else None
-        itu_zone = data["ituz"] if "ituz" in data else None
+        country = data.get("country", None)
+        dxcc_id = data.get("adif", None)
+        continent = data.get("continent", None)
+        cq_zone = data.get("cqz", None)
+        itu_zone = data.get("ituz", None)
         lat = float(data["latitude"]) if "latitude" in data else None
         lon = float(data["longitude"]) if "longitude" in data else None
         grid = None
         if lat and lon:
             grid = latlong_to_locator(lat, lon)
 
-        return Callsign(call=callsign,
-                        home_call=home_call,
-                        country=country,
-                        dxcc_id=dxcc_id,
-                        continent=continent,
-                        cq_zone=cq_zone,
-                        itu_zone=itu_zone,
-                        latitude=lat,
-                        longitude=lon,
-                        grid=grid,
-                        location_source="DXCC")
+        return Callsign(
+            call=callsign,
+            home_call=home_call,
+            country=country,
+            dxcc_id=dxcc_id,
+            continent=continent,
+            cq_zone=cq_zone,
+            itu_zone=itu_zone,
+            latitude=lat,
+            longitude=lon,
+            grid=grid,
+            location_source="DXCC",
+        )
 
     except (KeyError, ValueError):
         # Unknown callsign, can't look anything up, return a Callsign object with basic data so that gets cached

data/alert.py

@@ -1,4 +1,3 @@
-import copy
 import hashlib
 import json
 import logging
@@ -11,6 +10,8 @@ from core.call_lookup_helper import get_call_info
 from core.sig_lookup_helper import populate_missing_sig_ref_info
 from core.utils import get_flag_for_dxcc
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class Alert:
@@ -113,26 +114,23 @@ class Alert:
             if self.sig_refs and len(self.sig_refs) > 0 and self.sig_refs[0] and not self.sig:
                 self.sig = self.sig_refs[0].sig
 
-            # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
-            # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
-            # apart from that they were retrieved from the API at different times. Note that the simple Python hash()
-            # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
-            # use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
-            # The ID is computed before the online lookups below so that it is stable regardless of whether credentials
-            # are provided, allowing the enriched API response to be matched to the stored alert by ID.
+            # Create an ID based on the source and source ID if possible, as these guaranee uniqueness. If there is no
+            # source ID, use a combination of callsign and start time. Excluding things like the comment here allows for
+            # user updates of their alert comments without duplicating in the system.
             if not self.id:
-                self_copy = copy.deepcopy(self)
-                self_copy.received_time = 0
-                self_copy.received_time_iso = ""
-                self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
+                if self.source and self.source_id:
+                    self.id = hashlib.sha256(str({"s": self.source, "sid": self.source_id}).encode("utf-8")).hexdigest()
+                else:
+                    self.id = hashlib.sha256(
+                        str({"s": self.source, "c": self.dx_calls, "t": self.start_time}).encode("utf-8")
+                    ).hexdigest()
 
             # DX operator name lookup, using QRZ.com/HamQTH.
             if self.dx_calls and not self.dx_names:
-                self.dx_names = list(
-                    map(lambda c: get_call_info(c, credentials).name, self.dx_calls))
+                self.dx_names = [get_call_info(c, credentials).name for c in self.dx_calls]
 
-        except Exception as e:
-            logging.error("Exception while inferring missing data from spot", e, exc_info=True)
+        except Exception:
+            logger.exception("Exception while inferring missing data from spot")
 
     def to_json(self):
         """JSON serialise"""
@@ -145,5 +143,8 @@ class Alert:
         either having an end_time in the past, or if it only has a start_time, then that start time was more than 3 hours
         ago. If it somehow doesn't have a start_time either, it is considered to be expired."""
 
-        return not self.start_time or (self.end_time and self.end_time < datetime.now(pytz.UTC).timestamp()) or (
-                not self.end_time and self.start_time < (datetime.now(pytz.UTC) - timedelta(hours=3)).timestamp())
+        return (
+            not self.start_time
+            or (self.end_time and self.end_time < datetime.now(pytz.UTC).timestamp())
+            or (not self.end_time and self.start_time < (datetime.now(pytz.UTC) - timedelta(hours=3)).timestamp())
+        )

data/callsign.py

@@ -13,18 +13,18 @@ class Callsign:
     # "Home" call, i.e. with any prefixes and suffixes stripped off
     home_call: str | None = None
     # Operator name
-    name : str | None = None
+    name: str | None = None
     # QTH (location), free text
-    qth : str | None = None
+    qth: str | None = None
     # Maidenhead grid locator. Depending on the source of lookup this could be a home location from QRZ/HamQTH or just
     # the centre of the country they're operating in if no other data is available.
-    grid : str | None = None
+    grid: str | None = None
     # Latitude. Depending on the source of lookup this could be a home location from QRZ/HamQTH or just
     # the centre of the country they're operating in if no other data is available.
-    latitude : float | None = None
+    latitude: float | None = None
     # Longitude. Depending on the source of lookup this could be a home location from QRZ/HamQTH or just
     # the centre of the country they're operating in if no other data is available.
-    longitude : float | None = None
+    longitude: float | None = None
     # Country in which the callsign indicates they are operating
     country: str | None = None
     # Continent in which the callsign indicates they are operating
@@ -42,6 +42,16 @@ class Callsign:
         """Utility method to indicate that the callsign data is fully populated. Multiple providers can return data for
         a callsign, and we try them in sequence until we have all the data we can, in which case there's no point
         querying any other providers."""
-        return self.home_call is not None and self.name is not None and self.qth is not None and self.grid is not None\
-            and self.latitude is not None and self.longitude is not None and self.country is not None and self.continent\
-            is not None and self.dxcc_id is not None and self.cq_zone is not None and self.itu_zone is not None
+        return (
+            self.home_call is not None
+            and self.name is not None
+            and self.qth is not None
+            and self.grid is not None
+            and self.latitude is not None
+            and self.longitude is not None
+            and self.country is not None
+            and self.continent is not None
+            and self.dxcc_id is not None
+            and self.cq_zone is not None
+            and self.itu_zone is not None
+        )

data/lookup_credentials.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 @dataclass
 class LookupCredentials:
     """Per-request credentials for QRZ.com and HamQTH online callsign lookups."""
+
     qrz_username: str = ""
     qrz_password: str = ""
     qrz_session_key: str = ""  # alternative to username/password

data/sig.py

@@ -4,12 +4,12 @@ from dataclasses import dataclass, field
 @dataclass
 class SIG:
     """Data class that defines a Special Interest Group. Each contains a name and a longer form description.
-     They also contain comment_names which attempts to separate out the way people might refer to it in
-     cluster comments from how it is referred to in the UI & API. (For example, "TOTA" in cluster spot comments
-     almost always means Towers on the Air, but no single programme is referred to in the UI as "TOTA" as
-     it's ambiguous between Towers, Toilets and Tiles. And while Beaches got the name "BOTA" first, "BOTA" spots
-     are much more likely to be bunkers.) Finally, there is a ref_regex which provides a regular expression to
-     match what references (such as parks and summits) look like for that programme."""
+    They also contain comment_names which attempts to separate out the way people might refer to it in
+    cluster comments from how it is referred to in the UI & API. (For example, "TOTA" in cluster spot comments
+    almost always means Towers on the Air, but no single programme is referred to in the UI as "TOTA" as
+    it's ambiguous between Towers, Toilets and Tiles. And while Beaches got the name "BOTA" first, "BOTA" spots
+    are much more likely to be bunkers.) Finally, there is a ref_regex which provides a regular expression to
+    match what references (such as parks and summits) look like for that programme."""
 
     # SIG name as used in the UI and API, e.g. "Towers"
     name: str

data/sig_ref.py

@@ -12,13 +12,17 @@ class SIGRef:
     sig: str
     # Name of the reference, e.g. "Null Country Park", if known.
     name: str | None = None
+    # Type of the reference, e.g. "Park", if known.
+    ref_type: str | None = None
     # URL to look up more information about the reference, if known.
     url: str | None = None
-    # Latitude of the reference, if known.
+    # Latitude of the reference, in degrees, if known.
     latitude: float | None = None
-    # Longitude of the reference, if known.
+    # Longitude of the reference, in degrees, if known.
     longitude: float | None = None
-    # Maidenhead grid reference of the reference, if known.
+    # Altitude of the reference, in metres, if known.
+    altitude: float | None = None
+    # Maidenhead grid ref of the reference, if known.
     grid: str | None = None
     # Activation score. SOTA only
     activation_score: int | None = None

data/solar_conditions.py

@@ -82,9 +82,9 @@ def _xray_blackout_scale(xray):
         number = float(xray[1:])
     except ValueError:
         return 0
-    if letter == 'M':
+    if letter == "M":
         return 1 if number < 5 else 2
-    if letter == 'X':
+    if letter == "X":
         if number < 10:
             return 3
         if number < 20:

data/spot.py

@@ -1,24 +1,35 @@
-import copy
 import hashlib
 import json
 import logging
 import re
 from dataclasses import dataclass
 from datetime import datetime, timedelta
+from math import isnan
 
 import pytz
-from pyhamtools.locator import locator_to_latlong, latlong_to_locator
+from pyhamtools.locator import latlong_to_locator, locator_to_latlong
 
+from core.call_lookup_helper import get_call_info
 from core.config import MAX_SPOT_AGE
 from core.constants import MODE_ALIASES, PROPAGATION_MODES
 from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone
-from core.call_lookup_helper import get_call_info
-from core.sig_utils import ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name
 from core.sig_lookup_helper import populate_missing_sig_ref_info
-from core.utils import infer_band_from_freq, infer_mode_from_comment, \
-    infer_mode_from_frequency, infer_mode_type_from_mode, get_flag_for_dxcc
+from core.sig_utils import (
+    ANY_SIG_REGEX,
+    get_ref_regex_for_sig,
+    get_sig_name_from_comment_name,
+)
+from core.utils import (
+    get_flag_for_dxcc,
+    infer_band_from_freq,
+    infer_mode_from_comment,
+    infer_mode_from_frequency,
+    infer_mode_type_from_mode,
+)
 from data.sig_ref import SIGRef
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class Spot:
@@ -141,10 +152,7 @@ class Spot:
         objects such as the sig_refs list.."""
 
         if self.sig_refs:
-            self.sig_refs = [
-                sig_ref if isinstance(sig_ref, SIGRef) else SIGRef(**sig_ref)
-                for sig_ref in self.sig_refs
-            ]
+            self.sig_refs = [sig_ref if isinstance(sig_ref, SIGRef) else SIGRef(**sig_ref) for sig_ref in self.sig_refs]
 
     def infer_missing(self, credentials=None):
         """Infer missing parameters where possible"""
@@ -207,8 +215,11 @@ class Spot:
             # Spotter country, continent, zones etc. from callsign.
             # DE call with no digits, or APRS servers starting "T2" are not things we can look up location for
             de_call_info = get_call_info(self.de_call, credentials)
-            if self.de_call and any(char.isdigit() for char in self.de_call) and not (
-                    self.de_call.startswith("T2") and self.source == "APRS-IS"):
+            if (
+                self.de_call
+                and any(char.isdigit() for char in self.de_call)
+                and not (self.de_call.startswith("T2") and self.source == "APRS-IS")
+            ):
                 if not self.de_country:
                     self.de_country = de_call_info.country
                 if not self.de_continent:
@@ -219,7 +230,7 @@ class Spot:
                     self.de_flag = get_flag_for_dxcc(self.de_dxcc_id)
 
             # Remove NaNs in frequency
-            if self.freq and self.freq == float("nan"):
+            if self.freq and isnan(self.freq):
                 self.freq = None
 
             # Band from frequency
@@ -279,8 +290,11 @@ class Spot:
                     # If so, add that to the sig_refs list for this spot.
                     ref_regex = get_ref_regex_for_sig(found_sig)
                     if ref_regex:
-                        ref_matches = re.finditer(r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)", self.comment,
-                                                  re.IGNORECASE)
+                        ref_matches = re.finditer(
+                            r"(^|\W)" + found_sig + r"([ -])(" + ref_regex + r")($|\W)",
+                            self.comment,
+                            re.IGNORECASE,
+                        )
                         for ref_match in ref_matches:
                             self._append_sig_ref_if_missing(SIGRef(id=ref_match.group(3).upper(), sig=found_sig))
 
@@ -293,7 +307,7 @@ class Spot:
                     # If the spot itself doesn't have location yet, but the SIG ref does, extract it
                     if sig_ref.grid and not self.dx_grid:
                         self.dx_grid = sig_ref.grid
-                    if sig_ref.latitude and not self.dx_latitude:
+                    if sig_ref.latitude and not self.dx_latitude and sig_ref.longitude and not self.dx_longitude:
                         self.dx_latitude = sig_ref.latitude
                         self.dx_longitude = sig_ref.longitude
                         if self.sig == "WAB" or self.sig == "WAI" or self.sig == "Tiles":
@@ -311,8 +325,9 @@ class Spot:
             # being the only source we have for propagation mode. Brace for nightmare regex from hell.
             if self.comment:
                 grid_mode_grid_match = re.search(
-                    r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
-                    self.comment)
+                    r"\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)(?:<([^>]*)>|\(([^)]*)\))([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b",
+                    self.comment,
+                )
                 if grid_mode_grid_match:
                     # regex matches, so extract grids:
                     if not self.de_grid:
@@ -328,13 +343,14 @@ class Spot:
                             self.propagation_mode = PROPAGATION_MODES[mode_tag]
                         else:
                             self.propagation_mode = mode_tag
-                            logging.info("Seen a new propagation mode tag not yet in the system: %s", mode_tag)
+                            logger.info(f"Seen a new propagation mode tag not yet in the system: {mode_tag}")
 
             # Parse "de_grid -> dx_grid" structures from the comment
             if self.comment:
                 grid_mode_grid_match = re.search(
-                    r'\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b',
-                    self.comment)
+                    r"\b([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\s*->\s*([A-Ra-r]{2}\d{2}(?:[A-Xa-x]{2}(?:\d{2})?)?)\b",
+                    self.comment,
+                )
                 if grid_mode_grid_match:
                     # regex matches, so extract grids:
                     if not self.dx_grid:
@@ -344,42 +360,40 @@ class Spot:
                         self.de_grid = grid_mode_grid_match.group(2).upper()
 
             # DX Grid to lat/lon and vice versa in case one is missing
-            if self.dx_grid and not self.dx_latitude:
+            if self.dx_grid and (not self.dx_latitude or not self.dx_longitude):
                 try:
                     ll = locator_to_latlong(self.dx_grid)
                     self.dx_latitude = ll[0]
                     self.dx_longitude = ll[1]
-                except:
-                    logging.debug("Invalid grid received for spot")
+                except Exception:
+                    logger.debug("Invalid grid received for spot")
             if self.dx_latitude and self.dx_longitude and not self.dx_grid:
                 try:
                     self.dx_grid = latlong_to_locator(self.dx_latitude, self.dx_longitude, 8)
-                except:
-                    logging.debug("Invalid lat/lon received for spot")
+                except Exception:
+                    logger.debug("Invalid lat/lon received for spot")
 
             # QRT comment detection
             if self.comment and not self.qrt:
                 self.qrt = "QRT" in self.comment.upper()
 
-            # Always create an ID based on a hash of every parameter *except* received_time. This is used as the index
-            # to a map, which as a byproduct avoids us having multiple duplicate copies of the object that are identical
-            # apart from that they were retrieved from the API at different times. Note that the simple Python hash()
-            # function includes a seed randomly generated at runtime; this is therefore not consistent between runs. But we
-            # use diskcache to store our data between runs, so we use SHA256 which does not include this random element.
-            # The ID is computed before the online lookups below so that it is stable regardless of whether credentials
-            # are provided, allowing the enriched API response to be matched to the stored spot by ID.
+            # Create an ID based on the source and source ID if possible, as these guaranee uniqueness. If there is no
+            # source ID, use a combination of callsign and spot time. Spot time is down to the second or even the
+            # millisecond, so we can be reasonably sure two spots that match are the same spot.
             if not self.id:
-                self_copy = copy.deepcopy(self)
-                self_copy.received_time = 0
-                self_copy.received_time_iso = ""
-                self.id = hashlib.sha256(str(self_copy).encode("utf-8")).hexdigest()
+                if self.source and self.source_id:
+                    self.id = hashlib.sha256(str({"s": self.source, "sid": self.source_id}).encode("utf-8")).hexdigest()
+                else:
+                    self.id = hashlib.sha256(
+                        str({"s": self.source, "c": self.dx_call, "t": self.time}).encode("utf-8")
+                    ).hexdigest()
 
             # DX operator details lookup. This should be the last resort compared to taking the data from the actual
             # spotting service, e.g. we don't want to accidentally use a user's QRZ.com home lat/lon or DXCC lat/lon
             # instead of the one from the park reference they're at.
             if self.dx_call and not self.dx_name:
                 self.dx_name = dx_call_info.name
-            if self.dx_call and not self.dx_latitude:
+            if self.dx_call and (not self.dx_latitude or not self.dx_longitude):
                 self.dx_latitude = dx_call_info.latitude
                 self.dx_longitude = dx_call_info.longitude
                 self.dx_grid = dx_call_info.grid
@@ -390,41 +404,50 @@ class Spot:
             if self.sig_refs and len(self.sig_refs) > 0:
                 qth = self.sig_refs[0].id
                 if self.sig_refs[0].name:
-                    qth += " " + self.sig_refs[0].name
+                    qth += f" {self.sig_refs[0].name}"
                 self.dx_qth = qth
             else:
                 self.dx_qth = dx_call_info.qth
 
             # CQ and ITU zone lookup, preferably from location but failing that, from callsign
             if not self.dx_cq_zone:
-                if self.dx_latitude:
+                if self.dx_latitude and self.dx_longitude:
                     self.dx_cq_zone = lat_lon_to_cq_zone(self.dx_latitude, self.dx_longitude)
                 elif self.dx_call:
                     self.dx_cq_zone = dx_call_info.cq_zone
             if not self.dx_itu_zone:
-                if self.dx_latitude:
+                if self.dx_latitude and self.dx_longitude:
                     self.dx_itu_zone = lat_lon_to_itu_zone(self.dx_latitude, self.dx_longitude)
                 elif self.dx_call:
                     self.dx_itu_zone = dx_call_info.itu_zone
 
             # DX Location is "good" if it is from a spot, or from QRZ if the callsign doesn't contain a slash, so the operator
             # is likely at home.
-            self.dx_location_good = bool(self.dx_latitude and self.dx_longitude and (
-                    self.dx_location_source == "SPOT" or self.dx_location_source == "SIG REF LOOKUP"
+            self.dx_location_good = bool(
+                self.dx_latitude
+                and self.dx_longitude
+                and (
+                    self.dx_location_source == "SPOT"
+                    or self.dx_location_source == "SIG REF LOOKUP"
                     or self.dx_location_source == "GRID"
-                    or (self.dx_location_source == "HOME QTH" and "/" not in (self.dx_call or ""))))
+                    or (self.dx_location_source == "HOME QTH" and "/" not in (self.dx_call or ""))
+                )
+            )
 
             # DE with no digits and APRS servers starting "T2" are not things we can look up location for
-            if self.de_call and any(char.isdigit() for char in self.de_call) and not (
-                    self.de_call.startswith("T2") and self.source == "APRS-IS"):
+            if (
+                self.de_call
+                and any(char.isdigit() for char in str(self.de_call))
+                and not (self.de_call.startswith("T2") and self.source == "APRS-IS")
+                and (not self.de_latitude or not self.de_longitude)
+            ):
                 # DE operator location lookup
-                if not self.de_latitude:
-                    self.de_latitude = de_call_info.latitude
-                    self.de_longitude = de_call_info.longitude
-                    self.de_grid = de_call_info.grid
+                self.de_latitude = de_call_info.latitude
+                self.de_longitude = de_call_info.longitude
+                self.de_grid = de_call_info.grid
 
-        except Exception as e:
-            logging.error("Exception while inferring missing data from spot", e, exc_info=True)
+        except Exception:
+            logger.exception("Exception while inferring missing data from spot")
 
     def to_json(self):
         """JSON serialise"""

datafiles/toilets.csv

@@ -27,4 +27,9 @@ EH T-31,50.3636495,7.5584857
 EH T-33,50.3636495,7.5584857
 EH T-34,50.3636495,7.5584857
 EH T-41,50.3636495,7.5584857
-EH T-51,50.3636495,7.5584857
+EH T-51,50.3636495,7.5584857
+HOPE T-01,40.7527998,-73.9935015
+HOPE T-02,40.7526614,-73.9935752
+HOPE T-03,40.7528663,-73.9936566
+HOPE T-04,40.7527845,-73.9937284
+HOPE T-05,40.7518292,-73.9932781

docs/running.md

@@ -3,7 +3,7 @@
 If you want to run a copy of Spothole with different configuration settings than the main instance, you can download it
 and run it on your own local machine or server.
 
-You will require Python version 3.8 or later. If you encounter an error about `gdal-config` during the following
+You will require Python version 3.10 or later. If you encounter an error about `gdal-config` during the following
 process, you will also need `libgdal-dev` installed.
 
 To download and set up Spothole on a Debian server, run the following commands. Other operating systems will likely be

providers/alert/alert_provider.py

@@ -29,7 +29,7 @@ class AlertProvider:
 
         # Sort the batch so that earliest ones go in first. This helps keep the ordering correct when alerts are fired
         # off to SSE listeners.
-        alerts = sorted(alerts, key=lambda a: (a.start_time if a and a.start_time else 0))
+        alerts = sorted(alerts, key=lambda a: a.start_time if a and a.start_time else 0)
         for alert in alerts:
             # Fill in any blanks and add to the list
             alert.infer_missing()

providers/alert/bota.py

@@ -3,9 +3,9 @@ from datetime import datetime, timedelta
 import pytz
 from bs4 import BeautifulSoup
 
-from providers.alert.http_alert_provider import HTTPAlertProvider
 from data.alert import Alert
 from data.sig_ref import SIGRef
+from providers.alert.http_alert_provider import HTTPAlertProvider
 
 
 class BOTA(HTTPAlertProvider):
@@ -23,17 +23,17 @@ class BOTA(HTTPAlertProvider):
         bs = BeautifulSoup(http_response.content.decode("utf-8-sig"), features="lxml")
         if not bs.body:
             return new_alerts
-        div = bs.body.find('div', attrs={'class': 'view-activations-public'})
+        div = bs.body.find("div", attrs={"class": "view-activations-public"})
         if div:
-            table = div.find('table', attrs={'class': 'views-table'})
+            table = div.find("table", attrs={"class": "views-table"})
             if table:
-                tbody = table.find('tbody')
+                tbody = table.find("tbody")
                 if not tbody:
                     return new_alerts
-                for row in tbody.find_all('tr'):
-                    cells = row.find_all('td')
-                    first_cell_anchor = cells[0].find('a') if len(cells) > 0 else None
-                    second_cell_anchor = cells[1].find('a') if len(cells) > 1 else None
+                for row in tbody.find_all("tr"):
+                    cells = row.find_all("td")
+                    first_cell_anchor = cells[0].find("a") if len(cells) > 0 else None
+                    second_cell_anchor = cells[1].find("a") if len(cells) > 1 else None
                     if not first_cell_anchor or not second_cell_anchor:
                         continue
                     first_cell_text = first_cell_anchor.get_text().strip()
@@ -41,7 +41,7 @@ class BOTA(HTTPAlertProvider):
                     dx_call = second_cell_anchor.get_text().strip().upper()
 
                     # Get the date, dealing with the fact we get no year so have to figure out if it's last year or next year
-                    date_span = cells[2].find('span') if len(cells) > 2 else None
+                    date_span = cells[2].find("span") if len(cells) > 2 else None
                     if not date_span:
                         continue
                     date_text = date_span.get_text().strip()
@@ -52,11 +52,13 @@ class BOTA(HTTPAlertProvider):
                         date_time = date_time.replace(year=datetime.now(pytz.UTC).year + 1)
 
                     # Convert to our alert format
-                    alert = Alert(source=self.name,
-                                  dx_calls=[dx_call],
-                                  sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
-                                  start_time=date_time.timestamp(),
-                                  is_dxpedition=False)
+                    alert = Alert(
+                        source=self.name,
+                        dx_calls=[dx_call],
+                        sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
+                        start_time=date_time.timestamp(),
+                        is_dxpedition=False,
+                    )
 
                     new_alerts.append(alert)
         return new_alerts

providers/alert/http_alert_provider.py

@@ -1,13 +1,15 @@
 import logging
 from datetime import datetime
-from threading import Thread, Event
+from threading import Event, Thread
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
-from providers.alert.alert_provider import AlertProvider
 from core.constants import HTTP_HEADERS
+from providers.alert.alert_provider import AlertProvider
+
+logger = logging.getLogger(__name__)
 
 
 class HTTPAlertProvider(AlertProvider):
@@ -24,7 +26,7 @@ class HTTPAlertProvider(AlertProvider):
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
-        logging.info("Set up query of " + self.name + " alert API every " + str(self._poll_interval) + " seconds.")
+        logger.info(f"Set up query of {self.name} alert API every {self._poll_interval!s} seconds.")
         self._thread = Thread(target=self._run, name=f"HTTPAlertProvider-{self.name}")
         self._thread.start()
 
@@ -40,7 +42,7 @@ class HTTPAlertProvider(AlertProvider):
     def _poll(self):
         try:
             # Request data from API
-            logging.debug("Polling " + self.name + " alert API...")
+            logger.debug(f"Polling {self.name} alert API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
             # Check response code was good
             if http_response.ok:
@@ -52,18 +54,18 @@ class HTTPAlertProvider(AlertProvider):
 
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
-                logging.debug("Received data from " + self.name + " alert API.")
+                logger.debug(f"Received data from {self.name} alert API.")
             else:
                 self.status = "Error"
-                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} alerts API.")
+                logger.warning(f"HTTP {http_response.status_code} when calling {self.name} alerts API.")
 
         except ConnectionError:
-            logging.warning(f"Connection error when accessing {self.name} alerts API.")
+            logger.warning(f"Connection error when accessing {self.name} alerts API.")
         except (ConnectTimeout, ReadTimeout):
-            logging.warning(f"Timeout when accessing {self.name} alerts API.")
+            logger.warning(f"Timeout when accessing {self.name} alerts API.")
         except Exception:
             self.status = "Error"
-            logging.exception("Exception in HTTP JSON Alert Provider (" + self.name + ")")
+            logger.exception(f"Exception in HTTP JSON Alert Provider ({self.name})")
             # Brief pause on error before the next poll, but still respond promptly to stop()
             self._stop_event.wait(timeout=1)
 

providers/alert/ng3k.py

@@ -6,8 +6,8 @@ import pytz
 from rss_parser import Parser
 from rss_parser.models.rss import RSS
 
-from providers.alert.http_alert_provider import HTTPAlertProvider
 from data.alert import Alert
+from providers.alert.http_alert_provider import HTTPAlertProvider
 
 
 class NG3K(HTTPAlertProvider):
@@ -49,11 +49,16 @@ class NG3K(HTTPAlertProvider):
                 end_day = end_string.split(", ")[0].strip()
                 end_mon = start_mon
 
-            start_timestamp = datetime.strptime(start_year + " " + start_mon + " " + start_day, "%Y %b %d").replace(
-                tzinfo=pytz.UTC).timestamp()
-            end_timestamp = datetime.strptime(end_year + " " + end_mon + " " + end_day + " 23:59",
-                                              "%Y %b %d %H:%M").replace(
-                tzinfo=pytz.UTC).timestamp()
+            start_timestamp = (
+                datetime.strptime(f"{start_year} {start_mon} {start_day}", "%Y %b %d")
+                .replace(tzinfo=pytz.UTC)
+                .timestamp()
+            )
+            end_timestamp = (
+                datetime.strptime(f"{end_year} {end_mon} {end_day} 23:59", "%Y %b %d %H:%M")
+                .replace(tzinfo=pytz.UTC)
+                .timestamp()
+            )
 
             # Sometimes the DX callsign is "real", sometimes you just get a prefix with the real working callsigns being
             # provided in the "by" field. e.g. call="JW", by="By LA7XK as JW7XK, LA6VM as JW6VM, LA9DL as JW9DL". So
@@ -75,14 +80,16 @@ class NG3K(HTTPAlertProvider):
             comment = extra_parts[3] if len(extra_parts) > 3 else ""
 
             # Convert to our alert format
-            alert = Alert(source=self.name,
-                          dx_calls=dx_calls,
-                          dx_country=dx_country,
-                          freqs_modes=bands + (("; " + modes) if modes != "" else ""),
-                          comment=by + "; " + comment + "; " + qsl_info,
-                          start_time=start_timestamp,
-                          end_time=end_timestamp,
-                          is_dxpedition=True)
+            alert = Alert(
+                source=self.name,
+                dx_calls=dx_calls,
+                dx_country=dx_country,
+                freqs_modes=bands + (f"; {modes}" if modes != "" else ""),
+                comment=f"{by}; {comment}; {qsl_info}",
+                start_time=start_timestamp,
+                end_time=end_timestamp,
+                is_dxpedition=True,
+            )
 
             # Add to our list.
             new_alerts.append(alert)

providers/alert/parksnpeaks.py

@@ -3,9 +3,11 @@ from datetime import datetime
 
 import pytz
 
-from providers.alert.http_alert_provider import HTTPAlertProvider
 from data.alert import Alert
 from data.sig_ref import SIGRef
+from providers.alert.http_alert_provider import HTTPAlertProvider
+
+logger = logging.getLogger(__name__)
 
 
 class ParksNPeaks(HTTPAlertProvider):
@@ -30,8 +32,9 @@ class ParksNPeaks(HTTPAlertProvider):
             else:
                 sig_ref = source_alert["WWFFID"]
                 sig_ref_name = source_alert["Location"]
-            start_time = datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(
-                tzinfo=pytz.UTC).timestamp()
+            start_time = (
+                datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp()
+            )
 
             sigrefs = []
             # PnP can give us an alert of class "QRP" which is the only one that's not a real SIG in Spothole's list,
@@ -40,18 +43,30 @@ class ParksNPeaks(HTTPAlertProvider):
                 sigrefs = [SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)]
 
             # Convert to our alert format
-            alert = Alert(source=self.name,
-                          source_id=source_alert["alID"],
-                          dx_calls=[source_alert["CallSign"].upper()],
-                          freqs_modes=source_alert["Freq"] + " " + source_alert["MODE"],
-                          comment=source_alert["Comments"],
-                          sig_refs=sigrefs,
-                          start_time=start_time,
-                          is_dxpedition=False)
+            alert = Alert(
+                source=self.name,
+                source_id=source_alert["alID"],
+                dx_calls=[source_alert["CallSign"].upper()],
+                freqs_modes=f"{source_alert['Freq']} {source_alert['MODE']}",
+                comment=source_alert["Comments"],
+                sig_refs=sigrefs,
+                start_time=start_time,
+                is_dxpedition=False,
+            )
 
             # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
-            if sig and sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "SANPCPA", "LLOTA", "QRP"]:
-                logging.warning("PNP alert found with sig " + sig + ", developer needs to add support for this!")
+            if sig and sig not in [
+                "POTA",
+                "SOTA",
+                "WWFF",
+                "SIOTA",
+                "ZLOTA",
+                "KRMNPA",
+                "SANPCPA",
+                "LLOTA",
+                "QRP",
+            ]:
+                logger.warning(f"PNP alert found with sig {sig}, developer needs to add support for this!")
 
             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
             # the alert list. Note that while ZLOTA has its own spots API, it doesn't have its own alerts API. So that

providers/alert/pota.py

@@ -2,9 +2,9 @@ from datetime import datetime
 
 import pytz
 
-from providers.alert.http_alert_provider import HTTPAlertProvider
 from data.alert import Alert
 from data.sig_ref import SIGRef
+from providers.alert.http_alert_provider import HTTPAlertProvider
 
 
 class POTA(HTTPAlertProvider):
@@ -21,18 +21,31 @@ class POTA(HTTPAlertProvider):
         # Iterate through source data
         for source_alert in http_response.json():
             # Convert to our alert format
-            alert = Alert(source=self.name,
-                          source_id=source_alert["scheduledActivitiesId"],
-                          dx_calls=[source_alert["activator"].upper()],
-                          freqs_modes=source_alert["frequencies"],
-                          comment=source_alert["comments"],
-                          sig_refs=[SIGRef(id=source_alert["reference"], sig="POTA", name=source_alert["name"],
-                                           url="https://pota.app/#/park/" + source_alert["reference"])],
-                          start_time=datetime.strptime(source_alert["startDate"] + source_alert["startTime"],
-                                                       "%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
-                          end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"],
-                                                     "%Y-%m-%d%H:%M").replace(tzinfo=pytz.UTC).timestamp(),
-                          is_dxpedition=False)
+            alert = Alert(
+                source=self.name,
+                source_id=source_alert["scheduledActivitiesId"],
+                dx_calls=[source_alert["activator"].upper()],
+                freqs_modes=source_alert["frequencies"],
+                comment=source_alert["comments"],
+                sig_refs=[
+                    SIGRef(
+                        id=source_alert["reference"],
+                        sig="POTA",
+                        name=source_alert["name"],
+                        url=f"https://pota.app/#/park/{source_alert['reference']}",
+                    )
+                ],
+                start_time=datetime.strptime(
+                    source_alert["startDate"] + source_alert["startTime"],
+                    "%Y-%m-%d%H:%M",
+                )
+                .replace(tzinfo=pytz.UTC)
+                .timestamp(),
+                end_time=datetime.strptime(source_alert["endDate"] + source_alert["endTime"], "%Y-%m-%d%H:%M")
+                .replace(tzinfo=pytz.UTC)
+                .timestamp(),
+                is_dxpedition=False,
+            )
 
             # Add to our list, but exclude any old spots that POTA can sometimes give us where even the end time is
             # in the past. Don't worry about de-duping, removing old alerts etc. at this point; other code will do

providers/alert/sota.py

@@ -2,9 +2,9 @@ from datetime import datetime
 
 import pytz
 
-from providers.alert.http_alert_provider import HTTPAlertProvider
 from data.alert import Alert
 from data.sig_ref import SIGRef
+from providers.alert.http_alert_provider import HTTPAlertProvider
 
 
 class SOTA(HTTPAlertProvider):
@@ -26,18 +26,26 @@ class SOTA(HTTPAlertProvider):
             summit_points = None
             if len(details) > 2:
                 summit_points = int(details[-1].split(" ")[0])
-            alert = Alert(source=self.name,
-                          source_id=source_alert["id"],
-                          dx_calls=[source_alert["activatingCallsign"].upper()],
-                          dx_names=[source_alert["activatorName"].upper()],
-                          freqs_modes=source_alert["frequency"],
-                          comment=source_alert["comments"],
-                          sig_refs=[
-                              SIGRef(id=source_alert["associationCode"] + "/" + source_alert["summitCode"], sig="SOTA",
-                                     name=summit_name, activation_score=summit_points)],
-                          start_time=datetime.strptime(source_alert["dateActivated"],
-                                                       "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=pytz.UTC).timestamp(),
-                          is_dxpedition=False)
+            alert = Alert(
+                source=self.name,
+                source_id=source_alert["id"],
+                dx_calls=[source_alert["activatingCallsign"].upper()],
+                dx_names=[source_alert["activatorName"].upper()],
+                freqs_modes=source_alert["frequency"],
+                comment=source_alert["comments"],
+                sig_refs=[
+                    SIGRef(
+                        id=f"{source_alert['associationCode']}/{source_alert['summitCode']}",
+                        sig="SOTA",
+                        name=summit_name,
+                        activation_score=summit_points,
+                    )
+                ],
+                start_time=datetime.strptime(source_alert["dateActivated"], "%Y-%m-%dT%H:%M:%SZ")
+                .replace(tzinfo=pytz.UTC)
+                .timestamp(),
+                is_dxpedition=False,
+            )
 
             # Add to our list
             new_alerts.append(alert)

providers/alert/wota.py

@@ -1,13 +1,17 @@
+import logging
 from datetime import datetime
 from typing import cast
+from xml.parsers.expat import ExpatError
 
 import pytz
 from rss_parser import Parser as RSSParser
 from rss_parser.models.rss import RSS
 
-from providers.alert.http_alert_provider import HTTPAlertProvider
 from data.alert import Alert
 from data.sig_ref import SIGRef
+from providers.alert.http_alert_provider import HTTPAlertProvider
+
+logger = logging.getLogger(__name__)
 
 
 class WOTA(HTTPAlertProvider):
@@ -22,43 +26,62 @@ class WOTA(HTTPAlertProvider):
 
     def _http_response_to_alerts(self, http_response):
         new_alerts = []
-        rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig")))
-        # Iterate through source data
-        for source_alert in rss.channel.items:
 
-            # Reject GUID missing or zero
-            if not source_alert.guid or not source_alert.guid.content or source_alert.guid.content == "http://www.wota.org.uk/alerts/0":
-                continue
+        try:
+            rss = cast(RSS, RSSParser.parse(http_response.content.decode("utf-8-sig")))
+            # Iterate through source data
+            for source_alert in rss.channel.items:
+                # Reject GUID missing or zero
+                if (
+                    not source_alert.guid
+                    or not source_alert.guid.content
+                    or source_alert.guid.content == "http://www.wota.org.uk/alerts/0"
+                ):
+                    continue
 
-            # Pick apart the title
-            title_split = source_alert.title.split(" on ")
-            dx_call = title_split[0]
-            ref = None
-            ref_name = None
-            if len(title_split) > 1:
-                ref_split = title_split[1].split(" - ")
-                ref = str(ref_split[0])
-                if len(ref_split) > 1:
-                    ref_name = str(ref_split[1])
+                # Pick apart the title
+                dx_call = None
+                ref = None
+                ref_name = None
+                try:
+                    title_split = source_alert.title.split(" on ")
+                    dx_call = title_split[0]
+                    if len(title_split) > 1:
+                        ref_split = title_split[1].split(" - ")
+                        ref = str(ref_split[0])
+                        if len(ref_split) > 1:
+                            ref_name = str(ref_split[1])
+                except Exception:
+                    logger.warning(f"Could not parse WOTA alert title: {source_alert.description}")
 
-            # Pick apart the description
-            desc_split = source_alert.description.split(". ")
-            freqs_modes = desc_split[0].replace("Frequencies/modes:", "").strip()
-            comment = None
-            if len(desc_split) > 1:
-                comment = desc_split[1].strip()
+                # Pick apart the description
+                comment = None
+                freqs_modes = None
+                try:
+                    desc_split = source_alert.description.split(". ")
+                    freqs_modes = desc_split[0].replace("Frequencies/modes:", "").strip()
+                    if len(desc_split) > 1:
+                        comment = desc_split[1].strip()
+                except Exception:
+                    logger.warning(f"Could not parse WOTA alert description: {source_alert.description}")
 
-            time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+                time = datetime.strptime(source_alert.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
 
-            # Convert to our alert format
-            alert = Alert(source=self.name,
-                          source_id=source_alert.guid.content,
-                          dx_calls=[dx_call],
-                          freqs_modes=freqs_modes,
-                          comment=comment,
-                          sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
-                          start_time=time.timestamp())
+                # Convert to our alert format
+                alert = Alert(
+                    source=self.name,
+                    source_id=source_alert.guid.content,
+                    dx_calls=[dx_call],
+                    freqs_modes=freqs_modes,
+                    comment=comment,
+                    sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
+                    start_time=time.timestamp(),
+                )
+
+                # Add to our list.
+                new_alerts.append(alert)
+
+        except ExpatError:
+            logger.warning("WOTA alert RSS feed was fetched but was invalid")
 
-            # Add to our list.
-            new_alerts.append(alert)
         return new_alerts

providers/alert/wwff.py

@@ -2,9 +2,9 @@ from datetime import datetime
 
 import pytz
 
-from providers.alert.http_alert_provider import HTTPAlertProvider
 from data.alert import Alert
 from data.sig_ref import SIGRef
+from providers.alert.http_alert_provider import HTTPAlertProvider
 
 
 class WWFF(HTTPAlertProvider):
@@ -21,17 +21,21 @@ class WWFF(HTTPAlertProvider):
         # Iterate through source data
         for source_alert in http_response.json():
             # Convert to our alert format
-            alert = Alert(source=self.name,
-                          source_id=source_alert["id"],
-                          dx_calls=[source_alert["activator_call"].upper()],
-                          freqs_modes=source_alert["band"] + " " + source_alert["mode"],
-                          comment=source_alert["remarks"],
-                          sig_refs=[SIGRef(id=source_alert["reference"], sig="WWFF")],
-                          start_time=datetime.strptime(source_alert["utc_start"],
-                                                       "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
-                          end_time=datetime.strptime(source_alert["utc_end"],
-                                                     "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp(),
-                          is_dxpedition=False)
+            alert = Alert(
+                source=self.name,
+                source_id=source_alert["id"],
+                dx_calls=[source_alert["activator_call"].upper()],
+                freqs_modes=f"{source_alert['band']} {source_alert['mode']}",
+                comment=source_alert["remarks"],
+                sig_refs=[SIGRef(id=source_alert["reference"], sig="WWFF")],
+                start_time=datetime.strptime(source_alert["utc_start"], "%Y-%m-%d %H:%M:%S")
+                .replace(tzinfo=pytz.UTC)
+                .timestamp(),
+                end_time=datetime.strptime(source_alert["utc_end"], "%Y-%m-%d %H:%M:%S")
+                .replace(tzinfo=pytz.UTC)
+                .timestamp(),
+                is_dxpedition=False,
+            )
 
             # Add to our list
             new_alerts.append(alert)

providers/callsigndata/api_query_callsign_data_provider.py

@@ -1,13 +1,3 @@
-import logging
-from datetime import datetime
-from threading import Thread, Event
-
-import pytz
-from requests import ReadTimeout
-from requests.exceptions import ConnectionError, ConnectTimeout
-
-from core.constants import HTTP_HEADERS
-from core.url_data_cache import URLDataCache
 from providers.callsigndata.callsign_data_provider import CallsignDataProvider
 
 
@@ -15,7 +5,7 @@ class APIQueryCallsignDataProvider(CallsignDataProvider):
     """Generic callsign data provider class for providers that fetch their data from the web on-demand using an API."""
 
     def __init__(self, name, provider_config, storage):
-        """ Set up the provider."""
+        """Set up the provider."""
         super().__init__(name, provider_config, storage)
 
         if self.enabled:

providers/callsigndata/callsign_data_provider.py

@@ -1,4 +1,3 @@
-import logging
 from datetime import datetime
 
 import pytz
@@ -52,7 +51,6 @@ class CallsignDataProvider:
         else:
             return None
 
-
     def _perform_new_lookup(self, callsign, lookup_credentials):
         """Makes a new request to the data source for callsign data."""
 

providers/callsigndata/clublogapi.py

@@ -2,12 +2,16 @@ import logging
 from datetime import datetime
 
 import pytz
-from pyhamtools import LookupLib, Callinfo
+from pyhamtools import Callinfo, LookupLib
 
 from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
 from data.callsign import Callsign
-from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
+from providers.callsigndata.api_query_callsign_data_provider import (
+    APIQueryCallsignDataProvider,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class ClublogAPI(APIQueryCallsignDataProvider):
@@ -23,12 +27,12 @@ class ClublogAPI(APIQueryCallsignDataProvider):
             self._callinfo = Callinfo(lookuplib)
         else:
             provider_config["enabled"] = False
-            logging.warning(
-                "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
+            logger.warning(
+                "Clublog XML callsign data provider configured but no api key was provided, this has been disabled."
+            )
 
         super().__init__("Clublog API", provider_config, DATA_STORE.callsign_data_clublogapi)
 
-
     def _perform_new_lookup(self, callsign, lookup_credentials):
         callsign_data = Callsign(call=callsign)
 
@@ -41,8 +45,8 @@ class ClublogAPI(APIQueryCallsignDataProvider):
             else:
                 return None
 
-        except Exception as e:
+        except Exception:
             self.status = "Error"
-            logging.error("Exception when looking up data from Clublog API", e, exc_info=True)
+            logger.exception("Exception when looking up data from Clublog API")
 
         return callsign_data

providers/callsigndata/clublogxml.py

@@ -1,14 +1,16 @@
 import gzip
 import logging
-from datetime import datetime
 
-import pytz
-from pyhamtools import LookupLib, Callinfo
+from pyhamtools import Callinfo, LookupLib
 
 from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
 from data.callsign import Callsign
-from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
+from providers.callsigndata.file_download_callsign_data_provider import (
+    FileDownloadCallsignDataProvider,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class ClublogXML(FileDownloadCallsignDataProvider):
@@ -25,11 +27,18 @@ class ClublogXML(FileDownloadCallsignDataProvider):
         self._api_key = provider_config.get("api_key", "")
         if self._api_key == "":
             provider_config["enabled"] = False
-            logging.warning(
-                "Clublog XML callsign data provider configured but no api key was provided, this has been disabled.")
+            logger.warning(
+                "Clublog XML callsign data provider configured but no api key was provided, this has been disabled."
+            )
 
-        super().__init__("Clublog XML", provider_config, self.DATA_URL + "?api=" + self._api_key,
-                         self.CACHE_PATH_ZIPPED, self.POLL_INTERVAL_DAYS, DATA_STORE.callsign_data_clublogxml)
+        super().__init__(
+            "Clublog XML",
+            provider_config,
+            f"{self.DATA_URL}?api={self._api_key}",
+            self.CACHE_PATH_ZIPPED,
+            self.POLL_INTERVAL_DAYS,
+            DATA_STORE.callsign_data_clublogxml,
+        )
 
     def _handle_file(self, path):
         try:
@@ -47,8 +56,8 @@ class ClublogXML(FileDownloadCallsignDataProvider):
             self._callinfo = Callinfo(lookuplib)
             return True
 
-        except Exception as e:
-            logging.error("Exception when loading Clublog XML.", e, exc_info=True)
+        except Exception:
+            logger.exception("Exception when loading Clublog XML.")
             return False
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
@@ -62,8 +71,8 @@ class ClublogXML(FileDownloadCallsignDataProvider):
             else:
                 return None
 
-        except Exception as e:
+        except Exception:
             self.status = "Error"
-            logging.error("Exception when looking up data from Clublog XML data", e, exc_info=True)
+            logger.exception("Exception when looking up data from Clublog XML data")
 
         return callsign_data

providers/callsigndata/countryfiles.py

@@ -1,11 +1,15 @@
 import logging
 
-from pyhamtools import LookupLib, Callinfo
+from pyhamtools import Callinfo, LookupLib
 
 from core.data_store import DATA_STORE
 from core.utils import get_callsign_object_from_pyhamtools_callinfo
 from data.callsign import Callsign
-from providers.callsigndata.file_download_callsign_data_provider import FileDownloadCallsignDataProvider
+from providers.callsigndata.file_download_callsign_data_provider import (
+    FileDownloadCallsignDataProvider,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class CountryFiles(FileDownloadCallsignDataProvider):
@@ -17,8 +21,14 @@ class CountryFiles(FileDownloadCallsignDataProvider):
     _callinfo = None
 
     def __init__(self, provider_config):
-        super().__init__("CountryFiles.com", provider_config, self.DATA_URL, self.CACHE_PATH, self.POLL_INTERVAL_DAYS,
-                         DATA_STORE.callsign_data_countryfiles)
+        super().__init__(
+            "CountryFiles.com",
+            provider_config,
+            self.DATA_URL,
+            self.CACHE_PATH,
+            self.POLL_INTERVAL_DAYS,
+            DATA_STORE.callsign_data_countryfiles,
+        )
 
     def _handle_file(self, path):
         try:
@@ -26,8 +36,8 @@ class CountryFiles(FileDownloadCallsignDataProvider):
             self._callinfo = Callinfo(lookuplib)
             return True
 
-        except Exception as e:
-            logging.error("Exception when loading Country Files cty.plist.", e, exc_info=True)
+        except Exception:
+            logger.exception("Exception when loading Country Files cty.plist.")
             return False
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
@@ -41,8 +51,8 @@ class CountryFiles(FileDownloadCallsignDataProvider):
             else:
                 return None
 
-        except Exception as e:
+        except Exception:
             self.status = "Error"
-            logging.error("Exception when looking up data from Country file", e, exc_info=True)
+            logger.exception("Exception when looking up data from Country file")
 
         return callsign_data

providers/callsigndata/file_download_callsign_data_provider.py

@@ -1,28 +1,29 @@
 import logging
 from datetime import datetime
-from threading import Thread, Event
+from threading import Event, Thread
 
 import pytz
-from requests import ReadTimeout
-from requests.exceptions import ConnectionError, ConnectTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from core.url_data_cache import URLDataCache
 from providers.callsigndata.callsign_data_provider import CallsignDataProvider
 
+logger = logging.getLogger(__name__)
+
 
 class FileDownloadCallsignDataProvider(CallsignDataProvider):
     """Generic callsign data provider class for providers that fetch their data from the web by downloading a file."""
 
     def __init__(self, name, provider_config, url, cache_file_path, poll_interval, storage):
-        """ Set up the provider, note poll_interval is in *days*."""
+        """Set up the provider, note poll_interval is in *days*."""
         super().__init__(name, provider_config, storage)
         self._url = url
         self._cache_file_path = cache_file_path
         self._poll_interval = poll_interval
         self._thread = None
         self._stop_event = Event()
-        self._url_data_cache = URLDataCache("callsigndata_" + name)
+        self._url_data_cache = URLDataCache(f"callsigndata_{name}")
 
         if self.enabled:
             self.status = "Ready"
@@ -30,8 +31,7 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
-        logging.info(
-            "Set up query of " + self.name + " callsign reference data every " + str(self._poll_interval) + " days.")
+        logger.info(f"Set up query of {self.name} callsign reference data every {self._poll_interval!s} days.")
         self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}")
         self._thread.start()
 
@@ -48,7 +48,7 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
         try:
             # Request the file. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
             # caching, but just so continual restarts of the software during testing don't hammer the servers.
-            logging.debug("Downloading " + self.name + " callsign reference data...")
+            logger.debug(f"Downloading {self.name} callsign reference data...")
             http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
             # Check response code was good
             if http_response.ok:
@@ -61,24 +61,26 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
                     if ok:
                         self.status = "OK"
                         self.last_update_time = datetime.now(pytz.UTC)
-                        logging.info("Updated callsign reference data from " + self.name)
+                        logger.info(f"Updated callsign reference data from {self.name}")
                     else:
                         self.status = "Error"
-                        logging.warning(f"Error updating callsign reference data from {self.name}.")
+                        logger.warning(f"Error updating callsign reference data from {self.name}.")
 
             else:
                 self.status = "Error"
-                logging.warning(f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}.")
+                logger.warning(
+                    f"HTTP {http_response.status_code} when downloading callsign reference data from {self.name}."
+                )
 
         except ConnectionError:
             self.status = "Error"
-            logging.warning(f"Connection error when downloading callsign reference data from {self.name}.")
+            logger.warning(f"Connection error when downloading callsign reference data from {self.name}.")
         except (ConnectTimeout, ReadTimeout):
             self.status = "Error"
-            logging.warning(f"Timeout when downloading callsign reference data from {self.name}.")
+            logger.warning(f"Timeout when downloading callsign reference data from {self.name}.")
         except Exception:
             self.status = "Error"
-            logging.exception("Exception in callsign reference data provider (" + self.name + ")")
+            logger.exception(f"Exception in callsign reference data provider ({self.name})")
             self._stop_event.wait(timeout=1)
 
     def _handle_file(self, path):

providers/callsigndata/hamqth.py

@@ -1,19 +1,23 @@
 import logging
 import urllib.parse
-from datetime import timedelta, datetime
+from datetime import datetime, timedelta
 
 import pytz
 import xmltodict
 from pyhamtools import callinfo
-from requests import ConnectTimeout, ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 from requests_cache import CachedSession
 
 from core.config import SERVER_OWNER_CALLSIGN
 from core.constants import HTTP_HEADERS, SOFTWARE_VERSION
-from core.data_store import DATA_STORE, CACHE_DIR
+from core.data_store import CACHE_DIR, DATA_STORE
 from core.url_data_cache import URLDataCache
 from data.callsign import Callsign
-from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
+from providers.callsigndata.api_query_callsign_data_provider import (
+    APIQueryCallsignDataProvider,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class HamQTH(APIQueryCallsignDataProvider):
@@ -22,18 +26,19 @@ class HamQTH(APIQueryCallsignDataProvider):
     def __init__(self, provider_config):
         super().__init__("HamQTH", provider_config, DATA_STORE.callsign_data_hamqth)
         self._HAMQTH_BASE_URL = "https://www.hamqth.com/xml.php"
-        self._PRG = ("Spothole v" + SOFTWARE_VERSION + " operated by " + SERVER_OWNER_CALLSIGN).replace(" ", "_")
+        self._PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}".replace(" ", "_")
         self._URL_DATA_CACHE = URLDataCache("hamqth")
         # Separate URL cache for session key lookups. Once a session key is returned from logging in with a username
         # and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
-        self._CREDENTIALS_CACHE = CachedSession(CACHE_DIR + "/urls/hamqth-creds",
-                                                expire_after=timedelta(minutes=55))
+        self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/hamqth-creds", expire_after=timedelta(minutes=55))
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
         # If we don't have HamQTH credentials, skip this lookup Return None so we don't *cache* the lack of data, because
         #         # someone might provide credentials next time around.
-        if not lookup_credentials or not ((lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
-                or lookup_credentials.hamqth_session_id):
+        if not lookup_credentials or not (
+            (lookup_credentials.hamqth_username and lookup_credentials.hamqth_password)
+            or lookup_credentials.hamqth_session_id
+        ):
             return None
 
         try:
@@ -44,18 +49,18 @@ class HamQTH(APIQueryCallsignDataProvider):
             elif lookup_credentials.hamqth_username and lookup_credentials.hamqth_password:
                 try:
                     session_data = self._CREDENTIALS_CACHE.get(
-                        self._HAMQTH_BASE_URL + "?u=" + urllib.parse.quote_plus(lookup_credentials.hamqth_username) +
-                        "&p=" + urllib.parse.quote_plus(lookup_credentials.hamqth_password),
-                        headers=HTTP_HEADERS).content
+                        f"{self._HAMQTH_BASE_URL}?u={urllib.parse.quote_plus(lookup_credentials.hamqth_username)}&p={urllib.parse.quote_plus(lookup_credentials.hamqth_password)}",
+                        headers=HTTP_HEADERS,
+                    ).content
                     dict_data = xmltodict.parse(session_data)
                     if "session_id" in dict_data["HamQTH"]["session"]:
                         session_id = str(dict_data["HamQTH"]["session"]["session_id"])
                     else:
                         # Log this failure at debug level only, not our problem if user entered the wrong password.
-                        logging.debug("HamQTH login details incorrect, failed to look up with HamQTH.")
+                        logger.debug("HamQTH login details incorrect, failed to look up with HamQTH.")
                         return None
                 except Exception:
-                    logging.error("Exception when getting HamQTH session key")
+                    logger.error("Exception when getting HamQTH session key")
                     return None
 
             if not session_id:
@@ -68,14 +73,16 @@ class HamQTH(APIQueryCallsignDataProvider):
                 if home_call != callsign:
                     calls_to_try.append(home_call)
             except ValueError:
-                logging.debug("Could not look up home call for callsign %s", callsign)
+                logger.debug(f"Could not look up home call for callsign {callsign}")
 
             # Try looking up each call using the API
             for lookup_call in calls_to_try:
                 try:
                     response = self._URL_DATA_CACHE.get(
-                        self._HAMQTH_BASE_URL + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
-                            lookup_call) + "&prg=" + self._PRG, headers=HTTP_HEADERS, timeout=10)
+                        f"{self._HAMQTH_BASE_URL}?id={session_id}&callsign={urllib.parse.quote_plus(lookup_call)}&prg={self._PRG}",
+                        headers=HTTP_HEADERS,
+                        timeout=10,
+                    )
                     if response.ok:
                         # Found data, convert it to our object and return it
                         data = xmltodict.parse(response.content)["HamQTH"]["search"]
@@ -85,27 +92,26 @@ class HamQTH(APIQueryCallsignDataProvider):
                         return self.hamqth_response_to_callsign(callsign, data)
 
                     elif not response.from_cache:
-                        logging.warning("HTTP %d looking up callsign %s using HamQTH", response.status_code,
-                                        lookup_call)
+                        logger.warning(f"HTTP {response.status_code} looking up callsign {lookup_call} using HamQTH")
 
                 except (KeyError, ValueError):
                     continue
                 except ConnectionError:
-                    logging.warning(f"Connection error when looking up callsign %s using HamQTH", lookup_call)
+                    logger.warning(f"Connection error when looking up callsign {lookup_call} using HamQTH")
                     continue
                 except (ConnectTimeout, ReadTimeout):
-                    logging.warning(f"Timeout when looking up callsign %s using HamQTH", lookup_call)
+                    logger.warning(f"Timeout when looking up callsign {lookup_call} using HamQTH")
                     continue
                 except Exception:
-                    logging.error("Exception when looking up callsign %s using HamQTH", lookup_call, exc_info=True)
+                    logger.exception(f"Exception when looking up callsign {lookup_call} using HamQTH")
                     continue
 
             # Not found in HamQTH; return a Callsign object with no data so we cache that and don't keep retrying
             return Callsign(call=callsign)
 
-        except Exception as e:
+        except Exception:
             self.status = "Error"
-            logging.error("Exception when looking up data from HamQTH", e, exc_info=True)
+            logger.exception("Exception when looking up data from HamQTH")
             # Return None, this won't be cached so we will be asked to query data again for this call next time.
             return None
 
@@ -116,9 +122,12 @@ class HamQTH(APIQueryCallsignDataProvider):
         # Check for sensible latitudes
         lat = None
         lon = None
-        if "latitude" in data and "longitude" in data and (
-                float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float(
-                data["latitude"]) < 89.9:
+        if (
+            "latitude" in data
+            and "longitude" in data
+            and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
+            and -89.9 < float(data["latitude"]) < 89.9
+        ):
             lat = float(data["latitude"])
             lon = float(data["longitude"])
 
@@ -127,16 +136,18 @@ class HamQTH(APIQueryCallsignDataProvider):
         if "grid" in data and not data["grid"].startswith("AA00"):
             grid = data["grid"]
 
-        return Callsign(call=callsign,
-                        home_call=callinfo.Callinfo.get_homecall(callsign),
-                        name=data["nick"] if "nick" in data else None,
-                        qth=data["qth"] if "qth" in data else None,
-                        country=data["country"] if "country" in data else None,
-                        continent=data["continent"] if "continent" in data else None,
-                        latitude=lat,
-                        longitude=lon,
-                        grid=grid,
-                        dxcc_id=int(data["adif"]) if "adif" in data else None,
-                        cq_zone=int(data["cq"]) if "cq" in data else None,
-                        itu_zone=int(data["itu"]) if "itu" in data else None,
-                        location_source="HOME QTH")
+        return Callsign(
+            call=callsign,
+            home_call=callinfo.Callinfo.get_homecall(callsign),
+            name=data.get("nick", None),
+            qth=data.get("qth", None),
+            country=data.get("country", None),
+            continent=data.get("continent", None),
+            latitude=lat,
+            longitude=lon,
+            grid=grid,
+            dxcc_id=int(data["adif"]) if "adif" in data else None,
+            cq_zone=int(data["cq"]) if "cq" in data else None,
+            itu_zone=int(data["itu"]) if "itu" in data else None,
+            location_source="HOME QTH",
+        )

providers/callsigndata/qrz.py

@@ -1,18 +1,22 @@
 import logging
 import urllib.parse
-from datetime import timedelta, datetime
+from datetime import datetime, timedelta
 
 import pytz
 import xmltodict
 from pyhamtools import callinfo
-from requests import ConnectTimeout, ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 from requests_cache import CachedSession
 
 from core.constants import HTTP_HEADERS
-from core.data_store import DATA_STORE, CACHE_DIR
+from core.data_store import CACHE_DIR, DATA_STORE
 from core.url_data_cache import URLDataCache
 from data.callsign import Callsign
-from providers.callsigndata.api_query_callsign_data_provider import APIQueryCallsignDataProvider
+from providers.callsigndata.api_query_callsign_data_provider import (
+    APIQueryCallsignDataProvider,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class QRZ(APIQueryCallsignDataProvider):
@@ -24,14 +28,14 @@ class QRZ(APIQueryCallsignDataProvider):
         self._URL_DATA_CACHE = URLDataCache("qrz")
         # Separate URL cache for session key lookups. Once a session key is returned from logging in with a username
         # and password, this is valid for an hour, so our cache stores this specifically for 55 minutes.
-        self._CREDENTIALS_CACHE = CachedSession(CACHE_DIR + "/urls/qrz-creds",
-                                                expire_after=timedelta(minutes=55))
+        self._CREDENTIALS_CACHE = CachedSession(f"{CACHE_DIR}/urls/qrz-creds", expire_after=timedelta(minutes=55))
 
     def _perform_new_lookup(self, callsign, lookup_credentials):
         # If we don't have QRZ credentials, skip this lookup. Return None so we don't *cache* the lack of data, because
         # someone might provide credentials next time around.
-        if not lookup_credentials or not ((lookup_credentials.qrz_username and lookup_credentials.qrz_password)
-                or lookup_credentials.qrz_session_key):
+        if not lookup_credentials or not (
+            (lookup_credentials.qrz_username and lookup_credentials.qrz_password) or lookup_credentials.qrz_session_key
+        ):
             return None
 
         try:
@@ -42,19 +46,19 @@ class QRZ(APIQueryCallsignDataProvider):
             elif lookup_credentials.qrz_username and lookup_credentials.qrz_password:
                 try:
                     login_response = self._CREDENTIALS_CACHE.get(
-                        self._QRZ_BASE_URL + "?username=" + urllib.parse.quote_plus(lookup_credentials.qrz_username) +
-                        "&password=" + urllib.parse.quote_plus(lookup_credentials.qrz_password) + "&agent=spothole",
-                        headers=HTTP_HEADERS).content
+                        f"{self._QRZ_BASE_URL}?username={urllib.parse.quote_plus(lookup_credentials.qrz_username)}&password={urllib.parse.quote_plus(lookup_credentials.qrz_password)}&agent=spothole",
+                        headers=HTTP_HEADERS,
+                    ).content
                     login_data = xmltodict.parse(login_response)
                     session = login_data.get("QRZDatabase", {}).get("Session", {})
                     if "Key" in session:
                         session_key = str(session["Key"])
                     else:
                         # Log this failure at debug level only, not our problem if user entered the wrong password.
-                        logging.debug("QRZ.com login details incorrect, failed to look up with QRZ.")
+                        logger.debug("QRZ.com login details incorrect, failed to look up with QRZ.")
                         return None
                 except Exception:
-                    logging.error("Exception when getting QRZ.com session key")
+                    logger.error("Exception when getting QRZ.com session key")
                     return None
 
             if not session_key:
@@ -67,14 +71,16 @@ class QRZ(APIQueryCallsignDataProvider):
                 if home_call != callsign:
                     calls_to_try.append(home_call)
             except ValueError:
-                logging.debug("Could not look up home call for callsign %s", callsign)
+                logger.debug(f"Could not look up home call for callsign {callsign}")
 
             # Try looking up each call using the API
             for lookup_call in calls_to_try:
                 try:
                     response = self._URL_DATA_CACHE.get(
-                        self._QRZ_BASE_URL + "?s=" + session_key + "&callsign=" + urllib.parse.quote_plus(lookup_call),
-                        headers=HTTP_HEADERS, timeout=10)
+                        f"{self._QRZ_BASE_URL}?s={session_key}&callsign={urllib.parse.quote_plus(lookup_call)}",
+                        headers=HTTP_HEADERS,
+                        timeout=10,
+                    )
                     if response.ok:
                         qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
                         if qrz_response:
@@ -89,32 +95,33 @@ class QRZ(APIQueryCallsignDataProvider):
                             elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
                                 # Errors here are normally just "callsign not in database", no need to log that ourselves
                                 # above debug level.
-                                logging.debug("QRZ returned an error looking up callsign %s: %s", lookup_call,
-                                              qrz_response.get("Session").get("Error"))
+                                logger.debug(
+                                    f"QRZ returned an error looking up callsign {lookup_call}: {qrz_response.get('Session').get('Error')}"
+                                )
 
                         elif not response.from_cache:
-                            logging.warning("QRZ returned a malformed response looking up callsign %s", lookup_call)
+                            logger.warning(f"QRZ returned a malformed response looking up callsign {lookup_call}")
                     elif not response.from_cache:
-                        logging.warning("HTTP %d looking up callsign %s using QRZ", lookup_call)
+                        logger.warning(f"HTTP {response.status_code} looking up callsign {lookup_call} using QRZ")
 
                 except (KeyError, ValueError):
                     continue
                 except ConnectionError:
-                    logging.warning(f"Connection error when looking up callsign %s using QRZ", lookup_call)
+                    logger.warning(f"Connection error when looking up callsign {lookup_call} using QRZ")
                     continue
                 except (ConnectTimeout, ReadTimeout):
-                    logging.warning(f"Timeout when looking up callsign %s using QRZ.", lookup_call)
+                    logger.warning(f"Timeout when looking up callsign {lookup_call} using QRZ.")
                     continue
                 except Exception:
-                    logging.error("Exception when looking up callsign %s using QRZ", lookup_call, exc_info=True)
+                    logger.exception(f"Exception when looking up callsign {lookup_call} using QRZ")
                     continue
 
             # Not found in QRZ; return a Callsign object with no data so we cache that and don't keep retrying
             return Callsign(call=callsign)
 
-        except Exception as e:
+        except Exception:
             self.status = "Error"
-            logging.error("Exception when looking up data from QRZ.com", e, exc_info=True)
+            logger.exception("Exception when looking up data from QRZ.com")
             # Return None, this won't be cached so we will be asked to query data again for this call next time.
             return None
 
@@ -122,6 +129,13 @@ class QRZ(APIQueryCallsignDataProvider):
     def qrz_response_to_callsign(callsign, data):
         """Convert the "Callsign" block in QRZ's API response to our own Callsign object."""
 
+        # I have encountered a user passing multiple callsigns to the QRZ lookup function in a way that QRZ actually
+        # does handle, and returns a list of callsign data. If we encounter that, just take the first one, as our own
+        # functions can't deal with multiple calls this way.
+        if isinstance(data, list):
+            data = data[0]
+            callsign = data["call"]
+
         # Get a name
         name = None
         if "name_fmt" in data:
@@ -129,16 +143,19 @@ class QRZ(APIQueryCallsignDataProvider):
         if "fname" in data:
             name = data["fname"]
             if "nick" in data:
-                name = name + " \"" + data["nick"] + "\""
+                name = f'{name} "{data["nick"]}"'
             if "name" in data:
-                name = name + " " + data["name"]
+                name = f"{name} {data['name']}"
 
         # Check for sensible latitudes
         lat = None
         lon = None
-        if "latitude" in data and "longitude" in data and (
-                float(data["latitude"]) != 0 or float(data["longitude"]) != 0) and -89.9 < float(
-                data["latitude"]) < 89.9:
+        if (
+            "latitude" in data
+            and "longitude" in data
+            and (float(data["latitude"]) != 0 or float(data["longitude"]) != 0)
+            and -89.9 < float(data["latitude"]) < 89.9
+        ):
             lat = float(data["latitude"])
             lon = float(data["longitude"])
 
@@ -147,16 +164,18 @@ class QRZ(APIQueryCallsignDataProvider):
         if "grid" in data and not data["grid"].startswith("AA00"):
             grid = data["grid"]
 
-        return Callsign(call=callsign,
-                        home_call=callinfo.Callinfo.get_homecall(callsign),
-                        name=name,
-                        qth=data["addr2"] if "addr2" in data else None,
-                        country=data["country"] if "country" in data else None,
-                        continent=data["continent"] if "continent" in data else None,
-                        latitude=lat,
-                        longitude=lon,
-                        grid=grid,
-                        dxcc_id=int(data["adif"]) if "adif" in data else None,
-                        cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
-                        itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
-                        location_source="HOME QTH")
+        return Callsign(
+            call=callsign,
+            home_call=callinfo.Callinfo.get_homecall(callsign),
+            name=name,
+            qth=data.get("addr2", None),
+            country=data.get("country", None),
+            continent=data.get("continent", None),
+            latitude=lat,
+            longitude=lon,
+            grid=grid,
+            dxcc_id=int(data["adif"]) if "adif" in data else None,
+            cq_zone=int(data["cqzone"]) if "cqzone" in data else None,
+            itu_zone=int(data["ituzone"]) if "ituzone" in data else None,
+            location_source="HOME QTH",
+        )

providers/sigrefdata/arlhs.py

@@ -2,7 +2,9 @@ import csv
 from time import sleep
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class ARLHS(FileDownloadSIGRefDataProvider):
@@ -20,13 +22,18 @@ class ARLHS(FileDownloadSIGRefDataProvider):
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
             if "ARLHS" in row and row["ARLHS"] != "":
                 ref_id = row["ARLHS"]
-                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
-                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
-                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
-                                              "Latitude"] != "" else None,
-                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
-                                              "Longitude"] != "" else None,
-                                          grid=row["Maidenhead Locator"]))
+                new_data.append(
+                    SIGRef(
+                        sig=self.SIG,
+                        id=ref_id,
+                        name=row.get("Name", None),
+                        ref_type="Lighthouse",
+                        url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
+                        latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
+                        longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
+                        grid=row["Maidenhead Locator"],
+                    )
+                )
 
             # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
             # the data in this case

providers/sigrefdata/dme.py

@@ -4,7 +4,9 @@ from time import sleep
 from pyhamtools.locator import latlong_to_locator
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
+from providers.sigrefdata.local_file_sig_ref_data_provider import (
+    LocalFileSIGRefDataProvider,
+)
 
 
 class DME(LocalFileSIGRefDataProvider):
@@ -21,15 +23,25 @@ class DME(LocalFileSIGRefDataProvider):
         with open(path, encoding="latin-1") as _f:
             for row in csv.DictReader(_f, delimiter=";"):
                 ref_id = row["COD_INE"][:5]
-                latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
-                    "LATITUD_ETRS89_REGCAN95") else None
-                longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
-                    "LONGITUD_ETRS89_REGCAN95") else None
+                latitude = (
+                    float(row["LATITUD_ETRS89_REGCAN95"].replace(",", "."))
+                    if row.get("LATITUD_ETRS89_REGCAN95")
+                    else None
+                )
+                longitude = (
+                    float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", "."))
+                    if row.get("LONGITUD_ETRS89_REGCAN95")
+                    else None
+                )
 
-                ref = SIGRef(sig=self.SIG, id=ref_id,
-                                          name=row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"],
-                                          latitude=latitude,
-                                          longitude=longitude)
+                ref = SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    ref_type="Town",
+                    name=f"{row['NOMBRE_ACTUAL']}, {row['PROVINCIA']}",
+                    latitude=latitude,
+                    longitude=longitude,
+                )
                 if latitude and longitude:
                     ref.grid = latlong_to_locator(latitude, longitude, 6)
                 new_data.append(ref)

providers/sigrefdata/file_download_sig_ref_data_provider.py

@@ -1,33 +1,33 @@
 import logging
 from datetime import datetime
-from threading import Thread, Event
+from threading import Event, Thread
 
 import pytz
-from requests import ReadTimeout
-from requests.exceptions import ConnectionError, ConnectTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from core.url_data_cache import URLDataCache
 from providers.sigrefdata.sig_ref_data_provider import SIGRefDataProvider
 
+logger = logging.getLogger(__name__)
+
 
 class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
     """Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file."""
 
     def __init__(self, sig_name, provider_config, url, poll_interval):
-        """ Set up the provider, note poll_interval is in *days*."""
+        """Set up the provider, note poll_interval is in *days*."""
         super().__init__(sig_name, provider_config)
         self._url = url
         self._poll_interval = poll_interval
         self._thread = None
         self._stop_event = Event()
-        self._url_data_cache = URLDataCache("sigrefdata_" + sig_name)
+        self._url_data_cache = URLDataCache(f"sigrefdata_{sig_name}")
 
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
-        logging.info(
-            "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " days.")
+        logger.info(f"Set up query of {self.sig_name} SIG ref data every {self._poll_interval!s} days.")
         self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}")
         self._thread.start()
 
@@ -45,7 +45,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         try:
             # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
             # caching, but just so continual restarts of the software during testing don't hammer the servers.
-            logging.debug("Downloading " + self.sig_name + " SIG ref data...")
+            logger.debug(f"Downloading {self.sig_name} SIG ref data...")
             http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
             # Check response code was good
             if http_response.ok:
@@ -57,20 +57,20 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
 
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
-                logging.debug("Received SIG ref data for " + self.sig_name)
+                logger.debug(f"Received SIG ref data for {self.sig_name}")
             else:
                 self.status = "Error"
-                logging.warning(f"HTTP {http_response.status_code} when downloading SIG ref data for {self.sig_name}.")
+                logger.warning(f"HTTP {http_response.status_code} when downloading SIG ref data for {self.sig_name}.")
 
         except ConnectionError:
             self.status = "Error"
-            logging.warning(f"Connection error when downloading SIG ref data for {self.sig_name}.")
+            logger.warning(f"Connection error when downloading SIG ref data for {self.sig_name}.")
         except (ConnectTimeout, ReadTimeout):
             self.status = "Error"
-            logging.warning(f"Timeout when downloading SIG ref data for {self.sig_name}.")
+            logger.warning(f"Timeout when downloading SIG ref data for {self.sig_name}.")
         except Exception:
             self.status = "Error"
-            logging.exception("Exception in HTTP SIG Ref Data Provider (" + self.sig_name + ")")
+            logger.exception(f"Exception in HTTP SIG Ref Data Provider ({self.sig_name})")
             self._stop_event.wait(timeout=1)
 
     def _http_response_to_data(self, http_response):

providers/sigrefdata/gma.py

@@ -2,7 +2,9 @@ import csv
 from time import sleep
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class GMA(FileDownloadSIGRefDataProvider):
@@ -19,13 +21,21 @@ class GMA(FileDownloadSIGRefDataProvider):
         new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
             ref_id = row["Reference"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
-                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
-                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
-                                          "Latitude"] != "" else None,
-                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
-                                          "Longitude"] != "" else None,
-                                      grid=row["Maidenhead Locator"]))
+            new_data.append(
+                SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    name=row.get("Name", None),
+                    ref_type="Summit",
+                    url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
+                    latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
+                    longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
+                    altitude=float(row["Height (m)"].replace("m", ""))
+                    if "Height (m)" in row and row["Height (m)"] != ""
+                    else None,
+                    grid=row["Maidenhead Locator"],
+                )
+            )
 
             # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
             # the data in this case

providers/sigrefdata/illw.py

@@ -2,7 +2,9 @@ import csv
 from time import sleep
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class ILLW(FileDownloadSIGRefDataProvider):
@@ -20,13 +22,18 @@ class ILLW(FileDownloadSIGRefDataProvider):
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
             if "ILLW" in row and row["ILLW"] != "":
                 ref_id = row["ILLW"]
-                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
-                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
-                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
-                                              "Latitude"] != "" else None,
-                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
-                                              "Longitude"] != "" else None,
-                                          grid=row["Maidenhead Locator"]))
+                new_data.append(
+                    SIGRef(
+                        sig=self.SIG,
+                        id=ref_id,
+                        name=row.get("Name", None),
+                        ref_type="Lighthouse",
+                        url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
+                        latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
+                        longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
+                        grid=row["Maidenhead Locator"],
+                    )
+                )
 
             # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
             # the data in this case

providers/sigrefdata/iota.py

@@ -4,7 +4,11 @@ from time import sleep
 from pyhamtools.locator import latlong_to_locator
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class IOTA(FileDownloadSIGRefDataProvider):
@@ -23,16 +27,29 @@ class IOTA(FileDownloadSIGRefDataProvider):
         if isinstance(data, list):
             for ref in data:
                 ref_id = ref["refno"]
-                latitude = float(ref["latitude_min"]) + float(ref["latitude_max"]) / 2.0
-                longitude = float(ref["longitude_min"]) + float(ref["longitude_max"]) / 2.0
+                latitude = (float(ref["latitude_min"]) + float(ref["latitude_max"])) / 2.0
+                longitude = (float(ref["longitude_min"]) + float(ref["longitude_max"])) / 2.0
                 grid = None
                 try:
                     grid = latlong_to_locator(latitude, longitude, 6)
                 except ValueError:
-                    logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude)
+                    logger.debug(
+                        "Error converting lat/lon to locator for an IOTA reference %f %f",
+                        latitude,
+                        longitude,
+                    )
 
-                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
-                                          grid=grid, latitude=latitude, longitude=longitude))
+                new_data.append(
+                    SIGRef(
+                        sig=self.SIG,
+                        id=ref_id,
+                        name=ref["name"],
+                        ref_type="Island",
+                        grid=grid,
+                        latitude=latitude,
+                        longitude=longitude,
+                    )
+                )
 
                 # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
                 # of the data in this case

providers/sigrefdata/krmnpa.py

@@ -1,4 +1,6 @@
-from providers.sigrefdata.pnp_kml_sig_ref_data_provider import ParksNPeaksKMLSIGRefDataProvider
+from providers.sigrefdata.pnp_kml_sig_ref_data_provider import (
+    ParksNPeaksKMLSIGRefDataProvider,
+)
 
 
 class KRMNPA(ParksNPeaksKMLSIGRefDataProvider):
@@ -9,4 +11,4 @@ class KRMNPA(ParksNPeaksKMLSIGRefDataProvider):
     DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=KRMNPA&poiFormat=4"
 
     def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)

providers/sigrefdata/llota.py

@@ -3,7 +3,9 @@ from time import sleep
 from pyhamtools.locator import locator_to_latlong
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class LLOTA(FileDownloadSIGRefDataProvider):
@@ -25,11 +27,18 @@ class LLOTA(FileDownloadSIGRefDataProvider):
                 grid = str(ref["grid_locator"])
                 ll = locator_to_latlong(grid)
 
-                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
-                                          url="https://llota.app/list/ref/" + ref_id,
-                                          grid=grid,
-                                          latitude=ll[0],
-                                          longitude=ll[1]))
+                new_data.append(
+                    SIGRef(
+                        sig=self.SIG,
+                        id=ref_id,
+                        name=str(ref["name"]),
+                        ref_type="Lake",
+                        url=f"https://llota.app/list/ref/{ref_id}",
+                        grid=grid,
+                        latitude=ll[0],
+                        longitude=ll[1],
+                    )
+                )
 
                 # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
                 # of the data in this case

providers/sigrefdata/local_file_sig_ref_data_provider.py

@@ -5,6 +5,8 @@ import pytz
 
 from providers.sigrefdata.sig_ref_data_provider import SIGRefDataProvider
 
+logger = logging.getLogger(__name__)
+
 
 class LocalFileSIGRefDataProvider(SIGRefDataProvider):
     """Generic SIG ref data provider class for providers that fetch their data from a local file on startup."""
@@ -14,7 +16,7 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
         self._path = path
 
     def start(self):
-        logging.debug("Loading " + self.sig_name + " SIG ref data from file.")
+        logger.debug(f"Loading {self.sig_name} SIG ref data from file.")
         try:
             new_data = self._file_to_data(self._path)
             if new_data:
@@ -23,10 +25,10 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
                 self.last_update_time = datetime.now(pytz.UTC)
             else:
                 self.status = "Error"
-                logging.info("Failed to load SIG ref data for " + self.sig_name)
-        except Exception as e:
+                logger.info(f"Failed to load SIG ref data for {self.sig_name}")
+        except Exception:
             self.status = "Error"
-            logging.error("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")", e, exc_info=True)
+            logger.exception(f"Exception in local file SIG Ref Data Provider ({self.sig_name})")
 
     def _file_to_data(self, path):
         """Load a file on the given path and turn it into SIG Ref data."""

providers/sigrefdata/mota.py

@@ -2,7 +2,9 @@ import csv
 from time import sleep
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class MOTA(FileDownloadSIGRefDataProvider):
@@ -19,13 +21,18 @@ class MOTA(FileDownloadSIGRefDataProvider):
         new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
             ref_id = row["Reference"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
-                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
-                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
-                                          "Latitude"] != "" else None,
-                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
-                                          "Longitude"] != "" else None,
-                                      grid=row["Maidenhead Locator"]))
+            new_data.append(
+                SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    name=row.get("Name", None),
+                    ref_type="Mill",
+                    url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
+                    latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
+                    longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
+                    grid=row["Maidenhead Locator"],
+                )
+            )
 
             # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
             # the data in this case

providers/sigrefdata/pnp_kml_sig_ref_data_provider.py

@@ -5,7 +5,9 @@ from fastkml import kml
 from pyhamtools.locator import latlong_to_locator
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
@@ -15,7 +17,7 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
     REF_PATTERN = re.compile(r"VKFF-\d+")
 
     def __init__(self, sig_name, provider_config, url, poll_interval):
-        """ Set up the provider, note poll_interval is in *days*."""
+        """Set up the provider, note poll_interval is in *days*."""
         super().__init__(sig_name, provider_config, url, poll_interval)
 
     def _http_response_to_data(self, http_response):
@@ -24,7 +26,9 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
         k = kml.KML.from_string(http_response.content)
 
         for document in k.features:
+            # noinspection unresolved-references
             for folder in document.features:
+                # noinspection unresolved-references
                 for placemark in folder.features:
                     description = placemark.description or ""
                     match = self.REF_PATTERN.search(description)
@@ -35,10 +39,15 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
 
                     longitude, latitude = placemark.geometry.x, placemark.geometry.y
 
-                    ref = SIGRef(sig=self.sig_name, id=ref_id, name=placemark.name,
-                                              url="https://parksnpeaks.org/getPark.php?actPark=" + ref_id,
-                                              latitude=latitude,
-                                              longitude=longitude)
+                    ref = SIGRef(
+                        sig=self.sig_name,
+                        id=ref_id,
+                        name=placemark.name,
+                        ref_type="Park",
+                        url=f"https://parksnpeaks.org/getPark.php?actPark={ref_id}",
+                        latitude=latitude,
+                        longitude=longitude,
+                    )
                     if latitude and longitude:
                         ref.grid = latlong_to_locator(latitude, longitude, 6)
                     new_data.append(ref)

providers/sigrefdata/pota.py

@@ -2,7 +2,9 @@ import csv
 from time import sleep
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class POTA(FileDownloadSIGRefDataProvider):
@@ -19,13 +21,18 @@ class POTA(FileDownloadSIGRefDataProvider):
         new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["reference"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
-                                      url="https://pota.app/#/park/" + ref_id,
-                                      grid=row["grid"] if "grid" in row else None,
-                                      latitude=float(row["latitude"]) if "latitude" in row and row[
-                                          "latitude"] != "" else None,
-                                      longitude=float(row["longitude"]) if "longitude" in row and row[
-                                          "longitude"] != "" else None))
+            new_data.append(
+                SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    name=row.get("name", None),
+                    ref_type="Park",
+                    url=f"https://pota.app/#/park/{ref_id}",
+                    grid=row.get("grid", None),
+                    latitude=float(row["latitude"]) if "latitude" in row and row["latitude"] != "" else None,
+                    longitude=float(row["longitude"]) if "longitude" in row and row["longitude"] != "" else None,
+                )
+            )
 
             # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
             # the data in this case

providers/sigrefdata/sanpcpa.py

@@ -1,4 +1,6 @@
-from providers.sigrefdata.pnp_kml_sig_ref_data_provider import ParksNPeaksKMLSIGRefDataProvider
+from providers.sigrefdata.pnp_kml_sig_ref_data_provider import (
+    ParksNPeaksKMLSIGRefDataProvider,
+)
 
 
 class SANPCPA(ParksNPeaksKMLSIGRefDataProvider):

providers/sigrefdata/sig_ref_data_provider.py

@@ -5,6 +5,8 @@ import pytz
 
 from core.data_store import DATA_STORE
 
+logger = logging.getLogger(__name__)
+
 
 class SIGRefDataProvider:
     """Generic SIG reference data provider class. Subclasses of this query the individual URLs or files for data."""
@@ -19,13 +21,11 @@ class SIGRefDataProvider:
         self.reference_count = 0
         self._stop = False
 
-
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
 
         raise NotImplementedError("Subclasses must implement this method")
 
-
     def stop(self):
         """Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
         super()."""
@@ -38,7 +38,7 @@ class SIGRefDataProvider:
         # with transact() batches all writes together to save making thousands of individual sqlite writes
         with DATA_STORE.sigrefs.transact():
             for d in new_data:
-                DATA_STORE.sigrefs.set(self.sig_name + ":" + d.id, d)
+                DATA_STORE.sigrefs.set(f"{self.sig_name}:{d.id}", d)
 
                 # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
                 # within the first few minutes of startup, we need a way to abort this expensive process of filling up the
@@ -47,4 +47,4 @@ class SIGRefDataProvider:
                     break
 
         self.reference_count = len(new_data)
-        logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)
+        logger.info(f"Loaded {self.reference_count} references for {self.sig_name} into the data store.")

providers/sigrefdata/siota.py

@@ -2,7 +2,9 @@ import csv
 from time import sleep
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class SIOTA(FileDownloadSIGRefDataProvider):
@@ -19,10 +21,17 @@ class SIOTA(FileDownloadSIGRefDataProvider):
         new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["SILO_CODE"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
-                                      grid=row["LOCATOR"] if "LOCATOR" in row else None,
-                                      latitude=float(row["LAT"]) if "LAT" in row else None,
-                                      longitude=float(row["LNG"]) if "LNG" in row else None))
+            new_data.append(
+                SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    name=row.get("NAME", None),
+                    ref_type="Silo",
+                    grid=row.get("LOCATOR", None),
+                    latitude=float(row["LAT"]) if "LAT" in row else None,
+                    longitude=float(row["LNG"]) if "LNG" in row else None,
+                )
+            )
 
             # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
             # the data in this case

providers/sigrefdata/sota.py

@@ -4,7 +4,9 @@ from time import sleep
 from pyhamtools.locator import latlong_to_locator
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class SOTA(FileDownloadSIGRefDataProvider):
@@ -23,11 +25,18 @@ class SOTA(FileDownloadSIGRefDataProvider):
             ref_id = row["SummitCode"]
             latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
             longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
-            ref = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
-                                      url="https://www.sotadata.org.uk/en/summit/" + ref_id,
-                                      latitude=latitude,
-                                      longitude=longitude,
-                                      activation_score=int(row["Points"]) if "Points" in row else None)
+            altitude = float(row["AltM"]) if "AltM" in row and row["AltM"] != "" else None
+            ref = SIGRef(
+                sig=self.SIG,
+                id=ref_id,
+                name=row.get("SummitName", None),
+                ref_type="Summit",
+                url=f"https://www.sotadata.org.uk/en/summit/{ref_id}",
+                latitude=latitude,
+                longitude=longitude,
+                altitude=altitude,
+                activation_score=int(row["Points"]) if "Points" in row else None,
+            )
             if latitude and longitude:
                 ref.grid = latlong_to_locator(latitude, longitude, 6)
             new_data.append(ref)

providers/sigrefdata/toilets.py

@@ -1,7 +1,9 @@
 import csv
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
+from providers.sigrefdata.local_file_sig_ref_data_provider import (
+    LocalFileSIGRefDataProvider,
+)
 
 
 class Toilets(LocalFileSIGRefDataProvider):
@@ -19,8 +21,16 @@ class Toilets(LocalFileSIGRefDataProvider):
             csv_data = _f.read()
             dr = csv.DictReader(csv_data.splitlines())
             for row in dr:
-                new_data.append(SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]),
-                                              longitude=float(row["lon"])))
+                new_data.append(
+                    SIGRef(
+                        sig=self.SIG,
+                        id=row["ref"],
+                        name=row["ref"],
+                        ref_type="Toilet",
+                        latitude=float(row["lat"]),
+                        longitude=float(row["lon"]),
+                    )
+                )
 
                 # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
                 # of the data in this case

providers/sigrefdata/towers.py

@@ -2,7 +2,9 @@ import csv
 from time import sleep
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class Towers(FileDownloadSIGRefDataProvider):
@@ -19,11 +21,18 @@ class Towers(FileDownloadSIGRefDataProvider):
         new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
             ref_id = row["Ref"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
-                                      url="https://wwtota.com/seznam/karta_rozhledny.php?ref=" + ref_id,
-                                      grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
-                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
-                                      longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None))
+            new_data.append(
+                SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    name=row.get("Nazev", None),
+                    ref_type="Tower",
+                    url=f"https://wwtota.com/seznam/karta_rozhledny.php?ref={ref_id}",
+                    grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
+                    latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
+                    longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None,
+                )
+            )
 
             # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
             # the data in this case

providers/sigrefdata/wca.py

@@ -5,7 +5,11 @@ from time import sleep
 from pyhamtools.locator import latlong_to_locator
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class WCA(FileDownloadSIGRefDataProvider):
@@ -34,13 +38,20 @@ class WCA(FileDownloadSIGRefDataProvider):
                     longitude = float(split[1])
                     grid = latlong_to_locator(latitude, longitude)
             except ValueError:
-                logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id)
+                logger.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for {ref_id}")
 
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
-                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
-                                      latitude=latitude,
-                                      longitude=longitude,
-                                      grid=grid))
+            new_data.append(
+                SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    name=row.get("CLEAN NAME", None),
+                    ref_type="Castle",
+                    url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
+                    latitude=latitude,
+                    longitude=longitude,
+                    grid=grid,
+                )
+            )
 
             # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
             # the data in this case

providers/sigrefdata/wota.py

@@ -1,7 +1,9 @@
 from time import sleep
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class WOTA(FileDownloadSIGRefDataProvider):
@@ -20,15 +22,24 @@ class WOTA(FileDownloadSIGRefDataProvider):
             ref_id = feature["properties"]["wotaId"]
             # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
             # added to them
-            url = "https://www.wota.org.uk/MM_" + ref_id
+            url = f"https://www.wota.org.uk/MM_{ref_id}"
             if ref_id.upper().startswith("LDO-"):
                 number = int(ref_id.upper().replace("LDO-", ""))
-                url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
+                url = f"https://www.wota.org.uk/MM_LDO-{number + 214!s}"
 
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
-                                      grid=feature["properties"]["qthLocator"],
-                                      latitude=feature["geometry"]["coordinates"][1],
-                                      longitude=feature["geometry"]["coordinates"][0]))
+            new_data.append(
+                SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    name=feature["properties"]["title"],
+                    url=url,
+                    ref_type="Summit",
+                    grid=feature["properties"]["qthLocator"],
+                    latitude=feature["geometry"]["coordinates"][1],
+                    longitude=feature["geometry"]["coordinates"][0],
+                    altitude=feature["properties"]["height"],
+                )
+            )
 
             # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
             # the data in this case

providers/sigrefdata/wwbota.py

@@ -2,7 +2,9 @@ import csv
 from time import sleep
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class WWBOTA(FileDownloadSIGRefDataProvider):
@@ -19,11 +21,18 @@ class WWBOTA(FileDownloadSIGRefDataProvider):
         new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["Reference"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
-                                      url="https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None,
-                                      grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
-                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
-                                      longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None))
+            new_data.append(
+                SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    name=row.get("Name", None),
+                    ref_type="Bunker",
+                    url=f"https://bunkerwiki.org/?s={ref_id}" if ref_id.startswith("B/G") else None,
+                    grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
+                    latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
+                    longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None,
+                )
+            )
 
             # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
             # the data in this case

providers/sigrefdata/wwff.py

@@ -2,7 +2,9 @@ import csv
 from time import sleep
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class WWFF(FileDownloadSIGRefDataProvider):
@@ -19,14 +21,22 @@ class WWFF(FileDownloadSIGRefDataProvider):
         new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["reference"]
-            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
-                                      url="https://wwff.co/directory/?showRef=" + ref_id,
-                                      grid=row["iaruLocator"] if "iaruLocator" in row and row[
-                                          "iaruLocator"] != "-" else None,
-                                      latitude=float(row["latitude"]) if "latitude" in row and row[
-                                          "latitude"] != "" and row["latitude"] != "-" else None,
-                                      longitude=float(row["longitude"]) if "longitude" in row and row[
-                                          "longitude"] != "" and row["longitude"] != "-" else None))
+            new_data.append(
+                SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    name=row.get("name", None),
+                    ref_type="Park",
+                    url=f"https://wwff.co/directory/?showRef={ref_id}",
+                    grid=row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None,
+                    latitude=float(row["latitude"])
+                    if "latitude" in row and row["latitude"] != "" and row["latitude"] != "-"
+                    else None,
+                    longitude=float(row["longitude"])
+                    if "longitude" in row and row["longitude"] != "" and row["longitude"] != "-"
+                    else None,
+                )
+            )
 
             # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
             # the data in this case

providers/sigrefdata/zlota.py

@@ -3,7 +3,9 @@ from time import sleep
 from pyhamtools.locator import latlong_to_locator
 
 from data.sig_ref import SIGRef
-from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+from providers.sigrefdata.file_download_sig_ref_data_provider import (
+    FileDownloadSIGRefDataProvider,
+)
 
 
 class ZLOTA(FileDownloadSIGRefDataProvider):
@@ -22,16 +24,29 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
         if isinstance(data, list):
             for ref in data:
                 ref_id = ref["code"]
-                latitude = ref["y"]
-                longitude = ref["x"]
+                latitude = ref["latitude"]
+                longitude = ref["longitude"]
 
-                ref = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
-                                          url="https://ontheair.nz/assets/" + ref_id.replace("/", "_"),
-                                          latitude=latitude,
-                                          longitude=longitude)
+                new_ref = SIGRef(
+                    sig=self.SIG,
+                    id=ref_id,
+                    name=ref["name"],
+                    ref_type=ref["asset_type"].title(),
+                    url=f"https://ontheair.nz/assets/{ref_id.replace('/', '_')}",
+                    latitude=latitude,
+                    longitude=longitude,
+                )
+
+                # Check lat/lon validity and update grid accordingly
                 if latitude and longitude:
-                    ref.grid = latlong_to_locator(latitude, longitude, 6)
-                new_data.append(ref)
+                    try:
+                        new_ref.grid = latlong_to_locator(latitude, longitude, 6)
+                    except ValueError:
+                        # Junk lat/lon, remove from the spot
+                        new_ref.latitude = None
+                        new_ref.longitude = None
+
+                new_data.append(new_ref)
 
                 # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest
                 # of the data in this case

providers/solarconditions/giroionosonde.py

@@ -1,20 +1,22 @@
 import csv
 import logging
-from datetime import datetime, timezone, timedelta
-from threading import Thread, Event
+from datetime import datetime, timedelta, timezone
+from threading import Event, Thread
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from providers.solarconditions.ionosonde_utils import compute_band_states
 from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider
 
-# Each station gets polled roughly once every hour (3600 seconds). Note that to avoid a burst of requests to the server
-# every hour, the requests for data from each station are spaced out throughout the hour, leading to one request being
-# sent every 1-2 minutes.
-POLL_INTERVAL = 3600
+logger = logging.getLogger(__name__)
+
+# Each station gets polled roughly once every two hours (7200 seconds). Note that to avoid a burst of requests to the
+# server every two hours, the requests for data from each station are spaced out throughout the period, leading to one
+# request being sent every 3-4 minutes.
+POLL_INTERVAL = 7200
 # To avoid looking up all stations in the GIRO system and working out which ones are providing live data, this has been
 # manually determined and a CSV provided of all the stations that we can query for live data.
 STATIONS_INDEX = "datafiles/didbase-stations.csv"
@@ -40,9 +42,16 @@ class GIROIonosonde(SolarConditionsProvider):
         # entries so KC2G cache data is preserved.
         existing = self._solar_conditions.ionosonde_data or {}
         new_entries = {
-            s["ursi"]: {"ursi": s["ursi"], "name": s["name"], "fof2": None, "muf": None,
-                        "luf": None, "band_states": None}
-            for s in self._stations if s["ursi"] not in existing
+            s["ursi"]: {
+                "ursi": s["ursi"],
+                "name": s["name"],
+                "fof2": None,
+                "muf": None,
+                "luf": None,
+                "band_states": None,
+            }
+            for s in self._stations
+            if s["ursi"] not in existing
         }
         if new_entries:
             self.update_data({"ionosonde_data": {**existing, **new_entries}})
@@ -50,14 +59,14 @@ class GIROIonosonde(SolarConditionsProvider):
     @staticmethod
     def _load_stations():
         stations = []
-        with open(STATIONS_INDEX, newline='') as f:
+        with open(STATIONS_INDEX, newline="") as f:
             for row in csv.reader(f):
                 if len(row) >= 2:
                     stations.append({"ursi": row[0].strip(), "name": row[1].strip()})
         return stations
 
     def start(self):
-        logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
+        logger.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
         self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider")
         self._thread.start()
 
@@ -79,7 +88,7 @@ class GIROIonosonde(SolarConditionsProvider):
         ursi = station["ursi"]
         name = station["name"]
         try:
-            logging.debug(f"Polling GIRO ionosonde data for {ursi} ({name})...")
+            logger.debug(f"Polling GIRO ionosonde data for {ursi} ({name})...")
             now = datetime.now(timezone.utc)
             from_time = now - timedelta(hours=HISTORY_HOURS)
             cutoff_ts = from_time.timestamp()
@@ -94,8 +103,14 @@ class GIROIonosonde(SolarConditionsProvider):
 
             # Merge GIRO's readings into any existing data for this station.
             existing = ionosonde_data.get(ursi, {})
-            merged_fof2 = {**{float(t): v for t, v in (existing.get("fof2") or {}).items()}, **fof2}
-            merged_muf = {**{float(t): v for t, v in (existing.get("muf") or {}).items()}, **muf}
+            merged_fof2 = {
+                **{float(t): v for t, v in (existing.get("fof2") or {}).items()},
+                **fof2,
+            }
+            merged_muf = {
+                **{float(t): v for t, v in (existing.get("muf") or {}).items()},
+                **muf,
+            }
             merged_luf = dict(luf) if luf else {}
 
             merged_fof2 = {t: v for t, v in merged_fof2.items() if t >= cutoff_ts}
@@ -104,7 +119,8 @@ class GIROIonosonde(SolarConditionsProvider):
 
             band_states = compute_band_states(merged_fof2, merged_muf, merged_luf)
             ionosonde_data[ursi] = {
-                "ursi": ursi, "name": name,
+                "ursi": ursi,
+                "name": name,
                 "fof2": merged_fof2 or None,
                 "muf": merged_muf or None,
                 "luf": merged_luf or None,
@@ -113,11 +129,11 @@ class GIROIonosonde(SolarConditionsProvider):
             self.update_data({"ionosonde_data": ionosonde_data})
             self.status = "OK"
             self.last_update_time = datetime.now(pytz.UTC)
-            logging.debug(f"Updated ionosonde data for {ursi} ({name}).")
+            logger.debug(f"Updated ionosonde data for {ursi} ({name}).")
 
         except Exception:
             self.status = "Error"
-            logging.exception(f"Exception fetching GIRO ionosonde data for {ursi} ({name})")
+            logger.exception(f"Exception fetching GIRO ionosonde data for {ursi} ({name})")
 
     def _fetch_station_data(self, ursi, from_time, to_time):
         """Fetch foF2, MUF and LUF readings for a station. Returns (fof2_dict, muf_dict, luf_dict) keyed by UNIX timestamp."""
@@ -128,14 +144,14 @@ class GIROIonosonde(SolarConditionsProvider):
         try:
             http_response = requests.get(url, headers=HTTP_HEADERS, timeout=(5, 15))
             if not http_response.ok:
-                logging.warning(f"HTTP {http_response.status_code} when calling Giro ionosonde API.")
+                logger.warning(f"HTTP {http_response.status_code} when calling Giro ionosonde API.")
                 return None, None, None
             return self._parse_all(http_response.text)
         except (ConnectTimeout, ReadTimeout):
-            logging.warning(f"Timeout when accessing Giro ionosonde API.")
+            logger.warning("Timeout when accessing Giro ionosonde API.")
             return None, None, None
         except ConnectionError:
-            logging.warning("Connection error when accessing Giro ionosonde API.")
+            logger.warning("Connection error when accessing Giro ionosonde API.")
             return None, None, None
 
     @staticmethod
@@ -147,14 +163,14 @@ class GIROIonosonde(SolarConditionsProvider):
         luf_data = {}
         for line in text.splitlines():
             line = line.strip()
-            if not line or line.startswith('#'):
+            if not line or line.startswith("#"):
                 continue
             # Data rows have the following format: timestamp CS foF2 QD MUFD QD fmin QD
             parts = line.split()
             if len(parts) >= 5:
                 try:
                     # Python 3.8 TZ parsing fudge
-                    ts = datetime.fromisoformat(parts[0].replace('Z', '+00:00')).timestamp()
+                    ts = datetime.fromisoformat(parts[0].replace("Z", "+00:00")).timestamp()
                 except ValueError:
                     continue
                 try:

providers/solarconditions/hamqsl.py

@@ -2,9 +2,14 @@ import logging
 from xml.etree import ElementTree
 
 import pytz
-from dateutil import parser as dateutil_parser, tz as dateutil_tz
+from dateutil import parser as dateutil_parser
+from dateutil import tz as dateutil_tz
 
-from providers.solarconditions.http_solar_conditions_provider import HTTPSolarConditionsProvider
+from providers.solarconditions.http_solar_conditions_provider import (
+    HTTPSolarConditionsProvider,
+)
+
+logger = logging.getLogger(__name__)
 
 POLL_INTERVAL = 3600  # 1 hour
 URL = "https://www.hamqsl.com/solarxml.php"
@@ -21,14 +26,14 @@ class HamQSL(HTTPSolarConditionsProvider):
         root = ElementTree.fromstring(http_response.text)
         sd = root.find("solardata")
         if sd is None:
-            logging.warning("HamQSL solar conditions API returned unexpected XML structure")
+            logger.warning("HamQSL solar conditions API returned unexpected XML structure")
             return None
 
         # Some error checking functions in case the data is janky.
 
         def text(tag, default=None):
             if sd is None:
-                logging.warning("HamQSL solar conditions API returned unexpected XML structure")
+                logger.warning("HamQSL solar conditions API returned unexpected XML structure")
                 return default
             el = sd.find(tag)
             return el.text.strip() if el is not None and el.text else default
@@ -72,11 +77,11 @@ class HamQSL(HTTPSolarConditionsProvider):
                 tz_abbr = updated_str.split()[-1]
                 timezone = dateutil_tz.gettz(tz_abbr)
                 if timezone is None:
-                    raise ValueError("Unknown timezone abbreviation: " + tz_abbr)
+                    raise ValueError(f"Unknown timezone abbreviation: {tz_abbr}")
                 dt = dateutil_parser.parse(updated_str, tzinfos={tz_abbr: timezone})
                 updated = dt.astimezone(pytz.UTC).timestamp()
             except (ValueError, IndexError):
-                logging.warning("HamQSL solar conditions API returned unrecognised timestamp format: " + updated_str)
+                logger.warning(f"HamQSL solar conditions API returned unrecognised timestamp format: {updated_str}")
 
         # Return the data ready to be put into the solar conditions object.
         return {
@@ -92,7 +97,8 @@ class HamQSL(HTTPSolarConditionsProvider):
             "aurora_latitude": float_val("latdegree"),
             "solar_wind": float_val("solarwind"),
             "magnetic_field": float_val("magneticfield"),
-            "geomag_field": text("geomagfield").title()
+            "geomag_field": text("geomagfield")
+            .title()
             .replace("Vr Quiet", "Very Quiet")
             .replace("Unsettld", "Unsettled")
             .replace("Min Strm", "Minor Storm")
@@ -102,8 +108,10 @@ class HamQSL(HTTPSolarConditionsProvider):
             "geomag_noise": text("signalnoise"),
             "hf_conditions": hf_conditions,
             "vhf_conditions": {
-                "vhf_aurora_northern_hemi": (vhf_map.get(("vhf-aurora", "northern_hemi")) or "").title().replace(
-                    "Lat Aur", "Latitude") or None,
+                "vhf_aurora_northern_hemi": (vhf_map.get(("vhf-aurora", "northern_hemi")) or "")
+                .title()
+                .replace("Lat Aur", "Latitude")
+                or None,
                 "es_2m_europe": vhf_map.get(("E-Skip", "europe")),
                 "es_4m_europe": vhf_map.get(("E-Skip", "europe_4m")),
                 "es_6m_europe": vhf_map.get(("E-Skip", "europe_6m")),

providers/solarconditions/http_solar_conditions_provider.py

@@ -1,14 +1,16 @@
 import logging
 from datetime import datetime
-from threading import Thread, Event
+from threading import Event, Thread
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider
 
+logger = logging.getLogger(__name__)
+
 
 class HTTPSolarConditionsProvider(SolarConditionsProvider):
     """Generic solar conditions provider for providers that request data via HTTP(S). Subclasses implement
@@ -22,8 +24,7 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
         self._stop_event = Event()
 
     def start(self):
-        logging.info(
-            "Set up query of " + self.name + " solar conditions API every " + str(self._poll_interval) + " seconds.")
+        logger.info(f"Set up query of {self.name} solar conditions API every {self._poll_interval!s} seconds.")
         self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}")
         self._thread.start()
 
@@ -38,7 +39,7 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
 
     def _poll(self):
         try:
-            logging.debug("Polling " + self.name + " solar conditions API...")
+            logger.debug(f"Polling {self.name} solar conditions API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
             # Check response code was good
             if http_response.ok:
@@ -47,18 +48,18 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
 
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
-                logging.debug("Received data from " + self.name + " solar conditions API.")
+                logger.debug(f"Received data from {self.name} solar conditions API.")
             else:
                 self.status = "Error"
-                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} solar conditions API.")
+                logger.warning(f"HTTP {http_response.status_code} when calling {self.name} solar conditions API.")
 
         except ConnectionError:
-            logging.warning(f"Connection error when accessing {self.name} solar conditions API.")
+            logger.warning(f"Connection error when accessing {self.name} solar conditions API.")
         except (ConnectTimeout, ReadTimeout):
-            logging.warning(f"Timeout when accessing {self.name} solar conditions API.")
+            logger.warning(f"Timeout when accessing {self.name} solar conditions API.")
         except Exception:
             self.status = "Error"
-            logging.exception("Exception in HTTP Solar Conditions Provider (" + self.name + ")")
+            logger.exception(f"Exception in HTTP Solar Conditions Provider ({self.name})")
             self._stop_event.wait(timeout=1)
 
     def _http_response_to_solar_conditions(self, http_response):

providers/solarconditions/kc2gprop.py

@@ -1,15 +1,17 @@
 import logging
-from datetime import datetime, timezone, timedelta
-from threading import Thread, Event
+from datetime import datetime, timedelta, timezone
+from threading import Event, Thread
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from providers.solarconditions.ionosonde_utils import compute_band_states
 from providers.solarconditions.solar_conditions_provider import SolarConditionsProvider
 
+logger = logging.getLogger(__name__)
+
 POLL_INTERVAL = 900  # 15 minutes
 KC2G_URL = "https://prop.kc2g.com/api/stations.json"
 HISTORY_HOURS = 24
@@ -29,7 +31,7 @@ class KC2GProp(SolarConditionsProvider):
         self._stop_event = Event()
 
     def start(self):
-        logging.info(f"Set up query of KC2G ionosonde data API every {POLL_INTERVAL} seconds.")
+        logger.info(f"Set up query of KC2G ionosonde data API every {POLL_INTERVAL} seconds.")
         self._thread = Thread(target=self._run, name="KC2GPropProvider")
         self._thread.start()
 
@@ -44,10 +46,10 @@ class KC2GProp(SolarConditionsProvider):
 
     def _poll(self):
         try:
-            logging.debug("Polling KC2G ionosonde data...")
+            logger.debug("Polling KC2G ionosonde data...")
             http_response = requests.get(KC2G_URL, headers=HTTP_HEADERS, timeout=(5, 30))
             if not http_response.ok:
-                logging.warning(f"HTTP {http_response.status_code} when calling KG2G ionosonde API.")
+                logger.warning(f"HTTP {http_response.status_code} when calling KG2G ionosonde API.")
                 return
 
             now = datetime.now(timezone.utc)
@@ -114,13 +116,13 @@ class KC2GProp(SolarConditionsProvider):
             self.update_data({"ionosonde_data": ionosonde_data})
             self.status = "OK"
             self.last_update_time = datetime.now(pytz.UTC)
-            logging.debug(f"Updated KC2G ionosonde data for {updated_count} stations.")
+            logger.debug(f"Updated KC2G ionosonde data for {updated_count} stations.")
 
         except ConnectionError:
-            logging.warning("Connection error when accessing KC2G ionosonde API.")
+            logger.warning("Connection error when accessing KC2G ionosonde API.")
         except (ConnectTimeout, ReadTimeout):
-            logging.warning(f"Timeout when accessing KC2G ionosonde API.")
+            logger.warning("Timeout when accessing KC2G ionosonde API.")
         except Exception:
             self.status = "Error"
-            logging.exception("Exception in KC2G ionosonde data provider")
+            logger.exception("Exception in KC2G ionosonde data provider")
             self._stop_event.wait(timeout=1)

providers/solarconditions/noaa3dayforecast.py

@@ -2,7 +2,11 @@ import logging
 import re
 from datetime import datetime, timezone
 
-from providers.solarconditions.http_solar_conditions_provider import HTTPSolarConditionsProvider
+from providers.solarconditions.http_solar_conditions_provider import (
+    HTTPSolarConditionsProvider,
+)
+
+logger = logging.getLogger(__name__)
 
 POLL_INTERVAL = 10800  # Every 3 hours
 URL = "https://services.swpc.noaa.gov/text/3-day-forecast.txt"
@@ -26,21 +30,21 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
                 start_idx = i
                 break
         if start_idx is None:
-            logging.warning(f"NOAA 3-day forecast: could not find '{section_header}' section")
+            logger.warning(f"NOAA 3-day forecast: could not find '{section_header}' section")
             return None
 
         # Find the date header line by scanning the next few lines for month & day patterns
         date_header_idx = None
         for j in range(start_idx + 1, min(start_idx + 6, len(lines))):
-            if re.search(r'[A-Za-z]{3}\s+\d{2}', lines[j]):
+            if re.search(r"[A-Za-z]{3}\s+\d{2}", lines[j]):
                 date_header_idx = j
                 break
         if date_header_idx is None:
-            logging.warning(f"NOAA 3-day forecast: could not find date header after '{section_header}'")
+            logger.warning(f"NOAA 3-day forecast: could not find date header after '{section_header}'")
             return None
-        date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', lines[date_header_idx])
+        date_matches = re.findall(r"([A-Za-z]{3})\s+(\d{2})", lines[date_header_idx])
         if not date_matches:
-            logging.warning(f"NOAA 3-day forecast: no dates in header: {lines[date_header_idx]}")
+            logger.warning(f"NOAA 3-day forecast: no dates in header: {lines[date_header_idx]}")
             return None
 
         # Figure out the date based on the line found
@@ -50,25 +54,25 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
                 dt = datetime.strptime(f"{day_str} {month_str} {year}", "%d %b %Y").replace(tzinfo=timezone.utc)
                 column_timestamps.append(dt.timestamp())
             except ValueError:
-                logging.warning(f"NOAA 3-day forecast: could not parse date: {month_str} {day_str} {year}")
+                logger.warning(f"NOAA 3-day forecast: could not parse date: {month_str} {day_str} {year}")
                 return None
 
         # Parse data rows. Each non-empty line should have a text label followed by percentage values
         result = {}
-        for line in lines[date_header_idx + 1:]:
+        for line in lines[date_header_idx + 1 :]:
             line_stripped = line.strip()
             if not line_stripped:
                 if result:
                     break
                 continue
-            pct_matches = list(re.finditer(r'\b(\d+)%', line_stripped))
+            pct_matches = list(re.finditer(r"\b(\d+)%", line_stripped))
             if not pct_matches:
                 if result:
                     break
                 continue
 
             # Row label is everything before the first percentage value
-            row_label = line_stripped[:line_stripped.index(pct_matches[0].group())].strip()
+            row_label = line_stripped[: line_stripped.index(pct_matches[0].group())].strip()
             row_data = {}
             for j, match in enumerate(pct_matches):
                 if j >= len(column_timestamps):
@@ -89,26 +93,26 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
                 start_idx = i
                 break
         if start_idx is None:
-            logging.warning("NOAA K-index forecast: could not find 'NOAA Kp index breakdown' section")
+            logger.warning("NOAA K-index forecast: could not find 'NOAA Kp index breakdown' section")
             return None
 
         # Extract the year from the header line, e.g. "NOAA Kp index breakdown Apr 2-Apr 4, 2026"
         header_line = lines[start_idx]
-        year_match = re.search(r'\b(\d{4})\b', header_line)
+        year_match = re.search(r"\b(\d{4})\b", header_line)
         if not year_match:
-            logging.warning("NOAA K-index forecast: could not extract year from: " + header_line)
+            logger.warning(f"NOAA K-index forecast: could not extract year from: {header_line}")
             return None
         year = int(year_match.group(1))
 
         # Parse the column date headers on the next line, e.g. "              Apr 02     Apr 03     Apr 04"
         if start_idx + 1 >= len(lines):
-            logging.warning("NOAA K-index forecast: missing date header line")
+            logger.warning("NOAA K-index forecast: missing date header line")
             return None
 
         date_header_line = lines[start_idx + 2]
-        date_matches = re.findall(r'([A-Za-z]{3})\s+(\d{2})', date_header_line)
+        date_matches = re.findall(r"([A-Za-z]{3})\s+(\d{2})", date_header_line)
         if not date_matches:
-            logging.warning("NOAA K-index forecast: could not parse date headers from: " + date_header_line)
+            logger.warning(f"NOAA K-index forecast: could not parse date headers from: {date_header_line}")
             return None
 
         column_dates = []
@@ -116,13 +120,13 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
             try:
                 column_dates.append(datetime.strptime(f"{day_str} {month_str} {year}", "%d %b %Y").date())
             except ValueError:
-                logging.warning(f"NOAA K-index forecast: could not parse date: {month_str} {day_str} {year}")
+                logger.warning(f"NOAA K-index forecast: could not parse date: {month_str} {day_str} {year}")
                 return None
 
         # Parse each data row, e.g. "00-03UT          2.00          3.00          2.00"
         k_index_forecast = {}
-        for line in lines[start_idx + 3:]:
-            time_match = re.match(r'^(\d{2})-(\d{2})UT\s+(.*)', line.strip())
+        for line in lines[start_idx + 3 :]:
+            time_match = re.match(r"^(\d{2})-(\d{2})UT\s+(.*)", line.strip())
             if not time_match:
                 if k_index_forecast:
                     break
@@ -130,7 +134,7 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
 
             start_hour = int(time_match.group(1))
             # Split on 2 or more spaces so that e.g. "5.67 (G2)" stays as one token per column
-            raw_values = re.split(r' {2,}', time_match.group(3).strip())
+            raw_values = re.split(r" {2,}", time_match.group(3).strip())
 
             for i, val in enumerate(raw_values):
                 if i >= len(column_dates):
@@ -142,14 +146,22 @@ class NOAA3dayForecast(HTTPSolarConditionsProvider):
                     continue
 
                 date = column_dates[i]
-                start_dt = datetime(date.year, date.month, date.day, start_hour, 0, 0, tzinfo=timezone.utc)
+                start_dt = datetime(
+                    date.year,
+                    date.month,
+                    date.day,
+                    start_hour,
+                    0,
+                    0,
+                    tzinfo=timezone.utc,
+                )
 
                 # Key the data dict by start time
                 key = start_dt.timestamp()
                 k_index_forecast[key] = kp
 
         if not k_index_forecast:
-            logging.warning("NOAA K-index forecast: no data rows parsed")
+            logger.warning("NOAA K-index forecast: no data rows parsed")
             return None
 
         # Parse Solar Radiation Storm Forecast (single row: "S1 or greater")

providers/solarconditions/solar_conditions_provider.py

@@ -7,7 +7,7 @@ from core.data_store import DATA_STORE
 
 class SolarConditionsProvider:
     """Generic solar conditions provider class. Subclasses of this query individual APIs for space weather and
-     propagation data."""
+    propagation data."""
 
     def __init__(self, name, provider_config):
         """Constructor"""

providers/spot/aprsis.py

@@ -9,6 +9,8 @@ from core.config import SERVER_OWNER_CALLSIGN
 from data.spot import Spot
 from providers.spot.spot_provider import SpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class APRSIS(SpotProvider):
     """Spot provider for the APRS-IS."""
@@ -25,10 +27,10 @@ class APRSIS(SpotProvider):
     def _connect(self):
         self._aprsis = aprslib.IS(SERVER_OWNER_CALLSIGN)
         self.status = "Connecting"
-        logging.info("APRS-IS connecting...")
+        logger.info("APRS-IS connecting...")
         self._aprsis.connect()
         self._aprsis.consumer(self._handle)
-        logging.info("APRS-IS connected.")
+        logger.info("APRS-IS connected.")
 
     def stop(self):
         self.status = "Shutting down"
@@ -43,20 +45,21 @@ class APRSIS(SpotProvider):
         via_parts = str(data["via"]).split("-")
         de_call = via_parts[0].upper()
         de_ssid = via_parts[1].upper() if len(via_parts) > 1 else None
-        spot = Spot(source="APRS-IS",
-                    dx_call=dx_call,
-                    dx_ssid=dx_ssid,
-                    de_call=de_call,
-                    de_ssid=de_ssid,
-                    comment=str(data["comment"]) if "comment" in data else None,
-                    dx_latitude=float(data["latitude"]) if "latitude" in data else None,
-                    dx_longitude=float(data["longitude"]) if "longitude" in data else None,
-                    time=datetime.now(
-                        pytz.UTC).timestamp())  # APRS-IS spots are live so we can assume spot time is "now"
+        spot = Spot(
+            source="APRS-IS",
+            dx_call=dx_call,
+            dx_ssid=dx_ssid,
+            de_call=de_call,
+            de_ssid=de_ssid,
+            comment=str(data["comment"]) if "comment" in data else None,
+            dx_latitude=float(data["latitude"]) if "latitude" in data else None,
+            dx_longitude=float(data["longitude"]) if "longitude" in data else None,
+            time=datetime.now(pytz.UTC).timestamp(),
+        )  # APRS-IS spots are live so we can assume spot time is "now"
 
         # Add to our list
         self._submit(spot)
 
         self.status = "OK"
         self.last_update_time = datetime.now(pytz.UTC)
-        logging.debug("Data received from APRS-IS.")
+        logger.debug("Data received from APRS-IS.")

providers/spot/dxcluster.py

@@ -11,6 +11,8 @@ from core.config import SERVER_OWNER_CALLSIGN
 from data.spot import Spot
 from providers.spot.spot_provider import SpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class DXCluster(SpotProvider):
     """Spot provider for a DX Cluster. Hostname, port, login_prompt, login_callsign and allow_rbn_spots are provided in config.
@@ -18,23 +20,26 @@ class DXCluster(SpotProvider):
 
     _LINE_PATTERN_EXCLUDE_RBN = re.compile(
         r"^DX de ([a-z0-9/]+):\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
-        re.IGNORECASE)
+        re.IGNORECASE,
+    )
     _LINE_PATTERN_ALLOW_RBN = re.compile(
         r"^DX de ([a-z0-9/]+)-?#?:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
-        re.IGNORECASE)
+        re.IGNORECASE,
+    )
 
     def __init__(self, provider_config):
         """Constructor requires hostname and port"""
 
-        name = provider_config["name"] if "name" in provider_config else "Cluster"
+        name = provider_config.get("name", "Cluster")
         super().__init__(name, provider_config)
         self._hostname = provider_config["host"]
         self._port = provider_config["port"]
-        self._login_prompt = provider_config["login_prompt"] if "login_prompt" in provider_config else "login:"
-        self._login_callsign = provider_config[
-            "login_callsign"] if "login_callsign" in provider_config else SERVER_OWNER_CALLSIGN
-        self._allow_rbn_spots = provider_config["allow_rbn_spots"] if "allow_rbn_spots" in provider_config else False
-        self._spot_line_pattern = self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
+        self._login_prompt = provider_config.get("login_prompt", "login:")
+        self._login_callsign = provider_config.get("login_callsign", SERVER_OWNER_CALLSIGN)
+        self._allow_rbn_spots = provider_config.get("allow_rbn_spots", False)
+        self._spot_line_pattern = (
+            self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
+        )
         self._telnet = None
         self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}")
         self._thread.daemon = True
@@ -45,7 +50,8 @@ class DXCluster(SpotProvider):
 
     def stop(self):
         self._running = False
-        self._telnet.close()
+        if self._telnet:
+            self._telnet.close()
         self._thread.join()
 
     def _handle(self):
@@ -54,19 +60,19 @@ class DXCluster(SpotProvider):
             while not connected and self._running:
                 try:
                     self.status = "Connecting"
-                    logging.info("DX Cluster " + self._hostname + " connecting...")
+                    logger.info(f"DX Cluster {self._hostname} connecting...")
                     self._telnet = telnetlib3.Telnet(self._hostname, self._port)
                     self._telnet.read_until(self._login_prompt.encode("latin-1"))
-                    self._telnet.write((self._login_callsign + "\n").encode("latin-1"))
+                    self._telnet.write(f"{self._login_callsign}\n".encode("latin-1"))
                     connected = True
-                    logging.info("DX Cluster " + self._hostname + " connected.")
+                    logger.info(f"DX Cluster {self._hostname} connected.")
                 except ConnectionRefusedError:
                     self.status = "Error"
-                    logging.warning("Connection refused to DX cluster " + self._hostname)
+                    logger.warning(f"Connection refused to DX cluster {self._hostname}")
                     sleep(300)
                 except Exception:
                     self.status = "Error"
-                    logging.exception("Exception while connecting to DX Cluster Provider (" + self._hostname + ").")
+                    logger.exception(f"Exception while connecting to DX Cluster Provider ({self._hostname}).")
                     sleep(5)
 
             self.status = "Waiting for Data"
@@ -77,30 +83,44 @@ class DXCluster(SpotProvider):
                     match = self._spot_line_pattern.match(telnet_output.decode("latin-1"))
                     if match:
                         spot_time = datetime.strptime(match.group(5), "%H%MZ")
-                        spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(),
-                                                         tzinfo=pytz.UTC)
-                        spot = Spot(source=self.name,
-                                    dx_call=match.group(3),
-                                    de_call=match.group(1),
-                                    freq=float(match.group(2)) * 1000,
-                                    comment=match.group(4).strip(),
-                                    time=spot_datetime.timestamp())
+                        spot_datetime = datetime.combine(
+                            datetime.now(pytz.UTC).date(),
+                            spot_time.time(),
+                            tzinfo=pytz.UTC,
+                        )
+                        spot = Spot(
+                            source=self.name,
+                            dx_call=match.group(3),
+                            de_call=match.group(1),
+                            freq=float(match.group(2)) * 1000,
+                            comment=match.group(4).strip(),
+                            time=spot_datetime.timestamp(),
+                        )
 
                         # Add to our list
                         self._submit(spot)
 
                         self.status = "OK"
                         self.last_update_time = datetime.now(pytz.UTC)
-                        logging.debug("Data received from DX Cluster " + self._hostname + ".")
+                        logger.debug(f"Data received from DX Cluster {self._hostname}.")
 
+                except EOFError:
+                    connected = False
+                    if self._running:
+                        self.status = "Restarting"
+                        logger.warning(f"Disconnected from DX Cluster {self._hostname}. Reconnecting...")
+                        sleep(5)
+                    else:
+                        logger.info(f"DX Cluster {self._hostname} shutting down...")
+                        self.status = "Shutting down"
                 except Exception:
                     connected = False
                     if self._running:
                         self.status = "Error"
-                        logging.exception("Exception in DX Cluster Provider (" + self._hostname + ")")
+                        logger.exception(f"Exception in DX Cluster Provider ({self._hostname})")
                         sleep(5)
                     else:
-                        logging.info("DX Cluster " + self._hostname + " shutting down...")
+                        logger.info(f"DX Cluster {self._hostname} shutting down...")
                         self.status = "Shutting down"
 
             self.status = "Disconnected"

providers/spot/gma.py

@@ -9,6 +9,8 @@ from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class GMA(HTTPSpotProvider):
     """Spot provider for General Mountain Activity"""
@@ -24,10 +26,15 @@ class GMA(HTTPSpotProvider):
         self._api_key = provider_config.get("api_key", "")
         if self._api_key == "":
             provider_config["enabled"] = False
-            logging.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
+            logger.warning("GMA spot provider configured but no api key was provided, this API will not be queried.")
         self._url_data_cache = URLDataCache("GMA")
 
-        super().__init__("GMA", provider_config, self.SPOTS_URL + "?key=" + self._api_key, self.POLL_INTERVAL_SEC)
+        super().__init__(
+            "GMA",
+            provider_config,
+            f"{self.SPOTS_URL}?key={self._api_key}",
+            self.POLL_INTERVAL_SEC,
+        )
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
@@ -41,43 +48,68 @@ class GMA(HTTPSpotProvider):
 
                 # Seen some real janky times from GMA, if we don't understand it just ignore this spot
                 try:
-                    time = datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M").replace(
-                                tzinfo=pytz.UTC).timestamp()
+                    time = (
+                        datetime.strptime(source_spot["DATE"] + source_spot["TIME"], "%Y%m%d%H%M")
+                        .replace(tzinfo=pytz.UTC)
+                        .timestamp()
+                    )
                 except ValueError:
                     continue
 
-                spot = Spot(source=self.name,
-                            dx_call=source_spot["ACTIVATOR"].upper(),
-                            de_call=source_spot["SPOTTER"].upper(),
-                            # Seen GMA spots with no frequency or with "QRT" in this field
-                            freq=float(source_spot["QRG"]) * 1000 if (
-                                    source_spot["QRG"] != "" and source_spot["QRG"] != "QRT") else None,
-                            # Filter out some weird mode strings
-                            mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
-                            comment=source_spot["TEXT"],
-                            sig_refs=[SIGRef(id=source_spot["REF"], sig="", name=source_spot["NAME"], latitude=lat,
-                                             longitude=lon)],
-                            time=time,
-                            dx_latitude=lat,
-                            dx_longitude=lon,
-                            qrt=source_spot["QRG"] == "QRT")
+                spot = Spot(
+                    source=self.name,
+                    dx_call=source_spot["ACTIVATOR"].upper(),
+                    de_call=source_spot["SPOTTER"].upper(),
+                    # Seen GMA spots with no frequency or with "QRT" in this field
+                    freq=float(source_spot["QRG"]) * 1000
+                    if (source_spot["QRG"] != "" and source_spot["QRG"] != "QRT")
+                    else None,
+                    # Filter out some weird mode strings
+                    mode=source_spot["MODE"].upper() if "<>" not in source_spot["MODE"] else None,
+                    comment=source_spot["TEXT"],
+                    sig_refs=[
+                        SIGRef(
+                            id=source_spot["REF"],
+                            sig="",
+                            name=source_spot["NAME"],
+                            latitude=lat,
+                            longitude=lon,
+                        )
+                    ],
+                    time=time,
+                    dx_latitude=lat,
+                    dx_longitude=lon,
+                    qrt=source_spot["QRG"] == "QRT",
+                )
 
                 # GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
                 if "REF" in source_spot:
                     try:
-                        ref_response = self._url_data_cache.get(self.REF_INFO_URL_ROOT + source_spot["REF"],
-                                                          headers=HTTP_HEADERS)
+                        ref_response = self._url_data_cache.get(
+                            self.REF_INFO_URL_ROOT + source_spot["REF"],
+                            headers=HTTP_HEADERS,
+                        )
                         # Sometimes this is blank even if it's a 200 response, so handle that
-                        if ref_response.ok and ref_response.text is not None and ref_response.text != "" and ref_response.text != "\n":
+                        if (
+                            ref_response.ok
+                            and ref_response.text is not None
+                            and ref_response.text != ""
+                            and ref_response.text != "\n"
+                        ):
                             ref_info = ref_response.json()
                             # If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. POTA and WWFF
                             # spots come through with reftype=POTA or reftype=WWFF. SOTA is harder to figure out because both SOTA
                             # and GMA summits come through with reftype=Summit, so we must check for the presence of a "sota" entry
                             # to determine if it's a SOTA summit.
-                            if spot.sig_refs and "reftype" in ref_info and ref_info["reftype"] not in ["POTA",
-                                                                                                       "WWFF"] and (
-                                    ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info[
-                                "sota"] == ""):
+                            if (
+                                spot.sig_refs
+                                and ref_info
+                                and "reftype" in ref_info
+                                and ref_info["reftype"] not in ["POTA", "WWFF"]
+                                and (
+                                    ref_info["reftype"] != "Summit" or "sota" not in ref_info or ref_info["sota"] == ""
+                                )
+                            ):
                                 match ref_info["reftype"]:
                                     case "Summit":
                                         spot.sig_refs[0].sig = "GMA"
@@ -85,6 +117,9 @@ class GMA(HTTPSpotProvider):
                                     case "IOTA Island":
                                         spot.sig_refs[0].sig = "IOTA"
                                         spot.sig = "IOTA"
+                                    case "GMA Island":
+                                        spot.sig_refs[0].sig = "GMA Island"
+                                        spot.sig = "GMA Island"
                                     case "Lighthouse (ILLW)":
                                         spot.sig_refs[0].sig = "ILLW"
                                         spot.sig = "ILLW"
@@ -98,27 +133,31 @@ class GMA(HTTPSpotProvider):
                                         spot.sig_refs[0].sig = "MOTA"
                                         spot.sig = "MOTA"
                                     case _:
-                                        logging.warning("GMA spot found with ref type " + ref_info[
-                                            "reftype"] + ", developer needs to add support for this!")
+                                        logger.warning(
+                                            f"GMA spot found with ref type {ref_info['reftype']}, developer needs to add support for this!"
+                                        )
                                         spot.sig_refs[0].sig = ref_info["reftype"]
                                         spot.sig = ref_info["reftype"]
 
-                                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point;
-                                # other code will do that for us.
-                                new_spots.append(spot)
-
                         elif not ref_response.from_cache:
                             if not ref_response.ok:
-                                logging.warning(
-                                    f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot['REF']}")
+                                logger.warning(
+                                    f"HTTP {ref_response.status_code} when looking up GMA ref {source_spot['REF']}"
+                                )
                             else:
-                                logging.warning(
-                                    f"GMA API returned a malformed response when looking up ref {source_spot['REF']}")
-                    except:
-                        logging.warning("Exception when looking up " + self.REF_INFO_URL_ROOT + source_spot[
-                            "REF"] + ", ignoring this spot for now", exc_info=True)
+                                logger.debug(
+                                    f"GMA API had no data for {source_spot['REF']}, it may not be a valid reference."
+                                )
+                    except Exception:
+                        logger.exception(
+                            f"Exception when looking up {self.REF_INFO_URL_ROOT}{source_spot['REF']}, SIG data will not be populated for the spot."
+                        )
+
+                # Add to our list. Don't worry about de-duping, removing old spots etc. at this point;
+                # other code will do that for us.
+                new_spots.append(spot)
         else:
-            logging.warning(f"The GMA API returned an unexpected response (HTTP {http_response.status_code}).")
+            logger.warning(f"The GMA API returned an unexpected response (HTTP {http_response.status_code}).")
 
         return new_spots
 

providers/spot/hema.py

@@ -4,13 +4,15 @@ from datetime import datetime
 
 import pytz
 import requests
-from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class HEMA(HTTPSpotProvider):
     """Spot provider for HuMPs Excluding Marilyns Award"""
@@ -52,27 +54,37 @@ class HEMA(HTTPSpotProvider):
                             continue
 
                         # Convert to our spot format
-                        spot = Spot(source=self.name,
-                                    dx_call=spot_items[2].upper(),
-                                    de_call=spotter_comment_match.group(1).upper(),
-                                    freq=float(freq_mode_match.group(1)) * 1000000,
-                                    mode=freq_mode_match.group(2).upper(),
-                                    comment=spotter_comment_match.group(2),
+                        spot = Spot(
+                            source=self.name,
+                            dx_call=spot_items[2].upper(),
+                            de_call=spotter_comment_match.group(1).upper(),
+                            freq=float(freq_mode_match.group(1)) * 1000000,
+                            mode=freq_mode_match.group(2).upper(),
+                            comment=spotter_comment_match.group(2),
+                            sig="HEMA",
+                            sig_refs=[
+                                SIGRef(
+                                    id=spot_items[3].upper(),
                                     sig="HEMA",
-                                    sig_refs=[SIGRef(id=spot_items[3].upper(), sig="HEMA", name=spot_items[4],
-                                                     latitude=float(spot_items[7]), longitude=float(spot_items[8]))],
-                                    time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M").replace(
-                                        tzinfo=pytz.UTC).timestamp(),
-                                    dx_latitude=float(spot_items[7]),
-                                    dx_longitude=float(spot_items[8]))
+                                    name=spot_items[4],
+                                    latitude=float(spot_items[7]),
+                                    longitude=float(spot_items[8]),
+                                )
+                            ],
+                            time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M")
+                            .replace(tzinfo=pytz.UTC)
+                            .timestamp(),
+                            dx_latitude=float(spot_items[7]),
+                            dx_longitude=float(spot_items[8]),
+                        )
 
                         # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other
                         # code will do that for us.
                         new_spots.append(spot)
             except (ConnectTimeout, ReadTimeout):
-                logging.warning(f"Timeout when accessing HEMA spots API.")
+                logger.warning("Timeout when accessing HEMA spots API.")
             except ConnectionError:
-                logging.warning("Connection error when accessing HEMA spots API.")
+                logger.warning("Connection error when accessing HEMA spots API.")
         return new_spots
 
     def can_submit_spot(self, sig):

providers/spot/http_spot_provider.py

@@ -1,15 +1,16 @@
 import logging
 from datetime import datetime
-from threading import Thread, Event
+from threading import Event, Thread
 
 import pytz
 import requests
-from requests import ReadTimeout
-from requests.exceptions import ConnectionError, ConnectTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
 from core.constants import HTTP_HEADERS
 from providers.spot.spot_provider import SpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class HTTPSpotProvider(SpotProvider):
     """Generic spot provider class for providers that request data via HTTP(S). Just for convenience to avoid code
@@ -26,7 +27,7 @@ class HTTPSpotProvider(SpotProvider):
     def start(self):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
-        logging.info("Set up query of " + self.name + " spot API every " + str(self._poll_interval) + " seconds.")
+        logger.info(f"Set up query of {self.name} spot API every {self._poll_interval!s} seconds.")
         self._thread = Thread(target=self._run, name=f"HTTPSpotProvider-{self.name}")
         self._thread.start()
 
@@ -50,7 +51,7 @@ class HTTPSpotProvider(SpotProvider):
     def _poll(self):
         try:
             # Request data from API
-            logging.debug("Polling " + self.name + " spot API...")
+            logger.debug(f"Polling {self.name} spot API...")
             http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
             # Check response code was good
             if http_response.ok:
@@ -62,18 +63,18 @@ class HTTPSpotProvider(SpotProvider):
 
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
-                logging.debug("Received data from " + self.name + " spot API.")
+                logger.debug(f"Received data from {self.name} spot API.")
             else:
                 self.status = "Error"
-                logging.warning(f"HTTP {http_response.status_code} when calling {self.name} spot API.")
+                logger.warning(f"HTTP {http_response.status_code} when calling {self.name} spot API.")
 
         except ConnectionError:
-            logging.warning(f"Connection error when accessing {self.name} spots API.")
+            logger.warning(f"Connection error when accessing {self.name} spots API.")
         except (ConnectTimeout, ReadTimeout):
-            logging.warning(f"Timeout when accessing {self.name} spots API.")
+            logger.warning(f"Timeout when accessing {self.name} spots API.")
         except Exception:
             self.status = "Error"
-            logging.exception("Exception in HTTP Spot Provider (" + self.name + ")")
+            logger.exception(f"Exception in HTTP Spot Provider ({self.name})")
             self._stop_event.wait(timeout=1)
 
     def _http_response_to_spots(self, http_response):

providers/spot/llota.py

@@ -25,16 +25,24 @@ class LLOTA(HTTPSpotProvider):
                 comment = str(source_spot["history"][-1]["comment"])
                 spotter = str(source_spot["history"][-1]["spotter_callsign"])
             # Convert to our spot format
-            spot = Spot(source=self.name,
-                        source_id=source_spot["id"],
-                        dx_call=source_spot["callsign"].upper(),
-                        de_call=spotter.upper() if spotter else None,
-                        freq=float(source_spot["frequency"]) * 1000000,
-                        mode=source_spot["mode"].upper(),
-                        comment=comment,
+            spot = Spot(
+                source=self.name,
+                source_id=source_spot["id"],
+                dx_call=source_spot["callsign"].upper(),
+                de_call=spotter.upper() if spotter else None,
+                freq=float(source_spot["frequency"]) * 1000000,
+                mode=source_spot["mode"].upper(),
+                comment=comment,
+                sig="LLOTA",
+                sig_refs=[
+                    SIGRef(
+                        id=source_spot["reference"],
                         sig="LLOTA",
-                        sig_refs=[SIGRef(id=source_spot["reference"], sig="LLOTA", name=source_spot["reference_name"])],
-                        time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp())
+                        name=source_spot["reference_name"],
+                    )
+                ],
+                time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp(),
+            )
 
             # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
             # that for us.

providers/spot/parksnpeaks.py

@@ -10,6 +10,8 @@ from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class ParksNPeaks(HTTPSpotProvider):
     """Spot provider for Parks n Peaks"""
@@ -18,7 +20,17 @@ class ParksNPeaks(HTTPSpotProvider):
     SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
     SUBMIT_URL = "https://www.parksnpeaks.org/api/SPOT/"
     SIOTA_LIST_URL = "https://www.silosontheair.com/data/silos.csv"
-    SUBMITTABLE_SIGS = ["POTA", "SOTA", "WWFF", "HEMA", "WOTA", "ZLOTA", "SIOTA", "KRMNPA", "SANPCPA"]
+    SUBMITTABLE_SIGS = [
+        "POTA",
+        "SOTA",
+        "WWFF",
+        "HEMA",
+        "WOTA",
+        "ZLOTA",
+        "SIOTA",
+        "KRMNPA",
+        "SANPCPA",
+    ]
 
     def __init__(self, provider_config):
         super().__init__("ParksNPeaks", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -29,18 +41,23 @@ class ParksNPeaks(HTTPSpotProvider):
         if http_response and http_response != "":
             for source_spot in http_response.json():
                 # Convert to our spot format
-                spot = Spot(source=self.name,
-                            source_id=source_spot["actID"],
-                            dx_call=source_spot["actCallsign"].upper(),
-                            de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,
-                            # typo exists in API
-                            freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "")
-                                       .replace("+/-", "").strip()) * 1000000 if (source_spot["actFreq"] != "") else None,
-                            # Seen PNP spots with empty frequency, and with comma-separated thousands digits
-                            mode=source_spot["actMode"].upper(),
-                            comment=source_spot["actComments"],
-                            time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S").replace(
-                                tzinfo=pytz.UTC).timestamp())
+                spot = Spot(
+                    source=self.name,
+                    source_id=source_spot["actID"],
+                    dx_call=source_spot["actCallsign"].upper(),
+                    de_call=source_spot["actSpoter"].upper() if source_spot["actSpoter"] != "" else None,
+                    # typo exists in API
+                    freq=float(source_spot["actFreq"].replace(",", "").replace("+-", "").replace("+/-", "").strip())
+                    * 1000000
+                    if (source_spot["actFreq"] != "")
+                    else None,
+                    # Seen PNP spots with empty frequency, and with comma-separated thousands digits
+                    mode=source_spot["actMode"].upper(),
+                    comment=source_spot["actComments"],
+                    time=datetime.strptime(source_spot["actTime"], "%Y-%m-%d %H:%M:%S")
+                    .replace(tzinfo=pytz.UTC)
+                    .timestamp(),
+                )
 
                 # Extract a de_call if it's in the comment but not in the "actSpoter" field
                 m = re.search(r"\(de ([A-Za-z0-9]*)\)", spot.comment or "")
@@ -53,7 +70,12 @@ class ParksNPeaks(HTTPSpotProvider):
                 sig_ref = source_spot["actSiteID"]
                 if sig and sig != "" and sig != "QRP" and sig_ref and sig_ref != "":
                     spot.sig = sig
-                    sig_refs = [SIGRef(id=source_spot["actSiteID"], sig=source_spot["actClass"].upper())]
+                    sig_refs = [
+                        SIGRef(
+                            id=source_spot["actSiteID"],
+                            sig=source_spot["actClass"].upper(),
+                        )
+                    ]
                     spot.sig_refs = sig_refs
 
                     # Free text location is not present in all spots, so only add it if it's set
@@ -61,8 +83,17 @@ class ParksNPeaks(HTTPSpotProvider):
                         sig_refs[0].name = source_spot["actLocation"]
 
                     # Log a warning for the developer if PnP gives us an unknown programme we've never seen before
-                    if sig not in ["POTA", "SOTA", "WWFF", "SIOTA", "ZLOTA", "KRMNPA", "SANPCPA", "LLOTA"]:
-                        logging.warning("PNP spot found with sig " + sig + ", developer needs to add support for this!")
+                    if sig not in [
+                        "POTA",
+                        "SOTA",
+                        "WWFF",
+                        "SIOTA",
+                        "ZLOTA",
+                        "KRMNPA",
+                        "SANPCPA",
+                        "LLOTA",
+                    ]:
+                        logger.warning(f"PNP spot found with sig {sig}, developer needs to add support for this!")
 
                 # Add new spot to the list
                 new_spots.append(spot)
@@ -77,7 +108,8 @@ class ParksNPeaks(HTTPSpotProvider):
         api_key = credentials.get("api_key", "")
         if not user_id or not api_key:
             raise ValueError(
-                "Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account.")
+                "Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account."
+            )
         sig_ref = spot.sig_refs[0].id if spot.sig_refs else ""
         body = {
             "actClass": spot.sig or "",
@@ -91,4 +123,4 @@ class ParksNPeaks(HTTPSpotProvider):
         }
         response = requests.post(self.SUBMIT_URL, json=body, headers=HTTP_HEADERS, timeout=(5, 30))
         if not response.ok:
-            raise RuntimeError("Parks N Peaks API returned " + str(response.status_code) + ": " + response.text)
+            raise RuntimeError(f"Parks N Peaks API returned {response.status_code!s}: {response.text}")

providers/spot/pota.py

@@ -24,21 +24,31 @@ class POTA(HTTPSpotProvider):
         # Iterate through source data
         for source_spot in http_response.json():
             # Convert to our spot format
-            spot = Spot(source=self.name,
-                        source_id=source_spot["spotId"],
-                        dx_call=source_spot["activator"].upper(),
-                        de_call=source_spot["spotter"].upper(),
-                        freq=float(source_spot["frequency"]) * 1000,
-                        mode=source_spot["mode"].upper(),
-                        comment=source_spot["comments"],
+            spot = Spot(
+                source=self.name,
+                source_id=source_spot["spotId"],
+                dx_call=source_spot["activator"].upper(),
+                de_call=source_spot["spotter"].upper(),
+                freq=float(source_spot["frequency"]) * 1000,
+                mode=source_spot["mode"].upper(),
+                comment=source_spot["comments"],
+                sig="POTA",
+                sig_refs=[
+                    SIGRef(
+                        id=source_spot["reference"],
                         sig="POTA",
-                        sig_refs=[SIGRef(id=source_spot["reference"], sig="POTA", name=source_spot["name"],
-                                         latitude=source_spot["latitude"], longitude=source_spot["longitude"])],
-                        time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S").replace(
-                            tzinfo=pytz.UTC).timestamp(),
-                        dx_grid=source_spot["grid6"],
-                        dx_latitude=source_spot["latitude"],
-                        dx_longitude=source_spot["longitude"])
+                        name=source_spot["name"],
+                        latitude=source_spot["latitude"],
+                        longitude=source_spot["longitude"],
+                    )
+                ],
+                time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S")
+                .replace(tzinfo=pytz.UTC)
+                .timestamp(),
+                dx_grid=source_spot["grid6"],
+                dx_latitude=source_spot["latitude"],
+                dx_longitude=source_spot["longitude"],
+            )
 
             # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
             # that for us.
@@ -63,6 +73,6 @@ class POTA(HTTPSpotProvider):
             headers = {**HTTP_HEADERS, "Content-Type": "application/json"}
             response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
             if not response.ok:
-                raise RuntimeError("POTA API returned " + str(response.status_code) + ": " + response.text)
+                raise RuntimeError(f"POTA API returned {response.status_code!s}: {response.text}")
         else:
             raise RuntimeError("Park reference is required for submitting POTA spots.")

providers/spot/rbn.py

@@ -11,6 +11,8 @@ from core.config import SERVER_OWNER_CALLSIGN
 from data.spot import Spot
 from providers.spot.spot_provider import SpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class RBN(SpotProvider):
     """Spot provider for the Reverse Beacon Network. Connects to a single port, if you want both CW/RTTY (port 7000) and FT8
@@ -18,12 +20,13 @@ class RBN(SpotProvider):
 
     _LINE_PATTERN = re.compile(
         r"^DX de ([a-z0-9/]+)-.*:\s+([0-9.]+)\s+([a-z0-9/]+)\s+(.*)\s+(\d{4}Z)",
-        re.IGNORECASE)
+        re.IGNORECASE,
+    )
 
     def __init__(self, provider_config):
         """Constructor requires port number."""
 
-        name = provider_config["name"] if "name" in provider_config else "RBN"
+        name = provider_config.get("name", "RBN")
         super().__init__(name, provider_config)
         self._port = provider_config["port"]
         self._telnet = None
@@ -36,7 +39,8 @@ class RBN(SpotProvider):
 
     def stop(self):
         self._running = False
-        self._telnet.close()
+        if self._telnet:
+            self._telnet.close()
         self._thread.join()
 
     def _handle(self):
@@ -45,15 +49,15 @@ class RBN(SpotProvider):
             while not connected and self._running:
                 try:
                     self.status = "Connecting"
-                    logging.info("RBN port " + str(self._port) + " connecting...")
+                    logger.info(f"RBN port {self._port!s} connecting...")
                     self._telnet = telnetlib3.Telnet("telnet.reversebeacon.net", self._port)
                     self._telnet.read_until("Please enter your call: ".encode("latin-1"))
-                    self._telnet.write((SERVER_OWNER_CALLSIGN + "\n").encode("latin-1"))
+                    self._telnet.write(f"{SERVER_OWNER_CALLSIGN}\n".encode("latin-1"))
                     connected = True
-                    logging.info("RBN port " + str(self._port) + " connected.")
+                    logger.info(f"RBN port {self._port!s} connected.")
                 except Exception:
                     self.status = "Error"
-                    logging.exception("Exception while connecting to RBN (port " + str(self._port) + ").")
+                    logger.exception(f"Exception while connecting to RBN (port {self._port!s}).")
                     sleep(5)
 
             self.status = "Waiting for Data"
@@ -64,30 +68,44 @@ class RBN(SpotProvider):
                     match = self._LINE_PATTERN.match(telnet_output.decode("latin-1"))
                     if match:
                         spot_time = datetime.strptime(match.group(5), "%H%MZ")
-                        spot_datetime = datetime.combine(datetime.now(pytz.UTC).date(), spot_time.time(),
-                                                         tzinfo=pytz.UTC)
-                        spot = Spot(source=self.name,
-                                    dx_call=match.group(3),
-                                    de_call=match.group(1),
-                                    freq=float(match.group(2)) * 1000,
-                                    comment=match.group(4).strip(),
-                                    time=spot_datetime.timestamp())
+                        spot_datetime = datetime.combine(
+                            datetime.now(pytz.UTC).date(),
+                            spot_time.time(),
+                            tzinfo=pytz.UTC,
+                        )
+                        spot = Spot(
+                            source=self.name,
+                            dx_call=match.group(3),
+                            de_call=match.group(1),
+                            freq=float(match.group(2)) * 1000,
+                            comment=match.group(4).strip(),
+                            time=spot_datetime.timestamp(),
+                        )
 
                         # Add to our list
                         self._submit(spot)
 
                         self.status = "OK"
                         self.last_update_time = datetime.now(pytz.UTC)
-                        logging.debug("Data received from RBN on port " + str(self._port) + ".")
+                        logger.debug(f"Data received from RBN on port {self._port!s}.")
 
+                except EOFError:
+                    connected = False
+                    if self._running:
+                        self.status = "Restarting"
+                        logger.warning(f"Disconnected from RBN provider (port {self._port!s}). Reconnecting...")
+                        sleep(5)
+                    else:
+                        logger.info(f"RBN provider (port {self._port!s}) shutting down...")
+                        self.status = "Shutting down"
                 except Exception:
                     connected = False
                     if self._running:
                         self.status = "Error"
-                        logging.exception("Exception in RBN provider (port " + str(self._port) + ")")
+                        logger.exception(f"Exception in RBN provider (port {self._port!s})")
                         sleep(5)
                     else:
-                        logging.info("RBN provider (port " + str(self._port) + ") shutting down...")
+                        logger.info(f"RBN provider (port {self._port!s}) shutting down...")
                         self.status = "Shutting down"
 
             self.status = "Disconnected"

providers/spot/sota.py

@@ -2,13 +2,15 @@ import logging
 from datetime import datetime
 
 import requests
-from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
-from core.constants import HTTP_HEADERS, SSB_SUB_MODES, DV_SUB_MODES
+from core.constants import DV_SUB_MODES, HTTP_HEADERS, SSB_SUB_MODES
 from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class SOTA(HTTPSpotProvider):
     """Spot provider for Summits on the Air"""
@@ -41,32 +43,41 @@ class SOTA(HTTPSpotProvider):
                 # Iterate through source data
                 for source_spot in source_data:
                     # Convert to our spot format
-                    spot = Spot(source=self.name,
-                                source_id=source_spot["id"],
-                                dx_call=source_spot["activatorCallsign"].upper(),
-                                dx_name=source_spot["activatorName"],
-                                de_call=source_spot["callsign"].upper(),
-                                freq=(float(source_spot["frequency"]) * 1000000) if (
-                                        source_spot["frequency"] is not None) else None,
-                                # Seen SOTA spots with no frequency!
-                                mode=source_spot["mode"].upper(),
-                                comment=source_spot["comments"],
+                    spot = Spot(
+                        source=self.name,
+                        source_id=source_spot["id"],
+                        dx_call=source_spot["activatorCallsign"].upper(),
+                        dx_name=source_spot["activatorName"],
+                        de_call=source_spot["callsign"].upper(),
+                        freq=(float(source_spot["frequency"]) * 1000000)
+                        if (source_spot["frequency"] is not None)
+                        else None,
+                        # Seen SOTA spots with no frequency!
+                        mode=source_spot["mode"].upper(),
+                        comment=source_spot["comments"],
+                        sig="SOTA",
+                        sig_refs=[
+                            SIGRef(
+                                id=source_spot["summitCode"],
                                 sig="SOTA",
-                                sig_refs=[SIGRef(id=source_spot["summitCode"], sig="SOTA",
-                                                 name=source_spot["summitName"], latitude=source_spot["latitude"],
-                                                 longitude=source_spot["longitude"],
-                                                 activation_score=source_spot["points"])],
-                                dx_latitude=source_spot["latitude"],
-                                dx_longitude=source_spot["longitude"],
-                                time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp())
+                                name=source_spot["summitName"],
+                                latitude=source_spot["latitude"],
+                                longitude=source_spot["longitude"],
+                                activation_score=source_spot["points"],
+                            )
+                        ],
+                        dx_latitude=source_spot["latitude"],
+                        dx_longitude=source_spot["longitude"],
+                        time=datetime.fromisoformat(source_spot["timeStamp"].replace("Z", "+00:00")).timestamp(),
+                    )
 
                     # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
                     # that for us.
                     new_spots.append(spot)
             except ConnectionError:
-                logging.warning("Connection error when accessing SOTA spots API")
+                logger.warning("Connection error when accessing SOTA spots API")
             except (ConnectTimeout, ReadTimeout):
-                logging.warning(f"Timeout when accessing SOTA spots API.")
+                logger.warning("Timeout when accessing SOTA spots API.")
         return new_spots
 
     def can_submit_spot(self, sig):
@@ -102,12 +113,16 @@ class SOTA(HTTPSpotProvider):
                 "mode": mode or "",
                 "callsign": spot.de_call,
                 "comments": spot.comment or "",
-                "type": "TEST"  # todo replatce with NORMAL/QRT once testing complete
+                "type": "TEST",  # todo replatce with NORMAL/QRT once testing complete
+            }
+            headers = {
+                **HTTP_HEADERS,
+                "Authorization": f"bearer {access_token}",
+                "id_token": id_token,
+                "Content-Type": "application/json",
             }
-            headers = {**HTTP_HEADERS, "Authorization": "bearer " + access_token, "id_token": id_token,
-                       "Content-Type": "application/json"}
             response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
             if not response.ok:
-                raise RuntimeError("SOTA API returned " + str(response.status_code) + ": " + response.text)
+                raise RuntimeError(f"SOTA API returned {response.status_code!s}: {response.text}")
         else:
             raise RuntimeError("Summit reference is required for submitting SOTA spots.")

providers/spot/spot_provider.py

@@ -32,14 +32,14 @@ class SpotProvider:
 
         # Sort the batch so that earliest ones go in first. This helps keep the ordering correct when spots are fired
         # off to SSE listeners.
-        spots = sorted(spots, key=lambda s: (s.time if s and s.time else 0))
+        spots = sorted(spots, key=lambda s: s.time if s and s.time else 0)
         for spot in spots:
             if datetime.fromtimestamp(spot.time, pytz.UTC) > self.last_spot_time:
                 # Fill in any blanks and add to the list
                 spot.infer_missing()
                 self._add_spot(spot)
         if spots:
-            self.last_spot_time = datetime.fromtimestamp(max(map(lambda s: s.time, spots)), pytz.UTC)
+            self.last_spot_time = datetime.fromtimestamp(max(s.time for s in spots), pytz.UTC)
 
     def _submit(self, spot):
         """Submit a single spot retrieved from the provider. This will be added to the list regardless of its age. Spots

providers/spot/sse_spot_provider.py

@@ -3,11 +3,13 @@ from datetime import datetime
 from threading import Event, Lock, Thread
 
 import pytz
-from requests_sse import EventSource
+from requests_sse import EventSource, InvalidStatusCodeError
 
 from core.constants import HTTP_HEADERS
 from providers.spot.spot_provider import SpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class SSESpotProvider(SpotProvider):
     """Spot provider using Server-Sent Events."""
@@ -22,7 +24,7 @@ class SSESpotProvider(SpotProvider):
         self._event_source = None
 
     def start(self):
-        logging.info("Set up SSE connection to " + self.name + " spot API.")
+        logger.info(f"Set up SSE connection to {self.name} spot API.")
         self._stop_event.clear()
         self._thread = Thread(target=self._run, name=f"SSESpotProvider-{self.name}")
         self._thread.daemon = True
@@ -37,13 +39,12 @@ class SSESpotProvider(SpotProvider):
             try:
                 event_source.close()
             except Exception:
-                logging.exception(
-                    "Exception closing SSE connection for " + self.name + " during stop()")
+                logger.exception(f"Exception closing SSE connection for {self.name} during stop()")
 
         if self._thread:
             self._thread.join(timeout=15)
             if self._thread.is_alive():
-                logging.warning(self.name + " SSE worker thread did not exit on time and will be killed.")
+                logger.warning(f"{self.name} SSE worker thread did not exit on time and will be killed.")
 
     def _on_open(self):
         self.status = "Waiting for Data"
@@ -58,16 +59,22 @@ class SSESpotProvider(SpotProvider):
     def _run(self):
         while not self._stop_event.is_set():
             try:
-                logging.debug("Connecting to " + self.name + " spot API...")
+                logger.debug(f"Connecting to {self.name} spot API...")
                 self.status = "Connecting"
-                with EventSource(self._url, headers=HTTP_HEADERS, latest_event_id=self._last_event_id, timeout=10,
-                                 on_open=self._on_open, on_error=self._on_error) as event_source:
+                with EventSource(
+                    self._url,
+                    headers=HTTP_HEADERS,
+                    latest_event_id=self._last_event_id,
+                    timeout=10,
+                    on_open=self._on_open,
+                    on_error=self._on_error,
+                ) as event_source:
                     self._set_event_source(event_source)
                     try:
                         for event in event_source:
                             if self._stop_event.is_set():
                                 break
-                            if event.type == 'message':
+                            if event.type == "message":
                                 try:
                                     self._last_event_id = event.last_event_id
                                     new_spot = self._sse_message_to_spot(event.data)
@@ -76,20 +83,24 @@ class SSESpotProvider(SpotProvider):
 
                                     self.status = "OK"
                                     self.last_update_time = datetime.now(pytz.UTC)
-                                    logging.debug("Received data from " + self.name + " spot API.")
+                                    logger.debug(f"Received data from {self.name} spot API.")
 
                                 except Exception:
-                                    logging.exception(
-                                        "Exception processing message from SSE Spot Provider (" + self.name + ")")
+                                    logger.exception(
+                                        f"Exception processing message from SSE Spot Provider ({self.name})"
+                                    )
+                    except InvalidStatusCodeError as e:
+                        self.status = "Error"
+                        logger.warning(f"HTTP {e.status_code} returned from SSE Spot Provider ({self.name}).")
                     finally:
                         self._set_event_source(None)
 
             except Exception:
                 self.status = "Error"
-                logging.exception("Exception in SSE Spot Provider (" + self.name + ")")
+                logger.exception(f"Exception in SSE Spot Provider ({self.name})")
             else:
                 self.status = "Disconnected"
-            self._stop_event.wait(timeout=5) # Wait before trying to reconnect
+            self._stop_event.wait(timeout=5)  # Wait before trying to reconnect
 
     def _sse_message_to_spot(self, message_data):
         """Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass

providers/spot/tiles.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 
 import requests
@@ -7,6 +8,8 @@ from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class Tiles(HTTPSpotProvider):
     """Spot provider for Tiles on the Air"""
@@ -14,8 +17,22 @@ class Tiles(HTTPSpotProvider):
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/spots?active_hours=24"
     SUBMIT_URL = "https://icneuzxitdqtofutxbla.supabase.co/functions/v1/self-spot"
-    VALID_MODES = ["SSB", "CW", "FT8", "FT4", "FM", "DMR", "D-STAR", "M17", "AX.25", "JS8Call", "PSK31", "Olivia",
-                   "VarAC", "Other"]
+    VALID_MODES = [
+        "SSB",
+        "CW",
+        "FT8",
+        "FT4",
+        "FM",
+        "DMR",
+        "D-STAR",
+        "M17",
+        "AX.25",
+        "JS8Call",
+        "PSK31",
+        "Olivia",
+        "VarAC",
+        "Other",
+    ]
 
     def __init__(self, provider_config):
         super().__init__("Tiles", provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC)
@@ -25,25 +42,39 @@ class Tiles(HTTPSpotProvider):
         # Iterate through source data
         for source_spot in http_response.json()["spots"]:
             # Convert to our spot format
-            spot = Spot(source=self.name,
-                        source_id=source_spot["id"],
-                        dx_call=source_spot["call_sign"].upper(),
-                        # No separate spotter callsign, assume all spots are self-spots
-                        de_call=source_spot["call_sign"].upper(),
-                        freq=float(strip_extra_decimal_points(source_spot["frequency"])) * 1000000,
-                        mode=source_spot["mode"].upper(),
-                        comment=source_spot["notes"],
+            freq = None
+            try:
+                freq = float(strip_extra_decimal_points(source_spot["frequency"])) * 1000000
+            except ValueError:
+                logger.debug(f"Malformed frequency received from Tiles API: {source_spot['frequency']}")
+
+            spot = Spot(
+                source=self.name,
+                source_id=source_spot["id"],
+                dx_call=source_spot["call_sign"].upper(),
+                # No separate spotter callsign, assume all spots are self-spots
+                de_call=source_spot["call_sign"].upper(),
+                freq=freq,
+                mode=source_spot["mode"].upper(),
+                comment=source_spot["notes"],
+                sig="Tiles",
+                # Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway.
+                # Just take the grid reference itself as the single Tiles SIG reference.
+                sig_refs=[
+                    SIGRef(
+                        id=source_spot["maidenhead_grid"],
                         sig="Tiles",
-                        # Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway.
-                        # Just take the grid reference itself as the single Tiles SIG reference.
-                        sig_refs=[SIGRef(id=source_spot["maidenhead_grid"], sig="Tiles",
-                                         name=source_spot["maidenhead_grid"], latitude=source_spot["latitude"],
-                                         longitude=source_spot["longitude"])],
-                        time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
-                        dx_grid=source_spot["maidenhead_grid"],
-                        dx_latitude=source_spot["latitude"],
-                        dx_longitude=source_spot["longitude"],
-                        dx_location_source="GRID")
+                        name=source_spot["maidenhead_grid"],
+                        latitude=source_spot["latitude"],
+                        longitude=source_spot["longitude"],
+                    )
+                ],
+                time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
+                dx_grid=source_spot["maidenhead_grid"],
+                dx_latitude=source_spot["latitude"],
+                dx_longitude=source_spot["longitude"],
+                dx_location_source="GRID",
+            )
 
             # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
             # that for us.
@@ -56,7 +87,6 @@ class Tiles(HTTPSpotProvider):
     def submit_spot(self, spot, credentials):
         # Tiles on the air currently only supports *self* spots
         if spot.dx_call == spot.de_call:
-
             # Figure out a valid mode. Borrowed this from PoLo :)
             # https://github.com/ham2k/app-polo/blob/main/src/extensions/activities/sota/SOTAPostSelfSpot.js
             if spot.mode:
@@ -80,24 +110,24 @@ class Tiles(HTTPSpotProvider):
                     "lat": spot.dx_latitude or None,
                     "lon": spot.dx_longitude or None,
                     "qrt": spot.qrt or False,
-                    "pin": credentials.get("offline_spot_gateway_pin", "")
+                    "pin": credentials.get("offline_spot_gateway_pin", ""),
                 }
                 headers = {**HTTP_HEADERS, "Content-Type": "application/json"}
                 response = requests.post(self.SUBMIT_URL, json=body, headers=headers, timeout=(5, 30))
                 if not response.ok:
-                    raise RuntimeError(
-                        "Tiles on the Air API returned " + str(response.status_code) + ": " + response.text)
+                    raise RuntimeError(f"Tiles on the Air API returned {response.status_code!s}: {response.text}")
             else:
                 raise RuntimeError("The Tiles on the Air API requires a mode to be set.")
         else:
             raise RuntimeError(
-                "The Tiles on the Air API only supports self-spots, the DX call and spotter call must match.")
+                "The Tiles on the Air API only supports self-spots, the DX call and spotter call must match."
+            )
 
 
 # Utility function to keep the first decimal point in a given string but remove any others. Used to parse Tiles'
 # strange frequency format where we can sometimes have e.g. "14.123.5".
 def strip_extra_decimal_points(s):
-    parts = s.split('.', 1)
+    parts = s.split(".", 1)
     if len(parts) == 1:
         return s
-    return parts[0] + '.' + parts[1].replace('.', '')
+    return f"{parts[0]}.{parts[1].replace('.', '')}"

providers/spot/towers.py

@@ -26,14 +26,17 @@ class Towers(HTTPSpotProvider):
             likely_freq = float(source_spot["freq"]) * 1000
             if likely_freq < 1000000:
                 likely_freq = likely_freq * 1000
-            spot = Spot(source=self.name,
-                        dx_call=source_spot["call"].upper(),
-                        freq=likely_freq,
-                        comment=source_spot["comment"],
-                        sig="Towers",
-                        sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
-                        time=datetime.strptime(response_json["updated"][:10] + source_spot["time"],
-                                               "%Y-%m-%d%H:%M").timestamp())
+            spot = Spot(
+                source=self.name,
+                dx_call=source_spot["call"].upper(),
+                freq=likely_freq,
+                comment=source_spot["comment"],
+                sig="Towers",
+                sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers")],
+                time=datetime.strptime(
+                    response_json["updated"][:10] + source_spot["time"], "%Y-%m-%d%H:%M"
+                ).timestamp(),
+            )
 
             # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
             # that for us.

providers/spot/ukpacketnet.py

@@ -20,7 +20,7 @@ class UKPacketNet(HTTPSpotProvider):
         new_spots = []
         # Iterate through source data
         nodes = http_response.json()["nodes"]
-        for callsign, node in nodes.items():
+        for node in nodes.values():
             # The node corresponse to the spotter here. It has an "mheard" section which indicates which nodes it has
             # recently heard, which will be our "DX". But "mheard" stations are not necessarily over RF, they could be
             # via the internet, so we also need to look up the "port" on which the station was heard, and check that it
@@ -34,46 +34,58 @@ class UKPacketNet(HTTPSpotProvider):
                                 # This is another packet station heard over RF, so we are good to create a Spot object.
 
                                 # First build a "full" comment combining some of the extra info
-                                comment = listed_port["comment"] if "comment" in listed_port else ""
-                                comment = (comment + " " + listed_port["mode"]) if "mode" in listed_port else comment
-                                comment = (comment + " " + listed_port[
-                                    "modulation"]) if "modulation" in listed_port else comment
-                                comment = (comment + " " + str(
-                                    listed_port["baud"]) + " baud") if "baud" in listed_port and listed_port[
-                                    "baud"] > 0 else comment
+                                comment = listed_port.get("comment", "")
+                                comment = f"{comment} {listed_port['mode']}" if "mode" in listed_port else comment
+                                comment = (
+                                    f"{comment} {listed_port['modulation']}" if "modulation" in listed_port else comment
+                                )
+                                comment = (
+                                    f"{comment} {listed_port['baud']!s} baud"
+                                    if "baud" in listed_port and listed_port["baud"] > 0
+                                    else comment
+                                )
 
                                 # Get frequency from the comment if it's not set properly in the data structure. This is
                                 # very hacky but a lot of node comments contain their frequency as the first or second
                                 # word of their comment, but not in the proper data structure field.
-                                freq = listed_port["freq"] if "freq" in listed_port and listed_port[
-                                    "freq"] > 0 else None
+                                freq = (
+                                    listed_port["freq"] if "freq" in listed_port and listed_port["freq"] > 0 else None
+                                )
                                 if not freq and comment:
                                     possible_freq = comment.split(" ")[0].upper().replace("MHZ", "")
-                                    if re.match(r"^[0-9.]+$",
-                                                possible_freq) and possible_freq != "1200" and possible_freq != "9600":
+                                    if (
+                                        re.match(r"^[0-9.]+$", possible_freq)
+                                        and possible_freq != "1200"
+                                        and possible_freq != "9600"
+                                    ):
                                         freq = float(possible_freq) * 1000000
                                     if not freq and len(comment.split(" ")) > 1:
                                         possible_freq = comment.split(" ")[1].upper().replace("MHZ", "")
-                                        if re.match(r"^[0-9.]+$",
-                                                    possible_freq) and possible_freq != "1200" and possible_freq != "9600":
+                                        if (
+                                            re.match(r"^[0-9.]+$", possible_freq)
+                                            and possible_freq != "1200"
+                                            and possible_freq != "9600"
+                                        ):
                                             freq = float(possible_freq) * 1000000
                                     # Check for a found frequency likely having been in kHz, sorry to all GHz packet folks
                                     if freq and freq > 1000000000:
                                         freq = freq / 1000
 
                                 # Now build the spot object
-                                spot = Spot(source=self.name,
-                                            dx_call=heard["callsign"].upper(),
-                                            de_call=node["callsign"].upper(),
-                                            freq=freq,
-                                            mode="PKT",
-                                            comment=comment,
-                                            time=datetime.strptime(heard["lastHeard"], "%Y-%m-%d %H:%M:%S").replace(
-                                                tzinfo=pytz.UTC).timestamp(),
-                                            de_grid=node["location"]["locator"] if "locator" in node[
-                                                "location"] else None,
-                                            de_latitude=node["location"]["coords"]["lat"],
-                                            de_longitude=node["location"]["coords"]["lon"])
+                                spot = Spot(
+                                    source=self.name,
+                                    dx_call=heard["callsign"].upper(),
+                                    de_call=node["callsign"].upper(),
+                                    freq=freq,
+                                    mode="PKT",
+                                    comment=comment,
+                                    time=datetime.strptime(heard["lastHeard"], "%Y-%m-%d %H:%M:%S")
+                                    .replace(tzinfo=pytz.UTC)
+                                    .timestamp(),
+                                    de_grid=node["location"].get("locator", None),
+                                    de_latitude=node["location"]["coords"]["lat"],
+                                    de_longitude=node["location"]["coords"]["lon"],
+                                )
 
                                 # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
                                 # that for us.
@@ -86,8 +98,7 @@ class UKPacketNet(HTTPSpotProvider):
         # data, and we can use that to look these up.
         for spot in new_spots:
             if spot.dx_call in nodes:
-                spot.dx_grid = nodes[spot.dx_call]["location"]["locator"] if "locator" in nodes[spot.dx_call][
-                    "location"] else None
+                spot.dx_grid = nodes[spot.dx_call]["location"].get("locator", None)
                 spot.dx_latitude = nodes[spot.dx_call]["location"]["coords"]["lat"]
                 spot.dx_longitude = nodes[spot.dx_call]["location"]["coords"]["lon"]
 

providers/spot/websocket_spot_provider.py

@@ -9,6 +9,8 @@ from websocket import create_connection
 from core.constants import HTTP_HEADERS
 from providers.spot.spot_provider import SpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class WebsocketSpotProvider(SpotProvider):
     """Spot provider using websockets."""
@@ -22,7 +24,7 @@ class WebsocketSpotProvider(SpotProvider):
         self._last_event_id = None
 
     def start(self):
-        logging.info("Set up websocket connection to " + self.name + " spot API.")
+        logger.info(f"Set up websocket connection to {self.name} spot API.")
         self._stopped = False
         self._thread = Thread(target=self._run, name=f"WebsocketSpotProvider-{self.name}")
         self._thread.daemon = True
@@ -44,7 +46,7 @@ class WebsocketSpotProvider(SpotProvider):
     def _run(self):
         while not self._stopped:
             try:
-                logging.debug("Connecting to " + self.name + " spot API...")
+                logger.debug(f"Connecting to {self.name} spot API...")
                 self.status = "Connecting"
                 self._ws = create_connection(self._url, header=HTTP_HEADERS)
                 self.status = "Connected"
@@ -57,15 +59,14 @@ class WebsocketSpotProvider(SpotProvider):
 
                         self.status = "OK"
                         self.last_update_time = datetime.now(pytz.UTC)
-                        logging.debug("Received data from " + self.name + " spot API.")
+                        logger.debug(f"Received data from {self.name} spot API.")
 
                     except Exception:
-                        logging.exception(
-                            "Exception processing message from Websocket Spot Provider (" + self.name + ")")
+                        logger.exception(f"Exception processing message from Websocket Spot Provider ({self.name})")
 
-            except Exception as e:
+            except Exception:
                 self.status = "Error"
-                logging.exception("Exception in Websocket Spot Provider (" + self.name + ")", e)
+                logger.exception(f"Exception in Websocket Spot Provider ({self.name})")
             else:
                 self.status = "Disconnected"
             sleep(5)  # Wait before trying to reconnect

providers/spot/wota.py

@@ -2,6 +2,7 @@ import logging
 import re
 from datetime import datetime
 from typing import cast
+from xml.parsers.expat import ExpatError
 
 import pytz
 from rss_parser import Parser
@@ -11,13 +12,14 @@ from data.sig_ref import SIGRef
 from data.spot import Spot
 from providers.spot.http_spot_provider import HTTPSpotProvider
 
+logger = logging.getLogger(__name__)
+
 
 class WOTA(HTTPSpotProvider):
     """Spot provider for Wainwrights on the Air"""
 
     POLL_INTERVAL_SEC = 120
     SPOTS_URL = "https://www.wota.org.uk/spots_rss.php"
-    LIST_URL = "https://www.wota.org.uk/mapping/data/summits.json"
     RSS_DATE_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
 
     def __init__(self, provider_config):
@@ -25,59 +27,78 @@ class WOTA(HTTPSpotProvider):
 
     def _http_response_to_spots(self, http_response):
         new_spots = []
-        rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
-        # Iterate through source data
-        for source_spot in rss.channel.items:
+        try:
+            rss = cast(RSS, Parser.parse(http_response.content.decode("utf-8-sig")))
+            # Iterate through source data
+            for source_spot in rss.channel.items:
+                try:
+                    # Reject GUID missing or zero
+                    if (
+                        not source_spot.guid
+                        or not source_spot.guid.content
+                        or source_spot.guid.content == "http://www.wota.org.uk/spots/0"
+                    ):
+                        continue
 
-            try:
-                # Reject GUID missing or zero
-                if not source_spot.guid or not source_spot.guid.content or source_spot.guid.content == "http://www.wota.org.uk/spots/0":
-                    continue
+                    # Pick apart the title
+                    dx_call = None
+                    ref = None
+                    ref_name = None
+                    try:
+                        title_split = source_spot.title.split(" on ")
+                        dx_call = title_split[0]
+                        if len(title_split) > 1:
+                            ref_split = title_split[1].split(" - ")
+                            ref = str(ref_split[0])
+                            if len(ref_split) > 1:
+                                ref_name = str(ref_split[1])
+                    except Exception:
+                        logger.warning(f"Could not parse WOTA spot title: {source_spot.title}")
 
-                # Pick apart the title
-                title_split = source_spot.title.split(" on ")
-                dx_call = title_split[0]
-                ref = None
-                ref_name = None
-                if len(title_split) > 1:
-                    ref_split = title_split[1].split(" - ")
-                    ref = str(ref_split[0])
-                    if len(ref_split) > 1:
-                        ref_name = str(ref_split[1])
+                    # Pick apart the description
+                    freq_hz = None
+                    mode = None
+                    comment = None
+                    spotter = None
+                    try:
+                        desc_split = source_spot.description.split(". ")
+                        freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
+                        if freq_mode and freq_mode != "-":
+                            freq_mode_split = re.split(r"[\-\s]+", freq_mode)
+                            freq_hz = float(freq_mode_split[0].replace("'", ".")) * 1000000
+                            if len(freq_mode_split) > 1:
+                                mode = freq_mode_split[1].upper()
 
-                # Pick apart the description
-                desc_split = source_spot.description.split(". ")
-                freq_mode = desc_split[0].replace("Frequencies/modes:", "").strip()
-                freq_mode_split = re.split(r'[\-\s]+', freq_mode)
-                freq_hz = float(freq_mode_split[0].replace("'", ".")) * 1000000
-                mode = None
-                if len(freq_mode_split) > 1:
-                    mode = freq_mode_split[1].upper()
+                        if len(desc_split) > 1:
+                            comment = desc_split[1].strip()
+                        if len(desc_split) > 2:
+                            spotter = desc_split[2].replace("Spotted by ", "").replace(".", "").upper().strip()
+                    except Exception:
+                        logger.warning(f"Could not parse WOTA spot description: {source_spot.description}")
 
-                comment = None
-                if len(desc_split) > 1:
-                    comment = desc_split[1].strip()
-                spotter = None
-                if len(desc_split) > 2:
-                    spotter = desc_split[2].replace("Spotted by ", "").replace(".", "").upper().strip()
+                    time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
 
-                time = datetime.strptime(source_spot.pub_date.content, self.RSS_DATE_TIME_FORMAT).astimezone(pytz.UTC)
+                    # Convert to our spot format
+                    spot = Spot(
+                        source=self.name,
+                        source_id=source_spot.guid.content,
+                        dx_call=dx_call,
+                        de_call=spotter,
+                        freq=freq_hz,
+                        mode=mode,
+                        comment=comment,
+                        sig="WOTA",
+                        sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
+                        time=time.timestamp(),
+                    )
 
-                # Convert to our spot format
-                spot = Spot(source=self.name,
-                            source_id=source_spot.guid.content,
-                            dx_call=dx_call,
-                            de_call=spotter,
-                            freq=freq_hz,
-                            mode=mode,
-                            comment=comment,
-                            sig="WOTA",
-                            sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
-                            time=time.timestamp())
+                    new_spots.append(spot)
+                except Exception:
+                    logger.exception("Exception parsing WOTA spot")
+
+        except ExpatError:
+            logger.warning("WOTA spot RSS feed was fetched but was invalid")
 
-                new_spots.append(spot)
-            except Exception as e:
-                logging.error("Exception parsing WOTA spot", e)
         return new_spots
 
     def can_submit_spot(self, sig):

providers/spot/wwbota.py

@@ -20,25 +20,32 @@ class WWBOTA(SSESpotProvider):
         # n-fer activations.
         refs = []
         for ref in source_spot["references"]:
-            sigref = SIGRef(id=ref["reference"], sig="WWBOTA", name=ref["name"], latitude=ref["lat"],
-                            longitude=ref["long"])
+            sigref = SIGRef(
+                id=ref["reference"],
+                sig="WWBOTA",
+                name=ref["name"],
+                latitude=ref["lat"],
+                longitude=ref["long"],
+            )
             refs.append(sigref)
 
-        spot = Spot(source=self.name,
-                    dx_call=source_spot["call"].upper(),
-                    de_call=source_spot["spotter"].upper(),
-                    freq=float(source_spot["freq"]) * 1000000,
-                    mode=source_spot["mode"].upper(),
-                    comment=source_spot["comment"],
-                    sig="WWBOTA",
-                    sig_refs=refs,
-                    time=datetime.fromisoformat(source_spot["time"].replace("Z", "+00:00")).timestamp(),
-                    # WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
-                    # now, we will just pick the first one to use as our grid, latitude and longitude.
-                    dx_grid=source_spot["references"][0]["locator"],
-                    dx_latitude=source_spot["references"][0]["lat"],
-                    dx_longitude=source_spot["references"][0]["long"],
-                    qrt=source_spot["type"] == "QRT")
+        spot = Spot(
+            source=self.name,
+            dx_call=source_spot["call"].upper(),
+            de_call=source_spot["spotter"].upper(),
+            freq=float(source_spot["freq"]) * 1000000,
+            mode=source_spot["mode"].upper() if "mode" in source_spot else None,
+            comment=source_spot["comment"],
+            sig="WWBOTA",
+            sig_refs=refs,
+            time=datetime.fromisoformat(source_spot["time"].replace("Z", "+00:00")).timestamp(),
+            # WWBOTA spots can contain multiple references for bunkers being activated simultaneously. For
+            # now, we will just pick the first one to use as our grid, latitude and longitude.
+            dx_grid=source_spot["references"][0]["locator"],
+            dx_latitude=source_spot["references"][0]["lat"],
+            dx_longitude=source_spot["references"][0]["long"],
+            qrt=source_spot["type"] == "QRT",
+        )
 
         # WWBOTA does support a special "Test" spot type, we need to avoid adding that.
         return spot if source_spot["type"] != "Test" else None

providers/spot/wwff.py

@@ -21,19 +21,28 @@ class WWFF(HTTPSpotProvider):
         # Iterate through source data
         for source_spot in http_response.json():
             # Convert to our spot format
-            spot = Spot(source=self.name,
-                        source_id=source_spot["id"],
-                        dx_call=source_spot["activator"].upper(),
-                        de_call=source_spot["spotter"].upper(),
-                        freq=float(source_spot["frequency_khz"]) * 1000,
-                        mode=source_spot["mode"].upper(),
-                        comment=source_spot["remarks"],
+            spot = Spot(
+                source=self.name,
+                source_id=source_spot["id"],
+                dx_call=source_spot["activator"].upper(),
+                de_call=source_spot["spotter"].upper(),
+                freq=float(source_spot["frequency_khz"]) * 1000,
+                mode=source_spot["mode"].upper(),
+                comment=source_spot["remarks"],
+                sig="WWFF",
+                sig_refs=[
+                    SIGRef(
+                        id=source_spot["reference"],
                         sig="WWFF",
-                        sig_refs=[SIGRef(id=source_spot["reference"], sig="WWFF", name=source_spot["reference_name"],
-                                         latitude=source_spot["latitude"], longitude=source_spot["longitude"])],
-                        time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
-                        dx_latitude=source_spot["latitude"],
-                        dx_longitude=source_spot["longitude"])
+                        name=source_spot["reference_name"],
+                        latitude=source_spot["latitude"],
+                        longitude=source_spot["longitude"],
+                    )
+                ],
+                time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
+                dx_latitude=source_spot["latitude"],
+                dx_longitude=source_spot["longitude"],
+            )
 
             # Add to our list. Don't worry about de-duping, removing old spots etc. at this point; other code will do
             # that for us.