Refactor of caching & data storage part 17 #118

providers/alert/http_alert_provider.py

@@ -25,7 +25,7 @@ class HTTPAlertProvider(AlertProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info("Set up query of " + self.name + " alert API every " + str(self._poll_interval) + " seconds.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"HTTPAlertProvider-{self.name}")
         self._thread.start()
 
     def stop(self):

providers/callsigndata/file_download_callsign_data_provider.py

@@ -32,7 +32,7 @@ class FileDownloadCallsignDataProvider(CallsignDataProvider):
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info(
             "Set up query of " + self.name + " callsign reference data every " + str(self._poll_interval) + " days.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"FileDownloadCallsignDataProvider-{self.name}")
         self._thread.start()
 
     def stop(self):

providers/callsigndata/hamqth.py

@@ -77,11 +77,11 @@ class HamQTH(APIQueryCallsignDataProvider):
                         self._HAMQTH_BASE_URL + "?id=" + session_id + "&callsign=" + urllib.parse.quote_plus(
                             lookup_call) + "&prg=" + self._PRG, headers=HTTP_HEADERS, timeout=10)
                     if response.ok:
+                        # Found data, convert it to our object and return it
+                        data = xmltodict.parse(response.content)["HamQTH"]["search"]
                         self.status = "OK"
                         self.last_update_time = datetime.now(pytz.UTC)
                         self.lookup_count += 1
-                        # Found data, convert it to our object and return it
-                        data = xmltodict.parse(response.content)["HamQTH"]["search"]
                         return self.hamqth_response_to_callsign(callsign, data)
 
                     elif not response.from_cache:

providers/callsigndata/qrz.py

@@ -79,11 +79,12 @@ class QRZ(APIQueryCallsignDataProvider):
                         qrz_response = xmltodict.parse(response.content).get("QRZDatabase", {})
                         if qrz_response:
                             if "Callsign" in qrz_response:
+                                qrz_data = qrz_response.get("Callsign")
                                 self.status = "OK"
                                 self.last_update_time = datetime.now(pytz.UTC)
                                 self.lookup_count += 1
                                 # Found data, convert it to our object and return it
-                                return self.qrz_response_to_callsign(callsign, qrz_response.get("Callsign"))
+                                return self.qrz_response_to_callsign(callsign, qrz_data)
 
                             elif "Session" in qrz_response and "Error" in qrz_response.get("Session"):
                                 # Errors here are normally just "callsign not in database", no need to log that ourselves

providers/sigrefdata/file_download_sig_ref_data_provider.py

@@ -28,7 +28,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info(
             "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " days.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}")
         self._thread.start()
 
     def stop(self):

providers/solarconditions/giroionosonde.py

@@ -58,7 +58,7 @@ class GIROIonosonde(SolarConditionsProvider):
 
     def start(self):
         logging.info(f"Set up query of GIRO ionosonde data API every {POLL_INTERVAL} seconds.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name="GIROIonosondeDataProvider")
         self._thread.start()
 
     def stop(self):

providers/solarconditions/http_solar_conditions_provider.py

@@ -24,7 +24,7 @@ class HTTPSolarConditionsProvider(SolarConditionsProvider):
     def start(self):
         logging.info(
             "Set up query of " + self.name + " solar conditions API every " + str(self._poll_interval) + " seconds.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"HTTPSolarConditionsProvider-{self.name}")
         self._thread.start()
 
     def stop(self):

providers/solarconditions/kc2gprop.py

@@ -30,7 +30,7 @@ class KC2GProp(SolarConditionsProvider):
 
     def start(self):
         logging.info(f"Set up query of KC2G ionosonde data API every {POLL_INTERVAL} seconds.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name="KC2GPropProvider")
         self._thread.start()
 
     def stop(self):

providers/spot/aprsis.py

@@ -15,7 +15,7 @@ class APRSIS(SpotProvider):
 
     def __init__(self, provider_config):
         super().__init__("APRS-IS", provider_config)
-        self._thread = Thread(target=self._connect)
+        self._thread = Thread(target=self._connect, name="APRSISSpotProvider")
         self._thread.daemon = True
         self._aprsis = None
 

providers/spot/dxcluster.py

@@ -36,7 +36,7 @@ class DXCluster(SpotProvider):
         self._allow_rbn_spots = provider_config["allow_rbn_spots"] if "allow_rbn_spots" in provider_config else False
         self._spot_line_pattern = self._LINE_PATTERN_ALLOW_RBN if self._allow_rbn_spots else self._LINE_PATTERN_EXCLUDE_RBN
         self._telnet = None
-        self._thread = Thread(target=self._handle)
+        self._thread = Thread(target=self._handle, name=f"DXClusterSpotProvider-{self.name}")
         self._thread.daemon = True
         self._running = True
 

providers/spot/http_spot_provider.py

@@ -26,7 +26,7 @@ class HTTPSpotProvider(SpotProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info("Set up query of " + self.name + " spot API every " + str(self._poll_interval) + " seconds.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"HTTPSpotProvider-{self.name}")
         self._thread.start()
 
     def stop(self):

providers/spot/rbn.py

@@ -27,7 +27,7 @@ class RBN(SpotProvider):
         super().__init__(name, provider_config)
         self._port = provider_config["port"]
         self._telnet = None
-        self._thread = Thread(target=self._handle)
+        self._thread = Thread(target=self._handle, name=f"RBNSpotProvider-{self.name}")
         self._thread.daemon = True
         self._running = True
 

providers/spot/sse_spot_provider.py

@@ -1,7 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Thread
-from time import sleep
+from threading import Event, Lock, Thread
 
 import pytz
 from requests_sse import EventSource
@@ -16,24 +15,35 @@ class SSESpotProvider(SpotProvider):
     def __init__(self, name, provider_config, url):
         super().__init__(name, provider_config)
         self._url = url
-        self._event_source = None
         self._thread = None
-        self._stopped = False
         self._last_event_id = None
+        self._stop_event = Event()
+        self._event_source_lock = Lock()
+        self._event_source = None
 
     def start(self):
         logging.info("Set up SSE connection to " + self.name + " spot API.")
-        self._stopped = False
-        self._thread = Thread(target=self._run)
+        self._stop_event.clear()
+        self._thread = Thread(target=self._run, name=f"SSESpotProvider-{self.name}")
         self._thread.daemon = True
         self._thread.start()
 
     def stop(self):
-        self._stopped = True
-        if self._event_source:
-            self._event_source.close()
+        self._stop_event.set()
+
+        with self._event_source_lock:
+            event_source = self._event_source
+        if event_source:
+            try:
+                event_source.close()
+            except Exception:
+                logging.exception(
+                    "Exception closing SSE connection for " + self.name + " during stop()")
+
         if self._thread:
-            self._thread.join()
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logging.warning(self.name + " SSE worker thread did not exit on time and will be killed.")
 
     def _on_open(self):
         self.status = "Waiting for Data"
@@ -41,36 +51,45 @@ class SSESpotProvider(SpotProvider):
     def _on_error(self):
         self.status = "Connecting"
 
+    def _set_event_source(self, event_source):
+        with self._event_source_lock:
+            self._event_source = event_source
+
     def _run(self):
-        while not self._stopped:
+        while not self._stop_event.is_set():
             try:
                 logging.debug("Connecting to " + self.name + " spot API...")
                 self.status = "Connecting"
-                with EventSource(self._url, headers=HTTP_HEADERS, latest_event_id=self._last_event_id, timeout=30,
+                with EventSource(self._url, headers=HTTP_HEADERS, latest_event_id=self._last_event_id, timeout=10,
                                  on_open=self._on_open, on_error=self._on_error) as event_source:
-                    self._event_source = event_source
-                    for event in self._event_source:
-                        if event.type == 'message':
-                            try:
-                                self._last_event_id = event.last_event_id
-                                new_spot = self._sse_message_to_spot(event.data)
-                                if new_spot:
-                                    self._submit(new_spot)
+                    self._set_event_source(event_source)
+                    try:
+                        for event in event_source:
+                            if self._stop_event.is_set():
+                                break
+                            if event.type == 'message':
+                                try:
+                                    self._last_event_id = event.last_event_id
+                                    new_spot = self._sse_message_to_spot(event.data)
+                                    if new_spot:
+                                        self._submit(new_spot)
 
-                                self.status = "OK"
-                                self.last_update_time = datetime.now(pytz.UTC)
-                                logging.debug("Received data from " + self.name + " spot API.")
+                                    self.status = "OK"
+                                    self.last_update_time = datetime.now(pytz.UTC)
+                                    logging.debug("Received data from " + self.name + " spot API.")
 
-                            except Exception:
-                                logging.exception(
-                                    "Exception processing message from SSE Spot Provider (" + self.name + ")")
+                                except Exception:
+                                    logging.exception(
+                                        "Exception processing message from SSE Spot Provider (" + self.name + ")")
+                    finally:
+                        self._set_event_source(None)
 
             except Exception:
                 self.status = "Error"
                 logging.exception("Exception in SSE Spot Provider (" + self.name + ")")
             else:
                 self.status = "Disconnected"
-            sleep(5)  # Wait before trying to reconnect
+            self._stop_event.wait(timeout=5) # Wait before trying to reconnect
 
     def _sse_message_to_spot(self, message_data):
         """Convert an SSE message received from the API into a spot. The whole message data is provided here so the subclass

providers/spot/websocket_spot_provider.py

@@ -24,7 +24,7 @@ class WebsocketSpotProvider(SpotProvider):
     def start(self):
         logging.info("Set up websocket connection to " + self.name + " spot API.")
         self._stopped = False
-        self._thread = Thread(target=self._run)
+        self._thread = Thread(target=self._run, name=f"WebsocketSpotProvider-{self.name}")
         self._thread.daemon = True
         self._thread.start()
 

providers/staticdata/file_download_static_data_provider.py

@@ -29,7 +29,7 @@ class FileDownloadStaticDataProvider(StaticDataProvider):
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info(
             "Set up query of " + self.name + " static reference data every " + str(self._poll_interval) + " days.")
-        self._thread = Thread(target=self._run, daemon=True)
+        self._thread = Thread(target=self._run, name=f"FileDownloadStaticDataProvider-{self.name}")
         self._thread.start()
 
     def stop(self):