Attempt to fix CPU utilisation bug by preventing the heartbeat callback leak in the SSE stream handlers and replacing Timer-based with Event-based threads. Also compiled regexes in advance for DXCC callsign lookups for efficiency, and fixed my misunderstanding of what Queue.empty() does

core/cleanup.py

@@ -1,6 +1,6 @@
 import logging
 from datetime import datetime
-from threading import Timer
+from threading import Timer, Event, Thread
 from time import sleep
 
 import pytz
@@ -18,17 +18,23 @@ class CleanupTimer:
         self.cleanup_timer = None
         self.last_cleanup_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Starting"
+        self._stop_event = Event()
 
     # Start the cleanup timer
     def start(self):
-        self.cleanup()
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
 
     # Stop any threads and prepare for application shutdown
     def stop(self):
-        self.cleanup_timer.cancel()
+        self._stop_event.set()
+
+    def _run(self):
+        while not self._stop_event.wait(timeout=self.cleanup_interval):
+            self._cleanup()
 
     # Perform cleanup and reschedule next timer
-    def cleanup(self):
+    def _cleanup(self):
         try:
             # Perform cleanup via letting the data expire
             self.spots.expire()
@@ -61,7 +67,4 @@ class CleanupTimer:
         except Exception as e:
             self.status = "Error"
             logging.exception("Exception in Cleanup thread")
-            sleep(1)
-
-        self.cleanup_timer = Timer(self.cleanup_interval, self.cleanup)
-        self.cleanup_timer.start()
+            self._stop_event.wait(timeout=1)

core/lookup_helper.py

@@ -101,6 +101,10 @@ class LookupHelper:
         else:
             logging.error("Could not download DXCC data, flags and similar data may be missing!")
 
+        # Precompile regex matches for DXCCs to improve efficiency when iterating through them
+        for dxcc in self.DXCC_DATA.values():
+            dxcc["_prefixRegexCompiled"] = re.compile(dxcc["prefixRegex"])
+
     # Download the cty.plist file from country-files.com on first startup. The pyhamtools lib can actually download and use
     # this itself, but it's occasionally offline which causes it to throw an error. By downloading it separately, we can
     # catch errors and handle them, falling back to a previous copy of the file in the cache, and we can use the
@@ -559,7 +563,7 @@ class LookupHelper:
     # Utility method to get generic DXCC data from our lookup table, if we can find it
     def get_dxcc_data_for_callsign(self, call):
         for entry in self.DXCC_DATA.values():
-            if re.match(entry["prefixRegex"], call):
+            if entry["_prefixRegexCompiled"].match(call):
                 return entry
         return None
 

core/status_reporter.py

@@ -1,6 +1,6 @@
 import os
 from datetime import datetime
-from threading import Timer
+from threading import Thread, Event
 
 import psutil
 import pytz
@@ -24,22 +24,30 @@ class StatusReporter:
         self.spot_providers = spot_providers
         self.alerts = alerts
         self.alert_providers = alert_providers
-        self.run_timer = None
+        self._stop_event = Event()
         self.startup_time = datetime.now(pytz.UTC)
 
         self.status_data["software-version"] = SOFTWARE_VERSION
         self.status_data["server-owner-callsign"] = SERVER_OWNER_CALLSIGN
 
-    # Start the cleanup timer
+    # Start the reporter thread
     def start(self):
-        self.run()
+        self._thread = Thread(target=self._run, daemon=True)
+        self._thread.start()
 
     # Stop any threads and prepare for application shutdown
     def stop(self):
-        self.run_timer.cancel()
+        self._stop_event.set()
 
-    # Write status information and reschedule next timer
-    def run(self):
+    # Thread entry point: report immediately on startup, then on each interval until stopped
+    def _run(self):
+        while True:
+            self._report()
+            if self._stop_event.wait(timeout=self.run_interval):
+                break
+
+    # Write status information
+    def _report(self):
         self.status_data["uptime"] = (datetime.now(pytz.UTC) - self.startup_time).total_seconds()
         self.status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3)
         self.status_data["num_spots"] = len(self.spots)
@@ -73,7 +81,4 @@ class StatusReporter:
         # Update Prometheus metrics
         memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss * 1024)
         spots_gauge.set(len(self.spots))
-        alerts_gauge.set(len(self.alerts))
-
-        self.run_timer = Timer(self.run_interval, self.run)
-        self.run_timer.start()
+        alerts_gauge.set(len(self.alerts))

core/utils.py

@@ -3,3 +3,12 @@
 # to receive spots without complex handling.
 def serialize_everything(obj):
     return obj.__dict__
+
+
+# Empty a queue
+def empty_queue(q):
+    while not q.empty():
+        try:
+            q.get_nowait()
+        except:
+            break