Use _stop_event consistently across all threads as the way to signal that it should stop. Add thread joins with timeouts to allow the program to exit cleanly

webserver/webserver.py

@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import os
+import threading
 
 import tornado
 from tornado.web import StaticFileHandler
@@ -53,6 +54,8 @@ class WebServer:
         self._port = WEB_SERVER_PORT
         self._api_only_mode = API_ONLY_MODE
         self._shutdown_event = asyncio.Event()
+        self._loop = None
+        self._thread = None
         self.web_server_metrics = WebServerMetrics()
 
     def setup(self):
@@ -69,16 +72,34 @@ class WebServer:
     def start(self):
         """Start the web server"""
 
-        asyncio.run(self._start_inner())
+        self._thread = threading.Thread(target=asyncio.run, args=(self._start_inner(),), name="WebServer", daemon=True)
+        self._thread.start()
 
     def stop(self):
         """Stop the web server"""
 
-        self._shutdown_event.set()
+        if self._loop and self._loop.is_running():
+            self._loop.call_soon_threadsafe(self._shutdown_event.set)
+        if self._thread:
+            self._thread.join(timeout=15)
+            if self._thread.is_alive():
+                logger.warning("Web server background thread did not exit on time and will be killed.")
+
+    def _handle_loop_exception(self, loop, context):
+        """Ignore "cancelled" exceptions from the asyncio loop to avoid printing exceptions to the log on shutdown when
+        we cancel the SSE handler threads"""
+
+        exception = context.get("exception")
+        if isinstance(exception, asyncio.CancelledError):
+            return
+        loop.default_exception_handler(context)
 
     async def _start_inner(self):
         """Start method (async). Sets up the Tornado application."""
 
+        self._loop = asyncio.get_running_loop()
+        self._loop.set_exception_handler(self._handle_loop_exception)
+
         # Bind the SSE broadcasters to the web server's loop, so they fire correctly
         self._spot_broadcaster.bind_to_web_server_loop()
         self._alert_broadcaster.bind_to_web_server_loop()