Improve slow startup times. #126

core/data_providers.py

@@ -1,3 +1,6 @@
+import logging
+import threading
+
 from core.config import config, create_provider_from_config
 
 
@@ -27,27 +30,24 @@ class DataProviders:
         for entry in config.get("callsign_data_providers", []):
             self.callsign_data_providers.append(create_provider_from_config("providers.callsigndata", entry))
 
+    @staticmethod
+    def start_providers(providers, type):
+        """Helper method to activate enabled providers in the list."""
+
+        logging.info(f"Starting %s providers...", type)
+        for p in providers:
+            if p.enabled:
+                p.start()
 
     def start(self):
         # Start data providers before spot/alert providers so the lookup data is there already for incoming spots.
-        for p in self.static_data_providers:
+        # Each category is fired off after a small delay to give the rest of Spothole chance to start up.
-            if p.enabled:
+        threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")).start()
-                p.start()
+        threading.Timer(10.0, lambda: self.start_providers(self.callsign_data_providers, "callsign data")).start()
-        for p in self.sig_ref_data_providers:
+        threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")).start()
-            if p.enabled:
+        threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")).start()
-                p.start()
+        threading.Timer(25.0, lambda: self.start_providers(self.solar_condition_providers, "solar condition")).start()
-        for p in self.callsign_data_providers:
+        threading.Timer(30.0, lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data")).start()
-            if p.enabled:
-                p.start()
-        for p in self.spot_providers:
-            if p.enabled:
-                p.start()
-        for p in self.alert_providers:
-            if p.enabled:
-                p.start()
-        for p in self.solar_condition_providers:
-            if p.enabled:
-                p.start()
 
     def stop(self):
         for sp in self.spot_providers:

docs/running.md

@@ -46,8 +46,8 @@ source .venv/bin/activate
 python3 spothole.py
 ```
 
-The software can take a few seconds to start up, mostly because it is downloading an updated file to match callsigns to
+The software can take a few seconds to start up, particularly if it's been run previously and has a large amount of
-countries. This is normal, don't panic! Once you see `You can access your copy of Spothole at
+cache data to sort through. This is normal, don't panic! Once you see `You can access your copy of Spothole at
 http://localhost:8080` in the log, your server is good to go.
 
 If you see some errors on startup, check your configuration, e.g. in case you have specified a port for the web server

providers/sigrefdata/file_download_sig_ref_data_provider.py

@@ -32,6 +32,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         self._thread.start()
 
     def stop(self):
+        super().stop()
         self._stop_event.set()
 
     def _run(self):

providers/sigrefdata/local_file_sig_ref_data_provider.py

@@ -12,7 +12,6 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
     def __init__(self, sig, provider_config, path):
         super().__init__(sig, provider_config)
         self._path = path
-        self._stop = False
 
     def start(self):
         logging.debug("Loading " + self.sig_name + " SIG ref data from file.")
@@ -29,9 +28,6 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
             self.status = "Error"
             logging.error("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")", e, exc_info=True)
 
-    def stop(self):
-        self._stop = True
-
     def _file_to_data(self, path):
         """Load a file on the given path and turn it into SIG Ref data."""
 

providers/sigrefdata/sig_ref_data_provider.py

@@ -17,6 +17,7 @@ class SIGRefDataProvider:
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
         self.reference_count = 0
+        self._stop = False
 
 
     def start(self):
@@ -26,14 +27,22 @@ class SIGRefDataProvider:
 
 
     def stop(self):
-        """Stop any threads and prepare for application shutdown"""
+        """Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
+        super()."""
 
-        raise NotImplementedError("Subclasses must implement this method")
+        self._stop = True
 
     def _add_data(self, new_data):
         """Add all the provided reference data objects to the data store."""
 
         for d in new_data:
             DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d
+
+            # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
+            # within the first few minutes of startup, we need a way to abort this expensive process of filling up the
+            # disk cache.
+            if self._stop:
+                break
+
         self.reference_count = len(new_data)
         logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)

spothole.py

@@ -59,8 +59,6 @@ if __name__ == '__main__':
     # Set up the web server
     WEB_SERVER.setup()
 
-    logging.info("Startup complete.")
-
     # Run the web server. This is the blocking call that keeps the application running in the main thread, so this must
     # be the last thing we do. web_server.stop() triggers an await condition in the web server which finishes the main
     # thread.