From d5d91ef50e1d319c95675576305e10296d9f627f Mon Sep 17 00:00:00 2001 From: Ian Renton Date: Mon, 10 Aug 2026 21:25:52 +0100 Subject: [PATCH] Improve slow startup times. #126 --- core/data_providers.py | 36 +++++++++---------- docs/running.md | 4 +-- .../file_download_sig_ref_data_provider.py | 1 + .../local_file_sig_ref_data_provider.py | 4 --- providers/sigrefdata/sig_ref_data_provider.py | 13 +++++-- spothole.py | 2 -- 6 files changed, 32 insertions(+), 28 deletions(-) diff --git a/core/data_providers.py b/core/data_providers.py index 8280248..385723f 100644 --- a/core/data_providers.py +++ b/core/data_providers.py @@ -1,3 +1,6 @@ +import logging +import threading + from core.config import config, create_provider_from_config @@ -27,27 +30,24 @@ class DataProviders: for entry in config.get("callsign_data_providers", []): self.callsign_data_providers.append(create_provider_from_config("providers.callsigndata", entry)) + @staticmethod + def start_providers(providers, type): + """Helper method to activate enabled providers in the list.""" + + logging.info(f"Starting %s providers...", type) + for p in providers: + if p.enabled: + p.start() def start(self): # Start data providers before spot/alert providers so the lookup data is there already for incoming spots. - for p in self.static_data_providers: - if p.enabled: - p.start() - for p in self.sig_ref_data_providers: - if p.enabled: - p.start() - for p in self.callsign_data_providers: - if p.enabled: - p.start() - for p in self.spot_providers: - if p.enabled: - p.start() - for p in self.alert_providers: - if p.enabled: - p.start() - for p in self.solar_condition_providers: - if p.enabled: - p.start() + # Each category is fired off after a small delay to give the rest of Spothole chance to start up. + threading.Timer(5.0, lambda: self.start_providers(self.static_data_providers, "static data")).start() + threading.Timer(10.0, lambda: self.start_providers(self.callsign_data_providers, "callsign data")).start() + threading.Timer(15.0, lambda: self.start_providers(self.spot_providers, "spot")).start() + threading.Timer(20.0, lambda: self.start_providers(self.alert_providers, "alert")).start() + threading.Timer(25.0, lambda: self.start_providers(self.solar_condition_providers, "solar condition")).start() + threading.Timer(30.0, lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data")).start() def stop(self): for sp in self.spot_providers: diff --git a/docs/running.md b/docs/running.md index 340aabd..e58c765 100644 --- a/docs/running.md +++ b/docs/running.md @@ -46,8 +46,8 @@ source .venv/bin/activate python3 spothole.py ``` -The software can take a few seconds to start up, mostly because it is downloading an updated file to match callsigns to -countries. This is normal, don't panic! Once you see `You can access your copy of Spothole at +The software can take a few seconds to start up, particularly if it's been run previously and has a large amount of +cache data to sort through. This is normal, don't panic! Once you see `You can access your copy of Spothole at http://localhost:8080` in the log, your server is good to go. If you see some errors on startup, check your configuration, e.g. in case you have specified a port for the web server diff --git a/providers/sigrefdata/file_download_sig_ref_data_provider.py b/providers/sigrefdata/file_download_sig_ref_data_provider.py index aafe82d..55653d9 100644 --- a/providers/sigrefdata/file_download_sig_ref_data_provider.py +++ b/providers/sigrefdata/file_download_sig_ref_data_provider.py @@ -32,6 +32,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider): self._thread.start() def stop(self): + super().stop() self._stop_event.set() def _run(self): diff --git a/providers/sigrefdata/local_file_sig_ref_data_provider.py b/providers/sigrefdata/local_file_sig_ref_data_provider.py index 740bf41..8a99a50 100644 --- a/providers/sigrefdata/local_file_sig_ref_data_provider.py +++ b/providers/sigrefdata/local_file_sig_ref_data_provider.py @@ -12,7 +12,6 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider): def __init__(self, sig, provider_config, path): super().__init__(sig, provider_config) self._path = path - self._stop = False def start(self): logging.debug("Loading " + self.sig_name + " SIG ref data from file.") @@ -29,9 +28,6 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider): self.status = "Error" logging.error("Exception in local file SIG Ref Data Provider (" + self.sig_name + ")", e, exc_info=True) - def stop(self): - self._stop = True - def _file_to_data(self, path): """Load a file on the given path and turn it into SIG Ref data.""" diff --git a/providers/sigrefdata/sig_ref_data_provider.py b/providers/sigrefdata/sig_ref_data_provider.py index 8393e49..6bbb0a6 100644 --- a/providers/sigrefdata/sig_ref_data_provider.py +++ b/providers/sigrefdata/sig_ref_data_provider.py @@ -17,6 +17,7 @@ class SIGRefDataProvider: self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC) self.status = "Not Started" if self.enabled else "Disabled" self.reference_count = 0 + self._stop = False def start(self): @@ -26,14 +27,22 @@ class SIGRefDataProvider: def stop(self): - """Stop any threads and prepare for application shutdown""" + """Stop any threads and prepare for application shutdown. Subclasses should implement this method and call + super().""" - raise NotImplementedError("Subclasses must implement this method") + self._stop = True def _add_data(self, new_data): """Add all the provided reference data objects to the data store.""" for d in new_data: DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d + + # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly + # within the first few minutes of startup, we need a way to abort this expensive process of filling up the + # disk cache. + if self._stop: + break + self.reference_count = len(new_data) logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name) diff --git a/spothole.py b/spothole.py index a0e6d1c..fac892c 100644 --- a/spothole.py +++ b/spothole.py @@ -59,8 +59,6 @@ if __name__ == '__main__': # Set up the web server WEB_SERVER.setup() - logging.info("Startup complete.") - # Run the web server. This is the blocking call that keeps the application running in the main thread, so this must # be the last thing we do. web_server.stop() triggers an await condition in the web server which finishes the main # thread.