From 3df861189e93242762bc4bce71add084d7a9acc7 Mon Sep 17 00:00:00 2001 From: Ian Renton Date: Mon, 10 Aug 2026 21:40:07 +0100 Subject: [PATCH] Further speed & stability improvements when loading sig ref datasets #126 --- providers/sigrefdata/arlhs.py | 5 +++++ providers/sigrefdata/dme.py | 5 +++++ providers/sigrefdata/gma.py | 5 +++++ providers/sigrefdata/illw.py | 5 +++++ providers/sigrefdata/iota.py | 5 +++++ providers/sigrefdata/llota.py | 6 ++++++ providers/sigrefdata/mota.py | 5 +++++ .../sigrefdata/pnp_kml_sig_ref_data_provider.py | 5 +++++ providers/sigrefdata/pota.py | 5 +++++ providers/sigrefdata/sig_ref_data_provider.py | 16 +++++++++------- providers/sigrefdata/siota.py | 5 +++++ providers/sigrefdata/sota.py | 5 +++++ providers/sigrefdata/towers.py | 5 +++++ providers/sigrefdata/wca.py | 5 +++++ providers/sigrefdata/wota.py | 6 ++++++ providers/sigrefdata/wwbota.py | 5 +++++ providers/sigrefdata/wwff.py | 5 +++++ providers/sigrefdata/zlota.py | 6 ++++++ 18 files changed, 97 insertions(+), 7 deletions(-) diff --git a/providers/sigrefdata/arlhs.py b/providers/sigrefdata/arlhs.py index 9375eff..9960a44 100644 --- a/providers/sigrefdata/arlhs.py +++ b/providers/sigrefdata/arlhs.py @@ -1,4 +1,5 @@ import csv +from time import sleep from data.sig_ref import SIGRef from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider @@ -32,4 +33,8 @@ class ARLHS(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/dme.py b/providers/sigrefdata/dme.py index d940322..96c9f94 100644 --- a/providers/sigrefdata/dme.py +++ b/providers/sigrefdata/dme.py @@ -1,4 +1,5 @@ import csv +from time import sleep from pyhamtools.locator import latlong_to_locator @@ -38,4 +39,8 @@ class DME(LocalFileSIGRefDataProvider): if self._stop: break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/gma.py b/providers/sigrefdata/gma.py index c894993..01da1cc 100644 --- a/providers/sigrefdata/gma.py +++ b/providers/sigrefdata/gma.py @@ -1,4 +1,5 @@ import csv +from time import sleep from data.sig_ref import SIGRef from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider @@ -31,4 +32,8 @@ class GMA(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/illw.py b/providers/sigrefdata/illw.py index 3b53fec..04d0219 100644 --- a/providers/sigrefdata/illw.py +++ b/providers/sigrefdata/illw.py @@ -1,4 +1,5 @@ import csv +from time import sleep from data.sig_ref import SIGRef from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider @@ -32,4 +33,8 @@ class ILLW(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/iota.py b/providers/sigrefdata/iota.py index 7326124..f3d78da 100644 --- a/providers/sigrefdata/iota.py +++ b/providers/sigrefdata/iota.py @@ -1,4 +1,5 @@ import logging +from time import sleep from pyhamtools.locator import latlong_to_locator @@ -38,4 +39,8 @@ class IOTA(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/llota.py b/providers/sigrefdata/llota.py index 0d81a9c..b5ad1e9 100644 --- a/providers/sigrefdata/llota.py +++ b/providers/sigrefdata/llota.py @@ -1,3 +1,5 @@ +from time import sleep + from pyhamtools.locator import locator_to_latlong from data.sig_ref import SIGRef @@ -34,4 +36,8 @@ class LLOTA(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/mota.py b/providers/sigrefdata/mota.py index 83643b1..b0db137 100644 --- a/providers/sigrefdata/mota.py +++ b/providers/sigrefdata/mota.py @@ -1,4 +1,5 @@ import csv +from time import sleep from data.sig_ref import SIGRef from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider @@ -31,4 +32,8 @@ class MOTA(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/pnp_kml_sig_ref_data_provider.py b/providers/sigrefdata/pnp_kml_sig_ref_data_provider.py index c7d018a..0c4e4fe 100644 --- a/providers/sigrefdata/pnp_kml_sig_ref_data_provider.py +++ b/providers/sigrefdata/pnp_kml_sig_ref_data_provider.py @@ -1,4 +1,5 @@ import re +from time import sleep from fastkml import kml from pyhamtools.locator import latlong_to_locator @@ -47,4 +48,8 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): return new_data + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/pota.py b/providers/sigrefdata/pota.py index dfb381c..c57e6ca 100644 --- a/providers/sigrefdata/pota.py +++ b/providers/sigrefdata/pota.py @@ -1,4 +1,5 @@ import csv +from time import sleep from data.sig_ref import SIGRef from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider @@ -31,4 +32,8 @@ class POTA(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/sig_ref_data_provider.py b/providers/sigrefdata/sig_ref_data_provider.py index 6bbb0a6..75c752f 100644 --- a/providers/sigrefdata/sig_ref_data_provider.py +++ b/providers/sigrefdata/sig_ref_data_provider.py @@ -35,14 +35,16 @@ class SIGRefDataProvider: def _add_data(self, new_data): """Add all the provided reference data objects to the data store.""" - for d in new_data: - DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d + # with transact() batches all writes together to save making thousands of individual sqlite writes + with DATA_STORE.sigrefs.transact(): + for d in new_data: + DATA_STORE.sigrefs.set(self.sig_name + ":" + d.id, d) - # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly - # within the first few minutes of startup, we need a way to abort this expensive process of filling up the - # disk cache. - if self._stop: - break + # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly + # within the first few minutes of startup, we need a way to abort this expensive process of filling up the + # disk cache. + if self._stop: + break self.reference_count = len(new_data) logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name) diff --git a/providers/sigrefdata/siota.py b/providers/sigrefdata/siota.py index 8e613d8..add41d1 100644 --- a/providers/sigrefdata/siota.py +++ b/providers/sigrefdata/siota.py @@ -1,4 +1,5 @@ import csv +from time import sleep from data.sig_ref import SIGRef from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider @@ -28,4 +29,8 @@ class SIOTA(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/sota.py b/providers/sigrefdata/sota.py index 4aa56f6..0853a19 100644 --- a/providers/sigrefdata/sota.py +++ b/providers/sigrefdata/sota.py @@ -1,4 +1,5 @@ import csv +from time import sleep from pyhamtools.locator import latlong_to_locator @@ -36,4 +37,8 @@ class SOTA(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/towers.py b/providers/sigrefdata/towers.py index cd62d6b..c968d72 100644 --- a/providers/sigrefdata/towers.py +++ b/providers/sigrefdata/towers.py @@ -1,4 +1,5 @@ import csv +from time import sleep from data.sig_ref import SIGRef from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider @@ -29,4 +30,8 @@ class Towers(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/wca.py b/providers/sigrefdata/wca.py index aa05bd2..339b360 100644 --- a/providers/sigrefdata/wca.py +++ b/providers/sigrefdata/wca.py @@ -1,5 +1,6 @@ import csv import logging +from time import sleep from pyhamtools.locator import latlong_to_locator @@ -46,4 +47,8 @@ class WCA(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/wota.py b/providers/sigrefdata/wota.py index 36376c3..dba4459 100644 --- a/providers/sigrefdata/wota.py +++ b/providers/sigrefdata/wota.py @@ -1,3 +1,5 @@ +from time import sleep + from data.sig_ref import SIGRef from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider @@ -33,4 +35,8 @@ class WOTA(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/wwbota.py b/providers/sigrefdata/wwbota.py index d0a9c3e..6e0535d 100644 --- a/providers/sigrefdata/wwbota.py +++ b/providers/sigrefdata/wwbota.py @@ -1,4 +1,5 @@ import csv +from time import sleep from data.sig_ref import SIGRef from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider @@ -29,4 +30,8 @@ class WWBOTA(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/wwff.py b/providers/sigrefdata/wwff.py index 31ea48e..7f02ed1 100644 --- a/providers/sigrefdata/wwff.py +++ b/providers/sigrefdata/wwff.py @@ -1,4 +1,5 @@ import csv +from time import sleep from data.sig_ref import SIGRef from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider @@ -32,4 +33,8 @@ class WWFF(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data diff --git a/providers/sigrefdata/zlota.py b/providers/sigrefdata/zlota.py index 0142689..c960c2a 100644 --- a/providers/sigrefdata/zlota.py +++ b/providers/sigrefdata/zlota.py @@ -1,3 +1,5 @@ +from time import sleep + from pyhamtools.locator import latlong_to_locator from data.sig_ref import SIGRef @@ -36,4 +38,8 @@ class ZLOTA(FileDownloadSIGRefDataProvider): if self._stop_event.is_set(): break + # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time + # is available for other threads e.g. the web server. + sleep(0.001) + return new_data