Further speed & stability improvements when loading sig ref datasets #126

providers/sigrefdata/arlhs.py

@@ -1,4 +1,5 @@
 import csv
+from time import sleep
 
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -32,4 +33,8 @@ class ARLHS(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/dme.py

@@ -1,4 +1,5 @@
 import csv
+from time import sleep
 
 from pyhamtools.locator import latlong_to_locator
 
@@ -38,4 +39,8 @@ class DME(LocalFileSIGRefDataProvider):
                 if self._stop:
                     break
 
+                # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+                # is available for other threads e.g. the web server.
+                sleep(0.001)
+
         return new_data

providers/sigrefdata/gma.py

@@ -1,4 +1,5 @@
 import csv
+from time import sleep
 
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -31,4 +32,8 @@ class GMA(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/illw.py

@@ -1,4 +1,5 @@
 import csv
+from time import sleep
 
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -32,4 +33,8 @@ class ILLW(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/iota.py

@@ -1,4 +1,5 @@
 import logging
+from time import sleep
 
 from pyhamtools.locator import latlong_to_locator
 
@@ -38,4 +39,8 @@ class IOTA(FileDownloadSIGRefDataProvider):
                 if self._stop_event.is_set():
                     break
 
+                # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+                # is available for other threads e.g. the web server.
+                sleep(0.001)
+
         return new_data

providers/sigrefdata/llota.py

@@ -1,3 +1,5 @@
+from time import sleep
+
 from pyhamtools.locator import locator_to_latlong
 
 from data.sig_ref import SIGRef
@@ -34,4 +36,8 @@ class LLOTA(FileDownloadSIGRefDataProvider):
                 if self._stop_event.is_set():
                     break
 
+                # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+                # is available for other threads e.g. the web server.
+                sleep(0.001)
+
         return new_data

providers/sigrefdata/mota.py

@@ -1,4 +1,5 @@
 import csv
+from time import sleep
 
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -31,4 +32,8 @@ class MOTA(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/pnp_kml_sig_ref_data_provider.py

@@ -1,4 +1,5 @@
 import re
+from time import sleep
 
 from fastkml import kml
 from pyhamtools.locator import latlong_to_locator
@@ -47,4 +48,8 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
                     if self._stop_event.is_set():
                         return new_data
 
+                    # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+                    # is available for other threads e.g. the web server.
+                    sleep(0.001)
+
         return new_data

providers/sigrefdata/pota.py

@@ -1,4 +1,5 @@
 import csv
+from time import sleep
 
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -31,4 +32,8 @@ class POTA(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/sig_ref_data_provider.py

@@ -35,14 +35,16 @@ class SIGRefDataProvider:
     def _add_data(self, new_data):
         """Add all the provided reference data objects to the data store."""
 
-        for d in new_data:
+        # with transact() batches all writes together to save making thousands of individual sqlite writes
-            DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d
+        with DATA_STORE.sigrefs.transact():
+            for d in new_data:
+                DATA_STORE.sigrefs.set(self.sig_name + ":" + d.id, d)
 
-            # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
+                # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
-            # within the first few minutes of startup, we need a way to abort this expensive process of filling up the
+                # within the first few minutes of startup, we need a way to abort this expensive process of filling up the
-            # disk cache.
+                # disk cache.
-            if self._stop:
+                if self._stop:
-                break
+                    break
 
         self.reference_count = len(new_data)
         logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)

providers/sigrefdata/siota.py

@@ -1,4 +1,5 @@
 import csv
+from time import sleep
 
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -28,4 +29,8 @@ class SIOTA(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/sota.py

@@ -1,4 +1,5 @@
 import csv
+from time import sleep
 
 from pyhamtools.locator import latlong_to_locator
 
@@ -36,4 +37,8 @@ class SOTA(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/towers.py

@@ -1,4 +1,5 @@
 import csv
+from time import sleep
 
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -29,4 +30,8 @@ class Towers(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/wca.py

@@ -1,5 +1,6 @@
 import csv
 import logging
+from time import sleep
 
 from pyhamtools.locator import latlong_to_locator
 
@@ -46,4 +47,8 @@ class WCA(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/wota.py

@@ -1,3 +1,5 @@
+from time import sleep
+
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
 
@@ -33,4 +35,8 @@ class WOTA(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/wwbota.py

@@ -1,4 +1,5 @@
 import csv
+from time import sleep
 
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -29,4 +30,8 @@ class WWBOTA(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/wwff.py

@@ -1,4 +1,5 @@
 import csv
+from time import sleep
 
 from data.sig_ref import SIGRef
 from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -32,4 +33,8 @@ class WWFF(FileDownloadSIGRefDataProvider):
             if self._stop_event.is_set():
                 break
 
+            # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+            # is available for other threads e.g. the web server.
+            sleep(0.001)
+
         return new_data

providers/sigrefdata/zlota.py

@@ -1,3 +1,5 @@
+from time import sleep
+
 from pyhamtools.locator import latlong_to_locator
 
 from data.sig_ref import SIGRef
@@ -36,4 +38,8 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
                 if self._stop_event.is_set():
                     break
 
+                # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
+                # is available for other threads e.g. the web server.
+                sleep(0.001)
+
         return new_data