w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-11 07:01:41 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Further speed & stability improvements when loading sig ref datasets #126

Browse Source
This commit is contained in:
Ian Renton
2026-08-10 21:40:07 +01:00
parent 0d2e1369b6
commit 3df861189e
18 changed files with 97 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
providers/sigrefdata/arlhs.py
+5
View File
@@ -1,4 +1,5 @@
import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -32,4 +33,8 @@ class ARLHS(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/dme.py
+5
View File
@@ -1,4 +1,5 @@
import csv
from time import sleep
from pyhamtools.locator import latlong_to_locator
@@ -38,4 +39,8 @@ class DME(LocalFileSIGRefDataProvider):
if self._stop:
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/gma.py
+5
View File
@@ -1,4 +1,5 @@
import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -31,4 +32,8 @@ class GMA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/illw.py
+5
View File
@@ -1,4 +1,5 @@
import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -32,4 +33,8 @@ class ILLW(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/iota.py
+5
View File
@@ -1,4 +1,5 @@
import logging
from time import sleep
from pyhamtools.locator import latlong_to_locator
@@ -38,4 +39,8 @@ class IOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/llota.py
+6
View File
@@ -1,3 +1,5 @@
from time import sleep
from pyhamtools.locator import locator_to_latlong
from data.sig_ref import SIGRef
@@ -34,4 +36,8 @@ class LLOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/mota.py
+5
View File
@@ -1,4 +1,5 @@
import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -31,4 +32,8 @@ class MOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/pnp_kml_sig_ref_data_provider.py
+5
View File
@@ -1,4 +1,5 @@
import re
from time import sleep
from fastkml import kml
from pyhamtools.locator import latlong_to_locator
@@ -47,4 +48,8 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
return new_data
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/pota.py
+5
View File
@@ -1,4 +1,5 @@
import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -31,4 +32,8 @@ class POTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/sig_ref_data_provider.py
+9 -7
View File
@@ -35,14 +35,16 @@ class SIGRefDataProvider:
def _add_data(self, new_data):
"""Add all the provided reference data objects to the data store."""
for d in new_data:
DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d
# with transact() batches all writes together to save making thousands of individual sqlite writes
with DATA_STORE.sigrefs.transact():
for d in new_data:
DATA_STORE.sigrefs.set(self.sig_name + ":" + d.id, d)
# For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
# within the first few minutes of startup, we need a way to abort this expensive process of filling up the
# disk cache.
if self._stop:
break
# For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
# within the first few minutes of startup, we need a way to abort this expensive process of filling up the
# disk cache.
if self._stop:
break
self.reference_count = len(new_data)
logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)
providers/sigrefdata/siota.py
+5
View File
@@ -1,4 +1,5 @@
import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -28,4 +29,8 @@ class SIOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/sota.py
+5
View File
@@ -1,4 +1,5 @@
import csv
from time import sleep
from pyhamtools.locator import latlong_to_locator
@@ -36,4 +37,8 @@ class SOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/towers.py
+5
View File
@@ -1,4 +1,5 @@
import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -29,4 +30,8 @@ class Towers(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/wca.py
+5
View File
@@ -1,5 +1,6 @@
import csv
import logging
from time import sleep
from pyhamtools.locator import latlong_to_locator
@@ -46,4 +47,8 @@ class WCA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/wota.py
+6
View File
@@ -1,3 +1,5 @@
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -33,4 +35,8 @@ class WOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/wwbota.py
+5
View File
@@ -1,4 +1,5 @@
import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -29,4 +30,8 @@ class WWBOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/wwff.py
+5
View File
@@ -1,4 +1,5 @@
import csv
from time import sleep
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
@@ -32,4 +33,8 @@ class WWFF(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data
providers/sigrefdata/zlota.py
+6
View File
@@ -1,3 +1,5 @@
from time import sleep
from pyhamtools.locator import latlong_to_locator
from data.sig_ref import SIGRef
@@ -36,4 +38,8 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
return new_data