w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-06 02:21:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 5 #118

Browse Source
This commit is contained in:
Ian Renton
2026-07-31 22:35:52 +01:00
parent 55a265eb1a
commit fe10943ecc
22 changed files with 69 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sigrefdataproviders/arlhs.py
+4 -4
View File
@@ -15,16 +15,16 @@ class ARLHS(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
if "ARLHS" in row:
if "ARLHS" in row and row["ARLHS"] != "":
ref_id = row["ARLHS"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"])
grid=row["Maidenhead Locator"]))
return new_data
sigrefdataproviders/dme.py
+4 -3
View File
@@ -16,7 +16,7 @@ class DME(LocalFileSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.PATH)
def _file_to_data(self, path):
new_data = {}
new_data = []
with open(path, encoding="latin-1") as _f:
for row in csv.DictReader(_f, delimiter=";"):
ref_id = row["COD_INE"][:5]
@@ -25,10 +25,11 @@ class DME(LocalFileSIGRefDataProvider):
longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
"LONGITUD_ETRS89_REGCAN95") else None
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id,
ref = SIGRef(sig=self.SIG, id=ref_id,
name=row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"],
latitude=latitude,
longitude=longitude)
if latitude and longitude:
new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
return new_data
sigrefdataproviders/file_download_sig_ref_data_provider.py
+1 -1
View File
@@ -51,7 +51,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
new_data = self._http_response_to_data(http_response)
# Submit the new spots for processing. There might not be any spots for the less popular programs.
if new_data:
self._replace_data(new_data)
self._add_data(new_data)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
sigrefdataproviders/gma.py
+3 -3
View File
@@ -15,15 +15,15 @@ class GMA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"])
grid=row["Maidenhead Locator"]))
return new_data
sigrefdataproviders/illw.py
+4 -4
View File
@@ -15,16 +15,16 @@ class ILLW(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
if "ILLW" in row:
if "ILLW" in row and row["ILLW"] != "":
ref_id = row["ILLW"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"])
grid=row["Maidenhead Locator"]))
return new_data
sigrefdataproviders/iota.py
+3 -3
View File
@@ -17,7 +17,7 @@ class IOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
data = http_response.json()
if isinstance(data, list):
for ref in data:
@@ -30,7 +30,7 @@ class IOTA(FileDownloadSIGRefDataProvider):
except ValueError:
logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude)
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
grid=grid, latitude=latitude, longitude=longitude)
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
grid=grid, latitude=latitude, longitude=longitude))
return new_data
sigrefdataproviders/llota.py
+3 -3
View File
@@ -15,7 +15,7 @@ class LLOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
data = http_response.json()
if isinstance(data, list):
for ref in data:
@@ -23,10 +23,10 @@ class LLOTA(FileDownloadSIGRefDataProvider):
grid = str(ref["grid_locator"])
ll = locator_to_latlong(grid)
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
url="https://llota.app/list/ref/" + ref_id,
grid=grid,
latitude=ll[0],
longitude=ll[1])
longitude=ll[1]))
return new_data
sigrefdataproviders/local_file_sig_ref_data_provider.py
+1 -1
View File
@@ -18,7 +18,7 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
try:
new_data = self._file_to_data(self._path)
if new_data:
self._replace_data(new_data)
self._add_data(new_data)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
else:
sigrefdataproviders/mota.py
+3 -3
View File
@@ -15,15 +15,15 @@ class MOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"])
grid=row["Maidenhead Locator"]))
return new_data
sigrefdataproviders/pota.py
+3 -3
View File
@@ -15,15 +15,15 @@ class POTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
url="https://pota.app/#/park/" + ref_id,
grid=row["grid"] if "grid" in row else None,
latitude=float(row["latitude"]) if "latitude" in row and row[
"latitude"] != "" else None,
longitude=float(row["longitude"]) if "longitude" in row and row[
"longitude"] != "" else None)
"longitude"] != "" else None))
return new_data
sigrefdataproviders/sig_ref_data_provider.py
+4 -8
View File
@@ -19,10 +19,6 @@ class SIGRefDataProvider:
self.status = "Not Started" if self.enabled else "Disabled"
self.reference_count = 0
# Create an empty dict to store data if one doesn't already exist
if not sig_name in DATA_STORE.sigrefs:
DATA_STORE.sigrefs[sig_name] = {}
def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -35,10 +31,10 @@ class SIGRefDataProvider:
raise NotImplementedError("Subclasses must implement this method")
def _replace_data(self, new_data):
"""Replace all data for the named sig with the new data. new_data should be a map of reference ID to SIGRef
objects."""
def _add_data(self, new_data):
"""Add all the provided reference data objects to the data store."""
DATA_STORE.sigrefs[self.sig_name] = new_data
for d in new_data:
DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d
self.reference_count = len(new_data)
logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)
sigrefdataproviders/siota.py
+3 -3
View File
@@ -15,12 +15,12 @@ class SIOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["SILO_CODE"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
grid=row["LOCATOR"] if "LOCATOR" in row else None,
latitude=float(row["LAT"]) if "LAT" in row else None,
longitude=float(row["LNG"]) if "LNG" in row else None)
longitude=float(row["LNG"]) if "LNG" in row else None))
return new_data
sigrefdataproviders/sota.py
+4 -3
View File
@@ -17,17 +17,18 @@ class SOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["SummitCode"]
latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
ref = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
url="https://www.sotadata.org.uk/en/summit/" + ref_id,
latitude=latitude,
longitude=longitude,
activation_score=int(row["Points"]) if "Points" in row else None)
if latitude and longitude:
new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
return new_data
sigrefdataproviders/toilets.py
+3 -3
View File
@@ -14,11 +14,11 @@ class Toilets(LocalFileSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.PATH)
def _file_to_data(self, path):
new_data = {}
new_data = []
f = open(path)
csv_data = f.read()
dr = csv.DictReader(csv_data.splitlines())
for row in dr:
new_data[row["ref"]] = SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]),
longitude=float(row["lon"]))
new_data.append(SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]),
longitude=float(row["lon"])))
return new_data
sigrefdataproviders/towers.py
+3 -3
View File
@@ -15,13 +15,13 @@ class Towers(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
ref_id = row["Ref"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
url="https://wwtota.com/seznam/karta_rozhledny.php?ref=" + ref_id,
grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None)
longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None))
return new_data
sigrefdataproviders/wca.py
+3 -3
View File
@@ -18,7 +18,7 @@ class WCA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["REF"]
@@ -35,10 +35,10 @@ class WCA(FileDownloadSIGRefDataProvider):
except ValueError:
logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id)
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
latitude=latitude,
longitude=longitude,
grid=grid)
grid=grid))
return new_data
sigrefdataproviders/wota.py
+3 -3
View File
@@ -13,7 +13,7 @@ class WOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for feature in http_response.json().get("features", []):
ref_id = feature["properties"]["wotaId"]
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
@@ -23,9 +23,9 @@ class WOTA(FileDownloadSIGRefDataProvider):
number = int(ref_id.upper().replace("LDO-", ""))
url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
grid=feature["properties"]["qthLocator"],
latitude=feature["geometry"]["coordinates"][1],
longitude=feature["geometry"]["coordinates"][0])
longitude=feature["geometry"]["coordinates"][0]))
return new_data
sigrefdataproviders/wwbota.py
+3 -3
View File
@@ -15,13 +15,13 @@ class WWBOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["Reference"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
url="https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None,
grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None)
longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None))
return new_data
sigrefdataproviders/wwff.py
+3 -3
View File
@@ -15,16 +15,16 @@ class WWFF(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
url="https://wwff.co/directory/?showRef=" + ref_id,
grid=row["iaruLocator"] if "iaruLocator" in row and row[
"iaruLocator"] != "-" else None,
latitude=float(row["latitude"]) if "latitude" in row and row[
"latitude"] != "" and row["latitude"] != "-" else None,
longitude=float(row["longitude"]) if "longitude" in row and row[
"longitude"] != "" and row["longitude"] != "-" else None)
"longitude"] != "" and row["longitude"] != "-" else None))
return new_data
sigrefdataproviders/zlota.py
+4 -3
View File
@@ -15,7 +15,7 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
data = http_response.json()
if isinstance(data, list):
for ref in data:
@@ -23,11 +23,12 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
latitude = ref["y"]
longitude = ref["x"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
ref = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
url="https://ontheair.nz/assets/" + ref_id.replace("/", "_"),
latitude=latitude,
longitude=longitude)
if latitude and longitude:
new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
return new_data