mirror of
https://git.ianrenton.com/ian/spothole.git
synced 2026-08-06 02:21:42 +00:00
Refactor of caching & data storage part 3 #118
This commit is contained in:
Ian Renton
+54
-131
@@ -1,4 +1,3 @@
|
||||
import csv
|
||||
import logging
|
||||
|
||||
from pyhamtools.locator import latlong_to_locator, locator_to_latlong
|
||||
@@ -9,11 +8,6 @@ from core.data_store import DATA_STORE
|
||||
from core.geo_utils import wab_wai_square_to_lat_lon
|
||||
from core.url_data_cache import URL_DATA_CACHE
|
||||
|
||||
# Load Spanish municipality data for the DME programme. There's no convenient lookup API for this, so we embed the data
|
||||
# file in Spothole and load it on startup.
|
||||
with open("datafiles/MUNICIPIOS.csv", encoding="latin-1") as _f:
|
||||
for row in csv.DictReader(_f, delimiter=";"):
|
||||
DATA_STORE.sigrefs_dme.add(row["COD_INE"][:5], row)
|
||||
|
||||
def get_ref_regex_for_sig(sig):
|
||||
"""Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
|
||||
@@ -35,12 +29,15 @@ def get_sig_name_from_comment_name(sig):
|
||||
|
||||
|
||||
def populate_sig_ref_info(sig_ref):
|
||||
"""Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object which
|
||||
must at minimum have a "sig" and an "id". The rest of the object will be populated and returned.
|
||||
"""Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
|
||||
which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes
|
||||
use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
|
||||
SIG we are getting data for.
|
||||
Note there is currently no support for KRMNPA location lookup, see issue #61."""
|
||||
|
||||
if sig_ref.sig is None or sig_ref.id is None:
|
||||
logging.warning("Failed to look up sig_ref info, sig or id were not set.")
|
||||
if sig_ref.sig is None or sig_ref.id is None or sig_ref.id == "":
|
||||
logging.debug("Failed to look up sig_ref info, sig or id were not set.")
|
||||
return sig_ref
|
||||
|
||||
sig = sig_ref.sig or ""
|
||||
ref_id = sig_ref.id
|
||||
@@ -123,95 +120,36 @@ def populate_sig_ref_info(sig_ref):
|
||||
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
|
||||
|
||||
elif sig.upper() == "WWFF":
|
||||
response = URL_DATA_CACHE.get("https://wwff.co/wwff-data/wwff_directory.csv",
|
||||
headers=HTTP_HEADERS)
|
||||
if response.ok:
|
||||
if not bool(DATA_STORE.sigrefs_wwff) or not response.from_cache:
|
||||
# New data from WWFF, update our internal map
|
||||
for row in csv.DictReader(response.content.decode().splitlines()):
|
||||
DATA_STORE.sigrefs_wwff.add(row["reference"], row)
|
||||
row = DATA_STORE.sigrefs_wwff.get(ref_id)
|
||||
if row:
|
||||
sig_ref.name = row["name"] if "name" in row else None
|
||||
sig_ref.url = "https://wwff.co/directory/?showRef=" + ref_id
|
||||
sig_ref.grid = row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None
|
||||
sig_ref.latitude = float(row["latitude"]) if "latitude" in row and row["latitude"] != "-" else None
|
||||
sig_ref.longitude = float(row["longitude"]) if "longitude" in row and row[
|
||||
"longitude"] != "-" else None
|
||||
elif not response.from_cache:
|
||||
logging.warning("WWFF database did not contain data for ref %s", ref_id)
|
||||
elif not response.from_cache:
|
||||
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
|
||||
lookup_data = DATA_STORE.sigrefs["WWFF"][ref_id] if ref_id in DATA_STORE.sigrefs["WWFF"] else None
|
||||
if lookup_data:
|
||||
# Copy new sig ref data into existing object
|
||||
sig_ref.__dict__.update(lookup_data.__dict__)
|
||||
else:
|
||||
logging.warning("WWFF database did not contain data for ref %s", ref_id)
|
||||
|
||||
elif sig.upper() == "SIOTA":
|
||||
response = URL_DATA_CACHE.get("https://www.silosontheair.com/data/silos.csv",
|
||||
headers=HTTP_HEADERS)
|
||||
if response.ok:
|
||||
if not bool(DATA_STORE.sigrefs_siota) or not response.from_cache:
|
||||
# New data from SIOTA, update our internal map
|
||||
for row in csv.DictReader(response.content.decode().splitlines()):
|
||||
DATA_STORE.sigrefs_siota.add(row["SILO_CODE"], row)
|
||||
row = DATA_STORE.sigrefs_siota.get(ref_id)
|
||||
if row:
|
||||
sig_ref.name = row["NAME"] if "NAME" in row else None
|
||||
sig_ref.grid = row["LOCATOR"] if "LOCATOR" in row else None
|
||||
sig_ref.latitude = float(row["LAT"]) if "LAT" in row else None
|
||||
sig_ref.longitude = float(row["LNG"]) if "LNG" in row else None
|
||||
elif not response.from_cache:
|
||||
logging.warning("SIOTA database did not contain data for ref %s", ref_id)
|
||||
elif not response.from_cache:
|
||||
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
|
||||
lookup_data = DATA_STORE.sigrefs["SIOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["SIOTA"] else None
|
||||
if lookup_data:
|
||||
# Copy new sig ref data into existing object
|
||||
sig_ref.__dict__.update(lookup_data.__dict__)
|
||||
else:
|
||||
logging.warning("SIOTA database did not contain data for ref %s", ref_id)
|
||||
|
||||
elif sig.upper() == "WOTA":
|
||||
response = URL_DATA_CACHE.get("https://www.wota.org.uk/mapping/data/summits.json",
|
||||
headers=HTTP_HEADERS)
|
||||
if response.ok:
|
||||
data = response.json()
|
||||
if data:
|
||||
if not bool(DATA_STORE.sigrefs_wota) or not response.from_cache:
|
||||
# New data from WOTA, update our internal map
|
||||
for feature in data.get("features", []):
|
||||
DATA_STORE.sigrefs_wota.add(feature["properties"]["wotaId"], feature)
|
||||
feature = DATA_STORE.sigrefs_wota.get(ref_id)
|
||||
if feature:
|
||||
sig_ref.name = feature["properties"]["title"]
|
||||
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
|
||||
# added to them
|
||||
sig_ref.url = "https://www.wota.org.uk/MM_" + ref_id
|
||||
if ref_id.upper().startswith("LDO-"):
|
||||
number = int(ref_id.upper().replace("LDO-", ""))
|
||||
sig_ref.url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
|
||||
sig_ref.grid = feature["properties"]["qthLocator"]
|
||||
sig_ref.latitude = feature["geometry"]["coordinates"][1]
|
||||
sig_ref.longitude = feature["geometry"]["coordinates"][0]
|
||||
elif not response.from_cache:
|
||||
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
|
||||
elif not response.from_cache:
|
||||
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
|
||||
lookup_data = DATA_STORE.sigrefs["WOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["WOTA"] else None
|
||||
if lookup_data:
|
||||
# Copy new sig ref data into existing object
|
||||
sig_ref.__dict__.update(lookup_data.__dict__)
|
||||
else:
|
||||
logging.warning("WOTA database did not contain data for ref %s", ref_id)
|
||||
|
||||
elif sig.upper() == "ZLOTA":
|
||||
response = URL_DATA_CACHE.get("https://ontheair.nz/assets/assets.json", headers=HTTP_HEADERS)
|
||||
if response.ok:
|
||||
data = response.json()
|
||||
if isinstance(data, list):
|
||||
if not bool(DATA_STORE.sigrefs_zlota) or not response.from_cache:
|
||||
# New data from ZLOTA, update our internal map
|
||||
for ref in data:
|
||||
DATA_STORE.sigrefs_zlota.add(ref["code"], ref)
|
||||
ref = DATA_STORE.sigrefs_zlota.get(ref_id)
|
||||
if ref:
|
||||
sig_ref.name = ref["name"]
|
||||
sig_ref.url = "https://ontheair.nz/assets/" + ref_id.replace("/", "_")
|
||||
try:
|
||||
sig_ref.grid = latlong_to_locator(ref["y"], ref["x"], 6)
|
||||
except:
|
||||
logging.debug("Invalid lat/lon received for reference")
|
||||
sig_ref.latitude = ref["y"]
|
||||
sig_ref.longitude = ref["x"]
|
||||
elif not response.from_cache:
|
||||
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
|
||||
elif not response.from_cache:
|
||||
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
|
||||
lookup_data = DATA_STORE.sigrefs["ZLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["ZLOTA"] else None
|
||||
if lookup_data:
|
||||
# Copy new sig ref data into existing object
|
||||
sig_ref.__dict__.update(lookup_data.__dict__)
|
||||
else:
|
||||
logging.warning("ZLOTA database did not contain data for ref %s", ref_id)
|
||||
|
||||
elif sig.upper() == "BOTA":
|
||||
if not sig_ref.name:
|
||||
@@ -219,27 +157,29 @@ def populate_sig_ref_info(sig_ref):
|
||||
sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-")
|
||||
|
||||
elif sig.upper() == "LLOTA":
|
||||
response = URL_DATA_CACHE.get("https://llota.app/api/public/references",
|
||||
headers=HTTP_HEADERS)
|
||||
if response.ok:
|
||||
data = response.json()
|
||||
if isinstance(data, list):
|
||||
if not bool(DATA_STORE.sigrefs_llota) or not response.from_cache:
|
||||
# New data from LLOTA, update our internal map
|
||||
for ref in data:
|
||||
DATA_STORE.sigrefs_llota.add(ref["reference_code"], ref)
|
||||
ref = DATA_STORE.sigrefs_llota.get(ref_id)
|
||||
if ref:
|
||||
sig_ref.name = str(ref["name"])
|
||||
sig_ref.url = "https://llota.app/list/ref/" + ref_id
|
||||
sig_ref.grid = str(ref["grid_locator"])
|
||||
ll = locator_to_latlong(sig_ref.grid)
|
||||
sig_ref.latitude = ll[0]
|
||||
sig_ref.longitude = ll[1]
|
||||
elif not response.from_cache:
|
||||
logging.warning("Malformed response looking up %s ref %s", sig, ref_id)
|
||||
elif not response.from_cache:
|
||||
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
|
||||
lookup_data = DATA_STORE.sigrefs["LLOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["LLOTA"] else None
|
||||
if lookup_data:
|
||||
# Copy new sig ref data into existing object
|
||||
sig_ref.__dict__.update(lookup_data.__dict__)
|
||||
else:
|
||||
logging.warning("LLOTA database did not contain data for ref %s", ref_id)
|
||||
|
||||
elif sig.upper() == "DME":
|
||||
# Zero-pad to 5 digits to match our source data
|
||||
lookup_data = DATA_STORE.sigrefs["DME"][ref_id.zfill(5)] if ref_id.zfill(5) in DATA_STORE.sigrefs["DME"] else None
|
||||
if lookup_data:
|
||||
# Copy new sig ref data into existing object
|
||||
sig_ref.__dict__.update(lookup_data.__dict__)
|
||||
else:
|
||||
logging.warning("DME database did not contain data for ref %s", ref_id)
|
||||
|
||||
elif sig.upper() == "TOTA":
|
||||
lookup_data = DATA_STORE.sigrefs["TOTA"][ref_id] if ref_id in DATA_STORE.sigrefs["TOTA"] else None
|
||||
if lookup_data:
|
||||
# Copy new sig ref data into existing object
|
||||
sig_ref.__dict__.update(lookup_data.__dict__)
|
||||
else:
|
||||
logging.warning("TOTA database did not contain data for ref %s", ref_id)
|
||||
|
||||
elif sig.upper() == "WWTOTA":
|
||||
if not sig_ref.name:
|
||||
@@ -268,23 +208,6 @@ def populate_sig_ref_info(sig_ref):
|
||||
except:
|
||||
logging.warning("Invalid lat/lon received for WAB/WAI reference")
|
||||
|
||||
elif sig.upper() == "DME":
|
||||
# Zero-pad to 5 digits to match our source data
|
||||
row = DATA_STORE.sigrefs_dme.get(ref_id.zfill(5))
|
||||
if row:
|
||||
sig_ref.name = row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"]
|
||||
sig_ref.latitude = float(row["LATITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
|
||||
"LATITUD_ETRS89_REGCAN95") else None
|
||||
sig_ref.longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
|
||||
"LONGITUD_ETRS89_REGCAN95") else None
|
||||
if sig_ref.latitude and sig_ref.longitude:
|
||||
try:
|
||||
sig_ref.grid = latlong_to_locator(sig_ref.latitude, sig_ref.longitude, 6)
|
||||
except Exception:
|
||||
logging.warning("Invalid lat/lon received for DME reference")
|
||||
else:
|
||||
logging.warning("DME database did not contain data for ref %s", ref_id)
|
||||
|
||||
except ConnectionError:
|
||||
logging.warning("Connection error when looking up sig_ref info for " + sig + " ref " + ref_id)
|
||||
except (ConnectTimeout, ReadTimeout):
|
||||
|
||||
Reference in New Issue
Block a user