w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-08-05 18:11:41 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Refactor of caching & data storage part 5 #118

Browse Source
This commit is contained in:
Ian Renton
2026-07-31 22:35:52 +01:00
parent 55a265eb1a
commit fe10943ecc
22 changed files with 69 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/data_store.py
+4 -4
View File
@@ -43,11 +43,11 @@ class DataStore:
self.status_data = self._status.get("status_data")
# Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
# but there's no need for a TTL since old data is better than no data. This is a two-layer dict, keys are SIG
# name and then reference ID, with the final value being a SIGRef object.
# but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
# and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
# so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
self.sigrefs = diskcache.Cache(self._CACHE_DIR + "/sigrefs")
for k in list(self.sigrefs.iterkeys()):
logging.info(f"Loaded data for %d references in %s SIG.", len(self.sigrefs[k]), k)
logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
core/sig_utils.py
+3 -33
View File
@@ -1,12 +1,10 @@
import logging
from pyhamtools.locator import latlong_to_locator, locator_to_latlong
from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
from core.constants import SIGS, HTTP_HEADERS
from core.constants import SIGS
from core.data_store import DATA_STORE
from core.geo_utils import wab_wai_square_to_lat_lon
from core.url_data_cache import URL_DATA_CACHE
def get_ref_regex_for_sig(sig):
@@ -85,42 +83,14 @@ def populate_sig_ref_info(sig_ref):
# OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
# this SIG. If so, check for the reference data and use that.
elif sig in DATA_STORE.sigrefs:
lookup_data = DATA_STORE.sigrefs[sig][ref_id] if ref_id in DATA_STORE.sigrefs[sig] else None
key = sig + ":" + ref_id
lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
if lookup_data:
# Copy new sig ref data into existing object
sig_ref.__dict__.update(lookup_data.__dict__)
else:
logging.warning("%s database did not contain data for ref %s", sig, ref_id)
elif False:
# TODO remove
# OK, this is not a SIG we have stored data for. Maybe it's of a type we can query information for live.
if sig.upper() == "IOTA":
response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
headers=HTTP_HEADERS)
if response.ok:
data = response.json()
if data:
sig_ref.name = data["name"] if "name" in data else None
sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
sig_ref.grid = data["locator"] if "locator" in data else None
# For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
# the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
# and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
# just use the single point we have.
if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
"lat2") is not None and data.get("lng2") is not None:
sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
else:
sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
elif not response.from_cache:
logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
elif not response.from_cache:
logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
else:
logging.warning(f"Tried to look up a SIG called %s but Spothole does not know what that is.", sig)
sigrefdataproviders/arlhs.py
+4 -4
View File
@@ -15,16 +15,16 @@ class ARLHS(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
if "ARLHS" in row:
if "ARLHS" in row and row["ARLHS"] != "":
ref_id = row["ARLHS"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"])
grid=row["Maidenhead Locator"]))
return new_data
sigrefdataproviders/dme.py
+4 -3
View File
@@ -16,7 +16,7 @@ class DME(LocalFileSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.PATH)
def _file_to_data(self, path):
new_data = {}
new_data = []
with open(path, encoding="latin-1") as _f:
for row in csv.DictReader(_f, delimiter=";"):
ref_id = row["COD_INE"][:5]
@@ -25,10 +25,11 @@ class DME(LocalFileSIGRefDataProvider):
longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
"LONGITUD_ETRS89_REGCAN95") else None
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id,
ref = SIGRef(sig=self.SIG, id=ref_id,
name=row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"],
latitude=latitude,
longitude=longitude)
if latitude and longitude:
new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
return new_data
sigrefdataproviders/file_download_sig_ref_data_provider.py
+1 -1
View File
@@ -51,7 +51,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
new_data = self._http_response_to_data(http_response)
# Submit the new spots for processing. There might not be any spots for the less popular programs.
if new_data:
self._replace_data(new_data)
self._add_data(new_data)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
sigrefdataproviders/gma.py
+3 -3
View File
@@ -15,15 +15,15 @@ class GMA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"])
grid=row["Maidenhead Locator"]))
return new_data
sigrefdataproviders/illw.py
+4 -4
View File
@@ -15,16 +15,16 @@ class ILLW(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
if "ILLW" in row:
if "ILLW" in row and row["ILLW"] != "":
ref_id = row["ILLW"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"])
grid=row["Maidenhead Locator"]))
return new_data
sigrefdataproviders/iota.py
+3 -3
View File
@@ -17,7 +17,7 @@ class IOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
data = http_response.json()
if isinstance(data, list):
for ref in data:
@@ -30,7 +30,7 @@ class IOTA(FileDownloadSIGRefDataProvider):
except ValueError:
logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude)
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
grid=grid, latitude=latitude, longitude=longitude)
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
grid=grid, latitude=latitude, longitude=longitude))
return new_data
sigrefdataproviders/llota.py
+3 -3
View File
@@ -15,7 +15,7 @@ class LLOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
data = http_response.json()
if isinstance(data, list):
for ref in data:
@@ -23,10 +23,10 @@ class LLOTA(FileDownloadSIGRefDataProvider):
grid = str(ref["grid_locator"])
ll = locator_to_latlong(grid)
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
url="https://llota.app/list/ref/" + ref_id,
grid=grid,
latitude=ll[0],
longitude=ll[1])
longitude=ll[1]))
return new_data
sigrefdataproviders/local_file_sig_ref_data_provider.py
+1 -1
View File
@@ -18,7 +18,7 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
try:
new_data = self._file_to_data(self._path)
if new_data:
self._replace_data(new_data)
self._add_data(new_data)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
else:
sigrefdataproviders/mota.py
+3 -3
View File
@@ -15,15 +15,15 @@ class MOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
latitude=float(row["Latitude"]) if "Latitude" in row and row[
"Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row[
"Longitude"] != "" else None,
grid=row["Maidenhead Locator"])
grid=row["Maidenhead Locator"]))
return new_data
sigrefdataproviders/pota.py
+3 -3
View File
@@ -15,15 +15,15 @@ class POTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
url="https://pota.app/#/park/" + ref_id,
grid=row["grid"] if "grid" in row else None,
latitude=float(row["latitude"]) if "latitude" in row and row[
"latitude"] != "" else None,
longitude=float(row["longitude"]) if "longitude" in row and row[
"longitude"] != "" else None)
"longitude"] != "" else None))
return new_data
sigrefdataproviders/sig_ref_data_provider.py
+4 -8
View File
@@ -19,10 +19,6 @@ class SIGRefDataProvider:
self.status = "Not Started" if self.enabled else "Disabled"
self.reference_count = 0
# Create an empty dict to store data if one doesn't already exist
if not sig_name in DATA_STORE.sigrefs:
DATA_STORE.sigrefs[sig_name] = {}
def start(self):
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -35,10 +31,10 @@ class SIGRefDataProvider:
raise NotImplementedError("Subclasses must implement this method")
def _replace_data(self, new_data):
"""Replace all data for the named sig with the new data. new_data should be a map of reference ID to SIGRef
objects."""
def _add_data(self, new_data):
"""Add all the provided reference data objects to the data store."""
DATA_STORE.sigrefs[self.sig_name] = new_data
for d in new_data:
DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d
self.reference_count = len(new_data)
logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)
sigrefdataproviders/siota.py
+3 -3
View File
@@ -15,12 +15,12 @@ class SIOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["SILO_CODE"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
grid=row["LOCATOR"] if "LOCATOR" in row else None,
latitude=float(row["LAT"]) if "LAT" in row else None,
longitude=float(row["LNG"]) if "LNG" in row else None)
longitude=float(row["LNG"]) if "LNG" in row else None))
return new_data
sigrefdataproviders/sota.py
+4 -3
View File
@@ -17,17 +17,18 @@ class SOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["SummitCode"]
latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
ref = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
url="https://www.sotadata.org.uk/en/summit/" + ref_id,
latitude=latitude,
longitude=longitude,
activation_score=int(row["Points"]) if "Points" in row else None)
if latitude and longitude:
new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
return new_data
sigrefdataproviders/toilets.py
+3 -3
View File
@@ -14,11 +14,11 @@ class Toilets(LocalFileSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.PATH)
def _file_to_data(self, path):
new_data = {}
new_data = []
f = open(path)
csv_data = f.read()
dr = csv.DictReader(csv_data.splitlines())
for row in dr:
new_data[row["ref"]] = SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]),
longitude=float(row["lon"]))
new_data.append(SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]),
longitude=float(row["lon"])))
return new_data
sigrefdataproviders/towers.py
+3 -3
View File
@@ -15,13 +15,13 @@ class Towers(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
ref_id = row["Ref"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
url="https://wwtota.com/seznam/karta_rozhledny.php?ref=" + ref_id,
grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None)
longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None))
return new_data
sigrefdataproviders/wca.py
+3 -3
View File
@@ -18,7 +18,7 @@ class WCA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["REF"]
@@ -35,10 +35,10 @@ class WCA(FileDownloadSIGRefDataProvider):
except ValueError:
logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id)
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
latitude=latitude,
longitude=longitude,
grid=grid)
grid=grid))
return new_data
sigrefdataproviders/wota.py
+3 -3
View File
@@ -13,7 +13,7 @@ class WOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for feature in http_response.json().get("features", []):
ref_id = feature["properties"]["wotaId"]
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
@@ -23,9 +23,9 @@ class WOTA(FileDownloadSIGRefDataProvider):
number = int(ref_id.upper().replace("LDO-", ""))
url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
grid=feature["properties"]["qthLocator"],
latitude=feature["geometry"]["coordinates"][1],
longitude=feature["geometry"]["coordinates"][0])
longitude=feature["geometry"]["coordinates"][0]))
return new_data
sigrefdataproviders/wwbota.py
+3 -3
View File
@@ -15,13 +15,13 @@ class WWBOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["Reference"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
url="https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None,
grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None)
longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None))
return new_data
sigrefdataproviders/wwff.py
+3 -3
View File
@@ -15,16 +15,16 @@ class WWFF(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
url="https://wwff.co/directory/?showRef=" + ref_id,
grid=row["iaruLocator"] if "iaruLocator" in row and row[
"iaruLocator"] != "-" else None,
latitude=float(row["latitude"]) if "latitude" in row and row[
"latitude"] != "" and row["latitude"] != "-" else None,
longitude=float(row["longitude"]) if "longitude" in row and row[
"longitude"] != "" and row["longitude"] != "-" else None)
"longitude"] != "" and row["longitude"] != "-" else None))
return new_data
sigrefdataproviders/zlota.py
+4 -3
View File
@@ -15,7 +15,7 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = {}
new_data = []
data = http_response.json()
if isinstance(data, list):
for ref in data:
@@ -23,11 +23,12 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
latitude = ref["y"]
longitude = ref["x"]
new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
ref = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
url="https://ontheair.nz/assets/" + ref_id.replace("/", "_"),
latitude=latitude,
longitude=longitude)
if latitude and longitude:
new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
ref.grid = latlong_to_locator(latitude, longitude, 6)
new_data.append(ref)
return new_data