diff --git a/core/data_store.py b/core/data_store.py index c5b3ec7..33c900a 100644 --- a/core/data_store.py +++ b/core/data_store.py @@ -43,11 +43,11 @@ class DataStore: self.status_data = self._status.get("status_data") # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis - # but there's no need for a TTL since old data is better than no data. This is a two-layer dict, keys are SIG - # name and then reference ID, with the final value being a SIGRef object. + # but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference, + # and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts, + # so we have an ugly "SIG:ref" syntax for keys to keep it a single level. self.sigrefs = diskcache.Cache(self._CACHE_DIR + "/sigrefs") - for k in list(self.sigrefs.iterkeys()): - logging.info(f"Loaded data for %d references in %s SIG.", len(self.sigrefs[k]), k) + logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs)) # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup. # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign diff --git a/core/sig_utils.py b/core/sig_utils.py index 68c2a34..486f8e0 100644 --- a/core/sig_utils.py +++ b/core/sig_utils.py @@ -1,12 +1,10 @@ import logging from pyhamtools.locator import latlong_to_locator, locator_to_latlong -from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout -from core.constants import SIGS, HTTP_HEADERS +from core.constants import SIGS from core.data_store import DATA_STORE from core.geo_utils import wab_wai_square_to_lat_lon -from core.url_data_cache import URL_DATA_CACHE def get_ref_regex_for_sig(sig): @@ -85,42 +83,14 @@ def populate_sig_ref_info(sig_ref): # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for # this SIG. If so, check for the reference data and use that. elif sig in DATA_STORE.sigrefs: - lookup_data = DATA_STORE.sigrefs[sig][ref_id] if ref_id in DATA_STORE.sigrefs[sig] else None + key = sig + ":" + ref_id + lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None if lookup_data: # Copy new sig ref data into existing object sig_ref.__dict__.update(lookup_data.__dict__) else: logging.warning("%s database did not contain data for ref %s", sig, ref_id) - elif False: - # TODO remove - # OK, this is not a SIG we have stored data for. Maybe it's of a type we can query information for live. - if sig.upper() == "IOTA": - response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id, - headers=HTTP_HEADERS) - if response.ok: - data = response.json() - if data: - sig_ref.name = data["name"] if "name" in data else None - sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id - sig_ref.grid = data["locator"] if "locator" in data else None - - # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are - # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this - # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters, - # just use the single point we have. - if data.get("latitude") is not None and data.get("longitude") is not None and data.get( - "lat2") is not None and data.get("lng2") is not None: - sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0 - sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0 - else: - sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None - sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None - elif not response.from_cache: - logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id) - elif not response.from_cache: - logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id) - else: logging.warning(f"Tried to look up a SIG called %s but Spothole does not know what that is.", sig) diff --git a/sigrefdataproviders/arlhs.py b/sigrefdataproviders/arlhs.py index 20d8a2a..2b23b88 100644 --- a/sigrefdataproviders/arlhs.py +++ b/sigrefdataproviders/arlhs.py @@ -15,16 +15,16 @@ class ARLHS(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): - if "ARLHS" in row: + if "ARLHS" in row and row["ARLHS"] != "": ref_id = row["ARLHS"] - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, url="https://www.cqgma.org/zinfo.php?ref=" + ref_id, latitude=float(row["Latitude"]) if "Latitude" in row and row[ "Latitude"] != "" else None, longitude=float(row["Longitude"]) if "Longitude" in row and row[ "Longitude"] != "" else None, - grid=row["Maidenhead Locator"]) + grid=row["Maidenhead Locator"])) return new_data diff --git a/sigrefdataproviders/dme.py b/sigrefdataproviders/dme.py index b2a8e47..492fc29 100644 --- a/sigrefdataproviders/dme.py +++ b/sigrefdataproviders/dme.py @@ -16,7 +16,7 @@ class DME(LocalFileSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.PATH) def _file_to_data(self, path): - new_data = {} + new_data = [] with open(path, encoding="latin-1") as _f: for row in csv.DictReader(_f, delimiter=";"): ref_id = row["COD_INE"][:5] @@ -25,10 +25,11 @@ class DME(LocalFileSIGRefDataProvider): longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get( "LONGITUD_ETRS89_REGCAN95") else None - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, + ref = SIGRef(sig=self.SIG, id=ref_id, name=row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"], latitude=latitude, longitude=longitude) if latitude and longitude: - new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6) + ref.grid = latlong_to_locator(latitude, longitude, 6) + new_data.append(ref) return new_data diff --git a/sigrefdataproviders/file_download_sig_ref_data_provider.py b/sigrefdataproviders/file_download_sig_ref_data_provider.py index 364b564..8920fe3 100644 --- a/sigrefdataproviders/file_download_sig_ref_data_provider.py +++ b/sigrefdataproviders/file_download_sig_ref_data_provider.py @@ -51,7 +51,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider): new_data = self._http_response_to_data(http_response) # Submit the new spots for processing. There might not be any spots for the less popular programs. if new_data: - self._replace_data(new_data) + self._add_data(new_data) self.status = "OK" self.last_update_time = datetime.now(pytz.UTC) diff --git a/sigrefdataproviders/gma.py b/sigrefdataproviders/gma.py index ae58549..f8753b6 100644 --- a/sigrefdataproviders/gma.py +++ b/sigrefdataproviders/gma.py @@ -15,15 +15,15 @@ class GMA(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): ref_id = row["Reference"] - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, url="https://www.cqgma.org/zinfo.php?ref=" + ref_id, latitude=float(row["Latitude"]) if "Latitude" in row and row[ "Latitude"] != "" else None, longitude=float(row["Longitude"]) if "Longitude" in row and row[ "Longitude"] != "" else None, - grid=row["Maidenhead Locator"]) + grid=row["Maidenhead Locator"])) return new_data diff --git a/sigrefdataproviders/illw.py b/sigrefdataproviders/illw.py index af17391..9e2f7b8 100644 --- a/sigrefdataproviders/illw.py +++ b/sigrefdataproviders/illw.py @@ -15,16 +15,16 @@ class ILLW(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): - if "ILLW" in row: + if "ILLW" in row and row["ILLW"] != "": ref_id = row["ILLW"] - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, url="https://www.cqgma.org/zinfo.php?ref=" + ref_id, latitude=float(row["Latitude"]) if "Latitude" in row and row[ "Latitude"] != "" else None, longitude=float(row["Longitude"]) if "Longitude" in row and row[ "Longitude"] != "" else None, - grid=row["Maidenhead Locator"]) + grid=row["Maidenhead Locator"])) return new_data diff --git a/sigrefdataproviders/iota.py b/sigrefdataproviders/iota.py index 3213a24..1925967 100644 --- a/sigrefdataproviders/iota.py +++ b/sigrefdataproviders/iota.py @@ -17,7 +17,7 @@ class IOTA(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] data = http_response.json() if isinstance(data, list): for ref in data: @@ -30,7 +30,7 @@ class IOTA(FileDownloadSIGRefDataProvider): except ValueError: logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude) - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"], - grid=grid, latitude=latitude, longitude=longitude) + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref["name"], + grid=grid, latitude=latitude, longitude=longitude)) return new_data diff --git a/sigrefdataproviders/llota.py b/sigrefdataproviders/llota.py index d0513d4..6faa48f 100644 --- a/sigrefdataproviders/llota.py +++ b/sigrefdataproviders/llota.py @@ -15,7 +15,7 @@ class LLOTA(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] data = http_response.json() if isinstance(data, list): for ref in data: @@ -23,10 +23,10 @@ class LLOTA(FileDownloadSIGRefDataProvider): grid = str(ref["grid_locator"]) ll = locator_to_latlong(grid) - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]), + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]), url="https://llota.app/list/ref/" + ref_id, grid=grid, latitude=ll[0], - longitude=ll[1]) + longitude=ll[1])) return new_data diff --git a/sigrefdataproviders/local_file_sig_ref_data_provider.py b/sigrefdataproviders/local_file_sig_ref_data_provider.py index 780a0a6..05a3152 100644 --- a/sigrefdataproviders/local_file_sig_ref_data_provider.py +++ b/sigrefdataproviders/local_file_sig_ref_data_provider.py @@ -18,7 +18,7 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider): try: new_data = self._file_to_data(self._path) if new_data: - self._replace_data(new_data) + self._add_data(new_data) self.status = "OK" self.last_update_time = datetime.now(pytz.UTC) else: diff --git a/sigrefdataproviders/mota.py b/sigrefdataproviders/mota.py index f5579dc..1fac35a 100644 --- a/sigrefdataproviders/mota.py +++ b/sigrefdataproviders/mota.py @@ -15,15 +15,15 @@ class MOTA(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): ref_id = row["Reference"] - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, url="https://www.cqgma.org/zinfo.php?ref=" + ref_id, latitude=float(row["Latitude"]) if "Latitude" in row and row[ "Latitude"] != "" else None, longitude=float(row["Longitude"]) if "Longitude" in row and row[ "Longitude"] != "" else None, - grid=row["Maidenhead Locator"]) + grid=row["Maidenhead Locator"])) return new_data diff --git a/sigrefdataproviders/pota.py b/sigrefdataproviders/pota.py index 5a4e87c..a0a549f 100644 --- a/sigrefdataproviders/pota.py +++ b/sigrefdataproviders/pota.py @@ -15,15 +15,15 @@ class POTA(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): ref_id = row["reference"] - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None, + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None, url="https://pota.app/#/park/" + ref_id, grid=row["grid"] if "grid" in row else None, latitude=float(row["latitude"]) if "latitude" in row and row[ "latitude"] != "" else None, longitude=float(row["longitude"]) if "longitude" in row and row[ - "longitude"] != "" else None) + "longitude"] != "" else None)) return new_data diff --git a/sigrefdataproviders/sig_ref_data_provider.py b/sigrefdataproviders/sig_ref_data_provider.py index 9664731..efe1977 100644 --- a/sigrefdataproviders/sig_ref_data_provider.py +++ b/sigrefdataproviders/sig_ref_data_provider.py @@ -19,10 +19,6 @@ class SIGRefDataProvider: self.status = "Not Started" if self.enabled else "Disabled" self.reference_count = 0 - # Create an empty dict to store data if one doesn't already exist - if not sig_name in DATA_STORE.sigrefs: - DATA_STORE.sigrefs[sig_name] = {} - def start(self): """Start the provider. This should return immediately after spawning threads to access the remote resources""" @@ -35,10 +31,10 @@ class SIGRefDataProvider: raise NotImplementedError("Subclasses must implement this method") - def _replace_data(self, new_data): - """Replace all data for the named sig with the new data. new_data should be a map of reference ID to SIGRef - objects.""" + def _add_data(self, new_data): + """Add all the provided reference data objects to the data store.""" - DATA_STORE.sigrefs[self.sig_name] = new_data + for d in new_data: + DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d self.reference_count = len(new_data) logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name) diff --git a/sigrefdataproviders/siota.py b/sigrefdataproviders/siota.py index 9b20593..f763cfd 100644 --- a/sigrefdataproviders/siota.py +++ b/sigrefdataproviders/siota.py @@ -15,12 +15,12 @@ class SIOTA(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): ref_id = row["SILO_CODE"] - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None, + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None, grid=row["LOCATOR"] if "LOCATOR" in row else None, latitude=float(row["LAT"]) if "LAT" in row else None, - longitude=float(row["LNG"]) if "LNG" in row else None) + longitude=float(row["LNG"]) if "LNG" in row else None)) return new_data diff --git a/sigrefdataproviders/sota.py b/sigrefdataproviders/sota.py index 2f5bdd5..fbfbb36 100644 --- a/sigrefdataproviders/sota.py +++ b/sigrefdataproviders/sota.py @@ -17,17 +17,18 @@ class SOTA(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]): ref_id = row["SummitCode"] latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None, + ref = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None, url="https://www.sotadata.org.uk/en/summit/" + ref_id, latitude=latitude, longitude=longitude, activation_score=int(row["Points"]) if "Points" in row else None) if latitude and longitude: - new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6) + ref.grid = latlong_to_locator(latitude, longitude, 6) + new_data.append(ref) return new_data diff --git a/sigrefdataproviders/toilets.py b/sigrefdataproviders/toilets.py index 7bffab3..defdc37 100644 --- a/sigrefdataproviders/toilets.py +++ b/sigrefdataproviders/toilets.py @@ -14,11 +14,11 @@ class Toilets(LocalFileSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.PATH) def _file_to_data(self, path): - new_data = {} + new_data = [] f = open(path) csv_data = f.read() dr = csv.DictReader(csv_data.splitlines()) for row in dr: - new_data[row["ref"]] = SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]), - longitude=float(row["lon"])) + new_data.append(SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]), + longitude=float(row["lon"]))) return new_data diff --git a/sigrefdataproviders/towers.py b/sigrefdataproviders/towers.py index bf122f4..80e3be7 100644 --- a/sigrefdataproviders/towers.py +++ b/sigrefdataproviders/towers.py @@ -15,13 +15,13 @@ class Towers(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"): ref_id = row["Ref"] - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None, + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None, url="https://wwtota.com/seznam/karta_rozhledny.php?ref=" + ref_id, grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None, latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None, - longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None) + longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None)) return new_data diff --git a/sigrefdataproviders/wca.py b/sigrefdataproviders/wca.py index 07a85ff..bb874cf 100644 --- a/sigrefdataproviders/wca.py +++ b/sigrefdataproviders/wca.py @@ -18,7 +18,7 @@ class WCA(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): ref_id = row["REF"] @@ -35,10 +35,10 @@ class WCA(FileDownloadSIGRefDataProvider): except ValueError: logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id) - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None, + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None, url="https://www.cqgma.org/zinfo.php?ref=" + ref_id, latitude=latitude, longitude=longitude, - grid=grid) + grid=grid)) return new_data diff --git a/sigrefdataproviders/wota.py b/sigrefdataproviders/wota.py index c221845..1e8c1f3 100644 --- a/sigrefdataproviders/wota.py +++ b/sigrefdataproviders/wota.py @@ -13,7 +13,7 @@ class WOTA(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for feature in http_response.json().get("features", []): ref_id = feature["properties"]["wotaId"] # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be @@ -23,9 +23,9 @@ class WOTA(FileDownloadSIGRefDataProvider): number = int(ref_id.upper().replace("LDO-", "")) url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214) - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url, + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url, grid=feature["properties"]["qthLocator"], latitude=feature["geometry"]["coordinates"][1], - longitude=feature["geometry"]["coordinates"][0]) + longitude=feature["geometry"]["coordinates"][0])) return new_data diff --git a/sigrefdataproviders/wwbota.py b/sigrefdataproviders/wwbota.py index 52dd296..11f77d9 100644 --- a/sigrefdataproviders/wwbota.py +++ b/sigrefdataproviders/wwbota.py @@ -15,13 +15,13 @@ class WWBOTA(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): ref_id = row["Reference"] - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None, url="https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None, grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None, latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None, - longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None) + longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None)) return new_data diff --git a/sigrefdataproviders/wwff.py b/sigrefdataproviders/wwff.py index bb8f033..c840f23 100644 --- a/sigrefdataproviders/wwff.py +++ b/sigrefdataproviders/wwff.py @@ -15,16 +15,16 @@ class WWFF(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()): ref_id = row["reference"] - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None, + new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None, url="https://wwff.co/directory/?showRef=" + ref_id, grid=row["iaruLocator"] if "iaruLocator" in row and row[ "iaruLocator"] != "-" else None, latitude=float(row["latitude"]) if "latitude" in row and row[ "latitude"] != "" and row["latitude"] != "-" else None, longitude=float(row["longitude"]) if "longitude" in row and row[ - "longitude"] != "" and row["longitude"] != "-" else None) + "longitude"] != "" and row["longitude"] != "-" else None)) return new_data diff --git a/sigrefdataproviders/zlota.py b/sigrefdataproviders/zlota.py index 6323f4d..876e48d 100644 --- a/sigrefdataproviders/zlota.py +++ b/sigrefdataproviders/zlota.py @@ -15,7 +15,7 @@ class ZLOTA(FileDownloadSIGRefDataProvider): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): - new_data = {} + new_data = [] data = http_response.json() if isinstance(data, list): for ref in data: @@ -23,11 +23,12 @@ class ZLOTA(FileDownloadSIGRefDataProvider): latitude = ref["y"] longitude = ref["x"] - new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"], + ref = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"], url="https://ontheair.nz/assets/" + ref_id.replace("/", "_"), latitude=latitude, longitude=longitude) if latitude and longitude: - new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6) + ref.grid = latlong_to_locator(latitude, longitude, 6) + new_data.append(ref) return new_data