Refactor of caching & data storage part 5 #118

core/data_store.py

@@ -43,11 +43,11 @@ class DataStore:
         self.status_data = self._status.get("status_data")
 
         # Standard disk cache for SIG ref data. Separate provider threads will repopulate theis on a regular basis
-        # but there's no need for a TTL since old data is better than no data. This is a two-layer dict, keys are SIG
+        # but there's no need for a TTL since old data is better than no data. We need to key on both SIG and reference,
-        # name and then reference ID, with the final value being a SIGRef object.
+        # and trying to do two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts,
+        # so we have an ugly "SIG:ref" syntax for keys to keep it a single level.
         self.sigrefs = diskcache.Cache(self._CACHE_DIR + "/sigrefs")
-        for k in list(self.sigrefs.iterkeys()):
+        logging.info(f"Loaded data for %d SIG references.", len(self.sigrefs))
-            logging.info(f"Loaded data for %d references in %s SIG.", len(self.sigrefs[k]), k)
 
         # Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
         # Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign

core/sig_utils.py

@@ -1,12 +1,10 @@
 import logging
 
 from pyhamtools.locator import latlong_to_locator, locator_to_latlong
-from requests.exceptions import ConnectionError, ReadTimeout, ConnectTimeout
 
-from core.constants import SIGS, HTTP_HEADERS
+from core.constants import SIGS
 from core.data_store import DATA_STORE
 from core.geo_utils import wab_wai_square_to_lat_lon
-from core.url_data_cache import URL_DATA_CACHE
 
 
 def get_ref_regex_for_sig(sig):
@@ -85,42 +83,14 @@ def populate_sig_ref_info(sig_ref):
         # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for
         # this SIG. If so, check for the reference data and use that.
         elif sig in DATA_STORE.sigrefs:
-            lookup_data = DATA_STORE.sigrefs[sig][ref_id] if ref_id in DATA_STORE.sigrefs[sig] else None
+            key = sig + ":" + ref_id
+            lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
             if lookup_data:
                 # Copy new sig ref data into existing object
                 sig_ref.__dict__.update(lookup_data.__dict__)
             else:
                 logging.warning("%s database did not contain data for ref %s", sig, ref_id)
 
-        elif False:
-            # TODO remove
-            # OK, this is not a SIG we have stored data for. Maybe it's of a type we can query information for live.
-            if sig.upper() == "IOTA":
-                response = URL_DATA_CACHE.get("https://www.cqgma.org/api/ref/?" + ref_id,
-                                                          headers=HTTP_HEADERS)
-                if response.ok:
-                    data = response.json()
-                    if data:
-                        sig_ref.name = data["name"] if "name" in data else None
-                        sig_ref.url = "https://www.cqgma.org/zinfo.php?ref=" + ref_id
-                        sig_ref.grid = data["locator"] if "locator" in data else None
-
-                        # For some things (just IOTA?) the GMA actually returns a box where "latitude" and "longitude" are
-                        # the zeroest corner of the box, then "lat2" and "lng2" provide the other corner. We detect this
-                        # and provide a single lat/lon for the centre. Otherwise if we don't have these extra parameters,
-                        # just use the single point we have.
-                        if data.get("latitude") is not None and data.get("longitude") is not None and data.get(
-                                "lat2") is not None and data.get("lng2") is not None:
-                            sig_ref.latitude = (float(data["latitude"]) + float(data["lat2"])) / 2.0
-                            sig_ref.longitude = (float(data["longitude"]) + float(data["lng2"])) / 2.0
-                        else:
-                            sig_ref.latitude = float(data["latitude"]) if data.get("latitude") is not None else None
-                            sig_ref.longitude = float(data["longitude"]) if data.get("longitude") is not None else None
-                    elif not response.from_cache:
-                        logging.warning("Malformed response looking up %s ref %s via GMA", sig, ref_id)
-                elif not response.from_cache:
-                    logging.warning("HTTP %d looking up %s ref %s", response.status_code, sig, ref_id)
-
         else:
             logging.warning(f"Tried to look up a SIG called %s but Spothole does not know what that is.", sig)
 

sigrefdataproviders/arlhs.py

@@ -15,16 +15,16 @@ class ARLHS(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
-            if "ARLHS" in row:
+            if "ARLHS" in row and row["ARLHS"] != "":
                 ref_id = row["ARLHS"]
-                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
                                           url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
                                           latitude=float(row["Latitude"]) if "Latitude" in row and row[
                                               "Latitude"] != "" else None,
                                           longitude=float(row["Longitude"]) if "Longitude" in row and row[
                                               "Longitude"] != "" else None,
-                                          grid=row["Maidenhead Locator"])
+                                          grid=row["Maidenhead Locator"]))
 
         return new_data

sigrefdataproviders/dme.py

@@ -16,7 +16,7 @@ class DME(LocalFileSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.PATH)
 
     def _file_to_data(self, path):
-        new_data = {}
+        new_data = []
         with open(path, encoding="latin-1") as _f:
             for row in csv.DictReader(_f, delimiter=";"):
                 ref_id = row["COD_INE"][:5]
@@ -25,10 +25,11 @@ class DME(LocalFileSIGRefDataProvider):
                 longitude = float(row["LONGITUD_ETRS89_REGCAN95"].replace(",", ".")) if row.get(
                     "LONGITUD_ETRS89_REGCAN95") else None
 
-                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id,
+                ref = SIGRef(sig=self.SIG, id=ref_id,
                                           name=row["NOMBRE_ACTUAL"] + ", " + row["PROVINCIA"],
                                           latitude=latitude,
                                           longitude=longitude)
                 if latitude and longitude:
-                    new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
+                    ref.grid = latlong_to_locator(latitude, longitude, 6)
+                new_data.append(ref)
         return new_data

sigrefdataproviders/file_download_sig_ref_data_provider.py

@@ -51,7 +51,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
                 new_data = self._http_response_to_data(http_response)
                 # Submit the new spots for processing. There might not be any spots for the less popular programs.
                 if new_data:
-                    self._replace_data(new_data)
+                    self._add_data(new_data)
 
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)

sigrefdataproviders/gma.py

@@ -15,15 +15,15 @@ class GMA(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
             ref_id = row["Reference"]
-            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
                                       url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
                                       latitude=float(row["Latitude"]) if "Latitude" in row and row[
                                           "Latitude"] != "" else None,
                                       longitude=float(row["Longitude"]) if "Longitude" in row and row[
                                           "Longitude"] != "" else None,
-                                      grid=row["Maidenhead Locator"])
+                                      grid=row["Maidenhead Locator"]))
 
         return new_data

sigrefdataproviders/illw.py

@@ -15,16 +15,16 @@ class ILLW(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
-            if "ILLW" in row:
+            if "ILLW" in row and row["ILLW"] != "":
                 ref_id = row["ILLW"]
-                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
                                           url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
                                           latitude=float(row["Latitude"]) if "Latitude" in row and row[
                                               "Latitude"] != "" else None,
                                           longitude=float(row["Longitude"]) if "Longitude" in row and row[
                                               "Longitude"] != "" else None,
-                                          grid=row["Maidenhead Locator"])
+                                          grid=row["Maidenhead Locator"]))
 
         return new_data

sigrefdataproviders/iota.py

@@ -17,7 +17,7 @@ class IOTA(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         data = http_response.json()
         if isinstance(data, list):
             for ref in data:
@@ -30,7 +30,7 @@ class IOTA(FileDownloadSIGRefDataProvider):
                 except ValueError:
                     logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude)
 
-                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
-                                          grid=grid, latitude=latitude, longitude=longitude)
+                                          grid=grid, latitude=latitude, longitude=longitude))
 
         return new_data

sigrefdataproviders/llota.py

@@ -15,7 +15,7 @@ class LLOTA(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         data = http_response.json()
         if isinstance(data, list):
             for ref in data:
@@ -23,10 +23,10 @@ class LLOTA(FileDownloadSIGRefDataProvider):
                 grid = str(ref["grid_locator"])
                 ll = locator_to_latlong(grid)
 
-                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
+                new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=str(ref["name"]),
                                           url="https://llota.app/list/ref/" + ref_id,
                                           grid=grid,
                                           latitude=ll[0],
-                                          longitude=ll[1])
+                                          longitude=ll[1]))
 
         return new_data

sigrefdataproviders/local_file_sig_ref_data_provider.py

@@ -18,7 +18,7 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
         try:
             new_data = self._file_to_data(self._path)
             if new_data:
-                self._replace_data(new_data)
+                self._add_data(new_data)
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
             else:

sigrefdataproviders/mota.py

@@ -15,15 +15,15 @@ class MOTA(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
             ref_id = row["Reference"]
-            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
                                       url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
                                       latitude=float(row["Latitude"]) if "Latitude" in row and row[
                                           "Latitude"] != "" else None,
                                       longitude=float(row["Longitude"]) if "Longitude" in row and row[
                                           "Longitude"] != "" else None,
-                                      grid=row["Maidenhead Locator"])
+                                      grid=row["Maidenhead Locator"]))
 
         return new_data

sigrefdataproviders/pota.py

@@ -15,15 +15,15 @@ class POTA(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["reference"]
-            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
                                       url="https://pota.app/#/park/" + ref_id,
                                       grid=row["grid"] if "grid" in row else None,
                                       latitude=float(row["latitude"]) if "latitude" in row and row[
                                           "latitude"] != "" else None,
                                       longitude=float(row["longitude"]) if "longitude" in row and row[
-                                          "longitude"] != "" else None)
+                                          "longitude"] != "" else None))
 
         return new_data

sigrefdataproviders/sig_ref_data_provider.py

@@ -19,10 +19,6 @@ class SIGRefDataProvider:
         self.status = "Not Started" if self.enabled else "Disabled"
         self.reference_count = 0
 
-        # Create an empty dict to store data if one doesn't already exist
-        if not sig_name in DATA_STORE.sigrefs:
-            DATA_STORE.sigrefs[sig_name] = {}
-
 
     def start(self):
         """Start the provider. This should return immediately after spawning threads to access the remote resources"""
@@ -35,10 +31,10 @@ class SIGRefDataProvider:
 
         raise NotImplementedError("Subclasses must implement this method")
 
-    def _replace_data(self, new_data):
+    def _add_data(self, new_data):
-        """Replace all data for the named sig with the new data. new_data should be a map of reference ID to SIGRef
+        """Add all the provided reference data objects to the data store."""
-        objects."""
 
-        DATA_STORE.sigrefs[self.sig_name] = new_data
+        for d in new_data:
+            DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d
         self.reference_count = len(new_data)
         logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)

sigrefdataproviders/siota.py

@@ -15,12 +15,12 @@ class SIOTA(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["SILO_CODE"]
-            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
                                       grid=row["LOCATOR"] if "LOCATOR" in row else None,
                                       latitude=float(row["LAT"]) if "LAT" in row else None,
-                                      longitude=float(row["LNG"]) if "LNG" in row else None)
+                                      longitude=float(row["LNG"]) if "LNG" in row else None))
 
         return new_data

sigrefdataproviders/sota.py

@@ -17,17 +17,18 @@ class SOTA(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
             ref_id = row["SummitCode"]
             latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
             longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
-            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
+            ref = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
                                       url="https://www.sotadata.org.uk/en/summit/" + ref_id,
                                       latitude=latitude,
                                       longitude=longitude,
                                       activation_score=int(row["Points"]) if "Points" in row else None)
             if latitude and longitude:
-                new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
+                ref.grid = latlong_to_locator(latitude, longitude, 6)
+            new_data.append(ref)
 
         return new_data

sigrefdataproviders/toilets.py

@@ -14,11 +14,11 @@ class Toilets(LocalFileSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.PATH)
 
     def _file_to_data(self, path):
-        new_data = {}
+        new_data = []
         f = open(path)
         csv_data = f.read()
         dr = csv.DictReader(csv_data.splitlines())
         for row in dr:
-            new_data[row["ref"]] = SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]),
+            new_data.append(SIGRef(sig=self.SIG, id=row["ref"], name=row["ref"], latitude=float(row["lat"]),
-                                          longitude=float(row["lon"]))
+                                          longitude=float(row["lon"])))
         return new_data

sigrefdataproviders/towers.py

@@ -15,13 +15,13 @@ class Towers(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
             ref_id = row["Ref"]
-            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
                                       url="https://wwtota.com/seznam/karta_rozhledny.php?ref=" + ref_id,
                                       grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
                                       latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
-                                      longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None)
+                                      longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None))
 
         return new_data

sigrefdataproviders/wca.py

@@ -18,7 +18,7 @@ class WCA(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["REF"]
 
@@ -35,10 +35,10 @@ class WCA(FileDownloadSIGRefDataProvider):
             except ValueError:
                 logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id)
 
-            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
                                       url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
                                       latitude=latitude,
                                       longitude=longitude,
-                                      grid=grid)
+                                      grid=grid))
 
         return new_data

sigrefdataproviders/wota.py

@@ -13,7 +13,7 @@ class WOTA(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for feature in http_response.json().get("features", []):
             ref_id = feature["properties"]["wotaId"]
             # Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
@@ -23,9 +23,9 @@ class WOTA(FileDownloadSIGRefDataProvider):
                 number = int(ref_id.upper().replace("LDO-", ""))
                 url = "https://www.wota.org.uk/MM_LDO-" + str(number + 214)
 
-            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=feature["properties"]["title"], url=url,
                                       grid=feature["properties"]["qthLocator"],
                                       latitude=feature["geometry"]["coordinates"][1],
-                                      longitude=feature["geometry"]["coordinates"][0])
+                                      longitude=feature["geometry"]["coordinates"][0]))
 
         return new_data

sigrefdataproviders/wwbota.py

@@ -15,13 +15,13 @@ class WWBOTA(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["Reference"]
-            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
                                       url="https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None,
                                       grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
                                       latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
-                                      longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None)
+                                      longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None))
 
         return new_data

sigrefdataproviders/wwff.py

@@ -15,16 +15,16 @@ class WWFF(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["reference"]
-            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
+            new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
                                       url="https://wwff.co/directory/?showRef=" + ref_id,
                                       grid=row["iaruLocator"] if "iaruLocator" in row and row[
                                           "iaruLocator"] != "-" else None,
                                       latitude=float(row["latitude"]) if "latitude" in row and row[
                                           "latitude"] != "" and row["latitude"] != "-" else None,
                                       longitude=float(row["longitude"]) if "longitude" in row and row[
-                                          "longitude"] != "" and row["longitude"] != "-" else None)
+                                          "longitude"] != "" and row["longitude"] != "-" else None))
 
         return new_data

sigrefdataproviders/zlota.py

@@ -15,7 +15,7 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
         super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
-        new_data = {}
+        new_data = []
         data = http_response.json()
         if isinstance(data, list):
             for ref in data:
@@ -23,11 +23,12 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
                 latitude = ref["y"]
                 longitude = ref["x"]
 
-                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
+                ref = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
                                           url="https://ontheair.nz/assets/" + ref_id.replace("/", "_"),
                                           latitude=latitude,
                                           longitude=longitude)
                 if latitude and longitude:
-                    new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
+                    ref.grid = latlong_to_locator(latitude, longitude, 6)
+                new_data.append(ref)
 
         return new_data