Refactor of caching & data storage part 8 #118

config-example.yml

@@ -172,6 +172,12 @@ static-data-providers:
   - class: "K0SWE"
     enabled: true
 
+  - class: "CQZoneData"
+    enabled: true
+
+  - class: "ITUZoneData"
+    enabled: true
+
 
 # SIG reference data providers to use. This allows Spothole to download, for example, the WWFF directory that maps WWFF
 # park IDs to their name and location.

core/data_store.py

@@ -16,10 +16,12 @@ class DataStore:
     lookup data using different caching strategies for each."""
 
     def __init__(self):
+        # Constants
         self._MAX_SPOT_COUNT = 100000
         self._MAX_ALERT_COUNT = 100000
         self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC = 300
         self._CALLSIGN_DATA_TTL_SEC = 30 * 24 * 60 * 60
+        # Caches
         self.alerts = None
         self.spots = None
         self.callsigns = None
@@ -30,6 +32,10 @@ class DataStore:
         self._status = None
         self.solar_conditions = None
         self._solar = None
+        # ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be
+        # caches, they can just be straight objects
+        self.cq_zone_data = None
+        self.itu_zone_data = None
 
     def setup(self):
         Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)

core/geo_utils.py

@@ -1,66 +1,58 @@
-import json
 import logging
 import re
 from math import floor
 
-import geopandas
 from pyproj import Transformer
-from shapely import prepare
 from shapely.geometry import Point, Polygon
 
+from core.data_store import DATA_STORE
+
 TRANSFORMER_OS_GRID_TO_WGS84 = Transformer.from_crs("EPSG:27700", "EPSG:4326")
 TRANSFORMER_IRISH_GRID_TO_WGS84 = Transformer.from_crs("EPSG:29903", "EPSG:4326")
 TRANSFORMER_CI_UTM_GRID_TO_WGS84 = Transformer.from_crs("+proj=utm +zone=30 +ellps=WGS84", "EPSG:4326")
 
-with open("datafiles/cqzones.geojson") as f:
-    cq_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
-with open("datafiles/ituzones.geojson") as f:
-    itu_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
-for idx in cq_zone_data.index:
-    prepare(cq_zone_data.at[idx, 'geometry'])
-for idx in itu_zone_data.index:
-    prepare(itu_zone_data.at[idx, 'geometry'])
-
 
 def lat_lon_to_cq_zone(lat, lon):
     """Finds out which CQ zone a lat/lon point is in."""
 
-    lon = ((lon + 180) % 360) - 180
+    if DATA_STORE.cq_zone_data is not None:
-    for index, row in cq_zone_data.iterrows():
+        lon = ((lon + 180) % 360) - 180
-        polygon = Polygon(row["geometry"])
+        for index, row in DATA_STORE.cq_zone_data.iterrows():
-        test_point = Point(lon, lat)
+            polygon = Polygon(row["geometry"])
-        if polygon.contains(test_point):
+            test_point = Point(lon, lat)
-            return int(row["name"])
+            if polygon.contains(test_point):
+                return int(row["name"])
 
-        # Might have problems around the antemeridian, so if we didn't find a match, try offsetting the point by + or -
+            # Might have problems around the antemeridian, so if we didn't find a match, try offsetting the point by + or -
-        # 360 degrees longitude to try the other side of the Earth
+            # 360 degrees longitude to try the other side of the Earth
-        if lon < 0:
+            if lon < 0:
-            test_point = Point(lon + 360, lat)
+                test_point = Point(lon + 360, lat)
-        else:
+            else:
-            test_point = Point(lon - 360, lat)
+                test_point = Point(lon - 360, lat)
-        if polygon.contains(test_point):
+            if polygon.contains(test_point):
-            return int(row["name"])
+                return int(row["name"])
     return None
 
 
 def lat_lon_to_itu_zone(lat, lon):
     """Finds out which ITU zone a lat/lon point is in."""
 
-    lon = ((lon + 180) % 360) - 180
+    if DATA_STORE.itu_zone_data is not None:
-    for index, row in itu_zone_data.iterrows():
+        lon = ((lon + 180) % 360) - 180
-        polygon = Polygon(row["geometry"])
+        for index, row in DATA_STORE.itu_zone_data.iterrows():
-        test_point = Point(lon, lat)
+            polygon = Polygon(row["geometry"])
-        if polygon.contains(test_point):
+            test_point = Point(lon, lat)
-            return int(row["name"])
+            if polygon.contains(test_point):
+                return int(row["name"])
 
-        # Might have problems around the antemeridian, so if we didn't find a match, try offsetting the point by + or -
+            # Might have problems around the antemeridian, so if we didn't find a match, try offsetting the point by + or -
-        # 360 degrees longitude to try the other side of the Earth
+            # 360 degrees longitude to try the other side of the Earth
-        if lon < 0:
+            if lon < 0:
-            test_point = Point(lon + 360, lat)
+                test_point = Point(lon + 360, lat)
-        else:
+            else:
-            test_point = Point(lon - 360, lat)
+                test_point = Point(lon - 360, lat)
-        if polygon.contains(test_point):
+            if polygon.contains(test_point):
-            return int(row["name"])
+                return int(row["name"])
     return None
 
 

staticdataproviders/cqzonedata.py

@@ -0,0 +1,30 @@
+import json
+import logging
+
+import geopandas
+from shapely import prepare
+
+from core.data_store import DATA_STORE
+from staticdataproviders.local_file_static_data_provider import LocalFileStaticDataProvider
+
+
+class CQZoneData(LocalFileStaticDataProvider):
+    """Static data provider for CQ zone geodata."""
+
+    PATH = "datafiles/cqzones.geojson"
+
+    def __init__(self, provider_config):
+        super().__init__("CQ Zone Data", provider_config, self.PATH)
+
+    def _load_data(self, path):
+        try:
+            with open(path) as f:
+                cq_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
+            for idx in cq_zone_data.index:
+                prepare(cq_zone_data.at[idx, 'geometry'])
+            DATA_STORE.cq_zone_data = cq_zone_data
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading CQ zone data.", e, exc_info=True)
+            return False

staticdataproviders/ituzonedata.py

@@ -0,0 +1,30 @@
+import json
+import logging
+
+import geopandas
+from shapely import prepare
+
+from core.data_store import DATA_STORE
+from staticdataproviders.local_file_static_data_provider import LocalFileStaticDataProvider
+
+
+class ITUZoneData(LocalFileStaticDataProvider):
+    """Static data provider for ITU zone geodata."""
+
+    PATH = "datafiles/ituzones.geojson"
+
+    def __init__(self, provider_config):
+        super().__init__("ITU Zone Data", provider_config, self.PATH)
+
+    def _load_data(self, path):
+        try:
+            with open(path) as f:
+                itu_zone_data = geopandas.GeoDataFrame.from_features(json.load(f)["features"])
+            for idx in itu_zone_data.index:
+                prepare(itu_zone_data.at[idx, 'geometry'])
+            DATA_STORE.itu_zone_data = itu_zone_data
+            return True
+
+        except Exception as e:
+            logging.error("Exception when loading ITU zone data.", e, exc_info=True)
+            return False

staticdataproviders/local_file_static_data_provider.py

@@ -20,9 +20,10 @@ class LocalFileStaticDataProvider(StaticDataProvider):
             if ok:
                 self.status = "OK"
                 self.last_update_time = datetime.now(pytz.UTC)
+                logging.info("Updated static reference data for " + self.name)
             else:
                 self.status = "Error"
-                logging.info("Failed to load data for " + self.name)
+                logging.error("Failed to load data for " + self.name)
         except Exception as e:
             self.status = "Error"
             logging.error("Exception in local file Static Data Provider (" + self.name + ")", e, exc_info=True)