Refactor of caching & data storage part 4 #118

sigrefdataproviders/arlhs.py

@@ -0,0 +1,30 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class ARLHS(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Amateur Radio Light House Society"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "ARLHS"
+    DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            if "ARLHS" in row:
+                ref_id = row["ARLHS"]
+                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                              "Latitude"] != "" else None,
+                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                              "Longitude"] != "" else None,
+                                          grid=row["Maidenhead Locator"])
+
+        return new_data

sigrefdataproviders/file_download_sig_ref_data_provider.py

@@ -3,11 +3,11 @@ from datetime import datetime
 from threading import Thread, Event
 
 import pytz
-import requests
 from requests import ReadTimeout
 from requests.exceptions import ConnectionError, ConnectTimeout
 
 from core.constants import HTTP_HEADERS
+from core.url_data_cache import URL_DATA_CACHE
 from sigrefdataproviders.sig_ref_data_provider import SIGRefDataProvider
 
 
@@ -15,6 +15,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
     """Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file."""
 
     def __init__(self, sig_name, provider_config, url, poll_interval):
+        """ Set up the provider, note poll_interval is in *days*."""
         super().__init__(sig_name, provider_config)
         self._url = url
         self._poll_interval = poll_interval
@@ -25,7 +26,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
         # Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
         # subsequent polls, so start() returns immediately and the application can continue starting.
         logging.info(
-            "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " seconds.")
+            "Set up query of " + self.sig_name + " SIG ref data every " + str(self._poll_interval) + " days.")
         self._thread = Thread(target=self._run, daemon=True)
         self._thread.start()
 
@@ -35,14 +36,15 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
     def _run(self):
         while True:
             self._poll()
-            if self._stop_event.wait(timeout=self._poll_interval):
+            if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
                 break
 
     def _poll(self):
         try:
-            # Request data from API
+            # Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
+            # caching, but just so continual restarts of the software during testing don't hammer the servers.
             logging.debug("Downloading " + self.sig_name + " SIG ref data...")
-            http_response = requests.get(self._url, headers=HTTP_HEADERS, timeout=(5, 30))
+            http_response = URL_DATA_CACHE.get(self._url, headers=HTTP_HEADERS)
             # Check response code was good
             if http_response.ok:
                 # Pass off to the subclass for processing

sigrefdataproviders/gma.py

@@ -0,0 +1,29 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class GMA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Global Mountain Activity"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "GMA"
+    DATA_URL = "https://www.gma.rocks/download/summits.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["Reference"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                          "Latitude"] != "" else None,
+                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                          "Longitude"] != "" else None,
+                                      grid=row["Maidenhead Locator"])
+
+        return new_data

sigrefdataproviders/illw.py

@@ -0,0 +1,30 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class ILLW(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for International Lighthouse & Lightship Weekend"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "ILLW"
+    DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            if "ILLW" in row:
+                ref_id = row["ILLW"]
+                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                          url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                          latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                              "Latitude"] != "" else None,
+                                          longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                              "Longitude"] != "" else None,
+                                          grid=row["Maidenhead Locator"])
+
+        return new_data

sigrefdataproviders/iota.py

@@ -0,0 +1,36 @@
+import logging
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class IOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Islands on the Air"""
+
+    POLL_INTERVAL_DAYS = 365
+    SIG = "IOTA"
+    DATA_URL = "https://www.iota-world.org/islands-on-the-air/downloads/download-file.html?path=groups.json"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        data = http_response.json()
+        if isinstance(data, list):
+            for ref in data:
+                ref_id = ref["refno"]
+                latitude = float(ref["latitude_min"]) + float(ref["latitude_max"]) / 2.0
+                longitude = float(ref["longitude_min"]) + float(ref["longitude_max"]) / 2.0
+                grid = None
+                try:
+                    grid = latlong_to_locator(latitude, longitude, 6)
+                except ValueError:
+                    logging.debug(f"Error converting lat/lon to locator for an IOTA reference %f %f", latitude, longitude)
+
+                new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=ref["name"],
+                                          grid=grid, latitude=latitude, longitude=longitude)
+
+        return new_data

sigrefdataproviders/llota.py

@@ -7,12 +7,12 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload
 class LLOTA(FileDownloadSIGRefDataProvider):
     """SIG ref data provider for Lagos y Lagunas on the Air"""
 
-    POLL_INTERVAL_SEC = 7 * 24 * 60 * 60  # 7 days
+    POLL_INTERVAL_DAYS = 7
     SIG = "LLOTA"
     DATA_URL = "https://llota.app/api/public/references"
 
     def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
         new_data = {}

sigrefdataproviders/local_file_sig_ref_data_provider.py

@@ -14,7 +14,7 @@ class LocalFileSIGRefDataProvider(SIGRefDataProvider):
         self._path = path
 
     def start(self):
-        logging.info("Loading " + self.sig_name + " SIG ref data from file.")
+        logging.debug("Loading " + self.sig_name + " SIG ref data from file.")
         try:
             new_data = self._file_to_data(self._path)
             if new_data:

sigrefdataproviders/mota.py

@@ -0,0 +1,29 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class MOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Mills on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "MOTA"
+    DATA_URL = "https://www.gma.rocks/download/mills.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["Reference"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=float(row["Latitude"]) if "Latitude" in row and row[
+                                          "Latitude"] != "" else None,
+                                      longitude=float(row["Longitude"]) if "Longitude" in row and row[
+                                          "Longitude"] != "" else None,
+                                      grid=row["Maidenhead Locator"])
+
+        return new_data

sigrefdataproviders/pota.py

@@ -0,0 +1,29 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class POTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Parks on the Air"""
+
+    POLL_INTERVAL_DAYS = 7
+    SIG = "POTA"
+    DATA_URL = "https://pota.app/all_parks_ext.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["reference"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
+                                      url="https://pota.app/#/park/" + ref_id,
+                                      grid=row["grid"] if "grid" in row else None,
+                                      latitude=float(row["latitude"]) if "latitude" in row and row[
+                                          "latitude"] != "" else None,
+                                      longitude=float(row["longitude"]) if "longitude" in row and row[
+                                          "longitude"] != "" else None)
+
+        return new_data

sigrefdataproviders/sig_ref_data_provider.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 
 import pytz
@@ -16,6 +17,7 @@ class SIGRefDataProvider:
         self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC)
         self.status = "Not Started" if self.enabled else "Disabled"
+        self.reference_count = 0
 
         # Create an empty dict to store data if one doesn't already exist
         if not sig_name in DATA_STORE.sigrefs:
@@ -38,3 +40,5 @@ class SIGRefDataProvider:
         objects."""
 
         DATA_STORE.sigrefs[self.sig_name] = new_data
+        self.reference_count = len(new_data)
+        logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)

sigrefdataproviders/siota.py

@@ -7,16 +7,16 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload
 class SIOTA(FileDownloadSIGRefDataProvider):
     """SIG ref data provider for Silos on the Air"""
 
-    POLL_INTERVAL_SEC = 30 * 24 * 60 * 60  # 30 days
+    POLL_INTERVAL_DAYS = 30
     SIG = "SIOTA"
     DATA_URL = "https://www.silosontheair.com/data/silos.csv"
 
     def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
         new_data = {}
-        for row in csv.DictReader(http_response.content.decode().splitlines()):
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["SILO_CODE"]
             new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["NAME"] if "NAME" in row else None,
                                       grid=row["LOCATOR"] if "LOCATOR" in row else None,

sigrefdataproviders/sota.py

@@ -0,0 +1,33 @@
+import csv
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class SOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Summits on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "SOTA"
+    DATA_URL = "https://storage.sota.org.uk/summitslist.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
+            ref_id = row["SummitCode"]
+            latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
+            longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["SummitName"] if "SummitName" in row else None,
+                                      url="https://www.sotadata.org.uk/en/summit/" + ref_id,
+                                      latitude=latitude,
+                                      longitude=longitude,
+                                      activation_score=int(row["Points"]) if "Points" in row else None)
+            if latitude and longitude:
+                new_data[ref_id].grid = latlong_to_locator(latitude, longitude, 6)
+
+        return new_data

sigrefdataproviders/toilets.py

@@ -4,11 +4,11 @@ from data.sig_ref import SIGRef
 from sigrefdataproviders.local_file_sig_ref_data_provider import LocalFileSIGRefDataProvider
 
 
-class TOTA(LocalFileSIGRefDataProvider):
+class Toilets(LocalFileSIGRefDataProvider):
     """SIG ref data provider for Toilets on the Air"""
 
-    SIG = "TOTA"
-    PATH = "datafiles/tota.csv"
+    SIG = "Toilets"
+    PATH = "datafiles/toilets.csv"
 
     def __init__(self, provider_config):
         super().__init__(self.SIG, provider_config, self.PATH)

sigrefdataproviders/towers.py

@@ -0,0 +1,27 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class Towers(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Towers on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "Towers"
+    DATA_URL = "https://wwtota.com/servis/generate_csv.php?ref=&filter=all"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
+            ref_id = row["Ref"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Nazev"] if "Nazev" in row else None,
+                                      url="https://wwtota.com/seznam/karta_rozhledny.php?ref=" + ref_id,
+                                      grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
+                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
+                                      longitude=float(row["Lon"]) if "Lon" in row and row["Lon"] != "" else None)
+
+        return new_data

sigrefdataproviders/wca.py

@@ -0,0 +1,44 @@
+import csv
+import logging
+
+from pyhamtools.locator import latlong_to_locator
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WCA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for World Castles Award"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "WCA"
+    DATA_URL = "https://polo.ham2k.com/data/activities/wca/all-castles.csv"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["REF"]
+
+            coordsStr = row["COORDINATES"]
+            latitude = None
+            longitude = None
+            grid = None
+            try:
+                if coordsStr:
+                    split = coordsStr.split(", ")
+                    latitude = float(split[0])
+                    longitude = float(split[1])
+                    grid = latlong_to_locator(latitude, longitude)
+            except ValueError:
+                logging.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for %s", ref_id)
+
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["CLEAN NAME"] if "CLEAN NAME" in row else None,
+                                      url="https://www.cqgma.org/zinfo.php?ref=" + ref_id,
+                                      latitude=latitude,
+                                      longitude=longitude,
+                                      grid=grid)
+
+        return new_data

sigrefdataproviders/wota.py

@@ -5,12 +5,12 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload
 class WOTA(FileDownloadSIGRefDataProvider):
     """SIG ref data provider for Wainwrights on the Air"""
 
-    POLL_INTERVAL_SEC = 365 * 24 * 60 * 60  # 365 days
+    POLL_INTERVAL_DAYS = 365
     SIG = "WOTA"
     DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json"
 
     def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
         new_data = {}

sigrefdataproviders/wwbota.py

@@ -0,0 +1,27 @@
+import csv
+
+from data.sig_ref import SIGRef
+from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
+
+
+class WWBOTA(FileDownloadSIGRefDataProvider):
+    """SIG ref data provider for Worldwide Bunkers on the Air"""
+
+    POLL_INTERVAL_DAYS = 30
+    SIG = "WWBOTA"
+    DATA_URL = "https://api.wwbota.org/bunkers/?format=CSV"
+
+    def __init__(self, provider_config):
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+
+    def _http_response_to_data(self, http_response):
+        new_data = {}
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
+            ref_id = row["Reference"]
+            new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["Name"] if "Name" in row else None,
+                                      url="https://bunkerwiki.org/?s=" + ref_id if ref_id.startswith("B/G") else None,
+                                      grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
+                                      latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
+                                      longitude=float(row["Long"]) if "Long" in row and row["Long"] != "" else None)
+
+        return new_data

sigrefdataproviders/wwff.py

@@ -7,16 +7,16 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload
 class WWFF(FileDownloadSIGRefDataProvider):
     """SIG ref data provider for Worldwide Flora & Fauna"""
 
-    POLL_INTERVAL_SEC = 30 * 24 * 60 * 60  # 30 days
+    POLL_INTERVAL_DAYS = 30
     SIG = "WWFF"
     DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv"
 
     def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
         new_data = {}
-        for row in csv.DictReader(http_response.content.decode().splitlines()):
+        for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
             ref_id = row["reference"]
             new_data[ref_id] = SIGRef(sig=self.SIG, id=ref_id, name=row["name"] if "name" in row else None,
                                       url="https://wwff.co/directory/?showRef=" + ref_id,

sigrefdataproviders/zlota.py

@@ -7,12 +7,12 @@ from sigrefdataproviders.file_download_sig_ref_data_provider import FileDownload
 class ZLOTA(FileDownloadSIGRefDataProvider):
     """SIG ref data provider for New Zealand on the Air"""
 
-    POLL_INTERVAL_SEC = 30 * 24 * 60 * 60  # 30 days
+    POLL_INTERVAL_DAYS = 30
     SIG = "ZLOTA"
     DATA_URL = "https://ontheair.nz/assets/assets.json"
 
     def __init__(self, provider_config):
-        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_SEC)
+        super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
 
     def _http_response_to_data(self, http_response):
         new_data = {}