mirror of
https://git.ianrenton.com/ian/spothole.git
synced 2026-09-21 06:47:42 +00:00
Autogenerated type safety parameterisation of all methods
This commit is contained in:
@@ -1,10 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Event
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
from core.data_store import DATA_STORE
|
||||
from data.activity_ref import ActivityRef
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -13,29 +17,29 @@ class ActivityRefDataProvider:
|
||||
"""Generic activity reference data provider class. Subclasses of this query the individual URLs or files for
|
||||
data."""
|
||||
|
||||
def __init__(self, sig_name, provider_config):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any]) -> None:
|
||||
"""Constructor. Note the parameter and attribute are still named "sig_name" for consistency with the API's
|
||||
"sig" field name."""
|
||||
|
||||
self.sig_name = sig_name
|
||||
self.enabled = provider_config["enabled"]
|
||||
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status = "Not Started" if self.enabled else "Disabled"
|
||||
self.reference_count = 0
|
||||
self.enabled: bool = provider_config["enabled"]
|
||||
self.last_update_time: datetime = datetime.min.replace(tzinfo=pytz.UTC)
|
||||
self.status: str = "Not Started" if self.enabled else "Disabled"
|
||||
self.reference_count: int = 0
|
||||
self._stop_event = Event()
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
"""Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
|
||||
super()."""
|
||||
|
||||
self._stop_event.set()
|
||||
|
||||
def _add_data(self, new_data):
|
||||
def _add_data(self, new_data: list[ActivityRef]) -> None:
|
||||
"""Add all the provided reference data objects to the data store."""
|
||||
|
||||
# with transact() batches all writes together to save making thousands of individual sqlite writes. However,
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class ARLHS(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.ARLHS
|
||||
DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
if "ARLHS" in row and row["ARLHS"] != "":
|
||||
ref_id = row["ARLHS"]
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -14,11 +18,11 @@ class COTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.COTA
|
||||
DATA_URL = "https://www.cotagroup.org/cotagroup/map/data/castles-all-7d90ee2a5e1175e5dece1bbf9dc87504.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data[2]:
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class DCE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DCE
|
||||
DATA_URL = "https://www.acracb.org/dce/descargas/General/directorio_referencias_dce.xls"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
file_stream = io.BytesIO(http_response.content)
|
||||
df = pd.read_excel(file_stream, engine="xlrd", header=None)
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class DEFE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DEFE
|
||||
DATA_URL = "https://www.acracb.org/defe/descargas/General/directorio_referencias_defe.xls"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
file_stream = io.BytesIO(http_response.content)
|
||||
df = pd.read_excel(file_stream, engine="xlrd", header=None)
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
@@ -16,11 +19,11 @@ class DME(LocalFileActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DME
|
||||
PATH = "datafiles/MUNICIPIOS.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.PATH)
|
||||
|
||||
def _file_to_data(self, path):
|
||||
new_data = []
|
||||
def _file_to_data(self, path: str) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
with open(path, encoding="latin-1") as _f:
|
||||
for row in csv.DictReader(_f, delimiter=";"):
|
||||
# Store reference IDs with the "DME-" prefix rather than just the number. This will prevent Spothole
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -13,11 +18,11 @@ class DMUE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DMUE
|
||||
DATA_URL = "https://dmue.radiogalena.es/nom_dmue.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
for row in csv.reader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
|
||||
if len(row) > 1 and row[0] and row[1]:
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class DMVE(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DMVE
|
||||
DATA_URL = "https://www.acracb.org/dmve/descargas/General/directorio_referencias_dmve.xls"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
file_stream = io.BytesIO(http_response.content)
|
||||
# Despide the .xls extension this is actually an xlsx file, so we need openpyxl not xlrd
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -12,11 +17,11 @@ class DTMBA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.DTMBA
|
||||
DATA_URL = "https://www.iu1fig.com/share/iz0eik/dtmba/export.php"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in http_response.content.decode("utf-8-sig").splitlines():
|
||||
split = row.split(";")
|
||||
ref_id = split[0]
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import pdfplumber
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +19,11 @@ class FEA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.FEA
|
||||
DATA_URL = "http://ea5ol.net/Lista%20Faros.pdf"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
# Use PDFPlumber to extract the tables in the PDF
|
||||
with pdfplumber.open(BytesIO(http_response.content)) as pdf:
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
|
||||
|
||||
from core.constants import HTTP_HEADERS
|
||||
from core.url_data_cache import URLDataCache
|
||||
from data.activity_ref import ActivityRef
|
||||
from providers.activityrefdata.activity_ref_data_provider import ActivityRefDataProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -16,35 +21,35 @@ class FileDownloadActivityRefDataProvider(ActivityRefDataProvider):
|
||||
"""Generic activity ref data provider class for providers that fetch their data from the web by downloading a
|
||||
file."""
|
||||
|
||||
def __init__(self, sig_name, provider_config, url, poll_interval):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None:
|
||||
"""Set up the provider, note poll_interval is in *days*."""
|
||||
super().__init__(sig_name, provider_config)
|
||||
self._url = url
|
||||
self._poll_interval = poll_interval
|
||||
self._thread = None
|
||||
self._thread: Thread | None = None
|
||||
self._url_data_cache = URLDataCache(f"activity_ref_data_{sig_name}")
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
|
||||
# subsequent polls, so start() returns immediately and the application can continue starting.
|
||||
logger.info(f"Set up query of {self.sig_name} activity ref data every {self._poll_interval!s} days.")
|
||||
self._thread = Thread(target=self._run, name=f"FileDownloadActivityRefDataProvider-{self.sig_name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
def stop(self) -> None:
|
||||
super().stop()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=12)
|
||||
if self._thread.is_alive():
|
||||
logger.warning(f"{self.sig_name} activity ref data worker thread did not exit on time and will be killed.")
|
||||
|
||||
def _run(self):
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._poll()
|
||||
if self._stop_event.wait(timeout=self._poll_interval * 60 * 60 * 24):
|
||||
break
|
||||
|
||||
def _poll(self):
|
||||
def _poll(self) -> None:
|
||||
try:
|
||||
# Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
|
||||
# caching, but just so continual restarts of the software during testing don't hammer the servers.
|
||||
@@ -76,7 +81,7 @@ class FileDownloadActivityRefDataProvider(ActivityRefDataProvider):
|
||||
logger.exception(f"Exception in HTTP Activity Ref Data Provider ({self.sig_name})")
|
||||
self._stop_event.wait(timeout=1)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
"""Convert an HTTP response returned by the server into activity ref data. The whole response is provided here
|
||||
so the subclass implementations can check for HTTP status codes if necessary, and handle the response as
|
||||
JSON, CSV, whatever the remote file actually is."""
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class GMA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.GMA
|
||||
DATA_URL = "https://www.gma.rocks/download/summits.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
ref_id = row["Reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class ILLW(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.ILLW
|
||||
DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
if "ILLW" in row and row["ILLW"] != "":
|
||||
ref_id = row["ILLW"]
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -19,11 +23,11 @@ class IOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.IOTA
|
||||
DATA_URL = "https://www.iota-world.org/islands-on-the-air/downloads/download-file.html?path=groups.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data:
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.enums import ActivityName
|
||||
from providers.activityrefdata.pnp_kml_activity_ref_data_provider import (
|
||||
ParksNPeaksKMLActivityRefDataProvider,
|
||||
@@ -11,5 +15,5 @@ class KRMNPA(ParksNPeaksKMLActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.KRMNPA
|
||||
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=KRMNPA&poiFormat=4"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import locator_to_latlong
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -16,11 +20,11 @@ class LLOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.LLOTA
|
||||
DATA_URL = "https://llota.app/api/public/references"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data:
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
from data.activity_ref import ActivityRef
|
||||
from providers.activityrefdata.activity_ref_data_provider import ActivityRefDataProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -11,11 +15,11 @@ logger = logging.getLogger(__name__)
|
||||
class LocalFileActivityRefDataProvider(ActivityRefDataProvider):
|
||||
"""Generic activity ref data provider class for providers that fetch their data from a local file on startup."""
|
||||
|
||||
def __init__(self, sig_name, provider_config, path):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any], path: str) -> None:
|
||||
super().__init__(sig_name, provider_config)
|
||||
self._path = path
|
||||
|
||||
def start(self):
|
||||
def start(self) -> None:
|
||||
logger.debug(f"Loading {self.sig_name} activity ref data from file.")
|
||||
try:
|
||||
new_data = self._file_to_data(self._path)
|
||||
@@ -30,7 +34,7 @@ class LocalFileActivityRefDataProvider(ActivityRefDataProvider):
|
||||
self.status = "Error"
|
||||
logger.exception(f"Exception in local file Activity Ref Data Provider ({self.sig_name})")
|
||||
|
||||
def _file_to_data(self, path):
|
||||
def _file_to_data(self, path: str) -> list[ActivityRef]:
|
||||
"""Load a file on the given path and turn it into activity ref data."""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement this method")
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class MOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.MOTA
|
||||
DATA_URL = "https://www.gma.rocks/download/mills.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
ref_id = row["Reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -14,11 +18,11 @@ class PGA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.PGA
|
||||
DATA_URL = "http://www.spga.pl/lista_pga2.php"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
soup = BeautifulSoup(http_response.text, "html.parser")
|
||||
|
||||
# Iterate through tables in the page
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from fastkml import kml
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
@@ -17,12 +21,12 @@ class ParksNPeaksKMLActivityRefDataProvider(FileDownloadActivityRefDataProvider)
|
||||
|
||||
REF_PATTERN = re.compile(r"VKFF-\d+")
|
||||
|
||||
def __init__(self, sig_name, provider_config, url, poll_interval):
|
||||
def __init__(self, sig_name: str, provider_config: dict[str, Any], url: str, poll_interval: float) -> None:
|
||||
"""Set up the provider, note poll_interval is in *days*."""
|
||||
super().__init__(sig_name, provider_config, url, poll_interval)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
|
||||
k = kml.KML.from_string(http_response.content)
|
||||
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class POTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.POTA
|
||||
DATA_URL = "https://pota.app/all_parks_ext.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.enums import ActivityName
|
||||
from providers.activityrefdata.pnp_kml_activity_ref_data_provider import (
|
||||
ParksNPeaksKMLActivityRefDataProvider,
|
||||
@@ -11,5 +15,5 @@ class SANPCPA(ParksNPeaksKMLActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.SANPCPA
|
||||
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=SANPCPA&poiFormat=4"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class SIOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.SIOTA
|
||||
DATA_URL = "https://www.silosontheair.com/data/silos.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["SILO_CODE"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -17,11 +21,11 @@ class SOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.SOTA
|
||||
DATA_URL = "https://storage.sota.org.uk/summitslist.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
|
||||
ref_id = row["SummitCode"]
|
||||
latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from typing import Any
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -13,11 +16,11 @@ class Toilets(LocalFileActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.TOILETS
|
||||
PATH = "datafiles/toilets.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.PATH)
|
||||
|
||||
def _file_to_data(self, path):
|
||||
new_data = []
|
||||
def _file_to_data(self, path: str) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
with open(path) as _f:
|
||||
csv_data = _f.read()
|
||||
dr = csv.DictReader(csv_data.splitlines())
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class Towers(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.TOWERS
|
||||
DATA_URL = "https://wwtota.com/servis/generate_csv.php?ref=&filter=all"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
|
||||
ref_id = row["Ref"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -20,11 +24,11 @@ class WCA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WCA
|
||||
DATA_URL = "https://polo.ham2k.com/data/activities/wca/all-castles.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["REF"]
|
||||
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -14,11 +19,11 @@ class WOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WOTA
|
||||
DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for feature in http_response.json().get("features", []):
|
||||
ref_id = feature["properties"]["wotaId"]
|
||||
# Fudge WOTA URLs. Outlying fell (LDO) URLs don't match their ID numbers but require 214 to be
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class WWBOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WWBOTA
|
||||
DATA_URL = "https://api.wwbota.org/bunkers/?format=CSV"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["Reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
from data.activity_ref import ActivityRef
|
||||
@@ -15,11 +20,11 @@ class WWFF(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.WWFF
|
||||
DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
|
||||
ref_id = row["reference"]
|
||||
new_data.append(
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from time import sleep
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyhamtools.locator import latlong_to_locator
|
||||
|
||||
from core.enums import ActivityName, ActivityRefType
|
||||
@@ -16,11 +20,11 @@ class ZLOTA(FileDownloadActivityRefDataProvider):
|
||||
ACTIVITY = ActivityName.ZLOTA
|
||||
DATA_URL = "https://ontheair.nz/assets/assets.json"
|
||||
|
||||
def __init__(self, provider_config):
|
||||
def __init__(self, provider_config: dict[str, Any]) -> None:
|
||||
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
||||
|
||||
def _http_response_to_data(self, http_response):
|
||||
new_data = []
|
||||
def _http_response_to_data(self, http_response: requests.Response) -> list[ActivityRef]:
|
||||
new_data: list[ActivityRef] = []
|
||||
data = http_response.json()
|
||||
if isinstance(data, list):
|
||||
for ref in data:
|
||||
|
||||
Reference in New Issue
Block a user