Giant refactor to rebrand "SIG" as "Activity" anywhere that doesn't touch API or config file (which is to be addressed in a future breaking change). #147

This commit is contained in:
Ian Renton
2026-09-18 14:54:11 +01:00
parent 556ea56378
commit 81cd686a00
96 changed files with 1208 additions and 1170 deletions
@@ -9,11 +9,13 @@ from core.data_store import DATA_STORE
logger = logging.getLogger(__name__)
class SIGRefDataProvider:
"""Generic SIG reference data provider class. Subclasses of this query the individual URLs or files for data."""
class ActivityRefDataProvider:
"""Generic activity reference data provider class. Subclasses of this query the individual URLs or files for
data."""
def __init__(self, sig_name, provider_config):
"""Constructor"""
"""Constructor. Note the parameter and attribute are still named "sig_name" for consistency with the API's
"sig" field name."""
self.sig_name = sig_name
self.enabled = provider_config["enabled"]
@@ -37,9 +39,9 @@ class SIGRefDataProvider:
"""Add all the provided reference data objects to the data store."""
# with transact() batches all writes together to save making thousands of individual sqlite writes
with DATA_STORE.sigrefs.transact():
with DATA_STORE.activity_refs.transact():
for d in new_data:
DATA_STORE.sigrefs.set(f"{self.sig_name}:{d.id}", d)
DATA_STORE.activity_refs.set(f"{self.sig_name}:{d.id}", d)
# For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
# within the first few minutes of startup, we need a way to abort this expensive process of filling up the
@@ -1,22 +1,22 @@
import csv
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class ARLHS(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Amateur Radio Light House Society"""
class ARLHS(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Amateur Radio Light House Society"""
POLL_INTERVAL_DAYS = 30
SIG = "ARLHS"
ACTIVITY = "ARLHS"
DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -24,11 +24,11 @@ class ARLHS(FileDownloadSIGRefDataProvider):
if "ARLHS" in row and row["ARLHS"] != "":
ref_id = row["ARLHS"]
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=row.get("Name", None),
ref_type=SIGRefType.LIGHTHOUSE,
ref_type=ActivityRefType.LIGHTHOUSE,
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
@@ -41,7 +41,7 @@ class ARLHS(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -2,20 +2,20 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import FileDownloadActivityRefDataProvider
class COTA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Castles on the Air"""
class COTA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Castles on the Air"""
POLL_INTERVAL_DAYS = 30
SIG = "COTA"
ACTIVITY = "COTA"
DATA_URL = "https://www.cotagroup.org/cotagroup/map/data/castles-all-7d90ee2a5e1175e5dece1bbf9dc87504.json"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -29,11 +29,11 @@ class COTA(FileDownloadSIGRefDataProvider):
grid = latlong_to_locator(lat, lon)
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=name,
ref_type=SIGRefType.CASTLE,
ref_type=ActivityRefType.CASTLE,
grid=grid,
latitude=lat,
longitude=lon,
@@ -45,7 +45,7 @@ class COTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -3,20 +3,20 @@ from time import sleep
import pandas as pd
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import FileDownloadActivityRefDataProvider
class DCE(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Diploma Castillos de España"""
class DCE(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Diploma Castillos de España"""
POLL_INTERVAL_DAYS = 365
SIG = "DCE"
ACTIVITY = "DCE"
DATA_URL = "https://www.acracb.org/dce/descargas/General/directorio_referencias_dce.xls"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -27,7 +27,7 @@ class DCE(FileDownloadSIGRefDataProvider):
for index, row in df.iterrows():
if row.iloc[0] and row.iloc[2]:
new_data.append(
SIGRef(sig=self.SIG, id=row.iloc[0].strip(), name=row.iloc[2].strip(), ref_type=SIGRefType.CASTLE)
ActivityRef(sig=self.ACTIVITY, id=row.iloc[0].strip(), name=row.iloc[2].strip(), ref_type=ActivityRefType.CASTLE)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
@@ -35,7 +35,7 @@ class DCE(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -3,20 +3,20 @@ from time import sleep
import pandas as pd
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import FileDownloadActivityRefDataProvider
class DEFE(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Diploma Estationes de Ferrocarril de España"""
class DEFE(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Diploma Estationes de Ferrocarril de España"""
POLL_INTERVAL_DAYS = 365
SIG = "DEFE"
ACTIVITY = "DEFE"
DATA_URL = "https://www.acracb.org/defe/descargas/General/directorio_referencias_defe.xls"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -31,7 +31,7 @@ class DEFE(FileDownloadSIGRefDataProvider):
if row.iloc[0] and row.iloc[1]:
new_data.append(
SIGRef(sig=self.SIG, id=row.iloc[0].strip(), name=row.iloc[1].strip(), ref_type=SIGRefType.BUILDING)
ActivityRef(sig=self.ACTIVITY, id=row.iloc[0].strip(), name=row.iloc[1].strip(), ref_type=ActivityRefType.BUILDING)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
@@ -39,7 +39,7 @@ class DEFE(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -3,21 +3,21 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.local_file_sig_ref_data_provider import (
LocalFileSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.local_file_activity_ref_data_provider import (
LocalFileActivityRefDataProvider,
)
class DME(LocalFileSIGRefDataProvider):
"""SIG ref data provider for Diploma Municipios de Espana"""
class DME(LocalFileActivityRefDataProvider):
"""Activity ref data provider for Diploma Municipios de Espana"""
SIG = "DME"
ACTIVITY = "DME"
PATH = "datafiles/MUNICIPIOS.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.PATH)
super().__init__(self.ACTIVITY, provider_config, self.PATH)
def _file_to_data(self, path):
new_data = []
@@ -26,7 +26,7 @@ class DME(LocalFileSIGRefDataProvider):
# Store reference IDs with the "DME-" prefix rather than just the number. This will prevent Spothole
# from agressively thinking every number in a spot comment is DME after it's seen "DME" once. The only
# numbers that count are straight after "DME " or "DME-". The dash versus space is normalised in
# sig_lookup_helper.py.
# activity_lookup_helper.py.
ref_id = "DME-" + row["COD_INE"][:5]
latitude = (
float(row["LATITUD_ETRS89_REGCAN95"].replace(",", "."))
@@ -39,10 +39,10 @@ class DME(LocalFileSIGRefDataProvider):
else None
)
ref = SIGRef(
sig=self.SIG,
ref = ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
ref_type=SIGRefType.TOWN,
ref_type=ActivityRefType.TOWN,
name=f"{row['NOMBRE_ACTUAL']}, {row['PROVINCIA']}",
latitude=latitude,
longitude=longitude,
@@ -56,7 +56,7 @@ class DME(LocalFileSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -1,20 +1,20 @@
import csv
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import FileDownloadActivityRefDataProvider
class DMUE(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Diploma Museos de España"""
class DMUE(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Diploma Museos de España"""
POLL_INTERVAL_DAYS = 365
SIG = "DMUE"
ACTIVITY = "DMUE"
DATA_URL = "https://dmue.radiogalena.es/nom_dmue.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -22,7 +22,7 @@ class DMUE(FileDownloadSIGRefDataProvider):
for row in csv.reader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
if len(row) > 1 and row[0] and row[1]:
new_data.append(
SIGRef(sig=self.SIG, id=row[0].strip(), name=row[1].strip(), ref_type=SIGRefType.BUILDING)
ActivityRef(sig=self.ACTIVITY, id=row[0].strip(), name=row[1].strip(), ref_type=ActivityRefType.BUILDING)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
@@ -30,7 +30,7 @@ class DMUE(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -3,20 +3,20 @@ from time import sleep
import pandas as pd
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import FileDownloadActivityRefDataProvider
class DMVE(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Diploma Monumentos y Vestigios de España"""
class DMVE(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Diploma Monumentos y Vestigios de España"""
POLL_INTERVAL_DAYS = 365
SIG = "DMVE"
ACTIVITY = "DMVE"
DATA_URL = "https://www.acracb.org/dmve/descargas/General/directorio_referencias_dmve.xls"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -36,14 +36,14 @@ class DMVE(FileDownloadSIGRefDataProvider):
continue
if ref and name:
new_data.append(SIGRef(sig=self.SIG, id=ref.strip(), name=name.strip(), ref_type=SIGRefType.BUILDING))
new_data.append(ActivityRef(sig=self.ACTIVITY, id=ref.strip(), name=name.strip(), ref_type=ActivityRefType.BUILDING))
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -1,19 +1,19 @@
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import FileDownloadActivityRefDataProvider
class DTMBA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Diploma Teatri Musei Belle Arti"""
class DTMBA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Diploma Teatri Musei Belle Arti"""
POLL_INTERVAL_DAYS = 30
SIG = "DTMBA"
ACTIVITY = "DTMBA"
DATA_URL = "https://www.iu1fig.com/share/iz0eik/dtmba/export.php"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -21,14 +21,14 @@ class DTMBA(FileDownloadSIGRefDataProvider):
split = row.split(";")
ref_id = split[0]
ref_name = split[1]
new_data.append(SIGRef(sig=self.SIG, id=ref_id, name=ref_name, ref_type=SIGRefType.BUILDING))
new_data.append(ActivityRef(sig=self.ACTIVITY, id=ref_id, name=ref_name, ref_type=ActivityRefType.BUILDING))
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -3,20 +3,20 @@ from time import sleep
import pdfplumber
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import FileDownloadActivityRefDataProvider
class FEA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Diploma Faros de España"""
class FEA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Diploma Faros de España"""
POLL_INTERVAL_DAYS = 30
SIG = "FEA"
ACTIVITY = "FEA"
DATA_URL = "http://ea5ol.net/Lista%20Faros.pdf"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -41,15 +41,15 @@ class FEA(FileDownloadSIGRefDataProvider):
# prefix and just use FEA-1234 or FEA 1234, so we add both copies to the database.
ref_id_1 = row[0].strip()
ref_id_2 = ref_id_1.replace("D-", "FEA-").replace("E-", "FEA-")
new_data.append(SIGRef(sig=self.SIG, id=ref_id_1, name=row[1].strip(), ref_type=SIGRefType.LIGHTHOUSE))
new_data.append(SIGRef(sig=self.SIG, id=ref_id_2, name=row[1].strip(), ref_type=SIGRefType.LIGHTHOUSE))
new_data.append(ActivityRef(sig=self.ACTIVITY, id=ref_id_1, name=row[1].strip(), ref_type=ActivityRefType.LIGHTHOUSE))
new_data.append(ActivityRef(sig=self.ACTIVITY, id=ref_id_2, name=row[1].strip(), ref_type=ActivityRefType.LIGHTHOUSE))
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -7,13 +7,14 @@ from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS
from core.url_data_cache import URLDataCache
from providers.sigrefdata.sig_ref_data_provider import SIGRefDataProvider
from providers.activityrefdata.activity_ref_data_provider import ActivityRefDataProvider
logger = logging.getLogger(__name__)
class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
"""Generic SIG ref data provider class for providers that fetch their data from the web by downloading a file."""
class FileDownloadActivityRefDataProvider(ActivityRefDataProvider):
"""Generic activity ref data provider class for providers that fetch their data from the web by downloading a
file."""
def __init__(self, sig_name, provider_config, url, poll_interval):
"""Set up the provider, note poll_interval is in *days*."""
@@ -21,13 +22,13 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
self._url = url
self._poll_interval = poll_interval
self._thread = None
self._url_data_cache = URLDataCache(f"sigrefdata_{sig_name}")
self._url_data_cache = URLDataCache(f"sigrefdata_{sig_name}") # cache dir name kept for continuity
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting.
logger.info(f"Set up query of {self.sig_name} SIG ref data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadSIGRefDataProvider-{self.sig_name}", daemon=True)
logger.info(f"Set up query of {self.sig_name} activity ref data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadActivityRefDataProvider-{self.sig_name}", daemon=True)
self._thread.start()
def stop(self):
@@ -35,7 +36,7 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
if self._thread:
self._thread.join(timeout=35)
if self._thread.is_alive():
logger.warning(f"{self.sig_name} SIG ref data worker thread did not exit on time and will be killed.")
logger.warning(f"{self.sig_name} activity ref data worker thread did not exit on time and will be killed.")
def _run(self):
while True:
@@ -47,37 +48,37 @@ class FileDownloadSIGRefDataProvider(SIGRefDataProvider):
try:
# Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
# caching, but just so continual restarts of the software during testing don't hammer the servers.
logger.debug(f"Downloading {self.sig_name} SIG ref data...")
logger.debug(f"Downloading {self.sig_name} activity ref data...")
http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
# Check response code was good
if http_response.ok:
# Pass off to the subclass for processing
new_data = self._http_response_to_data(http_response)
# Add the new data to the SIG Ref data store
# Add the new data to the activity ref data store
if new_data:
self._add_data(new_data)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
logger.debug(f"Received SIG ref data for {self.sig_name}")
logger.debug(f"Received activity ref data for {self.sig_name}")
else:
self.status = "Error"
logger.warning(f"HTTP {http_response.status_code} when downloading SIG ref data for {self.sig_name}.")
logger.warning(f"HTTP {http_response.status_code} when downloading activity ref data for {self.sig_name}.")
except ConnectionError:
self.status = "Error"
logger.warning(f"Connection error when downloading SIG ref data for {self.sig_name}.")
logger.warning(f"Connection error when downloading activity ref data for {self.sig_name}.")
except (ConnectTimeout, ReadTimeout):
self.status = "Error"
logger.warning(f"Timeout when downloading SIG ref data for {self.sig_name}.")
logger.warning(f"Timeout when downloading activity ref data for {self.sig_name}.")
except Exception:
self.status = "Error"
logger.exception(f"Exception in HTTP SIG Ref Data Provider ({self.sig_name})")
logger.exception(f"Exception in HTTP Activity Ref Data Provider ({self.sig_name})")
self._stop_event.wait(timeout=1)
def _http_response_to_data(self, http_response):
"""Convert an HTTP response returned by the server into SIG Ref data. The whole response is provided here so the
subclass implementations can check for HTTP status codes if necessary, and handle the response as JSON, CSV,
whatever the remote file actually is."""
"""Convert an HTTP response returned by the server into activity ref data. The whole response is provided here
so the subclass implementations can check for HTTP status codes if necessary, and handle the response as
JSON, CSV, whatever the remote file actually is."""
raise NotImplementedError("Subclasses must implement this method")
@@ -1,33 +1,33 @@
import csv
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class GMA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Global Mountain Activity"""
class GMA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Global Mountain Activity"""
POLL_INTERVAL_DAYS = 30
SIG = "GMA"
ACTIVITY = "GMA"
DATA_URL = "https://www.gma.rocks/download/summits.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"]
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=row.get("Name", None),
ref_type=SIGRefType.SUMMIT,
ref_type=ActivityRefType.SUMMIT,
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
@@ -43,7 +43,7 @@ class GMA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -1,22 +1,22 @@
import csv
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class ILLW(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for International Lighthouse & Lightship Weekend"""
class ILLW(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for International Lighthouse & Lightship Weekend"""
POLL_INTERVAL_DAYS = 30
SIG = "ILLW"
ACTIVITY = "ILLW"
DATA_URL = "https://www.gma.rocks/download/lighthouse.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -24,11 +24,11 @@ class ILLW(FileDownloadSIGRefDataProvider):
if "ILLW" in row and row["ILLW"] != "":
ref_id = row["ILLW"]
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=row.get("Name", None),
ref_type=SIGRefType.LIGHTHOUSE,
ref_type=ActivityRefType.LIGHTHOUSE,
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
@@ -41,7 +41,7 @@ class ILLW(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -3,24 +3,24 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
logger = logging.getLogger(__name__)
class IOTA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Islands on the Air"""
class IOTA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Islands on the Air"""
POLL_INTERVAL_DAYS = 365
SIG = "IOTA"
ACTIVITY = "IOTA"
DATA_URL = "https://www.iota-world.org/islands-on-the-air/downloads/download-file.html?path=groups.json"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -41,11 +41,11 @@ class IOTA(FileDownloadSIGRefDataProvider):
)
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=ref["name"],
ref_type=SIGRefType.ISLAND,
ref_type=ActivityRefType.ISLAND,
grid=grid,
latitude=latitude,
longitude=longitude,
@@ -57,7 +57,7 @@ class IOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
+14
View File
@@ -0,0 +1,14 @@
from providers.activityrefdata.pnp_kml_activity_ref_data_provider import (
ParksNPeaksKMLActivityRefDataProvider,
)
class KRMNPA(ParksNPeaksKMLActivityRefDataProvider):
"""Activity ref data provider for the Keith Roget Memorrial National Parks Award (KRMNPA)."""
POLL_INTERVAL_DAYS = 365
ACTIVITY = "KRMNPA"
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=KRMNPA&poiFormat=4"
def __init__(self, provider_config):
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
@@ -2,22 +2,22 @@ from time import sleep
from pyhamtools.locator import locator_to_latlong
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class LLOTA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Lagos y Lagunas on the Air"""
class LLOTA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Lagos y Lagunas on the Air"""
POLL_INTERVAL_DAYS = 7
SIG = "LLOTA"
ACTIVITY = "LLOTA"
DATA_URL = "https://llota.app/api/public/references"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -29,11 +29,11 @@ class LLOTA(FileDownloadSIGRefDataProvider):
ll = locator_to_latlong(grid)
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=str(ref["name"]),
ref_type=SIGRefType.LAKE,
ref_type=ActivityRefType.LAKE,
url=f"https://llota.app/list/ref/{ref_id}",
grid=grid,
latitude=ll[0],
@@ -46,7 +46,7 @@ class LLOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -0,0 +1,36 @@
import logging
from datetime import datetime
import pytz
from providers.activityrefdata.activity_ref_data_provider import ActivityRefDataProvider
logger = logging.getLogger(__name__)
class LocalFileActivityRefDataProvider(ActivityRefDataProvider):
"""Generic activity ref data provider class for providers that fetch their data from a local file on startup."""
def __init__(self, sig_name, provider_config, path):
super().__init__(sig_name, provider_config)
self._path = path
def start(self):
logger.debug(f"Loading {self.sig_name} activity ref data from file.")
try:
new_data = self._file_to_data(self._path)
if new_data:
self._add_data(new_data)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
else:
self.status = "Error"
logger.info(f"Failed to load activity ref data for {self.sig_name}")
except Exception:
self.status = "Error"
logger.exception(f"Exception in local file Activity Ref Data Provider ({self.sig_name})")
def _file_to_data(self, path):
"""Load a file on the given path and turn it into activity ref data."""
raise NotImplementedError("Subclasses must implement this method")
@@ -1,33 +1,33 @@
import csv
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class MOTA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Mills on the Air"""
class MOTA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Mills on the Air"""
POLL_INTERVAL_DAYS = 30
SIG = "MOTA"
ACTIVITY = "MOTA"
DATA_URL = "https://www.gma.rocks/download/mills.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()[1:]):
ref_id = row["Reference"]
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=row.get("Name", None),
ref_type=SIGRefType.MILL,
ref_type=ActivityRefType.MILL,
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None,
longitude=float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None,
@@ -40,7 +40,7 @@ class MOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -2,20 +2,20 @@ from time import sleep
from bs4 import BeautifulSoup
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import FileDownloadActivityRefDataProvider
class PGA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Polish Gmina Award"""
class PGA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Polish Gmina Award"""
POLL_INTERVAL_DAYS = 30
SIG = "PGA"
ACTIVITY = "PGA"
DATA_URL = "http://www.spga.pl/lista_pga2.php"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -38,11 +38,11 @@ class PGA(FileDownloadSIGRefDataProvider):
continue
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=name,
ref_type=SIGRefType.REGION,
ref_type=ActivityRefType.REGION,
)
)
@@ -51,7 +51,7 @@ class PGA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -4,16 +4,16 @@ from time import sleep
from fastkml import kml
from pyhamtools.locator import latlong_to_locator
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
"""Base class for SIG ref data providers that use parksnpeaks.org KML POI feeds and have references that use the
VKFF refs rather than their own system (i.e. KRMNPA and SANPCPA)."""
class ParksNPeaksKMLActivityRefDataProvider(FileDownloadActivityRefDataProvider):
"""Base class for activity ref data providers that use parksnpeaks.org KML POI feeds and have references that
use the VKFF refs rather than their own system (i.e. KRMNPA and SANPCPA)."""
REF_PATTERN = re.compile(r"VKFF-\d+")
@@ -40,11 +40,11 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
longitude, latitude = placemark.geometry.x, placemark.geometry.y
ref = SIGRef(
ref = ActivityRef(
sig=self.sig_name,
id=ref_id,
name=placemark.name,
ref_type=SIGRefType.PARK,
ref_type=ActivityRefType.PARK,
url=f"https://parksnpeaks.org/getPark.php?actPark={ref_id}",
latitude=latitude,
longitude=longitude,
@@ -58,7 +58,7 @@ class ParksNPeaksKMLSIGRefDataProvider(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
return new_data
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -1,33 +1,33 @@
import csv
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class POTA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Parks on the Air"""
class POTA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Parks on the Air"""
POLL_INTERVAL_DAYS = 7
SIG = "POTA"
ACTIVITY = "POTA"
DATA_URL = "https://pota.app/all_parks_ext.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"]
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=row.get("name", None),
ref_type=SIGRefType.PARK,
ref_type=ActivityRefType.PARK,
url=f"https://pota.app/#/park/{ref_id}",
grid=row.get("grid", None),
latitude=float(row["latitude"]) if "latitude" in row and row["latitude"] != "" else None,
@@ -40,7 +40,7 @@ class POTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
+14
View File
@@ -0,0 +1,14 @@
from providers.activityrefdata.pnp_kml_activity_ref_data_provider import (
ParksNPeaksKMLActivityRefDataProvider,
)
class SANPCPA(ParksNPeaksKMLActivityRefDataProvider):
"""Activity ref data provider for the South Australia National Parks and Conservation Parks Award (SANPCPA)."""
POLL_INTERVAL_DAYS = 365
ACTIVITY = "SANPCPA"
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=SANPCPA&poiFormat=4"
def __init__(self, provider_config):
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
@@ -1,33 +1,33 @@
import csv
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class SIOTA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Silos on the Air"""
class SIOTA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Silos on the Air"""
POLL_INTERVAL_DAYS = 30
SIG = "SIOTA"
ACTIVITY = "SIOTA"
DATA_URL = "https://www.silosontheair.com/data/silos.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["SILO_CODE"]
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=row.get("NAME", None),
ref_type=SIGRefType.SILO,
ref_type=ActivityRefType.SILO,
grid=row.get("LOCATOR", None),
latitude=float(row["LAT"]) if "LAT" in row else None,
longitude=float(row["LNG"]) if "LNG" in row else None,
@@ -39,7 +39,7 @@ class SIOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -3,22 +3,22 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class SOTA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Summits on the Air"""
class SOTA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Summits on the Air"""
POLL_INTERVAL_DAYS = 30
SIG = "SOTA"
ACTIVITY = "SOTA"
DATA_URL = "https://storage.sota.org.uk/summitslist.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -27,11 +27,11 @@ class SOTA(FileDownloadSIGRefDataProvider):
latitude = float(row["Latitude"]) if "Latitude" in row and row["Latitude"] != "" else None
longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
altitude = float(row["AltM"]) if "AltM" in row and row["AltM"] != "" else None
ref = SIGRef(
sig=self.SIG,
ref = ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=row.get("SummitName", None),
ref_type=SIGRefType.SUMMIT,
ref_type=ActivityRefType.SUMMIT,
url=f"https://www.sotadata.org.uk/en/summit/{ref_id}",
latitude=latitude,
longitude=longitude,
@@ -47,7 +47,7 @@ class SOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -1,20 +1,20 @@
import csv
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.local_file_sig_ref_data_provider import (
LocalFileSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.local_file_activity_ref_data_provider import (
LocalFileActivityRefDataProvider,
)
class Toilets(LocalFileSIGRefDataProvider):
"""SIG ref data provider for Toilets on the Air"""
class Toilets(LocalFileActivityRefDataProvider):
"""Activity ref data provider for Toilets on the Air"""
SIG = "Toilets"
ACTIVITY = "Toilets"
PATH = "datafiles/toilets.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.PATH)
super().__init__(self.ACTIVITY, provider_config, self.PATH)
def _file_to_data(self, path):
new_data = []
@@ -23,11 +23,11 @@ class Toilets(LocalFileSIGRefDataProvider):
dr = csv.DictReader(csv_data.splitlines())
for row in dr:
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=row["ref"],
name=row["ref"],
ref_type=SIGRefType.TOILET,
ref_type=ActivityRefType.TOILET,
latitude=float(row["lat"]),
longitude=float(row["lon"]),
)
@@ -1,33 +1,33 @@
import csv
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class Towers(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Towers on the Air"""
class Towers(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Towers on the Air"""
POLL_INTERVAL_DAYS = 30
SIG = "Towers"
ACTIVITY = "Towers"
DATA_URL = "https://wwtota.com/servis/generate_csv.php?ref=&filter=all"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
ref_id = row["Ref"]
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=row.get("Nazev", None),
ref_type=SIGRefType.TOWER,
ref_type=ActivityRefType.TOWER,
url=f"https://wwtota.com/seznam/karta_rozhledny.php?ref={ref_id}",
grid=row["Lokator"] if "Lokator" in row and row["Lokator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
@@ -40,7 +40,7 @@ class Towers(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -4,24 +4,24 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
logger = logging.getLogger(__name__)
class WCA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for World Castles Award"""
class WCA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for World Castles Award"""
POLL_INTERVAL_DAYS = 30
SIG = "WCA"
ACTIVITY = "WCA"
DATA_URL = "https://polo.ham2k.com/data/activities/wca/all-castles.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -42,11 +42,11 @@ class WCA(FileDownloadSIGRefDataProvider):
logger.debug(f"Encountered dodgy formatting in WCA CSV, skipping location data for {ref_id}")
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=row.get("CLEAN NAME", None),
ref_type=SIGRefType.CASTLE,
ref_type=ActivityRefType.CASTLE,
url=f"https://www.cqgma.org/zinfo.php?ref={ref_id}",
latitude=latitude,
longitude=longitude,
@@ -59,7 +59,7 @@ class WCA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -1,21 +1,21 @@
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class WOTA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Wainwrights on the Air"""
class WOTA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Wainwrights on the Air"""
POLL_INTERVAL_DAYS = 365
SIG = "WOTA"
ACTIVITY = "WOTA"
DATA_URL = "https://www.wota.org.uk/mapping/data/summits.json"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -29,12 +29,12 @@ class WOTA(FileDownloadSIGRefDataProvider):
url = f"https://www.wota.org.uk/MM_LDO-{number + 214!s}"
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=feature["properties"]["title"],
url=url,
ref_type=SIGRefType.SUMMIT,
ref_type=ActivityRefType.SUMMIT,
grid=feature["properties"]["qthLocator"],
latitude=feature["geometry"]["coordinates"][1],
longitude=feature["geometry"]["coordinates"][0],
@@ -47,7 +47,7 @@ class WOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -1,33 +1,33 @@
import csv
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class WWBOTA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Worldwide Bunkers on the Air"""
class WWBOTA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Worldwide Bunkers on the Air"""
POLL_INTERVAL_DAYS = 30
SIG = "WWBOTA"
ACTIVITY = "WWBOTA"
DATA_URL = "https://api.wwbota.org/bunkers/?format=CSV"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["Reference"]
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=row.get("Name", None),
ref_type=SIGRefType.BUNKER,
ref_type=ActivityRefType.BUNKER,
url=f"https://bunkerwiki.org/?s={ref_id}" if ref_id.startswith("B/G") else None,
grid=row["Locator"] if "Locator" in row and row["Locator"] != "" else None,
latitude=float(row["Lat"]) if "Lat" in row and row["Lat"] != "" else None,
@@ -40,7 +40,7 @@ class WWBOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -1,33 +1,33 @@
import csv
from time import sleep
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class WWFF(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for Worldwide Flora & Fauna"""
class WWFF(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for Worldwide Flora & Fauna"""
POLL_INTERVAL_DAYS = 30
SIG = "WWFF"
ACTIVITY = "WWFF"
DATA_URL = "https://wwff.co/wwff-data/wwff_directory.csv"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
for row in csv.DictReader(http_response.content.decode("utf-8-sig").splitlines()):
ref_id = row["reference"]
new_data.append(
SIGRef(
sig=self.SIG,
ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=row.get("name", None),
ref_type=SIGRefType.PARK,
ref_type=ActivityRefType.PARK,
url=f"https://wwff.co/directory/?showRef={ref_id}",
grid=row["iaruLocator"] if "iaruLocator" in row and row["iaruLocator"] != "-" else None,
latitude=float(row["latitude"])
@@ -44,7 +44,7 @@ class WWFF(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
@@ -2,22 +2,22 @@ from time import sleep
from pyhamtools.locator import latlong_to_locator
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from providers.sigrefdata.file_download_sig_ref_data_provider import (
FileDownloadSIGRefDataProvider,
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from providers.activityrefdata.file_download_activity_ref_data_provider import (
FileDownloadActivityRefDataProvider,
)
class ZLOTA(FileDownloadSIGRefDataProvider):
"""SIG ref data provider for New Zealand on the Air"""
class ZLOTA(FileDownloadActivityRefDataProvider):
"""Activity ref data provider for New Zealand on the Air"""
POLL_INTERVAL_DAYS = 30
SIG = "ZLOTA"
ACTIVITY = "ZLOTA"
DATA_URL = "https://ontheair.nz/assets/assets.json"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
def _http_response_to_data(self, http_response):
new_data = []
@@ -28,12 +28,12 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
latitude = ref["latitude"]
longitude = ref["longitude"]
try:
ref_type = SIGRefType(ref["asset_type"].title().upper())
ref_type = ActivityRefType(ref["asset_type"].title().upper())
except ValueError:
ref_type = None
new_ref = SIGRef(
sig=self.SIG,
new_ref = ActivityRef(
sig=self.ACTIVITY,
id=ref_id,
name=ref["name"],
ref_type=ref_type,
@@ -58,7 +58,7 @@ class ZLOTA(FileDownloadSIGRefDataProvider):
if self._stop_event.is_set():
break
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
# is available for other threads e.g. the web server.
sleep(0.001)
+2 -2
View File
@@ -4,8 +4,8 @@ import pytz
from bs4 import BeautifulSoup
from core.enums import AlertType
from data.activity_ref import ActivityRef
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
@@ -56,7 +56,7 @@ class BOTA(HTTPAlertProvider):
alert = Alert(
source=self.name,
dx_calls=[dx_call],
sig_refs=[SIGRef(id=ref_name, sig="BOTA")],
sig_refs=[ActivityRef(id=ref_name, sig="BOTA")],
start_time=date_time.timestamp(),
alert_type=AlertType.XOTA,
)
+3 -3
View File
@@ -3,8 +3,8 @@ from datetime import datetime
import pytz
from core.enums import AlertType
from data.activity_ref import ActivityRef
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
@@ -35,9 +35,9 @@ class Hamsat(HTTPAlertProvider):
dx_calls=[source_alert["callsign"].upper()],
freqs_modes=freqs_modes,
comment=source_alert["comment"],
# Fudge a SIG ref to provide the remaining bits of data we need: the satellite and the operator's grid
# Fudge an activity ref to provide the remaining bits of data we need: the satellite and the operator's grid
sig_refs=[
SIGRef(
ActivityRef(
sig="AMSAT",
id=f"{source_alert['satellite']['name']} from {source_alert['grids'][0]}",
)
+15 -15
View File
@@ -4,8 +4,8 @@ from datetime import datetime
import pytz
from core.enums import AlertType
from data.activity_ref import ActivityRef
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
logger = logging.getLogger(__name__)
@@ -25,23 +25,23 @@ class ParksNPeaks(HTTPAlertProvider):
# Iterate through source data
for source_alert in http_response.json():
# Calculate some things
sig = source_alert["Class"].upper()
activity = source_alert["Class"].upper()
if " - " in source_alert["Location"]:
split = source_alert["Location"].split(" - ")
sig_ref = split[0]
sig_ref_name = split[1]
ref_id = split[0]
ref_name = split[1]
else:
sig_ref = source_alert["WWFFID"]
sig_ref_name = source_alert["Location"]
ref_id = source_alert["WWFFID"]
ref_name = source_alert["Location"]
start_time = (
datetime.strptime(source_alert["alTime"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC).timestamp()
)
sigrefs = []
# PnP can give us an alert of class "QRP" which is the only one that's not a real SIG in Spothole's list,
# so mask this out if we got it.
if sig != "QRP":
sigrefs = [SIGRef(id=sig_ref, sig=sig, name=sig_ref_name)]
activity_refs = []
# PnP can give us an alert of class "QRP" which is the only one that's not a real activity in Spothole's
# list, so mask this out if we got it.
if activity != "QRP":
activity_refs = [ActivityRef(id=ref_id, sig=activity, name=ref_name)]
# Convert to our alert format
alert = Alert(
@@ -50,13 +50,13 @@ class ParksNPeaks(HTTPAlertProvider):
dx_calls=[source_alert["CallSign"].upper()],
freqs_modes=f"{source_alert['Freq']} {source_alert['MODE']}",
comment=source_alert["Comments"],
sig_refs=sigrefs,
sig_refs=activity_refs,
start_time=start_time,
alert_type=AlertType.XOTA,
)
# Log a warning for the developer if PnP gives us an unknown programme we've never seen before
if sig and sig not in [
if activity and activity not in [
"POTA",
"SOTA",
"WWFF",
@@ -68,11 +68,11 @@ class ParksNPeaks(HTTPAlertProvider):
"LLOTA",
"QRP",
]:
logger.warning(f"PNP alert found with sig {sig}, developer needs to add support for this!")
logger.warning(f"PNP alert found with activity {activity}, developer needs to add support for this!")
# If this is POTA, SOTA or WWFF data we already have it through other means, so ignore. Otherwise, add to
# the alert list. Note that while ZLOTA has its own spots API, it doesn't have its own alerts API. So that
# means the PnP *spot* provider rejects ZLOTA spots here, but the PnP *alerts* provider here allows ZLOTA.
if sig not in ["POTA", "SOTA", "WWFF"]:
if activity not in ["POTA", "SOTA", "WWFF"]:
new_alerts.append(alert)
return new_alerts
+2 -2
View File
@@ -3,8 +3,8 @@ from datetime import datetime
import pytz
from core.enums import AlertType
from data.activity_ref import ActivityRef
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
@@ -29,7 +29,7 @@ class POTA(HTTPAlertProvider):
freqs_modes=source_alert["frequencies"],
comment=source_alert["comments"],
sig_refs=[
SIGRef(
ActivityRef(
id=source_alert["reference"],
sig="POTA",
name=source_alert["name"],
+2 -2
View File
@@ -3,8 +3,8 @@ from datetime import datetime
import pytz
from core.enums import AlertType
from data.activity_ref import ActivityRef
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
@@ -35,7 +35,7 @@ class SOTA(HTTPAlertProvider):
freqs_modes=source_alert["frequency"],
comment=source_alert["comments"],
sig_refs=[
SIGRef(
ActivityRef(
id=f"{source_alert['associationCode']}/{source_alert['summitCode']}",
sig="SOTA",
name=summit_name,
+2 -2
View File
@@ -7,8 +7,8 @@ import pytz
from rss_parser import Parser as RSSParser
from rss_parser.models.rss import RSS
from data.activity_ref import ActivityRef
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
logger = logging.getLogger(__name__)
@@ -74,7 +74,7 @@ class WOTA(HTTPAlertProvider):
dx_calls=[dx_call],
freqs_modes=freqs_modes,
comment=comment,
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
sig_refs=[ActivityRef(id=ref, sig="WOTA", name=ref_name)] if ref else [],
start_time=time.timestamp(),
)
+2 -2
View File
@@ -3,8 +3,8 @@ from datetime import datetime
import pytz
from core.enums import AlertType
from data.activity_ref import ActivityRef
from data.alert import Alert
from data.sig_ref import SIGRef
from providers.alert.http_alert_provider import HTTPAlertProvider
@@ -28,7 +28,7 @@ class WWFF(HTTPAlertProvider):
dx_calls=[source_alert["activator_call"].upper()],
freqs_modes=f"{source_alert['band']} {source_alert['mode']}",
comment=source_alert["remarks"],
sig_refs=[SIGRef(id=source_alert["reference"], sig="WWFF")],
sig_refs=[ActivityRef(id=source_alert["reference"], sig="WWFF")],
start_time=datetime.strptime(source_alert["utc_start"], "%Y-%m-%d %H:%M:%S")
.replace(tzinfo=pytz.UTC)
.timestamp(),
-14
View File
@@ -1,14 +0,0 @@
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import (
ParksNPeaksKMLSIGRefDataProvider,
)
class KRMNPA(ParksNPeaksKMLSIGRefDataProvider):
"""SIG ref data provider for the Keith Roget Memorrial National Parks Award (KRMNPA)."""
POLL_INTERVAL_DAYS = 365
SIG = "KRMNPA"
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=KRMNPA&poiFormat=4"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
@@ -1,36 +0,0 @@
import logging
from datetime import datetime
import pytz
from providers.sigrefdata.sig_ref_data_provider import SIGRefDataProvider
logger = logging.getLogger(__name__)
class LocalFileSIGRefDataProvider(SIGRefDataProvider):
"""Generic SIG ref data provider class for providers that fetch their data from a local file on startup."""
def __init__(self, sig, provider_config, path):
super().__init__(sig, provider_config)
self._path = path
def start(self):
logger.debug(f"Loading {self.sig_name} SIG ref data from file.")
try:
new_data = self._file_to_data(self._path)
if new_data:
self._add_data(new_data)
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
else:
self.status = "Error"
logger.info(f"Failed to load SIG ref data for {self.sig_name}")
except Exception:
self.status = "Error"
logger.exception(f"Exception in local file SIG Ref Data Provider ({self.sig_name})")
def _file_to_data(self, path):
"""Load a file on the given path and turn it into SIG Ref data."""
raise NotImplementedError("Subclasses must implement this method")
-14
View File
@@ -1,14 +0,0 @@
from providers.sigrefdata.pnp_kml_sig_ref_data_provider import (
ParksNPeaksKMLSIGRefDataProvider,
)
class SANPCPA(ParksNPeaksKMLSIGRefDataProvider):
"""SIG ref data provider for the South Australia National Parks and Conservation Parks Award (SANPCPA)."""
POLL_INTERVAL_DAYS = 365
SIG = "SANPCPA"
DATA_URL = "https://parksnpeaks.org/getPOI.php?poiClass=SANPCPA&poiFormat=4"
def __init__(self, provider_config):
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
+14 -14
View File
@@ -4,9 +4,9 @@ from datetime import datetime
import pytz
from core.constants import HTTP_HEADERS
from core.enums import Mode, SIGRefType
from core.enums import ActivityRefType, Mode
from core.url_data_cache import URLDataCache
from data.sig_ref import SIGRef
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -69,7 +69,7 @@ class GMA(HTTPSpotProvider):
mode=Mode.from_name(source_spot["MODE"].upper()) if "<>" not in source_spot["MODE"] else None,
comment=source_spot["TEXT"],
sig_refs=[
SIGRef(
ActivityRef(
id=source_spot["REF"],
sig="",
name=source_spot["NAME"],
@@ -83,7 +83,7 @@ class GMA(HTTPSpotProvider):
qrt=source_spot["QRG"] == "QRT",
)
# GMA doesn't give what programme (SIG) the reference is for until we separately look it up.
# GMA doesn't give what programme (activity) the reference is for until we separately look it up.
if "REF" in source_spot:
try:
ref_response = self._url_data_cache.get(
@@ -114,31 +114,31 @@ class GMA(HTTPSpotProvider):
match ref_info["reftype"]:
case "Summit":
spot.sig_refs[0].sig = "GMA"
spot.sig_refs[0].ref_type = SIGRefType.SUMMIT
spot.sig_refs[0].ref_type = ActivityRefType.SUMMIT
spot.sig = "GMA"
case "IOTA Island":
spot.sig_refs[0].sig = "IOTA"
spot.sig_refs[0].ref_type = SIGRefType.ISLAND
spot.sig_refs[0].ref_type = ActivityRefType.ISLAND
spot.sig = "IOTA"
case "GMA Island":
spot.sig_refs[0].sig = "GMA Islands"
spot.sig_refs[0].ref_type = SIGRefType.ISLAND
spot.sig_refs[0].ref_type = ActivityRefType.ISLAND
spot.sig = "GMA Islands"
case "Lighthouse (ILLW)":
spot.sig_refs[0].sig = "ILLW"
spot.sig_refs[0].ref_type = SIGRefType.LIGHTHOUSE
spot.sig_refs[0].ref_type = ActivityRefType.LIGHTHOUSE
spot.sig = "ILLW"
case "Lighthouse (ARLHS)":
spot.sig_refs[0].sig = "ARLHS"
spot.sig_refs[0].ref_type = SIGRefType.LIGHTHOUSE
spot.sig_refs[0].ref_type = ActivityRefType.LIGHTHOUSE
spot.sig = "ARLHS"
case "Castle":
spot.sig_refs[0].sig = "WCA"
spot.sig_refs[0].ref_type = SIGRefType.CASTLE
spot.sig_refs[0].ref_type = ActivityRefType.CASTLE
spot.sig = "WCA"
case "Mill":
spot.sig_refs[0].sig = "MOTA"
spot.sig_refs[0].ref_type = SIGRefType.MILL
spot.sig_refs[0].ref_type = ActivityRefType.MILL
spot.sig = "MOTA"
case _:
logger.warning(
@@ -158,7 +158,7 @@ class GMA(HTTPSpotProvider):
)
except Exception:
logger.exception(
f"Exception when looking up {self.REF_INFO_URL_ROOT}{source_spot['REF']}, SIG data will not be populated for the spot."
f"Exception when looking up {self.REF_INFO_URL_ROOT}{source_spot['REF']}, activity data will not be populated for the spot."
)
# Add to our list. Don't worry about de-duping, removing old spots etc. at this point;
@@ -169,8 +169,8 @@ class GMA(HTTPSpotProvider):
return new_spots
def can_submit_spot(self, sig):
return sig == "GMA"
def can_submit_spot(self, activity):
return activity == "GMA"
def submit_spot(self, spot, credentials):
# TODO: Implement.
+6 -6
View File
@@ -7,8 +7,8 @@ import requests
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS
from core.enums import Mode, SIGRefType
from data.sig_ref import SIGRef
from core.enums import ActivityRefType, Mode
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -64,13 +64,13 @@ class HEMA(HTTPSpotProvider):
comment=spotter_comment_match.group(2),
sig="HEMA",
sig_refs=[
SIGRef(
ActivityRef(
id=spot_items[3].upper(),
sig="HEMA",
name=spot_items[4],
latitude=float(spot_items[7]),
longitude=float(spot_items[8]),
ref_type=SIGRefType.SUMMIT,
ref_type=ActivityRefType.SUMMIT,
)
],
time=datetime.strptime(spot_items[0], "%d/%m/%Y %H:%M")
@@ -89,8 +89,8 @@ class HEMA(HTTPSpotProvider):
logger.warning("Connection error when accessing HEMA spots API.")
return new_spots
def can_submit_spot(self, sig):
return sig == "HEMA"
def can_submit_spot(self, activity):
return activity == "HEMA"
def submit_spot(self, spot, credentials):
# TODO: Implement. Currently blocked awaiting their API team to make a change to allow us to spot with a
+4 -4
View File
@@ -1,7 +1,7 @@
from datetime import datetime
from core.enums import Mode, SIGRefType
from data.sig_ref import SIGRef
from core.enums import ActivityRefType, Mode
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -36,11 +36,11 @@ class LLOTA(HTTPSpotProvider):
comment=comment,
sig="LLOTA",
sig_refs=[
SIGRef(
ActivityRef(
id=source_spot["reference"],
sig="LLOTA",
name=source_spot["reference_name"],
ref_type=SIGRefType.LAKE,
ref_type=ActivityRefType.LAKE,
)
],
time=datetime.fromisoformat(source_spot["updated_at"].replace("Z", "+00:00")).timestamp(),
+18 -18
View File
@@ -7,7 +7,7 @@ import requests
from core.constants import HTTP_HEADERS
from core.enums import Mode
from data.sig_ref import SIGRef
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -20,7 +20,7 @@ class ParksNPeaks(HTTPSpotProvider):
POLL_INTERVAL_SEC = 120
SPOTS_URL = "https://www.parksnpeaks.org/api/ALL"
SUBMIT_URL = "https://www.parksnpeaks.org/api/SPOT/"
SUBMITTABLE_SIGS = [
SUBMITTABLE_ACTIVITIES = [
"POTA",
"SOTA",
"WWFF",
@@ -64,26 +64,26 @@ class ParksNPeaks(HTTPSpotProvider):
if not spot.de_call and m:
spot.de_call = str(m.group(1))
# Record SIG information. Sometimes we get a "SIG" of "QRP", which we ignore as it's not a programme with a
# defined set of references
sig = source_spot["actClass"].upper()
sig_ref = source_spot["actSiteID"]
if sig and sig != "" and sig != "QRP" and sig_ref and sig_ref != "":
spot.sig = sig
sig_refs = [
SIGRef(
# Record activity information. Sometimes we get an activity of "QRP", which we ignore as it's not a
# programme with a defined set of references
activity = source_spot["actClass"].upper()
ref_id = source_spot["actSiteID"]
if activity and activity != "" and activity != "QRP" and ref_id and ref_id != "":
spot.sig = activity
activity_refs = [
ActivityRef(
id=source_spot["actSiteID"],
sig=source_spot["actClass"].upper(),
)
]
spot.sig_refs = sig_refs
spot.sig_refs = activity_refs
# Free text location is not present in all spots, so only add it if it's set
if "actLocation" in source_spot and source_spot["actLocation"] != "":
sig_refs[0].name = source_spot["actLocation"]
activity_refs[0].name = source_spot["actLocation"]
# Log a warning for the developer if PnP gives us an unknown programme we've never seen before
if sig not in [
if activity not in [
"POTA",
"SOTA",
"WWFF",
@@ -94,14 +94,14 @@ class ParksNPeaks(HTTPSpotProvider):
"SANPCPA",
"LLOTA",
]:
logger.warning(f"PNP spot found with sig {sig}, developer needs to add support for this!")
logger.warning(f"PNP spot found with activity {activity}, developer needs to add support for this!")
# Add new spot to the list
new_spots.append(spot)
return new_spots
def can_submit_spot(self, sig):
return sig in self.SUBMITTABLE_SIGS
def can_submit_spot(self, activity):
return activity in self.SUBMITTABLE_ACTIVITIES
def submit_spot(self, spot, credentials):
# TODO test this works
@@ -111,11 +111,11 @@ class ParksNPeaks(HTTPSpotProvider):
raise ValueError(
"Parks N Peaks user ID and API key are required. Get yours from your Parks N Peaks account."
)
sig_ref = spot.sig_refs[0].id if spot.sig_refs else ""
ref_id = spot.sig_refs[0].id if spot.sig_refs else ""
body = {
"actClass": spot.sig or "",
"actCallsign": spot.dx_call,
"actSite": sig_ref,
"actSite": ref_id,
"mode": spot.mode or "",
"freq": str(spot.freq / 1000000.0),
"comments": spot.comment or "",
+6 -6
View File
@@ -4,8 +4,8 @@ import pytz
import requests
from core.constants import HTTP_HEADERS
from core.enums import Mode, SIGRefType
from data.sig_ref import SIGRef
from core.enums import ActivityRefType, Mode
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -35,13 +35,13 @@ class POTA(HTTPSpotProvider):
comment=source_spot["comments"],
sig="POTA",
sig_refs=[
SIGRef(
ActivityRef(
id=source_spot["reference"],
sig="POTA",
name=source_spot["name"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
ref_type=SIGRefType.PARK
ref_type=ActivityRefType.PARK
)
],
time=datetime.strptime(source_spot["spotTime"], "%Y-%m-%dT%H:%M:%S")
@@ -57,8 +57,8 @@ class POTA(HTTPSpotProvider):
new_spots.append(spot)
return new_spots
def can_submit_spot(self, sig):
return sig == "POTA"
def can_submit_spot(self, activity):
return activity == "POTA"
def submit_spot(self, spot, credentials):
sig_ref = spot.sig_refs[0].id if spot.sig_refs else None
+6 -6
View File
@@ -5,8 +5,8 @@ import requests
from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
from core.constants import HTTP_HEADERS
from core.enums import Mode, SIGRefType
from data.sig_ref import SIGRef
from core.enums import ActivityRefType, Mode
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -58,14 +58,14 @@ class SOTA(HTTPSpotProvider):
comment=source_spot["comments"],
sig="SOTA",
sig_refs=[
SIGRef(
ActivityRef(
id=source_spot["summitCode"],
sig="SOTA",
name=source_spot["summitName"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
activation_score=source_spot["points"],
ref_type=SIGRefType.SUMMIT,
ref_type=ActivityRefType.SUMMIT,
)
],
dx_latitude=source_spot["latitude"],
@@ -82,8 +82,8 @@ class SOTA(HTTPSpotProvider):
logger.warning("Timeout when accessing SOTA spots API.")
return new_spots
def can_submit_spot(self, sig):
return sig == "SOTA"
def can_submit_spot(self, activity):
return activity == "SOTA"
def submit_spot(self, spot, credentials):
# TODO test this method works
+2 -2
View File
@@ -60,8 +60,8 @@ class SpotProvider:
raise NotImplementedError("Subclasses must implement this method")
def can_submit_spot(self, sig):
"""Return True if this provider supports submitting spots upstream for the given SIG."""
def can_submit_spot(self, activity):
"""Return True if this provider supports submitting spots upstream for the given activity."""
return False
+7 -7
View File
@@ -4,8 +4,8 @@ from datetime import datetime
import requests
from core.constants import HTTP_HEADERS
from core.enums import LocationSourceForSpot, Mode, SIGRefType
from data.sig_ref import SIGRef
from core.enums import ActivityRefType, LocationSourceForSpot, Mode
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -60,15 +60,15 @@ class Tiles(HTTPSpotProvider):
comment=source_spot["notes"],
sig="Tiles",
# Tiles spots can include POTA & SOTA references, but ignore those on the basis that we will get them separately from the POTA/SOTA providers anyway.
# Just take the grid reference itself as the single Tiles SIG reference.
# Just take the grid reference itself as the single Tiles activity reference.
sig_refs=[
SIGRef(
ActivityRef(
id=source_spot["maidenhead_grid"],
sig="Tiles",
name=source_spot["maidenhead_grid"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
ref_type=SIGRefType.GRID
ref_type=ActivityRefType.GRID
)
],
time=datetime.fromisoformat(source_spot["created_at"].replace("Z", "+00:00")).timestamp(),
@@ -83,8 +83,8 @@ class Tiles(HTTPSpotProvider):
new_spots.append(spot)
return new_spots
def can_submit_spot(self, sig):
return sig == "Tiles"
def can_submit_spot(self, activity):
return activity == "Tiles"
def submit_spot(self, spot, credentials):
# Tiles on the air currently only supports *self* spots
+3 -3
View File
@@ -3,8 +3,8 @@ from datetime import datetime
import pytz
from core.enums import SIGRefType
from data.sig_ref import SIGRef
from core.enums import ActivityRefType
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -35,7 +35,7 @@ class Towers(HTTPSpotProvider):
freq=likely_freq,
comment=source_spot["comment"],
sig="Towers",
sig_refs=[SIGRef(id=source_spot["ref"], sig="Towers", ref_type=SIGRefType.TOWER)],
sig_refs=[ActivityRef(id=source_spot["ref"], sig="Towers", ref_type=ActivityRefType.TOWER)],
time=datetime.strptime(response_json["updated"][:10] + source_spot["time"], "%Y-%m-%d%H:%M")
.replace(tzinfo=pytz.utc)
.timestamp(),
+5 -5
View File
@@ -8,8 +8,8 @@ import pytz
from rss_parser import Parser
from rss_parser.models.rss import RSS
from core.enums import Mode, SIGRefType
from data.sig_ref import SIGRef
from core.enums import ActivityRefType, Mode
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -91,7 +91,7 @@ class WOTA(HTTPSpotProvider):
mode=Mode.from_name(mode),
comment=comment,
sig="WOTA",
sig_refs=[SIGRef(id=ref, sig="WOTA", name=ref_name, ref_type=SIGRefType.SUMMIT)] if ref else [],
sig_refs=[ActivityRef(id=ref, sig="WOTA", name=ref_name, ref_type=ActivityRefType.SUMMIT)] if ref else [],
time=time.timestamp(),
)
@@ -104,8 +104,8 @@ class WOTA(HTTPSpotProvider):
return new_spots
def can_submit_spot(self, sig):
return sig == "WOTA"
def can_submit_spot(self, activity):
return activity == "WOTA"
def submit_spot(self, spot, credentials):
# TODO Ask M5TEA if he's happy to share how this is done from his app
+7 -7
View File
@@ -1,8 +1,8 @@
import json
from datetime import datetime
from core.enums import Mode, SIGRefType
from data.sig_ref import SIGRef
from core.enums import ActivityRefType, Mode
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.sse_spot_provider import SSESpotProvider
@@ -21,15 +21,15 @@ class WWBOTA(SSESpotProvider):
# n-fer activations.
refs = []
for ref in source_spot["references"]:
sigref = SIGRef(
activity_ref = ActivityRef(
id=ref["reference"],
sig="WWBOTA",
name=ref["name"],
latitude=ref["lat"],
longitude=ref["long"],
ref_type=SIGRefType.BUNKER,
ref_type=ActivityRefType.BUNKER,
)
refs.append(sigref)
refs.append(activity_ref)
spot = Spot(
source=self.name,
@@ -52,8 +52,8 @@ class WWBOTA(SSESpotProvider):
# WWBOTA does support a special "Test" spot type, we need to avoid adding that.
return spot if source_spot["type"] != "Test" else None
def can_submit_spot(self, sig):
return sig == "WWBOTA"
def can_submit_spot(self, activity):
return activity == "WWBOTA"
def submit_spot(self, spot, credentials):
# TODO: Implement. WWBOTA API docs cover this: https://api.wwbota.org/#tag/Spots/operation/create_spot_spots__post
+6 -6
View File
@@ -2,8 +2,8 @@ from datetime import datetime
import pytz
from core.enums import Mode, SIGRefType
from data.sig_ref import SIGRef
from core.enums import ActivityRefType, Mode
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -32,13 +32,13 @@ class WWFF(HTTPSpotProvider):
comment=source_spot["remarks"],
sig="WWFF",
sig_refs=[
SIGRef(
ActivityRef(
id=source_spot["reference"],
sig="WWFF",
name=source_spot["reference_name"],
latitude=source_spot["latitude"],
longitude=source_spot["longitude"],
ref_type=SIGRefType.PARK
ref_type=ActivityRefType.PARK
)
],
time=datetime.fromtimestamp(source_spot["spot_time"], tz=pytz.UTC).timestamp(),
@@ -51,8 +51,8 @@ class WWFF(HTTPSpotProvider):
new_spots.append(spot)
return new_spots
def can_submit_spot(self, sig):
return sig == "WWFF"
def can_submit_spot(self, activity):
return activity == "WWFF"
def submit_spot(self, spot, credentials):
# TODO: Implement. Spotting to WWFF should be possible, need to look up the Spotline docs or copy approach from
+13 -13
View File
@@ -4,43 +4,43 @@ from datetime import datetime
import pytz
from core.enums import Mode
from data.sig_ref import SIGRef
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.websocket_spot_provider import WebsocketSpotProvider
class XOTA(WebsocketSpotProvider):
"""Spot provider for servers based on the "xOTA" software at https://github.com/nischu/xOTA/
The provider typically doesn't give us a lat/lon or SIG explicitly, so our own config provides a SIG which we can
then use for lookups. This functionality is implemented for Toilets on the Air events, of which there are
several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA, which is why we also
provide a sig_ref_prefix in our config. This is applied to the reference ID, so e.g. "T-01" at C3 might become
"C3 T-01". This allows us to provide location lookups for TOTA at several conferences."""
The provider typically doesn't give us a lat/lon or activity explicitly, so our own config provides an activity
which we can then use for lookups. This functionality is implemented for Toilets on the Air events, of which
there are several - so a plain lookup of a "TOTA reference" doesn't make sense, it depends on which TOTA, which
is why we also provide a sig_ref_prefix in our config. This is applied to the reference ID, so e.g. "T-01" at C3
might become "C3 T-01". This allows us to provide location lookups for TOTA at several conferences."""
LOCATION_DATA = {}
SIG = None
ACTIVITY = None
def __init__(self, provider_config):
name = provider_config.get("name", "xOTA")
super().__init__(name, provider_config, provider_config["url"])
self.SIG = str(provider_config["sig"]) if "sig" in provider_config else None
self._sig_ref_prefix = str(provider_config["sig_ref_prefix"]) if "sig_ref_prefix" in provider_config else ""
self.ACTIVITY = str(provider_config["sig"]) if "sig" in provider_config else None
self._activity_ref_prefix = str(provider_config["sig_ref_prefix"]) if "sig_ref_prefix" in provider_config else ""
def _ws_message_to_spot(self, b):
string = b.decode("utf-8")
source_spot = json.loads(string)
ref_id = f"{self._sig_ref_prefix} {source_spot['reference']['title']}"
ref_id = f"{self._activity_ref_prefix} {source_spot['reference']['title']}"
spot = Spot(
source=self.name,
source_id=source_spot["id"],
dx_call=source_spot["stationCallSign"].upper(),
freq=float(source_spot["freq"]) * 1000,
mode=Mode.from_name(source_spot["mode"].upper()),
sig=self.SIG,
sig=self.ACTIVITY,
sig_refs=[
SIGRef(
ActivityRef(
id=ref_id,
sig=self.SIG or "",
sig=self.ACTIVITY or "",
url=source_spot["reference"]["website"],
)
],
+4 -4
View File
@@ -3,7 +3,7 @@ from datetime import datetime
import pytz
from core.enums import Mode
from data.sig_ref import SIGRef
from data.activity_ref import ActivityRef
from data.spot import Spot
from providers.spot.http_spot_provider import HTTPSpotProvider
@@ -37,7 +37,7 @@ class ZLOTA(HTTPSpotProvider):
comment=source_spot["comments"],
sig="ZLOTA",
sig_refs=[
SIGRef(
ActivityRef(
id=source_spot["reference"],
sig="ZLOTA",
name=source_spot["name"],
@@ -51,8 +51,8 @@ class ZLOTA(HTTPSpotProvider):
new_spots.append(spot)
return new_spots
def can_submit_spot(self, sig):
return sig == "ZLOTA"
def can_submit_spot(self, activity):
return activity == "ZLOTA"
def submit_spot(self, spot, credentials):
# TODO: Implement. Spotting to ZLOTA is supported via POST, see https://ontheair.nz/api