w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-09-24 16:24:32 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

First attempt at converting "sig" to "activity" for v3

Browse Source
This commit is contained in:
Ian Renton
2026-09-24 07:09:37 +01:00
parent 1d0129f7bb
commit d91fa70655
90 changed files with 822 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files
providers/activityrefdata/activity_ref_data_provider.py
+5 -6
View File
@@ -13,11 +13,10 @@ class ActivityRefDataProvider:
"""Generic activity reference data provider class. Subclasses of this query the individual URLs or files for
data."""
def __init__(self, sig_name, provider_config):
"""Constructor. Note the parameter and attribute are still named "sig_name" for consistency with the API's
"sig" field name."""
def __init__(self, activity_name, provider_config):
"""Constructor"""
self.sig_name = sig_name
self.activity_name = activity_name
self.enabled = provider_config["enabled"]
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
self.status = "Not Started" if self.enabled else "Disabled"
@@ -43,7 +42,7 @@ class ActivityRefDataProvider:
# transact()s is to fail if they can't get the lock (?!). This behaviour is fixed by retry=True.
with DATA_STORE.activity_refs.transact(retry=True):
for d in new_data:
DATA_STORE.activity_refs.set(f"{self.sig_name}:{d.id}", d)
DATA_STORE.activity_refs.set(f"{self.activity_name}:{d.id}", d)
# For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
# within the first few minutes of startup, we need a way to abort this expensive process of filling up the
@@ -52,4 +51,4 @@ class ActivityRefDataProvider:
break
self.reference_count = len(new_data)
logger.info(f"Loaded {self.reference_count} references for {self.sig_name} into the data store.")
logger.info(f"Loaded {self.reference_count} references for {self.activity_name} into the data store.")
providers/activityrefdata/arlhs.py
+1 -1
View File
@@ -25,7 +25,7 @@ class ARLHS(FileDownloadActivityRefDataProvider):
ref_id = row["ARLHS"]
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=row.get("Name", None),
ref_type=ActivityRefType.LIGHTHOUSE,
providers/activityrefdata/cota.py
+1 -1
View File
@@ -30,7 +30,7 @@ class COTA(FileDownloadActivityRefDataProvider):
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=name,
ref_type=ActivityRefType.CASTLE,
providers/activityrefdata/dce.py
+6 -1
View File
@@ -27,7 +27,12 @@ class DCE(FileDownloadActivityRefDataProvider):
for index, row in df.iterrows():
if row.iloc[0] and row.iloc[2]:
new_data.append(
ActivityRef(sig=self.ACTIVITY, id=row.iloc[0].strip(), name=row.iloc[2].strip(), ref_type=ActivityRefType.CASTLE)
ActivityRef(
activity=self.ACTIVITY,
id=row.iloc[0].strip(),
name=row.iloc[2].strip(),
ref_type=ActivityRefType.CASTLE,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
providers/activityrefdata/defe.py
+6 -1
View File
@@ -31,7 +31,12 @@ class DEFE(FileDownloadActivityRefDataProvider):
if row.iloc[0] and row.iloc[1]:
new_data.append(
ActivityRef(sig=self.ACTIVITY, id=row.iloc[0].strip(), name=row.iloc[1].strip(), ref_type=ActivityRefType.BUILDING)
ActivityRef(
activity=self.ACTIVITY,
id=row.iloc[0].strip(),
name=row.iloc[1].strip(),
ref_type=ActivityRefType.BUILDING,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
providers/activityrefdata/dme.py
+1 -1
View File
@@ -40,7 +40,7 @@ class DME(LocalFileActivityRefDataProvider):
)
ref = ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
ref_type=ActivityRefType.TOWN,
name=f"{row['NOMBRE_ACTUAL']}, {row['PROVINCIA']}",
providers/activityrefdata/dmue.py
+6 -1
View File
@@ -22,7 +22,12 @@ class DMUE(FileDownloadActivityRefDataProvider):
for row in csv.reader(http_response.content.decode("utf-8-sig").splitlines(), delimiter=";"):
if len(row) > 1 and row[0] and row[1]:
new_data.append(
ActivityRef(sig=self.ACTIVITY, id=row[0].strip(), name=row[1].strip(), ref_type=ActivityRefType.BUILDING)
ActivityRef(
activity=self.ACTIVITY,
id=row[0].strip(),
name=row[1].strip(),
ref_type=ActivityRefType.BUILDING,
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
providers/activityrefdata/dmve.py
+5 -1
View File
@@ -36,7 +36,11 @@ class DMVE(FileDownloadActivityRefDataProvider):
continue
if ref and name:
new_data.append(ActivityRef(sig=self.ACTIVITY, id=ref.strip(), name=name.strip(), ref_type=ActivityRefType.BUILDING))
new_data.append(
ActivityRef(
activity=self.ACTIVITY, id=ref.strip(), name=name.strip(), ref_type=ActivityRefType.BUILDING
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/activityrefdata/dtmba.py
+3 -1
View File
@@ -21,7 +21,9 @@ class DTMBA(FileDownloadActivityRefDataProvider):
split = row.split(";")
ref_id = split[0]
ref_name = split[1]
new_data.append(ActivityRef(sig=self.ACTIVITY, id=ref_id, name=ref_name, ref_type=ActivityRefType.BUILDING))
new_data.append(
ActivityRef(activity=self.ACTIVITY, id=ref_id, name=ref_name, ref_type=ActivityRefType.BUILDING)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/activityrefdata/fea.py
+10 -2
View File
@@ -41,8 +41,16 @@ class FEA(FileDownloadActivityRefDataProvider):
# prefix and just use FEA-1234 or FEA 1234, so we add both copies to the database.
ref_id_1 = row[0].strip()
ref_id_2 = ref_id_1.replace("D-", "FEA-").replace("E-", "FEA-")
new_data.append(ActivityRef(sig=self.ACTIVITY, id=ref_id_1, name=row[1].strip(), ref_type=ActivityRefType.LIGHTHOUSE))
new_data.append(ActivityRef(sig=self.ACTIVITY, id=ref_id_2, name=row[1].strip(), ref_type=ActivityRefType.LIGHTHOUSE))
new_data.append(
ActivityRef(
activity=self.ACTIVITY, id=ref_id_1, name=row[1].strip(), ref_type=ActivityRefType.LIGHTHOUSE
)
)
new_data.append(
ActivityRef(
activity=self.ACTIVITY, id=ref_id_2, name=row[1].strip(), ref_type=ActivityRefType.LIGHTHOUSE
)
)
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
# the data in this case
providers/activityrefdata/file_download_activity_ref_data_provider.py
+18 -12
View File
@@ -16,19 +16,21 @@ class FileDownloadActivityRefDataProvider(ActivityRefDataProvider):
"""Generic activity ref data provider class for providers that fetch their data from the web by downloading a
file."""
def __init__(self, sig_name, provider_config, url, poll_interval):
def __init__(self, activity_name, provider_config, url, poll_interval):
"""Set up the provider, note poll_interval is in *days*."""
super().__init__(sig_name, provider_config)
super().__init__(activity_name, provider_config)
self._url = url
self._poll_interval = poll_interval
self._thread = None
self._url_data_cache = URLDataCache(f"activity_ref_data_{sig_name}")
self._url_data_cache = URLDataCache(f"activity_ref_data_{activity_name}")
def start(self):
# Fire off the polling thread. It will poll immediately on startup, then sleep for poll_interval between
# subsequent polls, so start() returns immediately and the application can continue starting.
logger.info(f"Set up query of {self.sig_name} activity ref data every {self._poll_interval!s} days.")
self._thread = Thread(target=self._run, name=f"FileDownloadActivityRefDataProvider-{self.sig_name}", daemon=True)
logger.info(f"Set up query of {self.activity_name} activity ref data every {self._poll_interval!s} days.")
self._thread = Thread(
target=self._run, name=f"FileDownloadActivityRefDataProvider-{self.activity_name}", daemon=True
)
self._thread.start()
def stop(self):
@@ -36,7 +38,9 @@ class FileDownloadActivityRefDataProvider(ActivityRefDataProvider):
if self._thread:
self._thread.join(timeout=12)
if self._thread.is_alive():
logger.warning(f"{self.sig_name} activity ref data worker thread did not exit on time and will be killed.")
logger.warning(
f"{self.activity_name} activity ref data worker thread did not exit on time and will be killed."
)
def _run(self):
while True:
@@ -48,7 +52,7 @@ class FileDownloadActivityRefDataProvider(ActivityRefDataProvider):
try:
# Request data from API. Use the data cache (with a TTL of 1 day) here, not as the main mechanism for
# caching, but just so continual restarts of the software during testing don't hammer the servers.
logger.debug(f"Downloading {self.sig_name} activity ref data...")
logger.debug(f"Downloading {self.activity_name} activity ref data...")
http_response = self._url_data_cache.get(self._url, headers=HTTP_HEADERS)
# Check response code was good
if http_response.ok:
@@ -60,20 +64,22 @@ class FileDownloadActivityRefDataProvider(ActivityRefDataProvider):
self.status = "OK"
self.last_update_time = datetime.now(pytz.UTC)
logger.debug(f"Received activity ref data for {self.sig_name}")
logger.debug(f"Received activity ref data for {self.activity_name}")
else:
self.status = "Error"
logger.warning(f"HTTP {http_response.status_code} when downloading activity ref data for {self.sig_name}.")
logger.warning(
f"HTTP {http_response.status_code} when downloading activity ref data for {self.activity_name}."
)
except ConnectionError:
self.status = "Error"
logger.warning(f"Connection error when downloading activity ref data for {self.sig_name}.")
logger.warning(f"Connection error when downloading activity ref data for {self.activity_name}.")
except (ConnectTimeout, ReadTimeout):
self.status = "Error"
logger.warning(f"Timeout when downloading activity ref data for {self.sig_name}.")
logger.warning(f"Timeout when downloading activity ref data for {self.activity_name}.")
except Exception:
self.status = "Error"
logger.exception(f"Exception in HTTP Activity Ref Data Provider ({self.sig_name})")
logger.exception(f"Exception in HTTP Activity Ref Data Provider ({self.activity_name})")
self._stop_event.wait(timeout=1)
def _http_response_to_data(self, http_response):
providers/activityrefdata/gma.py
+1 -1
View File
@@ -24,7 +24,7 @@ class GMA(FileDownloadActivityRefDataProvider):
ref_id = row["Reference"]
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=row.get("Name", None),
ref_type=ActivityRefType.SUMMIT,
providers/activityrefdata/illw.py
+1 -1
View File
@@ -25,7 +25,7 @@ class ILLW(FileDownloadActivityRefDataProvider):
ref_id = row["ILLW"]
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=row.get("Name", None),
ref_type=ActivityRefType.LIGHTHOUSE,
providers/activityrefdata/iota.py
+1 -1
View File
@@ -42,7 +42,7 @@ class IOTA(FileDownloadActivityRefDataProvider):
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=ref["name"],
ref_type=ActivityRefType.ISLAND,
providers/activityrefdata/llota.py
+1 -1
View File
@@ -30,7 +30,7 @@ class LLOTA(FileDownloadActivityRefDataProvider):
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=str(ref["name"]),
ref_type=ActivityRefType.LAKE,
providers/activityrefdata/local_file_activity_ref_data_provider.py
+5 -5
View File
@@ -11,12 +11,12 @@ logger = logging.getLogger(__name__)
class LocalFileActivityRefDataProvider(ActivityRefDataProvider):
"""Generic activity ref data provider class for providers that fetch their data from a local file on startup."""
def __init__(self, sig_name, provider_config, path):
super().__init__(sig_name, provider_config)
def __init__(self, activity_name, provider_config, path):
super().__init__(activity_name, provider_config)
self._path = path
def start(self):
logger.debug(f"Loading {self.sig_name} activity ref data from file.")
logger.debug(f"Loading {self.activity_name} activity ref data from file.")
try:
new_data = self._file_to_data(self._path)
if new_data:
@@ -25,10 +25,10 @@ class LocalFileActivityRefDataProvider(ActivityRefDataProvider):
self.last_update_time = datetime.now(pytz.UTC)
else:
self.status = "Error"
logger.info(f"Failed to load activity ref data for {self.sig_name}")
logger.info(f"Failed to load activity ref data for {self.activity_name}")
except Exception:
self.status = "Error"
logger.exception(f"Exception in local file Activity Ref Data Provider ({self.sig_name})")
logger.exception(f"Exception in local file Activity Ref Data Provider ({self.activity_name})")
def _file_to_data(self, path):
"""Load a file on the given path and turn it into activity ref data."""
providers/activityrefdata/mota.py
+1 -1
View File
@@ -24,7 +24,7 @@ class MOTA(FileDownloadActivityRefDataProvider):
ref_id = row["Reference"]
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=row.get("Name", None),
ref_type=ActivityRefType.MILL,
providers/activityrefdata/pga.py
+1 -1
View File
@@ -39,7 +39,7 @@ class PGA(FileDownloadActivityRefDataProvider):
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=name,
ref_type=ActivityRefType.REGION,
providers/activityrefdata/pnp_kml_activity_ref_data_provider.py
+3 -3
View File
@@ -17,9 +17,9 @@ class ParksNPeaksKMLActivityRefDataProvider(FileDownloadActivityRefDataProvider)
REF_PATTERN = re.compile(r"VKFF-\d+")
def __init__(self, sig_name, provider_config, url, poll_interval):
def __init__(self, activity_name, provider_config, url, poll_interval):
"""Set up the provider, note poll_interval is in *days*."""
super().__init__(sig_name, provider_config, url, poll_interval)
super().__init__(activity_name, provider_config, url, poll_interval)
def _http_response_to_data(self, http_response):
new_data = []
@@ -41,7 +41,7 @@ class ParksNPeaksKMLActivityRefDataProvider(FileDownloadActivityRefDataProvider)
longitude, latitude = placemark.geometry.x, placemark.geometry.y
ref = ActivityRef(
sig=self.sig_name,
activity=self.activity_name,
id=ref_id,
name=placemark.name,
ref_type=ActivityRefType.PARK,
providers/activityrefdata/pota.py
+1 -1
View File
@@ -24,7 +24,7 @@ class POTA(FileDownloadActivityRefDataProvider):
ref_id = row["reference"]
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=row.get("name", None),
ref_type=ActivityRefType.PARK,
providers/activityrefdata/siota.py
+1 -1
View File
@@ -24,7 +24,7 @@ class SIOTA(FileDownloadActivityRefDataProvider):
ref_id = row["SILO_CODE"]
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=row.get("NAME", None),
ref_type=ActivityRefType.SILO,
providers/activityrefdata/sota.py
+1 -1
View File
@@ -28,7 +28,7 @@ class SOTA(FileDownloadActivityRefDataProvider):
longitude = float(row["Longitude"]) if "Longitude" in row and row["Longitude"] != "" else None
altitude = float(row["AltM"]) if "AltM" in row and row["AltM"] != "" else None
ref = ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=row.get("SummitName", None),
ref_type=ActivityRefType.SUMMIT,
providers/activityrefdata/toilets.py
+1 -1
View File
@@ -24,7 +24,7 @@ class Toilets(LocalFileActivityRefDataProvider):
for row in dr:
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=row["ref"],
name=row["ref"],
ref_type=ActivityRefType.TOILET,
providers/activityrefdata/towers.py
+1 -1
View File
@@ -24,7 +24,7 @@ class Towers(FileDownloadActivityRefDataProvider):
ref_id = row["Ref"]
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=row.get("Nazev", None),
ref_type=ActivityRefType.TOWER,
providers/activityrefdata/wca.py
+1 -1
View File
@@ -43,7 +43,7 @@ class WCA(FileDownloadActivityRefDataProvider):
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=row.get("CLEAN NAME", None),
ref_type=ActivityRefType.CASTLE,
providers/activityrefdata/wota.py
+1 -1
View File
@@ -30,7 +30,7 @@ class WOTA(FileDownloadActivityRefDataProvider):
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=feature["properties"]["title"],
url=url,
providers/activityrefdata/wwbota.py
+1 -1
View File
@@ -24,7 +24,7 @@ class WWBOTA(FileDownloadActivityRefDataProvider):
ref_id = row["Reference"]
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=row.get("Name", None),
ref_type=ActivityRefType.BUNKER,
providers/activityrefdata/wwff.py
+1 -1
View File
@@ -24,7 +24,7 @@ class WWFF(FileDownloadActivityRefDataProvider):
ref_id = row["reference"]
new_data.append(
ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=row.get("name", None),
ref_type=ActivityRefType.PARK,
providers/activityrefdata/zlota.py
+1 -1
View File
@@ -33,7 +33,7 @@ class ZLOTA(FileDownloadActivityRefDataProvider):
ref_type = None
new_ref = ActivityRef(
sig=self.ACTIVITY,
activity=self.ACTIVITY,
id=ref_id,
name=ref["name"],
ref_type=ref_type,