mirror of
https://git.ianrenton.com/ian/spothole.git
synced 2026-09-20 14:27:42 +00:00
56 lines
2.3 KiB
Python
56 lines
2.3 KiB
Python
import logging
|
|
from datetime import datetime
|
|
from threading import Event
|
|
|
|
import pytz
|
|
|
|
from core.data_store import DATA_STORE
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ActivityRefDataProvider:
|
|
"""Generic activity reference data provider class. Subclasses of this query the individual URLs or files for
|
|
data."""
|
|
|
|
def __init__(self, sig_name, provider_config):
|
|
"""Constructor. Note the parameter and attribute are still named "sig_name" for consistency with the API's
|
|
"sig" field name."""
|
|
|
|
self.sig_name = sig_name
|
|
self.enabled = provider_config["enabled"]
|
|
self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
|
|
self.status = "Not Started" if self.enabled else "Disabled"
|
|
self.reference_count = 0
|
|
self._stop_event = Event()
|
|
|
|
def start(self):
|
|
"""Start the provider. This should return immediately after spawning threads to access the remote resources"""
|
|
|
|
raise NotImplementedError("Subclasses must implement this method")
|
|
|
|
def stop(self):
|
|
"""Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
|
|
super()."""
|
|
|
|
self._stop_event.set()
|
|
|
|
def _add_data(self, new_data):
|
|
"""Add all the provided reference data objects to the data store."""
|
|
|
|
# with transact() batches all writes together to save making thousands of individual sqlite writes. However,
|
|
# that means that each provider holds the lock while it writes, and the default behaviour for other attempted
|
|
# transact()s is to fail if they can't get the lock (?!). This behaviour is fixed by retry=True.
|
|
with DATA_STORE.activity_refs.transact(retry=True):
|
|
for d in new_data:
|
|
DATA_STORE.activity_refs.set(f"{self.sig_name}:{d.id}", d)
|
|
|
|
# For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
|
|
# within the first few minutes of startup, we need a way to abort this expensive process of filling up the
|
|
# disk cache.
|
|
if self._stop_event.is_set():
|
|
break
|
|
|
|
self.reference_count = len(new_data)
|
|
logger.info(f"Loaded {self.reference_count} references for {self.sig_name} into the data store.")
|