Giant refactor to rebrand "SIG" as "Activity" anywhere that doesn't touch API or config file (which is to be addressed in a future breaking change). #147

providers/activityrefdata/activity_ref_data_provider.py

@@ -0,0 +1,53 @@
+import logging
+from datetime import datetime
+from threading import Event
+
+import pytz
+
+from core.data_store import DATA_STORE
+
+logger = logging.getLogger(__name__)
+
+
+class ActivityRefDataProvider:
+    """Generic activity reference data provider class. Subclasses of this query the individual URLs or files for
+    data."""
+
+    def __init__(self, sig_name, provider_config):
+        """Constructor. Note the parameter and attribute are still named "sig_name" for consistency with the API's
+        "sig" field name."""
+
+        self.sig_name = sig_name
+        self.enabled = provider_config["enabled"]
+        self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC)
+        self.status = "Not Started" if self.enabled else "Disabled"
+        self.reference_count = 0
+        self._stop_event = Event()
+
+    def start(self):
+        """Start the provider. This should return immediately after spawning threads to access the remote resources"""
+
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def stop(self):
+        """Stop any threads and prepare for application shutdown. Subclasses should implement this method and call
+        super()."""
+
+        self._stop_event.set()
+
+    def _add_data(self, new_data):
+        """Add all the provided reference data objects to the data store."""
+
+        # with transact() batches all writes together to save making thousands of individual sqlite writes
+        with DATA_STORE.activity_refs.transact():
+            for d in new_data:
+                DATA_STORE.activity_refs.set(f"{self.sig_name}:{d.id}", d)
+
+                # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
+                # within the first few minutes of startup, we need a way to abort this expensive process of filling up the
+                # disk cache.
+                if self._stop_event.is_set():
+                    break
+
+        self.reference_count = len(new_data)
+        logger.info(f"Loaded {self.reference_count} references for {self.sig_name} into the data store.")