import logging from datetime import datetime import pytz from core.data_store import DATA_STORE class SIGRefDataProvider: """Generic SIG reference data provider class. Subclasses of this query the individual URLs or files for data.""" def __init__(self, sig_name, provider_config): """Constructor""" self.sig_name = sig_name self.enabled = provider_config["enabled"] self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC) self.status = "Not Started" if self.enabled else "Disabled" self.reference_count = 0 self._stop = False def start(self): """Start the provider. This should return immediately after spawning threads to access the remote resources""" raise NotImplementedError("Subclasses must implement this method") def stop(self): """Stop any threads and prepare for application shutdown. Subclasses should implement this method and call super().""" self._stop = True def _add_data(self, new_data): """Add all the provided reference data objects to the data store.""" # with transact() batches all writes together to save making thousands of individual sqlite writes with DATA_STORE.sigrefs.transact(): for d in new_data: DATA_STORE.sigrefs.set(self.sig_name + ":" + d.id, d) # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly # within the first few minutes of startup, we need a way to abort this expensive process of filling up the # disk cache. if self._stop: break self.reference_count = len(new_data) logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)