Further speed & stability improvements when loading sig ref datasets #126

providers/sigrefdata/sig_ref_data_provider.py

@@ -35,14 +35,16 @@ class SIGRefDataProvider:
     def _add_data(self, new_data):
         """Add all the provided reference data objects to the data store."""
 
-        for d in new_data:
-            DATA_STORE.sigrefs[self.sig_name + ":" + d.id] = d
+        # with transact() batches all writes together to save making thousands of individual sqlite writes
+        with DATA_STORE.sigrefs.transact():
+            for d in new_data:
+                DATA_STORE.sigrefs.set(self.sig_name + ":" + d.id, d)
 
-            # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
-            # within the first few minutes of startup, we need a way to abort this expensive process of filling up the
-            # disk cache.
-            if self._stop:
-                break
+                # For the big data sources, loading will take a few minutes. If we want to shut down the software neatly
+                # within the first few minutes of startup, we need a way to abort this expensive process of filling up the
+                # disk cache.
+                if self._stop:
+                    break
 
         self.reference_count = len(new_data)
         logging.info(f"Loaded %d references for %s into the data store.", self.reference_count, self.sig_name)