import io from time import sleep import pandas as pd from core.enums import SIGRefType from data.sig_ref import SIGRef from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider class DCE(FileDownloadSIGRefDataProvider): """SIG ref data provider for Diploma Castillos de EspaƱa""" POLL_INTERVAL_DAYS = 365 SIG = "DCE" DATA_URL = "https://www.acracb.org/dce/descargas/General/directorio_referencias_dce.xls" def __init__(self, provider_config): super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS) def _http_response_to_data(self, http_response): new_data = [] file_stream = io.BytesIO(http_response.content) df = pd.read_excel(file_stream, engine="xlrd", header=None) for index, row in df.iterrows(): if row.iloc[0] and row.iloc[2]: new_data.append( SIGRef(sig=self.SIG, id=row.iloc[0].strip(), name=row.iloc[2].strip(), ref_type=SIGRefType.CASTLE) ) # Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of # the data in this case if self._stop_event.is_set(): break # Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time # is available for other threads e.g. the web server. sleep(0.001) return new_data