mirror of
https://git.ianrenton.com/ian/spothole.git
synced 2026-09-20 22:37:44 +00:00
56 lines
2.2 KiB
Python
56 lines
2.2 KiB
Python
from io import BytesIO
|
|
from time import sleep
|
|
|
|
import pdfplumber
|
|
|
|
from data.sig_ref import SIGRef
|
|
from providers.sigrefdata.file_download_sig_ref_data_provider import FileDownloadSIGRefDataProvider
|
|
|
|
|
|
class FEA(FileDownloadSIGRefDataProvider):
|
|
"""SIG ref data provider for Diploma Faros de España"""
|
|
|
|
POLL_INTERVAL_DAYS = 30
|
|
SIG = "FEA"
|
|
DATA_URL = "http://ea5ol.net/Lista%20Faros.pdf"
|
|
|
|
def __init__(self, provider_config):
|
|
super().__init__(self.SIG, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
|
|
|
def _http_response_to_data(self, http_response):
|
|
new_data = []
|
|
|
|
# Use PDFPlumber to extract the tables in the PDF
|
|
with pdfplumber.open(BytesIO(http_response.content)) as pdf:
|
|
all_rows = []
|
|
|
|
for page_number, page in enumerate(pdf.pages, start=1):
|
|
tables = page.extract_tables()
|
|
|
|
for table_number, table in enumerate(tables, start=1):
|
|
if not table:
|
|
continue
|
|
|
|
rows = [row for row in table if any(cell and cell.strip() for cell in row)]
|
|
all_rows.extend(rows)
|
|
|
|
for row in all_rows:
|
|
if not "REF" in row[0] and not "\n" in row[0]:
|
|
# FEA references are technically [DE]\-\d{4}(\.\d)? but spotters always seem to miss out the D- or E-
|
|
# prefix and just use FEA-1234 or FEA 1234, so we add both copies to the database.
|
|
ref_id_1 = row[0].strip()
|
|
ref_id_2 = ref_id_1.replace("D-", "FEA-").replace("E-", "FEA-")
|
|
new_data.append(SIGRef(sig=self.SIG, id=ref_id_1, name=row[1].strip(), ref_type="Lighthouse"))
|
|
new_data.append(SIGRef(sig=self.SIG, id=ref_id_2, name=row[1].strip(), ref_type="Lighthouse"))
|
|
|
|
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
|
|
# the data in this case
|
|
if self._stop_event.is_set():
|
|
break
|
|
|
|
# Very short pause. This will extend the time to handle sig refs by a few seconds but will ensure some time
|
|
# is available for other threads e.g. the web server.
|
|
sleep(0.001)
|
|
|
|
return new_data
|