mirror of
https://git.ianrenton.com/ian/spothole.git
synced 2026-09-20 14:27:42 +00:00
57 lines
2.4 KiB
Python
57 lines
2.4 KiB
Python
from io import BytesIO
|
|
from time import sleep
|
|
|
|
import pdfplumber
|
|
|
|
from core.enums import ActivityName, ActivityRefType
|
|
from data.activity_ref import ActivityRef
|
|
from providers.activityrefdata.file_download_activity_ref_data_provider import FileDownloadActivityRefDataProvider
|
|
|
|
|
|
class FEA(FileDownloadActivityRefDataProvider):
|
|
"""Activity ref data provider for Diploma Faros de España"""
|
|
|
|
POLL_INTERVAL_DAYS = 30
|
|
ACTIVITY = ActivityName.FEA
|
|
DATA_URL = "http://ea5ol.net/Lista%20Faros.pdf"
|
|
|
|
def __init__(self, provider_config):
|
|
super().__init__(self.ACTIVITY, provider_config, self.DATA_URL, self.POLL_INTERVAL_DAYS)
|
|
|
|
def _http_response_to_data(self, http_response):
|
|
new_data = []
|
|
|
|
# Use PDFPlumber to extract the tables in the PDF
|
|
with pdfplumber.open(BytesIO(http_response.content)) as pdf:
|
|
all_rows = []
|
|
|
|
for page_number, page in enumerate(pdf.pages, start=1):
|
|
tables = page.extract_tables()
|
|
|
|
for table_number, table in enumerate(tables, start=1):
|
|
if not table:
|
|
continue
|
|
|
|
rows = [row for row in table if any(cell and cell.strip() for cell in row)]
|
|
all_rows.extend(rows)
|
|
|
|
for row in all_rows:
|
|
if not "REF" in row[0] and not "\n" in row[0]:
|
|
# FEA references are technically [DE]\-\d{4}(\.\d)? but spotters always seem to miss out the D- or E-
|
|
# prefix and just use FEA-1234 or FEA 1234, so we add both copies to the database.
|
|
ref_id_1 = row[0].strip()
|
|
ref_id_2 = ref_id_1.replace("D-", "FEA-").replace("E-", "FEA-")
|
|
new_data.append(ActivityRef(sig=self.ACTIVITY, id=ref_id_1, name=row[1].strip(), ref_type=ActivityRefType.LIGHTHOUSE))
|
|
new_data.append(ActivityRef(sig=self.ACTIVITY, id=ref_id_2, name=row[1].strip(), ref_type=ActivityRefType.LIGHTHOUSE))
|
|
|
|
# Bail out if a stop has been requested, i.e. the program is shutting down - no need to parse the rest of
|
|
# the data in this case
|
|
if self._stop_event.is_set():
|
|
break
|
|
|
|
# Very short pause. This will extend the time to handle activity refs by a few seconds but will ensure some time
|
|
# is available for other threads e.g. the web server.
|
|
sleep(0.001)
|
|
|
|
return new_data
|