diff --git a/core/lookup_helper.py b/core/call_lookup_helper.py similarity index 99% rename from core/lookup_helper.py rename to core/call_lookup_helper.py index 07b841a..fa393e0 100644 --- a/core/lookup_helper.py +++ b/core/call_lookup_helper.py @@ -329,7 +329,7 @@ class LookupHelper: def get_flag_for_dxcc(self, dxcc): """Get an emoji flag for a given DXCC entity ID""" - dxcc_data = DATA_STORE.dxcc_data[dxcc] + dxcc_data = DATA_STORE.dxcc_data[dxcc] if dxcc in DATA_STORE.dxcc_data else None return dxcc_data["flag"] if dxcc_data else None def infer_name_from_callsign_online_lookup(self, call, credentials=None): @@ -621,9 +621,9 @@ class LookupHelper: def _get_dxcc_data_for_callsign(self, call) -> dict | None: """Utility method to get generic DXCC data from our lookup table, if we can find it""" - for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]: - if entry["_prefixRegexCompiled"].match(call): - return entry + for pattern, entity_code in DATA_STORE.dxcc_lookup_by_call_regex: + if pattern.match(call): + return DATA_STORE.dxcc_data[entity_code] return None def stop(self): diff --git a/core/data_store.py b/core/data_store.py index dd9e94d..6018bdd 100644 --- a/core/data_store.py +++ b/core/data_store.py @@ -1,4 +1,5 @@ import logging +import re from pathlib import Path import diskcache @@ -23,6 +24,7 @@ class DataStore: self.spots = None self.callsigns = None self.dxcc_data = None + self.dxcc_lookup_by_call_regex = [] self.sigrefs = None self.status_data = None self._status = None @@ -46,6 +48,7 @@ class DataStore: # Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on # a regular basis but there's no need for a TTL since old data is better than no data. self.dxcc_data = diskcache.Cache(CACHE_DIR + "dxcc_data") + self.regenerateCallRegexToDXCCEntityMap() # For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers # of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref" @@ -72,6 +75,15 @@ class DataStore: snapshot_interval_sec=self._SPOT_ALERT_SNAPSHOT_INTERVAL_SEC) logging.info(f"Loaded %d alerts from a previous run.", len(self.alerts.keys())) + def regenerateCallRegexToDXCCEntityMap(self): + """DXCC entity data from K0SWE includes a regex which we can use to match a callsign, and determine which DXCC + entity it belongs to. But getting every DXCC entity data object out of DiskCache, iterating, compiling its regex + and testing the callsign every time is expensive. So instead we build a separate in-memory lookup of compiled + regex against DXCC entity code, as a list of tuples we can iterate through.""" + + for entry in [DATA_STORE.dxcc_data[key] for key in DATA_STORE.dxcc_data]: + self.dxcc_lookup_by_call_regex.append((re.compile(entry["prefixRegex"]), entry["entityCode"])) + def close(self): self.spots.close() self.alerts.close() diff --git a/core/sig_lookup_helper.py b/core/sig_lookup_helper.py new file mode 100644 index 0000000..1232f19 --- /dev/null +++ b/core/sig_lookup_helper.py @@ -0,0 +1,93 @@ +import logging + +from pyhamtools.locator import locator_to_latlong, latlong_to_locator + +from core.data_store import DATA_STORE +from core.geo_utils import wab_wai_square_to_lat_lon + + +def populate_sig_ref_info(sig_ref): + """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object + which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes + use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which + SIG we are getting data for. + Note there is currently no support for KRMNPA location lookup, see issue #61.""" + + if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "": + logging.debug("Failed to look up sig_ref info, sig or id were not set.") + return sig_ref + + sig = sig_ref.sig + ref_id = sig_ref.id + + try: + ### FUDGES ### + # + # DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all + # activators add leading zeros. + if sig.upper() == "DME": + ref_id = ref_id.zfill(5) + + # KRMNPA & SANPCPA fudge. These don't have their own reference system, they just use VKFF references, so pretend + # the sig is WWFF and carry on + if sig.upper() == "KRMNPA" or sig.upper() == "SANPCPA": + sig = "WWFF" + + ### SKIPS ### + # + # If the SIG is HEMA, we have no current lookup for this so just skip the lookup here. + if sig.upper() == "HEMA": + return sig_ref + + ### PROGRAMMATIC DATA GENERATION INSTEAD OF LOOKUPS ### + # + # If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can + # calculate all the information we are going to get directly. + if sig.upper() == "TILES": + # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same + if not sig_ref.name: + sig_ref.name = sig_ref.id + if not sig_ref.grid: + sig_ref.grid = sig_ref.id + if sig_ref.grid and not sig_ref.latitude: + ll = locator_to_latlong(str(sig_ref.grid)) + sig_ref.latitude = ll[0] + sig_ref.longitude = ll[1] + return sig_ref + + elif sig.upper() == "WAB" or sig.upper() == "WAI": + ll = wab_wai_square_to_lat_lon(ref_id) + if ll: + sig_ref.name = ref_id + try: + sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6) + sig_ref.latitude = ll[0] + sig_ref.longitude = ll[1] + except: + logging.warning("Invalid lat/lon received for WAB/WAI reference") + return sig_ref + + elif sig.upper() == "BOTA": + # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs + if not sig_ref.name: + sig_ref.name = sig_ref.id + sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-") + return sig_ref + + ### ACTUAL LOOKUP ### + # + # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for + # this SIG. If so, check for the reference data and use that. + key = sig + ":" + ref_id + lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None + if lookup_data: + # Copy new sig ref data into existing object where data was previously missing + for key, value in lookup_data.__dict__.items(): + if value is not None and sig_ref.__dict__.get(key) is None: + sig_ref.__dict__[key] = value + else: + logging.warning("%s database did not contain data for ref %s", sig, ref_id) + + except Exception: + logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True) + return sig_ref diff --git a/core/sig_utils.py b/core/sig_utils.py index ed74d84..100edc1 100644 --- a/core/sig_utils.py +++ b/core/sig_utils.py @@ -1,10 +1,4 @@ -import logging - -from pyhamtools.locator import latlong_to_locator, locator_to_latlong - from core.constants import SIGS -from core.data_store import DATA_STORE -from core.geo_utils import wab_wai_square_to_lat_lon def get_ref_regex_for_sig(sig): @@ -26,89 +20,5 @@ def get_sig_name_from_comment_name(sig): return None -def populate_sig_ref_info(sig_ref): - """Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object - which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. This makes - use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which - SIG we are getting data for. - Note there is currently no support for KRMNPA location lookup, see issue #61.""" - - if sig_ref.sig is None or sig_ref.sig == "" or sig_ref.id is None or sig_ref.id == "": - logging.debug("Failed to look up sig_ref info, sig or id were not set.") - return sig_ref - - sig = sig_ref.sig - ref_id = sig_ref.id - - try: - ### FUDGES ### - # - # DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all - # activators add leading zeros. - if sig.upper() == "DME": - ref_id = ref_id.zfill(5) - - # KRMNPA & SANPCPA fudge. These don't have their own reference system, they just use VKFF references, so pretend - # the sig is WWFF and carry on - if sig.upper() == "KRMNPA" or sig.upper() == "SANPCPA": - sig = "WWFF" - - ### SKIPS ### - # - # If the SIG is HEMA, we have no current lookup for this so just skip the lookup here. - if sig.upper() == "HEMA": - return sig_ref - - ### PROGRAMMATIC DATA GENERATION INSTEAD OF LOOKUPS ### - # - # If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can - # calculate all the information we are going to get directly. - if sig.upper() == "TILES": - # Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same - if not sig_ref.name: - sig_ref.name = sig_ref.id - if not sig_ref.grid: - sig_ref.grid = sig_ref.id - if sig_ref.grid and not sig_ref.latitude: - ll = locator_to_latlong(str(sig_ref.grid)) - sig_ref.latitude = ll[0] - sig_ref.longitude = ll[1] - - elif sig.upper() == "WAB" or sig.upper() == "WAI": - ll = wab_wai_square_to_lat_lon(ref_id) - if ll: - sig_ref.name = ref_id - try: - sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6) - sig_ref.latitude = ll[0] - sig_ref.longitude = ll[1] - except: - logging.warning("Invalid lat/lon received for WAB/WAI reference") - - elif sig.upper() == "BOTA": - # For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs - if not sig_ref.name: - sig_ref.name = sig_ref.id - sig_ref.url = "https://www.beachesontheair.com/beaches/" + sig_ref.name.lower().replace(" ", "-") - - ### ACTUAL LOOKUP ### - # - # OK, this is something we have to look up. Now check to see if our data store contains SIG ref information for - # this SIG. If so, check for the reference data and use that. - key = sig + ":" + ref_id - lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None - if lookup_data: - # Copy new sig ref data into existing object where data was previously missing - for key, value in lookup_data.__dict__.items(): - if value is not None and sig_ref.__dict__.get(key) is None: - sig_ref.__dict__[key] = value - else: - logging.warning("%s database did not contain data for ref %s", sig, ref_id) - - except Exception: - logging.error("Exception when looking up sig_ref info for " + sig + " ref " + ref_id, exc_info=True) - return sig_ref - - # Regex matching any SIG's "comment name", i.e. how it may be referred to in spot comments ANY_SIG_REGEX = r"(" + r"|".join(n for s in SIGS for n in s.comment_names) + r")" diff --git a/data/alert.py b/data/alert.py index 62c40f7..563fcef 100644 --- a/data/alert.py +++ b/data/alert.py @@ -7,8 +7,8 @@ from datetime import datetime, timedelta import pytz -from core.lookup_helper import lookup_helper -from core.sig_utils import populate_sig_ref_info +from core.call_lookup_helper import lookup_helper +from core.sig_lookup_helper import populate_sig_ref_info @dataclass diff --git a/data/spot.py b/data/spot.py index 886099c..e7f35f6 100644 --- a/data/spot.py +++ b/data/spot.py @@ -12,8 +12,9 @@ from pyhamtools.locator import locator_to_latlong, latlong_to_locator from core.config import MAX_SPOT_AGE from core.constants import MODE_ALIASES, PROPAGATION_MODES from core.geo_utils import lat_lon_to_cq_zone, lat_lon_to_itu_zone -from core.lookup_helper import lookup_helper -from core.sig_utils import populate_sig_ref_info, ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name +from core.call_lookup_helper import lookup_helper +from core.sig_utils import ANY_SIG_REGEX, get_ref_regex_for_sig, get_sig_name_from_comment_name +from core.sig_lookup_helper import populate_sig_ref_info from core.utils import infer_band_from_freq, infer_mode_from_comment, \ infer_mode_from_frequency, infer_mode_type_from_mode from data.sig_ref import SIGRef diff --git a/server/handlers/api/lookups.py b/server/handlers/api/lookups.py index 4475bdc..6d18a8f 100644 --- a/server/handlers/api/lookups.py +++ b/server/handlers/api/lookups.py @@ -11,7 +11,8 @@ from tornado.web import Application from core.constants import SIGS from core.geo_utils import lat_lon_for_grid_sw_corner_plus_size, lat_lon_to_cq_zone, lat_lon_to_itu_zone from core.prometheus_metrics_handler import api_requests_counter -from core.sig_utils import get_ref_regex_for_sig, populate_sig_ref_info +from core.sig_utils import get_ref_regex_for_sig +from core.sig_lookup_helper import populate_sig_ref_info from core.utils import safe_json_dumps from data.lookup_credentials import extract_credentials from data.sig_ref import SIGRef diff --git a/spothole.py b/spothole.py index 2510149..c2cfc74 100644 --- a/spothole.py +++ b/spothole.py @@ -9,7 +9,7 @@ from core.config import config, SERVER_OWNER_CALLSIGN, LOG_LEVEL, get_sig_ref_da get_static_data_provider_from_config from core.constants import SOFTWARE_VERSION from core.data_store import DATA_STORE -from core.lookup_helper import lookup_helper +from core.call_lookup_helper import lookup_helper from core.status_reporter import StatusReporter from server.webserver import WebServer diff --git a/static/js/status.js b/static/js/status.js index fd7e6ff..9ba0f78 100644 --- a/static/js/status.js +++ b/static/js/status.js @@ -43,7 +43,7 @@ function loadStatus() { jsonData["static_data_providers"].forEach(p => { $("#static-data-providers-status-container").append(`
-
${p["sig_name"]}
+
${p["name"]}
Status: ${p["status"]}
Last updated: ${(p["enabled"] && p["last_updated"] > 0) ? moment.unix(p["last_updated"]).utc().fromNow() : "N/A"}
`); diff --git a/staticdataproviders/k0swe.py b/staticdataproviders/k0swe.py index 2beb609..30c31ff 100644 --- a/staticdataproviders/k0swe.py +++ b/staticdataproviders/k0swe.py @@ -23,14 +23,13 @@ class K0SWE(FileDownloadStaticDataProvider): for dxcc in dxcc_list: dxcc_map[dxcc["entityCode"]] = dxcc - # Precompile regex matches for DXCCs to improve efficiency when iterating through them - for dxcc in dxcc_map.values(): - dxcc["_prefixRegexCompiled"] = re.compile(dxcc["prefixRegex"]) - # Add to data store for k, v in dxcc_map.items(): DATA_STORE.dxcc_data[k] = v + # Regenerate in-memory regex-to-DXCC-entity-code map. + DATA_STORE.regenerateCallRegexToDXCCEntityMap() + return True except Exception as e: