w1cdn/spothole
1
0
Fork 0
mirror of https://git.ianrenton.com/ian/spothole.git synced 2026-09-20 14:27:42 +00:00
Code Issues 13 Packages Projects Releases Wiki Activity

Giant refactor to rebrand "SIG" as "Activity" anywhere that doesn't touch API or config file (which is to be addressed in a future breaking change). #147

Browse Source
This commit is contained in:
Ian Renton
2026-09-18 14:54:11 +01:00
parent 556ea56378
commit 81cd686a00
96 changed files with 1208 additions and 1170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/activity_lookup_helper.py
+156
View File
@@ -0,0 +1,156 @@
import logging
import re
from pyhamtools.locator import latlong_to_locator, locator_to_latlong
from core.constants import ACTIVITIES
from core.data_store import DATA_STORE
from core.enums import ActivityRefType
from core.geo_utils import wab_wai_square_to_lat_lon
from data.activity_ref import ActivityRef
logger = logging.getLogger(__name__)
def get_activity_ref_info(activity_name, ref_id):
"""Look up details of an activity reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in an
activity name and a reference ID (both strings) and returns an ActivityRef object populated with as much data
as we can find. This makes use of activity ref data in the data store, live lookups from the web, or just
automatic calculation depending on which activity we are getting data for."""
if activity_name is None or activity_name == "" or ref_id is None or ref_id == "":
logger.debug("Failed to look up activity ref info, activity or ref were not set.")
return None
# Sometimes we allow spaces instead of dashes in references due to common usage that way, but official reference
# lists never do, so convert them here.
ref_id = ref_id.replace(" ", "-")
# Prepare the object to be returned
activity_ref = ActivityRef(sig=activity_name, id=ref_id)
# We can always get the reference type and the icon from the activity itself
for activity in ACTIVITIES:
if activity.name.upper() == activity_name.upper():
activity_ref.ref_type = activity.ref_type
activity_ref.icon = activity.icon
try:
### FUDGES ###
#
# DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all
# activators add leading zeros. We also need to normalise "DME 01234" to "DME-01234" to match what's in our
# database.
if activity_name.upper() == "DME":
match = re.match(r"DME[\- ](\d{3,5})", ref_id, re.IGNORECASE)
if match:
number = match.group(1)
ref_id = f"DME-{number.zfill(5)}"
# DTMBA spotters sometimes include spaces and dashes, our regex allows them but they must be removed here so we
# can look up against the official list which doesn't have them
if activity_name.upper() == "DTMBA":
ref_id = ref_id.replace("-", "").replace(" ", "")
### NO DATA ACTIVITIES ###
#
# If the activity is HEMA or BIWOTA, we have no way to either generate useful data or look it up on a
# reference list, so just skip the lookup here.
if activity_name.upper() == "HEMA" or activity_name.upper() == "BIWOTA":
return activity_ref
### PROGRAMMATIC DATA GENERATION INSTEAD OF LOOKUPS ###
#
# If the activity is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data
# store, we can calculate all the information we are going to get directly.
if activity_name.upper() == "TILES":
# Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
if not activity_ref.name:
activity_ref.name = activity_ref.id
if not activity_ref.grid:
activity_ref.grid = activity_ref.id
if activity_ref.grid and (not activity_ref.latitude or not activity_ref.longitude):
ll = locator_to_latlong(str(activity_ref.grid))
activity_ref.latitude = ll[0]
activity_ref.longitude = ll[1]
return activity_ref
elif activity_name.upper() == "WAB" or activity_name.upper() == "WAI":
ll = wab_wai_square_to_lat_lon(ref_id)
if ll:
activity_ref.name = ref_id
try:
activity_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
activity_ref.latitude = ll[0]
activity_ref.longitude = ll[1]
except Exception:
logger.warning("Invalid lat/lon received for WAB/WAI reference")
return activity_ref
elif activity_name.upper() == "BOTA":
# For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
if not activity_ref.name:
activity_ref.name = activity_ref.id
if activity_ref.name:
activity_ref.url = (
f"https://www.beachesontheair.com/beaches/{activity_ref.name.lower().replace(' ', '-')}"
)
return activity_ref
elif activity_name.upper() == "GMA Islands":
# GMA Islands is a bit of a mess of GMA and IOTA references. Try looking them both up and see what returns
# the best result.
iota_lookup = get_activity_ref_info("IOTA", ref_id)
gma_lookup = get_activity_ref_info("GMA", ref_id)
for key, value in iota_lookup.__dict__.items():
if value is not None and activity_ref.__dict__.get(key) is None:
activity_ref.__dict__[key] = value
for key, value in gma_lookup.__dict__.items():
if value is not None and activity_ref.__dict__.get(key) is None:
activity_ref.__dict__[key] = value
activity_ref.ref_type = ActivityRefType.ISLAND
return activity_ref
### ACTUAL LOOKUP ###
#
# OK, this is something we have to look up. Now check to see if our data store contains reference data and if
# so, copy the data into the activity_ref object
key = f"{activity_name}:{ref_id}"
try:
lookup_data = DATA_STORE.activity_refs.get(key) if key in DATA_STORE.activity_refs else None
if lookup_data:
for attr, value in lookup_data.__dict__.items():
if value is not None and activity_ref.__dict__.get(attr) is None:
activity_ref.__dict__[attr] = value
else:
# Maybe a super new reference we don't know about yet, but more likely a typo or a test reference,
# just silently ignore it.
logger.debug(f"{activity_name} database did not contain data for ref {ref_id}")
except (ValueError, KeyError):
# Catch exceptions due to e.g. old versions of objects in the cache that are no longer compatible,
# and remove them from the cache.
del DATA_STORE.activity_refs[key]
return None
except Exception:
logger.exception(f"Exception when looking up activity ref info for {activity_name} ref {ref_id}")
return activity_ref
def populate_missing_activity_ref_info(activity_ref):
"""Look up details of an activity reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in an
activity_ref object which must at minimum have a "sig" and an "id". The rest of the object will be populated
and returned. Any data currently in the object will be kept, only missing data in the object will be populated
if it can be determined."""
lookup_data = get_activity_ref_info(activity_ref.sig, activity_ref.id)
if lookup_data:
# Copy new activity ref data into existing object where data was previously missing
for key, value in lookup_data.__dict__.items():
if value is not None and activity_ref.__dict__.get(key) is None:
activity_ref.__dict__[key] = value
return activity_ref
core/activity_utils.py
+26
View File
@@ -0,0 +1,26 @@
from core.constants import ACTIVITIES
def get_ref_regex_for_activity(activity):
"""Utility function to get the regex string for an activity reference for a named activity. If no match is
found, None will be returned."""
for a in ACTIVITIES:
if a.name.upper() == activity.upper():
return a.ref_regex
return None
def get_activity_name_from_comment_name(activity):
"""Utility function to get the name of an activity from its "comment name". Generally these will be the same
but there are some cases (e.g. is "TOTA" Towers, Tiles or Toilets?) where we need to transform one to the
other."""
for a in ACTIVITIES:
if any(n.upper() == activity.upper() for n in a.comment_names):
return a.name
return None
# Regex matching any activity's "comment name", i.e. how it may be referred to in spot comments
ANY_ACTIVITY_REGEX = rf"({'|'.join(n for a in ACTIVITIES for n in a.comment_names)})"
core/constants.py
+113 -113
View File
@@ -1,7 +1,7 @@
from core.config import SERVER_OWNER_CALLSIGN
from core.enums import SIGRefType, SIGType
from core.enums import ActivityRefType, ActivityType
from data.activity import Activity
from data.band import Band
from data.sig import SIG
# General software
SOFTWARE_VERSION = "2.1"
@@ -10,387 +10,387 @@ SOFTWARE_VERSION = "2.1"
HTTP_HEADERS = {"User-Agent": f"Spothole v{SOFTWARE_VERSION} (operated by {SERVER_OWNER_CALLSIGN})"}
HAMQTH_PRG = f"Spothole v{SOFTWARE_VERSION} operated by {SERVER_OWNER_CALLSIGN}".replace(" ", "_")
# Special Interest Groups
SIGS = [
SIG(
# Activities
ACTIVITIES = [
Activity(
name="AMSAT",
comment_names=[],
description="Amateur Radio Satellites",
sig_type=SIGType.WORLDWIDE,
sig_type=ActivityType.WORLDWIDE,
icon="fa-satellite",
refs_globally_unique=False,
),
SIG(
Activity(
name="EME",
comment_names=[],
description="Moonbounce",
sig_type=SIGType.WORLDWIDE,
sig_type=ActivityType.WORLDWIDE,
icon="fa-moon",
refs_globally_unique=False,
),
SIG(
Activity(
name="POTA",
comment_names=["POTA"],
description="Parks on the Air",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.PARK,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.PARK,
ref_regex=r"[A-Z]{2}\-\d{4,5}|K\-TEST",
icon="fa-tree",
refs_globally_unique=False,
),
SIG(
Activity(
name="SOTA",
comment_names=["SOTA"],
description="Summits on the Air",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.SUMMIT,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.SUMMIT,
ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
icon="fa-mountain-sun",
refs_globally_unique=False,
),
SIG(
Activity(
name="WWFF",
comment_names=["WWFF"],
description="World Wide Flora & Fauna",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.PARK,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.PARK,
ref_regex=r"[A-Z0-9]{1,3}FF\-\d{4}",
icon="fa-seedling",
refs_globally_unique=True,
),
SIG(
Activity(
name="GMA",
comment_names=["GMA"],
description="Global Mountain Activity",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.SUMMIT,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.SUMMIT,
ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}",
icon="fa-person-hiking",
refs_globally_unique=False,
),
SIG(
Activity(
name="WWBOTA",
comment_names=["WWBOTA", "BOTA"],
description="Worldwide Bunkers on the Air",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.BUNKER,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.BUNKER,
ref_regex=r"B\/[A-Z0-9]{1,3}\-\d{3,4}",
icon="fa-radiation",
refs_globally_unique=True,
),
SIG(
Activity(
name="HEMA",
comment_names=["HEMA"],
description="HuMPs Excluding Marilyns Award",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.SUMMIT,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.SUMMIT,
ref_regex=r"[A-Z0-9]{1,3}\/[A-Z]{3}\-\d{3}",
icon="fa-mound",
refs_globally_unique=False,
),
SIG(
Activity(
name="IOTA",
comment_names=["IOTA"],
description="Islands on the Air",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.ISLAND,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.ISLAND,
ref_regex=r"[A-Z]{2}\-\d{3}",
icon="fa-book-atlas",
refs_globally_unique=False,
),
SIG(
Activity(
name="GMA Islands",
comment_names=[],
description="Global Mountain Activity - Islands",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.ISLAND,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.ISLAND,
ref_regex=r"(([A-Z]{2}\-\d{3})|([A-Z0-9]{1,3}\/[A-Z]{2}\-\d{3}))",
icon="fa-person-hiking",
refs_globally_unique=False,
),
SIG(
Activity(
name="ARLHS",
comment_names=["ARLHS"],
description="Amateur Radio Lighthouse Society",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.LIGHTHOUSE,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.LIGHTHOUSE,
ref_regex=r"[A-Z]{3}[\- ]\d{3,4}",
icon="fa-house-flood-water",
refs_globally_unique=False,
),
SIG(
Activity(
name="ILLW",
comment_names=["ILLW"],
description="International Lighthouse & Lightship Weekend",
sig_type=SIGType.EVENT,
ref_type=SIGRefType.LIGHTHOUSE,
sig_type=ActivityType.EVENT,
ref_type=ActivityRefType.LIGHTHOUSE,
ref_regex=r"[A-Z]{2}\d{4}",
icon="fa-house-flood-water",
refs_globally_unique=False,
),
SIG(
Activity(
name="MOTA",
comment_names=["MOTA"],
description="Mills on the Air",
sig_type=SIGType.EVENT,
ref_type=SIGRefType.MILL,
sig_type=ActivityType.EVENT,
ref_type=ActivityRefType.MILL,
ref_regex=r"X\d{4,6}",
icon="fa-fan",
refs_globally_unique=True,
),
SIG(
Activity(
name="SIOTA",
comment_names=["SIOTA"],
description="Silos on the Air",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.SILO,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.SILO,
ref_regex=r"[A-Z]{2}\-[A-Z]{3}\d",
icon="fa-wheat-awn",
refs_globally_unique=False,
),
SIG(
Activity(
name="WCA",
comment_names=["WCA"],
description="World Castles Award",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.CASTLE,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.CASTLE,
ref_regex=r"[A-Z0-9]{1,3}\-\d{5}",
icon="fa-chess-rook",
refs_globally_unique=False,
),
SIG(
Activity(
name="ZLOTA",
comment_names=["ZLOTA"],
description="New Zealand on the Air",
sig_type=SIGType.REGIONAL,
sig_type=ActivityType.REGIONAL,
ref_type=None,
ref_regex=r"ZL[A-Z]/[A-Z]{2}\-\d{3,4}",
icon="fa-kiwi-bird",
region_flag="🇳🇿",
refs_globally_unique=True,
),
SIG(
Activity(
name="WOTA",
comment_names=["WOTA"],
description="Wainwrights on the Air",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.SUMMIT,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.SUMMIT,
ref_regex=r"[A-Z]{3}-[0-9]{2}",
icon="fa-w",
region_flag="🇬🇧",
refs_globally_unique=False,
),
SIG(
Activity(
name="BOTA",
comment_names=[],
description="Beaches on the Air",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.BEACH,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.BEACH,
icon="fa-umbrella-beach",
refs_globally_unique=False,
),
SIG(
Activity(
name="KRMNPA",
comment_names=["KRMNPA"],
description="Keith Roget Memorial National Parks Award",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.PARK,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.PARK,
ref_regex=r"VKFF\-\d{4}",
icon="fa-earth-oceania",
region_flag="🇦🇺",
refs_globally_unique=False,
),
SIG(
Activity(
name="SANPCPA",
comment_names=["SANPCPA"],
description="South Australian National Parks and Conservation Parks Award",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.PARK,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.PARK,
ref_regex=r"VKFF\-\d{4}",
icon="fa-earth-oceania",
region_flag="🇦🇺",
refs_globally_unique=False,
),
SIG(
Activity(
name="LLOTA",
comment_names=["LLOTA"],
description="Lagos y Lagunas on the Air",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.LAKE,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.LAKE,
ref_regex=r"LL[A-Z]{2}\-\d{4}",
icon="fa-water",
refs_globally_unique=True,
),
SIG(
Activity(
name="Towers",
comment_names=["TOTA"],
description="Towers on the Air",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.TOWER,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.TOWER,
ref_regex=r"[A-Z]{2,3}R\-\d{4}",
icon="fa-tower-observation",
refs_globally_unique=False,
),
SIG(
Activity(
name="Tiles",
comment_names=[],
description="Tiles on the Air",
sig_type=SIGType.WORLDWIDE,
ref_type=SIGRefType.GRID,
sig_type=ActivityType.WORLDWIDE,
ref_type=ActivityRefType.GRID,
ref_regex=r"[A-Za-z]{2}[0-9]{2}[A-Za-z]{2}",
icon="fa-square",
refs_globally_unique=False,
),
SIG(
Activity(
name="WAB",
comment_names=["WAB"],
description="Worked All Britain",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.GRID,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.GRID,
ref_regex=r"[A-Z]{1,2}[0-9]{2}",
icon="fa-table-cells-large",
region_flag="🇬🇧",
refs_globally_unique=False,
),
SIG(
Activity(
name="WAI",
comment_names=["WAI"],
description="Worked All Ireland",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.GRID,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.GRID,
ref_regex=r"[A-Z][0-9]{2}",
icon="fa-table-cells-large",
region_flag="🇮🇪",
refs_globally_unique=False,
),
SIG(
Activity(
name="DMF",
comment_names=["DMF"],
description="Diplôme des Moulins de France",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.MILL,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.MILL,
icon="fa-fan",
region_flag="🇫🇷",
refs_globally_unique=False,
),
SIG(
Activity(
name="DME",
comment_names=["DME"],
description="Diploma Municipios de España",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.TOWN,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.TOWN,
ref_regex=r"DME[\- ]\d{3,5}",
icon="fa-building",
region_flag="🇪🇸",
refs_globally_unique=True,
),
SIG(
Activity(
name="FEA",
comment_names=["FEA"],
description="Diploma Faros de España",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.LIGHTHOUSE,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.LIGHTHOUSE,
# FEA references are technically [DE]\-\d{4}(\.\d)? but spotters always seem to miss out the D- or E-
# prefix and just use FEA-1234 or FEA 1234, so allow for that. The FEA sigref data provider adds both
# prefix and just use FEA-1234 or FEA 1234, so allow for that. The FEA activity ref data provider adds both
# forms to the database.
ref_regex=r"([DE]|FEA)[\- ]\d{4}(\.\d)?",
icon="fa-house-flood-water",
region_flag="🇪🇸",
refs_globally_unique=True,
),
SIG(
Activity(
name="DMUE",
comment_names=["DMUE"],
description="Diploma Museos de España",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.BUILDING,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.BUILDING,
ref_regex=r"MUE[A-Z]{2}-\d{3}",
icon="fa-landmark",
region_flag="🇪🇸",
refs_globally_unique=True,
),
SIG(
Activity(
name="DMVE",
comment_names=["DMVE"],
description="Diploma Monumentos y Vestigios de España",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.BUILDING,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.BUILDING,
ref_regex=r"MV[A-Z]{1,2}-\d{4}",
icon="fa-monument",
region_flag="🇪🇸",
refs_globally_unique=True,
),
SIG(
Activity(
name="DCE",
comment_names=["DCE"],
description="Diploma Castillos de España",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.CASTLE,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.CASTLE,
ref_regex=r"C[A-Z]{1,2}-\d{3}",
icon="fa-chess-rook",
region_flag="🇪🇸",
refs_globally_unique=False,
),
SIG(
Activity(
name="DEFE",
comment_names=["DEFE"],
description="Diploma Estaciones de Ferrocarril de España",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.BUILDING,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.BUILDING,
ref_regex=r"EF[A-Z]{1,2}-\d{3}",
icon="fa-train",
region_flag="🇪🇸",
refs_globally_unique=True,
),
SIG(
Activity(
name="DTMBA",
comment_names=["DTMBA"],
description="Diploma Teatri Musei e Belle Arti",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.BUILDING,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.BUILDING,
ref_regex=r"I-?[0-9]{3,4}\s?[A-Z]{2}",
icon="fa-landmark",
region_flag="🇮🇹",
refs_globally_unique=True,
),
SIG(
Activity(
name="BIWOTA",
comment_names=["BIWOTA"],
description="British Inland Waterways on the Air",
sig_type=SIGType.EVENT,
ref_type=SIGRefType.WATERWAY,
sig_type=ActivityType.EVENT,
ref_type=ActivityRefType.WATERWAY,
icon="fa-ship",
region_flag="🇬🇧",
refs_globally_unique=False,
),
SIG(
Activity(
name="COTA",
comment_names=["COTA"],
description="Castles on the Air",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.CASTLE,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.CASTLE,
ref_regex=r"[A-Z]{3}\-[0-9]{3,5}",
icon="fa-chess-rook",
region_flag="🇩🇪",
refs_globally_unique=False,
),
SIG(
Activity(
name="PGA",
comment_names=["PGA"],
description="Polish Gmina Award",
sig_type=SIGType.REGIONAL,
ref_type=SIGRefType.REGION,
sig_type=ActivityType.REGIONAL,
ref_type=ActivityRefType.REGION,
ref_regex=r"[A-Z]{2}[0-9]{2}",
icon="fa-g",
region_flag="🇵🇱",
refs_globally_unique=False,
),
SIG(
Activity(
name="Toilets",
comment_names=[],
description="Toilets on the Air",
sig_type=SIGType.EVENT,
ref_type=SIGRefType.TOILET,
sig_type=ActivityType.EVENT,
ref_type=ActivityRefType.TOILET,
ref_regex=r"T\-[0-9]{2}",
icon="fa-toilet",
region_flag="🏴‍☠️",
core/data_providers.py
+2 -2
View File
@@ -29,7 +29,7 @@ class DataProviders:
for entry in config.get("static_data_providers", []):
self.static_data_providers.append(create_provider_from_config("providers.staticdata", entry))
for entry in config.get("sig_ref_data_providers", []):
self.sig_ref_data_providers.append(create_provider_from_config("providers.sigrefdata", entry))
self.sig_ref_data_providers.append(create_provider_from_config("providers.activityrefdata", entry))
for entry in config.get("callsign_data_providers", []):
self.callsign_data_providers.append(create_provider_from_config("providers.callsigndata", entry))
@@ -54,7 +54,7 @@ class DataProviders:
25.0,
lambda: self.start_providers(self.solar_condition_providers, "solar condition"),
),
threading.Timer(30.0, lambda: self.start_providers(self.sig_ref_data_providers, "SIG ref data")),
threading.Timer(30.0, lambda: self.start_providers(self.sig_ref_data_providers, "activity ref data")),
]
for t in self._startup_timers:
t.daemon = True
core/data_store.py
+13 -13
View File
@@ -15,8 +15,8 @@ CACHE_DIR = "./cache/"
class DataStore:
"""Data caching/storage object. Handles storage of spots, alerts, solar conditions, SIG reference data, and callsign
lookup data using different caching strategies for each."""
"""Data caching/storage object. Handles storage of spots, alerts, solar conditions, activity reference data, and
callsign lookup data using different caching strategies for each."""
def __init__(self):
# Constants
@@ -34,7 +34,7 @@ class DataStore:
self.callsign_data_hamqth = None
self.dxcc_data = None
self.dxcc_lookup_by_call_regex = []
self.sigrefs = None
self.activity_refs = None
self.status = None
self.solar_conditions = None
# ITU/CQ zone GeoJSON data is only ever loaded statically from a local file so these don't even need to be
@@ -51,16 +51,16 @@ class DataStore:
self.solar_conditions = SingleObjectDataCache(f"{CACHE_DIR}solar", SolarConditions())
self.status = SingleObjectDataCache(f"{CACHE_DIR}status", {})
# Standard disk cache for static reference and SIG ref data. Separate provider threads will repopulate these on
# a regular basis but there's no need for a TTL since old data is better than no data.
# Standard disk cache for static reference and activity ref data. Separate provider threads will repopulate
# these on a regular basis but there's no need for a TTL since old data is better than no data.
self.dxcc_data = diskcache.Cache(f"{CACHE_DIR}dxcc_data")
self.regenerate_call_regex_to_dxcc_entity_map()
# For SIG reference data specifically, we need to key on both SIG *and* reference, and trying to do two layers
# of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an ugly "SIG:ref"
# syntax for keys to keep it a single level.
self.sigrefs = diskcache.Cache(f"{CACHE_DIR}sigrefs")
logger.info(f"Loaded data for {len(self.sigrefs)} SIG references.")
# For activity reference data specifically, we need to key on both activity *and* reference, and trying to do
# two layers of dict in diskcache absolutely destroys performance with unpickling huge dicts, so we have an
# ugly "activity:ref" syntax for keys to keep it a single level.
self.activity_refs = diskcache.Cache(f"{CACHE_DIR}activity_refs")
logger.info(f"Loaded data for {len(self.activity_refs)} activity references.")
# Standard disk cache for callsign data. This data does have a TTL to trigger an occasional re-lookup.
# Old data *is* better than no data, but we can't have a background thread re-looking-up every callsign
@@ -82,8 +82,8 @@ class DataStore:
logger.info(f"Loaded data for {len(unique_keys)} callsigns.")
# Special caches for spots and alerts, which have TTL and write snapshots to disk at an interval. We
# specifically load these caches *last* so that any sigref and callsign data is already loaded from disk cache
# before the spots and alerts are live in the system.
# specifically load these caches *last* so that any activity ref and callsign data is already loaded from disk
# cache before the spots and alerts are live in the system.
self.spots = LiveDataCache(
maxsize=self._MAX_SPOT_COUNT,
ttl=MAX_SPOT_AGE,
@@ -116,7 +116,7 @@ class DataStore:
self.solar_conditions.close()
self.status.close()
self.dxcc_data.close()
self.sigrefs.close()
self.activity_refs.close()
self.callsign_data_countryfiles.close()
self.callsign_data_clublogxml.close()
self.callsign_data_clublogapi.close()
core/enums.py
+4 -4
View File
@@ -90,8 +90,8 @@ class LocationSourceForCallsign(str, Enum):
DXCC = "DXCC"
class SIGRefType(str, Enum):
"""Type of a Special Interest Group reference."""
class ActivityRefType(str, Enum):
"""Type of an activity reference."""
PARK = "PARK"
SUMMIT = "SUMMIT"
@@ -121,8 +121,8 @@ class AlertType(str, Enum):
CONTEST = "CONTEST"
class SIGType(str, Enum):
"""Type of a Special Interest Group. Used to group them in the web UI."""
class ActivityType(str, Enum):
"""Type of an activity. Used to group them in the web UI."""
WORLDWIDE = "WORLDWIDE"
REGIONAL = "REGIONAL"
core/sig_lookup_helper.py
-153
View File
@@ -1,153 +0,0 @@
import logging
import re
from pyhamtools.locator import latlong_to_locator, locator_to_latlong
from core.constants import SIGS
from core.data_store import DATA_STORE
from core.enums import SIGRefType
from core.geo_utils import wab_wai_square_to_lat_lon
from data.sig_ref import SIGRef
logger = logging.getLogger(__name__)
def get_sig_ref_info(sig_name, ref_id):
"""Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig name and
a reference ID (both strings) and returns a SigRef object populated with as much data as we can find. This makes
use of SIG ref data in the data store, live lookups from the web, or just automatic calculation depending on which
SIG we are getting data for."""
if sig_name is None or sig_name == "" or ref_id is None or ref_id == "":
logger.debug("Failed to look up sig_ref info, sig or ref were not set.")
return None
# Sometimes we allow spaces instead of dashes in references due to common usage that way, but official reference
# lists never do, so convert them here.
ref_id = ref_id.replace(" ", "-")
# Prepare the object to be returned
sig_ref = SIGRef(sig=sig_name, id=ref_id)
# We can always get the reference type and the icon from the SIG itself
for sig in SIGS:
if sig.name.upper() == sig_name.upper():
sig_ref.ref_type = sig.ref_type
sig_ref.icon = sig.icon
try:
### FUDGES ###
#
# DME fudge. Our database has leading zeros padding to 5 digits which is the expected format, but not all
# activators add leading zeros. We also need to normalise "DME 01234" to "DME-01234" to match what's in our
# database.
if sig_name.upper() == "DME":
match = re.match(r"DME[\- ](\d{3,5})", ref_id, re.IGNORECASE)
if match:
number = match.group(1)
ref_id = f"DME-{number.zfill(5)}"
# DTMBA spotters sometimes include spaces and dashes, our regex allows them but they must be removed here so we
# can look up against the official list which doesn't have them
if sig_name.upper() == "DTMBA":
ref_id = ref_id.replace("-", "").replace(" ", "")
### NO DATA SIGS ###
#
# If the SIG is HEMA or BIWOTA, we have no way to either generate useful data or look it up on a reference list,
# so just skip the lookup here.
if sig_name.upper() == "HEMA" or sig_name.upper() == "BIWOTA":
return sig_ref
### PROGRAMMATIC DATA GENERATION INSTEAD OF LOOKUPS ###
#
# If the SIG is Tiles, WAB, WAI or BOTA (Beaches), we don't have anything to look up from the data store, we can
# calculate all the information we are going to get directly.
if sig_name.upper() == "TILES":
# Tiles on the Air just uses Maidenhead 6-digit squares, so ID, Name and Grid are all the same
if not sig_ref.name:
sig_ref.name = sig_ref.id
if not sig_ref.grid:
sig_ref.grid = sig_ref.id
if sig_ref.grid and (not sig_ref.latitude or not sig_ref.longitude):
ll = locator_to_latlong(str(sig_ref.grid))
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
return sig_ref
elif sig_name.upper() == "WAB" or sig_name.upper() == "WAI":
ll = wab_wai_square_to_lat_lon(ref_id)
if ll:
sig_ref.name = ref_id
try:
sig_ref.grid = latlong_to_locator(ll[0], ll[1], 6)
sig_ref.latitude = ll[0]
sig_ref.longitude = ll[1]
except Exception:
logger.warning("Invalid lat/lon received for WAB/WAI reference")
return sig_ref
elif sig_name.upper() == "BOTA":
# For BOTA all we can ever generate is the URL, there is no data file or lookup for lat/longs
if not sig_ref.name:
sig_ref.name = sig_ref.id
if sig_ref.name:
sig_ref.url = f"https://www.beachesontheair.com/beaches/{sig_ref.name.lower().replace(' ', '-')}"
return sig_ref
elif sig_name.upper() == "GMA Islands":
# GMA Islands is a bit of a mess of GMA and IOTA references. Try looking them both up and see what returns
# the best result.
iota_lookup = get_sig_ref_info("IOTA", ref_id)
gma_lookup = get_sig_ref_info("GMA", ref_id)
for key, value in iota_lookup.__dict__.items():
if value is not None and sig_ref.__dict__.get(key) is None:
sig_ref.__dict__[key] = value
for key, value in gma_lookup.__dict__.items():
if value is not None and sig_ref.__dict__.get(key) is None:
sig_ref.__dict__[key] = value
sig_ref.ref_type = SIGRefType.ISLAND
return sig_ref
### ACTUAL LOOKUP ###
#
# OK, this is something we have to look up. Now check to see if our data store contains reference data and if
# so, copy the data into the sig_ref object
key = f"{sig_name}:{ref_id}"
try:
lookup_data = DATA_STORE.sigrefs.get(key) if key in DATA_STORE.sigrefs else None
if lookup_data:
for attr, value in lookup_data.__dict__.items():
if value is not None and sig_ref.__dict__.get(attr) is None:
sig_ref.__dict__[attr] = value
else:
# Maybe a super new reference we don't know about yet, but more likely a typo or a test reference,
# just silently ignore it.
logger.debug(f"{sig_name} database did not contain data for ref {ref_id}")
except (ValueError, KeyError):
# Catch exceptions due to e.g. old versions of objects in the cache that are no longer compatible,
# and remove them from the cache.
del DATA_STORE.sigrefs[key]
return None
except Exception:
logger.exception(f"Exception when looking up sig_ref info for {sig_name} ref {ref_id}")
return sig_ref
def populate_missing_sig_ref_info(sig_ref):
"""Look up details of a SIG reference (e.g. POTA park) such as name, lat/lon, and grid. Takes in a sig_ref object
which must at minimum have a "sig" and an "id". The rest of the object will be populated and returned. Any data
currently in the object will be kept, only missing data in the object will be populated if it can be determined."""
lookup_data = get_sig_ref_info(sig_ref.sig, sig_ref.id)
if lookup_data:
# Copy new sig ref data into existing object where data was previously missing
for key, value in lookup_data.__dict__.items():
if value is not None and sig_ref.__dict__.get(key) is None:
sig_ref.__dict__[key] = value
return sig_ref
core/sig_utils.py
-24
View File
@@ -1,24 +0,0 @@
from core.constants import SIGS
def get_ref_regex_for_sig(sig):
"""Utility function to get the regex string for a SIG reference for a named SIG. If no match is found, None will be returned."""
for s in SIGS:
if s.name.upper() == sig.upper():
return s.ref_regex
return None
def get_sig_name_from_comment_name(sig):
"""Utility function to get the name of a SIG from its "comment name". Generally these will be the same but there are
some cases (e.g. is "TOTA" Towers, Tiles or Toilets?) where we need to transform one to the other."""
for s in SIGS:
if any(n.upper() == sig.upper() for n in s.comment_names):
return s.name
return None
# Regex matching any SIG's "comment name", i.e. how it may be referred to in spot comments
ANY_SIG_REGEX = rf"({'|'.join(n for s in SIGS for n in s.comment_names)})"
core/single_object_data_cache.py
+4 -4
View File
@@ -7,10 +7,10 @@ logger = logging.getLogger(__name__)
class SingleObjectDataCache:
"""Cache for status and solar conditions. This uses DiskCache, but unlike the standard DiskCache users like SIG and
callsign lookup handlers, status and solar conditions are persisted as a single object. If we just load the object
from DiskCache and modify it, DiskCache doesn't know that it's been updated and needs re-caching, so we provide a
store() method that any functions updating the object can call afterwards."""
"""Cache for status and solar conditions. This uses DiskCache, but unlike the standard DiskCache users like
activity ref and callsign lookup handlers, status and solar conditions are persisted as a single object. If we
just load the object from DiskCache and modify it, DiskCache doesn't know that it's been updated and needs
re-caching, so we provide a store() method that any functions updating the object can call afterwards."""
def __init__(self, cache_dir, object_if_empty):
"""Initialize a SingleObjectDataCache. Provide the directory to load the cache from and save it to. If the cache