diff --git a/config-example.yml b/config-example.yml index ffb1035..3eb01a4 100644 --- a/config-example.yml +++ b/config-example.yml @@ -7,31 +7,60 @@ server-owner-callsign: "N0CALL" # Data providers to use. This is an example set, tailor it to your liking by commenting and uncommenting. -# RBN is supported but has such a high data rate, you probably don't want it enabled. -# APRS-IS support is not yet implemented. +# RBN and APRS-IS are supported but have such a high data rate, you probably don't want them enabled. +# Each provider needs a type, a name, and an enabled/disabled state. Some require more config such as hostnames/IP +# addresses and ports. You can duplicate them if you like, e.g. to support several DX clusters. RBN uses two ports, 7000 +# for CW/RTTY and 7001 for FT8, so if you want both, you need two entries, as shown below. # Feel free to write your own provider classes! providers: -# Some providers don't require any config: - - type: "POTA" - - type: "SOTA" - - type: "WWFF" - - type: "WWBOTA" - - type: "GMA" - - type: "HEMA" - - type: "ParksNPeaks" -# - type: "APRS-IS" -# Some, like DX Clusters, require extra config. You can add multiple DX clusters if you want! + - + type: "POTA" + name: "POTA" + enabled: true + - + type: "SOTA" + name: "SOTA" + enabled: true + - + type: "WWFF" + name: "WWFF" + enabled: true + - + type: "WWBOTA" + name: "WWBOTA" + enabled: true + - + type: "GMA" + name: "GMA" + enabled: true + - + type: "HEMA" + name: "HEMA" + enabled: true + - + type: "ParksNPeaks" + name: "ParksNPeaks" + enabled: true + - + type: "APRS-IS" + name: "APRS-IS" + enabled: false - type: "DXCluster" + name: "HRD Dx Cluster" + enabled: true host: "hrd.wa9pie.net" port: 8000 -# RBN uses two ports, 7000 for CW/RTTY and 7001 for FT8, so if you want both, you need to add two entries: -# - -# type: "RBN" -# port: 7000 -# - -# type: "RBN" -# port: 7001 + - + type: "RBN" + name: "RBN CW/RTTY" + enabled: false + port: 7000 + - + type: "RBN" + name: "RBN FT8" + enabled: false + port: 7001 # Port to open the local web server on web-server-port: 8080 diff --git a/core/constants.py b/core/constants.py index c572c8e..3316771 100644 --- a/core/constants.py +++ b/core/constants.py @@ -1,7 +1,7 @@ from data.band import Band # General software -SOFTWARE_NAME = "Spothole by M0TRT" +SOFTWARE_NAME = "(S)pothole by M0TRT" SOFTWARE_VERSION = "0.1" # Sources diff --git a/main.py b/main.py index 903f6b2..70c0796 100644 --- a/main.py +++ b/main.py @@ -35,32 +35,34 @@ def shutdown(sig, frame): logging.info("Stopping program, this may take a few seconds...") global run run = False - for p in providers: p.stop() + for p in providers: + if p.enabled: + p.stop() cleanup_timer.stop() # Utility method to get a data provider based on its config entry. def get_provider_from_config(config_providers_entry): match config_providers_entry["type"]: case "POTA": - return POTA() + return POTA(config_providers_entry) case "SOTA": - return SOTA() + return SOTA(config_providers_entry) case "WWFF": - return WWFF() + return WWFF(config_providers_entry) case "GMA": - return GMA() + return GMA(config_providers_entry) case "WWBOTA": - return WWBOTA() + return WWBOTA(config_providers_entry) case "HEMA": - return HEMA() + return HEMA(config_providers_entry) case "ParksNPeaks": - return ParksNPeaks() + return ParksNPeaks(config_providers_entry) case "DXCluster": - return DXCluster(config_providers_entry["host"], config_providers_entry["port"]) + return DXCluster(config_providers_entry) case "RBN": - return RBN(config_providers_entry["port"]) + return RBN(config_providers_entry) case "APRS-IS": - return APRSIS() + return APRSIS(config_providers_entry) return None @@ -85,7 +87,9 @@ if __name__ == '__main__': # Set up data providers for p in providers: p.setup(spot_list=spot_list) # Start data providers - for p in providers: p.start() + for p in providers: + if p.enabled: + p.start() # Set up timer to clear spot list of old data cleanup_timer = CleanupTimer(spot_list=spot_list, cleanup_interval=60, max_spot_age=config["max-spot-age-sec"]) @@ -103,6 +107,6 @@ if __name__ == '__main__': status_data["uptime"] = str(datetime.now(pytz.UTC) - startup_time).split(".")[0] status_data["mem_use_mb"] = round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 3) status_data["num_spots"] = len(spot_list) - status_data["providers"] = list(map(lambda p: {"name": p.name(), "status": p.status, "last_updated": p.last_update_time, "last_spot": p.last_spot_time}, providers)) + status_data["providers"] = list(map(lambda p: {"name": p.name, "enabled": p.enabled, "status": p.status, "last_updated": p.last_update_time, "last_spot": p.last_spot_time}, providers)) status_data["cleanup"] = {"status": cleanup_timer.status, "last_ran": cleanup_timer.last_cleanup_time} status_data["webserver"] = {"status": web_server.status, "last_api_access": web_server.last_api_access_time, "last_page_access": web_server.last_page_access_time} diff --git a/providers/aprsis.py b/providers/aprsis.py index ac2d48e..64bb299 100644 --- a/providers/aprsis.py +++ b/providers/aprsis.py @@ -13,15 +13,12 @@ from providers.provider import Provider # Provider for the APRS-IS. class APRSIS(Provider): - def __init__(self): - super().__init__() + def __init__(self, provider_config): + super().__init__(provider_config) self.thread = Thread(target=self.connect) self.thread.daemon = True self.aprsis = None - def name(self): - return "APRS-IS" - def start(self): self.thread.start() diff --git a/providers/dxcluster.py b/providers/dxcluster.py index 4dc945e..93da002 100644 --- a/providers/dxcluster.py +++ b/providers/dxcluster.py @@ -21,18 +21,15 @@ class DXCluster(Provider): re.IGNORECASE) # Constructor requires hostname and port - def __init__(self, hostname, port): - super().__init__() - self.hostname = hostname - self.port = port + def __init__(self, provider_config): + super().__init__(provider_config) + self.hostname = provider_config["host"] + self.port = provider_config["port"] self.telnet = None self.thread = Thread(target=self.handle) self.thread.daemon = True self.run = True - def name(self): - return "DX Cluster " + self.hostname - def start(self): self.thread.start() @@ -67,7 +64,7 @@ class DXCluster(Provider): if match: spot_time = datetime.strptime(match.group(5), "%H%MZ") spot_datetime = datetime.combine(datetime.today(), spot_time.time()).replace(tzinfo=pytz.UTC) - spot = Spot(source="Cluster", + spot = Spot(source=self.name, dx_call=match.group(3), de_call=match.group(1), freq=float(match.group(2)), diff --git a/providers/gma.py b/providers/gma.py index 54e6b47..5d6a67b 100644 --- a/providers/gma.py +++ b/providers/gma.py @@ -16,18 +16,15 @@ class GMA(HTTPProvider): REF_INFO_CACHE_TIME_DAYS = 30 REF_INFO_CACHE = CachedSession("gma_ref_info_cache", expire_after=timedelta(days=REF_INFO_CACHE_TIME_DAYS)) - def __init__(self): - super().__init__(self.SPOTS_URL, self.POLL_INTERVAL_SEC) - - def name(self): - return "GMA" + def __init__(self, provider_config): + super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC) def http_response_to_spots(self, http_response): new_spots = [] # Iterate through source data for source_spot in http_response.json()["RCD"]: # Convert to our spot format - spot = Spot(source=self.name(), + spot = Spot(source=self.name, dx_call=source_spot["ACTIVATOR"].upper(), de_call=source_spot["SPOTTER"].upper(), freq=float(source_spot["QRG"]) if (source_spot["QRG"] != "") else None, # Seen GMA spots with no frequency diff --git a/providers/hema.py b/providers/hema.py index f8c02e8..fa23cff 100644 --- a/providers/hema.py +++ b/providers/hema.py @@ -20,13 +20,10 @@ class HEMA(HTTPProvider): FREQ_MODE_PATTERN = re.compile("^([\\d.]*) \\((.*)\\)$") SPOTTER_COMMENT_PATTERN = re.compile("^\\((.*)\\) (.*)$") - def __init__(self): - super().__init__(self.SPOT_SEED_URL, self.POLL_INTERVAL_SEC) + def __init__(self, provider_config): + super().__init__(provider_config, self.SPOT_SEED_URL, self.POLL_INTERVAL_SEC) self.spot_seed = "" - def name(self): - return "HEMA" - def http_response_to_spots(self, http_response): # OK, source data is actually just the spot seed at this point. We'll then go on to fetch real data if we know # this has changed. @@ -48,7 +45,7 @@ class HEMA(HTTPProvider): spotter_comment_match = re.search(self.SPOTTER_COMMENT_PATTERN, spot_items[6]) # Convert to our spot format - spot = Spot(source=self.name(), + spot = Spot(source=self.name, dx_call=spot_items[2].upper(), de_call=spotter_comment_match.group(1).upper(), freq=float(freq_mode_match.group(1)) * 1000, diff --git a/providers/http_provider.py b/providers/http_provider.py index e28cd59..d31c3f2 100644 --- a/providers/http_provider.py +++ b/providers/http_provider.py @@ -13,20 +13,17 @@ from providers.provider import Provider # duplication. Subclasses of this query the individual APIs for data. class HTTPProvider(Provider): - def __init__(self, url, poll_interval): - super().__init__() + def __init__(self, provider_config, url, poll_interval): + super().__init__(provider_config) self.url = url self.poll_interval = poll_interval self.poll_timer = None - def name(self): - raise NotImplementedError("Subclasses must implement this method") - def start(self): # Fire off a one-shot thread to run poll() for the first time, just to ensure start() returns immediately and # the application can continue starting. The thread itself will then die, and the timer will kick in on its own # thread. - logging.info("Set up query of " + self.name() + " API every " + str(self.poll_interval) + " seconds.") + logging.info("Set up query of " + self.name + " API every " + str(self.poll_interval) + " seconds.") thread = Thread(target=self.poll) thread.daemon = True thread.start() @@ -37,7 +34,7 @@ class HTTPProvider(Provider): def poll(self): try: # Request data from API - logging.debug("Polling " + self.name() + " API...") + logging.debug("Polling " + self.name + " API...") http_response = requests.get(self.url, headers=self.HTTP_HEADERS) # Pass off to the subclass for processing new_spots = self.http_response_to_spots(http_response) @@ -47,11 +44,11 @@ class HTTPProvider(Provider): self.status = "OK" self.last_update_time = datetime.now(pytz.UTC) - logging.debug("Received data from " + self.name() + " API.") + logging.debug("Received data from " + self.name + " API.") except Exception as e: self.status = "Error" - logging.exception("Exception in HTTP JSON Provider (" + self.name() + ")") + logging.exception("Exception in HTTP JSON Provider (" + self.name + ")") sleep(1) self.poll_timer = Timer(self.poll_interval, self.poll) diff --git a/providers/parksnpeaks.py b/providers/parksnpeaks.py index 627b5f5..1bc585d 100644 --- a/providers/parksnpeaks.py +++ b/providers/parksnpeaks.py @@ -12,18 +12,15 @@ class ParksNPeaks(HTTPProvider): POLL_INTERVAL_SEC = 120 SPOTS_URL = "https://www.parksnpeaks.org/api/ALL" - def __init__(self): - super().__init__(self.SPOTS_URL, self.POLL_INTERVAL_SEC) - - def name(self): - return "ParksNPeaks" + def __init__(self, provider_config): + super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC) def http_response_to_spots(self, http_response): new_spots = [] # Iterate through source data for source_spot in http_response.json(): # Convert to our spot format - spot = Spot(source=self.name(), + spot = Spot(source=self.name, source_id=source_spot["actID"], dx_call=source_spot["actCallsign"].upper(), de_call=source_spot["actSpoter"].upper(), # typo exists in API diff --git a/providers/pota.py b/providers/pota.py index ba3dc68..6921de2 100644 --- a/providers/pota.py +++ b/providers/pota.py @@ -11,18 +11,15 @@ class POTA(HTTPProvider): POLL_INTERVAL_SEC = 120 SPOTS_URL = "https://api.pota.app/spot/activator" - def __init__(self): - super().__init__(self.SPOTS_URL, self.POLL_INTERVAL_SEC) - - def name(self): - return "POTA" + def __init__(self, provider_config): + super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC) def http_response_to_spots(self, http_response): new_spots = [] # Iterate through source data for source_spot in http_response.json(): # Convert to our spot format - spot = Spot(source=self.name(), + spot = Spot(source=self.name, source_id=source_spot["spotId"], dx_call=source_spot["activator"].upper(), de_call=source_spot["spotter"].upper(), diff --git a/providers/provider.py b/providers/provider.py index 29a9415..4984115 100644 --- a/providers/provider.py +++ b/providers/provider.py @@ -13,16 +13,14 @@ class Provider: HTTP_HEADERS = { "User-Agent": SOFTWARE_NAME + " " + SOFTWARE_VERSION + " (operated by " + config["server-owner-callsign"] + ")" } # Constructor - def __init__(self): + def __init__(self, provider_config): + self.name = provider_config["name"] + self.enabled = provider_config["enabled"] self.last_update_time = datetime.min.replace(tzinfo=pytz.UTC) self.last_spot_time = datetime.min.replace(tzinfo=pytz.UTC) - self.status = "Not Started" + self.status = "Not Started" if self.enabled else "Disabled" self.spot_list = None - # Return the name of the provider - def name(self): - raise NotImplementedError("Subclasses must implement this method") - # Set up the provider, e.g. giving it the spot list to work from def setup(self, spot_list): self.spot_list = spot_list diff --git a/providers/rbn.py b/providers/rbn.py index 671eb01..85a8819 100644 --- a/providers/rbn.py +++ b/providers/rbn.py @@ -22,17 +22,14 @@ class RBN(Provider): re.IGNORECASE) # Constructor requires port number. - def __init__(self, port): - super().__init__() - self.port = port + def __init__(self, provider_config): + super().__init__(provider_config) + self.port = provider_config["port"] self.telnet = None self.thread = Thread(target=self.handle) self.thread.daemon = True self.run = True - def name(self): - return "RBN port " + str(self.port) - def start(self): self.thread.start() @@ -68,7 +65,7 @@ class RBN(Provider): if match: spot_time = datetime.strptime(match.group(5), "%H%MZ") spot_datetime = datetime.combine(datetime.today(), spot_time.time()).replace(tzinfo=pytz.UTC) - spot = Spot(source="RBN", + spot = Spot(source=self.name, dx_call=match.group(3), de_call=match.group(1), freq=float(match.group(2)), diff --git a/providers/sota.py b/providers/sota.py index 45580c5..e529c27 100644 --- a/providers/sota.py +++ b/providers/sota.py @@ -20,13 +20,10 @@ class SOTA(HTTPProvider): SUMMIT_DATA_CACHE_TIME_DAYS = 30 SUMMIT_DATA_CACHE = CachedSession("sota_summit_data_cache", expire_after=timedelta(days=SUMMIT_DATA_CACHE_TIME_DAYS)) - def __init__(self): - super().__init__(self.EPOCH_URL, self.POLL_INTERVAL_SEC) + def __init__(self, provider_config): + super().__init__(provider_config, self.EPOCH_URL, self.POLL_INTERVAL_SEC) self.api_epoch = "" - def name(self): - return "SOTA" - def http_response_to_spots(self, http_response): # OK, source data is actually just the epoch at this point. We'll then go on to fetch real data if we know this # has changed. @@ -40,7 +37,7 @@ class SOTA(HTTPProvider): # Iterate through source data for source_spot in source_data: # Convert to our spot format - spot = Spot(source=self.name(), + spot = Spot(source=self.name, source_id=source_spot["id"], dx_call=source_spot["activatorCallsign"].upper(), dx_name=source_spot["activatorName"], diff --git a/providers/wwbota.py b/providers/wwbota.py index b5ed89d..4ba21dc 100644 --- a/providers/wwbota.py +++ b/providers/wwbota.py @@ -9,11 +9,8 @@ class WWBOTA(HTTPProvider): POLL_INTERVAL_SEC = 120 SPOTS_URL = "https://api.wwbota.org/spots/" - def __init__(self): - super().__init__(self.SPOTS_URL, self.POLL_INTERVAL_SEC) - - def name(self): - return "WWBOTA" + def __init__(self, provider_config): + super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC) def http_response_to_spots(self, http_response): new_spots = [] @@ -26,7 +23,7 @@ class WWBOTA(HTTPProvider): for ref in source_spot["references"]: refs.append(ref["reference"]) ref_names.append(ref["name"]) - spot = Spot(source=self.name(), + spot = Spot(source=self.name, dx_call=source_spot["call"].upper(), de_call=source_spot["spotter"].upper(), freq=float(source_spot["freq"]) * 1000, # MHz to kHz diff --git a/providers/wwff.py b/providers/wwff.py index ca40158..03ffafa 100644 --- a/providers/wwff.py +++ b/providers/wwff.py @@ -11,18 +11,15 @@ class WWFF(HTTPProvider): POLL_INTERVAL_SEC = 120 SPOTS_URL = "https://spots.wwff.co/static/spots.json" - def __init__(self): - super().__init__(self.SPOTS_URL, self.POLL_INTERVAL_SEC) - - def name(self): - return "WWFF" + def __init__(self, provider_config): + super().__init__(provider_config, self.SPOTS_URL, self.POLL_INTERVAL_SEC) def http_response_to_spots(self, http_response): new_spots = [] # Iterate through source data for source_spot in http_response.json(): # Convert to our spot format - spot = Spot(source=self.name(), + spot = Spot(source=self.name, source_id=source_spot["id"], dx_call=source_spot["activator"].upper(), de_call=source_spot["spotter"].upper(), diff --git a/server/webserver.py b/server/webserver.py index 0714e6b..c2afd25 100644 --- a/server/webserver.py +++ b/server/webserver.py @@ -30,7 +30,6 @@ class WebServer: bottle.get("/api/options")(self.serve_api_options) bottle.get("/api/status")(self.serve_api_status) bottle.get("/")(self.serve_index) - bottle.get("/apidocs")(self.serve_apidocs) bottle.get("/")(self.serve_static_file) # Start the web server @@ -69,20 +68,21 @@ class WebServer: # Serve the home page. This would be accessible as /index.html but we need this workaround to make it available as / def serve_index(self): - self.last_page_access_time = datetime.now(pytz.UTC) - self.status = "OK" - return bottle.static_file("index.html", root="webassets") + return self.serve_static_file("") - # Serve the API docs page. This would be accessible as /apidocs/index.html but we need this workaround to make it - # available as /apidocs - def serve_apidocs(self): - self.last_page_access_time = datetime.now(pytz.UTC) - self.status = "OK" - return bottle.static_file("index.html", root="webassets/apidocs") - - # Serve general static files from "webassets" directory + # Serve general static files from "webassets" directory, along with some extra workarounds to make URLs such as + # "/", "/about" and "/apidocs" work. def serve_static_file(self, filepath): - return bottle.static_file(filepath, root="webassets") + self.last_page_access_time = datetime.now(pytz.UTC) + self.status = "OK" + if filepath == "": + return bottle.static_file("index.html", root="webassets") + elif filepath == "about": + return bottle.static_file("about.html", root="webassets") + elif filepath == "apidocs": + return bottle.static_file("index.html", root="webassets/apidocs") + else: + return bottle.static_file(filepath, root="webassets") # Utility method to apply filters to the overall spot list and return only a subset. Enables query parameters in # the main "spots" GET call. The "query" parameter should be the result of bottle's request.query, and is a MultiDict diff --git a/webassets/apidocs/openapi.yml b/webassets/apidocs/openapi.yml index ae37128..d58b9ae 100644 --- a/webassets/apidocs/openapi.yml +++ b/webassets/apidocs/openapi.yml @@ -1,10 +1,10 @@ openapi: 3.0.4 info: - title: Spothole API + title: (S)pothole API description: |- - Spothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open JSON API as well as a website to browse the data. + (S)pothole is a utility to aggregate "spots" from amateur radio DX clusters and xOTA spotting sites, and provide an open JSON API as well as a website to browse the data. - While there are other web-based interfaces to DX clusters, and sites that aggregate spots from various outfoor activity programmes for amateur radio, Spothole differentiates itself by supporting a large number of data sources, and by being "API first" rather than just providing a web front-end. This allows other software to be built on top of it. Spothole itself is also open source, Public Domain licenced code that anyone can take and modify. + While there are other web-based interfaces to DX clusters, and sites that aggregate spots from various outfoor activity programmes for amateur radio, (S)pothole differentiates itself by supporting a large number of data sources, and by being "API first" rather than just providing a web front-end. This allows other software to be built on top of it. Spothole itself is also open source, Public Domain licenced code that anyone can take and modify. contact: email: ian@ianrenton.com license: @@ -531,6 +531,10 @@ components: type: string description: The name of the provider. example: POTA + enabled: + type: boolean + description: Whether the provider is enabled or not. + example: true status: type: string description: The status of the provider. diff --git a/webassets/index.html b/webassets/index.html index f0977be..5373dce 100644 --- a/webassets/index.html +++ b/webassets/index.html @@ -4,7 +4,7 @@ - Spothole + (S)pothole
- Spothole + (S)pothole