Implement web server metrics in Tornado #3

core/status_reporter.py

@@ -58,13 +58,17 @@ class StatusReporter:
         self.status_data["cleanup"] = {"status": self.cleanup_timer.status,
                                        "last_ran": self.cleanup_timer.last_cleanup_time.replace(
                                            tzinfo=pytz.UTC).timestamp() if self.cleanup_timer.last_cleanup_time else 0}
-        self.status_data["webserver"] = {"status": self.web_server.status,
+        self.status_data["webserver"] = {"status": self.web_server.web_server_metrics["status"],
-                                         "last_api_access": self.web_server.last_api_access_time.replace(
+                                         "last_api_access": self.web_server.web_server_metrics[
-                                             tzinfo=pytz.UTC).timestamp() if self.web_server.last_api_access_time else 0,
+                                             "last_api_access_time"].replace(
-                                         "api_access_count": self.web_server.api_access_counter,
+                                             tzinfo=pytz.UTC).timestamp() if self.web_server.web_server_metrics[
-                                         "last_page_access": self.web_server.last_page_access_time.replace(
+                                             "last_api_access_time"] else 0,
-                                             tzinfo=pytz.UTC).timestamp() if self.web_server.last_page_access_time else 0,
+                                         "api_access_count": self.web_server.web_server_metrics["api_access_counter"],
-                                         "page_access_count": self.web_server.page_access_counter}
+                                         "last_page_access": self.web_server.web_server_metrics[
+                                             "last_page_access_time"].replace(
+                                             tzinfo=pytz.UTC).timestamp() if self.web_server.web_server_metrics[
+                                             "last_page_access_time"] else 0,
+                                         "page_access_count": self.web_server.web_server_metrics["page_access_counter"]}
 
         # Update Prometheus metrics
         memory_use_gauge.set(psutil.Process(os.getpid()).memory_info().rss * 1024)

server/handlers/api/addspot.py

@@ -1,12 +1,15 @@
 import json
 import logging
 import re
+from datetime import datetime
 
+import pytz
 import tornado
 
 from core.config import ALLOW_SPOTTING, MAX_SPOT_AGE
 from core.constants import UNKNOWN_BAND
 from core.lookup_helper import lookup_helper
+from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig
 from core.utils import serialize_everything
 from data.sig_ref import SIGRef
@@ -15,11 +18,18 @@ from data.spot import Spot
 
 # API request handler for /api/v1/spot (POST)
 class APISpotHandler(tornado.web.RequestHandler):
-    def initialize(self, spots):
+    def initialize(self, spots, web_server_metrics):
         self.spots = spots
+        self.web_server_metrics = web_server_metrics
 
     def post(self):
         try:
+            # Metrics
+            self.web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self.web_server_metrics["api_access_counter"] += 1
+            self.web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
             # Reject if not allowed
             if not ALLOW_SPOTTING:
                 self.set_status(401)

server/handlers/api/alerts.py

@@ -5,16 +5,24 @@ from datetime import datetime
 import pytz
 import tornado
 
+from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import serialize_everything
 
 
 # API request handler for /api/v1/alerts
 class APIAlertsHandler(tornado.web.RequestHandler):
-    def initialize(self, alerts):
+    def initialize(self, alerts, web_server_metrics):
         self.alerts = alerts
+        self.web_server_metrics = web_server_metrics
 
     def get(self):
         try:
+            # Metrics
+            self.web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self.web_server_metrics["api_access_counter"] += 1
+            self.web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
@@ -39,6 +47,9 @@ class APIAlertsStreamHandler(tornado.web.RequestHandler):
     def get(self):
         # todo
         # try:
+        #     # Metrics
+        #     api_requests_counter.inc()
+        #
         #     response.content_type  = 'text/event-stream'
         #     response.cache_control = 'no-cache'
         #     yield 'retry: 1000\n\n'

server/handlers/api/lookups.py

@@ -1,10 +1,13 @@
 import json
 import logging
 import re
+from datetime import datetime
 
+import pytz
 import tornado
 
 from core.constants import SIGS
+from core.prometheus_metrics_handler import api_requests_counter
 from core.sig_utils import get_ref_regex_for_sig, populate_sig_ref_info
 from core.utils import serialize_everything
 from data.sig_ref import SIGRef
@@ -13,8 +16,17 @@ from data.spot import Spot
 
 # API request handler for /api/v1/lookup/call
 class APILookupCallHandler(tornado.web.RequestHandler):
+    def initialize(self, web_server_metrics):
+        self.web_server_metrics = web_server_metrics
+
     def get(self):
         try:
+            # Metrics
+            self.web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self.web_server_metrics["api_access_counter"] += 1
+            self.web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
@@ -63,8 +75,17 @@ class APILookupCallHandler(tornado.web.RequestHandler):
 
 # API request handler for /api/v1/lookup/sigref
 class APILookupSIGRefHandler(tornado.web.RequestHandler):
+    def initialize(self, web_server_metrics):
+        self.web_server_metrics = web_server_metrics
+
     def get(self):
         try:
+            # Metrics
+            self.web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self.web_server_metrics["api_access_counter"] += 1
+            self.web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}

server/handlers/api/options.py

@@ -1,18 +1,28 @@
 import json
+from datetime import datetime
 
+import pytz
 import tornado
 
 from core.config import MAX_SPOT_AGE, ALLOW_SPOTTING, WEB_UI_OPTIONS
 from core.constants import BANDS, ALL_MODES, MODE_TYPES, SIGS, CONTINENTS
+from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import serialize_everything
 
 
 # API request handler for /api/v1/options
 class APIOptionsHandler(tornado.web.RequestHandler):
-    def initialize(self, status_data):
+    def initialize(self, status_data, web_server_metrics):
         self.status_data = status_data
+        self.web_server_metrics = web_server_metrics
 
     def get(self):
+        # Metrics
+        self.web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+        self.web_server_metrics["api_access_counter"] += 1
+        self.web_server_metrics["status"] = "OK"
+        api_requests_counter.inc()
+
         options = {"bands": BANDS,
                    "modes": ALL_MODES,
                    "mode_types": MODE_TYPES,

server/handlers/api/spots.py

@@ -5,16 +5,24 @@ from datetime import datetime, timedelta
 import pytz
 import tornado
 
+from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import serialize_everything
 
 
 # API request handler for /api/v1/spots
 class APISpotsHandler(tornado.web.RequestHandler):
-    def initialize(self, spots):
+    def initialize(self, spots, web_server_metrics):
         self.spots = spots
+        self.web_server_metrics = web_server_metrics
 
     def get(self):
         try:
+            # Metrics
+            self.web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+            self.web_server_metrics["api_access_counter"] += 1
+            self.web_server_metrics["status"] = "OK"
+            api_requests_counter.inc()
+
             # request.arguments contains lists for each param key because technically the client can supply multiple,
             # reduce that to just the first entry, and convert bytes to string
             query_params = {k: v[0].decode("utf-8") for k, v in self.request.arguments.items()}
@@ -40,6 +48,9 @@ class APISpotsStreamHandler(tornado.web.RequestHandler):
     def get(self):
         # todo
         # try:
+        #     # Metrics
+        #     api_requests_counter.inc()
+        #
         #     response.content_type  = 'text/event-stream'
         #     response.cache_control = 'no-cache'
         #     yield 'retry: 1000\n\n'

server/handlers/api/status.py

@@ -1,16 +1,26 @@
 import json
+from datetime import datetime
 
+import pytz
 import tornado
 
+from core.prometheus_metrics_handler import api_requests_counter
 from core.utils import serialize_everything
 
 
 # API request handler for /api/v1/status
 class APIStatusHandler(tornado.web.RequestHandler):
-    def initialize(self, status_data):
+    def initialize(self, status_data, web_server_metrics):
         self.status_data = status_data
+        self.web_server_metrics = web_server_metrics
 
     def get(self):
+        # Metrics
+        self.web_server_metrics["last_api_access_time"] = datetime.now(pytz.UTC)
+        self.web_server_metrics["api_access_counter"] += 1
+        self.web_server_metrics["status"] = "OK"
+        api_requests_counter.inc()
+
         self.write(json.dumps(self.status_data, default=serialize_everything))
         self.set_status(200)
         self.set_header("Cache-Control", "no-store")

server/handlers/pagetemplate.py

@@ -1,14 +1,26 @@
+from datetime import datetime
+
+import pytz
 import tornado
 
 from core.config import ALLOW_SPOTTING
 from core.constants import SOFTWARE_VERSION
+from core.prometheus_metrics_handler import page_requests_counter
 
 
 # Handler for all HTML pages generated from templates
 class PageTemplateHandler(tornado.web.RequestHandler):
-    def initialize(self, template_name):
+    def initialize(self, template_name, web_server_metrics):
         self.template_name = template_name
+        self.web_server_metrics = web_server_metrics
 
     def get(self):
+        # Metrics
+        self.web_server_metrics["last_page_access_time"] = datetime.now(pytz.UTC)
+        self.web_server_metrics["page_access_counter"] += 1
+        self.web_server_metrics["status"] = "OK"
+        page_requests_counter.inc()
+
         # Load named template, and provide variables used in templates
         self.render(self.template_name + ".html", software_version=SOFTWARE_VERSION, allow_spotting=ALLOW_SPOTTING)
+

server/webserver.py

@@ -15,35 +15,25 @@ from server.handlers.pagetemplate import PageTemplateHandler
 
 
 # Provides the public-facing web server.
-# TODO test lookups
 # TODO SSE API responses
 # TODO clean_up_sse_queues
-# TODO page & API access counters - how to do from a subclass handler? e.g.
-#         self.last_api_access_time = datetime.now(pytz.UTC)
-#         self.api_access_counter += 1
-#         api_requests_counter.inc()
-#         self.status = "OK"
-#
-#         self.last_page_access_time = datetime.now(pytz.UTC)
-#         self.page_access_counter += 1
-#         page_requests_counter.inc()
-#         self.status = "OK"
-
 class WebServer:
     # Constructor
     def __init__(self, spots, alerts, status_data, port):
-        self.last_page_access_time = None
-        self.last_api_access_time = None
-        self.page_access_counter = 0
-        self.api_access_counter = 0
         self.spots = spots
         self.alerts = alerts
         self.sse_spot_queues = []
         self.sse_alert_queues = []
         self.status_data = status_data
         self.port = port
-        self.status = "Starting"
         self.shutdown_event = asyncio.Event()
+        self.web_server_metrics = {
+            "last_page_access_time": None,
+            "last_api_access_time": None,
+            "page_access_counter": 0,
+            "api_access_counter": 0,
+            "status": "Starting"
+        }
 
     # Start the web server
     def start(self):
@@ -57,24 +47,24 @@ class WebServer:
     async def start_inner(self):
         app = tornado.web.Application([
             # Routes for API calls
-            (r"/api/v1/spots", APISpotsHandler, {"spots": self.spots}),
+            (r"/api/v1/spots", APISpotsHandler, {"spots": self.spots, "web_server_metrics": self.web_server_metrics}),
-            (r"/api/v1/alerts", APIAlertsHandler, {"alerts": self.alerts}),
+            (r"/api/v1/alerts", APIAlertsHandler, {"alerts": self.alerts, "web_server_metrics": self.web_server_metrics}),
             (r"/api/v1/spots/stream", APISpotsStreamHandler), # todo provide queues?
             (r"/api/v1/alerts/stream", APIAlertsStreamHandler), # todo provide queues?
-            (r"/api/v1/options", APIOptionsHandler, {"status_data": self.status_data}),
+            (r"/api/v1/options", APIOptionsHandler, {"status_data": self.status_data, "web_server_metrics": self.web_server_metrics}),
-            (r"/api/v1/status", APIStatusHandler, {"status_data": self.status_data}),
+            (r"/api/v1/status", APIStatusHandler, {"status_data": self.status_data, "web_server_metrics": self.web_server_metrics}),
-            (r"/api/v1/lookup/call", APILookupCallHandler),
+            (r"/api/v1/lookup/call", APILookupCallHandler, {"web_server_metrics": self.web_server_metrics}),
-            (r"/api/v1/lookup/sigref", APILookupSIGRefHandler),
+            (r"/api/v1/lookup/sigref", APILookupSIGRefHandler, {"web_server_metrics": self.web_server_metrics}),
-            (r"/api/v1/spot", APISpotHandler, {"spots": self.spots}),
+            (r"/api/v1/spot", APISpotHandler, {"spots": self.spots, "web_server_metrics": self.web_server_metrics}),
             # Routes for templated pages
-            (r"/", PageTemplateHandler, {"template_name": "spots"}),
+            (r"/", PageTemplateHandler, {"template_name": "spots", "web_server_metrics": self.web_server_metrics}),
-            (r"/map", PageTemplateHandler, {"template_name": "map"}),
+            (r"/map", PageTemplateHandler, {"template_name": "map", "web_server_metrics": self.web_server_metrics}),
-            (r"/bands", PageTemplateHandler, {"template_name": "bands"}),
+            (r"/bands", PageTemplateHandler, {"template_name": "bands", "web_server_metrics": self.web_server_metrics}),
-            (r"/alerts", PageTemplateHandler, {"template_name": "alerts"}),
+            (r"/alerts", PageTemplateHandler, {"template_name": "alerts", "web_server_metrics": self.web_server_metrics}),
-            (r"/add-spot", PageTemplateHandler, {"template_name": "add_spot"}),
+            (r"/add-spot", PageTemplateHandler, {"template_name": "add_spot", "web_server_metrics": self.web_server_metrics}),
-            (r"/status", PageTemplateHandler, {"template_name": "status"}),
+            (r"/status", PageTemplateHandler, {"template_name": "status", "web_server_metrics": self.web_server_metrics}),
-            (r"/about", PageTemplateHandler, {"template_name": "about"}),
+            (r"/about", PageTemplateHandler, {"template_name": "about", "web_server_metrics": self.web_server_metrics}),
-            (r"/apidocs", PageTemplateHandler, {"template_name": "apidocs"}),
+            (r"/apidocs", PageTemplateHandler, {"template_name": "apidocs", "web_server_metrics": self.web_server_metrics}),
             # Route for Prometheus metrics
             (r"/metrics", PrometheusMetricsHandler),
             # Default route to serve from "webassets"