Merge pull request #12754 from kdelee/fix_metrics_consumed_capacity

calcuate consumed capacity in same way in metrics

awx/main/analytics/collectors.py

@@ -16,6 +16,7 @@ from awx.conf.license import get_license
 from awx.main.utils import get_awx_version, camelcase_to_underscore, datetime_hook
 from awx.main import models
 from awx.main.analytics import register
+from awx.main.scheduler.task_manager_models import TaskManagerInstances
 
 """
 This module is used to define metrics collected by awx.main.analytics.gather()
@@ -235,25 +236,25 @@ def projects_by_scm_type(since, **kwargs):
 @register('instance_info', '1.2', description=_('Cluster topology and capacity'))
 def instance_info(since, include_hostnames=False, **kwargs):
     info = {}
-    instances = models.Instance.objects.values_list('hostname').values(
+    # Use same method that the TaskManager does to compute consumed capacity without querying all running jobs for each Instance
-        'uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'hostname', 'enabled'
+    active_tasks = models.UnifiedJob.objects.filter(status__in=['running', 'waiting']).only('task_impact', 'controller_node', 'execution_node')
-    )
+    tm_instances = TaskManagerInstances(active_tasks, instance_fields=['uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'enabled'])
-    for instance in instances:
+    for tm_instance in tm_instances.instances_by_hostname.values():
-        consumed_capacity = sum(x.task_impact for x in models.UnifiedJob.objects.filter(execution_node=instance['hostname'], status__in=('running', 'waiting')))
+        instance = tm_instance.obj
         instance_info = {
-            'uuid': instance['uuid'],
+            'uuid': instance.uuid,
-            'version': instance['version'],
+            'version': instance.version,
-            'capacity': instance['capacity'],
+            'capacity': instance.capacity,
-            'cpu': instance['cpu'],
+            'cpu': instance.cpu,
-            'memory': instance['memory'],
+            'memory': instance.memory,
-            'managed_by_policy': instance['managed_by_policy'],
+            'managed_by_policy': instance.managed_by_policy,
-            'enabled': instance['enabled'],
+            'enabled': instance.enabled,
-            'consumed_capacity': consumed_capacity,
+            'consumed_capacity': tm_instance.consumed_capacity,
-            'remaining_capacity': instance['capacity'] - consumed_capacity,
+            'remaining_capacity': instance.capacity - tm_instance.consumed_capacity,
         }
         if include_hostnames is True:
-            instance_info['hostname'] = instance['hostname']
+            instance_info['hostname'] = instance.hostname
-        info[instance['uuid']] = instance_info
+        info[instance.uuid] = instance_info
     return info
 
 

awx/main/scheduler/task_manager_models.py

@@ -34,12 +34,10 @@ class TaskManagerInstance:
 
 
 class TaskManagerInstances:
-    def __init__(self, active_tasks, instances=None):
+    def __init__(self, active_tasks, instances=None, instance_fields=('node_type', 'capacity', 'hostname', 'enabled')):
         self.instances_by_hostname = dict()
         if instances is None:
-            instances = (
+            instances = Instance.objects.filter(hostname__isnull=False, enabled=True).exclude(node_type='hop').only(*instance_fields)
-                Instance.objects.filter(hostname__isnull=False, enabled=True).exclude(node_type='hop').only('node_type', 'capacity', 'hostname', 'enabled')
-            )
         for instance in instances:
             self.instances_by_hostname[instance.hostname] = TaskManagerInstance(instance)
 

tools/grafana/dashboards/demo_dashboard.json

@@ -85,10 +85,96 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
+        "w": 24,
         "x": 0,
         "y": 0
       },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "awx_status_total",
+          "refId": "A"
+        }
+      ],
+      "title": "job status",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
       "id": 12,
       "options": {
         "legend": {
@@ -199,7 +285,7 @@
       "gridPos": {
         "h": 8,
         "w": 12,
-        "x": 0,
+        "x": 12,
         "y": 8
       },
       "id": 10,
@@ -458,8 +544,8 @@
       "gridPos": {
         "h": 8,
         "w": 12,
-        "x": 0,
+        "x": 12,
-        "y": 24
+        "y": 16
       },
       "id": 18,
       "options": {
@@ -556,9 +642,9 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 32
+        "y": 24
       },
-      "id": 8,
+      "id": 14,
       "options": {
         "legend": {
           "calcs": [],
@@ -576,11 +662,14 @@
             "type": "prometheus",
             "uid": "PBFA97CFB590B2093"
           },
-          "expr": "awx_status_total",
+          "editorMode": "builder",
+          "expr": "awx_database_connections_total",
+          "legendFormat": "__auto",
+          "range": true,
           "refId": "A"
         }
       ],
-      "title": "job status",
+      "title": "Database",
       "type": "timeseries"
     },
     {
@@ -641,10 +730,10 @@
       "gridPos": {
         "h": 8,
         "w": 12,
-        "x": 0,
+        "x": 12,
-        "y": 40
+        "y": 24
       },
-      "id": 14,
+      "id": 20,
       "options": {
         "legend": {
           "calcs": [],
@@ -663,13 +752,13 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "builder",
-          "expr": "awx_database_connections_total",
+          "expr": "awx_instance_consumed_capacity",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Database",
+      "title": "Consumed Instance Capacity",
       "type": "timeseries"
     }
   ],
@@ -688,6 +777,6 @@
   "timezone": "",
   "title": "awx-demo",
   "uid": "GISWZOXnk",
-  "version": 2,
+  "version": 4,
   "weekStart": ""
 }