Automate bootstrapping of Prometheus in the development environment

.gitignore

@@ -153,9 +153,6 @@ use_dev_supervisor.txt
 /sanity/
 /awx_collection_build/
 
-# Setup for metrics gathering
-tools/prometheus/prometheus.yml
-
 .idea/*
 *.unison.tmp
 *.#

docs/prometheus.md

@@ -1,18 +0,0 @@
-# Prometheus Container
-
-## Development
-AWX comes with an example Prometheus container and `make` target.  To use it:
-
-1.  Edit `tools/prometheus/prometheus.yml` and update the `basic_auth` section
-    to specify a valid user/password for an AWX user you've created.
-    Alternatively, you can provide an OAuth2 token (which can be generated at
-    `/api/v2/users/N/personal_tokens/`).  
-
-    > Note: By default, the config assumes a user with username=admin and password=password.
-
-2.  Start the Prometheus container:
-    `make prometheus`
-3.  The Prometheus UI will now be accessible at `http://localhost:9090/graph`.
-
-There should be no extra setup needed.  You can try executing this query in the
-UI to get back the number of active sessions: `awx_sessions_total`

tools/docker-compose/README.md

@@ -469,28 +469,14 @@ Once the playbook is done running Splunk should now be setup in your development
 
 ### Prometheus and Grafana integration
 
-Prometheus is a metrics collecting tool, and we support prometheus formatted data at the `api/v2/metrics` endpoint.
-
-Before you run anything, you should perform this basic setup:
-
-1. Copy the prometheus configuration:
+Prometheus is a metrics collecting tool, and we support prometheus formatted data at the `api/v2/metrics` endpoint. To run the development environment (see [docs](https://github.com/ansible/awx/blob/devel/tools/docker-compose/README.md)) with Prometheus and Grafana enabled, set the following variables:
 
 ```
-cp tools/prometheus/prometheus.yml.example tools/prometheus/prometheus.yml
+$ PROMETHEUS=yes GRAFANA=yes make docker-compose
 ```
 
-Set the `username` and `password` in that file to your AWX user. You can also change the scrape interval.
-
-2. (optional) if you are in a clustered environment, you can change the target to `haproxy:8043` so that the incoming prometheus requests go through the load balancer. Leaving it set to `awx1` also works.
-
-You can use this as part of the docker-compose target:
-
-```
-PROMETHEUS=true GRAFANA=true make docker-compose
-```
-
-3. navigate to `http://localhost:9090/targets` and check that the metrics endpoint State is Up.
-4. Click the Graph tab, start typing a metric name, or use the Open metrics explorer button to find a metric to display (next to `Execute` button)
-5. Navigate to `http://localhost:3001`. Sign in, using `admin` for both username and password.
-6. In the left navigation menu go to Dashboards->Browse, find the "awx-demo" and click. These should have graphs.
-6. Now you can modify these and add panels for whichever metrics you like.
+1. navigate to `http://localhost:9090/targets` and check that the metrics endpoint State is Up.
+2. Click the Graph tab, start typing a metric name, or use the Open metrics explorer button to find a metric to display (next to `Execute` button)
+3. Navigate to `http://localhost:3001`. Sign in, using `admin` for both username and password.
+4. In the left navigation menu go to Dashboards->Browse, find the "awx-demo" and click. These should have graphs.
+5. Now you can modify these and add panels for whichever metrics you like.

tools/docker-compose/ansible/roles/sources/tasks/main.yml

@@ -122,3 +122,9 @@
     mode: '0600'
   with_sequence: start=1 end={{ execution_node_count if execution_node_count | int > 0 else 1}}
   when: execution_node_count | int > 0
+
+- name: Render prometheus config
+  template:
+    src: "prometheus.yml.j2"
+    dest: "{{ sources_dest }}/prometheus.yml"
+  when: enable_prometheus|bool

tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2

@@ -1,3 +1,4 @@
+#jinja2: lstrip_blocks: True
 ---
 version: '2.1'
 services:
@@ -142,28 +143,27 @@ services:
   prometheus:
     image: prom/prometheus:latest
     container_name: tools_prometheus_1
-    hostname: splunk
+    hostname: prometheus
     ports:
       - "9090:9090"
     volumes:
-      - "../../prometheus:/etc/prometheus"
+      - "../../docker-compose/_sources/prometheus.yml:/etc/prometheus/prometheus.yml"
       - "prometheus_storage:/prometheus:rw"
     links:
-      - awx_1:awx1
+{% for i in range(control_plane_node_count|int) %}
+      - awx_{{ loop.index }}:awx{{ loop.index }} # because underscores are not valid in hostnames
+{% endfor %}
 {% endif %}
 {% if enable_grafana|bool %}
   grafana:
     image: grafana/grafana-enterprise:latest
     container_name: tools_grafana_1
-    hostname: splunk
+    hostname: grafana
     ports:
       - "3001:3000"
     volumes:
       - "../../grafana:/etc/grafana/provisioning"
       - "grafana_storage:/var/lib/grafana:rw"
-    environment:
-      SPLUNK_START_ARGS: --accept-license
-      SPLUNK_PASSWORD: splunk_admin
     links:
       - prometheus
     depends_on:
@@ -200,7 +200,7 @@ services:
       - "5555:5555"
     volumes:
       - "../../docker-compose/_sources/receptor/receptor-hop.conf:/etc/receptor/receptor.conf"
-  {% for i in range(execution_node_count|int) -%}
+  {% for i in range(execution_node_count|int) %}
   receptor-{{ loop.index }}:
     image: "{{ awx_image }}:{{ awx_image_tag }}"
     user: "{{ ansible_user_uid }}"

tools/docker-compose/ansible/roles/sources/templates/prometheus.yml.j2

@@ -1,3 +1,4 @@
+#jinja2: lstrip_blocks: True
 ---
 global:
   scrape_interval: 15s  # Set the scrape interval to every 15 seconds. Default is every 1 minute.
@@ -5,14 +6,15 @@ global:
 scrape_configs:
   - job_name: 'awx'
     static_configs:
-      - targets: ['awx1:8043']  # or haproxy:8043 in cluster env
-    tls_config:
-      insecure_skip_verify: true
+    - targets:
+      # metrics are broadcast to all nodes in the cluster,
+      # so no need to track nodes individually.
+      - awx1:8013
     metrics_path: /api/v2/metrics
     scrape_interval: 5s
-    scheme: https
+    scheme: http
     params:
       format: ['txt']
     basic_auth:
-      username: admin  # change this
-      password: password  # change this
+      username: admin
+      password: {{ admin_password }}