Merge pull request #8075 from ryanpetrello/redis-capacity-check

if redis is unreachable, set instance capacity to zero

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

awx/main/models/ha.py

@@ -12,6 +12,7 @@ from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
 from django.utils.timezone import now, timedelta
 
+import redis
 from solo.models import SingletonModel
 
 from awx import __version__ as awx_application_version
@@ -152,6 +153,14 @@ class Instance(HasPolicyEditsMixin, BaseModel):
             self.capacity = get_system_task_capacity(self.capacity_adjustment)
         else:
             self.capacity = 0
+
+        try:
+            # if redis is down for some reason, that means we can't persist
+            # playbook event data; we should consider this a zero capacity event
+            redis.Redis.from_url(settings.BROKER_URL).ping()
+        except redis.ConnectionError:
+            self.capacity = 0
+
         self.cpu = cpu[0]
         self.memory = mem[0]
         self.cpu_capacity = cpu[1]

awx/main/tests/functional/test_jobs.py

@@ -1,3 +1,4 @@
+import redis
 import pytest
 from unittest import mock
 import json
@@ -25,7 +26,8 @@ def test_orphan_unified_job_creation(instance, inventory):
 @mock.patch('awx.main.utils.common.get_mem_capacity', lambda: (8000,62))
 def test_job_capacity_and_with_inactive_node():
     i = Instance.objects.create(hostname='test-1')
-    i.refresh_capacity()
+    with mock.patch.object(redis.client.Redis, 'ping', lambda self: True):
+        i.refresh_capacity()
     assert i.capacity == 62
     i.enabled = False
     i.save()
@@ -35,6 +37,19 @@ def test_job_capacity_and_with_inactive_node():
         assert i.capacity == 0
 
 
+@pytest.mark.django_db
+@mock.patch('awx.main.utils.common.get_cpu_capacity', lambda: (2,8))
+@mock.patch('awx.main.utils.common.get_mem_capacity', lambda: (8000,62))
+def test_job_capacity_with_redis_disabled():
+    i = Instance.objects.create(hostname='test-1')
+
+    def _raise(self):
+        raise redis.ConnectionError()
+    with mock.patch.object(redis.client.Redis, 'ping', _raise):
+        i.refresh_capacity()
+    assert i.capacity == 0
+
+
 @pytest.mark.django_db
 def test_job_type_name():
     job = Job.objects.create()