Clean up work_type processing and fix execution vs control capacity (#10930)

* Clean up added work_type processing for mesh_code branch

* track both execution and control capacity

* Remove unused execution_capacity property

* Count all forms of capacity to make test pass

* Force jobs to be on execution nodes, updates on control nodes

* Introduce capacity_type property to abstract some details out

* Update test to cover all job types at same time

* Register OpenShift nodes as control types

* Remove unqualified consumed_capacity from task manager and make unit tests work

* Remove unqualified consumed_capacity from task manager and make unit tests work

* Update unit test to execution vs control TM logic changes

* Fix bug, else handling for work_type method

awx/main/tests/functional/conftest.py

@@ -121,7 +121,7 @@ def run_computed_fields_right_away(request):
 
 @pytest.fixture
 @mock.patch.object(Project, "update", lambda self, **kwargs: None)
-def project(instance, organization):
+def project(organization):
     prj = Project.objects.create(
         name="test-proj",
         description="test-proj-desc",
@@ -136,7 +136,7 @@ def project(instance, organization):
 
 @pytest.fixture
 @mock.patch.object(Project, "update", lambda self, **kwargs: None)
-def manual_project(instance, organization):
+def manual_project(organization):
     prj = Project.objects.create(
         name="test-manual-proj",
         description="manual-proj-desc",
@@ -196,7 +196,7 @@ def instance(settings):
 
 
 @pytest.fixture
-def organization(instance):
+def organization():
     return Organization.objects.create(name="test-org", description="test-org-desc")
 
 

awx/main/tests/functional/task_management/test_scheduler.py

@@ -7,6 +7,7 @@ from awx.main.scheduler import TaskManager
 from awx.main.scheduler.dependency_graph import DependencyGraph
 from awx.main.utils import encrypt_field
 from awx.main.models import WorkflowJobTemplate, JobTemplate, Job
+from awx.main.models.ha import Instance, InstanceGroup
 
 
 @pytest.mark.django_db
@@ -99,6 +100,48 @@ class TestJobLifeCycle:
                 self.run_tm(tm, expect_schedule=[mock.call()])
             wfjts[0].refresh_from_db()
 
+    @pytest.fixture
+    def control_instance(self):
+        '''Control instance in the controlplane automatic IG'''
+        ig = InstanceGroup.objects.create(name='controlplane')
+        inst = Instance.objects.create(hostname='control-1', node_type='control', capacity=500)
+        ig.instances.add(inst)
+        return inst
+
+    @pytest.fixture
+    def execution_instance(self):
+        '''Execution node in the automatic default IG'''
+        ig = InstanceGroup.objects.create(name='default')
+        inst = Instance.objects.create(hostname='receptor-1', node_type='execution', capacity=500)
+        ig.instances.add(inst)
+        return inst
+
+    def test_control_and_execution_instance(self, project, system_job_template, job_template, inventory_source, control_instance, execution_instance):
+        assert Instance.objects.count() == 2
+
+        pu = project.create_unified_job()
+        sj = system_job_template.create_unified_job()
+        job = job_template.create_unified_job()
+        inv_update = inventory_source.create_unified_job()
+
+        all_ujs = (pu, sj, job, inv_update)
+        for uj in all_ujs:
+            uj.signal_start()
+
+        tm = TaskManager()
+        self.run_tm(tm)
+
+        for uj in all_ujs:
+            uj.refresh_from_db()
+            assert uj.status == 'waiting'
+
+        for uj in (pu, sj):  # control plane jobs
+            assert uj.capacity_type == 'control'
+            assert [uj.execution_node, uj.controller_node] == [control_instance.hostname, control_instance.hostname], uj
+        for uj in (job, inv_update):  # user-space jobs
+            assert uj.capacity_type == 'execution'
+            assert [uj.execution_node, uj.controller_node] == [execution_instance.hostname, control_instance.hostname], uj
+
 
 @pytest.mark.django_db
 def test_single_jt_multi_job_launch_blocks_last(default_instance_group, job_template_factory, mocker):

awx/main/tests/functional/test_instances.py

@@ -68,7 +68,7 @@ class TestPolicyTaskScheduling:
 
 
 @pytest.mark.django_db
-def test_instance_dup(org_admin, organization, project, instance_factory, instance_group_factory, get, system_auditor):
+def test_instance_dup(org_admin, organization, project, instance_factory, instance_group_factory, get, system_auditor, instance):
     i1 = instance_factory("i1")
     i2 = instance_factory("i2")
     i3 = instance_factory("i3")
@@ -83,7 +83,7 @@ def test_instance_dup(org_admin, organization, project, instance_factory, instan
     api_num_instances_oa = list(list_response2.data.items())[0][1]
 
     assert actual_num_instances == api_num_instances_auditor
-    # Note: The org_admin will not see the default 'tower' node because it is not in it's group, as expected
+    # Note: The org_admin will not see the default 'tower' node (instance fixture) because it is not in it's group, as expected
     assert api_num_instances_oa == (actual_num_instances - 1)
 
 

awx/main/tests/unit/models/test_ha.py

@@ -17,8 +17,9 @@ def test_capacity_adjustment_no_save(capacity_adjustment):
 
 
 def T(impact):
-    j = mock.Mock(spec_set=['task_impact'])
+    j = mock.Mock(spec_set=['task_impact', 'capacity_type'])
     j.task_impact = impact
+    j.capacity_type = 'execution'
     return j
 
 
@@ -35,11 +36,13 @@ def Is(param):
             inst = Mock()
             inst.capacity = capacity
             inst.jobs_running = jobs_running
+            inst.node_type = 'execution'
             instances.append(inst)
     else:
         for i in param:
             inst = Mock()
             inst.remaining_capacity = i
+            inst.node_type = 'execution'
             instances.append(inst)
     return instances
 

awx/main/tests/unit/test_capacity.py

@@ -3,10 +3,16 @@ import pytest
 from awx.main.models import InstanceGroup
 
 
+class FakeMeta(object):
+    model_name = 'job'
+
+
 class FakeObject(object):
     def __init__(self, **kwargs):
         for k, v in kwargs.items():
             setattr(self, k, v)
+            self._meta = FakeMeta()
+            self._meta.concrete_model = self
 
 
 class Job(FakeObject):
@@ -85,7 +91,7 @@ def test_offline_node_running(sample_cluster):
     ig_small.instance_list[0].capacity = 0
     tasks = [Job(status='running', execution_node='i1', instance_group=ig_small)]
     capacities = InstanceGroup.objects.capacity_values(qs=[default, ig_large, ig_small], tasks=tasks)
-    assert capacities['ig_small']['consumed_capacity'] == 43
+    assert capacities['ig_small']['consumed_execution_capacity'] == 43
 
 
 def test_offline_node_waiting(sample_cluster):
@@ -96,7 +102,7 @@ def test_offline_node_waiting(sample_cluster):
     ig_small.instance_list[0].capacity = 0
     tasks = [Job(status='waiting', instance_group=ig_small)]
     capacities = InstanceGroup.objects.capacity_values(qs=[default, ig_large, ig_small], tasks=tasks)
-    assert capacities['ig_small']['consumed_capacity'] == 43
+    assert capacities['ig_small']['consumed_execution_capacity'] == 43
 
 
 def test_RBAC_reduced_filter(sample_cluster):

awx/main/tests/unit/utils/test_common.py

@@ -104,18 +104,32 @@ def test_get_type_for_model(model, name):
     assert common.get_type_for_model(model) == name
 
 
-@pytest.mark.django_db
 def test_get_model_for_invalid_type():
     with pytest.raises(LookupError):
         common.get_model_for_type('foobar')
 
 
-@pytest.mark.django_db
 @pytest.mark.parametrize("model_type,model_class", [(name, cls) for cls, name in TEST_MODELS])
 def test_get_model_for_valid_type(model_type, model_class):
     assert common.get_model_for_type(model_type) == model_class
 
 
+@pytest.mark.parametrize("model_type,model_class", [(name, cls) for cls, name in TEST_MODELS])
+def test_get_capacity_type(model_type, model_class):
+    if model_type in ('job', 'ad_hoc_command', 'inventory_update', 'job_template'):
+        expectation = 'execution'
+    elif model_type in ('project_update', 'system_job'):
+        expectation = 'control'
+    else:
+        expectation = None
+    if model_type in ('unified_job', 'unified_job_template', 'inventory'):
+        with pytest.raises(RuntimeError):
+            common.get_capacity_type(model_class)
+    else:
+        assert common.get_capacity_type(model_class) == expectation
+        assert common.get_capacity_type(model_class()) == expectation
+
+
 @pytest.fixture
 def memoized_function(mocker, mock_cache):
     with mock.patch('awx.main.utils.common.get_memoize_cache', return_value=mock_cache):