Consolidate cleanup actions under new ansible-runner worker cleanup command (#11160)

* Primary development of integrating runner cleanup command

* Fixup image cleanup signals and their tests

* Use alphabetical sort to solve the cluster coordination problem

* Update test to new pattern

* Clarity edits to interface with ansible-runner cleanup method

* Another change corresponding to ansible-runner CLI updates

* Fix incomplete implementation of receptor remote cleanup

* Share receptor utils code between worker_info and cleanup

* Complete task logging from calling runner cleanup command

* Wrap up unit tests and some contract changes that fall out of those

* Fix bug in CLI construction

* Fix queryset filter bug

awx/main/tests/functional/models/test_execution_environment.py

@@ -0,0 +1,46 @@
+import pytest
+
+from awx.main.models.execution_environments import ExecutionEnvironment
+
+
+@pytest.fixture
+def cleanup_patch(mocker):
+    return mocker.patch('awx.main.signals.handle_removed_image')
+
+
+@pytest.mark.django_db
+def test_image_unchanged_no_delete_task(cleanup_patch):
+    """When an irrelevant EE field is changed, we do not run the image cleanup task"""
+    execution_environment = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+    execution_environment.description = 'foobar'
+    execution_environment.save()
+
+    cleanup_patch.delay.assert_not_called()
+
+
+@pytest.mark.django_db
+def test_image_changed_creates_delete_task(cleanup_patch):
+    execution_environment = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+    execution_environment.image = 'quay.io/new/image'
+    execution_environment.save()
+
+    cleanup_patch.delay.assert_called_once_with(remove_images=['quay.io/foo/bar'])
+
+
+@pytest.mark.django_db
+def test_image_still_in_use(cleanup_patch):
+    """When an image is still in use by another EE, we do not clean it up"""
+    ExecutionEnvironment.objects.create(name='unrelated-ee', image='quay.io/foo/bar')
+    execution_environment = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+    execution_environment.image = 'quay.io/new/image'
+    execution_environment.save()
+
+    cleanup_patch.delay.assert_not_called()
+
+
+@pytest.mark.django_db
+def test_image_deletion_creates_delete_task(cleanup_patch):
+    execution_environment = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+    execution_environment.delete()
+
+    cleanup_patch.delay.assert_called_once_with(remove_images=['quay.io/foo/bar'])

awx/main/tests/unit/models/test_ha.py

@@ -89,3 +89,19 @@ class TestInstanceGroup(object):
             assert ig.find_largest_idle_instance(instances_online_only) is None, reason
         else:
             assert ig.find_largest_idle_instance(instances_online_only) == instances[instance_fit_index], reason
+
+
+def test_cleanup_params_defaults():
+    inst = Instance(hostname='foobar')
+    assert inst.get_cleanup_task_kwargs(exclude_strings=['awx_423_']) == {'exclude_strings': ['awx_423_'], 'file_pattern': '/tmp/awx_*_*'}
+
+
+def test_cleanup_params_for_image_cleanup():
+    inst = Instance(hostname='foobar')
+    # see CLI conversion in awx.main.tests.unit.utils.test_receptor
+    assert inst.get_cleanup_task_kwargs(file_pattern='', remove_images=['quay.invalid/foo/bar'], image_prune=True) == {
+        'file_pattern': '',
+        'process_isolation_executable': 'podman',
+        'remove_images': ['quay.invalid/foo/bar'],
+        'image_prune': True,
+    }

awx/main/tests/unit/utils/test_receptor.py

@@ -0,0 +1,21 @@
+from awx.main.utils.receptor import _convert_args_to_cli
+
+
+def test_file_cleanup_scenario():
+    args = _convert_args_to_cli({'exclude_strings': ['awx_423_', 'awx_582_'], 'file_pattern': '/tmp/awx_*_*'})
+    assert ' '.join(args) == 'cleanup --exclude-strings=awx_423_ awx_582_ --file-pattern=/tmp/awx_*_*'
+
+
+def test_image_cleanup_scenario():
+    # See input dict in awx.main.tests.unit.models.test_ha
+    args = _convert_args_to_cli(
+        {
+            'file_pattern': '',
+            'process_isolation_executable': 'podman',
+            'remove_images': ['quay.invalid/foo/bar:latest', 'quay.invalid/foo/bar:devel'],
+            'image_prune': True,
+        }
+    )
+    assert (
+        ' '.join(args) == 'cleanup --remove-images=quay.invalid/foo/bar:latest quay.invalid/foo/bar:devel --image-prune --process-isolation-executable=podman'
+    )