From 7cca39d0691d1760e8903825a6e5970069b72e11 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Mon, 27 Feb 2023 20:53:07 -0500 Subject: [PATCH 1/8] change make Dockerfile to phony awx-kube-build and docker-compose-build share the same Dockerfile if u run awx-kube-build than docker-compose-build in succession the second command wont run the Dockerfile target and cause the image to be built with the incorrect Dockerfile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 4e402125b2..c90430d67e 100644 --- a/Makefile +++ b/Makefile @@ -572,6 +572,7 @@ VERSION: PYTHON_VERSION: @echo "$(PYTHON)" | sed 's:python::' +.PHONY: Dockerfile Dockerfile: tools/ansible/roles/dockerfile/templates/Dockerfile.j2 ansible-playbook tools/ansible/dockerfile.yml -e receptor_image=$(RECEPTOR_IMAGE) From d5de1f9d114808ef415e799c687efec2ea0f56f0 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 20 Feb 2023 09:56:38 -0500 Subject: [PATCH 2/8] Make use of new keepalive messages from ansible-runner Make setting API configurable and process keepalive events when seen in the event callback Use env var in pod spec and make it specific to K8S --- awx/main/conf.py | 10 ++++++++++ awx/main/tasks/callback.py | 2 ++ awx/main/tasks/receptor.py | 4 ++++ awx/settings/defaults.py | 5 +++++ 4 files changed, 21 insertions(+) diff --git a/awx/main/conf.py b/awx/main/conf.py index dab0543a1a..99b995b113 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -282,6 +282,16 @@ register( placeholder={'HTTP_PROXY': 'myproxy.local:8080'}, ) +register( + 'AWX_RUNNER_KEEPALIVE_SECONDS', + field_class=fields.IntegerField, + label=_('K8S Ansible Runner Keep-Alive Message Interval'), + help_text=_('Only applies to K8S deployments and container_group jobs. If not 0, send a message every so-many seconds to keep connection open.'), + category=_('Jobs'), + category_slug='jobs', + placeholder=240, # intended to be under common 5 minute idle timeout +) + register( 'GALAXY_TASK_ENV', field_class=fields.KeyValueField, diff --git a/awx/main/tasks/callback.py b/awx/main/tasks/callback.py index 92bfc40368..0046d07d82 100644 --- a/awx/main/tasks/callback.py +++ b/awx/main/tasks/callback.py @@ -85,6 +85,8 @@ class RunnerCallback: # which generate job events from two 'streams': # ansible-inventory and the awx.main.commands.inventory_import # logger + if event_data.get('event') == 'keepalive': + return if event_data.get(self.event_data_key, None): if self.event_data_key != 'job_id': diff --git a/awx/main/tasks/receptor.py b/awx/main/tasks/receptor.py index 006c805943..9cb4d49efe 100644 --- a/awx/main/tasks/receptor.py +++ b/awx/main/tasks/receptor.py @@ -526,6 +526,10 @@ class AWXReceptorJob: pod_spec['spec']['containers'][0]['image'] = ee.image pod_spec['spec']['containers'][0]['args'] = ['ansible-runner', 'worker', '--private-data-dir=/runner'] + if settings.AWX_RUNNER_KEEPALIVE_SECONDS: + pod_spec['spec']['containers'][0].setdefault('env', []) + pod_spec['spec']['containers'][0]['env'].append({'name': 'ANSIBLE_RUNNER_KEEPALIVE_SECONDS', 'value': str(settings.AWX_RUNNER_KEEPALIVE_SECONDS)}) + # Enforce EE Pull Policy pull_options = {"always": "Always", "missing": "IfNotPresent", "never": "Never"} if self.task and self.task.instance.execution_environment: diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 4d18540bcd..74a36b3e2d 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -929,6 +929,11 @@ AWX_RUNNER_OMIT_ENV_FILES = True # Allow ansible-runner to save ansible output (may cause performance issues) AWX_RUNNER_SUPPRESS_OUTPUT_FILE = True +# https://github.com/ansible/ansible-runner/pull/1191/files +# Interval in seconds between the last message and keep-alive messages that +# ansible-runner will send +AWX_RUNNER_KEEPALIVE_SECONDS = 0 + # Delete completed work units in receptor RECEPTOR_RELEASE_WORK = True From 6fa22f5be29262d52e9ee95bc37dc133ccb54de9 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Tue, 21 Feb 2023 16:17:48 -0500 Subject: [PATCH 3/8] Add UI for the new setting --- .../screens/Setting/Jobs/JobsEdit/JobsEdit.js | 5 +++++ .../Setting/Jobs/JobsEdit/JobsEdit.test.js | 1 + .../shared/data.allSettingOptions.json | 19 +++++++++++++++++++ .../Setting/shared/data.allSettings.json | 1 + .../Setting/shared/data.jobSettings.json | 1 + 5 files changed, 27 insertions(+) diff --git a/awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.js b/awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.js index 52e216e41e..d258fdec46 100644 --- a/awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.js +++ b/awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.js @@ -150,6 +150,11 @@ function JobsEdit() { type={options?.SCHEDULE_MAX_JOBS ? 'number' : undefined} isRequired={Boolean(options?.SCHEDULE_MAX_JOBS)} /> + ', () => { const { EVENT_STDOUT_MAX_BYTES_DISPLAY, STDOUT_MAX_BYTES_DISPLAY, + AWX_RUNNER_KEEPALIVE_SECONDS, ...jobRequest } = mockJobSettings; expect(SettingsAPI.updateAll).toHaveBeenCalledWith(jobRequest); diff --git a/awx/ui/src/screens/Setting/shared/data.allSettingOptions.json b/awx/ui/src/screens/Setting/shared/data.allSettingOptions.json index b654d1bd90..3eaf93eff0 100644 --- a/awx/ui/src/screens/Setting/shared/data.allSettingOptions.json +++ b/awx/ui/src/screens/Setting/shared/data.allSettingOptions.json @@ -344,6 +344,16 @@ "category_slug": "jobs", "default": 10 }, + "AWX_RUNNER_KEEPALIVE_SECONDS": { + "type": "integer", + "required": true, + "label": "K8S Ansible Runner Keep-Alive Message Interval", + "help_text": "Only applies to K8S deployments and container_group jobs. If not 0, send a message every so-many seconds to keep connection open.", + "category": "Jobs", + "category_slug": "jobs", + "placeholder": 240, + "default": 0 + }, "AWX_ANSIBLE_CALLBACK_PLUGINS": { "type": "list", "required": false, @@ -4098,6 +4108,15 @@ "category_slug": "jobs", "defined_in_file": false }, + "AWX_RUNNER_KEEPALIVE_SECONDS": { + "type": "integer", + "label": "K8S Ansible Runner Keep-Alive Message Interval", + "help_text": "Only applies to K8S deployments and container_group jobs. If not 0, send a message every so-many seconds to keep connection open.", + "category": "Jobs", + "category_slug": "jobs", + "placeholder": 240, + "default": 0 + }, "AWX_ANSIBLE_CALLBACK_PLUGINS": { "type": "list", "label": "Ansible Callback Plugins", diff --git a/awx/ui/src/screens/Setting/shared/data.allSettings.json b/awx/ui/src/screens/Setting/shared/data.allSettings.json index e5136f4b58..b2eaea2a12 100644 --- a/awx/ui/src/screens/Setting/shared/data.allSettings.json +++ b/awx/ui/src/screens/Setting/shared/data.allSettings.json @@ -51,6 +51,7 @@ "STDOUT_MAX_BYTES_DISPLAY":1048576, "EVENT_STDOUT_MAX_BYTES_DISPLAY":1024, "SCHEDULE_MAX_JOBS":10, + "AWX_RUNNER_KEEPALIVE_SECONDS": 0, "AWX_ANSIBLE_CALLBACK_PLUGINS":[], "DEFAULT_JOB_TIMEOUT":0, "DEFAULT_JOB_IDLE_TIMEOUT":0, diff --git a/awx/ui/src/screens/Setting/shared/data.jobSettings.json b/awx/ui/src/screens/Setting/shared/data.jobSettings.json index 29567a8f8c..6c001cca3f 100644 --- a/awx/ui/src/screens/Setting/shared/data.jobSettings.json +++ b/awx/ui/src/screens/Setting/shared/data.jobSettings.json @@ -19,6 +19,7 @@ "STDOUT_MAX_BYTES_DISPLAY": 1048576, "EVENT_STDOUT_MAX_BYTES_DISPLAY": 1024, "SCHEDULE_MAX_JOBS": 10, + "AWX_RUNNER_KEEPALIVE_SECONDS": 0, "AWX_ANSIBLE_CALLBACK_PLUGINS": [], "DEFAULT_JOB_TIMEOUT": 0, "DEFAULT_JOB_IDLE_TIMEOUT": 0, From b143df31834fe98afb2059999d07be81727dfa21 Mon Sep 17 00:00:00 2001 From: Michael Abashian Date: Tue, 21 Feb 2023 17:05:14 -0500 Subject: [PATCH 4/8] Fix broken UI test --- awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.test.js b/awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.test.js index 0d9d2a9ca2..3f8b6dd220 100644 --- a/awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.test.js +++ b/awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.test.js @@ -79,7 +79,6 @@ describe('', () => { const { EVENT_STDOUT_MAX_BYTES_DISPLAY, STDOUT_MAX_BYTES_DISPLAY, - AWX_RUNNER_KEEPALIVE_SECONDS, ...jobRequest } = mockJobSettings; expect(SettingsAPI.updateAll).toHaveBeenCalledWith(jobRequest); From 90f54b98cd0183091228b78c46d289f175f94e93 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 22 Feb 2023 14:32:30 -0500 Subject: [PATCH 5/8] Update keepalive setting help_text to be more direct Co-authored-by: Shane McDonald --- awx/main/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/conf.py b/awx/main/conf.py index 99b995b113..2dbf5e127e 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -286,7 +286,7 @@ register( 'AWX_RUNNER_KEEPALIVE_SECONDS', field_class=fields.IntegerField, label=_('K8S Ansible Runner Keep-Alive Message Interval'), - help_text=_('Only applies to K8S deployments and container_group jobs. If not 0, send a message every so-many seconds to keep connection open.'), + help_text=_('Only applies to jobs running in a Container Group. If not 0, send a message every so-many seconds to keep connection open.'), category=_('Jobs'), category_slug='jobs', placeholder=240, # intended to be under common 5 minute idle timeout From 217dc57c24b07717eca6bf178f41c713a969a100 Mon Sep 17 00:00:00 2001 From: Jesse Wattenbarger Date: Tue, 14 Feb 2023 10:06:15 -0500 Subject: [PATCH 6/8] Change docker-clean build rule in Makefile - Use a make foreach macro and rmi instead of grep and xargs. --- Makefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 896dcf2c6c..e6da7db8da 100644 --- a/Makefile +++ b/Makefile @@ -547,10 +547,8 @@ docker-compose-build: --cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) . docker-clean: - $(foreach container_id,$(shell docker ps -f name=tools_awx -aq && docker ps -f name=tools_receptor -aq),docker stop $(container_id); docker rm -f $(container_id);) - if [ "$(shell docker images | grep awx_devel)" ]; then \ - docker images | grep awx_devel | awk '{print $$3}' | xargs docker rmi --force; \ - fi + -$(foreach container_id,$(shell docker ps -f name=tools_awx -aq && docker ps -f name=tools_receptor -aq),docker stop $(container_id); docker rm -f $(container_id);) + -$(foreach image_id,$(shell docker images --filter=reference='*awx_devel*' -aq),docker rmi --force $(image_id);) docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean docker volume rm -f tools_awx_db tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q) From 0a2f1622f60518746dec532603049ccaa4758102 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Tue, 7 Mar 2023 18:24:04 +0100 Subject: [PATCH 7/8] Analytics: instance_info.json v1.3 (#13408) --- awx/main/analytics/collectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index 30bf730784..14d54c7ac2 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -233,7 +233,7 @@ def projects_by_scm_type(since, **kwargs): return counts -@register('instance_info', '1.2', description=_('Cluster topology and capacity')) +@register('instance_info', '1.3', description=_('Cluster topology and capacity')) def instance_info(since, include_hostnames=False, **kwargs): info = {} # Use same method that the TaskManager does to compute consumed capacity without querying all running jobs for each Instance From 3945db60eb396aefe6a60aba789648aab0d03ec0 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Tue, 7 Mar 2023 14:09:34 -0500 Subject: [PATCH 8/8] Automatically build image for feature branch - also will now publish awx image for devel --- .github/workflows/devel_images.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/devel_images.yml b/.github/workflows/devel_images.yml index dbc1a4937b..6f2bdf74c4 100644 --- a/.github/workflows/devel_images.yml +++ b/.github/workflows/devel_images.yml @@ -7,6 +7,7 @@ on: branches: - devel - release_* + - feature_* jobs: push: if: endsWith(github.repository, '/awx') || startsWith(github.ref, 'refs/heads/release_') @@ -20,6 +21,12 @@ jobs: - name: Get python version from Makefile run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV + - name: Set lower case owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Install python ${{ env.py_version }} uses: actions/setup-python@v2 with: @@ -31,15 +38,18 @@ jobs: - name: Pre-pull image to warm build cache run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${GITHUB_REF##*/} || : - docker pull ghcr.io/${{ github.repository_owner }}/awx_kube_devel:${GITHUB_REF##*/} || : + docker pull ghcr.io/${OWNER_LC}/awx_devel:${GITHUB_REF##*/} || : + docker pull ghcr.io/${OWNER_LC}/awx_kube_devel:${GITHUB_REF##*/} || : + docker pull ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/} || : - name: Build images run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${GITHUB_REF##*/} make docker-compose-build - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-dev-build + DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make docker-compose-build + DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-dev-build + DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-build - name: Push image run: | - docker push ghcr.io/${{ github.repository_owner }}/awx_devel:${GITHUB_REF##*/} - docker push ghcr.io/${{ github.repository_owner }}/awx_kube_devel:${GITHUB_REF##*/} + docker push ghcr.io/${OWNER_LC}/awx_devel:${GITHUB_REF##*/} + docker push ghcr.io/${OWNER_LC}/awx_kube_devel:${GITHUB_REF##*/} + docker push ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/}