From 2622a1e764306ba00406f2c4dc4d86d060142a18 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 17 Jul 2021 10:11:21 -0400 Subject: [PATCH 1/7] Port CI jobs to GitHub Actions --- .github/workflows/ci.yml | 149 ++++++++++++++++++++++++++++++ .github/workflows/devel_image.yml | 30 ++++++ 2 files changed, 179 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/devel_image.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..a17a1df304 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,149 @@ +--- +name: CI +on: + pull_request: +jobs: + api-test: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + + - name: Log in to registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pre-pull image to warm build cache + run: | + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + + - name: Build image + run: | + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build + + - name: Run API Tests + run: | + docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ + --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /start_tests.sh + api-lint: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + + - name: Log in to registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pre-pull image to warm build cache + run: | + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + + - name: Build image + run: | + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build + + - name: Run API Linters + run: | + docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ + --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /var/lib/awx/venv/awx/bin/tox -e linters + api-swagger: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + + - name: Log in to registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pre-pull image to warm build cache + run: | + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + + - name: Build image + run: | + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build + + - name: Generate API Reference + run: | + docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ + --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /start_tests.sh swagger + awx-collection: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + + - name: Log in to registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pre-pull image to warm build cache + run: | + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + + - name: Build image + run: | + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build + + - name: Run Collection Tests + run: | + docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ + --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /start_tests.sh test_collection_all + api-schema: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + + - name: Log in to registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pre-pull image to warm build cache + run: | + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + + - name: Build image + run: | + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build + + - name: Check API Schema + run: | + docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ + --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /start_tests.sh detect-schema-change + ui: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + + - name: Log in to registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pre-pull image to warm build cache + run: | + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + + - name: Build image + run: | + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build + + - name: Run UI Linters and Tests + run: | + docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ + --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} make ui-zuul-lint-and-test diff --git a/.github/workflows/devel_image.yml b/.github/workflows/devel_image.yml new file mode 100644 index 0000000000..44c16e33c3 --- /dev/null +++ b/.github/workflows/devel_image.yml @@ -0,0 +1,30 @@ +--- +name: Push Development Image +on: + push: + branches: + - devel +jobs: + push: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + + - name: Log in to registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pre-pull image to warm build cache + run: | + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + + - name: Build image + run: | + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build + + - name: Push image + run: | + docker push ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} From baca30ef83a3f9443cc9ecb48b89b8ca9acdcd5b Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 17 Jul 2021 10:11:51 -0400 Subject: [PATCH 2/7] Dont lint GHA files --- .yamllint | 1 + 1 file changed, 1 insertion(+) diff --git a/.yamllint b/.yamllint index 11c769cff1..1da43da59d 100644 --- a/.yamllint +++ b/.yamllint @@ -1,5 +1,6 @@ --- ignore: | + .github .tox awx/main/tests/data/inventory/plugins/** # vault files From e22612fc3e92594b82157e944d5abe22def5e24c Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 17 Jul 2021 11:59:09 -0400 Subject: [PATCH 3/7] Fix image push job --- .github/workflows/devel_image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/devel_image.yml b/.github/workflows/devel_image.yml index 44c16e33c3..b13bb4e92e 100644 --- a/.github/workflows/devel_image.yml +++ b/.github/workflows/devel_image.yml @@ -19,12 +19,12 @@ jobs: - name: Pre-pull image to warm build cache run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${GITHUB_REF##*/} - name: Build image run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${GITHUB_REF##*/} make docker-compose-build - name: Push image run: | - docker push ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + docker push ghcr.io/${{ github.repository_owner }}/awx_devel:${GITHUB_REF##*/} From 3353b3f3b70b7544975b11915dcf4e78ab4f798b Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 17 Jul 2021 11:59:49 -0400 Subject: [PATCH 4/7] Split out UI test and linters into different GHA jobs --- .github/workflows/ci.yml | 30 +++++++++++++++++++++++++++--- Makefile | 5 ++++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a17a1df304..7377b4af88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,7 +123,7 @@ jobs: run: | docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /start_tests.sh detect-schema-change - ui: + ui-lint: runs-on: ubuntu-latest permissions: packages: write @@ -143,7 +143,31 @@ jobs: run: | DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build - - name: Run UI Linters and Tests + - name: Run UI Linters run: | docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ - --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} make ui-zuul-lint-and-test + --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} make ui-lint + ui-test: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + + - name: Log in to registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pre-pull image to warm build cache + run: | + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + + - name: Build image + run: | + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build + + - name: Run UI Tests + run: | + docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ + --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} make ui-test diff --git a/Makefile b/Makefile index e3214871a3..56f8b3f26d 100644 --- a/Makefile +++ b/Makefile @@ -418,10 +418,13 @@ ui-devel-instrumented: awx/ui_next/node_modules ui-devel-test: awx/ui_next/node_modules $(NPM_BIN) --prefix awx/ui_next --loglevel warn run start -ui-zuul-lint-and-test: +ui-lint: $(NPM_BIN) --prefix awx/ui_next install $(NPM_BIN) run --prefix awx/ui_next lint $(NPM_BIN) run --prefix awx/ui_next prettier-check + +ui-test: + $(NPM_BIN) --prefix awx/ui_next install $(NPM_BIN) run --prefix awx/ui_next test -- --coverage --watchAll=false From dd510ab90cc0157d202ce9de0e3d6a971a1e3a04 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Mon, 19 Jul 2021 09:18:32 -0400 Subject: [PATCH 5/7] Set maxworkers to 4 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 56f8b3f26d..eff8133c21 100644 --- a/Makefile +++ b/Makefile @@ -425,7 +425,7 @@ ui-lint: ui-test: $(NPM_BIN) --prefix awx/ui_next install - $(NPM_BIN) run --prefix awx/ui_next test -- --coverage --watchAll=false + $(NPM_BIN) run --prefix awx/ui_next test -- --coverage --maxWorkers=4 --watchAll=false # Build a pip-installable package into dist/ with a timestamped version number. From aa934b1ddaf39122f690de774f8ddf61f4c225ec Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 19 Jul 2021 12:31:47 -0400 Subject: [PATCH 6/7] Add post-merge GHA job for uploading api schema to s3 --- .github/workflows/upload_schema.yml | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/upload_schema.yml diff --git a/.github/workflows/upload_schema.yml b/.github/workflows/upload_schema.yml new file mode 100644 index 0000000000..2806739d27 --- /dev/null +++ b/.github/workflows/upload_schema.yml @@ -0,0 +1,41 @@ +--- +name: Upload API Schema +on: + push: + branches: + - devel +jobs: + push: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + + - name: Log in to registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pre-pull image to warm build cache + run: | + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${GITHUB_REF##*/} + + - name: Build image + run: | + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${GITHUB_REF##*/} make docker-compose-build + + - name: Generate API Schema + run: | + docker run -u $(id -u) --rm -v ${{ github.workspace }}:/awx_devel/:Z \ + --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /start_tests.sh genschema + + - name: Upload API Schema + env: + AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} + run: | + ansible localhost -c local -m aws_s3 \ + -a 'src=${{ github.workspace }}/schema.json bucket=awx-public-ci-files object=schema.json mode=put permission=public-read' + + From d2017feb55d3ef7a13d988f2abf291783b57510f Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 19 Jul 2021 12:47:05 -0400 Subject: [PATCH 7/7] Specify which region to upload schema to --- .github/workflows/upload_schema.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/upload_schema.yml b/.github/workflows/upload_schema.yml index 2806739d27..ce8647ed4b 100644 --- a/.github/workflows/upload_schema.yml +++ b/.github/workflows/upload_schema.yml @@ -34,6 +34,7 @@ jobs: env: AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} + AWS_REGION: 'us-east-1' run: | ansible localhost -c local -m aws_s3 \ -a 'src=${{ github.workspace }}/schema.json bucket=awx-public-ci-files object=schema.json mode=put permission=public-read'