From 67474c8de11b5cc553eb010bca5f316e7eb2ede9 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Wed, 9 Aug 2017 15:53:25 -0400 Subject: [PATCH] Updating development tooling to be more generic * Not assuming GCR is being used for image hosting * Breaking out the dev environment bootstrapping from service starting --- Makefile | 35 +++++++++-------- awx/api/views.py | 2 +- tools/docker-compose-cluster.yml | 12 +++--- tools/docker-compose.yml | 2 +- tools/docker-compose/Dockerfile | 1 + tools/docker-compose/bootstrap_development.sh | 36 ++++++++++++++++++ tools/docker-compose/start_development.sh | 38 ++----------------- tools/docker-isolated-override.yml | 2 +- 8 files changed, 68 insertions(+), 60 deletions(-) create mode 100755 tools/docker-compose/bootstrap_development.sh diff --git a/Makefile b/Makefile index e4971d7477..a1ca534282 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,8 @@ NPM_BIN ?= npm DEPS_SCRIPT ?= packaging/bundle/deps.py GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) MANAGEMENT_COMMAND ?= awx-manage -GCLOUD_AUTH ?= +IMAGE_REPOSITORY_AUTH ?= +IMAGE_REPOSITORY_BASE ?= https://gcr.io VERSION=$(shell git describe --long) VERSION3=$(shell git describe --long | sed 's/\-g.*//') @@ -24,7 +25,7 @@ VENV_BASE ?= /venv SCL_PREFIX ?= CELERY_SCHEDULE_FILE ?= /celerybeat-schedule -DEV_DOCKER_TAG_BASE ?= gcr.io/ansible-tower-engineering/ +DEV_DOCKER_TAG_BASE ?= gcr.io/ansible-tower-engineering # Python packages to install only from source (not from binary wheels) # Comma separated list SRC_ONLY_PKGS ?= cffi,pycparser,psycopg2,twilio @@ -537,13 +538,13 @@ setup-bundle-build: mkdir -p $@ docker-auth: - if [ "$(GCLOUD_AUTH)" ]; then \ - docker login -u oauth2accesstoken -p "$(GCLOUD_AUTH)" https://gcr.io; \ + if [ "$(IMAGE_REPOSITORY_AUTH)" ]; then \ + docker login -u oauth2accesstoken -p "$(IMAGE_REPOSITORY_AUTH)" $(IMAGE_REPOSITORY_BASE); \ fi; # Docker isolated rampart docker-isolated: - TAG=$(COMPOSE_TAG) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml create + TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml create docker start tools_awx_1 docker start tools_isolated_1 if [ "`docker exec -i -t tools_isolated_1 cat /root/.ssh/authorized_keys`" == "`docker exec -t tools_awx_1 cat /root/.ssh/id_rsa.pub`" ]; then \ @@ -551,29 +552,31 @@ docker-isolated: else \ docker exec "tools_isolated_1" bash -c "mkdir -p /root/.ssh && rm -f /root/.ssh/authorized_keys && echo $$(docker exec -t tools_awx_1 cat /root/.ssh/id_rsa.pub) >> /root/.ssh/authorized_keys"; \ fi - TAG=$(COMPOSE_TAG) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml up + TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml up # Docker Compose Development environment docker-compose: docker-auth - TAG=$(COMPOSE_TAG) docker-compose -f tools/docker-compose.yml up --no-recreate awx + TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml up --no-recreate awx docker-compose-cluster: docker-auth - TAG=$(COMPOSE_TAG) docker-compose -f tools/docker-compose-cluster.yml up + TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose-cluster.yml up docker-compose-test: docker-auth - cd tools && TAG=$(COMPOSE_TAG) docker-compose run --rm --service-ports awx /bin/bash + cd tools && TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /bin/bash -docker-compose-build: awx-devel-build awx-isolated-build +docker-compose-build: awx-devel-build +# Base development image build awx-devel-build: docker build -t ansible/awx_devel -f tools/docker-compose/Dockerfile . - docker tag ansible/awx_devel $(DEV_DOCKER_TAG_BASE)awx_devel:$(COMPOSE_TAG) - #docker push $(DEV_DOCKER_TAG_BASE)awx_devel:$(COMPOSE_TAG) + docker tag ansible/awx_devel $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) + #docker push $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) +# For use when developing on "isolated" AWX deployments awx-isolated-build: docker build -t ansible/awx_isolated -f tools/docker-isolated/Dockerfile . - docker tag ansible/awx_isolated $(DEV_DOCKER_TAG_BASE)awx_isolated:$(COMPOSE_TAG) - #docker push $(DEV_DOCKER_TAG_BASE)awx_isolated:$(COMPOSE_TAG) + docker tag ansible/awx_isolated $(DEV_DOCKER_TAG_BASE)/awx_isolated:$(COMPOSE_TAG) + #docker push $(DEV_DOCKER_TAG_BASE)/awx_isolated:$(COMPOSE_TAG) MACHINE?=default docker-clean: @@ -585,10 +588,10 @@ docker-refresh: docker-clean docker-compose # Docker Development Environment with Elastic Stack Connected docker-compose-elk: docker-auth - TAG=$(COMPOSE_TAG) docker-compose -f tools/docker-compose.yml -f tools/elastic/docker-compose.logstash-link.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate + TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/elastic/docker-compose.logstash-link.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate docker-compose-cluster-elk: docker-auth - TAG=$(COMPOSE_TAG) docker-compose -f tools/docker-compose-cluster.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate + TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose-cluster.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate clean-elk: docker stop tools_kibana_1 diff --git a/awx/api/views.py b/awx/api/views.py index 100540ad6a..4f4d8d7ed0 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -140,7 +140,7 @@ class ApiRootView(APIView): v1 = reverse('api:api_v1_root_view', kwargs={'version': 'v1'}) v2 = reverse('api:api_v2_root_view', kwargs={'version': 'v2'}) data = dict( - description = _('Ansible Tower REST API'), + description = _('AWX REST API'), current_version = v2, available_versions = dict(v1 = v1, v2 = v2), ) diff --git a/tools/docker-compose-cluster.yml b/tools/docker-compose-cluster.yml index 9986e09972..09c02e3915 100644 --- a/tools/docker-compose-cluster.yml +++ b/tools/docker-compose-cluster.yml @@ -16,7 +16,7 @@ services: - "15672:15672" awx_1: privileged: true - image: gcr.io/ansible-tower-engineering/awx_devel:${TAG} + image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG} hostname: awx_1 environment: RABBITMQ_HOST: rabbitmq_1 @@ -30,7 +30,7 @@ services: awx_2: privileged: true - image: gcr.io/ansible-tower-engineering/awx_devel:${TAG} + image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG} hostname: awx_2 environment: RABBITMQ_HOST: rabbitmq_2 @@ -43,7 +43,7 @@ services: - "../:/awx_devel" awx_3: privileged: true - image: gcr.io/ansible-tower-engineering/awx_devel:${TAG} + image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG} hostname: awx_3 environment: RABBITMQ_HOST: rabbitmq_3 @@ -55,16 +55,16 @@ services: volumes: - "../:/awx_devel" rabbitmq_1: - image: gcr.io/ansible-tower-engineering/rabbit_cluster_node:latest + image: ${DEV_DOCKER_TAG_BASE}/rabbit_cluster_node:latest hostname: rabbitmq_1 rabbitmq_2: - image: gcr.io/ansible-tower-engineering/rabbit_cluster_node:latest + image: ${DEV_DOCKER_TAG_BASE}/rabbit_cluster_node:latest hostname: rabbitmq_2 environment: - CLUSTERED=true - CLUSTER_WITH=rabbitmq_1 rabbitmq_3: - image: gcr.io/ansible-tower-engineering/rabbit_cluster_node:latest + image: ${DEV_DOCKER_TAG_BASE}/rabbit_cluster_node:latest hostname: rabbitmq_3 environment: - CLUSTERED=true diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index 37a8586999..7a90b40cf9 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: # Primary AWX Development Container awx: - image: gcr.io/ansible-tower-engineering/awx_devel:${TAG} + image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG} hostname: awx environment: RABBITMQ_HOST: rabbitmq diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile index 0a648c3d5b..e6fb9c0d43 100644 --- a/tools/docker-compose/Dockerfile +++ b/tools/docker-compose/Dockerfile @@ -26,6 +26,7 @@ ADD tools/docker-compose/awx.egg-info /tmp/awx.egg-info RUN ln -Ffs /awx_devel/tools/docker-compose/nginx.conf /etc/nginx/nginx.conf RUN ln -Ffs /awx_devel/tools/docker-compose/nginx.vh.default.conf /etc/nginx/conf.d/nginx.vh.default.conf RUN ln -s /awx_devel/tools/docker-compose/start_development.sh /start_development.sh +RUN ln -s /awx_devel/tools/docker-compose/bootstrap_development.sh /bootstrap_development.sh RUN openssl req -nodes -newkey rsa:2048 -keyout /etc/nginx/nginx.key -out /etc/nginx/nginx.csr -subj "/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost" RUN openssl x509 -req -days 365 -in /etc/nginx/nginx.csr -signkey /etc/nginx/nginx.key -out /etc/nginx/nginx.crt WORKDIR /tmp diff --git a/tools/docker-compose/bootstrap_development.sh b/tools/docker-compose/bootstrap_development.sh new file mode 100755 index 0000000000..df6ecb2637 --- /dev/null +++ b/tools/docker-compose/bootstrap_development.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set +x + +# Wait for the databases to come up +ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=postgres port=5432" all +ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=memcached port=11211" all +ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=${RABBITMQ_HOST} port=5672" all + +# In case AWX in the container wants to connect to itself, use "docker exec" to attach to the container otherwise +# TODO: FIX +#/etc/init.d/ssh start + + +ansible -i "127.0.0.1," -c local -v -m postgresql_user -U postgres -a "name=awx-dev password=AWXsome1 login_user=postgres login_host=postgres" all +ansible -i "127.0.0.1," -c local -v -m postgresql_db -U postgres -a "name=awx-dev owner=awx-dev login_user=postgres login_host=postgres" all + +# Move to the source directory so we can bootstrap +if [ -f "/awx_devel/manage.py" ]; then + cd /awx_devel +else + echo "Failed to find awx source tree, map your development tree volume" +fi + +cp -R /tmp/awx.egg-info /awx_devel/ || true +sed -i "s/placeholder/$(git describe --long | sed 's/\./\\./g')/" /awx_devel/awx.egg-info/PKG-INFO +cp /tmp/awx.egg-link /venv/awx/lib/python2.7/site-packages/awx.egg-link +ln -s /awx_devel/tools/rdb.py /venv/awx/lib/python2.7/site-packages/rdb.py || true +yes | cp -rf /awx_devel/tools/docker-compose/supervisor.conf /supervisor.conf + +# AWX bootstrapping +make version_file +make migrate +make init + +mkdir -p /awx_devel/awx/public/static +mkdir -p /awx_devel/awx/ui/static diff --git a/tools/docker-compose/start_development.sh b/tools/docker-compose/start_development.sh index 9dd095aeed..b86614dd9b 100755 --- a/tools/docker-compose/start_development.sh +++ b/tools/docker-compose/start_development.sh @@ -1,42 +1,10 @@ #!/bin/bash set +x -# Wait for the databases to come up -ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=postgres port=5432" all -ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=memcached port=11211" all -ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=${RABBITMQ_HOST} port=5672" all - -# In case AWX in the container wants to connect to itself, use "docker exec" to attach to the container otherwise -# TODO: FIX -#/etc/init.d/ssh start - - -ansible -i "127.0.0.1," -c local -v -m postgresql_user -U postgres -a "name=awx-dev password=AWXsome1 login_user=postgres login_host=postgres" all -ansible -i "127.0.0.1," -c local -v -m postgresql_db -U postgres -a "name=awx-dev owner=awx-dev login_user=postgres login_host=postgres" all - -# Move to the source directory so we can bootstrap -if [ -f "/awx_devel/manage.py" ]; then - cd /awx_devel -else - echo "Failed to find awx source tree, map your development tree volume" -fi - -cp -R /tmp/awx.egg-info /awx_devel/ || true -sed -i "s/placeholder/$(git describe --long | sed 's/\./\\./g')/" /awx_devel/awx.egg-info/PKG-INFO -cp /tmp/awx.egg-link /venv/awx/lib/python2.7/site-packages/awx.egg-link -ln -s /awx_devel/tools/rdb.py /venv/awx/lib/python2.7/site-packages/rdb.py || true -yes | cp -rf /awx_devel/tools/docker-compose/supervisor.conf /supervisor.conf - -# AWX bootstrapping -make version_file -make migrate -make init - -mkdir -p /awx_devel/awx/public/static -mkdir -p /awx_devel/awx/ui/static - -# Start the service +/bootstrap_development.sh +cd /awx_devel +# Start the services if [ -f "/awx_devel/tools/docker-compose/use_dev_supervisor.txt" ]; then make supervisor else diff --git a/tools/docker-isolated-override.yml b/tools/docker-isolated-override.yml index 140f13de7a..db6efe54c0 100644 --- a/tools/docker-isolated-override.yml +++ b/tools/docker-isolated-override.yml @@ -8,7 +8,7 @@ services: - isolated # Isolated Rampart Container isolated: - image: gcr.io/ansible-tower-engineering/awx_isolated:${TAG} + image: ${DEV_DOCKER_TAG_BASE}/awx_isolated:${TAG} hostname: isolated volumes: - "../awx/main/isolated:/awx_devel"