Merge branch 'feature_web-task-split' into 13437-cache-clear

.github/triage_replies.md

@@ -106,6 +106,13 @@ The Ansible Community is looking at building an EE that corresponds to all of th
 ### Oracle AWX
 We'd be happy to help if you can reproduce this with AWX since we do not have Oracle's Linux Automation Manager. If you need help with this specific version of Oracles Linux Automation Manager you will need to contact your Oracle for support. 
 
+### Community Resolved
+Hi,
+
+We are happy to see that it appears a fix has been provided for your issue, so we will go ahead and close this ticket. Please feel free to reopen if any other problems arise.
+
+<name of community member who helped> thanks so much for taking the time to write a thoughtful and helpful response to this issue!
+
 ### AWX Release
 Subject: Announcing AWX Xa.Ya.za and AWX-Operator Xb.Yb.zb
 

.github/workflows/pr_body_check.yml

@@ -17,9 +17,9 @@ jobs:
         env:
           PR_BODY: ${{ github.event.pull_request.body }}
         run: |
-          echo $PR_BODY | grep "Bug, Docs Fix or other nominal change" > Z
-          echo $PR_BODY | grep "New or Enhanced Feature" > Y
-          echo $PR_BODY | grep "Breaking Change" > X
+          echo "$PR_BODY" | grep "Bug, Docs Fix or other nominal change" > Z
+          echo "$PR_BODY" | grep "New or Enhanced Feature" > Y
+          echo "$PR_BODY" | grep "Breaking Change" > X
           exit 0
         # We exit 0 and set the shell to prevent the returns from the greps from failing this step
         # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference

.github/workflows/promote.yml

@@ -38,9 +38,13 @@ jobs:
       - name: Build collection and publish to galaxy
         run: |
           COLLECTION_TEMPLATE_VERSION=true COLLECTION_NAMESPACE=${{ env.collection_namespace }} make build_collection
+          if [ "$(curl --head -sw '%{http_code}' https://galaxy.ansible.com/download/${{ env.collection_namespace }}-awx-${{ github.event.release.tag_name }}.tar.gz | tail -1)" == "302" ] ; then \
+              echo "Galaxy release already done"; \
+          else \
               ansible-galaxy collection publish \
                 --token=${{ secrets.GALAXY_TOKEN }} \
-            awx_collection_build/${{ env.collection_namespace }}-awx-${{ github.event.release.tag_name }}.tar.gz
+                awx_collection_build/${{ env.collection_namespace }}-awx-${{ github.event.release.tag_name }}.tar.gz; \
+          fi
 
       - name: Set official pypi info
         run: echo pypi_repo=pypi >> $GITHUB_ENV
@@ -52,6 +56,7 @@ jobs:
 
       - name: Build awxkit and upload to pypi
         run: |
+          git reset --hard
           cd awxkit && python3 setup.py bdist_wheel
           twine upload \
             -r ${{ env.pypi_repo }} \
@@ -74,4 +79,6 @@ jobs:
           docker tag ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }} quay.io/${{ github.repository }}:latest
           docker push quay.io/${{ github.repository }}:${{ github.event.release.tag_name }}
           docker push quay.io/${{ github.repository }}:latest
-
+          docker pull ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }}
+          docker tag ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }} quay.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }}
+          docker push quay.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }}

.github/workflows/stage.yml

@@ -84,6 +84,20 @@ jobs:
             -e push=yes \
             -e awx_official=yes
 
+      - name: Log in to GHCR
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Log in to Quay
+        run: |
+          echo ${{ secrets.QUAY_TOKEN }} | docker login quay.io -u ${{ secrets.QUAY_USER }} --password-stdin
+
+      - name: tag awx-ee:latest with version input
+        run: |
+          docker pull quay.io/ansible/awx-ee:latest
+          docker tag quay.io/ansible/awx-ee:latest ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.inputs.version }}
+          docker push ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.inputs.version }}
+
       - name: Build and stage awx-operator
         working-directory: awx-operator
         run: |
@@ -103,6 +117,7 @@ jobs:
         env:
           AWX_TEST_IMAGE: ${{ github.repository }}
           AWX_TEST_VERSION: ${{ github.event.inputs.version }}
+          AWX_EE_TEST_IMAGE: ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.inputs.version }}
 
       - name: Create draft release for AWX
         working-directory: awx

Makefile

@@ -2,6 +2,7 @@ PYTHON ?= python3.9
 OFFICIAL ?= no
 NODE ?= node
 NPM_BIN ?= npm
+KIND_BIN ?= $(shell which kind)
 CHROMIUM_BIN=/tmp/chrome-linux/chrome
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 MANAGEMENT_COMMAND ?= awx-manage
@@ -232,11 +233,17 @@ daphne:
 	fi; \
 	daphne -b 127.0.0.1 -p 8051 awx.asgi:channel_layer
 
-wsbroadcast:
+wsrelay:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	$(PYTHON) manage.py run_wsbroadcast
+	$(PYTHON) manage.py run_wsrelay
+
+heartbeet:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/awx/bin/activate; \
+	fi; \
+	$(PYTHON) manage.py run_heartbeet
 
 ## Run to start the background task dispatcher for development.
 dispatcher:
@@ -606,6 +613,9 @@ awx-kube-build: Dockerfile
 		--build-arg HEADLESS=$(HEADLESS) \
 		-t $(DEV_DOCKER_TAG_BASE)/awx:$(COMPOSE_TAG) .
 
+kind-dev-load: awx-kube-dev-build
+	$(KIND_BIN) load docker-image $(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG)
+
 # Translation TASKS
 # --------------------------------------
 

awx/api/conf.py

@@ -96,6 +96,15 @@ register(
     category=_('Authentication'),
     category_slug='authentication',
 )
+register(
+    'ALLOW_METRICS_FOR_ANONYMOUS_USERS',
+    field_class=fields.BooleanField,
+    default=False,
+    label=_('Allow anonymous users to poll metrics'),
+    help_text=_('If true, anonymous users are allowed to poll metrics.'),
+    category=_('Authentication'),
+    category_slug='authentication',
+)
 
 
 def authentication_validate(serializer, attrs):

awx/api/views/metrics.py

@@ -5,9 +5,11 @@
 import logging
 
 # Django
+from django.conf import settings
 from django.utils.translation import gettext_lazy as _
 
 # Django REST Framework
+from rest_framework.permissions import AllowAny
 from rest_framework.response import Response
 from rest_framework.exceptions import PermissionDenied
 
@@ -31,9 +33,14 @@ class MetricsView(APIView):
 
     renderer_classes = [renderers.PlainTextRenderer, renderers.PrometheusJSONRenderer, renderers.BrowsableAPIRenderer]
 
+    def initialize_request(self, request, *args, **kwargs):
+        if settings.ALLOW_METRICS_FOR_ANONYMOUS_USERS:
+            self.permission_classes = (AllowAny,)
+        return super(APIView, self).initialize_request(request, *args, **kwargs)
+
     def get(self, request):
         '''Show Metrics Details'''
-        if request.user.is_superuser or request.user.is_system_auditor:
+        if settings.ALLOW_METRICS_FOR_ANONYMOUS_USERS or request.user.is_superuser or request.user.is_system_auditor:
             metrics_to_show = ''
             if not request.query_params.get('subsystemonly', "0") == "1":
                 metrics_to_show += metrics().decode('UTF-8')

awx/conf/settings.py

@@ -5,6 +5,8 @@ import threading
 import time
 import os
 
+from concurrent.futures import ThreadPoolExecutor
+
 # Django
 from django.conf import LazySettings
 from django.conf import settings, UserSettingsHolder
@@ -158,7 +160,7 @@ class EncryptedCacheProxy(object):
             obj_id = self.cache.get(Setting.get_cache_id_key(key), default=empty)
             if obj_id is empty:
                 logger.info('Efficiency notice: Corresponding id not stored in cache %s', Setting.get_cache_id_key(key))
-                obj_id = getattr(self._get_setting_from_db(key), 'pk', None)
+                obj_id = getattr(_get_setting_from_db(self.registry, key), 'pk', None)
             elif obj_id == SETTING_CACHE_NONE:
                 obj_id = None
             return method(TransientSetting(pk=obj_id, value=value), 'value')
@@ -167,11 +169,6 @@ class EncryptedCacheProxy(object):
         # a no-op; it just returns the provided value
         return value
 
-    def _get_setting_from_db(self, key):
-        field = self.registry.get_setting_field(key)
-        if not field.read_only:
-            return Setting.objects.filter(key=key, user__isnull=True).order_by('pk').first()
-
     def __getattr__(self, name):
         return getattr(self.cache, name)
 
@@ -187,6 +184,18 @@ def get_settings_to_cache(registry):
     return dict([(key, SETTING_CACHE_NOTSET) for key in get_writeable_settings(registry)])
 
 
+# HACK: runs in thread in order to work in an asyncio context
+def _get_setting_from_db(registry, key):
+    def wrapped(registry, key):
+        field = registry.get_setting_field(key)
+        if not field.read_only:
+            return Setting.objects.filter(key=key, user__isnull=True).order_by('pk').first()
+
+    with ThreadPoolExecutor(max_workers=1) as executor:
+        future = executor.submit(wrapped, registry, key)
+        return future.result()
+
+
 def get_cache_value(value):
     """Returns the proper special cache setting for a value
     based on instance type.
@@ -346,7 +355,7 @@ class SettingsWrapper(UserSettingsHolder):
             setting_id = None
             # this value is read-only, however we *do* want to fetch its value from the database
             if not field.read_only or name == 'INSTALL_UUID':
-                setting = Setting.objects.filter(key=name, user__isnull=True).order_by('pk').first()
+                setting = _get_setting_from_db(self.registry, name)
             if setting:
                 if getattr(field, 'encrypted', False):
                     value = decrypt_field(setting, 'value')

awx/main/analytics/broadcast_websocket.py

@@ -65,7 +65,7 @@ class FixedSlidingWindow:
         return sum(self.buckets.values()) or 0
 
 
-class BroadcastWebsocketStatsManager:
+class RelayWebsocketStatsManager:
     def __init__(self, event_loop, local_hostname):
         self._local_hostname = local_hostname
 
@@ -74,7 +74,7 @@ class BroadcastWebsocketStatsManager:
         self._redis_key = BROADCAST_WEBSOCKET_REDIS_KEY_NAME
 
     def new_remote_host_stats(self, remote_hostname):
-        self._stats[remote_hostname] = BroadcastWebsocketStats(self._local_hostname, remote_hostname)
+        self._stats[remote_hostname] = RelayWebsocketStats(self._local_hostname, remote_hostname)
         return self._stats[remote_hostname]
 
     def delete_remote_host_stats(self, remote_hostname):
@@ -107,7 +107,7 @@ class BroadcastWebsocketStatsManager:
         return parser.text_string_to_metric_families(stats_str.decode('UTF-8'))
 
 
-class BroadcastWebsocketStats:
+class RelayWebsocketStats:
     def __init__(self, local_hostname, remote_hostname):
         self._local_hostname = local_hostname
         self._remote_hostname = remote_hostname

awx/main/analytics/subsystem_metrics.py

@@ -9,7 +9,7 @@ from django.apps import apps
 from awx.main.consumers import emit_channel_notification
 from awx.main.utils import is_testing
 
-root_key = 'awx_metrics'
+root_key = settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX
 logger = logging.getLogger('awx.main.analytics')
 
 
@@ -264,13 +264,6 @@ class Metrics:
             data[field] = self.METRICS[field].decode(self.conn)
         return data
 
-    def store_metrics(self, data_json):
-        # called when receiving metrics from other instances
-        data = json.loads(data_json)
-        if self.instance_name != data['instance']:
-            logger.debug(f"{self.instance_name} received subsystem metrics from {data['instance']}")
-        self.conn.set(root_key + "_instance_" + data['instance'], data['metrics'])
-
     def should_pipe_execute(self):
         if self.metrics_have_changed is False:
             return False
@@ -309,9 +302,9 @@ class Metrics:
                     'instance': self.instance_name,
                     'metrics': self.serialize_local_metrics(),
                 }
-                # store a local copy as well
-                self.store_metrics(json.dumps(payload))
+
                 emit_channel_notification("metrics", payload)
+
                 self.previous_send_metrics.set(current_time)
                 self.previous_send_metrics.store_value(self.conn)
         finally:

awx/main/consumers.py

@@ -3,6 +3,7 @@ import logging
 import time
 import hmac
 import asyncio
+import redis
 
 from django.core.serializers.json import DjangoJSONEncoder
 from django.conf import settings
@@ -80,7 +81,7 @@ class WebsocketSecretAuthHelper:
         WebsocketSecretAuthHelper.verify_secret(secret)
 
 
-class BroadcastConsumer(AsyncJsonWebsocketConsumer):
+class RelayConsumer(AsyncJsonWebsocketConsumer):
     async def connect(self):
         try:
             WebsocketSecretAuthHelper.is_authorized(self.scope)
@@ -100,6 +101,21 @@ class BroadcastConsumer(AsyncJsonWebsocketConsumer):
     async def internal_message(self, event):
         await self.send(event['text'])
 
+    async def receive_json(self, data):
+        (group, message) = unwrap_broadcast_msg(data)
+        if group == "metrics":
+            message = json.loads(message['text'])
+            conn = redis.Redis.from_url(settings.BROKER_URL)
+            conn.set(settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX + "_instance_" + message['instance'], message['metrics'])
+        else:
+            await self.channel_layer.group_send(group, message)
+
+    async def consumer_subscribe(self, event):
+        await self.send_json(event)
+
+    async def consumer_unsubscribe(self, event):
+        await self.send_json(event)
+
 
 class EventConsumer(AsyncJsonWebsocketConsumer):
     async def connect(self):
@@ -128,6 +144,11 @@ class EventConsumer(AsyncJsonWebsocketConsumer):
                 self.channel_name,
             )
 
+        await self.channel_layer.group_send(
+            settings.BROADCAST_WEBSOCKET_GROUP_NAME,
+            {"type": "consumer.unsubscribe", "groups": list(current_groups), "origin_channel": self.channel_name},
+        )
+
     @database_sync_to_async
     def user_can_see_object_id(self, user_access, oid):
         # At this point user is a channels.auth.UserLazyObject object
@@ -176,9 +197,20 @@ class EventConsumer(AsyncJsonWebsocketConsumer):
                     self.channel_name,
                 )
 
+            if len(old_groups):
+                await self.channel_layer.group_send(
+                    settings.BROADCAST_WEBSOCKET_GROUP_NAME,
+                    {"type": "consumer.unsubscribe", "groups": list(old_groups), "origin_channel": self.channel_name},
+                )
+
             new_groups_exclusive = new_groups - current_groups
             for group_name in new_groups_exclusive:
                 await self.channel_layer.group_add(group_name, self.channel_name)
+
+            await self.channel_layer.group_send(
+                settings.BROADCAST_WEBSOCKET_GROUP_NAME,
+                {"type": "consumer.subscribe", "groups": list(new_groups), "origin_channel": self.channel_name},
+            )
             self.scope['session']['groups'] = new_groups
             await self.send_json({"groups_current": list(new_groups), "groups_left": list(old_groups), "groups_joined": list(new_groups_exclusive)})
 
@@ -200,9 +232,11 @@ def _dump_payload(payload):
         return None
 
 
-def emit_channel_notification(group, payload):
-    from awx.main.wsbroadcast import wrap_broadcast_msg  # noqa
+def unwrap_broadcast_msg(payload: dict):
+    return (payload['group'], payload['message'])
 
+
+def emit_channel_notification(group, payload):
     payload_dumped = _dump_payload(payload)
     if payload_dumped is None:
         return
@@ -212,16 +246,6 @@ def emit_channel_notification(group, payload):
     run_sync(
         channel_layer.group_send(
             group,
-            {"type": "internal.message", "text": payload_dumped},
-        )
-    )
-
-    run_sync(
-        channel_layer.group_send(
-            settings.BROADCAST_WEBSOCKET_GROUP_NAME,
-            {
-                "type": "internal.message",
-                "text": wrap_broadcast_msg(group, payload_dumped),
-            },
+            {"type": "internal.message", "text": payload_dumped, "needs_relay": True},
         )
     )

awx/main/credential_plugins/hashivault.py

@@ -1,6 +1,7 @@
 import copy
 import os
 import pathlib
+import time
 from urllib.parse import urljoin
 
 from .plugin import CredentialPlugin, CertFiles, raise_for_status
@@ -247,7 +248,15 @@ def kv_backend(**kwargs):
     request_url = urljoin(url, '/'.join(['v1'] + path_segments)).rstrip('/')
     with CertFiles(cacert) as cert:
         request_kwargs['verify'] = cert
+        request_retries = 0
+        while request_retries < 5:
             response = sess.get(request_url, **request_kwargs)
+            # https://developer.hashicorp.com/vault/docs/enterprise/consistency
+            if response.status_code == 412:
+                request_retries += 1
+                time.sleep(1)
+            else:
+                break
     raise_for_status(response)
 
     json = response.json()
@@ -289,8 +298,15 @@ def ssh_backend(**kwargs):
 
     with CertFiles(cacert) as cert:
         request_kwargs['verify'] = cert
+        request_retries = 0
+        while request_retries < 5:
             resp = sess.post(request_url, **request_kwargs)
-
+            # https://developer.hashicorp.com/vault/docs/enterprise/consistency
+            if resp.status_code == 412:
+                request_retries += 1
+                time.sleep(1)
+            else:
+                break
     raise_for_status(resp)
     return resp.json()['data']['signed_key']
 

awx/main/db/profiled_pg/base.py

@@ -63,7 +63,7 @@ class RecordedQueryLog(object):
             if not os.path.isdir(self.dest):
                 os.makedirs(self.dest)
             progname = ' '.join(sys.argv)
-            for match in ('uwsgi', 'dispatcher', 'callback_receiver', 'wsbroadcast'):
+            for match in ('uwsgi', 'dispatcher', 'callback_receiver', 'wsrelay'):
                 if match in progname:
                     progname = match
                     break

awx/main/dispatch/worker/callback.py

@@ -3,14 +3,12 @@ import logging
 import os
 import signal
 import time
-import traceback
 import datetime
 
 from django.conf import settings
 from django.utils.functional import cached_property
 from django.utils.timezone import now as tz_now
-from django.db import DatabaseError, OperationalError, transaction, connection as django_connection
-from django.db.utils import InterfaceError, InternalError
+from django.db import transaction, connection as django_connection
 from django_guid import set_guid
 
 import psutil
@@ -64,6 +62,7 @@ class CallbackBrokerWorker(BaseWorker):
     """
 
     MAX_RETRIES = 2
+    INDIVIDUAL_EVENT_RETRIES = 3
     last_stats = time.time()
     last_flush = time.time()
     total = 0
@@ -164,38 +163,48 @@ class CallbackBrokerWorker(BaseWorker):
                     else:  # only calculate the seconds if the created time already has been set
                         metrics_total_job_event_processing_seconds += e.modified - e.created
                 metrics_duration_to_save = time.perf_counter()
+                saved_events = []
                 try:
                     cls.objects.bulk_create(events)
                     metrics_bulk_events_saved += len(events)
+                    saved_events = events
+                    self.buff[cls] = []
                 except Exception as exc:
-                    logger.warning(f'Error in events bulk_create, will try indiviually up to 5 errors, error {str(exc)}')
+                    # If the database is flaking, let ensure_connection throw a general exception
+                    # will be caught by the outer loop, which goes into a proper sleep and retry loop
+                    django_connection.ensure_connection()
+                    logger.warning(f'Error in events bulk_create, will try indiviually, error: {str(exc)}')
                     # if an exception occurs, we should re-attempt to save the
                     # events one-by-one, because something in the list is
                     # broken/stale
-                    consecutive_errors = 0
-                    events_saved = 0
                     metrics_events_batch_save_errors += 1
-                    for e in events:
+                    for e in events.copy():
                         try:
                             e.save()
-                            events_saved += 1
-                            consecutive_errors = 0
+                            metrics_singular_events_saved += 1
+                            events.remove(e)
+                            saved_events.append(e)  # Importantly, remove successfully saved events from the buffer
                         except Exception as exc_indv:
-                            consecutive_errors += 1
-                            logger.info(f'Database Error Saving individual Job Event, error {str(exc_indv)}')
-                        if consecutive_errors >= 5:
-                            raise
-                    metrics_singular_events_saved += events_saved
-                    if events_saved == 0:
-                        raise
+                            retry_count = getattr(e, '_retry_count', 0) + 1
+                            e._retry_count = retry_count
+
+                            # special sanitization logic for postgres treatment of NUL 0x00 char
+                            if (retry_count == 1) and isinstance(exc_indv, ValueError) and ("\x00" in e.stdout):
+                                e.stdout = e.stdout.replace("\x00", "")
+
+                            if retry_count >= self.INDIVIDUAL_EVENT_RETRIES:
+                                logger.error(f'Hit max retries ({retry_count}) saving individual Event error: {str(exc_indv)}\ndata:\n{e.__dict__}')
+                                events.remove(e)
+                            else:
+                                logger.info(f'Database Error Saving individual Event uuid={e.uuid} try={retry_count}, error: {str(exc_indv)}')
+
                 metrics_duration_to_save = time.perf_counter() - metrics_duration_to_save
-                for e in events:
+                for e in saved_events:
                     if not getattr(e, '_skip_websocket_message', False):
                         metrics_events_broadcast += 1
                         emit_event_detail(e)
                     if getattr(e, '_notification_trigger_event', False):
                         job_stats_wrapup(getattr(e, e.JOB_REFERENCE), event=e)
-            self.buff = {}
             self.last_flush = time.time()
             # only update metrics if we saved events
             if (metrics_bulk_events_saved + metrics_singular_events_saved) > 0:
@@ -267,20 +276,16 @@ class CallbackBrokerWorker(BaseWorker):
                 try:
                     self.flush(force=flush)
                     break
-                except (OperationalError, InterfaceError, InternalError) as exc:
+                except Exception as exc:
+                    # Aside form bugs, exceptions here are assumed to be due to database flake
                     if retries >= self.MAX_RETRIES:
                         logger.exception('Worker could not re-establish database connectivity, giving up on one or more events.')
+                        self.buff = {}
                         return
                     delay = 60 * retries
                     logger.warning(f'Database Error Flushing Job Events, retry #{retries + 1} in {delay} seconds: {str(exc)}')
                     django_connection.close()
                     time.sleep(delay)
                     retries += 1
-                except DatabaseError:
-                    logger.exception('Database Error Flushing Job Events')
-                    django_connection.close()
-                    break
-        except Exception as exc:
-            tb = traceback.format_exc()
-            logger.error('Callback Task Processor Raised Exception: %r', exc)
-            logger.error('Detail: {}'.format(tb))
+        except Exception:
+            logger.exception(f'Callback Task Processor Raised Unexpected Exception processing event data:\n{body}')

awx/main/management/commands/run_heartbeet.py

@@ -0,0 +1,67 @@
+import json
+import logging
+import os
+import time
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+from awx.main.dispatch import pg_bus_conn
+
+logger = logging.getLogger('awx.main.commands.run_heartbeet')
+
+
+class Command(BaseCommand):
+    help = 'Launch the web server beacon (heartbeet)'
+
+    def print_banner(self):
+        heartbeet = """
+   **********   **********
+ ************* *************
+*****************************
+ ***********HEART***********
+  *************************
+     *******************
+       *************** _._
+         *********** /`._ `'.      __
+           *******   \  .\| \   _'`  `)
+             ***  (``_)  \| ).'` /`- /
+              *   `\ `;\_ `\\//`-'` /
+                    \ `'.'.| /  __/`
+                     `'--v_|/`'`
+                         __||-._
+                      /'` `-``  `'\\
+                     /         .'` )
+                     \  BEET  '    )
+                      \.          /
+                        '.     /'`
+                           `) |
+                            //
+                            '(.
+                             `\`.
+                               ``"""
+        print(heartbeet)
+
+    def construct_payload(self, action='online'):
+        payload = {
+            'hostname': settings.CLUSTER_HOST_ID,
+            'ip': os.environ.get('MY_POD_IP'),
+            'action': action,
+        }
+        return json.dumps(payload)
+
+    def do_hearbeat_loop(self):
+        with pg_bus_conn(new_connection=True) as conn:
+            while True:
+                logger.debug('Sending heartbeat')
+                conn.notify('web_heartbeet', self.construct_payload())
+                time.sleep(settings.BROADCAST_WEBSOCKET_BEACON_FROM_WEB_RATE_SECONDS)
+
+    # TODO: Send a message with action=offline if we notice a SIGTERM or SIGINT
+    # (wsrelay can use this to remove the node quicker)
+    def handle(self, *arg, **options):
+        self.print_banner()
+
+        # Note: We don't really try any reconnect logic to pg_notify here,
+        # just let supervisor restart if we fail.
+        self.do_hearbeat_loop()

awx/main/management/commands/run_wsrelay.py

@@ -13,13 +13,13 @@ from django.db import connection
 from django.db.migrations.executor import MigrationExecutor
 
 from awx.main.analytics.broadcast_websocket import (
-    BroadcastWebsocketStatsManager,
+    RelayWebsocketStatsManager,
     safe_name,
 )
-from awx.main.wsbroadcast import BroadcastWebsocketManager
+from awx.main.wsrelay import WebSocketRelayManager
 
 
-logger = logging.getLogger('awx.main.wsbroadcast')
+logger = logging.getLogger('awx.main.wsrelay')
 
 
 class Command(BaseCommand):
@@ -99,7 +99,7 @@ class Command(BaseCommand):
             executor = MigrationExecutor(connection)
             migrating = bool(executor.migration_plan(executor.loader.graph.leaf_nodes()))
         except Exception as exc:
-            logger.info(f'Error on startup of run_wsbroadcast (error: {exc}), retry in 10s...')
+            logger.warning(f'Error on startup of run_wsrelay (error: {exc}), retry in 10s...')
             time.sleep(10)
             return
 
@@ -130,9 +130,9 @@ class Command(BaseCommand):
 
         if options.get('status'):
             try:
-                stats_all = BroadcastWebsocketStatsManager.get_stats_sync()
+                stats_all = RelayWebsocketStatsManager.get_stats_sync()
             except redis.exceptions.ConnectionError as e:
-                print(f"Unable to get Broadcast Websocket Status. Failed to connect to redis {e}")
+                print(f"Unable to get Relay Websocket Status. Failed to connect to redis {e}")
                 return
 
             data = {}
@@ -151,22 +151,19 @@ class Command(BaseCommand):
             host_stats = Command.get_connection_status(hostnames, data)
             lines = Command._format_lines(host_stats)
 
-            print(f'Broadcast websocket connection status from "{my_hostname}" to:')
+            print(f'Relay websocket connection status from "{my_hostname}" to:')
             print('\n'.join(lines))
 
             host_stats = Command.get_connection_stats(hostnames, data)
             lines = Command._format_lines(host_stats)
 
-            print(f'\nBroadcast websocket connection stats from "{my_hostname}" to:')
+            print(f'\nRelay websocket connection stats from "{my_hostname}" to:')
             print('\n'.join(lines))
 
             return
 
         try:
-            broadcast_websocket_mgr = BroadcastWebsocketManager()
-            task = broadcast_websocket_mgr.start()
-
-            loop = asyncio.get_event_loop()
-            loop.run_until_complete(task)
+            websocket_relay_manager = WebSocketRelayManager()
+            asyncio.run(websocket_relay_manager.run())
         except KeyboardInterrupt:
-            logger.debug('Terminating Websocket Broadcaster')
+            logger.info('Terminating Websocket Relayer')

awx/main/routing.py

@@ -28,7 +28,7 @@ class AWXProtocolTypeRouter(ProtocolTypeRouter):
 
 websocket_urlpatterns = [
     re_path(r'websocket/$', consumers.EventConsumer.as_asgi()),
-    re_path(r'websocket/broadcast/$', consumers.BroadcastConsumer.as_asgi()),
+    re_path(r'websocket/relay/$', consumers.RelayConsumer.as_asgi()),
 ]
 
 application = AWXProtocolTypeRouter(

awx/main/tasks/jobs.py

@@ -390,6 +390,7 @@ class BaseTask(object):
             logger.error("I/O error({0}) while trying to open lock file [{1}]: {2}".format(e.errno, lock_path, e.strerror))
             raise
 
+        emitted_lockfile_log = False
         start_time = time.time()
         while True:
             try:
@@ -401,6 +402,9 @@ class BaseTask(object):
                     logger.error("I/O error({0}) while trying to aquire lock on file [{1}]: {2}".format(e.errno, lock_path, e.strerror))
                     raise
                 else:
+                    if not emitted_lockfile_log:
+                        logger.info(f"exception acquiring lock {lock_path}: {e}")
+                        emitted_lockfile_log = True
                     time.sleep(1.0)
             self.instance.refresh_from_db(fields=['cancel_flag'])
             if self.instance.cancel_flag or signal_callback():

awx/main/tests/functional/commands/test_callback_receiver.py

@@ -1,7 +1,15 @@
 import pytest
+import time
+from unittest import mock
+from uuid import uuid4
+
+from django.test import TransactionTestCase
+
+from awx.main.dispatch.worker.callback import job_stats_wrapup, CallbackBrokerWorker
 
-from awx.main.dispatch.worker.callback import job_stats_wrapup
 from awx.main.models.jobs import Job
+from awx.main.models.inventory import InventoryUpdate, InventorySource
+from awx.main.models.events import InventoryUpdateEvent
 
 
 @pytest.mark.django_db
@@ -24,3 +32,108 @@ def test_wrapup_does_send_notifications(mocker):
     job.refresh_from_db()
     assert job.host_status_counts == {}
     mock.assert_called_once_with('succeeded')
+
+
+class FakeRedis:
+    def keys(self, *args, **kwargs):
+        return []
+
+    def set(self):
+        pass
+
+    def get(self):
+        return None
+
+    @classmethod
+    def from_url(cls, *args, **kwargs):
+        return cls()
+
+    def pipeline(self):
+        return self
+
+
+class TestCallbackBrokerWorker(TransactionTestCase):
+    @pytest.fixture(autouse=True)
+    def turn_off_websockets(self):
+        with mock.patch('awx.main.dispatch.worker.callback.emit_event_detail', lambda *a, **kw: None):
+            yield
+
+    def get_worker(self):
+        with mock.patch('redis.Redis', new=FakeRedis):  # turn off redis stuff
+            return CallbackBrokerWorker()
+
+    def event_create_kwargs(self):
+        inventory_update = InventoryUpdate.objects.create(source='file', inventory_source=InventorySource.objects.create(source='file'))
+        return dict(inventory_update=inventory_update, created=inventory_update.created)
+
+    def test_flush_with_valid_event(self):
+        worker = self.get_worker()
+        events = [InventoryUpdateEvent(uuid=str(uuid4()), **self.event_create_kwargs())]
+        worker.buff = {InventoryUpdateEvent: events}
+        worker.flush()
+        assert worker.buff.get(InventoryUpdateEvent, []) == []
+        assert InventoryUpdateEvent.objects.filter(uuid=events[0].uuid).count() == 1
+
+    def test_flush_with_invalid_event(self):
+        worker = self.get_worker()
+        kwargs = self.event_create_kwargs()
+        events = [
+            InventoryUpdateEvent(uuid=str(uuid4()), stdout='good1', **kwargs),
+            InventoryUpdateEvent(uuid=str(uuid4()), stdout='bad', counter=-2, **kwargs),
+            InventoryUpdateEvent(uuid=str(uuid4()), stdout='good2', **kwargs),
+        ]
+        worker.buff = {InventoryUpdateEvent: events.copy()}
+        worker.flush()
+        assert InventoryUpdateEvent.objects.filter(uuid=events[0].uuid).count() == 1
+        assert InventoryUpdateEvent.objects.filter(uuid=events[1].uuid).count() == 0
+        assert InventoryUpdateEvent.objects.filter(uuid=events[2].uuid).count() == 1
+        assert worker.buff == {InventoryUpdateEvent: [events[1]]}
+
+    def test_duplicate_key_not_saved_twice(self):
+        worker = self.get_worker()
+        events = [InventoryUpdateEvent(uuid=str(uuid4()), **self.event_create_kwargs())]
+        worker.buff = {InventoryUpdateEvent: events.copy()}
+        worker.flush()
+
+        # put current saved event in buffer (error case)
+        worker.buff = {InventoryUpdateEvent: [InventoryUpdateEvent.objects.get(uuid=events[0].uuid)]}
+        worker.last_flush = time.time() - 2.0
+        # here, the bulk_create will fail with UNIQUE constraint violation, but individual saves should resolve it
+        worker.flush()
+        assert InventoryUpdateEvent.objects.filter(uuid=events[0].uuid).count() == 1
+        assert worker.buff.get(InventoryUpdateEvent, []) == []
+
+    def test_give_up_on_bad_event(self):
+        worker = self.get_worker()
+        events = [InventoryUpdateEvent(uuid=str(uuid4()), counter=-2, **self.event_create_kwargs())]
+        worker.buff = {InventoryUpdateEvent: events.copy()}
+
+        for i in range(5):
+            worker.last_flush = time.time() - 2.0
+            worker.flush()
+
+        # Could not save, should be logged, and buffer should be cleared
+        assert worker.buff.get(InventoryUpdateEvent, []) == []
+        assert InventoryUpdateEvent.objects.filter(uuid=events[0].uuid).count() == 0  # sanity
+
+    def test_postgres_invalid_NUL_char(self):
+        # In postgres, text fields reject NUL character, 0x00
+        # tests use sqlite3 which will not raise an error
+        # but we can still test that it is sanitized before saving
+        worker = self.get_worker()
+        kwargs = self.event_create_kwargs()
+        events = [InventoryUpdateEvent(uuid=str(uuid4()), stdout="\x00", **kwargs)]
+        assert "\x00" in events[0].stdout  # sanity
+        worker.buff = {InventoryUpdateEvent: events.copy()}
+
+        with mock.patch.object(InventoryUpdateEvent.objects, 'bulk_create', side_effect=ValueError):
+            with mock.patch.object(events[0], 'save', side_effect=ValueError):
+                worker.flush()
+
+            assert "\x00" not in events[0].stdout
+
+            worker.last_flush = time.time() - 2.0
+            worker.flush()
+
+            event = InventoryUpdateEvent.objects.get(uuid=events[0].uuid)
+            assert "\x00" not in event.stdout

awx/main/utils/handlers.py

@@ -103,6 +103,10 @@ ColorHandler = logging.StreamHandler
 if settings.COLOR_LOGS is True:
     try:
         from logutils.colorize import ColorizingStreamHandler
+        import colorama
+
+        colorama.deinit()
+        colorama.init(wrap=False, convert=False, strip=False)
 
         class ColorHandler(ColorizingStreamHandler):
             def colorize(self, line, record):

awx/main/wsbroadcast.py

@@ -1,209 +0,0 @@
-import json
-import logging
-import asyncio
-
-import aiohttp
-from aiohttp import client_exceptions
-from asgiref.sync import sync_to_async
-
-from channels.layers import get_channel_layer
-
-from django.conf import settings
-from django.apps import apps
-from django.core.serializers.json import DjangoJSONEncoder
-
-from awx.main.analytics.broadcast_websocket import (
-    BroadcastWebsocketStats,
-    BroadcastWebsocketStatsManager,
-)
-import awx.main.analytics.subsystem_metrics as s_metrics
-
-logger = logging.getLogger('awx.main.wsbroadcast')
-
-
-def wrap_broadcast_msg(group, message: str):
-    # TODO: Maybe wrap as "group","message" so that we don't need to
-    # encode/decode as json.
-    return json.dumps(dict(group=group, message=message), cls=DjangoJSONEncoder)
-
-
-def unwrap_broadcast_msg(payload: dict):
-    return (payload['group'], payload['message'])
-
-
-@sync_to_async
-def get_broadcast_hosts():
-    Instance = apps.get_model('main', 'Instance')
-    instances = (
-        Instance.objects.exclude(hostname=Instance.objects.my_hostname())
-        .exclude(node_type='execution')
-        .exclude(node_type='hop')
-        .order_by('hostname')
-        .values('hostname', 'ip_address')
-        .distinct()
-    )
-    return {i['hostname']: i['ip_address'] or i['hostname'] for i in instances}
-
-
-def get_local_host():
-    Instance = apps.get_model('main', 'Instance')
-    return Instance.objects.my_hostname()
-
-
-class WebsocketTask:
-    def __init__(
-        self,
-        name,
-        event_loop,
-        stats: BroadcastWebsocketStats,
-        remote_host: str,
-        remote_port: int = settings.BROADCAST_WEBSOCKET_PORT,
-        protocol: str = settings.BROADCAST_WEBSOCKET_PROTOCOL,
-        verify_ssl: bool = settings.BROADCAST_WEBSOCKET_VERIFY_CERT,
-        endpoint: str = 'broadcast',
-    ):
-        self.name = name
-        self.event_loop = event_loop
-        self.stats = stats
-        self.remote_host = remote_host
-        self.remote_port = remote_port
-        self.endpoint = endpoint
-        self.protocol = protocol
-        self.verify_ssl = verify_ssl
-        self.channel_layer = None
-        self.subsystem_metrics = s_metrics.Metrics(instance_name=name)
-
-    async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse):
-        raise RuntimeError("Implement me")
-
-    async def connect(self, attempt):
-        from awx.main.consumers import WebsocketSecretAuthHelper  # noqa
-
-        logger.debug(f"Connection from {self.name} to {self.remote_host} attempt number {attempt}.")
-
-        '''
-        Can not put get_channel_layer() in the init code because it is in the init
-        path of channel layers i.e. RedisChannelLayer() calls our init code.
-        '''
-        if not self.channel_layer:
-            self.channel_layer = get_channel_layer()
-
-        try:
-            if attempt > 0:
-                await asyncio.sleep(settings.BROADCAST_WEBSOCKET_RECONNECT_RETRY_RATE_SECONDS)
-        except asyncio.CancelledError:
-            logger.warning(f"Connection from {self.name} to {self.remote_host} cancelled")
-            raise
-
-        uri = f"{self.protocol}://{self.remote_host}:{self.remote_port}/websocket/{self.endpoint}/"
-        timeout = aiohttp.ClientTimeout(total=10)
-
-        secret_val = WebsocketSecretAuthHelper.construct_secret()
-        try:
-            async with aiohttp.ClientSession(headers={'secret': secret_val}, timeout=timeout) as session:
-                async with session.ws_connect(uri, ssl=self.verify_ssl, heartbeat=20) as websocket:
-                    logger.info(f"Connection from {self.name} to {self.remote_host} established.")
-                    self.stats.record_connection_established()
-                    attempt = 0
-                    await self.run_loop(websocket)
-        except asyncio.CancelledError:
-            # TODO: Check if connected and disconnect
-            # Possibly use run_until_complete() if disconnect is async
-            logger.warning(f"Connection from {self.name} to {self.remote_host} cancelled.")
-            self.stats.record_connection_lost()
-            raise
-        except client_exceptions.ClientConnectorError as e:
-            logger.warning(f"Connection from {self.name} to {self.remote_host} failed: '{e}'.")
-        except asyncio.TimeoutError:
-            logger.warning(f"Connection from {self.name} to {self.remote_host} timed out.")
-        except Exception as e:
-            # Early on, this is our canary. I'm not sure what exceptions we can really encounter.
-            logger.exception(f"Connection from {self.name} to {self.remote_host} failed for unknown reason: '{e}'.")
-        else:
-            logger.warning(f"Connection from {self.name} to {self.remote_host} list.")
-
-        self.stats.record_connection_lost()
-        self.start(attempt=attempt + 1)
-
-    def start(self, attempt=0):
-        self.async_task = self.event_loop.create_task(self.connect(attempt=attempt))
-
-    def cancel(self):
-        self.async_task.cancel()
-
-
-class BroadcastWebsocketTask(WebsocketTask):
-    async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse):
-        async for msg in websocket:
-            self.stats.record_message_received()
-
-            if msg.type == aiohttp.WSMsgType.ERROR:
-                break
-            elif msg.type == aiohttp.WSMsgType.TEXT:
-                try:
-                    payload = json.loads(msg.data)
-                except json.JSONDecodeError:
-                    logmsg = "Failed to decode broadcast message"
-                    if logger.isEnabledFor(logging.DEBUG):
-                        logmsg = "{} {}".format(logmsg, payload)
-                    logger.warning(logmsg)
-                    continue
-                (group, message) = unwrap_broadcast_msg(payload)
-                if group == "metrics":
-                    self.subsystem_metrics.store_metrics(message)
-                    continue
-                await self.channel_layer.group_send(group, {"type": "internal.message", "text": message})
-
-
-class BroadcastWebsocketManager(object):
-    def __init__(self):
-        self.event_loop = asyncio.get_event_loop()
-        '''
-        {
-            'hostname1': BroadcastWebsocketTask(),
-            'hostname2': BroadcastWebsocketTask(),
-            'hostname3': BroadcastWebsocketTask(),
-        }
-        '''
-        self.broadcast_tasks = dict()
-        self.local_hostname = get_local_host()
-        self.stats_mgr = BroadcastWebsocketStatsManager(self.event_loop, self.local_hostname)
-
-    async def run_per_host_websocket(self):
-
-        while True:
-            known_hosts = await get_broadcast_hosts()
-            future_remote_hosts = known_hosts.keys()
-            current_remote_hosts = self.broadcast_tasks.keys()
-            deleted_remote_hosts = set(current_remote_hosts) - set(future_remote_hosts)
-            new_remote_hosts = set(future_remote_hosts) - set(current_remote_hosts)
-
-            remote_addresses = {k: v.remote_host for k, v in self.broadcast_tasks.items()}
-            for hostname, address in known_hosts.items():
-                if hostname in self.broadcast_tasks and address != remote_addresses[hostname]:
-                    deleted_remote_hosts.add(hostname)
-                    new_remote_hosts.add(hostname)
-
-            if deleted_remote_hosts:
-                logger.warning(f"Removing {deleted_remote_hosts} from websocket broadcast list")
-            if new_remote_hosts:
-                logger.warning(f"Adding {new_remote_hosts} to websocket broadcast list")
-
-            for h in deleted_remote_hosts:
-                self.broadcast_tasks[h].cancel()
-                del self.broadcast_tasks[h]
-                self.stats_mgr.delete_remote_host_stats(h)
-
-            for h in new_remote_hosts:
-                stats = self.stats_mgr.new_remote_host_stats(h)
-                broadcast_task = BroadcastWebsocketTask(name=self.local_hostname, event_loop=self.event_loop, stats=stats, remote_host=known_hosts[h])
-                broadcast_task.start()
-                self.broadcast_tasks[h] = broadcast_task
-
-            await asyncio.sleep(settings.BROADCAST_WEBSOCKET_NEW_INSTANCE_POLL_RATE_SECONDS)
-
-    def start(self):
-        self.stats_mgr.start()
-
-        self.async_task = self.event_loop.create_task(self.run_per_host_websocket())
-        return self.async_task

awx/main/wsrelay.py

@@ -0,0 +1,307 @@
+import json
+import logging
+import asyncio
+
+import aiohttp
+from aiohttp import client_exceptions
+from asgiref.sync import sync_to_async
+
+from channels.layers import get_channel_layer
+from channels.db import database_sync_to_async
+
+from django.conf import settings
+from django.apps import apps
+
+import psycopg
+
+from awx.main.analytics.broadcast_websocket import (
+    RelayWebsocketStats,
+    RelayWebsocketStatsManager,
+)
+import awx.main.analytics.subsystem_metrics as s_metrics
+
+logger = logging.getLogger('awx.main.wsrelay')
+
+
+def wrap_broadcast_msg(group, message: str):
+    # TODO: Maybe wrap as "group","message" so that we don't need to
+    # encode/decode as json.
+    return dict(group=group, message=message)
+
+
+def get_local_host():
+    Instance = apps.get_model('main', 'Instance')
+    return Instance.objects.my_hostname()
+
+
+class WebsocketRelayConnection:
+    def __init__(
+        self,
+        name,
+        stats: RelayWebsocketStats,
+        remote_host: str,
+        remote_port: int = settings.BROADCAST_WEBSOCKET_PORT,
+        protocol: str = settings.BROADCAST_WEBSOCKET_PROTOCOL,
+        verify_ssl: bool = settings.BROADCAST_WEBSOCKET_VERIFY_CERT,
+    ):
+        self.name = name
+        self.event_loop = asyncio.get_event_loop()
+        self.stats = stats
+        self.remote_host = remote_host
+        self.remote_port = remote_port
+        self.protocol = protocol
+        self.verify_ssl = verify_ssl
+        self.channel_layer = None
+        self.subsystem_metrics = s_metrics.Metrics(instance_name=name)
+        self.producers = dict()
+        self.connected = False
+
+    async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse):
+        raise RuntimeError("Implement me")
+
+    async def connect(self):
+        from awx.main.consumers import WebsocketSecretAuthHelper  # noqa
+
+        logger.debug(f"Connection attempt from {self.name} to {self.remote_host}")
+
+        '''
+        Can not put get_channel_layer() in the init code because it is in the init
+        path of channel layers i.e. RedisChannelLayer() calls our init code.
+        '''
+        if not self.channel_layer:
+            self.channel_layer = get_channel_layer()
+
+        uri = f"{self.protocol}://{self.remote_host}:{self.remote_port}/websocket/relay/"
+        timeout = aiohttp.ClientTimeout(total=10)
+
+        secret_val = WebsocketSecretAuthHelper.construct_secret()
+        try:
+            async with aiohttp.ClientSession(headers={'secret': secret_val}, timeout=timeout) as session:
+                async with session.ws_connect(uri, ssl=self.verify_ssl, heartbeat=20) as websocket:
+                    logger.info(f"Connection from {self.name} to {self.remote_host} established.")
+                    self.stats.record_connection_established()
+                    self.connected = True
+                    await self.run_connection(websocket)
+        except asyncio.CancelledError:
+            # TODO: Check if connected and disconnect
+            # Possibly use run_until_complete() if disconnect is async
+            logger.warning(f"Connection from {self.name} to {self.remote_host} cancelled.")
+        except client_exceptions.ClientConnectorError as e:
+            logger.warning(f"Connection from {self.name} to {self.remote_host} failed: '{e}'.", exc_info=True)
+        except asyncio.TimeoutError:
+            logger.warning(f"Connection from {self.name} to {self.remote_host} timed out.")
+        except Exception as e:
+            # Early on, this is our canary. I'm not sure what exceptions we can really encounter.
+            logger.warning(f"Connection from {self.name} to {self.remote_host} failed for unknown reason: '{e}'.", exc_info=True)
+        else:
+            logger.debug(f"Connection from {self.name} to {self.remote_host} lost, but no exception was raised.")
+        finally:
+            self.connected = False
+            self.stats.record_connection_lost()
+
+    def start(self):
+        self.async_task = self.event_loop.create_task(self.connect())
+        return self.async_task
+
+    def cancel(self):
+        self.async_task.cancel()
+
+    async def run_connection(self, websocket: aiohttp.ClientWebSocketResponse):
+        # create a dedicated subsystem metric producer to handle local subsystem
+        # metrics messages
+        # the "metrics" group is not subscribed to in the typical fashion, so we
+        # just explicitly create it
+        producer = self.event_loop.create_task(self.run_producer("metrics", websocket, "metrics"))
+        self.producers["metrics"] = {"task": producer, "subscriptions": {"metrics"}}
+        async for msg in websocket:
+            self.stats.record_message_received()
+
+            if msg.type == aiohttp.WSMsgType.ERROR:
+                break
+            elif msg.type == aiohttp.WSMsgType.TEXT:
+                try:
+                    payload = json.loads(msg.data)
+                except json.JSONDecodeError:
+                    logmsg = "Failed to decode message from web node"
+                    if logger.isEnabledFor(logging.DEBUG):
+                        logmsg = "{} {}".format(logmsg, payload)
+                    logger.warning(logmsg)
+                    continue
+
+            if payload.get("type") == "consumer.subscribe":
+                for group in payload['groups']:
+                    name = f"{self.remote_host}-{group}"
+                    origin_channel = payload['origin_channel']
+                    if not self.producers.get(name):
+                        producer = self.event_loop.create_task(self.run_producer(name, websocket, group))
+                        self.producers[name] = {"task": producer, "subscriptions": {origin_channel}}
+                        logger.debug(f"Producer {name} started.")
+                    else:
+                        self.producers[name]["subscriptions"].add(origin_channel)
+                        logger.debug(f"Connection from {self.name} to {self.remote_host} added subscription to {group}.")
+
+            if payload.get("type") == "consumer.unsubscribe":
+                for group in payload['groups']:
+                    name = f"{self.remote_host}-{group}"
+                    origin_channel = payload['origin_channel']
+                    try:
+                        self.producers[name]["subscriptions"].remove(origin_channel)
+                        logger.debug(f"Unsubscribed {origin_channel} from {name}")
+                    except KeyError:
+                        logger.warning(f"Producer {name} not found.")
+
+    async def run_producer(self, name, websocket, group):
+        try:
+            logger.info(f"Starting producer for {name}")
+
+            consumer_channel = await self.channel_layer.new_channel()
+            await self.channel_layer.group_add(group, consumer_channel)
+            logger.debug(f"Producer {name} added to group {group} and is now awaiting messages.")
+
+            while True:
+                try:
+                    msg = await asyncio.wait_for(self.channel_layer.receive(consumer_channel), timeout=10)
+                    if not msg.get("needs_relay"):
+                        # This is added in by emit_channel_notification(). It prevents us from looping
+                        # in the event that we are sharing a redis with a web instance. We'll see the
+                        # message once (it'll have needs_relay=True), we'll delete that, and then forward
+                        # the message along. The web instance will add it back to the same channels group,
+                        # but it won't have needs_relay=True, so we'll ignore it.
+                        continue
+
+                    # We need to copy the message because we're going to delete the needs_relay key
+                    # and we don't want to modify the original message because other producers may
+                    # still need to act on it. It seems weird, but it's necessary.
+                    msg = dict(msg)
+                    del msg["needs_relay"]
+                except asyncio.TimeoutError:
+                    current_subscriptions = self.producers[name]["subscriptions"]
+                    if len(current_subscriptions) == 0:
+                        logger.info(f"Producer {name} has no subscribers, shutting down.")
+                        return
+
+                    continue
+
+                await websocket.send_json(wrap_broadcast_msg(group, msg))
+        except ConnectionResetError:
+            # This can be hit when a web node is scaling down and we try to write to it.
+            # There's really nothing to do in this case and it's a fairly typical thing to happen.
+            # We'll log it as debug, but it's not really a problem.
+            logger.debug(f"Producer {name} connection reset.")
+            pass
+        except Exception:
+            # Note, this is very intentional and important since we do not otherwise
+            # ever check the result of this future. Without this line you will not see an error if
+            # something goes wrong in here.
+            logger.exception(f"Event relay producer {name} crashed")
+        finally:
+            await self.channel_layer.group_discard(group, consumer_channel)
+            del self.producers[name]
+
+
+class WebSocketRelayManager(object):
+    def __init__(self):
+
+        self.local_hostname = get_local_host()
+        self.relay_connections = dict()
+        # hostname -> ip
+        self.known_hosts: Dict[str, str] = dict()
+
+    async def pg_consumer(self, conn):
+        try:
+            await conn.execute("LISTEN web_heartbeet")
+            async for notif in conn.notifies():
+                if notif is not None and notif.channel == "web_heartbeet":
+                    try:
+                        payload = json.loads(notif.payload)
+                    except json.JSONDecodeError:
+                        logmsg = "Failed to decode message from pg_notify channel `web_heartbeet`"
+                        if logger.isEnabledFor(logging.DEBUG):
+                            logmsg = "{} {}".format(logmsg, payload)
+                        logger.warning(logmsg)
+                        continue
+
+                    # Skip if the message comes from the same host we are running on
+                    # In this case, we'll be sharing a redis, no need to relay.
+                    if payload.get("hostname") == self.local_hostname:
+                        continue
+
+                    if payload.get("action") == "online":
+                        hostname = payload["hostname"]
+                        ip = payload["ip"]
+                        if ip is None:
+                            # If we don't get an IP, just try the hostname, maybe it resolves
+                            ip = hostname
+                        self.known_hosts[hostname] = ip
+                        logger.debug(f"Web host {hostname} ({ip}) online heartbeat received.")
+                    elif payload.get("action") == "offline":
+                        hostname = payload["hostname"]
+                        del self.known_hosts[hostname]
+                        logger.debug(f"Web host {hostname} ({ip}) offline heartbeat received.")
+        except Exception as e:
+            # This catch-all is the same as the one above. asyncio will eat the exception
+            # but we want to know about it.
+            logger.exception(f"pg_consumer exception")
+
+    async def run(self):
+        event_loop = asyncio.get_running_loop()
+
+        stats_mgr = RelayWebsocketStatsManager(event_loop, self.local_hostname)
+        stats_mgr.start()
+
+        # Set up a pg_notify consumer for allowing web nodes to "provision" and "deprovision" themselves gracefully.
+        database_conf = settings.DATABASES['default']
+        async_conn = await psycopg.AsyncConnection.connect(
+            dbname=database_conf['NAME'],
+            host=database_conf['HOST'],
+            user=database_conf['USER'],
+            password=database_conf['PASSWORD'],
+            port=database_conf['PORT'],
+            **database_conf.get("OPTIONS", {}),
+        )
+        await async_conn.set_autocommit(True)
+        event_loop.create_task(self.pg_consumer(async_conn))
+
+        # Establishes a websocket connection to /websocket/relay on all API servers
+        while True:
+            # logger.info("Current known hosts: {}".format(self.known_hosts))
+            future_remote_hosts = self.known_hosts.keys()
+            current_remote_hosts = self.relay_connections.keys()
+            deleted_remote_hosts = set(current_remote_hosts) - set(future_remote_hosts)
+            new_remote_hosts = set(future_remote_hosts) - set(current_remote_hosts)
+
+            # This loop handles if we get an advertisement from a host we already know about but
+            # the advertisement has a different IP than we are currently connected to.
+            for hostname, address in self.known_hosts.items():
+                if hostname not in self.relay_connections:
+                    # We've picked up a new hostname that we don't know about yet.
+                    continue
+
+                if address != self.relay_connections[hostname].remote_host:
+                    deleted_remote_hosts.add(hostname)
+                    new_remote_hosts.add(hostname)
+
+            # Delete any hosts with closed connections
+            for hostname, relay_conn in self.relay_connections.items():
+                if not relay_conn.connected:
+                    deleted_remote_hosts.add(hostname)
+
+            if deleted_remote_hosts:
+                logger.info(f"Removing {deleted_remote_hosts} from websocket broadcast list")
+
+            if new_remote_hosts:
+                logger.info(f"Adding {new_remote_hosts} to websocket broadcast list")
+
+            for h in deleted_remote_hosts:
+                self.relay_connections[h].cancel()
+                del self.relay_connections[h]
+                del self.known_hosts[h]
+                stats_mgr.delete_remote_host_stats(h)
+
+            for h in new_remote_hosts:
+                stats = stats_mgr.new_remote_host_stats(h)
+                relay_connection = WebsocketRelayConnection(name=self.local_hostname, stats=stats, remote_host=self.known_hosts[h])
+                relay_connection.start()
+                self.relay_connections[h] = relay_connection
+
+            await asyncio.sleep(settings.BROADCAST_WEBSOCKET_NEW_INSTANCE_POLL_RATE_SECONDS)

awx/settings/defaults.py

@@ -215,6 +215,9 @@ JOB_EVENT_MAX_QUEUE_SIZE = 10000
 # The number of job events to migrate per-transaction when moving from int -> bigint
 JOB_EVENT_MIGRATION_CHUNK_SIZE = 1000000
 
+# The prefix of the redis key that stores metrics
+SUBSYSTEM_METRICS_REDIS_KEY_PREFIX = "awx_metrics"
+
 # Histogram buckets for the callback_receiver_batch_events_insert_db metric
 SUBSYSTEM_METRICS_BATCH_INSERT_BUCKETS = [10, 50, 150, 350, 650, 2000]
 
@@ -418,6 +421,9 @@ AUTH_BASIC_ENABLED = True
 # when trying to access a UI page that requries authentication.
 LOGIN_REDIRECT_OVERRIDE = ''
 
+# Note: This setting may be overridden by database settings.
+ALLOW_METRICS_FOR_ANONYMOUS_USERS = False
+
 DEVSERVER_DEFAULT_ADDR = '0.0.0.0'
 DEVSERVER_DEFAULT_PORT = '8013'
 
@@ -470,21 +476,15 @@ _SOCIAL_AUTH_PIPELINE_BASE = (
     'social_core.pipeline.user.get_username',
     'social_core.pipeline.social_auth.associate_by_email',
     'social_core.pipeline.user.create_user',
-    'awx.sso.pipeline.check_user_found_or_created',
+    'awx.sso.social_base_pipeline.check_user_found_or_created',
     'social_core.pipeline.social_auth.associate_user',
     'social_core.pipeline.social_auth.load_extra_data',
-    'awx.sso.pipeline.set_is_active_for_new_user',
+    'awx.sso.social_base_pipeline.set_is_active_for_new_user',
     'social_core.pipeline.user.user_details',
-    'awx.sso.pipeline.prevent_inactive_login',
-)
-SOCIAL_AUTH_PIPELINE = _SOCIAL_AUTH_PIPELINE_BASE + ('awx.sso.pipeline.update_user_orgs', 'awx.sso.pipeline.update_user_teams')
-SOCIAL_AUTH_SAML_PIPELINE = _SOCIAL_AUTH_PIPELINE_BASE + (
-    'awx.sso.pipeline.update_user_orgs_by_saml_attr',
-    'awx.sso.pipeline.update_user_teams_by_saml_attr',
-    'awx.sso.pipeline.update_user_orgs',
-    'awx.sso.pipeline.update_user_teams',
-    'awx.sso.pipeline.update_user_flags',
+    'awx.sso.social_base_pipeline.prevent_inactive_login',
 )
+SOCIAL_AUTH_PIPELINE = _SOCIAL_AUTH_PIPELINE_BASE + ('awx.sso.social_pipeline.update_user_orgs', 'awx.sso.social_pipeline.update_user_teams')
+SOCIAL_AUTH_SAML_PIPELINE = _SOCIAL_AUTH_PIPELINE_BASE + ('awx.sso.saml_pipeline.populate_user', 'awx.sso.saml_pipeline.update_user_flags')
 SAML_AUTO_CREATE_OBJECTS = True
 
 SOCIAL_AUTH_LOGIN_URL = '/'
@@ -844,7 +844,7 @@ LOGGING = {
         'awx.main.commands.run_callback_receiver': {'handlers': ['callback_receiver']},  # level handled by dynamic_level_filter
         'awx.main.dispatch': {'handlers': ['dispatcher']},
         'awx.main.consumers': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'INFO'},
-        'awx.main.wsbroadcast': {'handlers': ['wsbroadcast']},
+        'awx.main.wsrelay': {'handlers': ['wsrelay']},
         'awx.main.rsyslog_configurer': {'handlers': ['rsyslog_configurer']},
         'awx.main.cache_clear': {'handlers': ['cache_clear']},
         'awx.main.commands.inventory_import': {'handlers': ['inventory_import'], 'propagate': False},
@@ -855,7 +855,7 @@ LOGGING = {
         'awx.main.signals': {'level': 'INFO'},  # very verbose debug-level logs
         'awx.api.permissions': {'level': 'INFO'},  # very verbose debug-level logs
         'awx.analytics': {'handlers': ['external_logger'], 'level': 'INFO', 'propagate': False},
-        'awx.analytics.broadcast_websocket': {'handlers': ['console', 'file', 'wsbroadcast', 'external_logger'], 'level': 'INFO', 'propagate': False},
+        'awx.analytics.broadcast_websocket': {'handlers': ['console', 'file', 'wsrelay', 'external_logger'], 'level': 'INFO', 'propagate': False},
         'awx.analytics.performance': {'handlers': ['console', 'file', 'tower_warnings', 'external_logger'], 'level': 'DEBUG', 'propagate': False},
         'awx.analytics.job_lifecycle': {'handlers': ['console', 'job_lifecycle'], 'level': 'DEBUG', 'propagate': False},
         'django_auth_ldap': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'DEBUG'},
@@ -873,7 +873,7 @@ handler_config = {
     'tower_warnings': {'filename': 'tower.log'},
     'callback_receiver': {'filename': 'callback_receiver.log'},
     'dispatcher': {'filename': 'dispatcher.log', 'formatter': 'dispatcher'},
-    'wsbroadcast': {'filename': 'wsbroadcast.log'},
+    'wsrelay': {'filename': 'wsrelay.log'},
     'task_system': {'filename': 'task_system.log'},
     'rbac_migrations': {'filename': 'tower_rbac_migrations.log'},
     'job_lifecycle': {'filename': 'job_lifecycle.log', 'formatter': 'job_lifecycle'},
@@ -984,6 +984,9 @@ BROADCAST_WEBSOCKET_NEW_INSTANCE_POLL_RATE_SECONDS = 10
 # How often websocket process will generate stats
 BROADCAST_WEBSOCKET_STATS_POLL_RATE_SECONDS = 5
 
+# How often should web instances advertise themselves?
+BROADCAST_WEBSOCKET_BEACON_FROM_WEB_RATE_SECONDS = 15
+
 DJANGO_GUID = {'GUID_HEADER_NAME': 'X-API-Request-Id'}
 
 # Name of the default task queue

awx/sso/backends.py

@@ -11,11 +11,9 @@ import ldap
 # Django
 from django.dispatch import receiver
 from django.contrib.auth.models import User
-from django.contrib.contenttypes.models import ContentType
 from django.conf import settings as django_settings
 from django.core.signals import setting_changed
 from django.utils.encoding import force_str
-from django.db.utils import IntegrityError
 
 # django-auth-ldap
 from django_auth_ldap.backend import LDAPSettings as BaseLDAPSettings
@@ -36,6 +34,7 @@ from social_core.backends.saml import SAMLIdentityProvider as BaseSAMLIdentityPr
 
 # Ansible Tower
 from awx.sso.models import UserEnterpriseAuth
+from awx.sso.common import create_org_and_teams, reconcile_users_org_team_mappings
 
 logger = logging.getLogger('awx.sso.backends')
 
@@ -354,7 +353,9 @@ def _update_m2m_from_groups(ldap_user, opts, remove=True):
                 continue
             if ldap_user._get_groups().is_member_of(group_dn):
                 return True
+    if remove:
         return False
+    return None
 
 
 @receiver(populate_user, dispatch_uid='populate-ldap-user')
@@ -363,8 +364,6 @@ def on_populate_user(sender, **kwargs):
     Handle signal from LDAP backend to populate the user object.  Update user
     organization/team memberships according to their LDAP groups.
     """
-    from awx.main.models import Organization, Team
-
     user = kwargs['user']
     ldap_user = kwargs['ldap_user']
     backend = ldap_user.backend
@@ -388,43 +387,16 @@ def on_populate_user(sender, **kwargs):
 
     org_map = getattr(backend.settings, 'ORGANIZATION_MAP', {})
     team_map = getattr(backend.settings, 'TEAM_MAP', {})
-
-    # Move this junk into save of the settings for performance later, there is no need to do that here
-    #    with maybe the exception of someone defining this in settings before the server is started?
-    # ==============================================================================================================
-
-    # Get all of the IDs and names of orgs in the DB and create any new org defined in LDAP that does not exist in the DB
-    existing_orgs = {}
-    for (org_id, org_name) in Organization.objects.all().values_list('id', 'name'):
-        existing_orgs[org_name] = org_id
-
-    # Create any orgs (if needed) for all entries in the org and team maps
-    for org_name in set(list(org_map.keys()) + [item.get('organization', None) for item in team_map.values()]):
-        if org_name and org_name not in existing_orgs:
-            logger.info("LDAP adapter is creating org {}".format(org_name))
-            try:
-                new_org = Organization.objects.create(name=org_name)
-            except IntegrityError:
-                # Another thread must have created this org before we did so now we need to get it
-                new_org = Organization.objects.get(name=org_name)
-            # Add the org name to the existing orgs since we created it and we may need it to build the teams below
-            existing_orgs[org_name] = new_org.id
-
-    # Do the same for teams
-    existing_team_names = list(Team.objects.all().values_list('name', flat=True))
+    orgs_list = list(org_map.keys())
+    team_map = {}
     for team_name, team_opts in team_map.items():
         if not team_opts.get('organization', None):
             # You can't save the LDAP config in the UI w/o an org (or '' or null as the org) so if we somehow got this condition its an error
             logger.error("Team named {} in LDAP team map settings is invalid due to missing organization".format(team_name))
             continue
-        if team_name not in existing_team_names:
-            try:
-                Team.objects.create(name=team_name, organization_id=existing_orgs[team_opts['organization']])
-            except IntegrityError:
-                # If another process got here before us that is ok because we don't need the ID from this team or anything
-                pass
-    # End move some day
-    # ==============================================================================================================
+        team_map[team_name] = team_opts['organization']
+
+    create_org_and_teams(orgs_list, team_map, 'LDAP')
 
     # Compute in memory what the state is of the different LDAP orgs
     org_roles_and_ldap_attributes = {'admin_role': 'admins', 'auditor_role': 'auditors', 'member_role': 'users'}
@@ -452,7 +424,10 @@ def on_populate_user(sender, **kwargs):
         remove = bool(team_opts.get('remove', True))
         state = _update_m2m_from_groups(ldap_user, users_opts, remove)
         if state is not None:
-            desired_team_states[team_name] = {'member_role': state}
+            organization = team_opts['organization']
+            if organization not in desired_team_states:
+                desired_team_states[organization] = {}
+            desired_team_states[organization][team_name] = {'member_role': state}
 
     # Check if user.profile is available, otherwise force user.save()
     try:
@@ -470,60 +445,3 @@ def on_populate_user(sender, **kwargs):
         profile.save()
 
     reconcile_users_org_team_mappings(user, desired_org_states, desired_team_states, 'LDAP')
-
-
-def reconcile_users_org_team_mappings(user, desired_org_states, desired_team_states, source):
-    from awx.main.models import Organization, Team
-
-    content_types = []
-    reconcile_items = []
-    if desired_org_states:
-        content_types.append(ContentType.objects.get_for_model(Organization))
-        reconcile_items.append(('organization', desired_org_states, Organization))
-    if desired_team_states:
-        content_types.append(ContentType.objects.get_for_model(Team))
-        reconcile_items.append(('team', desired_team_states, Team))
-
-    if not content_types:
-        # If both desired states were empty we can simply return because there is nothing to reconcile
-        return
-
-    # users_roles is a flat set of IDs
-    users_roles = set(user.roles.filter(content_type__in=content_types).values_list('pk', flat=True))
-
-    for object_type, desired_states, model in reconcile_items:
-        # Get all of the roles in the desired states for efficient DB extraction
-        roles = []
-        for sub_dict in desired_states.values():
-            for role_name in sub_dict:
-                if sub_dict[role_name] is None:
-                    continue
-                if role_name not in roles:
-                    roles.append(role_name)
-
-        # Get a set of named tuples for the org/team name plus all of the roles we got above
-        model_roles = model.objects.filter(name__in=desired_states.keys()).values_list('name', *roles, named=True)
-        for row in model_roles:
-            for role_name in roles:
-                desired_state = desired_states.get(row.name, {})
-                if desired_state[role_name] is None:
-                    # The mapping was not defined for this [org/team]/role so we can just pass
-                    pass
-
-                # If somehow the auth adapter knows about an items role but that role is not defined in the DB we are going to print a pretty error
-                # This is your classic safety net that we should never hit; but here you are reading this comment... good luck and Godspeed.
-                role_id = getattr(row, role_name, None)
-                if role_id is None:
-                    logger.error("{} adapter wanted to manage role {} of {} {} but that role is not defined".format(source, role_name, object_type, row.name))
-                    continue
-
-                if desired_state[role_name]:
-                    # The desired state was the user mapped into the object_type, if the user was not mapped in map them in
-                    if role_id not in users_roles:
-                        logger.debug("{} adapter adding user {} to {} {} as {}".format(source, user.username, object_type, row.name, role_name))
-                        user.roles.add(role_id)
-                else:
-                    # The desired state was the user was not mapped into the org, if the user has the permission remove it
-                    if role_id in users_roles:
-                        logger.debug("{} adapter removing user {} permission of {} from {} {}".format(source, user.username, role_name, object_type, row.name))
-                        user.roles.remove(role_id)

awx/sso/common.py

@@ -0,0 +1,171 @@
+# Copyright (c) 2022 Ansible, Inc.
+# All Rights Reserved.
+
+import logging
+
+from django.contrib.contenttypes.models import ContentType
+from django.db.utils import IntegrityError
+from awx.main.models import Organization, Team
+
+logger = logging.getLogger('awx.sso.common')
+
+
+def get_orgs_by_ids():
+    existing_orgs = {}
+    for (org_id, org_name) in Organization.objects.all().values_list('id', 'name'):
+        existing_orgs[org_name] = org_id
+    return existing_orgs
+
+
+def reconcile_users_org_team_mappings(user, desired_org_states, desired_team_states, source):
+    #
+    # Arguments:
+    #   user - a user object
+    #   desired_org_states: { '<org_name>': { '<role>': <boolean> or None } }
+    #   desired_team_states: { '<org_name>': { '<team name>': { '<role>': <boolean> or None } } }
+    #   source - a text label indicating the "authentication adapter" for debug messages
+    #
+    # This function will load the users existing roles and then based on the desired states modify the users roles
+    #    True indicates the user needs to be a member of the role
+    #    False indicates the user should not be a member of the role
+    #    None means this function should not change the users membership of a role
+    #
+
+    content_types = []
+    reconcile_items = []
+    if desired_org_states:
+        content_types.append(ContentType.objects.get_for_model(Organization))
+        reconcile_items.append(('organization', desired_org_states))
+    if desired_team_states:
+        content_types.append(ContentType.objects.get_for_model(Team))
+        reconcile_items.append(('team', desired_team_states))
+
+    if not content_types:
+        # If both desired states were empty we can simply return because there is nothing to reconcile
+        return
+
+    # users_roles is a flat set of IDs
+    users_roles = set(user.roles.filter(content_type__in=content_types).values_list('pk', flat=True))
+
+    for object_type, desired_states in reconcile_items:
+        roles = []
+        # Get a set of named tuples for the org/team name plus all of the roles we got above
+        if object_type == 'organization':
+            for sub_dict in desired_states.values():
+                for role_name in sub_dict:
+                    if sub_dict[role_name] is None:
+                        continue
+                    if role_name not in roles:
+                        roles.append(role_name)
+            model_roles = Organization.objects.filter(name__in=desired_states.keys()).values_list('name', *roles, named=True)
+        else:
+            team_names = []
+            for teams_dict in desired_states.values():
+                team_names.extend(teams_dict.keys())
+                for sub_dict in teams_dict.values():
+                    for role_name in sub_dict:
+                        if sub_dict[role_name] is None:
+                            continue
+                        if role_name not in roles:
+                            roles.append(role_name)
+            model_roles = Team.objects.filter(name__in=team_names).values_list('name', 'organization__name', *roles, named=True)
+
+        for row in model_roles:
+            for role_name in roles:
+                if object_type == 'organization':
+                    desired_state = desired_states.get(row.name, {})
+                else:
+                    desired_state = desired_states.get(row.organization__name, {}).get(row.name, {})
+
+                if desired_state.get(role_name, None) is None:
+                    # The mapping was not defined for this [org/team]/role so we can just pass
+                    continue
+
+                # If somehow the auth adapter knows about an items role but that role is not defined in the DB we are going to print a pretty error
+                # This is your classic safety net that we should never hit; but here you are reading this comment... good luck and Godspeed.
+                role_id = getattr(row, role_name, None)
+                if role_id is None:
+                    logger.error("{} adapter wanted to manage role {} of {} {} but that role is not defined".format(source, role_name, object_type, row.name))
+                    continue
+
+                if desired_state[role_name]:
+                    # The desired state was the user mapped into the object_type, if the user was not mapped in map them in
+                    if role_id not in users_roles:
+                        logger.debug("{} adapter adding user {} to {} {} as {}".format(source, user.username, object_type, row.name, role_name))
+                        user.roles.add(role_id)
+                else:
+                    # The desired state was the user was not mapped into the org, if the user has the permission remove it
+                    if role_id in users_roles:
+                        logger.debug("{} adapter removing user {} permission of {} from {} {}".format(source, user.username, role_name, object_type, row.name))
+                        user.roles.remove(role_id)
+
+
+def create_org_and_teams(org_list, team_map, adapter, can_create=True):
+    #
+    # org_list is a set of organization names
+    # team_map is a dict of {<team_name>: <org name>}
+    #
+    # Move this junk into save of the settings for performance later, there is no need to do that here
+    #    with maybe the exception of someone defining this in settings before the server is started?
+    # ==============================================================================================================
+
+    if not can_create:
+        logger.debug(f"Adapter {adapter} is not allowed to create orgs/teams")
+        return
+
+    # Get all of the IDs and names of orgs in the DB and create any new org defined in LDAP that does not exist in the DB
+    existing_orgs = get_orgs_by_ids()
+
+    # Parse through orgs and teams provided and create a list of unique items we care about creating
+    all_orgs = list(set(org_list))
+    all_teams = []
+    for team_name in team_map:
+        org_name = team_map[team_name]
+        if org_name:
+            if org_name not in all_orgs:
+                all_orgs.append(org_name)
+            # We don't have to test if this is in all_teams because team_map is already a hash
+            all_teams.append(team_name)
+        else:
+            # The UI should prevent this condition so this is just a double check to prevent a stack trace....
+            #  although the rest of the login process might stack later on
+            logger.error("{} adapter is attempting to create a team {} but it does not have an org".format(adapter, team_name))
+
+    for org_name in all_orgs:
+        if org_name and org_name not in existing_orgs:
+            logger.info("{} adapter is creating org {}".format(adapter, org_name))
+            try:
+                new_org = get_or_create_org_with_default_galaxy_cred(name=org_name)
+            except IntegrityError:
+                # Another thread must have created this org before we did so now we need to get it
+                new_org = get_or_create_org_with_default_galaxy_cred(name=org_name)
+            # Add the org name to the existing orgs since we created it and we may need it to build the teams below
+            existing_orgs[org_name] = new_org.id
+
+    # Do the same for teams
+    existing_team_names = list(Team.objects.all().values_list('name', flat=True))
+    for team_name in all_teams:
+        if team_name not in existing_team_names:
+            logger.info("{} adapter is creating team {} in org {}".format(adapter, team_name, team_map[team_name]))
+            try:
+                Team.objects.create(name=team_name, organization_id=existing_orgs[team_map[team_name]])
+            except IntegrityError:
+                # If another process got here before us that is ok because we don't need the ID from this team or anything
+                pass
+    # End move some day
+    # ==============================================================================================================
+
+
+def get_or_create_org_with_default_galaxy_cred(**kwargs):
+    from awx.main.models import Organization, Credential
+
+    (org, org_created) = Organization.objects.get_or_create(**kwargs)
+    if org_created:
+        logger.debug("Created org {} (id {}) from {}".format(org.name, org.id, kwargs))
+        public_galaxy_credential = Credential.objects.filter(managed=True, name='Ansible Galaxy').first()
+        if public_galaxy_credential is not None:
+            org.galaxy_credentials.add(public_galaxy_credential)
+            logger.debug("Added default Ansible Galaxy credential to org")
+        else:
+            logger.debug("Could not find default Ansible Galaxy credential to add to org")
+    return org

awx/sso/saml_pipeline.py

@@ -5,59 +5,43 @@
 import re
 import logging
 
-
-# Python Social Auth
-from social_core.exceptions import AuthException
-
 # Django
-from django.core.exceptions import ObjectDoesNotExist
-from django.utils.translation import gettext_lazy as _
-from django.db.models import Q
+from django.conf import settings
+
+from awx.main.models import Team
+from awx.sso.common import create_org_and_teams, reconcile_users_org_team_mappings, get_orgs_by_ids
+
+logger = logging.getLogger('awx.sso.saml_pipeline')
 
 
-logger = logging.getLogger('awx.sso.pipeline')
-
-
-class AuthNotFound(AuthException):
-    def __init__(self, backend, email_or_uid, *args, **kwargs):
-        self.email_or_uid = email_or_uid
-        super(AuthNotFound, self).__init__(backend, *args, **kwargs)
-
-    def __str__(self):
-        return _('An account cannot be found for {0}').format(self.email_or_uid)
-
-
-class AuthInactive(AuthException):
-    def __str__(self):
-        return _('Your account is inactive')
-
-
-def check_user_found_or_created(backend, details, user=None, *args, **kwargs):
+def populate_user(backend, details, user=None, *args, **kwargs):
     if not user:
-        email_or_uid = details.get('email') or kwargs.get('email') or kwargs.get('uid') or '???'
-        raise AuthNotFound(backend, email_or_uid)
+        return
+
+    # Build the in-memory settings for how this user should be modeled
+    desired_org_state = {}
+    desired_team_state = {}
+    orgs_to_create = []
+    teams_to_create = {}
+    _update_user_orgs_by_saml_attr(backend, desired_org_state, orgs_to_create, **kwargs)
+    _update_user_teams_by_saml_attr(desired_team_state, teams_to_create, **kwargs)
+    _update_user_orgs(backend, desired_org_state, orgs_to_create, user)
+    _update_user_teams(backend, desired_team_state, teams_to_create, user)
+
+    # If the SAML adapter is allowed to create objects, lets do that first
+    create_org_and_teams(orgs_to_create, teams_to_create, 'SAML', settings.SAML_AUTO_CREATE_OBJECTS)
+
+    # Finally reconcile the user
+    reconcile_users_org_team_mappings(user, desired_org_state, desired_team_state, 'SAML')
 
 
-def set_is_active_for_new_user(strategy, details, user=None, *args, **kwargs):
-    if kwargs.get('is_new', False):
-        details['is_active'] = True
-        return {'details': details}
-
-
-def prevent_inactive_login(backend, details, user=None, *args, **kwargs):
-    if user and not user.is_active:
-        raise AuthInactive(backend)
-
-
-def _update_m2m_from_expression(user, related, expr, remove=True):
+def _update_m2m_from_expression(user, expr, remove=True):
     """
     Helper function to update m2m relationship based on user matching one or
     more expressions.
     """
     should_add = False
-    if expr is None:
-        return
-    elif not expr:
+    if expr is None or not expr:
         pass
     elif expr is True:
         should_add = True
@@ -72,35 +56,89 @@ def _update_m2m_from_expression(user, related, expr, remove=True):
                 if ex.match(user.username) or ex.match(user.email):
                     should_add = True
     if should_add:
-        related.add(user)
+        return True
     elif remove:
-        related.remove(user)
-
-
-def get_or_create_with_default_galaxy_cred(**kwargs):
-    from awx.main.models import Organization, Credential
-
-    (org, org_created) = Organization.objects.get_or_create(**kwargs)
-    if org_created:
-        logger.debug("Created org {} (id {}) from {}".format(org.name, org.id, kwargs))
-        public_galaxy_credential = Credential.objects.filter(managed=True, name='Ansible Galaxy').first()
-        if public_galaxy_credential is not None:
-            org.galaxy_credentials.add(public_galaxy_credential)
-            logger.debug("Added default Ansible Galaxy credential to org")
+        return False
     else:
-            logger.debug("Could not find default Ansible Galaxy credential to add to org")
-    return org
+        return None
 
 
-def _update_org_from_attr(user, related, attr, remove, remove_admins, remove_auditors, backend):
-    from awx.main.models import Organization
-    from django.conf import settings
+def _update_user_orgs(backend, desired_org_state, orgs_to_create, user=None):
+    """
+    Update organization memberships for the given user based on mapping rules
+    defined in settings.
+    """
+    org_map = backend.setting('ORGANIZATION_MAP') or {}
+    for org_name, org_opts in org_map.items():
+        organization_alias = org_opts.get('organization_alias')
+        if organization_alias:
+            organization_name = organization_alias
+        else:
+            organization_name = org_name
+        if organization_name not in orgs_to_create:
+            orgs_to_create.append(organization_name)
 
-    org_ids = []
+        remove = bool(org_opts.get('remove', True))
 
-    for org_name in attr:
-        try:
-            if settings.SAML_AUTO_CREATE_OBJECTS:
+        if organization_name not in desired_org_state:
+            desired_org_state[organization_name] = {}
+
+        for role_name, user_type in (('admin_role', 'admins'), ('member_role', 'users'), ('auditor_role', 'auditors')):
+            is_member_expression = org_opts.get(user_type, None)
+            remove_members = bool(org_opts.get('remove_{}'.format(user_type), remove))
+            has_role = _update_m2m_from_expression(user, is_member_expression, remove_members)
+            desired_org_state[organization_name][role_name] = has_role
+
+
+def _update_user_teams(backend, desired_team_state, teams_to_create, user=None):
+    """
+    Update team memberships for the given user based on mapping rules defined
+    in settings.
+    """
+
+    team_map = backend.setting('TEAM_MAP') or {}
+    for team_name, team_opts in team_map.items():
+        # Get or create the org to update.
+        if 'organization' not in team_opts:
+            continue
+        teams_to_create[team_name] = team_opts['organization']
+        users_expr = team_opts.get('users', None)
+        remove = bool(team_opts.get('remove', True))
+        add_or_remove = _update_m2m_from_expression(user, users_expr, remove)
+        if add_or_remove is not None:
+            org_name = team_opts['organization']
+            if org_name not in desired_team_state:
+                desired_team_state[org_name] = {}
+            desired_team_state[org_name][team_name] = {'member_role': add_or_remove}
+
+
+def _update_user_orgs_by_saml_attr(backend, desired_org_state, orgs_to_create, **kwargs):
+    org_map = settings.SOCIAL_AUTH_SAML_ORGANIZATION_ATTR
+    roles_and_flags = (
+        ('member_role', 'remove', 'saml_attr'),
+        ('admin_role', 'remove_admins', 'saml_admin_attr'),
+        ('auditor_role', 'remove_auditors', 'saml_auditor_attr'),
+    )
+
+    # If the remove_flag was present we need to load all of the orgs and remove the user from the role
+    all_orgs = None
+    for role, remove_flag, _ in roles_and_flags:
+        remove = bool(org_map.get(remove_flag, True))
+        if remove:
+            # Only get the all orgs once, and only if needed
+            if all_orgs is None:
+                all_orgs = get_orgs_by_ids()
+            for org_name in all_orgs.keys():
+                if org_name not in desired_org_state:
+                    desired_org_state[org_name] = {}
+                desired_org_state[org_name][role] = False
+
+    # Now we can add the user as a member/admin/auditor for any orgs they have specified
+    for role, _, attr_flag in roles_and_flags:
+        if org_map.get(attr_flag) is None:
+            continue
+        saml_attr_values = kwargs.get('response', {}).get('attributes', {}).get(org_map.get(attr_flag), [])
+        for org_name in saml_attr_values:
             try:
                 organization_alias = backend.setting('ORGANIZATION_MAP').get(org_name).get('organization_alias')
                 if organization_alias is not None:
@@ -109,112 +147,36 @@ def _update_org_from_attr(user, related, attr, remove, remove_admins, remove_aud
                     organization_name = org_name
             except Exception:
                 organization_name = org_name
-                org = get_or_create_with_default_galaxy_cred(name=organization_name)
-            else:
-                org = Organization.objects.get(name=org_name)
-        except ObjectDoesNotExist:
-            continue
-
-        org_ids.append(org.id)
-        getattr(org, related).members.add(user)
-
-    if remove:
-        [o.member_role.members.remove(user) for o in Organization.objects.filter(Q(member_role__members=user) & ~Q(id__in=org_ids))]
-
-    if remove_admins:
-        [o.admin_role.members.remove(user) for o in Organization.objects.filter(Q(admin_role__members=user) & ~Q(id__in=org_ids))]
-
-    if remove_auditors:
-        [o.auditor_role.members.remove(user) for o in Organization.objects.filter(Q(auditor_role__members=user) & ~Q(id__in=org_ids))]
+            if organization_name not in orgs_to_create:
+                orgs_to_create.append(organization_name)
+            if organization_name not in desired_org_state:
+                desired_org_state[organization_name] = {}
+            desired_org_state[organization_name][role] = True
 
 
-def update_user_orgs(backend, details, user=None, *args, **kwargs):
-    """
-    Update organization memberships for the given user based on mapping rules
-    defined in settings.
-    """
-    if not user:
-        return
-
-    org_map = backend.setting('ORGANIZATION_MAP') or {}
-    for org_name, org_opts in org_map.items():
-        organization_alias = org_opts.get('organization_alias')
-        if organization_alias:
-            organization_name = organization_alias
-        else:
-            organization_name = org_name
-        org = get_or_create_with_default_galaxy_cred(name=organization_name)
-
-        # Update org admins from expression(s).
-        remove = bool(org_opts.get('remove', True))
-        admins_expr = org_opts.get('admins', None)
-        remove_admins = bool(org_opts.get('remove_admins', remove))
-        _update_m2m_from_expression(user, org.admin_role.members, admins_expr, remove_admins)
-
-        # Update org users from expression(s).
-        users_expr = org_opts.get('users', None)
-        remove_users = bool(org_opts.get('remove_users', remove))
-        _update_m2m_from_expression(user, org.member_role.members, users_expr, remove_users)
-
-
-def update_user_teams(backend, details, user=None, *args, **kwargs):
-    """
-    Update team memberships for the given user based on mapping rules defined
-    in settings.
-    """
-    if not user:
-        return
-    from awx.main.models import Team
-
-    team_map = backend.setting('TEAM_MAP') or {}
-    for team_name, team_opts in team_map.items():
-        # Get or create the org to update.
-        if 'organization' not in team_opts:
-            continue
-        org = get_or_create_with_default_galaxy_cred(name=team_opts['organization'])
-
-        # Update team members from expression(s).
-        team = Team.objects.get_or_create(name=team_name, organization=org)[0]
-        users_expr = team_opts.get('users', None)
-        remove = bool(team_opts.get('remove', True))
-        _update_m2m_from_expression(user, team.member_role.members, users_expr, remove)
-
-
-def update_user_orgs_by_saml_attr(backend, details, user=None, *args, **kwargs):
-    if not user:
-        return
-    from django.conf import settings
-
-    org_map = settings.SOCIAL_AUTH_SAML_ORGANIZATION_ATTR
-    if org_map.get('saml_attr') is None and org_map.get('saml_admin_attr') is None and org_map.get('saml_auditor_attr') is None:
-        return
-
-    remove = bool(org_map.get('remove', True))
-    remove_admins = bool(org_map.get('remove_admins', True))
-    remove_auditors = bool(org_map.get('remove_auditors', True))
-
-    attr_values = kwargs.get('response', {}).get('attributes', {}).get(org_map.get('saml_attr'), [])
-    attr_admin_values = kwargs.get('response', {}).get('attributes', {}).get(org_map.get('saml_admin_attr'), [])
-    attr_auditor_values = kwargs.get('response', {}).get('attributes', {}).get(org_map.get('saml_auditor_attr'), [])
-
-    _update_org_from_attr(user, "member_role", attr_values, remove, False, False, backend)
-    _update_org_from_attr(user, "admin_role", attr_admin_values, False, remove_admins, False, backend)
-    _update_org_from_attr(user, "auditor_role", attr_auditor_values, False, False, remove_auditors, backend)
-
-
-def update_user_teams_by_saml_attr(backend, details, user=None, *args, **kwargs):
-    if not user:
-        return
-    from awx.main.models import Organization, Team
-    from django.conf import settings
-
+def _update_user_teams_by_saml_attr(desired_team_state, teams_to_create, **kwargs):
+    #
+    # Map users into organizations based on SOCIAL_AUTH_SAML_TEAM_ATTR setting
+    #
     team_map = settings.SOCIAL_AUTH_SAML_TEAM_ATTR
     if team_map.get('saml_attr') is None:
         return
 
+    all_teams = None
+    # The role and flag is hard coded here but intended to be flexible in case we ever wanted to add another team type
+    for role, remove_flag in [('member_role', 'remove')]:
+        remove = bool(team_map.get(remove_flag, True))
+        if remove:
+            # Only get the all orgs once, and only if needed
+            if all_teams is None:
+                all_teams = Team.objects.all().values_list('name', 'organization__name')
+            for (team_name, organization_name) in all_teams:
+                if organization_name not in desired_team_state:
+                    desired_team_state[organization_name] = {}
+                desired_team_state[organization_name][team_name] = {role: False}
+
     saml_team_names = set(kwargs.get('response', {}).get('attributes', {}).get(team_map['saml_attr'], []))
 
-    team_ids = []
     for team_name_map in team_map.get('team_org_map', []):
         team_name = team_name_map.get('team', None)
         team_alias = team_name_map.get('team_alias', None)
@@ -225,29 +187,17 @@ def update_user_teams_by_saml_attr(backend, details, user=None, *args, **kwargs)
                 logger.error("organization name invalid for team {}".format(team_name))
                 continue
 
-            try:
-                if settings.SAML_AUTO_CREATE_OBJECTS:
-                    org = get_or_create_with_default_galaxy_cred(name=organization_name)
-                else:
-                    org = Organization.objects.get(name=organization_name)
-            except ObjectDoesNotExist:
-                continue
-
             if team_alias:
                 team_name = team_alias
-            try:
-                if settings.SAML_AUTO_CREATE_OBJECTS:
-                    team = Team.objects.get_or_create(name=team_name, organization=org)[0]
+
+            teams_to_create[team_name] = organization_name
+            user_is_member_of_team = True
         else:
-                    team = Team.objects.get(name=team_name, organization=org)
-            except ObjectDoesNotExist:
-                continue
+            user_is_member_of_team = False
 
-            team_ids.append(team.id)
-            team.member_role.members.add(user)
-
-    if team_map.get('remove', True):
-        [t.member_role.members.remove(user) for t in Team.objects.filter(Q(member_role__members=user) & ~Q(id__in=team_ids))]
+        if organization_name not in desired_team_state:
+            desired_team_state[organization_name] = {}
+        desired_team_state[organization_name][team_name] = {'member_role': user_is_member_of_team}
 
 
 def _get_matches(list1, list2):
@@ -329,11 +279,6 @@ def _check_flag(user, flag, attributes, user_flags_settings):
 
 
 def update_user_flags(backend, details, user=None, *args, **kwargs):
-    if not user:
-        return
-
-    from django.conf import settings
-
     user_flags_settings = settings.SOCIAL_AUTH_SAML_USER_FLAGS_BY_ATTR
 
     attributes = kwargs.get('response', {}).get('attributes', {})

awx/sso/social_base_pipeline.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved.
+
+# Python Social Auth
+from social_core.exceptions import AuthException
+
+# Django
+from django.utils.translation import gettext_lazy as _
+
+
+class AuthNotFound(AuthException):
+    def __init__(self, backend, email_or_uid, *args, **kwargs):
+        self.email_or_uid = email_or_uid
+        super(AuthNotFound, self).__init__(backend, *args, **kwargs)
+
+    def __str__(self):
+        return _('An account cannot be found for {0}').format(self.email_or_uid)
+
+
+class AuthInactive(AuthException):
+    def __str__(self):
+        return _('Your account is inactive')
+
+
+def check_user_found_or_created(backend, details, user=None, *args, **kwargs):
+    if not user:
+        email_or_uid = details.get('email') or kwargs.get('email') or kwargs.get('uid') or '???'
+        raise AuthNotFound(backend, email_or_uid)
+
+
+def set_is_active_for_new_user(strategy, details, user=None, *args, **kwargs):
+    if kwargs.get('is_new', False):
+        details['is_active'] = True
+        return {'details': details}
+
+
+def prevent_inactive_login(backend, details, user=None, *args, **kwargs):
+    if user and not user.is_active:
+        raise AuthInactive(backend)

awx/sso/social_pipeline.py

@@ -0,0 +1,90 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved.
+
+# Python
+import re
+import logging
+
+from awx.sso.common import get_or_create_org_with_default_galaxy_cred
+
+logger = logging.getLogger('awx.sso.social_pipeline')
+
+
+def _update_m2m_from_expression(user, related, expr, remove=True):
+    """
+    Helper function to update m2m relationship based on user matching one or
+    more expressions.
+    """
+    should_add = False
+    if expr is None:
+        return
+    elif not expr:
+        pass
+    elif expr is True:
+        should_add = True
+    else:
+        if isinstance(expr, (str, type(re.compile('')))):
+            expr = [expr]
+        for ex in expr:
+            if isinstance(ex, str):
+                if user.username == ex or user.email == ex:
+                    should_add = True
+            elif isinstance(ex, type(re.compile(''))):
+                if ex.match(user.username) or ex.match(user.email):
+                    should_add = True
+    if should_add:
+        related.add(user)
+    elif remove:
+        related.remove(user)
+
+
+def update_user_orgs(backend, details, user=None, *args, **kwargs):
+    """
+    Update organization memberships for the given user based on mapping rules
+    defined in settings.
+    """
+    if not user:
+        return
+
+    org_map = backend.setting('ORGANIZATION_MAP') or {}
+    for org_name, org_opts in org_map.items():
+        organization_alias = org_opts.get('organization_alias')
+        if organization_alias:
+            organization_name = organization_alias
+        else:
+            organization_name = org_name
+        org = get_or_create_org_with_default_galaxy_cred(name=organization_name)
+
+        # Update org admins from expression(s).
+        remove = bool(org_opts.get('remove', True))
+        admins_expr = org_opts.get('admins', None)
+        remove_admins = bool(org_opts.get('remove_admins', remove))
+        _update_m2m_from_expression(user, org.admin_role.members, admins_expr, remove_admins)
+
+        # Update org users from expression(s).
+        users_expr = org_opts.get('users', None)
+        remove_users = bool(org_opts.get('remove_users', remove))
+        _update_m2m_from_expression(user, org.member_role.members, users_expr, remove_users)
+
+
+def update_user_teams(backend, details, user=None, *args, **kwargs):
+    """
+    Update team memberships for the given user based on mapping rules defined
+    in settings.
+    """
+    if not user:
+        return
+    from awx.main.models import Team
+
+    team_map = backend.setting('TEAM_MAP') or {}
+    for team_name, team_opts in team_map.items():
+        # Get or create the org to update.
+        if 'organization' not in team_opts:
+            continue
+        org = get_or_create_org_with_default_galaxy_cred(name=team_opts['organization'])
+
+        # Update team members from expression(s).
+        team = Team.objects.get_or_create(name=team_name, organization=org)[0]
+        users_expr = team_opts.get('users', None)
+        remove = bool(team_opts.get('remove', True))
+        _update_m2m_from_expression(user, team.member_role.members, users_expr, remove)

awx/sso/tests/functional/test_common.py

@@ -0,0 +1,280 @@
+import pytest
+from collections import Counter
+from django.core.exceptions import FieldError
+from django.utils.timezone import now
+
+from awx.main.models import Credential, CredentialType, Organization, Team, User
+from awx.sso.common import get_orgs_by_ids, reconcile_users_org_team_mappings, create_org_and_teams, get_or_create_org_with_default_galaxy_cred
+
+
+@pytest.mark.django_db
+class TestCommonFunctions:
+    @pytest.fixture
+    def orgs(self):
+        o1 = Organization.objects.create(name='Default1')
+        o2 = Organization.objects.create(name='Default2')
+        o3 = Organization.objects.create(name='Default3')
+        return (o1, o2, o3)
+
+    @pytest.fixture
+    def galaxy_credential(self):
+        galaxy_type = CredentialType.objects.create(kind='galaxy')
+        cred = Credential(
+            created=now(), modified=now(), name='Ansible Galaxy', managed=True, credential_type=galaxy_type, inputs={'url': 'https://galaxy.ansible.com/'}
+        )
+        cred.save()
+
+    def test_get_orgs_by_ids(self, orgs):
+        orgs_and_ids = get_orgs_by_ids()
+        o1, o2, o3 = orgs
+        assert Counter(orgs_and_ids.keys()) == Counter([o1.name, o2.name, o3.name])
+        assert Counter(orgs_and_ids.values()) == Counter([o1.id, o2.id, o3.id])
+
+    def test_reconcile_users_org_team_mappings(self):
+        # Create objects for us to play with
+        user = User.objects.create(username='user1@foo.com', last_name='foo', first_name='bar', email='user1@foo.com', is_active=True)
+        org1 = Organization.objects.create(name='Default1')
+        org2 = Organization.objects.create(name='Default2')
+        team1 = Team.objects.create(name='Team1', organization=org1)
+        team2 = Team.objects.create(name='Team1', organization=org2)
+
+        # Try adding nothing
+        reconcile_users_org_team_mappings(user, {}, {}, 'Nada')
+        assert list(user.roles.all()) == []
+
+        # Add a user to an org that does not exist (should have no affect)
+        reconcile_users_org_team_mappings(
+            user,
+            {
+                'junk': {'member_role': True},
+            },
+            {},
+            'Nada',
+        )
+        assert list(user.roles.all()) == []
+
+        # Remove a user to an org that does not exist (should have no affect)
+        reconcile_users_org_team_mappings(
+            user,
+            {
+                'junk': {'member_role': False},
+            },
+            {},
+            'Nada',
+        )
+        assert list(user.roles.all()) == []
+
+        # Add the user to the orgs
+        reconcile_users_org_team_mappings(user, {org1.name: {'member_role': True}, org2.name: {'member_role': True}}, {}, 'Nada')
+        assert len(user.roles.all()) == 2
+        assert user in org1.member_role
+        assert user in org2.member_role
+
+        # Remove the user from the orgs
+        reconcile_users_org_team_mappings(user, {org1.name: {'member_role': False}, org2.name: {'member_role': False}}, {}, 'Nada')
+        assert list(user.roles.all()) == []
+        assert user not in org1.member_role
+        assert user not in org2.member_role
+
+        # Remove the user from the orgs (again, should have no affect)
+        reconcile_users_org_team_mappings(user, {org1.name: {'member_role': False}, org2.name: {'member_role': False}}, {}, 'Nada')
+        assert list(user.roles.all()) == []
+        assert user not in org1.member_role
+        assert user not in org2.member_role
+
+        # Add a user back to the member role
+        reconcile_users_org_team_mappings(
+            user,
+            {
+                org1.name: {
+                    'member_role': True,
+                },
+            },
+            {},
+            'Nada',
+        )
+        users_roles = set(user.roles.values_list('pk', flat=True))
+        assert len(users_roles) == 1
+        assert user in org1.member_role
+
+        # Add the user to additional roles
+        reconcile_users_org_team_mappings(
+            user,
+            {
+                org1.name: {'admin_role': True, 'auditor_role': True},
+            },
+            {},
+            'Nada',
+        )
+        assert len(user.roles.all()) == 3
+        assert user in org1.member_role
+        assert user in org1.admin_role
+        assert user in org1.auditor_role
+
+        # Add a user to a non-existent role (results in FieldError exception)
+        with pytest.raises(FieldError):
+            reconcile_users_org_team_mappings(
+                user,
+                {
+                    org1.name: {
+                        'dne_role': True,
+                    },
+                },
+                {},
+                'Nada',
+            )
+
+        # Try adding a user to a role that should not exist on an org (technically this works at this time)
+        reconcile_users_org_team_mappings(
+            user,
+            {
+                org1.name: {
+                    'read_role_id': True,
+                },
+            },
+            {},
+            'Nada',
+        )
+        assert len(user.roles.all()) == 4
+        assert user in org1.member_role
+        assert user in org1.admin_role
+        assert user in org1.auditor_role
+
+        # Remove all of the org perms to test team perms
+        reconcile_users_org_team_mappings(
+            user,
+            {
+                org1.name: {
+                    'read_role_id': False,
+                    'member_role': False,
+                    'admin_role': False,
+                    'auditor_role': False,
+                },
+            },
+            {},
+            'Nada',
+        )
+        assert list(user.roles.all()) == []
+
+        # Add the user as a member to one of the teams
+        reconcile_users_org_team_mappings(user, {}, {org1.name: {team1.name: {'member_role': True}}}, 'Nada')
+        assert len(user.roles.all()) == 1
+        assert user in team1.member_role
+        # Validate that the user did not become a member of a team with the same name in a different org
+        assert user not in team2.member_role
+
+        # Remove the user from the team
+        reconcile_users_org_team_mappings(user, {}, {org1.name: {team1.name: {'member_role': False}}}, 'Nada')
+        assert list(user.roles.all()) == []
+        assert user not in team1.member_role
+
+        # Remove the user from the team again
+        reconcile_users_org_team_mappings(user, {}, {org1.name: {team1.name: {'member_role': False}}}, 'Nada')
+        assert list(user.roles.all()) == []
+
+        # Add the user to a team that does not exist (should have no affect)
+        reconcile_users_org_team_mappings(user, {}, {org1.name: {'junk': {'member_role': True}}}, 'Nada')
+        assert list(user.roles.all()) == []
+
+        # Remove the user from a team that does not exist (should have no affect)
+        reconcile_users_org_team_mappings(user, {}, {org1.name: {'junk': {'member_role': False}}}, 'Nada')
+        assert list(user.roles.all()) == []
+
+        # Test a None setting
+        reconcile_users_org_team_mappings(user, {}, {org1.name: {'junk': {'member_role': None}}}, 'Nada')
+        assert list(user.roles.all()) == []
+
+        # Add the user multiple teams in different orgs
+        reconcile_users_org_team_mappings(user, {}, {org1.name: {team1.name: {'member_role': True}}, org2.name: {team2.name: {'member_role': True}}}, 'Nada')
+        assert len(user.roles.all()) == 2
+        assert user in team1.member_role
+        assert user in team2.member_role
+
+        # Remove the user from just one of the teams
+        reconcile_users_org_team_mappings(user, {}, {org2.name: {team2.name: {'member_role': False}}}, 'Nada')
+        assert len(user.roles.all()) == 1
+        assert user in team1.member_role
+        assert user not in team2.member_role
+
+    @pytest.mark.parametrize(
+        "org_list, team_map, can_create, org_count, team_count",
+        [
+            # In this case we will only pass in organizations
+            (
+                ["org1", "org2"],
+                {},
+                True,
+                2,
+                0,
+            ),
+            # In this case we will only pass in teams but the orgs will be created from the teams
+            (
+                [],
+                {"team1": "org1", "team2": "org2"},
+                True,
+                2,
+                2,
+            ),
+            # In this case we will reuse an org
+            (
+                ["org1"],
+                {"team1": "org1", "team2": "org1"},
+                True,
+                1,
+                2,
+            ),
+            # In this case we have a combination of orgs, orgs reused and an org created by a team
+            (
+                ["org1", "org2", "org3"],
+                {"team1": "org1", "team2": "org4"},
+                True,
+                4,
+                2,
+            ),
+            # In this case we will test a case that the UI should prevent and have a team with no Org
+            #   This should create org1/2 but only team1
+            (
+                ["org1"],
+                {"team1": "org2", "team2": None},
+                True,
+                2,
+                1,
+            ),
+            # Block any creation with the can_create flag
+            (
+                ["org1"],
+                {"team1": "org2", "team2": None},
+                False,
+                0,
+                0,
+            ),
+        ],
+    )
+    def test_create_org_and_teams(self, galaxy_credential, org_list, team_map, can_create, org_count, team_count):
+        create_org_and_teams(org_list, team_map, 'py.test', can_create=can_create)
+        assert Organization.objects.count() == org_count
+        assert Team.objects.count() == team_count
+
+    def test_get_or_create_org_with_default_galaxy_cred_add_galaxy_cred(self, galaxy_credential):
+        # If this method creates the org it should get the default galaxy credential
+        num_orgs = 4
+        for number in range(1, (num_orgs + 1)):
+            get_or_create_org_with_default_galaxy_cred(name=f"Default {number}")
+
+        assert Organization.objects.count() == 4
+
+        for o in Organization.objects.all():
+            assert o.galaxy_credentials.count() == 1
+            assert o.galaxy_credentials.first().name == 'Ansible Galaxy'
+
+    def test_get_or_create_org_with_default_galaxy_cred_no_galaxy_cred(self, galaxy_credential):
+        # If the org is pre-created, we should not add the galaxy_credential
+        num_orgs = 4
+        for number in range(1, (num_orgs + 1)):
+            Organization.objects.create(name=f"Default {number}")
+            get_or_create_org_with_default_galaxy_cred(name=f"Default {number}")
+
+        assert Organization.objects.count() == 4
+
+        for o in Organization.objects.all():
+            assert o.galaxy_credentials.count() == 0

awx/sso/tests/functional/test_pipeline.py

@@ -1,566 +0,0 @@
-import pytest
-import re
-from unittest import mock
-
-from django.utils.timezone import now
-
-from awx.conf.registry import settings_registry
-from awx.sso.pipeline import update_user_orgs, update_user_teams, update_user_orgs_by_saml_attr, update_user_teams_by_saml_attr, _check_flag
-from awx.main.models import User, Team, Organization, Credential, CredentialType
-
-
-@pytest.fixture
-def galaxy_credential():
-    galaxy_type = CredentialType.objects.create(kind='galaxy')
-    cred = Credential(
-        created=now(), modified=now(), name='Ansible Galaxy', managed=True, credential_type=galaxy_type, inputs={'url': 'https://galaxy.ansible.com/'}
-    )
-    cred.save()
-
-
-@pytest.fixture
-def users():
-    u1 = User.objects.create(username='user1@foo.com', last_name='foo', first_name='bar', email='user1@foo.com')
-    u2 = User.objects.create(username='user2@foo.com', last_name='foo', first_name='bar', email='user2@foo.com')
-    u3 = User.objects.create(username='user3@foo.com', last_name='foo', first_name='bar', email='user3@foo.com')
-    return (u1, u2, u3)
-
-
-@pytest.mark.django_db
-class TestSAMLMap:
-    @pytest.fixture
-    def backend(self):
-        class Backend:
-            s = {
-                'ORGANIZATION_MAP': {
-                    'Default': {
-                        'remove': True,
-                        'admins': 'foobar',
-                        'remove_admins': True,
-                        'users': 'foo',
-                        'remove_users': True,
-                        'organization_alias': '',
-                    }
-                },
-                'TEAM_MAP': {'Blue': {'organization': 'Default', 'remove': True, 'users': ''}, 'Red': {'organization': 'Default', 'remove': True, 'users': ''}},
-            }
-
-            def setting(self, key):
-                return self.s[key]
-
-        return Backend()
-
-    @pytest.fixture
-    def org(self):
-        return Organization.objects.create(name="Default")
-
-    def test_update_user_orgs(self, org, backend, users, galaxy_credential):
-        u1, u2, u3 = users
-
-        # Test user membership logic with regular expressions
-        backend.setting('ORGANIZATION_MAP')['Default']['admins'] = re.compile('.*')
-        backend.setting('ORGANIZATION_MAP')['Default']['users'] = re.compile('.*')
-
-        update_user_orgs(backend, None, u1)
-        update_user_orgs(backend, None, u2)
-        update_user_orgs(backend, None, u3)
-
-        assert org.admin_role.members.count() == 3
-        assert org.member_role.members.count() == 3
-
-        # Test remove feature enabled
-        backend.setting('ORGANIZATION_MAP')['Default']['admins'] = ''
-        backend.setting('ORGANIZATION_MAP')['Default']['users'] = ''
-        backend.setting('ORGANIZATION_MAP')['Default']['remove_admins'] = True
-        backend.setting('ORGANIZATION_MAP')['Default']['remove_users'] = True
-        update_user_orgs(backend, None, u1)
-
-        assert org.admin_role.members.count() == 2
-        assert org.member_role.members.count() == 2
-
-        # Test remove feature disabled
-        backend.setting('ORGANIZATION_MAP')['Default']['remove_admins'] = False
-        backend.setting('ORGANIZATION_MAP')['Default']['remove_users'] = False
-        update_user_orgs(backend, None, u2)
-
-        assert org.admin_role.members.count() == 2
-        assert org.member_role.members.count() == 2
-
-        # Test organization alias feature
-        backend.setting('ORGANIZATION_MAP')['Default']['organization_alias'] = 'Default_Alias'
-        update_user_orgs(backend, None, u1)
-        assert Organization.objects.get(name="Default_Alias") is not None
-
-        for o in Organization.objects.all():
-            if o.name == 'Default':
-                # The default org was already created and should not have a galaxy credential
-                assert o.galaxy_credentials.count() == 0
-            else:
-                # The Default_Alias was created by SAML and should get the galaxy credential
-                assert o.galaxy_credentials.count() == 1
-                assert o.galaxy_credentials.first().name == 'Ansible Galaxy'
-
-    def test_update_user_teams(self, backend, users, galaxy_credential):
-        u1, u2, u3 = users
-
-        # Test user membership logic with regular expressions
-        backend.setting('TEAM_MAP')['Blue']['users'] = re.compile('.*')
-        backend.setting('TEAM_MAP')['Red']['users'] = re.compile('.*')
-
-        update_user_teams(backend, None, u1)
-        update_user_teams(backend, None, u2)
-        update_user_teams(backend, None, u3)
-
-        assert Team.objects.get(name="Red").member_role.members.count() == 3
-        assert Team.objects.get(name="Blue").member_role.members.count() == 3
-
-        # Test remove feature enabled
-        backend.setting('TEAM_MAP')['Blue']['remove'] = True
-        backend.setting('TEAM_MAP')['Red']['remove'] = True
-        backend.setting('TEAM_MAP')['Blue']['users'] = ''
-        backend.setting('TEAM_MAP')['Red']['users'] = ''
-
-        update_user_teams(backend, None, u1)
-
-        assert Team.objects.get(name="Red").member_role.members.count() == 2
-        assert Team.objects.get(name="Blue").member_role.members.count() == 2
-
-        # Test remove feature disabled
-        backend.setting('TEAM_MAP')['Blue']['remove'] = False
-        backend.setting('TEAM_MAP')['Red']['remove'] = False
-
-        update_user_teams(backend, None, u2)
-
-        assert Team.objects.get(name="Red").member_role.members.count() == 2
-        assert Team.objects.get(name="Blue").member_role.members.count() == 2
-
-        for o in Organization.objects.all():
-            assert o.galaxy_credentials.count() == 1
-            assert o.galaxy_credentials.first().name == 'Ansible Galaxy'
-
-
-@pytest.mark.django_db
-class TestSAMLAttr:
-    @pytest.fixture
-    def kwargs(self):
-        return {
-            'username': u'cmeyers@redhat.com',
-            'uid': 'idp:cmeyers@redhat.com',
-            'request': {u'SAMLResponse': [], u'RelayState': [u'idp']},
-            'is_new': False,
-            'response': {
-                'session_index': '_0728f0e0-b766-0135-75fa-02842b07c044',
-                'idp_name': u'idp',
-                'attributes': {
-                    'memberOf': ['Default1', 'Default2'],
-                    'admins': ['Default3'],
-                    'auditors': ['Default4'],
-                    'groups': ['Blue', 'Red'],
-                    'User.email': ['cmeyers@redhat.com'],
-                    'User.LastName': ['Meyers'],
-                    'name_id': 'cmeyers@redhat.com',
-                    'User.FirstName': ['Chris'],
-                    'PersonImmutableID': [],
-                },
-            },
-            # 'social': <UserSocialAuth: cmeyers@redhat.com>,
-            'social': None,
-            # 'strategy': <awx.sso.strategies.django_strategy.AWXDjangoStrategy object at 0x8523a10>,
-            'strategy': None,
-            'new_association': False,
-        }
-
-    @pytest.fixture
-    def orgs(self):
-        o1 = Organization.objects.create(name='Default1')
-        o2 = Organization.objects.create(name='Default2')
-        o3 = Organization.objects.create(name='Default3')
-        return (o1, o2, o3)
-
-    @pytest.fixture
-    def mock_settings(self, request):
-        fixture_args = request.node.get_closest_marker('fixture_args')
-        if fixture_args and 'autocreate' in fixture_args.kwargs:
-            autocreate = fixture_args.kwargs['autocreate']
-        else:
-            autocreate = True
-
-        class MockSettings:
-            SAML_AUTO_CREATE_OBJECTS = autocreate
-            SOCIAL_AUTH_SAML_ORGANIZATION_ATTR = {
-                'saml_attr': 'memberOf',
-                'saml_admin_attr': 'admins',
-                'saml_auditor_attr': 'auditors',
-                'remove': True,
-                'remove_admins': True,
-            }
-            SOCIAL_AUTH_SAML_TEAM_ATTR = {
-                'saml_attr': 'groups',
-                'remove': True,
-                'team_org_map': [
-                    {'team': 'Blue', 'organization': 'Default1'},
-                    {'team': 'Blue', 'organization': 'Default2'},
-                    {'team': 'Blue', 'organization': 'Default3'},
-                    {'team': 'Red', 'organization': 'Default1'},
-                    {'team': 'Green', 'organization': 'Default1'},
-                    {'team': 'Green', 'organization': 'Default3'},
-                    {'team': 'Yellow', 'team_alias': 'Yellow_Alias', 'organization': 'Default4', 'organization_alias': 'Default4_Alias'},
-                ],
-            }
-
-        mock_settings_obj = MockSettings()
-        for key in settings_registry.get_registered_settings(category_slug='logging'):
-            value = settings_registry.get_setting_field(key).get_default()
-            setattr(mock_settings_obj, key, value)
-        setattr(mock_settings_obj, 'DEBUG', True)
-
-        return mock_settings_obj
-
-    @pytest.fixture
-    def backend(self):
-        class Backend:
-            s = {
-                'ORGANIZATION_MAP': {
-                    'Default1': {
-                        'remove': True,
-                        'admins': 'foobar',
-                        'remove_admins': True,
-                        'users': 'foo',
-                        'remove_users': True,
-                        'organization_alias': 'o1_alias',
-                    }
-                }
-            }
-
-            def setting(self, key):
-                return self.s[key]
-
-        return Backend()
-
-    def test_update_user_orgs_by_saml_attr(self, orgs, users, galaxy_credential, kwargs, mock_settings, backend):
-        with mock.patch('django.conf.settings', mock_settings):
-            o1, o2, o3 = orgs
-            u1, u2, u3 = users
-
-            # Test getting orgs from attribute
-            update_user_orgs_by_saml_attr(None, None, u1, **kwargs)
-            update_user_orgs_by_saml_attr(None, None, u2, **kwargs)
-            update_user_orgs_by_saml_attr(None, None, u3, **kwargs)
-
-            assert o1.member_role.members.count() == 3
-            assert o2.member_role.members.count() == 3
-            assert o3.member_role.members.count() == 0
-
-            # Test remove logic enabled
-            kwargs['response']['attributes']['memberOf'] = ['Default3']
-
-            update_user_orgs_by_saml_attr(None, None, u1, **kwargs)
-
-            assert o1.member_role.members.count() == 2
-            assert o2.member_role.members.count() == 2
-            assert o3.member_role.members.count() == 1
-
-            # Test remove logic disabled
-            mock_settings.SOCIAL_AUTH_SAML_ORGANIZATION_ATTR['remove'] = False
-            kwargs['response']['attributes']['memberOf'] = ['Default1', 'Default2']
-
-            update_user_orgs_by_saml_attr(None, None, u1, **kwargs)
-
-            assert o1.member_role.members.count() == 3
-            assert o2.member_role.members.count() == 3
-            assert o3.member_role.members.count() == 1
-
-            update_user_orgs_by_saml_attr(backend, None, u1, **kwargs)
-            assert Organization.objects.get(name="o1_alias").member_role.members.count() == 1
-
-        for o in Organization.objects.all():
-            if o.id in [o1.id, o2.id, o3.id]:
-                # o[123] were created without a default galaxy cred
-                assert o.galaxy_credentials.count() == 0
-            else:
-                # anything else created should have a default galaxy cred
-                assert o.galaxy_credentials.count() == 1
-                assert o.galaxy_credentials.first().name == 'Ansible Galaxy'
-
-    def test_update_user_teams_by_saml_attr(self, orgs, users, galaxy_credential, kwargs, mock_settings):
-        with mock.patch('django.conf.settings', mock_settings):
-            o1, o2, o3 = orgs
-            u1, u2, u3 = users
-
-            # Test getting teams from attribute with team->org mapping
-
-            kwargs['response']['attributes']['groups'] = ['Blue', 'Red', 'Green']
-
-            # Ensure basic functionality
-            update_user_teams_by_saml_attr(None, None, u1, **kwargs)
-            update_user_teams_by_saml_attr(None, None, u2, **kwargs)
-            update_user_teams_by_saml_attr(None, None, u3, **kwargs)
-
-            assert Team.objects.get(name='Blue', organization__name='Default1').member_role.members.count() == 3
-            assert Team.objects.get(name='Blue', organization__name='Default2').member_role.members.count() == 3
-            assert Team.objects.get(name='Blue', organization__name='Default3').member_role.members.count() == 3
-
-            assert Team.objects.get(name='Red', organization__name='Default1').member_role.members.count() == 3
-
-            assert Team.objects.get(name='Green', organization__name='Default1').member_role.members.count() == 3
-            assert Team.objects.get(name='Green', organization__name='Default3').member_role.members.count() == 3
-
-            # Test remove logic
-            kwargs['response']['attributes']['groups'] = ['Green']
-            update_user_teams_by_saml_attr(None, None, u1, **kwargs)
-            update_user_teams_by_saml_attr(None, None, u2, **kwargs)
-            update_user_teams_by_saml_attr(None, None, u3, **kwargs)
-
-            assert Team.objects.get(name='Blue', organization__name='Default1').member_role.members.count() == 0
-            assert Team.objects.get(name='Blue', organization__name='Default2').member_role.members.count() == 0
-            assert Team.objects.get(name='Blue', organization__name='Default3').member_role.members.count() == 0
-
-            assert Team.objects.get(name='Red', organization__name='Default1').member_role.members.count() == 0
-
-            assert Team.objects.get(name='Green', organization__name='Default1').member_role.members.count() == 3
-            assert Team.objects.get(name='Green', organization__name='Default3').member_role.members.count() == 3
-
-            # Test remove logic disabled
-            mock_settings.SOCIAL_AUTH_SAML_TEAM_ATTR['remove'] = False
-            kwargs['response']['attributes']['groups'] = ['Blue']
-
-            update_user_teams_by_saml_attr(None, None, u1, **kwargs)
-            update_user_teams_by_saml_attr(None, None, u2, **kwargs)
-            update_user_teams_by_saml_attr(None, None, u3, **kwargs)
-
-            assert Team.objects.get(name='Blue', organization__name='Default1').member_role.members.count() == 3
-            assert Team.objects.get(name='Blue', organization__name='Default2').member_role.members.count() == 3
-            assert Team.objects.get(name='Blue', organization__name='Default3').member_role.members.count() == 3
-
-            assert Team.objects.get(name='Red', organization__name='Default1').member_role.members.count() == 0
-
-            assert Team.objects.get(name='Green', organization__name='Default1').member_role.members.count() == 3
-            assert Team.objects.get(name='Green', organization__name='Default3').member_role.members.count() == 3
-
-        for o in Organization.objects.all():
-            if o.id in [o1.id, o2.id, o3.id]:
-                # o[123] were created without a default galaxy cred
-                assert o.galaxy_credentials.count() == 0
-            else:
-                # anything else created should have a default galaxy cred
-                assert o.galaxy_credentials.count() == 1
-                assert o.galaxy_credentials.first().name == 'Ansible Galaxy'
-
-    def test_update_user_teams_alias_by_saml_attr(self, orgs, users, galaxy_credential, kwargs, mock_settings):
-        with mock.patch('django.conf.settings', mock_settings):
-            u1 = users[0]
-
-            # Test getting teams from attribute with team->org mapping
-            kwargs['response']['attributes']['groups'] = ['Yellow']
-
-            # Ensure team and org will be created
-            update_user_teams_by_saml_attr(None, None, u1, **kwargs)
-
-            assert Team.objects.filter(name='Yellow', organization__name='Default4').count() == 0
-            assert Team.objects.filter(name='Yellow_Alias', organization__name='Default4').count() == 1
-            assert Team.objects.get(name='Yellow_Alias', organization__name='Default4').member_role.members.count() == 1
-
-        # only Org 4 got created/updated
-        org = Organization.objects.get(name='Default4')
-        assert org.galaxy_credentials.count() == 1
-        assert org.galaxy_credentials.first().name == 'Ansible Galaxy'
-
-    @pytest.mark.fixture_args(autocreate=False)
-    def test_autocreate_disabled(self, users, kwargs, mock_settings):
-        kwargs['response']['attributes']['memberOf'] = ['Default1', 'Default2', 'Default3']
-        kwargs['response']['attributes']['groups'] = ['Blue', 'Red', 'Green']
-        with mock.patch('django.conf.settings', mock_settings):
-            for u in users:
-                update_user_orgs_by_saml_attr(None, None, u, **kwargs)
-                update_user_teams_by_saml_attr(None, None, u, **kwargs)
-                assert Organization.objects.count() == 0
-                assert Team.objects.count() == 0
-
-            # precreate everything
-            o1 = Organization.objects.create(name='Default1')
-            o2 = Organization.objects.create(name='Default2')
-            o3 = Organization.objects.create(name='Default3')
-            Team.objects.create(name='Blue', organization_id=o1.id)
-            Team.objects.create(name='Blue', organization_id=o2.id)
-            Team.objects.create(name='Blue', organization_id=o3.id)
-            Team.objects.create(name='Red', organization_id=o1.id)
-            Team.objects.create(name='Green', organization_id=o1.id)
-            Team.objects.create(name='Green', organization_id=o3.id)
-
-            for u in users:
-                update_user_orgs_by_saml_attr(None, None, u, **kwargs)
-                update_user_teams_by_saml_attr(None, None, u, **kwargs)
-
-        assert o1.member_role.members.count() == 3
-        assert o2.member_role.members.count() == 3
-        assert o3.member_role.members.count() == 3
-
-        assert Team.objects.get(name='Blue', organization__name='Default1').member_role.members.count() == 3
-        assert Team.objects.get(name='Blue', organization__name='Default2').member_role.members.count() == 3
-        assert Team.objects.get(name='Blue', organization__name='Default3').member_role.members.count() == 3
-
-        assert Team.objects.get(name='Red', organization__name='Default1').member_role.members.count() == 3
-
-        assert Team.objects.get(name='Green', organization__name='Default1').member_role.members.count() == 3
-        assert Team.objects.get(name='Green', organization__name='Default3').member_role.members.count() == 3
-
-    def test_galaxy_credential_auto_assign(self, users, kwargs, galaxy_credential, mock_settings):
-        kwargs['response']['attributes']['memberOf'] = ['Default1', 'Default2', 'Default3']
-        kwargs['response']['attributes']['groups'] = ['Blue', 'Red', 'Green']
-        with mock.patch('django.conf.settings', mock_settings):
-            for u in users:
-                update_user_orgs_by_saml_attr(None, None, u, **kwargs)
-                update_user_teams_by_saml_attr(None, None, u, **kwargs)
-
-        assert Organization.objects.count() == 4
-        for o in Organization.objects.all():
-            assert o.galaxy_credentials.count() == 1
-            assert o.galaxy_credentials.first().name == 'Ansible Galaxy'
-
-    def test_galaxy_credential_no_auto_assign(self, users, kwargs, galaxy_credential, mock_settings):
-        # A Galaxy credential should not be added to an existing org
-        o = Organization.objects.create(name='Default1')
-        o = Organization.objects.create(name='Default2')
-        o = Organization.objects.create(name='Default3')
-        o = Organization.objects.create(name='Default4')
-        kwargs['response']['attributes']['memberOf'] = ['Default1']
-        kwargs['response']['attributes']['groups'] = ['Blue']
-        with mock.patch('django.conf.settings', mock_settings):
-            for u in users:
-                update_user_orgs_by_saml_attr(None, None, u, **kwargs)
-                update_user_teams_by_saml_attr(None, None, u, **kwargs)
-
-        assert Organization.objects.count() == 4
-        for o in Organization.objects.all():
-            assert o.galaxy_credentials.count() == 0
-
-
-@pytest.mark.django_db
-class TestSAMLUserFlags:
-    @pytest.mark.parametrize(
-        "user_flags_settings, expected, is_superuser",
-        [
-            # In this case we will pass no user flags so new_flag should be false and changed will def be false
-            (
-                {},
-                (False, False),
-                False,
-            ),
-            # NOTE: The first handful of tests test role/value as string instead of lists.
-            #       This was from the initial implementation of these fields but the code should be able to handle this
-            #       There are a couple tests at the end of this which will validate arrays in these values.
-            #
-            # In this case we will give the user a group to make them an admin
-            (
-                {'is_superuser_role': 'test-role-1'},
-                (True, True),
-                False,
-            ),
-            # In this case we will give the user a flag that will make then an admin
-            (
-                {'is_superuser_attr': 'is_superuser'},
-                (True, True),
-                False,
-            ),
-            # In this case we will give the user a flag but the wrong value
-            (
-                {'is_superuser_attr': 'is_superuser', 'is_superuser_value': 'junk'},
-                (False, False),
-                False,
-            ),
-            # In this case we will give the user a flag and the right value
-            (
-                {'is_superuser_attr': 'is_superuser', 'is_superuser_value': 'true'},
-                (True, True),
-                False,
-            ),
-            # In this case we will give the user a proper role and an is_superuser_attr role that they dont have, this should make them an admin
-            (
-                {'is_superuser_role': 'test-role-1', 'is_superuser_attr': 'gibberish', 'is_superuser_value': 'true'},
-                (True, True),
-                False,
-            ),
-            # In this case we will give the user a proper role and an is_superuser_attr role that they have, this should make them an admin
-            (
-                {'is_superuser_role': 'test-role-1', 'is_superuser_attr': 'test-role-1'},
-                (True, True),
-                False,
-            ),
-            # In this case we will give the user a proper role and an is_superuser_attr role that they have but a bad value, this should make them an admin
-            (
-                {'is_superuser_role': 'test-role-1', 'is_superuser_attr': 'is_superuser', 'is_superuser_value': 'junk'},
-                (False, False),
-                False,
-            ),
-            # In this case we will give the user everything
-            (
-                {'is_superuser_role': 'test-role-1', 'is_superuser_attr': 'is_superuser', 'is_superuser_value': 'true'},
-                (True, True),
-                False,
-            ),
-            # In this test case we will validate that a single attribute (instead of a list) still works
-            (
-                {'is_superuser_attr': 'name_id', 'is_superuser_value': 'test_id'},
-                (True, True),
-                False,
-            ),
-            # This will be a negative test for a single atrribute
-            (
-                {'is_superuser_attr': 'name_id', 'is_superuser_value': 'junk'},
-                (False, False),
-                False,
-            ),
-            # The user is already a superuser so we should remove them
-            (
-                {'is_superuser_attr': 'name_id', 'is_superuser_value': 'junk', 'remove_superusers': True},
-                (False, True),
-                True,
-            ),
-            # The user is already a superuser but we don't have a remove field
-            (
-                {'is_superuser_attr': 'name_id', 'is_superuser_value': 'junk', 'remove_superusers': False},
-                (True, False),
-                True,
-            ),
-            # Positive test for multiple values for is_superuser_value
-            (
-                {'is_superuser_attr': 'is_superuser', 'is_superuser_value': ['junk', 'junk2', 'else', 'junk']},
-                (True, True),
-                False,
-            ),
-            # Negative test for multiple values for is_superuser_value
-            (
-                {'is_superuser_attr': 'is_superuser', 'is_superuser_value': ['junk', 'junk2', 'junk']},
-                (False, True),
-                True,
-            ),
-            # Positive test for multiple values of is_superuser_role
-            (
-                {'is_superuser_role': ['junk', 'junk2', 'something', 'junk']},
-                (True, True),
-                False,
-            ),
-            # Negative test for multiple values of is_superuser_role
-            (
-                {'is_superuser_role': ['junk', 'junk2', 'junk']},
-                (False, True),
-                True,
-            ),
-        ],
-    )
-    def test__check_flag(self, user_flags_settings, expected, is_superuser):
-        user = User()
-        user.username = 'John'
-        user.is_superuser = is_superuser
-
-        attributes = {
-            'email': ['noone@nowhere.com'],
-            'last_name': ['Westcott'],
-            'is_superuser': ['something', 'else', 'true'],
-            'username': ['test_id'],
-            'first_name': ['John'],
-            'Role': ['test-role-1', 'something', 'different'],
-            'name_id': 'test_id',
-        }
-
-        assert expected == _check_flag(user, 'superuser', attributes, user_flags_settings)

awx/sso/tests/functional/test_saml_pipeline.py

@@ -0,0 +1,639 @@
+import pytest
+import re
+
+from django.test.utils import override_settings
+from awx.main.models import User, Organization, Team
+from awx.sso.saml_pipeline import (
+    _update_m2m_from_expression,
+    _update_user_orgs,
+    _update_user_teams,
+    _update_user_orgs_by_saml_attr,
+    _update_user_teams_by_saml_attr,
+    _check_flag,
+)
+
+# from unittest import mock
+# from django.utils.timezone import now
+# , Credential, CredentialType
+
+
+@pytest.fixture
+def users():
+    u1 = User.objects.create(username='user1@foo.com', last_name='foo', first_name='bar', email='user1@foo.com')
+    u2 = User.objects.create(username='user2@foo.com', last_name='foo', first_name='bar', email='user2@foo.com')
+    u3 = User.objects.create(username='user3@foo.com', last_name='foo', first_name='bar', email='user3@foo.com')
+    return (u1, u2, u3)
+
+
+@pytest.mark.django_db
+class TestSAMLPopulateUser:
+    # The main populate_user does not need to be tested since its just a conglomeration of other functions that we test
+    # This test is here in case someone alters the code in the future in a way that does require testing
+    def test_populate_user(self):
+        assert True
+
+
+@pytest.mark.django_db
+class TestSAMLSimpleMaps:
+    # This tests __update_user_orgs and __update_user_teams
+    @pytest.fixture
+    def backend(self):
+        class Backend:
+            s = {
+                'ORGANIZATION_MAP': {
+                    'Default': {
+                        'remove': True,
+                        'admins': 'foobar',
+                        'remove_admins': True,
+                        'users': 'foo',
+                        'remove_users': True,
+                        'organization_alias': '',
+                    }
+                },
+                'TEAM_MAP': {'Blue': {'organization': 'Default', 'remove': True, 'users': ''}, 'Red': {'organization': 'Default', 'remove': True, 'users': ''}},
+            }
+
+            def setting(self, key):
+                return self.s[key]
+
+        return Backend()
+
+    def test__update_user_orgs(self, backend, users):
+        u1, u2, u3 = users
+
+        # Test user membership logic with regular expressions
+        backend.setting('ORGANIZATION_MAP')['Default']['admins'] = re.compile('.*')
+        backend.setting('ORGANIZATION_MAP')['Default']['users'] = re.compile('.*')
+
+        desired_org_state = {}
+        orgs_to_create = []
+        _update_user_orgs(backend, desired_org_state, orgs_to_create, u1)
+        _update_user_orgs(backend, desired_org_state, orgs_to_create, u2)
+        _update_user_orgs(backend, desired_org_state, orgs_to_create, u3)
+
+        assert desired_org_state == {'Default': {'member_role': True, 'admin_role': True, 'auditor_role': False}}
+        assert orgs_to_create == ['Default']
+
+        # Test remove feature enabled
+        backend.setting('ORGANIZATION_MAP')['Default']['admins'] = ''
+        backend.setting('ORGANIZATION_MAP')['Default']['users'] = ''
+        backend.setting('ORGANIZATION_MAP')['Default']['remove_admins'] = True
+        backend.setting('ORGANIZATION_MAP')['Default']['remove_users'] = True
+        desired_org_state = {}
+        orgs_to_create = []
+        _update_user_orgs(backend, desired_org_state, orgs_to_create, u1)
+        assert desired_org_state == {'Default': {'member_role': False, 'admin_role': False, 'auditor_role': False}}
+        assert orgs_to_create == ['Default']
+
+        # Test remove feature disabled
+        backend.setting('ORGANIZATION_MAP')['Default']['remove_admins'] = False
+        backend.setting('ORGANIZATION_MAP')['Default']['remove_users'] = False
+        desired_org_state = {}
+        orgs_to_create = []
+        _update_user_orgs(backend, desired_org_state, orgs_to_create, u2)
+
+        assert desired_org_state == {'Default': {'member_role': None, 'admin_role': None, 'auditor_role': False}}
+        assert orgs_to_create == ['Default']
+
+        # Test organization alias feature
+        backend.setting('ORGANIZATION_MAP')['Default']['organization_alias'] = 'Default_Alias'
+        orgs_to_create = []
+        _update_user_orgs(backend, {}, orgs_to_create, u1)
+        assert orgs_to_create == ['Default_Alias']
+
+    def test__update_user_teams(self, backend, users):
+        u1, u2, u3 = users
+
+        # Test user membership logic with regular expressions
+        backend.setting('TEAM_MAP')['Blue']['users'] = re.compile('.*')
+        backend.setting('TEAM_MAP')['Red']['users'] = re.compile('.*')
+
+        desired_team_state = {}
+        teams_to_create = {}
+        _update_user_teams(backend, desired_team_state, teams_to_create, u1)
+        assert teams_to_create == {'Red': 'Default', 'Blue': 'Default'}
+        assert desired_team_state == {'Default': {'Blue': {'member_role': True}, 'Red': {'member_role': True}}}
+
+        # Test remove feature enabled
+        backend.setting('TEAM_MAP')['Blue']['remove'] = True
+        backend.setting('TEAM_MAP')['Red']['remove'] = True
+        backend.setting('TEAM_MAP')['Blue']['users'] = ''
+        backend.setting('TEAM_MAP')['Red']['users'] = ''
+
+        desired_team_state = {}
+        teams_to_create = {}
+        _update_user_teams(backend, desired_team_state, teams_to_create, u1)
+        assert teams_to_create == {'Red': 'Default', 'Blue': 'Default'}
+        assert desired_team_state == {'Default': {'Blue': {'member_role': False}, 'Red': {'member_role': False}}}
+
+        # Test remove feature disabled
+        backend.setting('TEAM_MAP')['Blue']['remove'] = False
+        backend.setting('TEAM_MAP')['Red']['remove'] = False
+
+        desired_team_state = {}
+        teams_to_create = {}
+        _update_user_teams(backend, desired_team_state, teams_to_create, u2)
+        assert teams_to_create == {'Red': 'Default', 'Blue': 'Default'}
+        # If we don't care about team memberships we just don't add them to the hash so this would be an empty hash
+        assert desired_team_state == {}
+
+
+@pytest.mark.django_db
+class TestSAMLM2M:
+    @pytest.mark.parametrize(
+        "expression, remove, expected_return",
+        [
+            # No expression with no remove
+            (None, False, None),
+            ("", False, None),
+            # No expression with remove
+            (None, True, False),
+            # True expression with and without remove
+            (True, False, True),
+            (True, True, True),
+            # Single string matching the user name
+            ("user1", False, True),
+            # Single string matching the user email
+            ("user1@foo.com", False, True),
+            # Single string not matching username or email, no remove
+            ("user27", False, None),
+            # Single string not matching username or email, with remove
+            ("user27", True, False),
+            # Same tests with arrays instead of strings
+            (["user1"], False, True),
+            (["user1@foo.com"], False, True),
+            (["user27"], False, None),
+            (["user27"], True, False),
+            # Arrays with nothing matching
+            (["user27", "user28"], False, None),
+            (["user27", "user28"], True, False),
+            # Arrays with all matches
+            (["user1", "user1@foo.com"], False, True),
+            # Arrays with some match, some not
+            (["user1", "user28", "user27"], False, True),
+            #
+            # Note: For RE's, usually settings takes care of the compilation for us, so we have to do it manually for testing.
+            #       we also need to remove any / or flags for the compile to happen
+            #
+            # Matching username regex non-array
+            (re.compile("^user.*"), False, True),
+            (re.compile("^user.*"), True, True),
+            # Matching email regex non-array
+            (re.compile(".*@foo.com$"), False, True),
+            (re.compile(".*@foo.com$"), True, True),
+            # Non-array not matching username or email
+            (re.compile("^$"), False, None),
+            (re.compile("^$"), True, False),
+            # All re tests just in array form
+            ([re.compile("^user.*")], False, True),
+            ([re.compile("^user.*")], True, True),
+            ([re.compile(".*@foo.com$")], False, True),
+            ([re.compile(".*@foo.com$")], True, True),
+            ([re.compile("^$")], False, None),
+            ([re.compile("^$")], True, False),
+            # An re with username matching but not email
+            ([re.compile("^user.*"), re.compile(".*@bar.com$")], False, True),
+            # An re with email matching but not username
+            ([re.compile("^user27$"), re.compile(".*@foo.com$")], False, True),
+            # An re array with no matching
+            ([re.compile("^user27$"), re.compile(".*@bar.com$")], False, None),
+            ([re.compile("^user27$"), re.compile(".*@bar.com$")], True, False),
+            #
+            # A mix of re and strings
+            #
+            # String matches, re does not
+            (["user1", re.compile(".*@bar.com$")], False, True),
+            # String does not match, re does
+            (["user27", re.compile(".*@foo.com$")], False, True),
+            # Nothing matches
+            (["user27", re.compile(".*@bar.com$")], False, None),
+            (["user27", re.compile(".*@bar.com$")], True, False),
+        ],
+    )
+    def test__update_m2m_from_expression(self, expression, remove, expected_return):
+        user = User.objects.create(username='user1', last_name='foo', first_name='bar', email='user1@foo.com')
+        return_val = _update_m2m_from_expression(user, expression, remove)
+        assert return_val == expected_return
+
+
+@pytest.mark.django_db
+class TestSAMLAttrMaps:
+    @pytest.fixture
+    def backend(self):
+        class Backend:
+            s = {
+                'ORGANIZATION_MAP': {
+                    'Default1': {
+                        'remove': True,
+                        'admins': 'foobar',
+                        'remove_admins': True,
+                        'users': 'foo',
+                        'remove_users': True,
+                        'organization_alias': 'o1_alias',
+                    }
+                }
+            }
+
+            def setting(self, key):
+                return self.s[key]
+
+        return Backend()
+
+    @pytest.mark.parametrize(
+        "setting, expected_state, expected_orgs_to_create, kwargs_member_of_mods",
+        [
+            (
+                # Default test, make sure that our roles get applied and removed as specified (with an alias)
+                {
+                    'saml_attr': 'memberOf',
+                    'saml_admin_attr': 'admins',
+                    'saml_auditor_attr': 'auditors',
+                    'remove': True,
+                    'remove_admins': True,
+                },
+                {
+                    'Default2': {'member_role': True},
+                    'Default3': {'admin_role': True},
+                    'Default4': {'auditor_role': True},
+                    'o1_alias': {'member_role': True},
+                    'Rando1': {'admin_role': False, 'auditor_role': False, 'member_role': False},
+                },
+                [
+                    'o1_alias',
+                    'Default2',
+                    'Default3',
+                    'Default4',
+                ],
+                None,
+            ),
+            (
+                # Similar test, we are just going to override the values "coming from the IdP" to limit the teams
+                {
+                    'saml_attr': 'memberOf',
+                    'saml_admin_attr': 'admins',
+                    'saml_auditor_attr': 'auditors',
+                    'remove': True,
+                    'remove_admins': True,
+                },
+                {
+                    'Default3': {'admin_role': True, 'member_role': True},
+                    'Default4': {'auditor_role': True},
+                    'Rando1': {'admin_role': False, 'auditor_role': False, 'member_role': False},
+                },
+                [
+                    'Default3',
+                    'Default4',
+                ],
+                ['Default3'],
+            ),
+            (
+                # Test to make sure the remove logic is working
+                {
+                    'saml_attr': 'memberOf',
+                    'saml_admin_attr': 'admins',
+                    'saml_auditor_attr': 'auditors',
+                    'remove': False,
+                    'remove_admins': False,
+                    'remove_auditors': False,
+                },
+                {
+                    'Default2': {'member_role': True},
+                    'Default3': {'admin_role': True},
+                    'Default4': {'auditor_role': True},
+                    'o1_alias': {'member_role': True},
+                },
+                [
+                    'o1_alias',
+                    'Default2',
+                    'Default3',
+                    'Default4',
+                ],
+                ['Default1', 'Default2'],
+            ),
+        ],
+    )
+    def test__update_user_orgs_by_saml_attr(self, backend, setting, expected_state, expected_orgs_to_create, kwargs_member_of_mods):
+        kwargs = {
+            'username': u'cmeyers@redhat.com',
+            'uid': 'idp:cmeyers@redhat.com',
+            'request': {u'SAMLResponse': [], u'RelayState': [u'idp']},
+            'is_new': False,
+            'response': {
+                'session_index': '_0728f0e0-b766-0135-75fa-02842b07c044',
+                'idp_name': u'idp',
+                'attributes': {
+                    'memberOf': ['Default1', 'Default2'],
+                    'admins': ['Default3'],
+                    'auditors': ['Default4'],
+                    'groups': ['Blue', 'Red'],
+                    'User.email': ['cmeyers@redhat.com'],
+                    'User.LastName': ['Meyers'],
+                    'name_id': 'cmeyers@redhat.com',
+                    'User.FirstName': ['Chris'],
+                    'PersonImmutableID': [],
+                },
+            },
+            'social': None,
+            'strategy': None,
+            'new_association': False,
+        }
+        if kwargs_member_of_mods:
+            kwargs['response']['attributes']['memberOf'] = kwargs_member_of_mods
+
+        # Create a random organization in the database for testing
+        Organization.objects.create(name='Rando1')
+
+        with override_settings(SOCIAL_AUTH_SAML_ORGANIZATION_ATTR=setting):
+            desired_org_state = {}
+            orgs_to_create = []
+            _update_user_orgs_by_saml_attr(backend, desired_org_state, orgs_to_create, **kwargs)
+            assert desired_org_state == expected_state
+            assert orgs_to_create == expected_orgs_to_create
+
+    @pytest.mark.parametrize(
+        "setting, expected_team_state, expected_teams_to_create, kwargs_group_override",
+        [
+            (
+                {
+                    'saml_attr': 'groups',
+                    'remove': False,
+                    'team_org_map': [
+                        {'team': 'Blue', 'organization': 'Default1'},
+                        {'team': 'Blue', 'organization': 'Default2'},
+                        {'team': 'Blue', 'organization': 'Default3'},
+                        {'team': 'Red', 'organization': 'Default1'},
+                        {'team': 'Green', 'organization': 'Default1'},
+                        {'team': 'Green', 'organization': 'Default3'},
+                        {'team': 'Yellow', 'team_alias': 'Yellow_Alias', 'organization': 'Default4', 'organization_alias': 'Default4_Alias'},
+                    ],
+                },
+                {
+                    'Default1': {
+                        'Blue': {'member_role': True},
+                        'Green': {'member_role': False},
+                        'Red': {'member_role': True},
+                    },
+                    'Default2': {
+                        'Blue': {'member_role': True},
+                    },
+                    'Default3': {
+                        'Blue': {'member_role': True},
+                        'Green': {'member_role': False},
+                    },
+                    'Default4': {
+                        'Yellow': {'member_role': False},
+                    },
+                },
+                {
+                    'Blue': 'Default3',
+                    'Red': 'Default1',
+                },
+                None,
+            ),
+            (
+                {
+                    'saml_attr': 'groups',
+                    'remove': False,
+                    'team_org_map': [
+                        {'team': 'Blue', 'organization': 'Default1'},
+                        {'team': 'Blue', 'organization': 'Default2'},
+                        {'team': 'Blue', 'organization': 'Default3'},
+                        {'team': 'Red', 'organization': 'Default1'},
+                        {'team': 'Green', 'organization': 'Default1'},
+                        {'team': 'Green', 'organization': 'Default3'},
+                        {'team': 'Yellow', 'team_alias': 'Yellow_Alias', 'organization': 'Default4', 'organization_alias': 'Default4_Alias'},
+                    ],
+                },
+                {
+                    'Default1': {
+                        'Blue': {'member_role': True},
+                        'Green': {'member_role': True},
+                        'Red': {'member_role': True},
+                    },
+                    'Default2': {
+                        'Blue': {'member_role': True},
+                    },
+                    'Default3': {
+                        'Blue': {'member_role': True},
+                        'Green': {'member_role': True},
+                    },
+                    'Default4': {
+                        'Yellow': {'member_role': False},
+                    },
+                },
+                {
+                    'Blue': 'Default3',
+                    'Red': 'Default1',
+                    'Green': 'Default3',
+                },
+                ['Blue', 'Red', 'Green'],
+            ),
+            (
+                {
+                    'saml_attr': 'groups',
+                    'remove': True,
+                    'team_org_map': [
+                        {'team': 'Blue', 'organization': 'Default1'},
+                        {'team': 'Blue', 'organization': 'Default2'},
+                        {'team': 'Blue', 'organization': 'Default3'},
+                        {'team': 'Red', 'organization': 'Default1'},
+                        {'team': 'Green', 'organization': 'Default1'},
+                        {'team': 'Green', 'organization': 'Default3'},
+                        {'team': 'Yellow', 'team_alias': 'Yellow_Alias', 'organization': 'Default4', 'organization_alias': 'Default4_Alias'},
+                    ],
+                },
+                {
+                    'Default1': {
+                        'Blue': {'member_role': False},
+                        'Green': {'member_role': True},
+                        'Red': {'member_role': False},
+                    },
+                    'Default2': {
+                        'Blue': {'member_role': False},
+                    },
+                    'Default3': {
+                        'Blue': {'member_role': False},
+                        'Green': {'member_role': True},
+                    },
+                    'Default4': {
+                        'Yellow': {'member_role': False},
+                    },
+                    'Rando1': {
+                        'Rando1': {'member_role': False},
+                    },
+                },
+                {
+                    'Green': 'Default3',
+                },
+                ['Green'],
+            ),
+        ],
+    )
+    def test__update_user_teams_by_saml_attr(self, setting, expected_team_state, expected_teams_to_create, kwargs_group_override):
+        kwargs = {
+            'username': u'cmeyers@redhat.com',
+            'uid': 'idp:cmeyers@redhat.com',
+            'request': {u'SAMLResponse': [], u'RelayState': [u'idp']},
+            'is_new': False,
+            'response': {
+                'session_index': '_0728f0e0-b766-0135-75fa-02842b07c044',
+                'idp_name': u'idp',
+                'attributes': {
+                    'memberOf': ['Default1', 'Default2'],
+                    'admins': ['Default3'],
+                    'auditors': ['Default4'],
+                    'groups': ['Blue', 'Red'],
+                    'User.email': ['cmeyers@redhat.com'],
+                    'User.LastName': ['Meyers'],
+                    'name_id': 'cmeyers@redhat.com',
+                    'User.FirstName': ['Chris'],
+                    'PersonImmutableID': [],
+                },
+            },
+            'social': None,
+            'strategy': None,
+            'new_association': False,
+        }
+        if kwargs_group_override:
+            kwargs['response']['attributes']['groups'] = kwargs_group_override
+
+        o = Organization.objects.create(name='Rando1')
+        Team.objects.create(name='Rando1', organization_id=o.id)
+
+        with override_settings(SOCIAL_AUTH_SAML_TEAM_ATTR=setting):
+            desired_team_state = {}
+            teams_to_create = {}
+            _update_user_teams_by_saml_attr(desired_team_state, teams_to_create, **kwargs)
+            assert desired_team_state == expected_team_state
+            assert teams_to_create == expected_teams_to_create
+
+
+@pytest.mark.django_db
+class TestSAMLUserFlags:
+    @pytest.mark.parametrize(
+        "user_flags_settings, expected, is_superuser",
+        [
+            # In this case we will pass no user flags so new_flag should be false and changed will def be false
+            (
+                {},
+                (False, False),
+                False,
+            ),
+            # NOTE: The first handful of tests test role/value as string instead of lists.
+            #       This was from the initial implementation of these fields but the code should be able to handle this
+            #       There are a couple tests at the end of this which will validate arrays in these values.
+            #
+            # In this case we will give the user a group to make them an admin
+            (
+                {'is_superuser_role': 'test-role-1'},
+                (True, True),
+                False,
+            ),
+            # In this case we will give the user a flag that will make then an admin
+            (
+                {'is_superuser_attr': 'is_superuser'},
+                (True, True),
+                False,
+            ),
+            # In this case we will give the user a flag but the wrong value
+            (
+                {'is_superuser_attr': 'is_superuser', 'is_superuser_value': 'junk'},
+                (False, False),
+                False,
+            ),
+            # In this case we will give the user a flag and the right value
+            (
+                {'is_superuser_attr': 'is_superuser', 'is_superuser_value': 'true'},
+                (True, True),
+                False,
+            ),
+            # In this case we will give the user a proper role and an is_superuser_attr role that they don't have, this should make them an admin
+            (
+                {'is_superuser_role': 'test-role-1', 'is_superuser_attr': 'gibberish', 'is_superuser_value': 'true'},
+                (True, True),
+                False,
+            ),
+            # In this case we will give the user a proper role and an is_superuser_attr role that they have, this should make them an admin
+            (
+                {'is_superuser_role': 'test-role-1', 'is_superuser_attr': 'test-role-1'},
+                (True, True),
+                False,
+            ),
+            # In this case we will give the user a proper role and an is_superuser_attr role that they have but a bad value, this should make them an admin
+            (
+                {'is_superuser_role': 'test-role-1', 'is_superuser_attr': 'is_superuser', 'is_superuser_value': 'junk'},
+                (False, False),
+                False,
+            ),
+            # In this case we will give the user everything
+            (
+                {'is_superuser_role': 'test-role-1', 'is_superuser_attr': 'is_superuser', 'is_superuser_value': 'true'},
+                (True, True),
+                False,
+            ),
+            # In this test case we will validate that a single attribute (instead of a list) still works
+            (
+                {'is_superuser_attr': 'name_id', 'is_superuser_value': 'test_id'},
+                (True, True),
+                False,
+            ),
+            # This will be a negative test for a single attribute
+            (
+                {'is_superuser_attr': 'name_id', 'is_superuser_value': 'junk'},
+                (False, False),
+                False,
+            ),
+            # The user is already a superuser so we should remove them
+            (
+                {'is_superuser_attr': 'name_id', 'is_superuser_value': 'junk', 'remove_superusers': True},
+                (False, True),
+                True,
+            ),
+            # The user is already a superuser but we don't have a remove field
+            (
+                {'is_superuser_attr': 'name_id', 'is_superuser_value': 'junk', 'remove_superusers': False},
+                (True, False),
+                True,
+            ),
+            # Positive test for multiple values for is_superuser_value
+            (
+                {'is_superuser_attr': 'is_superuser', 'is_superuser_value': ['junk', 'junk2', 'else', 'junk']},
+                (True, True),
+                False,
+            ),
+            # Negative test for multiple values for is_superuser_value
+            (
+                {'is_superuser_attr': 'is_superuser', 'is_superuser_value': ['junk', 'junk2', 'junk']},
+                (False, True),
+                True,
+            ),
+            # Positive test for multiple values of is_superuser_role
+            (
+                {'is_superuser_role': ['junk', 'junk2', 'something', 'junk']},
+                (True, True),
+                False,
+            ),
+            # Negative test for multiple values of is_superuser_role
+            (
+                {'is_superuser_role': ['junk', 'junk2', 'junk']},
+                (False, True),
+                True,
+            ),
+        ],
+    )
+    def test__check_flag(self, user_flags_settings, expected, is_superuser):
+        user = User()
+        user.username = 'John'
+        user.is_superuser = is_superuser
+
+        attributes = {
+            'email': ['noone@nowhere.com'],
+            'last_name': ['Westcott'],
+            'is_superuser': ['something', 'else', 'true'],
+            'username': ['test_id'],
+            'first_name': ['John'],
+            'Role': ['test-role-1', 'something', 'different'],
+            'name_id': 'test_id',
+        }
+
+        assert expected == _check_flag(user, 'superuser', attributes, user_flags_settings)

awx/sso/tests/functional/test_social_base_pipeline.py

@@ -0,0 +1,76 @@
+import pytest
+
+from awx.main.models import User
+from awx.sso.social_base_pipeline import AuthNotFound, check_user_found_or_created, set_is_active_for_new_user, prevent_inactive_login, AuthInactive
+
+
+@pytest.mark.django_db
+class TestSocialBasePipeline:
+    def test_check_user_found_or_created_no_exception(self):
+        # If we have a user (the True param, we should not get an exception)
+        try:
+            check_user_found_or_created(None, {}, True)
+        except AuthNotFound:
+            assert False, 'check_user_found_or_created should not have raised an exception with a user'
+
+    @pytest.mark.parametrize(
+        "details, kwargs, expected_id",
+        [
+            (
+                {},
+                {},
+                '???',
+            ),
+            (
+                {},
+                {'uid': 'kwargs_uid'},
+                'kwargs_uid',
+            ),
+            (
+                {},
+                {'uid': 'kwargs_uid', 'email': 'kwargs_email'},
+                'kwargs_email',
+            ),
+            (
+                {'email': 'details_email'},
+                {'uid': 'kwargs_uid', 'email': 'kwargs_email'},
+                'details_email',
+            ),
+        ],
+    )
+    def test_check_user_found_or_created_exceptions(self, details, expected_id, kwargs):
+        with pytest.raises(AuthNotFound) as e:
+            check_user_found_or_created(None, details, False, None, **kwargs)
+        assert f'An account cannot be found for {expected_id}' == str(e.value)
+
+    @pytest.mark.parametrize(
+        "kwargs, expected_details, expected_response",
+        [
+            ({}, {}, None),
+            ({'is_new': False}, {}, None),
+            ({'is_new': True}, {'is_active': True}, {'details': {'is_active': True}}),
+        ],
+    )
+    def test_set_is_active_for_new_user(self, kwargs, expected_details, expected_response):
+        details = {}
+        response = set_is_active_for_new_user(None, details, None, None, **kwargs)
+        assert details == expected_details
+        assert response == expected_response
+
+    def test_prevent_inactive_login_no_exception_no_user(self):
+        try:
+            prevent_inactive_login(None, None, None, None, None)
+        except AuthInactive:
+            assert False, 'prevent_inactive_login should not have raised an exception with no user'
+
+    def test_prevent_inactive_login_no_exception_active_user(self):
+        user = User.objects.create(username='user1@foo.com', last_name='foo', first_name='bar', email='user1@foo.com', is_active=True)
+        try:
+            prevent_inactive_login(None, None, user, None, None)
+        except AuthInactive:
+            assert False, 'prevent_inactive_login should not have raised an exception with an active user'
+
+    def test_prevent_inactive_login_no_exception_inactive_user(self):
+        user = User.objects.create(username='user1@foo.com', last_name='foo', first_name='bar', email='user1@foo.com', is_active=False)
+        with pytest.raises(AuthInactive):
+            prevent_inactive_login(None, None, user, None, None)

awx/sso/tests/functional/test_social_pipeline.py

@@ -0,0 +1,113 @@
+import pytest
+import re
+
+from awx.sso.social_pipeline import update_user_orgs, update_user_teams
+from awx.main.models import User, Team, Organization
+
+
+@pytest.fixture
+def users():
+    u1 = User.objects.create(username='user1@foo.com', last_name='foo', first_name='bar', email='user1@foo.com')
+    u2 = User.objects.create(username='user2@foo.com', last_name='foo', first_name='bar', email='user2@foo.com')
+    u3 = User.objects.create(username='user3@foo.com', last_name='foo', first_name='bar', email='user3@foo.com')
+    return (u1, u2, u3)
+
+
+@pytest.mark.django_db
+class TestSocialPipeline:
+    @pytest.fixture
+    def backend(self):
+        class Backend:
+            s = {
+                'ORGANIZATION_MAP': {
+                    'Default': {
+                        'remove': True,
+                        'admins': 'foobar',
+                        'remove_admins': True,
+                        'users': 'foo',
+                        'remove_users': True,
+                        'organization_alias': '',
+                    }
+                },
+                'TEAM_MAP': {'Blue': {'organization': 'Default', 'remove': True, 'users': ''}, 'Red': {'organization': 'Default', 'remove': True, 'users': ''}},
+            }
+
+            def setting(self, key):
+                return self.s[key]
+
+        return Backend()
+
+    @pytest.fixture
+    def org(self):
+        return Organization.objects.create(name="Default")
+
+    def test_update_user_orgs(self, org, backend, users):
+        u1, u2, u3 = users
+
+        # Test user membership logic with regular expressions
+        backend.setting('ORGANIZATION_MAP')['Default']['admins'] = re.compile('.*')
+        backend.setting('ORGANIZATION_MAP')['Default']['users'] = re.compile('.*')
+
+        update_user_orgs(backend, None, u1)
+        update_user_orgs(backend, None, u2)
+        update_user_orgs(backend, None, u3)
+
+        assert org.admin_role.members.count() == 3
+        assert org.member_role.members.count() == 3
+
+        # Test remove feature enabled
+        backend.setting('ORGANIZATION_MAP')['Default']['admins'] = ''
+        backend.setting('ORGANIZATION_MAP')['Default']['users'] = ''
+        backend.setting('ORGANIZATION_MAP')['Default']['remove_admins'] = True
+        backend.setting('ORGANIZATION_MAP')['Default']['remove_users'] = True
+        update_user_orgs(backend, None, u1)
+
+        assert org.admin_role.members.count() == 2
+        assert org.member_role.members.count() == 2
+
+        # Test remove feature disabled
+        backend.setting('ORGANIZATION_MAP')['Default']['remove_admins'] = False
+        backend.setting('ORGANIZATION_MAP')['Default']['remove_users'] = False
+        update_user_orgs(backend, None, u2)
+
+        assert org.admin_role.members.count() == 2
+        assert org.member_role.members.count() == 2
+
+        # Test organization alias feature
+        backend.setting('ORGANIZATION_MAP')['Default']['organization_alias'] = 'Default_Alias'
+        update_user_orgs(backend, None, u1)
+        assert Organization.objects.get(name="Default_Alias") is not None
+
+    def test_update_user_teams(self, backend, users):
+        u1, u2, u3 = users
+
+        # Test user membership logic with regular expressions
+        backend.setting('TEAM_MAP')['Blue']['users'] = re.compile('.*')
+        backend.setting('TEAM_MAP')['Red']['users'] = re.compile('.*')
+
+        update_user_teams(backend, None, u1)
+        update_user_teams(backend, None, u2)
+        update_user_teams(backend, None, u3)
+
+        assert Team.objects.get(name="Red").member_role.members.count() == 3
+        assert Team.objects.get(name="Blue").member_role.members.count() == 3
+
+        # Test remove feature enabled
+        backend.setting('TEAM_MAP')['Blue']['remove'] = True
+        backend.setting('TEAM_MAP')['Red']['remove'] = True
+        backend.setting('TEAM_MAP')['Blue']['users'] = ''
+        backend.setting('TEAM_MAP')['Red']['users'] = ''
+
+        update_user_teams(backend, None, u1)
+
+        assert Team.objects.get(name="Red").member_role.members.count() == 2
+        assert Team.objects.get(name="Blue").member_role.members.count() == 2
+
+        # Test remove feature disabled
+        backend.setting('TEAM_MAP')['Blue']['remove'] = False
+        backend.setting('TEAM_MAP')['Red']['remove'] = False
+
+        update_user_teams(backend, None, u2)
+
+        assert Team.objects.get(name="Red").member_role.members.count() == 2
+        assert Team.objects.get(name="Blue").member_role.members.count() == 2

awx/sso/tests/unit/test_pipeline.py

@@ -1,2 +0,0 @@
-def test_module_loads():
-    from awx.sso import pipeline  # noqa

awx/sso/tests/unit/test_pipelines.py

@@ -0,0 +1,12 @@
+import pytest
+
+
+@pytest.mark.parametrize(
+    "lib",
+    [
+        ("saml_pipeline"),
+        ("social_pipeline"),
+    ],
+)
+def test_module_loads(lib):
+    module = __import__("awx.sso." + lib)  # noqa

awx/ui/src/components/RelatedTemplateList/RelatedTemplateList.js

@@ -34,8 +34,14 @@ const QS_CONFIG = getQSConfig('template', {
   order_by: 'name',
 });
 
-function RelatedTemplateList({ searchParams, projectName = null }) {
-  const { id: projectId } = useParams();
+const resources = {
+  projects: 'project',
+  inventories: 'inventory',
+  credentials: 'credentials',
+};
+
+function RelatedTemplateList({ searchParams, resourceName = null }) {
+  const { id } = useParams();
   const location = useLocation();
   const { addToast, Toast, toastProps } = useToast();
 
@@ -129,12 +135,19 @@ function RelatedTemplateList({ searchParams, projectName = null }) {
     actions && Object.prototype.hasOwnProperty.call(actions, 'POST');
 
   let linkTo = '';
-
-  if (projectName) {
-    const qs = encodeQueryString({
-      project_id: projectId,
-      project_name: projectName,
-    });
+  if (resourceName) {
+    const queryString = {
+      resource_id: id,
+      resource_name: resourceName,
+      resource_type: resources[location.pathname.split('/')[1]],
+      resource_kind: null,
+    };
+    if (Array.isArray(resourceName)) {
+      const [name, kind] = resourceName;
+      queryString.resource_name = name;
+      queryString.resource_kind = kind;
+    }
+    const qs = encodeQueryString(queryString);
     linkTo = `/templates/job_template/add/?${qs}`;
   } else {
     linkTo = '/templates/job_template/add';

awx/ui/src/components/RelatedTemplateList/relatedTemplateHelpers.js

@@ -0,0 +1 @@
+/* eslint-disable import/prefer-default-export */

awx/ui/src/components/Search/AdvancedSearch.test.js

@@ -420,7 +420,7 @@ describe('<AdvancedSearch />', () => {
     const selectOptions = wrapper.find(
       'Select[aria-label="Related search type"] SelectOption'
     );
-    expect(selectOptions).toHaveLength(2);
+    expect(selectOptions).toHaveLength(3);
     expect(
       selectOptions.find('SelectOption[id="name-option-select"]').prop('value')
     ).toBe('name__icontains');

awx/ui/src/components/Search/RelatedLookupTypeInput.js

@@ -31,6 +31,12 @@ function RelatedLookupTypeInput({
         value="name__icontains"
         description={t`Fuzzy search on name field.`}
       />
+      <SelectOption
+        id="name-exact-option-select"
+        key="name"
+        value="name"
+        description={t`Exact search on name field.`}
+      />
       <SelectOption
         id="id-option-select"
         key="id"

awx/ui/src/screens/Credential/Credential.js

@@ -22,6 +22,16 @@ import { CredentialsAPI } from 'api';
 import CredentialDetail from './CredentialDetail';
 import CredentialEdit from './CredentialEdit';
 
+const jobTemplateCredentialTypes = [
+  'machine',
+  'cloud',
+  'net',
+  'ssh',
+  'vault',
+  'kubernetes',
+  'cryptography',
+];
+
 function Credential({ setBreadcrumb }) {
   const { pathname } = useLocation();
 
@@ -75,13 +85,14 @@ function Credential({ setBreadcrumb }) {
       link: `/credentials/${id}/access`,
       id: 1,
     },
-    {
+  ];
+  if (jobTemplateCredentialTypes.includes(credential?.kind)) {
+    tabsArray.push({
       name: t`Job Templates`,
       link: `/credentials/${id}/job_templates`,
       id: 2,
-    },
-  ];
-
+    });
+  }
   let showCardHeader = true;
 
   if (pathname.endsWith('edit') || pathname.endsWith('add')) {
@@ -133,6 +144,7 @@ function Credential({ setBreadcrumb }) {
               <Route key="job_templates" path="/credentials/:id/job_templates">
                 <RelatedTemplateList
                   searchParams={{ credentials__id: credential.id }}
+                  resourceName={[credential.name, credential.kind]}
                 />
               </Route>,
               <Route key="not-found" path="*">

awx/ui/src/screens/Credential/Credential.test.js

@@ -6,7 +6,8 @@ import {
   mountWithContexts,
   waitForElement,
 } from '../../../testUtils/enzymeHelpers';
-import mockCredential from './shared/data.scmCredential.json';
+import mockMachineCredential from './shared/data.machineCredential.json';
+import mockSCMCredential from './shared/data.scmCredential.json';
 import Credential from './Credential';
 
 jest.mock('../../api');
@@ -21,13 +22,10 @@ jest.mock('react-router-dom', () => ({
 describe('<Credential />', () => {
   let wrapper;
 
-  beforeEach(() => {
+  test('initially renders user-based machine credential successfully', async () => {
     CredentialsAPI.readDetail.mockResolvedValueOnce({
-      data: mockCredential,
+      data: mockMachineCredential,
     });
-  });
-
-  test('initially renders user-based credential successfully', async () => {
     await act(async () => {
       wrapper = mountWithContexts(<Credential setBreadcrumb={() => {}} />);
     });
@@ -36,6 +34,18 @@ describe('<Credential />', () => {
     expect(wrapper.find('RoutedTabs li').length).toBe(4);
   });
 
+  test('initially renders user-based SCM credential successfully', async () => {
+    CredentialsAPI.readDetail.mockResolvedValueOnce({
+      data: mockSCMCredential,
+    });
+    await act(async () => {
+      wrapper = mountWithContexts(<Credential setBreadcrumb={() => {}} />);
+    });
+    wrapper.update();
+    expect(wrapper.find('Credential').length).toBe(1);
+    expect(wrapper.find('RoutedTabs li').length).toBe(3);
+  });
+
   test('should render expected tabs', async () => {
     const expectedTabs = [
       'Back to Credentials',

awx/ui/src/screens/InstanceGroup/InstanceDetails/InstanceDetails.js

@@ -81,35 +81,30 @@ function InstanceDetails({ setBreadcrumb, instanceGroup }) {
       const {
         data: { results },
       } = await InstanceGroupsAPI.readInstances(instanceGroup.id);
-      let instanceDetails;
       const isAssociated = results.some(
         ({ id: instId }) => instId === parseInt(instanceId, 10)
       );
 
       if (isAssociated) {
-        const [{ data: details }, { data: healthCheckData }] =
-          await Promise.all([
-            InstancesAPI.readDetail(instanceId),
-            InstancesAPI.readHealthCheckDetail(instanceId),
-          ]);
-
-        instanceDetails = details;
+        const { data: details } = await InstancesAPI.readDetail(instanceId);
+        if (details.node_type === 'execution') {
+          const { data: healthCheckData } =
+            await InstancesAPI.readHealthCheckDetail(instanceId);
           setHealthCheck(healthCheckData);
-      } else {
+        }
+        setBreadcrumb(instanceGroup, details);
+        setForks(
+          computeForks(
+            details.mem_capacity,
+            details.cpu_capacity,
+            details.capacity_adjustment
+          )
+        );
+        return { instance: details };
+      }
       throw new Error(
         `This instance is not associated with this instance group`
       );
-      }
-
-      setBreadcrumb(instanceGroup, instanceDetails);
-      setForks(
-        computeForks(
-          instanceDetails.mem_capacity,
-          instanceDetails.cpu_capacity,
-          instanceDetails.capacity_adjustment
-        )
-      );
-      return { instance: instanceDetails };
     }, [instanceId, setBreadcrumb, instanceGroup]),
     { instance: {}, isLoading: true }
   );

awx/ui/src/screens/Inventory/Inventory.js

@@ -181,6 +181,7 @@ function Inventory({ setBreadcrumb }) {
             >
               <RelatedTemplateList
                 searchParams={{ inventory__id: inventory.id }}
+                resourceName={inventory.name}
               />
             </Route>,
             <Route path="*" key="not-found">

awx/ui/src/screens/Job/JobOutput/JobOutput.js

@@ -187,7 +187,9 @@ function JobOutput({ job, eventRelatedSearchableKeys, eventSearchableKeys }) {
   useEffect(() => {
     const pendingRequests = Object.values(eventByUuidRequests.current || {});
     setHasContentLoading(true); // prevents "no content found" screen from flashing
+    if (location.search) {
       setIsFollowModeEnabled(false);
+    }
     Promise.allSettled(pendingRequests).then(() => {
       setRemoteRowCount(0);
       clearLoadedEvents();
@@ -251,6 +253,9 @@ function JobOutput({ job, eventRelatedSearchableKeys, eventSearchableKeys }) {
         });
         const updated = oldWsEvents.concat(newEvents);
         jobSocketCounter.current = updated.length;
+        if (!oldWsEvents.length && min > remoteRowCount + 1) {
+          loadJobEvents(min);
+        }
         return updated.sort((a, b) => a.counter - b.counter);
       });
       setCssMap((prevCssMap) => ({
@@ -358,7 +363,7 @@ function JobOutput({ job, eventRelatedSearchableKeys, eventSearchableKeys }) {
     }
   };
 
-  const loadJobEvents = async () => {
+  const loadJobEvents = async (firstWsCounter = null) => {
     const [params, loadRange] = getEventRequestParams(job, 50, [1, 50]);
 
     if (isMounted.current) {
@@ -371,6 +376,9 @@ function JobOutput({ job, eventRelatedSearchableKeys, eventSearchableKeys }) {
     if (isFlatMode) {
       params.not__stdout = '';
     }
+    if (firstWsCounter) {
+      params.counter__lt = firstWsCounter;
+    }
     const qsParams = parseQueryString(QS_CONFIG, location.search);
     const eventPromise = getJobModel(job.type).readEvents(job.id, {
       ...params,
@@ -435,7 +443,7 @@ function JobOutput({ job, eventRelatedSearchableKeys, eventSearchableKeys }) {
     if (getEvent(counter)) {
       return true;
     }
-    if (index > remoteRowCount && index < remoteRowCount + wsEvents.length) {
+    if (index >= remoteRowCount && index < remoteRowCount + wsEvents.length) {
       return true;
     }
     return currentlyLoading.includes(counter);
@@ -462,7 +470,7 @@ function JobOutput({ job, eventRelatedSearchableKeys, eventSearchableKeys }) {
     }
     if (
       !event &&
-      index > remoteRowCount &&
+      index >= remoteRowCount &&
       index < remoteRowCount + wsEvents.length
     ) {
       event = wsEvents[index - remoteRowCount];
@@ -629,10 +637,14 @@ function JobOutput({ job, eventRelatedSearchableKeys, eventSearchableKeys }) {
     setIsFollowModeEnabled(false);
   };
 
-  const scrollToEnd = () => {
+  const scrollToEnd = useCallback(() => {
     scrollToRow(-1);
+    let timeout;
+    if (isFollowModeEnabled) {
       setTimeout(() => scrollToRow(-1), 100);
-  };
+    }
+    return () => clearTimeout(timeout);
+  }, [isFollowModeEnabled]);
 
   const handleScrollLast = () => {
     scrollToEnd();

awx/ui/src/screens/Project/Project.js

@@ -179,7 +179,7 @@ function Project({ setBreadcrumb }) {
                 searchParams={{
                   project__id: project.id,
                 }}
-                projectName={project.name}
+                resourceName={project.name}
               />
             </Route>
             {project?.scm_type && project.scm_type !== '' && (

awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.js

@@ -141,14 +141,14 @@ function JobsEdit() {
               <FormColumnLayout>
                 <InputField
                   name="AWX_ISOLATION_BASE_PATH"
-                  config={jobs.AWX_ISOLATION_BASE_PATH}
-                  isRequired
+                  config={jobs.AWX_ISOLATION_BASE_PATH ?? null}
+                  isRequired={Boolean(options?.AWX_ISOLATION_BASE_PATH)}
                 />
                 <InputField
                   name="SCHEDULE_MAX_JOBS"
-                  config={jobs.SCHEDULE_MAX_JOBS}
-                  type="number"
-                  isRequired
+                  config={jobs.SCHEDULE_MAX_JOBS ?? null}
+                  type={options?.SCHEDULE_MAX_JOBS ? 'number' : undefined}
+                  isRequired={Boolean(options?.SCHEDULE_MAX_JOBS)}
                 />
                 <InputField
                   name="DEFAULT_JOB_TIMEOUT"

awx/ui/src/screens/Setting/Jobs/JobsEdit/JobsEdit.test.js

@@ -122,4 +122,22 @@ describe('<JobsEdit />', () => {
     await waitForElement(wrapper, 'ContentLoading', (el) => el.length === 0);
     expect(wrapper.find('ContentError').length).toBe(1);
   });
+
+  test('Form input fields that are invisible (due to being set manually via a settings file) should not prevent submitting the form', async () => {
+    const mockOptions = Object.assign({}, mockAllOptions);
+    // If AWX_ISOLATION_BASE_PATH has been set in a settings file it will be absent in the PUT options
+    delete mockOptions['actions']['PUT']['AWX_ISOLATION_BASE_PATH'];
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <SettingsProvider value={mockOptions.actions}>
+          <JobsEdit />
+        </SettingsProvider>
+      );
+    });
+    await waitForElement(wrapper, 'ContentLoading', (el) => el.length === 0);
+    await act(async () => {
+      wrapper.find('Form').invoke('onSubmit')();
+    });
+    expect(SettingsAPI.updateAll).toHaveBeenCalledTimes(1);
+  });
 });

awx/ui/src/screens/Setting/shared/SharedFields.js

@@ -397,7 +397,10 @@ const InputField = ({ name, config, type = 'text', isRequired = false }) => {
 };
 InputField.propTypes = {
   name: string.isRequired,
-  config: shape({}).isRequired,
+  config: shape({}),
+};
+InputField.defaultProps = {
+  config: null,
 };
 
 const TextAreaField = ({ name, config, isRequired = false }) => {

awx/ui/src/screens/Template/JobTemplateAdd/JobTemplateAdd.js

@@ -9,29 +9,31 @@ function JobTemplateAdd() {
   const [formSubmitError, setFormSubmitError] = useState(null);
   const history = useHistory();
 
-  const projectParams = {
-    project_id: null,
-    project_name: null,
+  const resourceParams = {
+    resource_id: null,
+    resource_name: null,
+    resource_type: null,
+    resource_kind: null,
   };
   history.location.search
     .replace(/^\?/, '')
     .split('&')
     .map((s) => s.split('='))
     .forEach(([key, val]) => {
-      if (!(key in projectParams)) {
+      if (!(key in resourceParams)) {
         return;
       }
-      projectParams[key] = decodeURIComponent(val);
+      resourceParams[key] = decodeURIComponent(val);
     });
 
-  let projectValues = null;
+  let resourceValues = null;
 
-  if (
-    Object.values(projectParams).filter((item) => item !== null).length === 2
-  ) {
-    projectValues = {
-      id: projectParams.project_id,
-      name: projectParams.project_name,
+  if (history.location.search.includes('resource_id' && 'resource_name')) {
+    resourceValues = {
+      id: resourceParams.resource_id,
+      name: resourceParams.resource_name,
+      type: resourceParams.resource_type,
+      kind: resourceParams.resource_kind, // refers to credential kind
     };
   }
 
@@ -122,7 +124,7 @@ function JobTemplateAdd() {
             handleCancel={handleCancel}
             handleSubmit={handleSubmit}
             submitError={formSubmitError}
-            projectValues={projectValues}
+            resourceValues={resourceValues}
             isOverrideDisabledLookup
           />
         </CardBody>

awx/ui/src/screens/Template/JobTemplateAdd/JobTemplateAdd.test.js

@@ -274,9 +274,14 @@ describe('<JobTemplateAdd />', () => {
   test('should parse and pre-fill project field from query params', async () => {
     const history = createMemoryHistory({
       initialEntries: [
-        '/templates/job_template/add/add?project_id=6&project_name=Demo%20Project',
+        '/templates/job_template/add?resource_id=6&resource_name=Demo%20Project&resource_type=project',
       ],
     });
+    ProjectsAPI.read.mockResolvedValueOnce({
+      count: 1,
+      results: [{ name: 'foo', id: 1, allow_override: true, organization: 1 }],
+    });
+    ProjectsAPI.readOptions.mockResolvedValueOnce({});
     let wrapper;
     await act(async () => {
       wrapper = mountWithContexts(<JobTemplateAdd />, {
@@ -284,8 +289,9 @@ describe('<JobTemplateAdd />', () => {
       });
     });
     await waitForElement(wrapper, 'EmptyStateBody', (el) => el.length === 0);
+
     expect(wrapper.find('input#project').prop('value')).toEqual('Demo Project');
-    expect(ProjectsAPI.readPlaybooks).toBeCalledWith('6');
+    expect(ProjectsAPI.readPlaybooks).toBeCalledWith(6);
   });
 
   test('should not call ProjectsAPI.readPlaybooks if there is no project', async () => {

awx/ui/src/screens/Template/shared/JobTemplateForm.js

@@ -690,7 +690,7 @@ JobTemplateForm.defaultProps = {
 };
 
 const FormikApp = withFormik({
-  mapPropsToValues({ projectValues = {}, template = {} }) {
+  mapPropsToValues({ resourceValues = null, template = {} }) {
     const {
       summary_fields = {
         labels: { results: [] },
@@ -698,7 +698,7 @@ const FormikApp = withFormik({
       },
     } = template;
 
-    return {
+    const initialValues = {
       allow_callbacks: template.allow_callbacks || false,
       allow_simultaneous: template.allow_simultaneous || false,
       ask_credential_on_launch: template.ask_credential_on_launch || false,
@@ -739,7 +739,7 @@ const FormikApp = withFormik({
       playbook: template.playbook || '',
       prevent_instance_group_fallback:
         template.prevent_instance_group_fallback || false,
-      project: summary_fields?.project || projectValues || null,
+      project: summary_fields?.project || null,
       scm_branch: template.scm_branch || '',
       skip_tags: template.skip_tags || '',
       timeout: template.timeout || 0,
@@ -756,6 +756,24 @@ const FormikApp = withFormik({
       execution_environment:
         template.summary_fields?.execution_environment || null,
     };
+    if (resourceValues !== null) {
+      if (resourceValues.type === 'credentials') {
+        initialValues[resourceValues.type] = [
+          {
+            id: parseInt(resourceValues.id, 10),
+            name: resourceValues.name,
+            kind: resourceValues.kind,
+          },
+        ];
+      } else {
+        initialValues[resourceValues.type] = {
+          id: parseInt(resourceValues.id, 10),
+          name: resourceValues.name,
+        };
+      }
+    }
+
+    return initialValues;
   },
   handleSubmit: async (values, { props, setErrors }) => {
     try {

awx/ui/src/screens/WorkflowApproval/WorkflowApprovalList/WorkflowApprovalList.js

@@ -12,11 +12,16 @@ import PaginatedTable, {
 import AlertModal from 'components/AlertModal';
 import ErrorDetail from 'components/ErrorDetail';
 import DataListToolbar from 'components/DataListToolbar';
-import useRequest, { useDeleteItems } from 'hooks/useRequest';
+import useRequest, {
+  useDeleteItems,
+  useDismissableError,
+} from 'hooks/useRequest';
 import useSelected from 'hooks/useSelected';
 import { getQSConfig, parseQueryString } from 'util/qs';
 import WorkflowApprovalListItem from './WorkflowApprovalListItem';
 import useWsWorkflowApprovals from './useWsWorkflowApprovals';
+import WorkflowApprovalListApproveButton from './WorkflowApprovalListApproveButton';
+import WorkflowApprovalListDenyButton from './WorkflowApprovalListDenyButton';
 
 const QS_CONFIG = getQSConfig('workflow_approvals', {
   page: 1,
@@ -104,7 +109,50 @@ function WorkflowApprovalsList() {
     clearSelected();
   };
 
-  const isLoading = isWorkflowApprovalsLoading || isDeleteLoading;
+  const {
+    error: approveApprovalError,
+    isLoading: isApproveLoading,
+    request: approveWorkflowApprovals,
+  } = useRequest(
+    useCallback(
+      async () =>
+        Promise.all(selected.map(({ id }) => WorkflowApprovalsAPI.approve(id))),
+      [selected]
+    ),
+    {}
+  );
+
+  const handleApprove = async () => {
+    await approveWorkflowApprovals();
+    clearSelected();
+  };
+
+  const {
+    error: denyApprovalError,
+    isLoading: isDenyLoading,
+    request: denyWorkflowApprovals,
+  } = useRequest(
+    useCallback(
+      async () =>
+        Promise.all(selected.map(({ id }) => WorkflowApprovalsAPI.deny(id))),
+      [selected]
+    ),
+    {}
+  );
+
+  const handleDeny = async () => {
+    await denyWorkflowApprovals();
+    clearSelected();
+  };
+
+  const { error: actionError, dismissError: dismissActionError } =
+    useDismissableError(approveApprovalError || denyApprovalError);
+
+  const isLoading =
+    isWorkflowApprovalsLoading ||
+    isDeleteLoading ||
+    isApproveLoading ||
+    isDenyLoading;
 
   return (
     <>
@@ -138,6 +186,16 @@ function WorkflowApprovalsList() {
                 onSelectAll={selectAll}
                 qsConfig={QS_CONFIG}
                 additionalControls={[
+                  <WorkflowApprovalListApproveButton
+                    key="approve"
+                    onApprove={handleApprove}
+                    selectedItems={selected}
+                  />,
+                  <WorkflowApprovalListDenyButton
+                    key="deny"
+                    onDeny={handleDeny}
+                    selectedItems={selected}
+                  />,
                   <ToolbarDeleteButton
                     key="delete"
                     onDelete={handleDelete}
@@ -193,6 +251,19 @@ function WorkflowApprovalsList() {
           <ErrorDetail error={deletionError} />
         </AlertModal>
       )}
+      {actionError && (
+        <AlertModal
+          isOpen={actionError}
+          variant="error"
+          title={t`Error!`}
+          onClose={dismissActionError}
+        >
+          {approveApprovalError
+            ? t`Failed to approve one or more workflow approval.`
+            : t`Failed to deny one or more workflow approval.`}
+          <ErrorDetail error={actionError} />
+        </AlertModal>
+      )}
     </>
   );
 }

awx/ui/src/screens/WorkflowApproval/WorkflowApprovalList/WorkflowApprovalListApproveButton.js

@@ -0,0 +1,76 @@
+import React, { useContext } from 'react';
+
+import { t } from '@lingui/macro';
+import PropTypes from 'prop-types';
+import { Button, DropdownItem, Tooltip } from '@patternfly/react-core';
+import { KebabifiedContext } from 'contexts/Kebabified';
+import { WorkflowApproval } from 'types';
+
+function cannotApprove(item) {
+  return !item.can_approve_or_deny;
+}
+
+function WorkflowApprovalListApproveButton({ onApprove, selectedItems }) {
+  const { isKebabified } = useContext(KebabifiedContext);
+
+  const renderTooltip = () => {
+    if (selectedItems.length === 0) {
+      return t`Select a row to approve`;
+    }
+
+    const itemsUnableToApprove = selectedItems
+      .filter(cannotApprove)
+      .map((item) => item.name)
+      .join(', ');
+
+    if (selectedItems.some(cannotApprove)) {
+      return t`You are unable to act on the following workflow approvals: ${itemsUnableToApprove}`;
+    }
+
+    return t`Approve`;
+  };
+
+  const isDisabled =
+    selectedItems.length === 0 || selectedItems.some(cannotApprove);
+
+  return (
+    /* eslint-disable-next-line react/jsx-no-useless-fragment */
+    <>
+      {isKebabified ? (
+        <DropdownItem
+          key="approve"
+          isDisabled={isDisabled}
+          component="button"
+          onClick={onApprove}
+        >
+          {t`Approve`}
+        </DropdownItem>
+      ) : (
+        <Tooltip content={renderTooltip()} position="top">
+          <div>
+            <Button
+              ouiaId="workflow-approval-approve-button"
+              isDisabled={isDisabled}
+              aria-label={t`Approve`}
+              variant="primary"
+              onClick={onApprove}
+            >
+              {t`Approve`}
+            </Button>
+          </div>
+        </Tooltip>
+      )}
+    </>
+  );
+}
+
+WorkflowApprovalListApproveButton.propTypes = {
+  onApprove: PropTypes.func.isRequired,
+  selectedItems: PropTypes.arrayOf(WorkflowApproval),
+};
+
+WorkflowApprovalListApproveButton.defaultProps = {
+  selectedItems: [],
+};
+
+export default WorkflowApprovalListApproveButton;

awx/ui/src/screens/WorkflowApproval/WorkflowApprovalList/WorkflowApprovalListApproveButton.test.js

@@ -0,0 +1,56 @@
+import React from 'react';
+import { mountWithContexts } from '../../../../testUtils/enzymeHelpers';
+import WorkflowApprovalListApproveButton from './WorkflowApprovalListApproveButton';
+
+const workflowApproval = {
+  id: 1,
+  name: 'Foo',
+  can_approve_or_deny: true,
+  url: '/api/v2/workflow_approvals/218/',
+};
+
+describe('<WorkflowApprovalListApproveButton />', () => {
+  test('should render button', () => {
+    const wrapper = mountWithContexts(
+      <WorkflowApprovalListApproveButton
+        onApprove={() => {}}
+        selectedItems={[]}
+      />
+    );
+    expect(wrapper.find('button')).toHaveLength(1);
+  });
+
+  test('should invoke onApprove prop', () => {
+    const onApprove = jest.fn();
+    const wrapper = mountWithContexts(
+      <WorkflowApprovalListApproveButton
+        onApprove={onApprove}
+        selectedItems={[workflowApproval]}
+      />
+    );
+    wrapper.find('button').simulate('click');
+    wrapper.update();
+    expect(onApprove).toHaveBeenCalled();
+  });
+
+  test('should disable button when no approve/deny permissions', () => {
+    const wrapper = mountWithContexts(
+      <WorkflowApprovalListApproveButton
+        onApprove={() => {}}
+        selectedItems={[{ ...workflowApproval, can_approve_or_deny: false }]}
+      />
+    );
+    expect(wrapper.find('button[disabled]')).toHaveLength(1);
+  });
+
+  test('should render tooltip', () => {
+    const wrapper = mountWithContexts(
+      <WorkflowApprovalListApproveButton
+        onApprove={() => {}}
+        selectedItems={[workflowApproval]}
+      />
+    );
+    expect(wrapper.find('Tooltip')).toHaveLength(1);
+    expect(wrapper.find('Tooltip').prop('content')).toEqual('Approve');
+  });
+});

awx/ui/src/screens/WorkflowApproval/WorkflowApprovalList/WorkflowApprovalListDenyButton.js

@@ -0,0 +1,76 @@
+import React, { useContext } from 'react';
+
+import { t } from '@lingui/macro';
+import PropTypes from 'prop-types';
+import { Button, DropdownItem, Tooltip } from '@patternfly/react-core';
+import { KebabifiedContext } from 'contexts/Kebabified';
+import { WorkflowApproval } from 'types';
+
+function cannotDeny(item) {
+  return !item.can_approve_or_deny;
+}
+
+function WorkflowApprovalListDenyButton({ onDeny, selectedItems }) {
+  const { isKebabified } = useContext(KebabifiedContext);
+
+  const renderTooltip = () => {
+    if (selectedItems.length === 0) {
+      return t`Select a row to deny`;
+    }
+
+    const itemsUnableToDeny = selectedItems
+      .filter(cannotDeny)
+      .map((item) => item.name)
+      .join(', ');
+
+    if (selectedItems.some(cannotDeny)) {
+      return t`You are unable to act on the following workflow approvals: ${itemsUnableToDeny}`;
+    }
+
+    return t`Deny`;
+  };
+
+  const isDisabled =
+    selectedItems.length === 0 || selectedItems.some(cannotDeny);
+
+  return (
+    /* eslint-disable-next-line react/jsx-no-useless-fragment */
+    <>
+      {isKebabified ? (
+        <DropdownItem
+          key="deny"
+          isDisabled={isDisabled}
+          component="button"
+          onClick={onDeny}
+        >
+          {t`Deny`}
+        </DropdownItem>
+      ) : (
+        <Tooltip content={renderTooltip()} position="top">
+          <div>
+            <Button
+              ouiaId="workflow-approval-deny-button"
+              isDisabled={isDisabled}
+              aria-label={t`Deny`}
+              variant="danger"
+              onClick={onDeny}
+            >
+              {t`Deny`}
+            </Button>
+          </div>
+        </Tooltip>
+      )}
+    </>
+  );
+}
+
+WorkflowApprovalListDenyButton.propTypes = {
+  onDeny: PropTypes.func.isRequired,
+  selectedItems: PropTypes.arrayOf(WorkflowApproval),
+};
+
+WorkflowApprovalListDenyButton.defaultProps = {
+  selectedItems: [],
+};
+
+export default WorkflowApprovalListDenyButton;

awx/ui/src/screens/WorkflowApproval/WorkflowApprovalList/WorkflowApprovalListDenyButton.test.js

@@ -0,0 +1,53 @@
+import React from 'react';
+import { mountWithContexts } from '../../../../testUtils/enzymeHelpers';
+import WorkflowApprovalListDenyButton from './WorkflowApprovalListDenyButton';
+
+const workflowApproval = {
+  id: 1,
+  name: 'Foo',
+  can_approve_or_deny: true,
+  url: '/api/v2/workflow_approvals/218/',
+};
+
+describe('<WorkflowApprovalListDenyButton />', () => {
+  test('should render button', () => {
+    const wrapper = mountWithContexts(
+      <WorkflowApprovalListDenyButton onDeny={() => {}} selectedItems={[]} />
+    );
+    expect(wrapper.find('button')).toHaveLength(1);
+  });
+
+  test('should invoke onDeny prop', () => {
+    const onDeny = jest.fn();
+    const wrapper = mountWithContexts(
+      <WorkflowApprovalListDenyButton
+        onDeny={onDeny}
+        selectedItems={[workflowApproval]}
+      />
+    );
+    wrapper.find('button').simulate('click');
+    wrapper.update();
+    expect(onDeny).toHaveBeenCalled();
+  });
+
+  test('should disable button when no approve/deny permissions', () => {
+    const wrapper = mountWithContexts(
+      <WorkflowApprovalListDenyButton
+        onDeny={() => {}}
+        selectedItems={[{ ...workflowApproval, can_approve_or_deny: false }]}
+      />
+    );
+    expect(wrapper.find('button[disabled]')).toHaveLength(1);
+  });
+
+  test('should render tooltip', () => {
+    const wrapper = mountWithContexts(
+      <WorkflowApprovalListDenyButton
+        onDeny={() => {}}
+        selectedItems={[workflowApproval]}
+      />
+    );
+    expect(wrapper.find('Tooltip')).toHaveLength(1);
+    expect(wrapper.find('Tooltip').prop('content')).toEqual('Deny');
+  });
+});

awx_collection/meta/runtime.yml

@@ -46,90 +46,216 @@ action_groups:
 plugin_routing:
   inventory:
     tower:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* plugins have been deprecated, use awx.awx.controller instead.
       redirect: awx.awx.controller
   lookup:
     tower_api:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* plugins have been deprecated, use awx.awx.controller_api instead.
       redirect: awx.awx.controller_api
     tower_schedule_rrule:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* plugins have been deprecated, use awx.awx.schedule_rrule instead.
       redirect: awx.awx.schedule_rrule
   modules:
     tower_ad_hoc_command_cancel:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.ad_hoc_command_cancel instead.
       redirect: awx.awx.ad_hoc_command_cancel
     tower_ad_hoc_command_wait:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.ad_hoc_command_wait instead.
       redirect: awx.awx.ad_hoc_command_wait
     tower_ad_hoc_command:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.ad_hoc_command instead.
       redirect: awx.awx.ad_hoc_command
     tower_application:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.application instead.
       redirect: awx.awx.application
     tower_meta:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.controller_meta instead.
       redirect: awx.awx.controller_meta
     tower_credential_input_source:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.credential_input_source instead.
       redirect: awx.awx.credential_input_source
     tower_credential_type:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.credential_type instead.
       redirect: awx.awx.credential_type
     tower_credential:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.credential instead.
       redirect: awx.awx.credential
     tower_execution_environment:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.execution_environment instead.
       redirect: awx.awx.execution_environment
     tower_export:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.export instead.
       redirect: awx.awx.export
     tower_group:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.group instead.
       redirect: awx.awx.group
     tower_host:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.host instead.
       redirect: awx.awx.host
     tower_import:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.import instead.
       redirect: awx.awx.import
     tower_instance_group:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.instance_group instead.
       redirect: awx.awx.instance_group
     tower_inventory_source_update:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.inventory_source_update instead.
       redirect: awx.awx.inventory_source_update
     tower_inventory_source:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.inventory_source instead.
       redirect: awx.awx.inventory_source
     tower_inventory:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.inventory instead.
       redirect: awx.awx.inventory
     tower_job_cancel:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.job_cancel instead.
       redirect: awx.awx.job_cancel
     tower_job_launch:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.job_launch instead.
       redirect: awx.awx.job_launch
     tower_job_list:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.job_list instead.
       redirect: awx.awx.job_list
     tower_job_template:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.job_template instead.
       redirect: awx.awx.job_template
     tower_job_wait:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.job_wait instead.
       redirect: awx.awx.job_wait
     tower_label:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.label instead.
       redirect: awx.awx.label
     tower_license:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.license instead.
       redirect: awx.awx.license
     tower_notification_template:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.notification_template instead.
       redirect: awx.awx.notification_template
     tower_notification:
       redirect: awx.awx.notification_template
     tower_organization:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.organization instead.
       redirect: awx.awx.organization
     tower_project_update:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.project_update instead.
       redirect: awx.awx.project_update
     tower_project:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.project instead.
       redirect: awx.awx.project
     tower_role:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.role instead.
       redirect: awx.awx.role
     tower_schedule:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.schedule instead.
       redirect: awx.awx.schedule
     tower_settings:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.settings instead.
       redirect: awx.awx.settings
     tower_team:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.team instead.
       redirect: awx.awx.team
     tower_token:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.token instead.
       redirect: awx.awx.token
     tower_user:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.user instead.
       redirect: awx.awx.user
     tower_workflow_approval:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.workflow_approval instead.
       redirect: awx.awx.workflow_approval
     tower_workflow_job_template_node:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.workflow_job_template_node instead.
       redirect: awx.awx.workflow_job_template_node
     tower_workflow_job_template:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.workflow_job_template instead.
       redirect: awx.awx.workflow_job_template
     tower_workflow_launch:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.workflow_launch instead.
       redirect: awx.awx.workflow_launch
     tower_workflow_node_wait:
+      deprecation:
+        removal_date: '2022-01-23'
+        warning_text: The tower_* modules have been deprecated, use awx.awx.workflow_node_wait instead.
       redirect: awx.awx.workflow_node_wait

awx_collection/plugins/modules/group.py

@@ -128,7 +128,7 @@ def main():
     description = module.params.get('description')
     state = module.params.pop('state')
     preserve_existing_hosts = module.params.get('preserve_existing_hosts')
-    preserve_existing_children = module.params.get('preserve_existing_groups')
+    preserve_existing_children = module.params.get('preserve_existing_children')
     variables = module.params.get('variables')
 
     # Attempt to look up the related items the user specified (these will fail the module if not found)

awx_collection/plugins/modules/tower_ad_hoc_command.py

@@ -1 +0,0 @@
-ad_hoc_command.py

awx_collection/plugins/modules/tower_ad_hoc_command_cancel.py

@@ -1 +0,0 @@
-ad_hoc_command_cancel.py

awx_collection/plugins/modules/tower_ad_hoc_command_wait.py

@@ -1 +0,0 @@
-ad_hoc_command_wait.py

awx_collection/plugins/modules/tower_application.py

@@ -1 +0,0 @@
-application.py

awx_collection/plugins/modules/tower_controller_meta.py

@@ -1 +0,0 @@
-controller_meta.py

awx_collection/plugins/modules/tower_credential.py

@@ -1 +0,0 @@
-credential.py

awx_collection/plugins/modules/tower_credential_input_source.py

@@ -1 +0,0 @@
-credential_input_source.py

awx_collection/plugins/modules/tower_credential_type.py

@@ -1 +0,0 @@
-credential_type.py

awx_collection/plugins/modules/tower_execution_environment.py

@@ -1 +0,0 @@
-execution_environment.py

awx_collection/plugins/modules/tower_export.py

@@ -1 +0,0 @@
-export.py

awx_collection/plugins/modules/tower_group.py

@@ -1 +0,0 @@
-group.py

awx_collection/plugins/modules/tower_host.py

@@ -1 +0,0 @@
-host.py

awx_collection/plugins/modules/tower_import.py

@@ -1 +0,0 @@
-import.py

awx_collection/plugins/modules/tower_instance_group.py

@@ -1 +0,0 @@
-instance_group.py

awx_collection/plugins/modules/tower_inventory.py

@@ -1 +0,0 @@
-inventory.py

awx_collection/plugins/modules/tower_inventory_source.py

@@ -1 +0,0 @@
-inventory_source.py

awx_collection/plugins/modules/tower_inventory_source_update.py

@@ -1 +0,0 @@
-inventory_source_update.py

awx_collection/plugins/modules/tower_job_cancel.py

@@ -1 +0,0 @@
-job_cancel.py

awx_collection/plugins/modules/tower_job_launch.py

@@ -1 +0,0 @@
-job_launch.py

awx_collection/plugins/modules/tower_job_list.py

@@ -1 +0,0 @@
-job_list.py

awx_collection/plugins/modules/tower_job_template.py

@@ -1 +0,0 @@
-job_template.py

awx_collection/plugins/modules/tower_job_wait.py

@@ -1 +0,0 @@
-job_wait.py

awx_collection/plugins/modules/tower_label.py

@@ -1 +0,0 @@
-label.py

awx_collection/plugins/modules/tower_license.py

@@ -1 +0,0 @@
-license.py

awx_collection/plugins/modules/tower_notification_template.py

@@ -1 +0,0 @@
-notification_template.py

awx_collection/plugins/modules/tower_organization.py

@@ -1 +0,0 @@
-organization.py

awx_collection/plugins/modules/tower_project.py

@@ -1 +0,0 @@
-project.py

awx_collection/plugins/modules/tower_project_update.py

@@ -1 +0,0 @@
-project_update.py

awx_collection/plugins/modules/tower_role.py

@@ -1 +0,0 @@
-role.py

awx_collection/plugins/modules/tower_schedule.py

@@ -1 +0,0 @@
-schedule.py

awx_collection/plugins/modules/tower_settings.py

@@ -1 +0,0 @@
-settings.py

awx_collection/plugins/modules/tower_team.py

@@ -1 +0,0 @@
-team.py

awx_collection/plugins/modules/tower_token.py

@@ -1 +0,0 @@
-token.py

awx_collection/plugins/modules/tower_user.py

@@ -1 +0,0 @@
-user.py

awx_collection/plugins/modules/tower_workflow_approval.py

@@ -1 +0,0 @@
-workflow_approval.py

awx_collection/plugins/modules/tower_workflow_job_template.py

@@ -1 +0,0 @@
-workflow_job_template.py

awx_collection/plugins/modules/tower_workflow_job_template_node.py

@@ -1 +0,0 @@
-workflow_job_template_node.py

awx_collection/plugins/modules/tower_workflow_launch.py

@@ -1 +0,0 @@
-workflow_launch.py

awx_collection/plugins/modules/tower_workflow_node_wait.py

@@ -1 +0,0 @@
-workflow_node_wait.py

awx_collection/plugins/modules/workflow_job_template.py

@@ -19,7 +19,6 @@ author: "John Westcott IV (@john-westcott-iv)"
 short_description: create, update, or destroy Automation Platform Controller workflow job templates.
 description:
     - Create, update, or destroy Automation Platform Controller workflow job templates.
-    - Replaces the deprecated tower_workflow_template module.
     - Use workflow_job_template_node after this, or use the workflow_nodes parameter to build the workflow's graph
 options:
     name:
@@ -614,6 +613,10 @@ def create_workflow_nodes(module, response, workflow_nodes, workflow_id):
                 if workflow_node['unified_job_template']['type'] != 'workflow_approval':
                     module.fail_json(msg="Unable to Find unified_job_template: {0}".format(search_fields))
 
+        inventory = workflow_node.get('inventory')
+        if inventory:
+            workflow_node_fields['inventory'] = module.resolve_name_to_id('inventories', inventory)
+
         # Lookup Values for other fields
 
         for field_name in (

awx_collection/plugins/modules/workflow_job_template_node.py

@@ -20,7 +20,6 @@ short_description: create, update, or destroy Automation Platform Controller wor
 description:
     - Create, update, or destroy Automation Platform Controller workflow job template nodes.
     - Use this to build a graph for a workflow, which dictates what the workflow runs.
-    - Replaces the deprecated tower_workflow_template module schema command.
     - You can create nodes first, and link them afterwards, and not worry about ordering.
       For failsafe referencing of a node, specify identifier, WFJT, and organization.
       With those specified, you can choose to modify or not modify any other parameter.

awx_collection/tools/roles/template_galaxy/templates/README.md.j2

@@ -74,6 +74,7 @@ Notable releases of the `{{ collection_namespace }}.{{ collection_package }}` co
  - 7.0.0 is intended to be identical to the content prior to the migration, aside from changes necessary to function as a collection.
  - 11.0.0 has no non-deprecated modules that depend on the deprecated `tower-cli` [PyPI](https://pypi.org/project/ansible-tower-cli/).
  - 19.2.1 large renaming purged "tower" names (like options and module names), adding redirects for old names
+ - 21.11.0 "tower" modules deprecated and symlinks removed.
  - 0.0.1-devel is the version you should see if installing from source, which is intended for development and expected to be unstable.
 {% else %}
  - 3.7.0 initial release