Merge branch 'stable' into devel

* stable: (275 commits)
  Install correct rpm-sign package in RPM builder
  Updating changelog for 3.1 release
  Switch job_type to check from sync when detecting delete_on_update
  use Unicode apostrophes - not single quotes - for French i18n strings
  pin appdirs==1.4.2
  only cancel deps if we can cancel the inv update
  fixing module_name check and adding support for the debug module
  cancel jobs dependent on inv update
  update tests
  CSS tweaks to workflow results panels
  like inventory updates, check if project update deps already processed
  Revert "Merge pull request #5553 from chrismeyersfsu/fix-waiting_blocked"
  Add awx/ui/client/languages to .gitignore
  Delete awx/ui/client/languages/*.json
  refactor based on review
  Add missing permission check.
  Make current_groups a set to easily avoid duplicates, update asgi-amqp requirement
  avoid duplicated related search fields
  Fix workflow audit items
  fixing module name, json blob, and stdout-for-yum-module on host event
  ...

awx/api/filters.py

@@ -9,9 +9,11 @@ from django.core.exceptions import FieldError, ValidationError
 from django.db import models
 from django.db.models import Q
 from django.db.models.fields import FieldDoesNotExist
-from django.db.models.fields.related import ForeignObjectRel
+from django.db.models.fields.related import ForeignObjectRel, ManyToManyField, ForeignKey
 from django.contrib.contenttypes.models import ContentType
+from django.contrib.contenttypes.fields import GenericForeignKey
 from django.utils.encoding import force_text
+from django.utils.translation import ugettext_lazy as _
 
 # Django REST Framework
 from rest_framework.exceptions import ParseError, PermissionDenied
@@ -88,8 +90,8 @@ class FieldLookupBackend(BaseFilterBackend):
         # those lookups combined with request.user.get_queryset(Model) to make
         # sure user cannot query using objects he could not view.
         new_parts = []
-        for n, name in enumerate(parts[:-1]):
 
+        for name in parts[:-1]:
             # HACK: Make project and inventory source filtering by old field names work for backwards compatibility.
             if model._meta.object_name in ('Project', 'InventorySource'):
                 name = {
@@ -99,15 +101,28 @@ class FieldLookupBackend(BaseFilterBackend):
                     'last_updated': 'last_job_run',
                 }.get(name, name)
 
-            new_parts.append(name)
+            if name == 'type' and 'polymorphic_ctype' in model._meta.get_all_field_names():
+                name = 'polymorphic_ctype'
+                new_parts.append('polymorphic_ctype__model')
+            else:
+                new_parts.append(name)
 
-            
             if name in getattr(model, 'PASSWORD_FIELDS', ()):
-                raise PermissionDenied('Filtering on password fields is not allowed.')
+                raise PermissionDenied(_('Filtering on password fields is not allowed.'))
             elif name == 'pk':
                 field = model._meta.pk
             else:
-                field = model._meta.get_field_by_name(name)[0]
+                name_alt = name.replace("_", "")
+                if name_alt in model._meta.fields_map.keys():
+                    field = model._meta.fields_map[name_alt]
+                    new_parts.pop()
+                    new_parts.append(name_alt)
+                else:
+                    field = model._meta.get_field_by_name(name)[0]
+                if isinstance(field, ForeignObjectRel) and getattr(field.field, '__prevent_search__', False):
+                    raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
+                elif getattr(field, '__prevent_search__', False):
+                    raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
             model = getattr(field, 'related_model', None) or field.model
 
         if parts:
@@ -127,14 +142,20 @@ class FieldLookupBackend(BaseFilterBackend):
             return to_python_boolean(value, allow_none=True)
         elif isinstance(field, models.BooleanField):
             return to_python_boolean(value)
-        elif isinstance(field, ForeignObjectRel):
+        elif isinstance(field, (ForeignObjectRel, ManyToManyField, GenericForeignKey, ForeignKey)):
             return self.to_python_related(value)
         else:
             return field.to_python(value)
 
     def value_to_python(self, model, lookup, value):
         field, new_lookup = self.get_field_from_lookup(model, lookup)
-        if new_lookup.endswith('__isnull'):
+
+        # Type names are stored without underscores internally, but are presented and
+        # and serialized over the API containing underscores so we remove `_`
+        # for polymorphic_ctype__model lookups.
+        if new_lookup.startswith('polymorphic_ctype__model'):
+            value = value.replace('_','')
+        elif new_lookup.endswith('__isnull'):
             value = to_python_boolean(value)
         elif new_lookup.endswith('__in'):
             items = []

awx/api/generics.py

@@ -9,6 +9,7 @@ import time
 # Django
 from django.conf import settings
 from django.db import connection
+from django.db.models.fields import FieldDoesNotExist
 from django.http import QueryDict
 from django.shortcuts import get_object_or_404
 from django.template.loader import render_to_string
@@ -26,6 +27,7 @@ from rest_framework import status
 from rest_framework import views
 
 # AWX
+from awx.api.filters import FieldLookupBackend
 from awx.main.models import *  # noqa
 from awx.main.utils import * # noqa
 from awx.api.serializers import ResourceAccessListElementSerializer
@@ -41,6 +43,7 @@ __all__ = ['APIView', 'GenericAPIView', 'ListAPIView', 'SimpleListAPIView',
            'DeleteLastUnattachLabelMixin',]
 
 logger = logging.getLogger('awx.api.generics')
+analytics_logger = logging.getLogger('awx.analytics.performance')
 
 
 def get_view_name(cls, suffix=None):
@@ -117,6 +120,8 @@ class APIView(views.APIView):
             q_times = [float(q['time']) for q in connection.queries[queries_before:]]
             response['X-API-Query-Count'] = len(q_times)
             response['X-API-Query-Time'] = '%0.3fs' % sum(q_times)
+
+        analytics_logger.info("api response", extra=dict(python_objects=dict(request=request, response=response)))
         return response
 
     def get_authenticate_header(self, request):
@@ -274,22 +279,48 @@ class ListAPIView(generics.ListAPIView, GenericAPIView):
 
     @property
     def related_search_fields(self):
-        fields = []
+        def skip_related_name(name):
+            return (
+                name is None or name.endswith('_role') or name.startswith('_') or
+                name.startswith('deprecated_') or name.endswith('_set') or
+                name == 'polymorphic_ctype')
+
+        fields = set([])
         for field in self.model._meta.fields:
-            if field.name.endswith('_role'):
+            if skip_related_name(field.name):
                 continue
             if getattr(field, 'related_model', None):
-                fields.append('{}__search'.format(field.name))
+                fields.add('{}__search'.format(field.name))
         for rel in self.model._meta.related_objects:
-            name = rel.get_accessor_name()
-            if name.endswith('_set'):
+            name = rel.related_model._meta.verbose_name.replace(" ", "_")
+            if skip_related_name(name):
+                continue
+            fields.add('{}__search'.format(name))
+        m2m_rel = []
+        m2m_rel += self.model._meta.local_many_to_many
+        if issubclass(self.model, UnifiedJobTemplate) and self.model != UnifiedJobTemplate:
+            m2m_rel += UnifiedJobTemplate._meta.local_many_to_many
+        if issubclass(self.model, UnifiedJob) and self.model != UnifiedJob:
+            m2m_rel += UnifiedJob._meta.local_many_to_many
+        for relationship in m2m_rel:
+            if skip_related_name(relationship.name):
                 continue
-            fields.append('{}__search'.format(name))
-        for relationship in self.model._meta.local_many_to_many:
             if relationship.related_model._meta.app_label != 'main':
                 continue
-            fields.append('{}__search'.format(relationship.name))
-        return fields
+            fields.add('{}__search'.format(relationship.name))
+        fields = list(fields)
+
+        allowed_fields = []
+        for field in fields:
+            try:
+                FieldLookupBackend().get_field_from_lookup(self.model, field)
+            except PermissionDenied:
+                pass
+            except FieldDoesNotExist:
+                allowed_fields.append(field)
+            else:
+                allowed_fields.append(field)
+        return allowed_fields
 
 
 class ListCreateAPIView(ListAPIView, generics.ListCreateAPIView):

awx/api/metadata.py

@@ -67,7 +67,10 @@ class Metadata(metadata.SimpleMetadata):
         # Indicate if a field has a default value.
         # FIXME: Still isn't showing all default values?
         try:
-            field_info['default'] = field.get_default()
+            default = field.get_default()
+            if field.field_name == 'TOWER_URL_BASE' and default == 'https://towerhost':
+                default = '{}://{}'.format(self.request.scheme, self.request.get_host())
+            field_info['default'] = default
         except serializers.SkipField:
             pass
 
@@ -120,19 +123,20 @@ class Metadata(metadata.SimpleMetadata):
         actions = {}
         for method in {'GET', 'PUT', 'POST'} & set(view.allowed_methods):
             view.request = clone_request(request, method)
+            obj = None
             try:
                 # Test global permissions
                 if hasattr(view, 'check_permissions'):
                     view.check_permissions(view.request)
                 # Test object permissions
                 if method == 'PUT' and hasattr(view, 'get_object'):
-                    view.get_object()
+                    obj = view.get_object()
             except (exceptions.APIException, PermissionDenied, Http404):
                 continue
             else:
                 # If user has appropriate permissions for the view, include
                 # appropriate metadata about the fields that should be supplied.
-                serializer = view.get_serializer()
+                serializer = view.get_serializer(instance=obj)
                 actions[method] = self.get_serializer_info(serializer)
             finally:
                 view.request = request
@@ -167,6 +171,10 @@ class Metadata(metadata.SimpleMetadata):
         return actions
 
     def determine_metadata(self, request, view):
+        # store request on self so we can use it to generate field defaults
+        # (such as TOWER_URL_BASE)
+        self.request = request
+
         metadata = super(Metadata, self).determine_metadata(request, view)
 
         # Add version number in which view was added to Tower.

awx/api/serializers.py

@@ -42,7 +42,9 @@ from awx.main.constants import SCHEDULEABLE_PROVIDERS
 from awx.main.models import * # noqa
 from awx.main.access import get_user_capabilities
 from awx.main.fields import ImplicitRoleField
-from awx.main.utils import get_type_for_model, get_model_for_type, build_url, timestamp_apiformat, camelcase_to_underscore, getattrd
+from awx.main.utils import (
+    get_type_for_model, get_model_for_type, build_url, timestamp_apiformat,
+    camelcase_to_underscore, getattrd, parse_yaml_or_json)
 from awx.main.validators import vars_validate_or_raise
 
 from awx.conf.license import feature_enabled
@@ -1307,10 +1309,7 @@ class BaseVariableDataSerializer(BaseSerializer):
         if obj is None:
             return {}
         ret = super(BaseVariableDataSerializer, self).to_representation(obj)
-        try:
-            return json.loads(ret.get('variables', '') or '{}')
-        except ValueError:
-            return yaml.safe_load(ret.get('variables', ''))
+        return parse_yaml_or_json(ret.get('variables', '') or '{}')
 
     def to_internal_value(self, data):
         data = {'variables': json.dumps(data)}
@@ -1622,8 +1621,11 @@ class ResourceAccessListElementSerializer(UserSerializer):
                 role_dict['user_capabilities'] = {'unattach': False}
             return { 'role': role_dict, 'descendant_roles': get_roles_on_resource(obj, role)}
 
-        def format_team_role_perm(team_role, permissive_role_ids):
+        def format_team_role_perm(naive_team_role, permissive_role_ids):
             ret = []
+            team_role = naive_team_role
+            if naive_team_role.role_field == 'admin_role':
+                team_role = naive_team_role.content_object.member_role
             for role in team_role.children.filter(id__in=permissive_role_ids).all():
                 role_dict = {
                     'id': role.id,
@@ -1682,11 +1684,11 @@ class ResourceAccessListElementSerializer(UserSerializer):
 
         ret['summary_fields']['direct_access'] \
             = [format_role_perm(r) for r in direct_access_roles.distinct()] \
-            + [y for x in (format_team_role_perm(r, direct_permissive_role_ids) for r in direct_team_roles.distinct()) for y in x]
+            + [y for x in (format_team_role_perm(r, direct_permissive_role_ids) for r in direct_team_roles.distinct()) for y in x] \
+            + [y for x in (format_team_role_perm(r, all_permissive_role_ids) for r in indirect_team_roles.distinct()) for y in x]
 
         ret['summary_fields']['indirect_access'] \
-            = [format_role_perm(r) for r in indirect_access_roles.distinct()] \
-            + [y for x in (format_team_role_perm(r, all_permissive_role_ids) for r in indirect_team_roles.distinct()) for y in x]
+            = [format_role_perm(r) for r in indirect_access_roles.distinct()]
 
         return ret
 

awx/api/views.py

@@ -22,7 +22,7 @@ from django.contrib.auth.models import User, AnonymousUser
 from django.core.cache import cache
 from django.core.urlresolvers import reverse
 from django.core.exceptions import FieldError
-from django.db.models import Q, Count
+from django.db.models import Q, Count, F
 from django.db import IntegrityError, transaction, connection
 from django.shortcuts import get_object_or_404
 from django.utils.encoding import smart_text, force_text
@@ -518,7 +518,7 @@ class AuthView(APIView):
     def get(self, request):
         data = OrderedDict()
         err_backend, err_message = request.session.get('social_auth_error', (None, None))
-        auth_backends = load_backends(settings.AUTHENTICATION_BACKENDS).items()
+        auth_backends = load_backends(settings.AUTHENTICATION_BACKENDS, force_load=True).items()
         # Return auth backends in consistent order: Google, GitHub, SAML.
         auth_backends.sort(key=lambda x: 'g' if x[0] == 'google-oauth2' else x[0])
         for name, backend in auth_backends:
@@ -646,15 +646,16 @@ class OrganizationCountsMixin(object):
             self.request.user, 'read_role').values('organization').annotate(
             Count('organization')).order_by('organization')
 
-        JT_reference = 'project__organization'
-        db_results['job_templates'] = JobTemplate.accessible_objects(
-            self.request.user, 'read_role').exclude(job_type='scan').values(JT_reference).annotate(
-            Count(JT_reference)).order_by(JT_reference)
+        JT_project_reference = 'project__organization'
+        JT_inventory_reference = 'inventory__organization'
+        db_results['job_templates_project'] = JobTemplate.accessible_objects(
+            self.request.user, 'read_role').exclude(
+            project__organization=F(JT_inventory_reference)).values(JT_project_reference).annotate(
+            Count(JT_project_reference)).order_by(JT_project_reference)
 
-        JT_scan_reference = 'inventory__organization'
-        db_results['job_templates_scan'] = JobTemplate.accessible_objects(
-            self.request.user, 'read_role').filter(job_type='scan').values(JT_scan_reference).annotate(
-            Count(JT_scan_reference)).order_by(JT_scan_reference)
+        db_results['job_templates_inventory'] = JobTemplate.accessible_objects(
+            self.request.user, 'read_role').values(JT_inventory_reference).annotate(
+            Count(JT_inventory_reference)).order_by(JT_inventory_reference)
 
         db_results['projects'] = project_qs\
             .values('organization').annotate(Count('organization')).order_by('organization')
@@ -672,16 +673,16 @@ class OrganizationCountsMixin(object):
                 'inventories': 0, 'teams': 0, 'users': 0, 'job_templates': 0,
                 'admins': 0, 'projects': 0}
 
-        for res in db_results:
-            if res == 'job_templates':
-                org_reference = JT_reference
-            elif res == 'job_templates_scan':
-                org_reference = JT_scan_reference
+        for res, count_qs in db_results.items():
+            if res == 'job_templates_project':
+                org_reference = JT_project_reference
+            elif res == 'job_templates_inventory':
+                org_reference = JT_inventory_reference
             elif res == 'users':
                 org_reference = 'id'
             else:
                 org_reference = 'organization'
-            for entry in db_results[res]:
+            for entry in count_qs:
                 org_id = entry[org_reference]
                 if org_id in count_context:
                     if res == 'users':
@@ -690,11 +691,13 @@ class OrganizationCountsMixin(object):
                         continue
                     count_context[org_id][res] = entry['%s__count' % org_reference]
 
-        # Combine the counts for job templates with scan job templates
+        # Combine the counts for job templates by project and inventory
         for org in org_id_list:
             org_id = org['id']
-            if 'job_templates_scan' in count_context[org_id]:
-                count_context[org_id]['job_templates'] += count_context[org_id].pop('job_templates_scan')
+            count_context[org_id]['job_templates'] = 0
+            for related_path in ['job_templates_project', 'job_templates_inventory']:
+                if related_path in count_context[org_id]:
+                    count_context[org_id]['job_templates'] += count_context[org_id].pop(related_path)
 
         full_context['related_field_counts'] = count_context
 
@@ -1865,6 +1868,16 @@ class GroupChildrenList(EnforceParentRelationshipMixin, SubListCreateAttachDetac
     relationship = 'children'
     enforce_parent_relationship = 'inventory'
 
+    def unattach(self, request, *args, **kwargs):
+        sub_id = request.data.get('id', None)
+        if sub_id is not None:
+            return super(GroupChildrenList, self).unattach(request, *args, **kwargs)
+        parent = self.get_parent_object()
+        if not request.user.can_access(self.model, 'delete', parent):
+            raise PermissionDenied()
+        parent.delete()
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
 
 class GroupPotentialChildrenList(SubListAPIView):
 
@@ -2484,7 +2497,7 @@ class JobTemplateSurveySpec(GenericAPIView):
                 return Response(dict(error=_("'required' missing from survey question %s.") % str(idx)), status=status.HTTP_400_BAD_REQUEST)
 
             if survey_item["type"] == "password":
-                if "default" in survey_item and survey_item["default"].startswith('$encrypted$'):
+                if survey_item.get("default") and survey_item["default"].startswith('$encrypted$'):
                     old_spec = obj.survey_spec
                     for old_item in old_spec['spec']:
                         if old_item['variable'] == survey_item['variable']:
@@ -3039,6 +3052,9 @@ class WorkflowJobTemplateWorkflowNodesList(WorkflowsEnforcementMixin, SubListCre
             data[fd] = None
         return super(WorkflowJobTemplateWorkflowNodesList, self).update_raw_data(data)
 
+    def get_queryset(self):
+        return super(WorkflowJobTemplateWorkflowNodesList, self).get_queryset().order_by('id')
+
 
 class WorkflowJobTemplateJobsList(WorkflowsEnforcementMixin, SubListAPIView):
 
@@ -3149,6 +3165,9 @@ class WorkflowJobWorkflowNodesList(WorkflowsEnforcementMixin, SubListAPIView):
     parent_key = 'workflow_job'
     new_in_310 = True
 
+    def get_queryset(self):
+        return super(WorkflowJobWorkflowNodesList, self).get_queryset().order_by('id')
+
 
 class WorkflowJobCancel(WorkflowsEnforcementMixin, RetrieveAPIView):