From 21c6dd6b1eb3e8f6aa84386c66e6e1528cabba40 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 18 Oct 2016 17:15:26 -0400 Subject: [PATCH] move survey models to mixins, start WFJT launch endpoint --- awx/api/serializers.py | 90 +++++++++++++++++++ awx/api/views.py | 27 ++++-- awx/main/models/jobs.py | 6 +- awx/main/models/mixins.py | 154 +++++++++++++++++++++++++++++++- awx/main/models/unified_jobs.py | 153 +------------------------------ awx/main/models/workflow.py | 13 +-- 6 files changed, 276 insertions(+), 167 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 2409dc1e74..f3e8419752 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2607,6 +2607,96 @@ class JobLaunchSerializer(BaseSerializer): obj.credential = JT_credential return attrs +class WorkflowJobLaunchSerializer(BaseSerializer): + + can_start_without_user_input = serializers.BooleanField(read_only=True) + variables_needed_to_start = serializers.ReadOnlyField() + + survey_enabled = serializers.SerializerMethodField() + extra_vars = VerbatimField(required=False, write_only=True) + workflow_job_template_data = serializers.SerializerMethodField() + + class Meta: + model = WorkflowJobTemplate + fields = ('can_start_without_user_input', + 'extra_vars', + 'survey_enabled', 'variables_needed_to_start', + 'workflow_job_template_data') + + def get_survey_enabled(self, obj): + if obj: + return obj.survey_enabled and 'spec' in obj.survey_spec + return False + + def get_workflow_job_template_data(self, obj): + return dict(name=obj.name, id=obj.id, description=obj.description) + + def validate(self, attrs): + errors = {} + obj = self.context.get('obj') + data = self.context.get('data') + + for field in obj.resources_needed_to_start: + if not (attrs.get(field, False) and obj._ask_for_vars_dict().get(field, False)): + errors[field] = "Job Template '%s' is missing or undefined." % field + + if (not obj.ask_credential_on_launch) or (not attrs.get('credential', None)): + credential = obj.credential + else: + credential = attrs.get('credential', None) + + # fill passwords dict with request data passwords + if credential and credential.passwords_needed: + passwords = self.context.get('passwords') + try: + for p in credential.passwords_needed: + passwords[p] = data[p] + except KeyError: + errors['passwords_needed_to_start'] = credential.passwords_needed + + extra_vars = attrs.get('extra_vars', {}) + + if isinstance(extra_vars, basestring): + try: + extra_vars = json.loads(extra_vars) + except (ValueError, TypeError): + try: + extra_vars = yaml.safe_load(extra_vars) + assert isinstance(extra_vars, dict) + except (yaml.YAMLError, TypeError, AttributeError, AssertionError): + errors['extra_vars'] = 'Must be a valid JSON or YAML dictionary.' + + if not isinstance(extra_vars, dict): + extra_vars = {} + + if self.get_survey_enabled(obj): + validation_errors = obj.survey_variable_validation(extra_vars) + if validation_errors: + errors['variables_needed_to_start'] = validation_errors + + # Special prohibited cases for scan jobs + errors.update(obj._extra_job_type_errors(data)) + + if errors: + raise serializers.ValidationError(errors) + + JT_extra_vars = obj.extra_vars + JT_limit = obj.limit + JT_job_type = obj.job_type + JT_job_tags = obj.job_tags + JT_skip_tags = obj.skip_tags + JT_inventory = obj.inventory + JT_credential = obj.credential + attrs = super(JobLaunchSerializer, self).validate(attrs) + obj.extra_vars = JT_extra_vars + obj.limit = JT_limit + obj.job_type = JT_job_type + obj.skip_tags = JT_skip_tags + obj.job_tags = JT_job_tags + obj.inventory = JT_inventory + obj.credential = JT_credential + return attrs + class NotificationTemplateSerializer(BaseSerializer): show_capabilities = ['edit', 'delete'] diff --git a/awx/api/views.py b/awx/api/views.py index 0ed269bc08..4431a6e825 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2790,21 +2790,36 @@ class WorkflowJobTemplateLabelList(JobTemplateLabelList): class WorkflowJobTemplateLaunch(GenericAPIView): model = WorkflowJobTemplate - serializer_class = EmptySerializer + serializer_class = WorkflowJobLaunchSerializer new_in_310 = True - def get(self, request, *args, **kwargs): - data = {} + def update_raw_data(self, data): obj = self.get_object() - data['warnings'] = obj.get_warnings() - data['passwords_needed_to_start'] = obj.passwords_needed_to_start - return Response(data) + extra_vars = data.pop('extra_vars', None) or {} + if obj: + for v in obj.variables_needed_to_start: + extra_vars.setdefault(v, u'') + if extra_vars: + data['extra_vars'] = extra_vars + return data + + # def get(self, request, *args, **kwargs): + # data = {} + # obj = self.get_object() + # data['warnings'] = obj.get_warnings() + # data['variables_needed_to_start'] = obj.variables_needed_to_start + # return Response(data) def post(self, request, *args, **kwargs): obj = self.get_object() if not request.user.can_access(self.model, 'start', obj): raise PermissionDenied() + +# serializer = self.serializer_class(instance=obj, data=request.data, context={'obj': obj, 'data': request.data, 'passwords': passwords}) +# if not serializer.is_valid(): +# return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + prompted_fields, ignored_fields = obj._accept_or_ignore_job_kwargs(**request.data) new_job = obj.create_unified_job(**prompted_fields) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index a5bc0d0a27..05f02bd5af 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -39,7 +39,7 @@ from awx.main.utils import ( ) from awx.main.redact import PlainTextCleaner from awx.main.fields import ImplicitRoleField -from awx.main.models.mixins import ResourceMixin +from awx.main.models.mixins import ResourceMixin, SurveyJobTemplateMixin, SurveyJobMixin from awx.main.models.base import PERM_INVENTORY_SCAN from awx.main.consumers import emit_channel_notification @@ -191,7 +191,7 @@ class JobOptions(BaseModel): else: return [] -class JobTemplate(UnifiedJobTemplate, SurveyJobTemplate, JobOptions, ResourceMixin): +class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin): ''' A job template is a reusable job definition for applying a project (with playbook) to an inventory source with a given credential. @@ -395,7 +395,7 @@ class JobTemplate(UnifiedJobTemplate, SurveyJobTemplate, JobOptions, ResourceMix any_notification_templates = set(any_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_any=self.project.organization))) return dict(error=list(error_notification_templates), success=list(success_notification_templates), any=list(any_notification_templates)) -class Job(UnifiedJob, SurveyJob, JobOptions, JobNotificationMixin): +class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin): ''' A job applies a project (with playbook) to an inventory source with a given credential. It represents a single invocation of ansible-playbook with the diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index 9cb6b18b77..39675cae7c 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -10,7 +10,7 @@ from awx.main.models.rbac import ( ) -__all__ = ['ResourceMixin'] +__all__ = ['ResourceMixin', 'SurveyJobTemplateMixin', 'SurveyJobMixin'] class ResourceMixin(models.Model): @@ -61,3 +61,155 @@ class ResourceMixin(models.Model): return get_roles_on_resource(self, accessor) + +class SurveyJobTemplateMixin(models.Model): + class Meta: + abstract = True + + survey_enabled = models.BooleanField( + default=False, + ) + survey_spec = JSONField( + blank=True, + default={}, + ) + + def survey_password_variables(self): + vars = [] + if self.survey_enabled and 'spec' in self.survey_spec: + # Get variables that are type password + for survey_element in self.survey_spec['spec']: + if survey_element['type'] == 'password': + vars.append(survey_element['variable']) + return vars + + @property + def variables_needed_to_start(self): + vars = [] + if self.survey_enabled and 'spec' in self.survey_spec: + for survey_element in self.survey_spec['spec']: + if survey_element['required']: + vars.append(survey_element['variable']) + return vars + + def _update_unified_job_kwargs(self, **kwargs): + ''' + Combine extra_vars with variable precedence order: + JT extra_vars -> JT survey defaults -> runtime extra_vars + ''' + if 'launch_type' in kwargs and kwargs['launch_type'] == 'relaunch': + return kwargs + + # Job Template extra_vars + extra_vars = self.extra_vars_dict + + # Overwrite with job template extra vars with survey default vars + if self.survey_enabled and 'spec' in self.survey_spec: + for survey_element in self.survey_spec.get("spec", []): + if 'default' in survey_element and survey_element['default']: + extra_vars[survey_element['variable']] = survey_element['default'] + + # transform to dict + if 'extra_vars' in kwargs: + kwargs_extra_vars = kwargs['extra_vars'] + kwargs_extra_vars = parse_yaml_or_json(kwargs_extra_vars) + else: + kwargs_extra_vars = {} + + # Overwrite job template extra vars with explicit job extra vars + # and add on job extra vars + extra_vars.update(kwargs_extra_vars) + kwargs['extra_vars'] = json.dumps(extra_vars) + return kwargs + + def survey_variable_validation(self, data): + errors = [] + if not self.survey_enabled: + return errors + if 'name' not in self.survey_spec: + errors.append("'name' missing from survey spec.") + if 'description' not in self.survey_spec: + errors.append("'description' missing from survey spec.") + for survey_element in self.survey_spec.get("spec", []): + if survey_element['variable'] not in data and \ + survey_element['required']: + errors.append("'%s' value missing" % survey_element['variable']) + elif survey_element['type'] in ["textarea", "text", "password"]: + if survey_element['variable'] in data: + if type(data[survey_element['variable']]) not in (str, unicode): + errors.append("Value %s for '%s' expected to be a string." % (data[survey_element['variable']], + survey_element['variable'])) + continue + if 'min' in survey_element and survey_element['min'] not in ["", None] and len(data[survey_element['variable']]) < int(survey_element['min']): + errors.append("'%s' value %s is too small (length is %s must be at least %s)." % + (survey_element['variable'], data[survey_element['variable']], len(data[survey_element['variable']]), survey_element['min'])) + if 'max' in survey_element and survey_element['max'] not in ["", None] and len(data[survey_element['variable']]) > int(survey_element['max']): + errors.append("'%s' value %s is too large (must be no more than %s)." % + (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) + elif survey_element['type'] == 'integer': + if survey_element['variable'] in data: + if type(data[survey_element['variable']]) != int: + errors.append("Value %s for '%s' expected to be an integer." % (data[survey_element['variable']], + survey_element['variable'])) + continue + if 'min' in survey_element and survey_element['min'] not in ["", None] and survey_element['variable'] in data and \ + data[survey_element['variable']] < int(survey_element['min']): + errors.append("'%s' value %s is too small (must be at least %s)." % + (survey_element['variable'], data[survey_element['variable']], survey_element['min'])) + if 'max' in survey_element and survey_element['max'] not in ["", None] and survey_element['variable'] in data and \ + data[survey_element['variable']] > int(survey_element['max']): + errors.append("'%s' value %s is too large (must be no more than %s)." % + (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) + elif survey_element['type'] == 'float': + if survey_element['variable'] in data: + if type(data[survey_element['variable']]) not in (float, int): + errors.append("Value %s for '%s' expected to be a numeric type." % (data[survey_element['variable']], + survey_element['variable'])) + continue + if 'min' in survey_element and survey_element['min'] not in ["", None] and data[survey_element['variable']] < float(survey_element['min']): + errors.append("'%s' value %s is too small (must be at least %s)." % + (survey_element['variable'], data[survey_element['variable']], survey_element['min'])) + if 'max' in survey_element and survey_element['max'] not in ["", None] and data[survey_element['variable']] > float(survey_element['max']): + errors.append("'%s' value %s is too large (must be no more than %s)." % + (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) + elif survey_element['type'] == 'multiselect': + if survey_element['variable'] in data: + if type(data[survey_element['variable']]) != list: + errors.append("'%s' value is expected to be a list." % survey_element['variable']) + else: + for val in data[survey_element['variable']]: + if val not in survey_element['choices']: + errors.append("Value %s for '%s' expected to be one of %s." % (val, survey_element['variable'], + survey_element['choices'])) + elif survey_element['type'] == 'multiplechoice': + if survey_element['variable'] in data: + if data[survey_element['variable']] not in survey_element['choices']: + errors.append("Value %s for '%s' expected to be one of %s." % (data[survey_element['variable']], + survey_element['variable'], + survey_element['choices'])) + return errors + + +class SurveyJobMixin(models.Model): + class Meta: + abstract = True + + survey_passwords = JSONField( + blank=True, + default={}, + editable=False, + ) + + def display_extra_vars(self): + ''' + Hides fields marked as passwords in survey. + ''' + if self.survey_passwords: + extra_vars = json.loads(self.extra_vars) + for key, value in self.survey_passwords.items(): + if key in extra_vars: + extra_vars[key] = value + return json.dumps(extra_vars) + else: + return self.extra_vars + diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index da37caeebd..e9925a6b17 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -36,7 +36,7 @@ from awx.main.utils import decrypt_field, _inventory_updates from awx.main.redact import UriCleaner, REPLACE_STR from awx.main.consumers import emit_channel_notification -__all__ = ['UnifiedJobTemplate', 'UnifiedJob', 'SurveyJobTemplate', 'SurveyJob'] +__all__ = ['UnifiedJobTemplate', 'UnifiedJob'] logger = logging.getLogger('awx.main.models.unified_jobs') @@ -937,154 +937,3 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique if settings.BROKER_URL.startswith('amqp://'): self._force_cancel() return self.cancel_flag - -class SurveyJobTemplate(models.Model): - class Meta: - abstract = True - - survey_enabled = models.BooleanField( - default=False, - ) - survey_spec = JSONField( - blank=True, - default={}, - ) - - def survey_password_variables(self): - vars = [] - if self.survey_enabled and 'spec' in self.survey_spec: - # Get variables that are type password - for survey_element in self.survey_spec['spec']: - if survey_element['type'] == 'password': - vars.append(survey_element['variable']) - return vars - - @property - def variables_needed_to_start(self): - vars = [] - if self.survey_enabled and 'spec' in self.survey_spec: - for survey_element in self.survey_spec['spec']: - if survey_element['required']: - vars.append(survey_element['variable']) - return vars - - def _update_unified_job_kwargs(self, **kwargs): - ''' - Combine extra_vars with variable precedence order: - JT extra_vars -> JT survey defaults -> runtime extra_vars - ''' - if 'launch_type' in kwargs and kwargs['launch_type'] == 'relaunch': - return kwargs - - # Job Template extra_vars - extra_vars = self.extra_vars_dict - - # Overwrite with job template extra vars with survey default vars - if self.survey_enabled and 'spec' in self.survey_spec: - for survey_element in self.survey_spec.get("spec", []): - if 'default' in survey_element and survey_element['default']: - extra_vars[survey_element['variable']] = survey_element['default'] - - # transform to dict - if 'extra_vars' in kwargs: - kwargs_extra_vars = kwargs['extra_vars'] - kwargs_extra_vars = parse_yaml_or_json(kwargs_extra_vars) - else: - kwargs_extra_vars = {} - - # Overwrite job template extra vars with explicit job extra vars - # and add on job extra vars - extra_vars.update(kwargs_extra_vars) - kwargs['extra_vars'] = json.dumps(extra_vars) - return kwargs - - def survey_variable_validation(self, data): - errors = [] - if not self.survey_enabled: - return errors - if 'name' not in self.survey_spec: - errors.append("'name' missing from survey spec.") - if 'description' not in self.survey_spec: - errors.append("'description' missing from survey spec.") - for survey_element in self.survey_spec.get("spec", []): - if survey_element['variable'] not in data and \ - survey_element['required']: - errors.append("'%s' value missing" % survey_element['variable']) - elif survey_element['type'] in ["textarea", "text", "password"]: - if survey_element['variable'] in data: - if type(data[survey_element['variable']]) not in (str, unicode): - errors.append("Value %s for '%s' expected to be a string." % (data[survey_element['variable']], - survey_element['variable'])) - continue - if 'min' in survey_element and survey_element['min'] not in ["", None] and len(data[survey_element['variable']]) < int(survey_element['min']): - errors.append("'%s' value %s is too small (length is %s must be at least %s)." % - (survey_element['variable'], data[survey_element['variable']], len(data[survey_element['variable']]), survey_element['min'])) - if 'max' in survey_element and survey_element['max'] not in ["", None] and len(data[survey_element['variable']]) > int(survey_element['max']): - errors.append("'%s' value %s is too large (must be no more than %s)." % - (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) - elif survey_element['type'] == 'integer': - if survey_element['variable'] in data: - if type(data[survey_element['variable']]) != int: - errors.append("Value %s for '%s' expected to be an integer." % (data[survey_element['variable']], - survey_element['variable'])) - continue - if 'min' in survey_element and survey_element['min'] not in ["", None] and survey_element['variable'] in data and \ - data[survey_element['variable']] < int(survey_element['min']): - errors.append("'%s' value %s is too small (must be at least %s)." % - (survey_element['variable'], data[survey_element['variable']], survey_element['min'])) - if 'max' in survey_element and survey_element['max'] not in ["", None] and survey_element['variable'] in data and \ - data[survey_element['variable']] > int(survey_element['max']): - errors.append("'%s' value %s is too large (must be no more than %s)." % - (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) - elif survey_element['type'] == 'float': - if survey_element['variable'] in data: - if type(data[survey_element['variable']]) not in (float, int): - errors.append("Value %s for '%s' expected to be a numeric type." % (data[survey_element['variable']], - survey_element['variable'])) - continue - if 'min' in survey_element and survey_element['min'] not in ["", None] and data[survey_element['variable']] < float(survey_element['min']): - errors.append("'%s' value %s is too small (must be at least %s)." % - (survey_element['variable'], data[survey_element['variable']], survey_element['min'])) - if 'max' in survey_element and survey_element['max'] not in ["", None] and data[survey_element['variable']] > float(survey_element['max']): - errors.append("'%s' value %s is too large (must be no more than %s)." % - (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) - elif survey_element['type'] == 'multiselect': - if survey_element['variable'] in data: - if type(data[survey_element['variable']]) != list: - errors.append("'%s' value is expected to be a list." % survey_element['variable']) - else: - for val in data[survey_element['variable']]: - if val not in survey_element['choices']: - errors.append("Value %s for '%s' expected to be one of %s." % (val, survey_element['variable'], - survey_element['choices'])) - elif survey_element['type'] == 'multiplechoice': - if survey_element['variable'] in data: - if data[survey_element['variable']] not in survey_element['choices']: - errors.append("Value %s for '%s' expected to be one of %s." % (data[survey_element['variable']], - survey_element['variable'], - survey_element['choices'])) - return errors - - -class SurveyJob(models.Model): - class Meta: - abstract = True - - survey_passwords = JSONField( - blank=True, - default={}, - editable=False, - ) - - def display_extra_vars(self): - ''' - Hides fields marked as passwords in survey. - ''' - if self.survey_passwords: - extra_vars = json.loads(self.extra_vars) - for key, value in self.survey_passwords.items(): - if key in extra_vars: - extra_vars[key] = value - return json.dumps(extra_vars) - else: - return self.extra_vars diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index a25bfebb3c..70e34e64eb 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -13,7 +13,6 @@ from jsonfield import JSONField # AWX from awx.main.models import UnifiedJobTemplate, UnifiedJob -from awx.main.models.unified_jobs import SurveyJobTemplate, SurveyJob from awx.main.models.notifications import JobNotificationMixin from awx.main.models.base import BaseModel, CreatedModifiedModel, VarsDictProperty from awx.main.models.rbac import ( @@ -21,7 +20,7 @@ from awx.main.models.rbac import ( ROLE_SINGLETON_SYSTEM_AUDITOR ) from awx.main.fields import ImplicitRoleField -from awx.main.models.mixins import ResourceMixin +from awx.main.models.mixins import ResourceMixin, SurveyJobTemplateMixin, SurveyJobMixin from awx.main.redact import REPLACE_STR from awx.main.utils import parse_yaml_or_json @@ -264,7 +263,7 @@ class WorkflowJobOptions(BaseModel): extra_vars_dict = VarsDictProperty('extra_vars', True) -class WorkflowJobTemplate(UnifiedJobTemplate, SurveyJobTemplate, WorkflowJobOptions, ResourceMixin): +class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTemplateMixin, ResourceMixin): class Meta: app_label = 'main' @@ -294,7 +293,7 @@ class WorkflowJobTemplate(UnifiedJobTemplate, SurveyJobTemplate, WorkflowJobOpti @classmethod def _get_unified_job_field_names(cls): - return ['name', 'description', 'extra_vars', 'labels', 'schedule', 'launch_type'] + return ['name', 'description', 'extra_vars', 'labels', 'survey_passwords', 'schedule', 'launch_type'] def get_absolute_url(self): return reverse('api:workflow_job_template_detail', args=(self.pk,)) @@ -341,6 +340,10 @@ class WorkflowJobTemplate(UnifiedJobTemplate, SurveyJobTemplate, WorkflowJobOpti return prompted_fields, ignored_fields + def can_start_without_user_input(self): + '''Return whether WFJT can be launched without survey passwords.''' + return bool(self.variables_needed_to_start) + def get_warnings(self): warning_data = {} for node in self.workflow_job_template_nodes.all(): @@ -395,7 +398,7 @@ class WorkflowJobInheritNodesMixin(object): self._inherit_relationship(old_node, new_node, node_ids_map, node_type) -class WorkflowJob(UnifiedJob, SurveyJob, WorkflowJobOptions, JobNotificationMixin, WorkflowJobInheritNodesMixin): +class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificationMixin, WorkflowJobInheritNodesMixin): class Meta: app_label = 'main'