zware/awx
1
0
Fork 0
mirror of https://github.com/ZwareBear/awx.git synced 2026-04-29 11:21:49 -05:00
Code Issues Packages Projects Releases Wiki Activity

move code linting to a stricter pep8-esque auto-formatting tool, black

Browse Source
This commit is contained in:
Ryan Petrello
2021-03-19 12:44:51 -04:00
parent 9b702e46fe
commit c2ef0a6500
671 changed files with 20538 additions and 21924 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
awx/main/models/__init__.py
View File

@@ -2,77 +2,79 @@
# All Rights Reserved.
# Django
from django.conf import settings # noqa
from django.conf import settings # noqa
from django.db import connection
from django.db.models.signals import pre_delete # noqa
# AWX
from awx.main.models.base import ( # noqa
BaseModel, PrimordialModel, prevent_search, accepts_json,
CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES
)
from awx.main.models.unified_jobs import ( # noqa
UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded
)
from awx.main.models.organization import ( # noqa
Organization, Profile, Team, UserSessionMembership
)
from awx.main.models.credential import ( # noqa
Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env
)
from awx.main.models.base import BaseModel, PrimordialModel, prevent_search, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES # noqa
from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded # noqa
from awx.main.models.organization import Organization, Profile, Team, UserSessionMembership # noqa
from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env # noqa
from awx.main.models.projects import Project, ProjectUpdate # noqa
from awx.main.models.inventory import ( # noqa
CustomInventoryScript, Group, Host, Inventory, InventorySource,
InventoryUpdate, SmartInventoryMembership
)
from awx.main.models.inventory import CustomInventoryScript, Group, Host, Inventory, InventorySource, InventoryUpdate, SmartInventoryMembership # noqa
from awx.main.models.jobs import ( # noqa
Job, JobHostSummary, JobLaunchConfig, JobTemplate, SystemJob,
Job,
JobHostSummary,
JobLaunchConfig,
JobTemplate,
SystemJob,
SystemJobTemplate,
)
from awx.main.models.events import ( # noqa
AdHocCommandEvent, InventoryUpdateEvent, JobEvent, ProjectUpdateEvent,
AdHocCommandEvent,
InventoryUpdateEvent,
JobEvent,
ProjectUpdateEvent,
SystemJobEvent,
)
from awx.main.models.ad_hoc_commands import AdHocCommand # noqa
from awx.main.models.schedules import Schedule # noqa
from awx.main.models.execution_environments import ExecutionEnvironment # noqa
from awx.main.models.activity_stream import ActivityStream # noqa
from awx.main.models.ad_hoc_commands import AdHocCommand # noqa
from awx.main.models.schedules import Schedule # noqa
from awx.main.models.execution_environments import ExecutionEnvironment # noqa
from awx.main.models.activity_stream import ActivityStream # noqa
from awx.main.models.ha import ( # noqa
Instance, InstanceGroup, TowerScheduleState,
Instance,
InstanceGroup,
TowerScheduleState,
)
from awx.main.models.rbac import ( # noqa
Role, batch_role_ancestor_rebuilding, get_roles_on_resource,
role_summary_fields_generator, ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
Role,
batch_role_ancestor_rebuilding,
get_roles_on_resource,
role_summary_fields_generator,
ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
ROLE_SINGLETON_SYSTEM_AUDITOR,
)
from awx.main.models.mixins import ( # noqa
CustomVirtualEnvMixin, ExecutionEnvironmentMixin, ResourceMixin, SurveyJobMixin,
SurveyJobTemplateMixin, TaskManagerInventoryUpdateMixin,
TaskManagerJobMixin, TaskManagerProjectUpdateMixin,
CustomVirtualEnvMixin,
ExecutionEnvironmentMixin,
ResourceMixin,
SurveyJobMixin,
SurveyJobTemplateMixin,
TaskManagerInventoryUpdateMixin,
TaskManagerJobMixin,
TaskManagerProjectUpdateMixin,
TaskManagerUnifiedJobMixin,
)
from awx.main.models.notifications import ( # noqa
Notification, NotificationTemplate,
JobNotificationMixin
)
from awx.main.models.label import Label # noqa
from awx.main.models.notifications import Notification, NotificationTemplate, JobNotificationMixin # noqa
from awx.main.models.label import Label # noqa
from awx.main.models.workflow import ( # noqa
WorkflowJob, WorkflowJobNode, WorkflowJobOptions, WorkflowJobTemplate,
WorkflowJobTemplateNode, WorkflowApproval, WorkflowApprovalTemplate,
WorkflowJob,
WorkflowJobNode,
WorkflowJobOptions,
WorkflowJobTemplate,
WorkflowJobTemplateNode,
WorkflowApproval,
WorkflowApprovalTemplate,
)
from awx.api.versioning import reverse
from awx.main.models.oauth import ( # noqa
OAuth2AccessToken, OAuth2Application
)
from oauth2_provider.models import Grant, RefreshToken # noqa -- needed django-oauth-toolkit model migrations
from awx.main.models.oauth import OAuth2AccessToken, OAuth2Application # noqa
from oauth2_provider.models import Grant, RefreshToken # noqa -- needed django-oauth-toolkit model migrations
# Add custom methods to User model for permissions checks.
from django.contrib.auth.models import User # noqa
from awx.main.access import ( # noqa
get_user_queryset, check_user_access, check_user_access_with_errors,
user_accessible_objects
)
from awx.main.access import get_user_queryset, check_user_access, check_user_access_with_errors, user_accessible_objects # noqa
User.add_to_class('get_queryset', get_user_queryset)
@@ -93,18 +95,12 @@ def enforce_bigint_pk_migration():
# from the *old* int primary key table to the replacement bigint table
# if not, attempt to migrate them in the background
#
for tblname in (
'main_jobevent', 'main_inventoryupdateevent',
'main_projectupdateevent', 'main_adhoccommandevent',
'main_systemjobevent'
):
for tblname in ('main_jobevent', 'main_inventoryupdateevent', 'main_projectupdateevent', 'main_adhoccommandevent', 'main_systemjobevent'):
with connection.cursor() as cursor:
cursor.execute(
'SELECT 1 FROM information_schema.tables WHERE table_name=%s',
(f'_old_{tblname}',)
)
cursor.execute('SELECT 1 FROM information_schema.tables WHERE table_name=%s', (f'_old_{tblname}',))
if bool(cursor.rowcount):
from awx.main.tasks import migrate_legacy_event_data
migrate_legacy_event_data.apply_async([tblname])
@@ -150,8 +146,7 @@ User.add_to_class('created', created)
def user_is_system_auditor(user):
if not hasattr(user, '_is_system_auditor'):
if user.pk:
user._is_system_auditor = user.roles.filter(
singleton_name='system_auditor', role_field='system_auditor').exists()
user._is_system_auditor = user.roles.filter(singleton_name='system_auditor', role_field='system_auditor').exists()
else:
# Odd case where user is unsaved, this should never be relied on
return False
@@ -195,8 +190,6 @@ def user_is_in_enterprise_category(user, category):
User.add_to_class('is_in_enterprise_category', user_is_in_enterprise_category)
def o_auth2_application_get_absolute_url(self, request=None):
return reverse('api:o_auth2_application_detail', kwargs={'pk': self.pk}, request=request)
@@ -210,18 +203,19 @@ def o_auth2_token_get_absolute_url(self, request=None):
OAuth2AccessToken.add_to_class('get_absolute_url', o_auth2_token_get_absolute_url)
from awx.main.registrar import activity_stream_registrar # noqa
from awx.main.registrar import activity_stream_registrar # noqa
activity_stream_registrar.connect(Organization)
activity_stream_registrar.connect(Inventory)
activity_stream_registrar.connect(Host)
activity_stream_registrar.connect(Group)
activity_stream_registrar.connect(InventorySource)
#activity_stream_registrar.connect(InventoryUpdate)
# activity_stream_registrar.connect(InventoryUpdate)
activity_stream_registrar.connect(Credential)
activity_stream_registrar.connect(CredentialType)
activity_stream_registrar.connect(Team)
activity_stream_registrar.connect(Project)
#activity_stream_registrar.connect(ProjectUpdate)
# activity_stream_registrar.connect(ProjectUpdate)
activity_stream_registrar.connect(ExecutionEnvironment)
activity_stream_registrar.connect(JobTemplate)
activity_stream_registrar.connect(Job)

8
awx/main/models/activity_stream.py
View File

@@ -16,9 +16,9 @@ __all__ = ['ActivityStream']
class ActivityStream(models.Model):
'''
"""
Model used to describe activity stream (audit) events
'''
"""
class Meta:
app_label = 'main'
@@ -29,7 +29,7 @@ class ActivityStream(models.Model):
('update', _("Entity Updated")),
('delete', _("Entity Deleted")),
('associate', _("Entity Associated with another Entity")),
('disassociate', _("Entity was Disassociated with another Entity"))
('disassociate', _("Entity was Disassociated with another Entity")),
]
actor = models.ForeignKey('auth.User', null=True, on_delete=models.SET_NULL, related_name='activity_stream')
@@ -85,8 +85,6 @@ class ActivityStream(models.Model):
o_auth2_application = models.ManyToManyField("OAuth2Application", blank=True)
o_auth2_access_token = models.ManyToManyField("OAuth2AccessToken", blank=True)
setting = JSONField(blank=True)
def __str__(self):

41
awx/main/models/ad_hoc_commands.py
View File

@@ -14,9 +14,7 @@ from django.core.exceptions import ValidationError
# AWX
from awx.api.versioning import reverse
from awx.main.models.base import (
prevent_search, AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty
)
from awx.main.models.base import prevent_search, AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty
from awx.main.models.events import AdHocCommandEvent
from awx.main.models.unified_jobs import UnifiedJob
from awx.main.models.notifications import JobNotificationMixin, NotificationTemplate
@@ -27,7 +25,6 @@ __all__ = ['AdHocCommand']
class AdHocCommand(UnifiedJob, JobNotificationMixin):
class Meta(object):
app_label = 'main'
ordering = ('id',)
@@ -84,10 +81,12 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
editable=False,
through='AdHocCommandEvent',
)
extra_vars = prevent_search(models.TextField(
blank=True,
default='',
))
extra_vars = prevent_search(
models.TextField(
blank=True,
default='',
)
)
extra_vars_dict = VarsDictProperty('extra_vars', True)
@@ -144,6 +143,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
@classmethod
def _get_task_class(cls):
from awx.main.tasks import RunAdHocCommand
return RunAdHocCommand
@classmethod
@@ -169,9 +169,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
all_orgs = set()
for h in self.hosts.all():
all_orgs.add(h.inventory.organization)
active_templates = dict(error=set(),
success=set(),
started=set())
active_templates = dict(error=set(), success=set(), started=set())
base_notification_templates = NotificationTemplate.objects
for org in all_orgs:
for templ in base_notification_templates.filter(organization_notification_templates_for_errors=org):
@@ -192,14 +190,26 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
def task_impact(self):
# NOTE: We sorta have to assume the host count matches and that forks default to 5
from awx.main.models.inventory import Host
count_hosts = Host.objects.filter( enabled=True, inventory__ad_hoc_commands__pk=self.pk).count()
count_hosts = Host.objects.filter(enabled=True, inventory__ad_hoc_commands__pk=self.pk).count()
return min(count_hosts, 5 if self.forks == 0 else self.forks) + 1
def copy(self):
data = {}
for field in ('job_type', 'inventory_id', 'limit', 'credential_id',
'execution_environment_id', 'module_name', 'module_args',
'forks', 'verbosity', 'extra_vars', 'become_enabled', 'diff_mode'):
for field in (
'job_type',
'inventory_id',
'limit',
'credential_id',
'execution_environment_id',
'module_name',
'module_args',
'forks',
'verbosity',
'extra_vars',
'become_enabled',
'diff_mode',
):
data[field] = getattr(self, field)
return AdHocCommand.objects.create(**data)
@@ -232,6 +242,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
'''
JobNotificationMixin
'''
def get_notification_templates(self):
return self.notification_templates

89
awx/main/models/base.py
View File

@@ -17,18 +17,29 @@ from crum import get_current_user
from awx.main.utils import encrypt_field, parse_yaml_or_json
from awx.main.constants import CLOUD_PROVIDERS
__all__ = ['prevent_search', 'VarsDictProperty', 'BaseModel', 'CreatedModifiedModel',
'PasswordFieldsModel', 'PrimordialModel', 'CommonModel',
'CommonModelNameNotUnique', 'NotificationFieldsModel',
'PERM_INVENTORY_DEPLOY', 'PERM_INVENTORY_SCAN',
'PERM_INVENTORY_CHECK', 'JOB_TYPE_CHOICES',
'AD_HOC_JOB_TYPE_CHOICES', 'PROJECT_UPDATE_JOB_TYPE_CHOICES',
'CLOUD_INVENTORY_SOURCES',
'VERBOSITY_CHOICES']
__all__ = [
'prevent_search',
'VarsDictProperty',
'BaseModel',
'CreatedModifiedModel',
'PasswordFieldsModel',
'PrimordialModel',
'CommonModel',
'CommonModelNameNotUnique',
'NotificationFieldsModel',
'PERM_INVENTORY_DEPLOY',
'PERM_INVENTORY_SCAN',
'PERM_INVENTORY_CHECK',
'JOB_TYPE_CHOICES',
'AD_HOC_JOB_TYPE_CHOICES',
'PROJECT_UPDATE_JOB_TYPE_CHOICES',
'CLOUD_INVENTORY_SOURCES',
'VERBOSITY_CHOICES',
]
PERM_INVENTORY_DEPLOY = 'run'
PERM_INVENTORY_CHECK = 'check'
PERM_INVENTORY_SCAN = 'scan'
PERM_INVENTORY_CHECK = 'check'
PERM_INVENTORY_SCAN = 'scan'
JOB_TYPE_CHOICES = [
(PERM_INVENTORY_DEPLOY, _('Run')),
@@ -64,9 +75,9 @@ VERBOSITY_CHOICES = [
class VarsDictProperty(object):
'''
"""
Retrieve a string of variables in YAML or JSON as a dictionary.
'''
"""
def __init__(self, field='variables', key_value=False):
self.field = field
@@ -86,9 +97,9 @@ class VarsDictProperty(object):
class BaseModel(models.Model):
'''
"""
Base model class with common methods for all models.
'''
"""
class Meta:
abstract = True
@@ -100,10 +111,10 @@ class BaseModel(models.Model):
return u'%s-%s' % (self._meta.verbose_name, self.pk)
def clean_fields(self, exclude=None):
'''
"""
Override default clean_fields to support methods for cleaning
individual model fields.
'''
"""
exclude = exclude or []
errors = {}
try:
@@ -134,11 +145,11 @@ class BaseModel(models.Model):
class CreatedModifiedModel(BaseModel):
'''
"""
Common model with created/modified timestamp fields. Allows explicitly
specifying created/modified timestamps in certain cases (migrations, job
events), calculates automatically if not specified.
'''
"""
class Meta:
abstract = True
@@ -166,10 +177,10 @@ class CreatedModifiedModel(BaseModel):
class PasswordFieldsModel(BaseModel):
'''
"""
Abstract base class for a model with password fields that should be stored
as encrypted values.
'''
"""
PASSWORD_FIELDS = ()
@@ -177,7 +188,7 @@ class PasswordFieldsModel(BaseModel):
abstract = True
def _password_field_allows_ask(self, field):
return False # Override in subclasses if needed.
return False # Override in subclasses if needed.
def save(self, *args, **kwargs):
new_instance = not bool(self.pk)
@@ -207,6 +218,7 @@ class PasswordFieldsModel(BaseModel):
self.mark_field_for_save(update_fields, field)
from awx.main.signals import disable_activity_stream
with disable_activity_stream():
# We've already got an activity stream record for the object
# creation, there's no need to have an extra one for the
@@ -255,18 +267,15 @@ class HasEditsMixin(BaseModel):
return new_values
def _values_have_edits(self, new_values):
return any(
new_values.get(fd_name, None) != self._prior_values_store.get(fd_name, None)
for fd_name in new_values.keys()
)
return any(new_values.get(fd_name, None) != self._prior_values_store.get(fd_name, None) for fd_name in new_values.keys())
class PrimordialModel(HasEditsMixin, CreatedModifiedModel):
'''
"""
Common model for all object types that have these standard fields
must use a subclass CommonModel or CommonModelNameNotUnique though
as this lacks a name field.
'''
"""
class Meta:
abstract = True
@@ -339,12 +348,7 @@ class PrimordialModel(HasEditsMixin, CreatedModifiedModel):
except ObjectDoesNotExist:
continue
if not (self.pk and self.pk == obj.pk):
errors.append(
'%s with this (%s) combination already exists.' % (
model.__name__,
', '.join(set(ut) - {'polymorphic_ctype'})
)
)
errors.append('%s with this (%s) combination already exists.' % (model.__name__, ', '.join(set(ut) - {'polymorphic_ctype'})))
if errors:
raise ValidationError(errors)
@@ -374,27 +378,14 @@ class CommonModelNameNotUnique(PrimordialModel):
class NotificationFieldsModel(BaseModel):
class Meta:
abstract = True
notification_templates_error = models.ManyToManyField(
"NotificationTemplate",
blank=True,
related_name='%(class)s_notification_templates_for_errors'
)
notification_templates_error = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_errors')
notification_templates_success = models.ManyToManyField(
"NotificationTemplate",
blank=True,
related_name='%(class)s_notification_templates_for_success'
)
notification_templates_success = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_success')
notification_templates_started = models.ManyToManyField(
"NotificationTemplate",
blank=True,
related_name='%(class)s_notification_templates_for_started'
)
notification_templates_started = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_started')
def prevent_search(relation):

998
awx/main/models/credential/__init__.py
View File

File diff suppressed because it is too large Load Diff

17
awx/main/models/credential/injectors.py
View File

@@ -19,12 +19,7 @@ def gce(cred, env, private_data_dir):
project = cred.get_input('project', default='')
username = cred.get_input('username', default='')
json_cred = {
'type': 'service_account',
'private_key': cred.get_input('ssh_key_data', default=''),
'client_email': username,
'project_id': project
}
json_cred = {'type': 'service_account', 'private_key': cred.get_input('ssh_key_data', default=''), 'client_email': username, 'project_id': project}
if 'INVENTORY_UPDATE_ID' not in env:
env['GCE_EMAIL'] = username
env['GCE_PROJECT'] = project
@@ -73,10 +68,12 @@ def vmware(cred, env, private_data_dir):
def _openstack_data(cred):
openstack_auth = dict(auth_url=cred.get_input('host', default=''),
username=cred.get_input('username', default=''),
password=cred.get_input('password', default=''),
project_name=cred.get_input('project', default=''))
openstack_auth = dict(
auth_url=cred.get_input('host', default=''),
username=cred.get_input('username', default=''),
password=cred.get_input('password', default=''),
project_name=cred.get_input('project', default=''),
)
if cred.has_input('project_domain_name'):
openstack_auth['project_domain_name'] = cred.get_input('project_domain_name', default='')
if cred.has_input('domain'):

119
awx/main/models/events.py
View File

@@ -24,8 +24,7 @@ analytics_logger = logging.getLogger('awx.analytics.job_events')
logger = logging.getLogger('awx.main.models.events')
__all__ = ['JobEvent', 'ProjectUpdateEvent', 'AdHocCommandEvent',
'InventoryUpdateEvent', 'SystemJobEvent']
__all__ = ['JobEvent', 'ProjectUpdateEvent', 'AdHocCommandEvent', 'InventoryUpdateEvent', 'SystemJobEvent']
def sanitize_event_keys(kwargs, valid_keys):
@@ -35,9 +34,7 @@ def sanitize_event_keys(kwargs, valid_keys):
kwargs.pop(key)
# Truncate certain values over 1k
for key in [
'play', 'role', 'task', 'playbook'
]:
for key in ['play', 'role', 'task', 'playbook']:
if isinstance(kwargs.get('event_data', {}).get(key), str):
if len(kwargs['event_data'][key]) > 1024:
kwargs['event_data'][key] = Truncator(kwargs['event_data'][key]).chars(1024)
@@ -59,17 +56,11 @@ def create_host_status_counts(event_data):
return dict(host_status_counts)
MINIMAL_EVENTS = set([
'playbook_on_play_start', 'playbook_on_task_start',
'playbook_on_stats', 'EOF'
])
MINIMAL_EVENTS = set(['playbook_on_play_start', 'playbook_on_task_start', 'playbook_on_stats', 'EOF'])
def emit_event_detail(event):
if (
settings.UI_LIVE_UPDATES_ENABLED is False and
event.event not in MINIMAL_EVENTS
):
if settings.UI_LIVE_UPDATES_ENABLED is False and event.event not in MINIMAL_EVENTS:
return
cls = event.__class__
relation = {
@@ -109,21 +100,32 @@ def emit_event_detail(event):
'play': getattr(event, 'play', ''),
'role': getattr(event, 'role', ''),
'task': getattr(event, 'task', ''),
}
},
)
class BasePlaybookEvent(CreatedModifiedModel):
'''
"""
An event/message logged from a playbook callback for each host.
'''
"""
VALID_KEYS = [
'event', 'event_data', 'playbook', 'play', 'role', 'task', 'created',
'counter', 'uuid', 'stdout', 'parent_uuid', 'start_line', 'end_line',
'host_id', 'host_name', 'verbosity',
'event',
'event_data',
'playbook',
'play',
'role',
'task',
'created',
'counter',
'uuid',
'stdout',
'parent_uuid',
'start_line',
'end_line',
'host_id',
'host_name',
'verbosity',
]
class Meta:
@@ -191,7 +193,6 @@ class BasePlaybookEvent(CreatedModifiedModel):
(2, 'playbook_on_not_import_for_host', _('internal: on Not Import for Host'), False),
(1, 'playbook_on_play_start', _('Play Started'), False),
(1, 'playbook_on_stats', _('Playbook Complete'), False),
# Additional event types for captured stdout not directly related to
# playbook or runner events.
(0, 'debug', _('Debug'), False),
@@ -342,8 +343,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
try:
failures_dict = event_data.get('failures', {})
dark_dict = event_data.get('dark', {})
self.failed = bool(sum(failures_dict.values()) +
sum(dark_dict.values()))
self.failed = bool(sum(failures_dict.values()) + sum(dark_dict.values()))
changed_dict = event_data.get('changed', {})
self.changed = bool(sum(changed_dict.values()))
except (AttributeError, TypeError):
@@ -364,33 +364,30 @@ class BasePlaybookEvent(CreatedModifiedModel):
logger.exception('Computed fields database error saving event {}'.format(self.pk))
# find parent links and progagate changed=T and failed=T
changed = job.job_events.filter(changed=True).exclude(parent_uuid=None).only('parent_uuid').values_list('parent_uuid', flat=True).distinct() # noqa
failed = job.job_events.filter(failed=True).exclude(parent_uuid=None).only('parent_uuid').values_list('parent_uuid', flat=True).distinct() # noqa
changed = (
job.job_events.filter(changed=True).exclude(parent_uuid=None).only('parent_uuid').values_list('parent_uuid', flat=True).distinct()
) # noqa
failed = (
job.job_events.filter(failed=True).exclude(parent_uuid=None).only('parent_uuid').values_list('parent_uuid', flat=True).distinct()
) # noqa
JobEvent.objects.filter(
job_id=self.job_id, uuid__in=changed
).update(changed=True)
JobEvent.objects.filter(
job_id=self.job_id, uuid__in=failed
).update(failed=True)
JobEvent.objects.filter(job_id=self.job_id, uuid__in=changed).update(changed=True)
JobEvent.objects.filter(job_id=self.job_id, uuid__in=failed).update(failed=True)
# send success/failure notifications when we've finished handling the playbook_on_stats event
from awx.main.tasks import handle_success_and_failure_notifications # circular import
def _send_notifications():
handle_success_and_failure_notifications.apply_async([job.id])
connection.on_commit(_send_notifications)
connection.on_commit(_send_notifications)
for field in ('playbook', 'play', 'task', 'role'):
value = force_text(event_data.get(field, '')).strip()
if value != getattr(self, field):
setattr(self, field, value)
if settings.LOG_AGGREGATOR_ENABLED:
analytics_logger.info(
'Event data saved.',
extra=dict(python_objects=dict(job_event=self))
)
analytics_logger.info('Event data saved.', extra=dict(python_objects=dict(job_event=self)))
@classmethod
def create_from_data(cls, **kwargs):
@@ -443,9 +440,9 @@ class BasePlaybookEvent(CreatedModifiedModel):
class JobEvent(BasePlaybookEvent):
'''
"""
An event/message logged from the callback when running a job.
'''
"""
VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['job_id', 'workflow_job_id']
@@ -513,9 +510,8 @@ class JobEvent(BasePlaybookEvent):
job = self.job
from awx.main.models import Host, JobHostSummary # circular import
all_hosts = Host.objects.filter(
pk__in=self.host_map.values()
).only('id')
all_hosts = Host.objects.filter(pk__in=self.host_map.values()).only('id')
existing_host_ids = set(h.id for h in all_hosts)
summaries = dict()
@@ -529,9 +525,7 @@ class JobEvent(BasePlaybookEvent):
host_stats[stat] = self.event_data.get(stat, {}).get(host, 0)
except AttributeError: # in case event_data[stat] isn't a dict.
pass
summary = JobHostSummary(
created=now(), modified=now(), job_id=job.id, host_id=host_id, host_name=host, **host_stats
)
summary = JobHostSummary(created=now(), modified=now(), job_id=job.id, host_id=host_id, host_name=host, **host_stats)
summary.failed = bool(summary.dark or summary.failures)
summaries[(host_id, host)] = summary
@@ -539,10 +533,7 @@ class JobEvent(BasePlaybookEvent):
# update the last_job_id and last_job_host_summary_id
# in single queries
host_mapping = dict(
(summary['host_id'], summary['id'])
for summary in JobHostSummary.objects.filter(job_id=job.id).values('id', 'host_id')
)
host_mapping = dict((summary['host_id'], summary['id']) for summary in JobHostSummary.objects.filter(job_id=job.id).values('id', 'host_id'))
updated_hosts = set()
for h in all_hosts:
# if the hostname *shows up* in the playbook_on_stats event
@@ -553,12 +544,7 @@ class JobEvent(BasePlaybookEvent):
h.last_job_host_summary_id = host_mapping[h.id]
updated_hosts.add(h)
Host.objects.bulk_update(
list(updated_hosts),
['last_job_id', 'last_job_host_summary_id'],
batch_size=100
)
Host.objects.bulk_update(list(updated_hosts), ['last_job_id', 'last_job_host_summary_id'], batch_size=100)
@property
def job_verbosity(self):
@@ -593,14 +579,11 @@ class ProjectUpdateEvent(BasePlaybookEvent):
class BaseCommandEvent(CreatedModifiedModel):
'''
"""
An event/message logged from a command for each host.
'''
"""
VALID_KEYS = [
'event_data', 'created', 'counter', 'uuid', 'stdout', 'start_line',
'end_line', 'verbosity'
]
VALID_KEYS = ['event_data', 'created', 'counter', 'uuid', 'stdout', 'start_line', 'end_line', 'verbosity']
class Meta:
abstract = True
@@ -671,9 +654,9 @@ class BaseCommandEvent(CreatedModifiedModel):
return event
def get_event_display(self):
'''
"""
Needed for __unicode__
'''
"""
return self.event
def get_event_display2(self):
@@ -688,9 +671,7 @@ class BaseCommandEvent(CreatedModifiedModel):
class AdHocCommandEvent(BaseCommandEvent):
VALID_KEYS = BaseCommandEvent.VALID_KEYS + [
'ad_hoc_command_id', 'event', 'host_name', 'host_id', 'workflow_job_id'
]
VALID_KEYS = BaseCommandEvent.VALID_KEYS + ['ad_hoc_command_id', 'event', 'host_name', 'host_id', 'workflow_job_id']
class Meta:
app_label = 'main'
@@ -718,7 +699,6 @@ class AdHocCommandEvent(BaseCommandEvent):
# ('runner_on_async_failed', _('Host Async Failure'), True),
# Tower does not yet support --diff mode.
# ('runner_on_file_diff', _('File Difference'), False),
# Additional event types for captured stdout not directly related to
# runner events.
('debug', _('Debug'), False),
@@ -775,10 +755,7 @@ class AdHocCommandEvent(BaseCommandEvent):
if isinstance(res, dict) and res.get('changed', False):
self.changed = True
analytics_logger.info(
'Event data saved.',
extra=dict(python_objects=dict(job_event=self))
)
analytics_logger.info('Event data saved.', extra=dict(python_objects=dict(job_event=self)))
class InventoryUpdateEvent(BaseCommandEvent):

2
awx/main/models/execution_environments.py
View File

@@ -15,7 +15,7 @@ class ExecutionEnvironment(CommonModel):
PULL_CHOICES = [
('always', _("Always pull container before running.")),
('missing', _("No pull option has been selected.")),
('never', _("Never pull container before running."))
('never', _("Never pull container before running.")),
]
organization = models.ForeignKey(

115
awx/main/models/ha.py
View File

@@ -28,7 +28,6 @@ __all__ = ('Instance', 'InstanceGroup', 'TowerScheduleState')
class HasPolicyEditsMixin(HasEditsMixin):
class Meta:
abstract = True
@@ -50,6 +49,7 @@ class HasPolicyEditsMixin(HasEditsMixin):
class Instance(HasPolicyEditsMixin, BaseModel):
"""A model representing an AWX instance running against this database."""
objects = InstanceManager()
uuid = models.CharField(max_length=40)
@@ -72,18 +72,9 @@ class Instance(HasPolicyEditsMixin, BaseModel):
default=100,
editable=False,
)
capacity_adjustment = models.DecimalField(
default=Decimal(1.0),
max_digits=3,
decimal_places=2,
validators=[MinValueValidator(0)]
)
enabled = models.BooleanField(
default=True
)
managed_by_policy = models.BooleanField(
default=True
)
capacity_adjustment = models.DecimalField(default=Decimal(1.0), max_digits=3, decimal_places=2, validators=[MinValueValidator(0)])
enabled = models.BooleanField(default=True)
managed_by_policy = models.BooleanField(default=True)
cpu = models.IntegerField(
default=0,
editable=False,
@@ -112,8 +103,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
@property
def consumed_capacity(self):
return sum(x.task_impact for x in UnifiedJob.objects.filter(execution_node=self.hostname,
status__in=('running', 'waiting')))
return sum(x.task_impact for x in UnifiedJob.objects.filter(execution_node=self.hostname, status__in=('running', 'waiting')))
@property
def remaining_capacity(self):
@@ -126,7 +116,13 @@ class Instance(HasPolicyEditsMixin, BaseModel):
@property
def jobs_running(self):
return UnifiedJob.objects.filter(execution_node=self.hostname, status__in=('running', 'waiting',)).count()
return UnifiedJob.objects.filter(
execution_node=self.hostname,
status__in=(
'running',
'waiting',
),
).count()
@property
def jobs_total(self):
@@ -150,8 +146,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
if settings.IS_K8S:
self.capacity = self.cpu = self.memory = self.cpu_capacity = self.mem_capacity = 0 # noqa
self.version = awx_application_version
self.save(update_fields=['capacity', 'version', 'modified', 'cpu',
'memory', 'cpu_capacity', 'mem_capacity'])
self.save(update_fields=['capacity', 'version', 'modified', 'cpu', 'memory', 'cpu_capacity', 'mem_capacity'])
return
cpu = get_cpu_capacity()
@@ -173,12 +168,12 @@ class Instance(HasPolicyEditsMixin, BaseModel):
self.cpu_capacity = cpu[1]
self.mem_capacity = mem[1]
self.version = awx_application_version
self.save(update_fields=['capacity', 'version', 'modified', 'cpu',
'memory', 'cpu_capacity', 'mem_capacity'])
self.save(update_fields=['capacity', 'version', 'modified', 'cpu', 'memory', 'cpu_capacity', 'mem_capacity'])
class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
"""A model representing a Queue/Group of AWX Instances."""
objects = InstanceGroupManager()
name = models.CharField(max_length=250, unique=True)
@@ -197,11 +192,9 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
editable=False,
default=None,
null=True,
on_delete=models.CASCADE
)
is_container_group = models.BooleanField(
default=False
on_delete=models.CASCADE,
)
is_container_group = models.BooleanField(default=False)
credential = models.ForeignKey(
'Credential',
related_name='%(class)ss',
@@ -210,27 +203,19 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
default=None,
on_delete=models.SET_NULL,
)
pod_spec_override = prevent_search(models.TextField(
blank=True,
default='',
))
policy_instance_percentage = models.IntegerField(
default=0,
help_text=_("Percentage of Instances to automatically assign to this group")
)
policy_instance_minimum = models.IntegerField(
default=0,
help_text=_("Static minimum number of Instances to automatically assign to this group")
pod_spec_override = prevent_search(
models.TextField(
blank=True,
default='',
)
)
policy_instance_percentage = models.IntegerField(default=0, help_text=_("Percentage of Instances to automatically assign to this group"))
policy_instance_minimum = models.IntegerField(default=0, help_text=_("Static minimum number of Instances to automatically assign to this group"))
policy_instance_list = JSONField(
default=[],
blank=True,
help_text=_("List of exact-match Instances that will always be automatically assigned to this group")
default=[], blank=True, help_text=_("List of exact-match Instances that will always be automatically assigned to this group")
)
POLICY_FIELDS = frozenset((
'policy_instance_list', 'policy_instance_minimum', 'policy_instance_percentage', 'controller'
))
POLICY_FIELDS = frozenset(('policy_instance_list', 'policy_instance_minimum', 'policy_instance_percentage', 'controller'))
def get_absolute_url(self, request=None):
return reverse('api:instance_group_detail', kwargs={'pk': self.pk}, request=request)
@@ -241,8 +226,7 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
@property
def jobs_running(self):
return UnifiedJob.objects.filter(status__in=('running', 'waiting'),
instance_group=self).count()
return UnifiedJob.objects.filter(status__in=('running', 'waiting'), instance_group=self).count()
@property
def jobs_total(self):
@@ -259,21 +243,20 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return UnifiedJob.objects.filter(instance_group=self)
class Meta:
app_label = 'main'
@staticmethod
def fit_task_to_most_remaining_capacity_instance(task, instances):
instance_most_capacity = None
for i in instances:
if i.remaining_capacity >= task.task_impact and \
(instance_most_capacity is None or
i.remaining_capacity > instance_most_capacity.remaining_capacity):
if i.remaining_capacity >= task.task_impact and (
instance_most_capacity is None or i.remaining_capacity > instance_most_capacity.remaining_capacity
):
instance_most_capacity = i
return instance_most_capacity
@@ -289,10 +272,7 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
return largest_instance
def choose_online_controller_node(self):
return random.choice(list(self.controller
.instances
.filter(capacity__gt=0, enabled=True)
.values_list('hostname', flat=True)))
return random.choice(list(self.controller.instances.filter(capacity__gt=0, enabled=True).values_list('hostname', flat=True)))
def set_default_policy_fields(self):
self.policy_instance_list = []
@@ -306,6 +286,7 @@ class TowerScheduleState(SingletonModel):
def schedule_policy_task():
from awx.main.tasks import apply_cluster_membership_policies
connection.on_commit(lambda: apply_cluster_membership_policies.apply_async())
@@ -337,14 +318,8 @@ def on_instance_deleted(sender, instance, using, **kwargs):
class UnifiedJobTemplateInstanceGroupMembership(models.Model):
unifiedjobtemplate = models.ForeignKey(
'UnifiedJobTemplate',
on_delete=models.CASCADE
)
instancegroup = models.ForeignKey(
'InstanceGroup',
on_delete=models.CASCADE
)
unifiedjobtemplate = models.ForeignKey('UnifiedJobTemplate', on_delete=models.CASCADE)
instancegroup = models.ForeignKey('InstanceGroup', on_delete=models.CASCADE)
position = models.PositiveIntegerField(
null=True,
default=None,
@@ -354,14 +329,8 @@ class UnifiedJobTemplateInstanceGroupMembership(models.Model):
class OrganizationInstanceGroupMembership(models.Model):
organization = models.ForeignKey(
'Organization',
on_delete=models.CASCADE
)
instancegroup = models.ForeignKey(
'InstanceGroup',
on_delete=models.CASCADE
)
organization = models.ForeignKey('Organization', on_delete=models.CASCADE)
instancegroup = models.ForeignKey('InstanceGroup', on_delete=models.CASCADE)
position = models.PositiveIntegerField(
null=True,
default=None,
@@ -371,14 +340,8 @@ class OrganizationInstanceGroupMembership(models.Model):
class InventoryInstanceGroupMembership(models.Model):
inventory = models.ForeignKey(
'Inventory',
on_delete=models.CASCADE
)
instancegroup = models.ForeignKey(
'InstanceGroup',
on_delete=models.CASCADE
)
inventory = models.ForeignKey('Inventory', on_delete=models.CASCADE)
instancegroup = models.ForeignKey('InstanceGroup', on_delete=models.CASCADE)
position = models.PositiveIntegerField(
null=True,
default=None,

378
awx/main/models/inventory.py
View File

@@ -34,13 +34,7 @@ from awx.main.fields import (
OrderedManyToManyField,
)
from awx.main.managers import HostManager
from awx.main.models.base import (
BaseModel,
CommonModelNameNotUnique,
VarsDictProperty,
CLOUD_INVENTORY_SOURCES,
prevent_search, accepts_json
)
from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, prevent_search, accepts_json
from awx.main.models.events import InventoryUpdateEvent
from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
from awx.main.models.mixins import (
@@ -58,16 +52,15 @@ from awx.main.utils import _inventory_updates
from awx.main.utils.safe_yaml import sanitize_jinja
__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate',
'CustomInventoryScript', 'SmartInventoryMembership']
__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'CustomInventoryScript', 'SmartInventoryMembership']
logger = logging.getLogger('awx.main.models.inventory')
class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
'''
"""
an inventory source contains lists and hosts.
'''
"""
FIELDS_TO_PRESERVE_AT_COPY = ['hosts', 'groups', 'instance_groups']
KIND_CHOICES = [
@@ -88,40 +81,39 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
on_delete=models.SET_NULL,
null=True,
)
variables = accepts_json(models.TextField(
blank=True,
default='',
help_text=_('Inventory variables in JSON or YAML format.'),
))
variables = accepts_json(
models.TextField(
blank=True,
default='',
help_text=_('Inventory variables in JSON or YAML format.'),
)
)
has_active_failures = models.BooleanField(
default=False,
editable=False,
help_text=_('This field is deprecated and will be removed in a future release. '
'Flag indicating whether any hosts in this inventory have failed.'),
help_text=_('This field is deprecated and will be removed in a future release. ' 'Flag indicating whether any hosts in this inventory have failed.'),
)
total_hosts = models.PositiveIntegerField(
default=0,
editable=False,
help_text=_('This field is deprecated and will be removed in a future release. '
'Total number of hosts in this inventory.'),
help_text=_('This field is deprecated and will be removed in a future release. ' 'Total number of hosts in this inventory.'),
)
hosts_with_active_failures = models.PositiveIntegerField(
default=0,
editable=False,
help_text=_('This field is deprecated and will be removed in a future release. '
'Number of hosts in this inventory with active failures.'),
help_text=_('This field is deprecated and will be removed in a future release. ' 'Number of hosts in this inventory with active failures.'),
)
total_groups = models.PositiveIntegerField(
default=0,
editable=False,
help_text=_('This field is deprecated and will be removed in a future release. '
'Total number of groups in this inventory.'),
help_text=_('This field is deprecated and will be removed in a future release. ' 'Total number of groups in this inventory.'),
)
has_inventory_sources = models.BooleanField(
default=False,
editable=False,
help_text=_('This field is deprecated and will be removed in a future release. '
'Flag indicating whether this inventory has any external inventory sources.'),
help_text=_(
'This field is deprecated and will be removed in a future release. ' 'Flag indicating whether this inventory has any external inventory sources.'
),
)
total_inventory_sources = models.PositiveIntegerField(
default=0,
@@ -163,12 +155,14 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
use_role = ImplicitRoleField(
parent_role='adhoc_role',
)
read_role = ImplicitRoleField(parent_role=[
'organization.auditor_role',
'update_role',
'use_role',
'admin_role',
])
read_role = ImplicitRoleField(
parent_role=[
'organization.auditor_role',
'update_role',
'use_role',
'admin_role',
]
)
insights_credential = models.ForeignKey(
'Credential',
related_name='insights_inventories',
@@ -184,16 +178,15 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
help_text=_('Flag indicating the inventory is being deleted.'),
)
def get_absolute_url(self, request=None):
return reverse('api:inventory_detail', kwargs={'pk': self.pk}, request=request)
variables_dict = VarsDictProperty('variables')
def get_group_hosts_map(self):
'''
"""
Return dictionary mapping group_id to set of child host_id's.
'''
"""
# FIXME: Cache this mapping?
group_hosts_kw = dict(group__inventory_id=self.pk, host__inventory_id=self.pk)
group_hosts_qs = Group.hosts.through.objects.filter(**group_hosts_kw)
@@ -205,9 +198,9 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
return group_hosts_map
def get_group_parents_map(self):
'''
"""
Return dictionary mapping group_id to set of parent group_id's.
'''
"""
# FIXME: Cache this mapping?
group_parents_kw = dict(from_group__inventory_id=self.pk, to_group__inventory_id=self.pk)
group_parents_qs = Group.parents.through.objects.filter(**group_parents_kw)
@@ -219,9 +212,9 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
return group_parents_map
def get_group_children_map(self):
'''
"""
Return dictionary mapping group_id to set of child group_id's.
'''
"""
# FIXME: Cache this mapping?
group_parents_kw = dict(from_group__inventory_id=self.pk, to_group__inventory_id=self.pk)
group_parents_qs = Group.parents.through.objects.filter(**group_parents_kw)
@@ -271,10 +264,9 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
grouped_hosts = set([])
# Build in-memory mapping of groups and their hosts.
group_hosts_qs = Group.hosts.through.objects.filter(
group__inventory_id=self.id,
host__inventory_id=self.id
).values_list('group_id', 'host_id', 'host__name')
group_hosts_qs = Group.hosts.through.objects.filter(group__inventory_id=self.id, host__inventory_id=self.id).values_list(
'group_id', 'host_id', 'host__name'
)
group_hosts_map = {}
for group_id, host_id, host_name in group_hosts_qs:
if host_name not in all_hostnames:
@@ -321,16 +313,15 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
for host in hosts:
data['_meta']['hostvars'][host.name] = host.variables_dict
if towervars:
tower_dict = dict(remote_tower_enabled=str(host.enabled).lower(),
remote_tower_id=host.id)
tower_dict = dict(remote_tower_enabled=str(host.enabled).lower(), remote_tower_id=host.id)
data['_meta']['hostvars'][host.name].update(tower_dict)
return data
def update_computed_fields(self):
'''
"""
Update model fields that are computed from database relationships.
'''
"""
logger.debug("Going to update inventory computed fields, pk={0}".format(self.pk))
start_time = time.time()
active_hosts = self.hosts
@@ -363,14 +354,12 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
computed_fields.pop(field)
if computed_fields:
iobj.save(update_fields=computed_fields.keys())
logger.debug("Finished updating inventory computed fields, pk={0}, in "
"{1:.3f} seconds".format(self.pk, time.time() - start_time))
logger.debug("Finished updating inventory computed fields, pk={0}, in " "{1:.3f} seconds".format(self.pk, time.time() - start_time))
def websocket_emit_status(self, status):
connection.on_commit(lambda: emit_channel_notification(
'inventories-status_changed',
{'group_name': 'inventories', 'inventory_id': self.id, 'status': status}
))
connection.on_commit(
lambda: emit_channel_notification('inventories-status_changed', {'group_name': 'inventories', 'inventory_id': self.id, 'status': status})
)
@property
def root_groups(self):
@@ -388,6 +377,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
def schedule_deletion(self, user_id=None):
from awx.main.tasks import delete_inventory
from awx.main.signals import activity_stream_delete
if self.pending_deletion is True:
raise RuntimeError("Inventory is already pending deletion.")
self.pending_deletion = True
@@ -399,16 +389,18 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
def _update_host_smart_inventory_memeberships(self):
if self.kind == 'smart' and settings.AWX_REBUILD_SMART_MEMBERSHIP:
def on_commit():
from awx.main.tasks import update_host_smart_inventory_memberships
update_host_smart_inventory_memberships.delay()
connection.on_commit(on_commit)
def save(self, *args, **kwargs):
self._update_host_smart_inventory_memeberships()
super(Inventory, self).save(*args, **kwargs)
if (self.kind == 'smart' and 'host_filter' in kwargs.get('update_fields', ['host_filter']) and
connection.vendor != 'sqlite'):
if self.kind == 'smart' and 'host_filter' in kwargs.get('update_fields', ['host_filter']) and connection.vendor != 'sqlite':
# Minimal update of host_count for smart inventory host filter changes
self.update_computed_fields()
@@ -419,18 +411,15 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return UnifiedJob.objects.non_polymorphic().filter(
Q(job__inventory=self) |
Q(inventoryupdate__inventory=self) |
Q(adhoccommand__inventory=self)
)
return UnifiedJob.objects.non_polymorphic().filter(Q(job__inventory=self) | Q(inventoryupdate__inventory=self) | Q(adhoccommand__inventory=self))
class SmartInventoryMembership(BaseModel):
'''
"""
A lookup table for Host membership in Smart Inventory
'''
"""
class Meta:
app_label = 'main'
@@ -441,17 +430,15 @@ class SmartInventoryMembership(BaseModel):
class Host(CommonModelNameNotUnique, RelatedJobsMixin):
'''
"""
A managed node
'''
"""
FIELDS_TO_PRESERVE_AT_COPY = [
'name', 'description', 'groups', 'inventory', 'enabled', 'instance_id', 'variables'
]
FIELDS_TO_PRESERVE_AT_COPY = ['name', 'description', 'groups', 'inventory', 'enabled', 'instance_id', 'variables']
class Meta:
app_label = 'main'
unique_together = (("name", "inventory"),) # FIXME: Add ('instance_id', 'inventory') after migration.
unique_together = (("name", "inventory"),) # FIXME: Add ('instance_id', 'inventory') after migration.
ordering = ('name',)
inventory = models.ForeignKey(
@@ -474,11 +461,13 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
default='',
help_text=_('The value used by the remote inventory source to uniquely identify the host'),
)
variables = accepts_json(models.TextField(
blank=True,
default='',
help_text=_('Host variables in JSON or YAML format.'),
))
variables = accepts_json(
models.TextField(
blank=True,
default='',
help_text=_('Host variables in JSON or YAML format.'),
)
)
last_job = models.ForeignKey(
'Job',
related_name='hosts_as_last_job+',
@@ -530,10 +519,10 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
@property
def all_groups(self):
'''
"""
Return all groups of which this host is a member, avoiding infinite
recursion in the case of cyclical group relations.
'''
"""
group_parents_map = self.inventory.get_group_parents_map()
group_pks = set(self.groups.values_list('pk', flat=True))
child_pks_to_check = set()
@@ -554,6 +543,7 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
'''
We don't use timestamp, but we may in the future.
'''
def update_ansible_facts(self, module, facts, timestamp=None):
if module == "ansible":
self.ansible_facts.update(facts)
@@ -562,10 +552,10 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
self.save()
def get_effective_host_name(self):
'''
"""
Return the name of the host that will be used in actual ansible
command run.
'''
"""
host_name = self.name
if 'ansible_ssh_host' in self.variables_dict:
host_name = self.variables_dict['ansible_ssh_host']
@@ -575,9 +565,12 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
def _update_host_smart_inventory_memeberships(self):
if settings.AWX_REBUILD_SMART_MEMBERSHIP:
def on_commit():
from awx.main.tasks import update_host_smart_inventory_memberships
update_host_smart_inventory_memberships.delay()
connection.on_commit(on_commit)
def clean_name(self):
@@ -598,19 +591,18 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return self.inventory._get_related_jobs()
class Group(CommonModelNameNotUnique, RelatedJobsMixin):
'''
"""
A group containing managed hosts. A group or host may belong to multiple
groups.
'''
"""
FIELDS_TO_PRESERVE_AT_COPY = [
'name', 'description', 'inventory', 'children', 'parents', 'hosts', 'variables'
]
FIELDS_TO_PRESERVE_AT_COPY = ['name', 'description', 'inventory', 'children', 'parents', 'hosts', 'variables']
class Meta:
app_label = 'main'
@@ -629,11 +621,13 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
related_name='children',
blank=True,
)
variables = accepts_json(models.TextField(
blank=True,
default='',
help_text=_('Group variables in JSON or YAML format.'),
))
variables = accepts_json(
models.TextField(
blank=True,
default='',
help_text=_('Group variables in JSON or YAML format.'),
)
)
hosts = models.ManyToManyField(
'Host',
related_name='groups',
@@ -656,7 +650,6 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
from awx.main.tasks import update_inventory_computed_fields
from awx.main.signals import disable_activity_stream, activity_stream_delete
def mark_actual():
all_group_hosts = Group.hosts.through.objects.select_related("host", "group").filter(group__inventory=self.inventory)
group_hosts = {'groups': {}, 'hosts': {}}
@@ -709,6 +702,7 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
Group.objects.filter(id__in=marked_groups).delete()
Host.objects.filter(id__in=marked_hosts).delete()
update_inventory_computed_fields.delay(self.inventory.id)
with ignore_inventory_computed_fields():
with disable_activity_stream():
mark_actual()
@@ -717,10 +711,10 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
variables_dict = VarsDictProperty('variables')
def get_all_parents(self, except_pks=None):
'''
"""
Return all parents of this group recursively. The group itself will
be excluded unless there is a cycle leading back to it.
'''
"""
group_parents_map = self.inventory.get_group_parents_map()
child_pks_to_check = set([self.pk])
child_pks_checked = set()
@@ -739,10 +733,10 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
return self.get_all_parents()
def get_all_children(self, except_pks=None):
'''
"""
Return all children of this group recursively. The group itself will
be excluded unless there is a cycle leading back to it.
'''
"""
group_children_map = self.inventory.get_group_children_map()
parent_pks_to_check = set([self.pk])
parent_pks_checked = set()
@@ -761,9 +755,9 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
return self.get_all_children()
def get_all_hosts(self, except_group_pks=None):
'''
"""
Return all hosts associated with this group or any of its children.
'''
"""
group_children_map = self.inventory.get_group_children_map()
group_hosts_map = self.inventory.get_group_hosts_map()
parent_pks_to_check = set([self.pk])
@@ -786,32 +780,33 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
@property
def job_host_summaries(self):
from awx.main.models.jobs import JobHostSummary
return JobHostSummary.objects.filter(host__in=self.all_hosts)
@property
def job_events(self):
from awx.main.models.jobs import JobEvent
return JobEvent.objects.filter(host__in=self.all_hosts)
@property
def ad_hoc_commands(self):
from awx.main.models.ad_hoc_commands import AdHocCommand
return AdHocCommand.objects.filter(hosts__in=self.all_hosts)
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return UnifiedJob.objects.non_polymorphic().filter(
Q(job__inventory=self.inventory) |
Q(inventoryupdate__inventory_source__groups=self)
)
return UnifiedJob.objects.non_polymorphic().filter(Q(job__inventory=self.inventory) | Q(inventoryupdate__inventory_source__groups=self))
class InventorySourceOptions(BaseModel):
'''
"""
Common fields for InventorySource and InventoryUpdate.
'''
"""
injectors = dict()
@@ -865,30 +860,34 @@ class InventorySourceOptions(BaseModel):
enabled_var = models.TextField(
blank=True,
default='',
help_text=_('Retrieve the enabled state from the given dict of host '
'variables. The enabled variable may be specified as "foo.bar", '
'in which case the lookup will traverse into nested dicts, '
'equivalent to: from_dict.get("foo", {}).get("bar", default)'),
help_text=_(
'Retrieve the enabled state from the given dict of host '
'variables. The enabled variable may be specified as "foo.bar", '
'in which case the lookup will traverse into nested dicts, '
'equivalent to: from_dict.get("foo", {}).get("bar", default)'
),
)
enabled_value = models.TextField(
blank=True,
default='',
help_text=_('Only used when enabled_var is set. Value when the host is '
'considered enabled. For example if enabled_var="status.power_state"'
'and enabled_value="powered_on" with host variables:'
'{'
' "status": {'
' "power_state": "powered_on",'
' "created": "2020-08-04T18:13:04+00:00",'
' "healthy": true'
' },'
' "name": "foobar",'
' "ip_address": "192.168.2.1"'
'}'
'The host would be marked enabled. If power_state where any '
'value other than powered_on then the host would be disabled '
'when imported into Tower. If the key is not found then the '
'host will be enabled'),
help_text=_(
'Only used when enabled_var is set. Value when the host is '
'considered enabled. For example if enabled_var="status.power_state"'
'and enabled_value="powered_on" with host variables:'
'{'
' "status": {'
' "power_state": "powered_on",'
' "created": "2020-08-04T18:13:04+00:00",'
' "healthy": true'
' },'
' "name": "foobar",'
' "ip_address": "192.168.2.1"'
'}'
'The host would be marked enabled. If power_state where any '
'value other than powered_on then the host would be disabled '
'when imported into Tower. If the key is not found then the '
'host will be enabled'
),
)
host_filter = models.TextField(
blank=True,
@@ -923,28 +922,20 @@ class InventorySourceOptions(BaseModel):
# the actual inventory source being used (Amazon requires Amazon
# credentials; Rackspace requires Rackspace credentials; etc...)
if source.replace('ec2', 'aws') != cred.kind:
return _('Cloud-based inventory sources (such as %s) require '
'credentials for the matching cloud service.') % source
return _('Cloud-based inventory sources (such as %s) require ' 'credentials for the matching cloud service.') % source
# Allow an EC2 source to omit the credential. If Tower is running on
# an EC2 instance with an IAM Role assigned, boto will use credentials
# from the instance metadata instead of those explicitly provided.
elif source in CLOUD_PROVIDERS and source != 'ec2':
return _('Credential is required for a cloud source.')
elif source == 'custom' and cred and cred.credential_type.kind in ('scm', 'ssh', 'insights', 'vault'):
return _(
'Credentials of type machine, source control, insights and vault are '
'disallowed for custom inventory sources.'
)
return _('Credentials of type machine, source control, insights and vault are ' 'disallowed for custom inventory sources.')
elif source == 'scm' and cred and cred.credential_type.kind in ('insights', 'vault'):
return _(
'Credentials of type insights and vault are '
'disallowed for scm inventory sources.'
)
return _('Credentials of type insights and vault are ' 'disallowed for scm inventory sources.')
return None
def get_cloud_credential(self):
"""Return the credential which is directly tied to the inventory source type.
"""
"""Return the credential which is directly tied to the inventory source type."""
credential = None
for cred in self.credentials.all():
if self.source in CLOUD_PROVIDERS:
@@ -978,7 +969,6 @@ class InventorySourceOptions(BaseModel):
if cred is not None:
return cred.pk
source_vars_dict = VarsDictProperty('source_vars')
@@ -1005,7 +995,7 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
on_delete=models.CASCADE,
blank=True,
default=None,
null=True
null=True,
)
scm_last_revision = models.CharField(
max_length=1024,
@@ -1029,9 +1019,7 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
@classmethod
def _get_unified_job_field_names(cls):
return set(f.name for f in InventorySourceOptions._meta.fields) | set(
['name', 'description', 'organization', 'credentials', 'inventory']
)
return set(f.name for f in InventorySourceOptions._meta.fields) | set(['name', 'description', 'organization', 'credentials', 'inventory'])
def save(self, *args, **kwargs):
# if this is a new object, inherit organization from its inventory
@@ -1059,7 +1047,7 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
if 'name' not in update_fields:
update_fields.append('name')
# Reset revision if SCM source has changed parameters
if self.source=='scm' and not is_new_instance:
if self.source == 'scm' and not is_new_instance:
before_is = self.__class__.objects.get(pk=self.pk)
if before_is.source_path != self.source_path or before_is.source_project_id != self.source_project_id:
# Reset the scm_revision if file changed to force update
@@ -1074,10 +1062,9 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
if replace_text in self.name:
self.name = self.name.replace(replace_text, str(self.pk))
super(InventorySource, self).save(update_fields=['name'])
if self.source=='scm' and is_new_instance and self.update_on_project_update:
if self.source == 'scm' and is_new_instance and self.update_on_project_update:
# Schedule a new Project update if one is not already queued
if self.source_project and not self.source_project.project_updates.filter(
status__in=['new', 'pending', 'waiting']).exists():
if self.source_project and not self.source_project.project_updates.filter(status__in=['new', 'pending', 'waiting']).exists():
self.update()
if not getattr(_inventory_updates, 'is_updating', False):
if self.inventory is not None:
@@ -1126,7 +1113,7 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
name = '{} - {}'.format(self.inventory.name, self.name)
name_field = self._meta.get_field('name')
if len(name) > name_field.max_length:
name = name[:name_field.max_length]
name = name[: name_field.max_length]
kwargs['_eager_fields']['name'] = name
return super(InventorySource, self).create_unified_job(**kwargs)
@@ -1150,39 +1137,41 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
@property
def notification_templates(self):
base_notification_templates = NotificationTemplate.objects
error_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
started_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
success_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
error_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
started_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
success_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
if self.inventory.organization is not None:
error_notification_templates = set(error_notification_templates + list(base_notification_templates
.filter(organization_notification_templates_for_errors=self.inventory.organization)))
started_notification_templates = set(started_notification_templates + list(base_notification_templates
.filter(organization_notification_templates_for_started=self.inventory.organization)))
success_notification_templates = set(success_notification_templates + list(base_notification_templates
.filter(organization_notification_templates_for_success=self.inventory.organization)))
return dict(error=list(error_notification_templates),
started=list(started_notification_templates),
success=list(success_notification_templates))
error_notification_templates = set(
error_notification_templates
+ list(base_notification_templates.filter(organization_notification_templates_for_errors=self.inventory.organization))
)
started_notification_templates = set(
started_notification_templates
+ list(base_notification_templates.filter(organization_notification_templates_for_started=self.inventory.organization))
)
success_notification_templates = set(
success_notification_templates
+ list(base_notification_templates.filter(organization_notification_templates_for_success=self.inventory.organization))
)
return dict(error=list(error_notification_templates), started=list(started_notification_templates), success=list(success_notification_templates))
def clean_update_on_project_update(self):
if self.update_on_project_update is True and \
self.source == 'scm' and \
InventorySource.objects.filter(
Q(inventory=self.inventory,
update_on_project_update=True, source='scm') &
~Q(id=self.id)).exists():
if (
self.update_on_project_update is True
and self.source == 'scm'
and InventorySource.objects.filter(Q(inventory=self.inventory, update_on_project_update=True, source='scm') & ~Q(id=self.id)).exists()
):
raise ValidationError(_("More than one SCM-based inventory source with update on project update per-inventory not allowed."))
return self.update_on_project_update
def clean_update_on_launch(self):
if self.update_on_project_update is True and \
self.source == 'scm' and \
self.update_on_launch is True:
raise ValidationError(_("Cannot update SCM-based inventory source on launch if set to update on project update. "
"Instead, configure the corresponding source project to update on launch."))
if self.update_on_project_update is True and self.source == 'scm' and self.update_on_launch is True:
raise ValidationError(
_(
"Cannot update SCM-based inventory source on launch if set to update on project update. "
"Instead, configure the corresponding source project to update on launch."
)
)
return self.update_on_launch
def clean_source_path(self):
@@ -1193,14 +1182,15 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return InventoryUpdate.objects.filter(inventory_source=self)
class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin, TaskManagerInventoryUpdateMixin, CustomVirtualEnvMixin):
'''
"""
Internal job for tracking inventory updates from external sources.
'''
"""
class Meta:
app_label = 'main'
@@ -1234,7 +1224,7 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
on_delete=models.CASCADE,
blank=True,
default=None,
null=True
null=True,
)
def _get_parent_field_name(self):
@@ -1243,6 +1233,7 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
@classmethod
def _get_task_class(cls):
from awx.main.tasks import RunInventoryUpdate
return RunInventoryUpdate
def _global_timeout_setting(self):
@@ -1267,9 +1258,7 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
'''Alias to source_path that combines with project path for for SCM file based sources'''
if self.inventory_source_id is None or self.inventory_source.source_project_id is None:
return self.source_path
return os.path.join(
self.inventory_source.source_project.get_project_path(check_if_exists=False),
self.source_path)
return os.path.join(self.inventory_source.source_project.get_project_path(check_if_exists=False), self.source_path)
@property
def event_class(self):
@@ -1292,6 +1281,7 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
'''
JobNotificationMixin
'''
def get_notification_templates(self):
return self.inventory_source.notification_templates
@@ -1332,17 +1322,18 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
class CustomInventoryScript(CommonModelNameNotUnique, ResourceMixin):
class Meta:
app_label = 'main'
unique_together = [('name', 'organization')]
ordering = ('name',)
script = prevent_search(models.TextField(
blank=True,
default='',
help_text=_('Inventory script contents'),
))
script = prevent_search(
models.TextField(
blank=True,
default='',
help_text=_('Inventory script contents'),
)
)
organization = models.ForeignKey(
'Organization',
related_name='custom_inventory_scripts',
@@ -1388,16 +1379,11 @@ class PluginFileInjector(object):
return '{0}.yml'.format(self.plugin_name)
def inventory_contents(self, inventory_update, private_data_dir):
"""Returns a string that is the content for the inventory file for the inventory plugin
"""
return yaml.safe_dump(
self.inventory_as_dict(inventory_update, private_data_dir),
default_flow_style=False,
width=1000
)
"""Returns a string that is the content for the inventory file for the inventory plugin"""
return yaml.safe_dump(self.inventory_as_dict(inventory_update, private_data_dir), default_flow_style=False, width=1000)
def inventory_as_dict(self, inventory_update, private_data_dir):
source_vars = dict(inventory_update.source_vars_dict) # make a copy
source_vars = dict(inventory_update.source_vars_dict) # make a copy
'''
None conveys that we should use the user-provided plugin.
Note that a plugin value of '' should still be overridden.
@@ -1414,8 +1400,7 @@ class PluginFileInjector(object):
return env
def _get_shared_env(self, inventory_update, private_data_dir, private_data_files):
"""By default, we will apply the standard managed_by_tower injectors
"""
"""By default, we will apply the standard managed_by_tower injectors"""
injected_env = {}
credential = inventory_update.get_cloud_credential()
# some sources may have no credential, specifically ec2
@@ -1425,15 +1410,14 @@ class PluginFileInjector(object):
injected_env['INVENTORY_UPDATE_ID'] = str(inventory_update.pk) # so injector knows this is inventory
if self.base_injector == 'managed':
from awx.main.models.credential import injectors as builtin_injectors
cred_kind = inventory_update.source.replace('ec2', 'aws')
if cred_kind in dir(builtin_injectors):
getattr(builtin_injectors, cred_kind)(credential, injected_env, private_data_dir)
elif self.base_injector == 'template':
safe_env = injected_env.copy()
args = []
credential.credential_type.inject_credential(
credential, injected_env, safe_env, args, private_data_dir
)
credential.credential_type.inject_credential(credential, injected_env, safe_env, args, private_data_dir)
# NOTE: safe_env is handled externally to injector class by build_safe_env static method
# that means that managed_by_tower injectors must only inject detectable env keys
# enforcement of this is accomplished by tests
@@ -1534,9 +1518,7 @@ class openstack(PluginFileInjector):
private_data = {'credentials': {}}
openstack_data = self._get_clouds_dict(inventory_update, credential, private_data_dir)
private_data['credentials'][credential] = yaml.safe_dump(
openstack_data, default_flow_style=False, allow_unicode=True
)
private_data['credentials'][credential] = yaml.safe_dump(openstack_data, default_flow_style=False, allow_unicode=True)
return private_data
def get_plugin_env(self, inventory_update, private_data_dir, private_data_files):
@@ -1548,8 +1530,8 @@ class openstack(PluginFileInjector):
class rhv(PluginFileInjector):
"""ovirt uses the custom credential templating, and that is all
"""
"""ovirt uses the custom credential templating, and that is all"""
plugin_name = 'ovirt'
base_injector = 'template'
initial_version = '2.9'

410
awx/main/models/jobs.py
View File

@@ -15,7 +15,8 @@ from urllib.parse import urljoin
from django.conf import settings
from django.core.exceptions import ValidationError
from django.db import models
#from django.core.cache import cache
# from django.core.cache import cache
from django.utils.encoding import smart_str
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _
@@ -27,15 +28,17 @@ from rest_framework.exceptions import ParseError
# AWX
from awx.api.versioning import reverse
from awx.main.models.base import (
BaseModel, CreatedModifiedModel,
prevent_search, accepts_json,
JOB_TYPE_CHOICES, NEW_JOB_TYPE_CHOICES, VERBOSITY_CHOICES,
VarsDictProperty
BaseModel,
CreatedModifiedModel,
prevent_search,
accepts_json,
JOB_TYPE_CHOICES,
NEW_JOB_TYPE_CHOICES,
VERBOSITY_CHOICES,
VarsDictProperty,
)
from awx.main.models.events import JobEvent, SystemJobEvent
from awx.main.models.unified_jobs import (
UnifiedJobTemplate, UnifiedJob
)
from awx.main.models.unified_jobs import UnifiedJobTemplate, UnifiedJob
from awx.main.models.notifications import (
NotificationTemplate,
JobNotificationMixin,
@@ -62,9 +65,9 @@ __all__ = ['JobTemplate', 'JobLaunchConfig', 'Job', 'JobHostSummary', 'SystemJob
class JobOptions(BaseModel):
'''
"""
Common options for job templates and jobs.
'''
"""
class Meta:
abstract = True
@@ -103,8 +106,7 @@ class JobOptions(BaseModel):
max_length=1024,
default='',
blank=True,
help_text=_('Branch to use in job run. Project default used if blank. '
'Only allowed if project allow_override field is set to true.'),
help_text=_('Branch to use in job run. Project default used if blank. ' 'Only allowed if project allow_override field is set to true.'),
)
forks = models.PositiveIntegerField(
blank=True,
@@ -119,10 +121,14 @@ class JobOptions(BaseModel):
blank=True,
default=0,
)
extra_vars = prevent_search(accepts_json(models.TextField(
blank=True,
default='',
)))
extra_vars = prevent_search(
accepts_json(
models.TextField(
blank=True,
default='',
)
)
)
job_tags = models.CharField(
max_length=1024,
blank=True,
@@ -157,7 +163,8 @@ class JobOptions(BaseModel):
default=False,
help_text=_(
"If enabled, Tower will act as an Ansible Fact Cache Plugin; persisting "
"facts at the end of a playbook run to the database and caching facts for use by Ansible."),
"facts at the end of a playbook run to the database and caching facts for use by Ansible."
),
)
extra_vars_dict = VarsDictProperty('extra_vars', True)
@@ -191,13 +198,12 @@ class JobOptions(BaseModel):
class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin, WebhookTemplateMixin):
'''
"""
A job template is a reusable job definition for applying a project (with
playbook) to an inventory source with a given credential.
'''
FIELDS_TO_PRESERVE_AT_COPY = [
'labels', 'instance_groups', 'credentials', 'survey_spec'
]
"""
FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'instance_groups', 'credentials', 'survey_spec']
FIELDS_TO_DISCARD_AT_COPY = ['vault_credential', 'credential']
SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')]
@@ -210,11 +216,13 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
choices=NEW_JOB_TYPE_CHOICES,
default='run',
)
host_config_key = prevent_search(models.CharField(
max_length=1024,
blank=True,
default='',
))
host_config_key = prevent_search(
models.CharField(
max_length=1024,
blank=True,
default='',
)
)
ask_diff_mode_on_launch = AskForField(
blank=True,
default=False,
@@ -223,11 +231,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
blank=True,
default=False,
)
ask_tags_on_launch = AskForField(
blank=True,
default=False,
allows_field='job_tags'
)
ask_tags_on_launch = AskForField(blank=True, default=False, allows_field='job_tags')
ask_skip_tags_on_launch = AskForField(
blank=True,
default=False,
@@ -244,26 +248,15 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
blank=True,
default=False,
)
ask_credential_on_launch = AskForField(
blank=True,
default=False,
allows_field='credentials'
)
ask_scm_branch_on_launch = AskForField(
blank=True,
default=False,
allows_field='scm_branch'
)
ask_credential_on_launch = AskForField(blank=True, default=False, allows_field='credentials')
ask_scm_branch_on_launch = AskForField(blank=True, default=False, allows_field='scm_branch')
job_slice_count = models.PositiveIntegerField(
blank=True,
default=1,
help_text=_("The number of jobs to slice into at runtime. "
"Will cause the Job Template to launch a workflow if value is greater than 1."),
help_text=_("The number of jobs to slice into at runtime. " "Will cause the Job Template to launch a workflow if value is greater than 1."),
)
admin_role = ImplicitRoleField(
parent_role=['organization.job_template_admin_role']
)
admin_role = ImplicitRoleField(parent_role=['organization.job_template_admin_role'])
execute_role = ImplicitRoleField(
parent_role=['admin_role', 'organization.execute_role'],
)
@@ -271,11 +264,11 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
parent_role=[
'organization.auditor_role',
'inventory.organization.auditor_role', # partial support for old inheritance via inventory
'execute_role', 'admin_role'
'execute_role',
'admin_role',
],
)
@classmethod
def _get_unified_job_class(cls):
return Job
@@ -283,20 +276,23 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
@classmethod
def _get_unified_job_field_names(cls):
return set(f.name for f in JobOptions._meta.fields) | set(
['name', 'description', 'organization', 'survey_passwords', 'labels', 'credentials',
'job_slice_number', 'job_slice_count', 'execution_environment']
['name', 'description', 'organization', 'survey_passwords', 'labels', 'credentials', 'job_slice_number', 'job_slice_count', 'execution_environment']
)
@property
def validation_errors(self):
'''
"""
Fields needed to start, which cannot be given on launch, invalid state.
'''
"""
validation_errors = {}
if self.inventory is None and not self.ask_inventory_on_launch:
validation_errors['inventory'] = [_("Job Template must provide 'inventory' or allow prompting for it."),]
validation_errors['inventory'] = [
_("Job Template must provide 'inventory' or allow prompting for it."),
]
if self.project is None:
validation_errors['project'] = [_("Job Templates must have a project assigned."),]
validation_errors['project'] = [
_("Job Templates must have a project assigned."),
]
return validation_errors
@property
@@ -309,9 +305,9 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
return self.forks
def create_job(self, **kwargs):
'''
"""
Create a new job based on this template.
'''
"""
return self.create_unified_job(**kwargs)
def get_effective_slice_ct(self, kwargs):
@@ -349,12 +345,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
if self.pk:
qs = qs.exclude(pk=self.pk)
if qs.exists():
errors.append(
'%s with this (%s) combination already exists.' % (
JobTemplate.__name__,
', '.join(set(ut) - {'polymorphic_ctype'})
)
)
errors.append('%s with this (%s) combination already exists.' % (JobTemplate.__name__, ', '.join(set(ut) - {'polymorphic_ctype'})))
if errors:
raise ValidationError(errors)
@@ -365,6 +356,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
if slice_event:
# A Slice Job Template will generate a WorkflowJob rather than a Job
from awx.main.models.workflow import WorkflowJobTemplate, WorkflowJobNode
kwargs['_unified_job_class'] = WorkflowJobTemplate._get_unified_job_class()
kwargs['_parent_field_name'] = "job_template"
kwargs.setdefault('_eager_fields', {})
@@ -379,9 +371,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
job = super(JobTemplate, self).create_unified_job(**kwargs)
if slice_event:
for idx in range(slice_ct):
create_kwargs = dict(workflow_job=job,
unified_job_template=self,
ancestor_artifacts=dict(job_slice=idx + 1))
create_kwargs = dict(workflow_job=job, unified_job_template=self, ancestor_artifacts=dict(job_slice=idx + 1))
WorkflowJobNode.objects.create(**create_kwargs)
return job
@@ -389,10 +379,10 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
return reverse('api:job_template_detail', kwargs={'pk': self.pk}, request=request)
def can_start_without_user_input(self, callback_extra_vars=None):
'''
"""
Return whether job template can be used to start a new job without
requiring any user input.
'''
"""
variables_needed = False
if callback_extra_vars:
extra_vars_dict = parse_yaml_or_json(callback_extra_vars)
@@ -411,18 +401,15 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
if getattr(self, ask_field_name):
prompting_needed = True
break
return (not prompting_needed and
not self.passwords_needed_to_start and
not variables_needed)
return not prompting_needed and not self.passwords_needed_to_start and not variables_needed
def _accept_or_ignore_job_kwargs(self, **kwargs):
exclude_errors = kwargs.pop('_exclude_errors', [])
prompted_data = {}
rejected_data = {}
accepted_vars, rejected_vars, errors_dict = self.accept_or_ignore_variables(
kwargs.get('extra_vars', {}),
_exclude_errors=exclude_errors,
extra_passwords=kwargs.get('survey_passwords', {}))
kwargs.get('extra_vars', {}), _exclude_errors=exclude_errors, extra_passwords=kwargs.get('survey_passwords', {})
)
if accepted_vars:
prompted_data['extra_vars'] = accepted_vars
if rejected_vars:
@@ -472,11 +459,8 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
if 'prompts' not in exclude_errors:
errors_dict[field_name] = _('Field is not configured to prompt on launch.')
if ('prompts' not in exclude_errors and
(not getattr(self, 'ask_credential_on_launch', False)) and
self.passwords_needed_to_start):
errors_dict['passwords_needed_to_start'] = _(
'Saved launch configurations cannot provide passwords needed to start.')
if 'prompts' not in exclude_errors and (not getattr(self, 'ask_credential_on_launch', False)) and self.passwords_needed_to_start:
errors_dict['passwords_needed_to_start'] = _('Saved launch configurations cannot provide passwords needed to start.')
needed = self.resources_needed_to_start
if needed:
@@ -493,8 +477,10 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
@property
def cache_timeout_blocked(self):
if Job.objects.filter(job_template=self, status__in=['pending', 'waiting', 'running']).count() >= getattr(settings, 'SCHEDULE_MAX_JOBS', 10):
logger.error("Job template %s could not be started because there are more than %s other jobs from that template waiting to run" %
(self.name, getattr(settings, 'SCHEDULE_MAX_JOBS', 10)))
logger.error(
"Job template %s could not be started because there are more than %s other jobs from that template waiting to run"
% (self.name, getattr(settings, 'SCHEDULE_MAX_JOBS', 10))
)
return True
return False
@@ -507,37 +493,40 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
# TODO: Currently there is no org fk on project so this will need to be added once that is
# available after the rbac pr
base_notification_templates = NotificationTemplate.objects
error_notification_templates = list(base_notification_templates.filter(
unifiedjobtemplate_notification_templates_for_errors__in=[self, self.project]))
started_notification_templates = list(base_notification_templates.filter(
unifiedjobtemplate_notification_templates_for_started__in=[self, self.project]))
success_notification_templates = list(base_notification_templates.filter(
unifiedjobtemplate_notification_templates_for_success__in=[self, self.project]))
error_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_errors__in=[self, self.project]))
started_notification_templates = list(
base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_started__in=[self, self.project])
)
success_notification_templates = list(
base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_success__in=[self, self.project])
)
# Get Organization NotificationTemplates
if self.organization is not None:
error_notification_templates = set(error_notification_templates + list(base_notification_templates.filter(
organization_notification_templates_for_errors=self.organization)))
started_notification_templates = set(started_notification_templates + list(base_notification_templates.filter(
organization_notification_templates_for_started=self.organization)))
success_notification_templates = set(success_notification_templates + list(base_notification_templates.filter(
organization_notification_templates_for_success=self.organization)))
return dict(error=list(error_notification_templates),
started=list(started_notification_templates),
success=list(success_notification_templates))
error_notification_templates = set(
error_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_errors=self.organization))
)
started_notification_templates = set(
started_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_started=self.organization))
)
success_notification_templates = set(
success_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_success=self.organization))
)
return dict(error=list(error_notification_templates), started=list(started_notification_templates), success=list(success_notification_templates))
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return UnifiedJob.objects.filter(unified_job_template=self)
class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskManagerJobMixin, CustomVirtualEnvMixin, WebhookMixin):
'''
"""
A job applies a project (with playbook) to an inventory source with a given
credential. It represents a single invocation of ansible-playbook with the
given parameters.
'''
"""
class Meta:
app_label = 'main'
@@ -581,23 +570,21 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
job_slice_number = models.PositiveIntegerField(
blank=True,
default=0,
help_text=_("If part of a sliced job, the ID of the inventory slice operated on. "
"If not part of sliced job, parameter is not used."),
help_text=_("If part of a sliced job, the ID of the inventory slice operated on. " "If not part of sliced job, parameter is not used."),
)
job_slice_count = models.PositiveIntegerField(
blank=True,
default=1,
help_text=_("If ran as part of sliced jobs, the total number of slices. "
"If 1, job is not part of a sliced job."),
help_text=_("If ran as part of sliced jobs, the total number of slices. " "If 1, job is not part of a sliced job."),
)
def _get_parent_field_name(self):
return 'job_template'
@classmethod
def _get_task_class(cls):
from awx.main.tasks import RunJob
return RunJob
@classmethod
@@ -623,7 +610,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
for virtualenv in (
self.job_template.custom_virtualenv if self.job_template else None,
self.project.custom_virtualenv,
self.organization.custom_virtualenv if self.organization else None
self.organization.custom_virtualenv if self.organization else None,
):
if virtualenv:
return virtualenv
@@ -651,10 +638,10 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
return Host.objects.filter(**kwargs)
def retry_qs(self, status):
'''
"""
Returns Host queryset that will be used to produce the `limit`
field in a retry on a subset of hosts
'''
"""
kwargs = {}
if status == 'all':
pass
@@ -668,9 +655,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
status_field = status
kwargs['job_host_summaries__{}__gt'.format(status_field)] = 0
else:
raise ParseError(_(
'{status_value} is not a valid status option.'
).format(status_value=status))
raise ParseError(_('{status_value} is not a valid status option.').format(status_value=status))
return self._get_hosts(**kwargs)
@property
@@ -736,31 +721,37 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
else:
summaries = self.job_host_summaries.all()
for h in self.job_host_summaries.all():
all_hosts[h.host_name] = dict(failed=h.failed,
changed=h.changed,
dark=h.dark,
failures=h.failures,
ok=h.ok,
processed=h.processed,
skipped=h.skipped,
rescued=h.rescued,
ignored=h.ignored)
data.update(dict(inventory=self.inventory.name if self.inventory else None,
project=self.project.name if self.project else None,
playbook=self.playbook,
credential=getattr(self.machine_credential, 'name', None),
limit=self.limit,
extra_vars=self.display_extra_vars(),
hosts=all_hosts))
all_hosts[h.host_name] = dict(
failed=h.failed,
changed=h.changed,
dark=h.dark,
failures=h.failures,
ok=h.ok,
processed=h.processed,
skipped=h.skipped,
rescued=h.rescued,
ignored=h.ignored,
)
data.update(
dict(
inventory=self.inventory.name if self.inventory else None,
project=self.project.name if self.project else None,
playbook=self.playbook,
credential=getattr(self.machine_credential, 'name', None),
limit=self.limit,
extra_vars=self.display_extra_vars(),
hosts=all_hosts,
)
)
return data
def _resources_sufficient_for_launch(self):
return not (self.inventory_id is None or self.project_id is None)
def display_artifacts(self):
'''
"""
Hides artifacts if they are marked as no_log type artifacts.
'''
"""
artifacts = self.artifacts
if artifacts.get('_ansible_no_log', False):
return "$hidden due to Ansible no_log flag$"
@@ -811,6 +802,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
'''
JobNotificationMixin
'''
def get_notification_templates(self):
if not self.job_template:
return NotificationTemplate.objects.none()
@@ -819,10 +811,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
def get_notification_friendly_name(self):
return "Job"
def _get_inventory_hosts(
self,
only=['name', 'ansible_facts', 'ansible_facts_modified', 'modified', 'inventory_id']
):
def _get_inventory_hosts(self, only=['name', 'ansible_facts', 'ansible_facts_modified', 'modified', 'inventory_id']):
if not self.inventory:
return []
return self.inventory.hosts.only(*only)
@@ -876,44 +865,46 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
ansible_facts_system_id = ansible_facts.get('system_id', None) if isinstance(ansible_facts, dict) else None
if ansible_local_system_id:
print("Setting local {}".format(ansible_local_system_id))
logger.debug("Insights system_id {} found for host <{}, {}> in"
" ansible local facts".format(ansible_local_system_id,
host.inventory.id,
host.name))
logger.debug(
"Insights system_id {} found for host <{}, {}> in"
" ansible local facts".format(ansible_local_system_id, host.inventory.id, host.name)
)
host.insights_system_id = ansible_local_system_id
elif ansible_facts_system_id:
logger.debug("Insights system_id {} found for host <{}, {}> in"
" insights facts".format(ansible_local_system_id,
host.inventory.id,
host.name))
logger.debug(
"Insights system_id {} found for host <{}, {}> in"
" insights facts".format(ansible_local_system_id, host.inventory.id, host.name)
)
host.insights_system_id = ansible_facts_system_id
host.save()
system_tracking_logger.info(
'New fact for inventory {} host {}'.format(
smart_str(host.inventory.name), smart_str(host.name)),
extra=dict(inventory_id=host.inventory.id, host_name=host.name,
ansible_facts=host.ansible_facts,
ansible_facts_modified=host.ansible_facts_modified.isoformat(),
job_id=self.id))
'New fact for inventory {} host {}'.format(smart_str(host.inventory.name), smart_str(host.name)),
extra=dict(
inventory_id=host.inventory.id,
host_name=host.name,
ansible_facts=host.ansible_facts,
ansible_facts_modified=host.ansible_facts_modified.isoformat(),
job_id=self.id,
),
)
else:
# if the file goes missing, ansible removed it (likely via clear_facts)
host.ansible_facts = {}
host.ansible_facts_modified = now()
system_tracking_logger.info(
'Facts cleared for inventory {} host {}'.format(
smart_str(host.inventory.name), smart_str(host.name)))
system_tracking_logger.info('Facts cleared for inventory {} host {}'.format(smart_str(host.inventory.name), smart_str(host.name)))
host.save()
class LaunchTimeConfigBase(BaseModel):
'''
"""
Needed as separate class from LaunchTimeConfig because some models
use `extra_data` and some use `extra_vars`. We cannot change the API,
so we force fake it in the model definitions
- model defines extra_vars - use this class
- model needs to use extra data - use LaunchTimeConfig
Use this for models which are SurveyMixins and UnifiedJobs or Templates
'''
"""
class Meta:
abstract = True
@@ -925,14 +916,11 @@ class LaunchTimeConfigBase(BaseModel):
null=True,
default=None,
on_delete=models.SET_NULL,
help_text=_('Inventory applied as a prompt, assuming job template prompts for inventory')
help_text=_('Inventory applied as a prompt, assuming job template prompts for inventory'),
)
# All standard fields are stored in this dictionary field
# This is a solution to the nullable CharField problem, specific to prompting
char_prompts = JSONField(
blank=True,
default=dict
)
char_prompts = JSONField(blank=True, default=dict)
def prompts_dict(self, display=False):
data = {}
@@ -976,28 +964,25 @@ for field_name in JobTemplate.get_ask_mapping().keys():
class LaunchTimeConfig(LaunchTimeConfigBase):
'''
"""
Common model for all objects that save details of a saved launch config
WFJT / WJ nodes, schedules, and job launch configs (not all implemented yet)
'''
"""
class Meta:
abstract = True
# Special case prompting fields, even more special than the other ones
extra_data = JSONField(
blank=True,
default=dict
extra_data = JSONField(blank=True, default=dict)
survey_passwords = prevent_search(
JSONField(
blank=True,
default=dict,
editable=False,
)
)
survey_passwords = prevent_search(JSONField(
blank=True,
default=dict,
editable=False,
))
# Credentials needed for non-unified job / unified JT models
credentials = models.ManyToManyField(
'Credential',
related_name='%(class)ss'
)
credentials = models.ManyToManyField('Credential', related_name='%(class)ss')
@property
def extra_vars(self):
@@ -1008,9 +993,9 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
self.extra_data = extra_vars
def display_extra_vars(self):
'''
"""
Hides fields marked as passwords in survey.
'''
"""
if hasattr(self, 'survey_passwords') and self.survey_passwords:
extra_vars = parse_yaml_or_json(self.extra_vars).copy()
for key, value in self.survey_passwords.items():
@@ -1025,11 +1010,12 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
class JobLaunchConfig(LaunchTimeConfig):
'''
"""
Historical record of user launch-time overrides for a job
Not exposed in the API
Used for relaunch, scheduling, etc.
'''
"""
class Meta:
app_label = 'main'
@@ -1041,18 +1027,18 @@ class JobLaunchConfig(LaunchTimeConfig):
)
def has_user_prompts(self, template):
'''
"""
Returns True if any fields exist in the launch config that are
not permissions exclusions
(has to exist because of callback relaunch exception)
'''
"""
return self._has_user_prompts(template, only_unprompted=False)
def has_unprompted(self, template):
'''
"""
returns True if the template has set ask_ fields to False after
launching with those prompts
'''
"""
return self._has_user_prompts(template, only_unprompted=True)
def _has_user_prompts(self, template, only_unprompted=True):
@@ -1061,10 +1047,7 @@ class JobLaunchConfig(LaunchTimeConfig):
if template.survey_enabled and (not template.ask_variables_on_launch):
ask_mapping.pop('extra_vars')
provided_vars = set(prompts.get('extra_vars', {}).keys())
survey_vars = set(
element.get('variable') for element in
template.survey_spec.get('spec', {}) if 'variable' in element
)
survey_vars = set(element.get('variable') for element in template.survey_spec.get('spec', {}) if 'variable' in element)
if (provided_vars and not only_unprompted) or (provided_vars - survey_vars):
return True
for field_name, ask_field_name in ask_mapping.items():
@@ -1077,9 +1060,9 @@ class JobLaunchConfig(LaunchTimeConfig):
class JobHostSummary(CreatedModifiedModel):
'''
"""
Per-host statistics for each job.
'''
"""
class Meta:
app_label = 'main'
@@ -1093,12 +1076,7 @@ class JobHostSummary(CreatedModifiedModel):
on_delete=models.CASCADE,
editable=False,
)
host = models.ForeignKey('Host',
related_name='job_host_summaries',
null=True,
default=None,
on_delete=models.SET_NULL,
editable=False)
host = models.ForeignKey('Host', related_name='job_host_summaries', null=True, default=None, on_delete=models.SET_NULL, editable=False)
host_name = models.CharField(
max_length=1024,
@@ -1119,9 +1097,17 @@ class JobHostSummary(CreatedModifiedModel):
def __str__(self):
host = getattr_dne(self, 'host')
hostname = host.name if host else 'N/A'
return '%s changed=%d dark=%d failures=%d ignored=%d ok=%d processed=%d rescued=%d skipped=%s' % \
(hostname, self.changed, self.dark, self.failures, self.ignored, self.ok,
self.processed, self.rescued, self.skipped)
return '%s changed=%d dark=%d failures=%d ignored=%d ok=%d processed=%d rescued=%d skipped=%s' % (
hostname,
self.changed,
self.dark,
self.failures,
self.ignored,
self.ok,
self.processed,
self.rescued,
self.skipped,
)
def get_absolute_url(self, request=None):
return reverse('api:job_host_summary_detail', kwargs={'pk': self.pk}, request=request)
@@ -1138,15 +1124,15 @@ class JobHostSummary(CreatedModifiedModel):
class SystemJobOptions(BaseModel):
'''
"""
Common fields for SystemJobTemplate and SystemJob.
'''
"""
SYSTEM_JOB_TYPE = [
('cleanup_jobs', _('Remove jobs older than a certain number of days')),
('cleanup_activitystream', _('Remove activity stream entries older than a certain number of days')),
('cleanup_sessions', _('Removes expired browser sessions from the database')),
('cleanup_tokens', _('Removes expired OAuth 2 access tokens and refresh tokens'))
('cleanup_tokens', _('Removes expired OAuth 2 access tokens and refresh tokens')),
]
class Meta:
@@ -1161,7 +1147,6 @@ class SystemJobOptions(BaseModel):
class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions):
class Meta:
app_label = 'main'
@@ -1184,15 +1169,10 @@ class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions):
def notification_templates(self):
# TODO: Go through RBAC instead of calling all(). Need to account for orphaned NotificationTemplates
base_notification_templates = NotificationTemplate.objects.all()
error_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
started_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
success_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
return dict(error=list(error_notification_templates),
started=list(started_notification_templates),
success=list(success_notification_templates))
error_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
started_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
success_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
return dict(error=list(error_notification_templates), started=list(started_notification_templates), success=list(success_notification_templates))
def _accept_or_ignore_job_kwargs(self, _exclude_errors=None, **kwargs):
extra_data = kwargs.pop('extra_vars', {})
@@ -1205,19 +1185,18 @@ class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions):
return (prompted_data, rejected_data, errors)
def _accept_or_ignore_variables(self, data, errors, _exclude_errors=()):
'''
"""
Unlike other templates, like project updates and inventory sources,
system job templates can accept a limited number of fields
used as options for the management commands.
'''
"""
rejected = {}
allowed_vars = set(['days', 'older_than', 'granularity'])
given_vars = set(data.keys())
unallowed_vars = given_vars - (allowed_vars & given_vars)
errors_list = []
if unallowed_vars:
errors_list.append(_('Variables {list_of_keys} are not allowed for system jobs.').format(
list_of_keys=', '.join(unallowed_vars)))
errors_list.append(_('Variables {list_of_keys} are not allowed for system jobs.').format(list_of_keys=', '.join(unallowed_vars)))
for key in unallowed_vars:
rejected[key] = data.pop(key)
@@ -1241,7 +1220,6 @@ class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions):
class SystemJob(UnifiedJob, SystemJobOptions, JobNotificationMixin):
class Meta:
app_label = 'main'
ordering = ('id',)
@@ -1255,10 +1233,12 @@ class SystemJob(UnifiedJob, SystemJobOptions, JobNotificationMixin):
on_delete=models.SET_NULL,
)
extra_vars = prevent_search(models.TextField(
blank=True,
default='',
))
extra_vars = prevent_search(
models.TextField(
blank=True,
default='',
)
)
extra_vars_dict = VarsDictProperty('extra_vars', True)
@@ -1269,6 +1249,7 @@ class SystemJob(UnifiedJob, SystemJobOptions, JobNotificationMixin):
@classmethod
def _get_task_class(cls):
from awx.main.tasks import RunSystemJob
return RunSystemJob
def websocket_emit_data(self):
@@ -1297,6 +1278,7 @@ class SystemJob(UnifiedJob, SystemJobOptions, JobNotificationMixin):
'''
JobNotificationMixin
'''
def get_notification_templates(self):
return self.system_job_template.notification_templates

19
awx/main/models/label.py
View File

@@ -10,13 +10,13 @@ from awx.api.versioning import reverse
from awx.main.models.base import CommonModelNameNotUnique
from awx.main.models.unified_jobs import UnifiedJobTemplate, UnifiedJob
__all__ = ('Label', )
__all__ = ('Label',)
class Label(CommonModelNameNotUnique):
'''
"""
Generic Tag. Designed for tagging Job Templates, but expandable to other models.
'''
"""
class Meta:
app_label = 'main'
@@ -35,19 +35,10 @@ class Label(CommonModelNameNotUnique):
@staticmethod
def get_orphaned_labels():
return \
Label.objects.filter(
organization=None,
unifiedjobtemplate_labels__isnull=True
)
return Label.objects.filter(organization=None, unifiedjobtemplate_labels__isnull=True)
def is_detached(self):
return bool(
Label.objects.filter(
id=self.id,
unifiedjob_labels__isnull=True,
unifiedjobtemplate_labels__isnull=True
).count())
return bool(Label.objects.filter(id=self.id, unifiedjob_labels__isnull=True, unifiedjobtemplate_labels__isnull=True).count())
def is_candidate_for_detach(self):
c1 = UnifiedJob.objects.filter(labels__in=[self.id]).count()

253
awx/main/models/mixins.py
View File

@@ -9,7 +9,7 @@ import requests
# Django
from django.apps import apps
from django.conf import settings
from django.contrib.auth.models import User # noqa
from django.contrib.auth.models import User # noqa
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.db import models
@@ -19,9 +19,7 @@ from django.utils.translation import ugettext_lazy as _
# AWX
from awx.main.models.base import prevent_search
from awx.main.models.rbac import (
Role, RoleAncestorEntry, get_roles_on_resource
)
from awx.main.models.rbac import Role, RoleAncestorEntry, get_roles_on_resource
from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic
from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted
from awx.main.utils.polymorphic import build_polymorphic_ctypes_map
@@ -32,19 +30,26 @@ from awx.main.constants import ACTIVE_STATES
logger = logging.getLogger('awx.main.models.mixins')
__all__ = ['ResourceMixin', 'SurveyJobTemplateMixin', 'SurveyJobMixin',
'TaskManagerUnifiedJobMixin', 'TaskManagerJobMixin', 'TaskManagerProjectUpdateMixin',
'TaskManagerInventoryUpdateMixin', 'ExecutionEnvironmentMixin', 'CustomVirtualEnvMixin']
__all__ = [
'ResourceMixin',
'SurveyJobTemplateMixin',
'SurveyJobMixin',
'TaskManagerUnifiedJobMixin',
'TaskManagerJobMixin',
'TaskManagerProjectUpdateMixin',
'TaskManagerInventoryUpdateMixin',
'ExecutionEnvironmentMixin',
'CustomVirtualEnvMixin',
]
class ResourceMixin(models.Model):
class Meta:
abstract = True
@classmethod
def accessible_objects(cls, accessor, role_field):
'''
"""
Use instead of `MyModel.objects` when you want to only consider
resources that a user has specific permissions for. For example:
MyModel.accessible_objects(user, 'read_role').filter(name__istartswith='bar');
@@ -52,7 +57,7 @@ class ResourceMixin(models.Model):
specific resource you want to check permissions on, it is more
performant to resolve the resource in question then call
`myresource.get_permissions(user)`.
'''
"""
return ResourceMixin._accessible_objects(cls, accessor, role_field)
@classmethod
@@ -67,32 +72,25 @@ class ResourceMixin(models.Model):
ancestor_roles = [accessor]
else:
accessor_type = ContentType.objects.get_for_model(accessor)
ancestor_roles = Role.objects.filter(content_type__pk=accessor_type.id,
object_id=accessor.id)
ancestor_roles = Role.objects.filter(content_type__pk=accessor_type.id, object_id=accessor.id)
if content_types is None:
ct_kwarg = dict(content_type_id = ContentType.objects.get_for_model(cls).id)
ct_kwarg = dict(content_type_id=ContentType.objects.get_for_model(cls).id)
else:
ct_kwarg = dict(content_type_id__in = content_types)
return RoleAncestorEntry.objects.filter(
ancestor__in = ancestor_roles,
role_field = role_field,
**ct_kwarg
).values_list('object_id').distinct()
ct_kwarg = dict(content_type_id__in=content_types)
return RoleAncestorEntry.objects.filter(ancestor__in=ancestor_roles, role_field=role_field, **ct_kwarg).values_list('object_id').distinct()
@staticmethod
def _accessible_objects(cls, accessor, role_field):
return cls.objects.filter(pk__in = ResourceMixin._accessible_pk_qs(cls, accessor, role_field))
return cls.objects.filter(pk__in=ResourceMixin._accessible_pk_qs(cls, accessor, role_field))
def get_permissions(self, accessor):
'''
"""
Returns a string list of the roles a accessor has for a given resource.
An accessor can be either a User, Role, or an arbitrary resource that
contains one or more Roles associated with it.
'''
"""
return get_roles_on_resource(self, accessor)
@@ -104,15 +102,13 @@ class SurveyJobTemplateMixin(models.Model):
survey_enabled = models.BooleanField(
default=False,
)
survey_spec = prevent_search(JSONField(
blank=True,
default=dict,
))
ask_variables_on_launch = AskForField(
blank=True,
default=False,
allows_field='extra_vars'
survey_spec = prevent_search(
JSONField(
blank=True,
default=dict,
)
)
ask_variables_on_launch = AskForField(blank=True, default=False, allows_field='extra_vars')
def survey_password_variables(self):
vars = []
@@ -133,7 +129,7 @@ class SurveyJobTemplateMixin(models.Model):
return vars
def _update_unified_job_kwargs(self, create_kwargs, kwargs):
'''
"""
Combine extra_vars with variable precedence order:
JT extra_vars -> JT survey defaults -> runtime extra_vars
@@ -143,7 +139,7 @@ class SurveyJobTemplateMixin(models.Model):
:type kwargs: dict
:return: modified create_kwargs.
:rtype: dict
'''
"""
# Job Template extra_vars
extra_vars = self.extra_vars_dict
@@ -170,11 +166,7 @@ class SurveyJobTemplateMixin(models.Model):
if default is not None:
decrypted_default = default
if (
survey_element['type'] == "password" and
isinstance(decrypted_default, str) and
decrypted_default.startswith('$encrypted$')
):
if survey_element['type'] == "password" and isinstance(decrypted_default, str) and decrypted_default.startswith('$encrypted$'):
decrypted_default = decrypt_value(get_encryption_key('value', pk=None), decrypted_default)
errors = self._survey_element_validation(survey_element, {variable_key: decrypted_default})
if not errors:
@@ -192,12 +184,9 @@ class SurveyJobTemplateMixin(models.Model):
# default (if any) will be validated against instead
errors = []
if (survey_element['type'] == "password"):
if survey_element['type'] == "password":
password_value = data.get(survey_element['variable'])
if (
isinstance(password_value, str) and
password_value == '$encrypted$'
):
if isinstance(password_value, str) and password_value == '$encrypted$':
if survey_element.get('default') is None and survey_element['required']:
if validate_required:
errors.append("'%s' value missing" % survey_element['variable'])
@@ -209,43 +198,60 @@ class SurveyJobTemplateMixin(models.Model):
elif survey_element['type'] in ["textarea", "text", "password"]:
if survey_element['variable'] in data:
if not isinstance(data[survey_element['variable']], str):
errors.append("Value %s for '%s' expected to be a string." % (data[survey_element['variable']],
survey_element['variable']))
errors.append("Value %s for '%s' expected to be a string." % (data[survey_element['variable']], survey_element['variable']))
return errors
if 'min' in survey_element and survey_element['min'] not in ["", None] and len(data[survey_element['variable']]) < int(survey_element['min']):
errors.append("'%s' value %s is too small (length is %s must be at least %s)." %
(survey_element['variable'], data[survey_element['variable']], len(data[survey_element['variable']]), survey_element['min']))
errors.append(
"'%s' value %s is too small (length is %s must be at least %s)."
% (survey_element['variable'], data[survey_element['variable']], len(data[survey_element['variable']]), survey_element['min'])
)
if 'max' in survey_element and survey_element['max'] not in ["", None] and len(data[survey_element['variable']]) > int(survey_element['max']):
errors.append("'%s' value %s is too large (must be no more than %s)." %
(survey_element['variable'], data[survey_element['variable']], survey_element['max']))
errors.append(
"'%s' value %s is too large (must be no more than %s)."
% (survey_element['variable'], data[survey_element['variable']], survey_element['max'])
)
elif survey_element['type'] == 'integer':
if survey_element['variable'] in data:
if type(data[survey_element['variable']]) != int:
errors.append("Value %s for '%s' expected to be an integer." % (data[survey_element['variable']],
survey_element['variable']))
errors.append("Value %s for '%s' expected to be an integer." % (data[survey_element['variable']], survey_element['variable']))
return errors
if 'min' in survey_element and survey_element['min'] not in ["", None] and survey_element['variable'] in data and \
data[survey_element['variable']] < int(survey_element['min']):
errors.append("'%s' value %s is too small (must be at least %s)." %
(survey_element['variable'], data[survey_element['variable']], survey_element['min']))
if 'max' in survey_element and survey_element['max'] not in ["", None] and survey_element['variable'] in data and \
data[survey_element['variable']] > int(survey_element['max']):
errors.append("'%s' value %s is too large (must be no more than %s)." %
(survey_element['variable'], data[survey_element['variable']], survey_element['max']))
if (
'min' in survey_element
and survey_element['min'] not in ["", None]
and survey_element['variable'] in data
and data[survey_element['variable']] < int(survey_element['min'])
):
errors.append(
"'%s' value %s is too small (must be at least %s)."
% (survey_element['variable'], data[survey_element['variable']], survey_element['min'])
)
if (
'max' in survey_element
and survey_element['max'] not in ["", None]
and survey_element['variable'] in data
and data[survey_element['variable']] > int(survey_element['max'])
):
errors.append(
"'%s' value %s is too large (must be no more than %s)."
% (survey_element['variable'], data[survey_element['variable']], survey_element['max'])
)
elif survey_element['type'] == 'float':
if survey_element['variable'] in data:
if type(data[survey_element['variable']]) not in (float, int):
errors.append("Value %s for '%s' expected to be a numeric type." % (data[survey_element['variable']],
survey_element['variable']))
errors.append("Value %s for '%s' expected to be a numeric type." % (data[survey_element['variable']], survey_element['variable']))
return errors
if 'min' in survey_element and survey_element['min'] not in ["", None] and data[survey_element['variable']] < float(survey_element['min']):
errors.append("'%s' value %s is too small (must be at least %s)." %
(survey_element['variable'], data[survey_element['variable']], survey_element['min']))
errors.append(
"'%s' value %s is too small (must be at least %s)."
% (survey_element['variable'], data[survey_element['variable']], survey_element['min'])
)
if 'max' in survey_element and survey_element['max'] not in ["", None] and data[survey_element['variable']] > float(survey_element['max']):
errors.append("'%s' value %s is too large (must be no more than %s)." %
(survey_element['variable'], data[survey_element['variable']], survey_element['max']))
errors.append(
"'%s' value %s is too large (must be no more than %s)."
% (survey_element['variable'], data[survey_element['variable']], survey_element['max'])
)
elif survey_element['type'] == 'multiselect':
if survey_element['variable'] in data:
if type(data[survey_element['variable']]) != list:
@@ -256,21 +262,18 @@ class SurveyJobTemplateMixin(models.Model):
choice_list = [choice for choice in choice_list.splitlines() if choice.strip() != '']
for val in data[survey_element['variable']]:
if val not in choice_list:
errors.append("Value %s for '%s' expected to be one of %s." % (val, survey_element['variable'],
choice_list))
errors.append("Value %s for '%s' expected to be one of %s." % (val, survey_element['variable'], choice_list))
elif survey_element['type'] == 'multiplechoice':
choice_list = copy(survey_element['choices'])
if isinstance(choice_list, str):
choice_list = [choice for choice in choice_list.splitlines() if choice.strip() != '']
if survey_element['variable'] in data:
if data[survey_element['variable']] not in choice_list:
errors.append("Value %s for '%s' expected to be one of %s." % (data[survey_element['variable']],
survey_element['variable'],
choice_list))
errors.append("Value %s for '%s' expected to be one of %s." % (data[survey_element['variable']], survey_element['variable'], choice_list))
return errors
def _accept_or_ignore_variables(self, data, errors=None, _exclude_errors=(), extra_passwords=None):
survey_is_enabled = (self.survey_enabled and self.survey_spec)
survey_is_enabled = self.survey_enabled and self.survey_spec
extra_vars = data.copy()
if errors is None:
errors = {}
@@ -285,12 +288,11 @@ class SurveyJobTemplateMixin(models.Model):
value = data.get(key, None)
validate_required = 'required' not in _exclude_errors
if extra_passwords and key in extra_passwords and is_encrypted(value):
element_errors = self._survey_element_validation(survey_element, {
key: decrypt_value(get_encryption_key('value', pk=None), value)
}, validate_required=validate_required)
else:
element_errors = self._survey_element_validation(
survey_element, data, validate_required=validate_required)
survey_element, {key: decrypt_value(get_encryption_key('value', pk=None), value)}, validate_required=validate_required
)
else:
element_errors = self._survey_element_validation(survey_element, data, validate_required=validate_required)
if element_errors:
survey_errors += element_errors
@@ -309,7 +311,7 @@ class SurveyJobTemplateMixin(models.Model):
if extra_vars:
# Prune the prompted variables for those identical to template
tmp_extra_vars = self.extra_vars_dict
for key in (set(tmp_extra_vars.keys()) & set(extra_vars.keys())):
for key in set(tmp_extra_vars.keys()) & set(extra_vars.keys()):
if tmp_extra_vars[key] == extra_vars[key]:
extra_vars.pop(key)
@@ -318,18 +320,20 @@ class SurveyJobTemplateMixin(models.Model):
rejected.update(extra_vars)
# ignored variables does not block manual launch
if 'prompts' not in _exclude_errors:
errors['extra_vars'] = [_('Variables {list_of_keys} are not allowed on launch. Check the Prompt on Launch setting '+
'on the {model_name} to include Extra Variables.').format(
list_of_keys=', '.join([str(key) for key in extra_vars.keys()]),
model_name=self._meta.verbose_name.title())]
errors['extra_vars'] = [
_(
'Variables {list_of_keys} are not allowed on launch. Check the Prompt on Launch setting '
+ 'on the {model_name} to include Extra Variables.'
).format(list_of_keys=', '.join([str(key) for key in extra_vars.keys()]), model_name=self._meta.verbose_name.title())
]
return (accepted, rejected, errors)
@staticmethod
def pivot_spec(spec):
'''
"""
Utility method that will return a dictionary keyed off variable names
'''
"""
pivoted = {}
for element_data in spec.get('spec', []):
if 'variable' in element_data:
@@ -349,9 +353,9 @@ class SurveyJobTemplateMixin(models.Model):
return errors
def display_survey_spec(self):
'''
"""
Hide encrypted default passwords in survey specs
'''
"""
survey_spec = deepcopy(self.survey_spec) if self.survey_spec else {}
for field in survey_spec.get('spec', []):
if field.get('type') == 'password':
@@ -364,16 +368,18 @@ class SurveyJobMixin(models.Model):
class Meta:
abstract = True
survey_passwords = prevent_search(JSONField(
blank=True,
default=dict,
editable=False,
))
survey_passwords = prevent_search(
JSONField(
blank=True,
default=dict,
editable=False,
)
)
def display_extra_vars(self):
'''
"""
Hides fields marked as passwords in survey.
'''
"""
if self.survey_passwords:
extra_vars = json.loads(self.extra_vars)
for key, value in self.survey_passwords.items():
@@ -384,9 +390,9 @@ class SurveyJobMixin(models.Model):
return self.extra_vars
def decrypted_extra_vars(self):
'''
"""
Decrypts fields marked as passwords in survey.
'''
"""
if self.survey_passwords:
extra_vars = json.loads(self.extra_vars)
for key in self.survey_passwords:
@@ -484,19 +490,13 @@ class CustomVirtualEnvMixin(models.Model):
abstract = True
custom_virtualenv = models.CharField(
blank=True,
null=True,
default=None,
max_length=100,
help_text=_('Local absolute file path containing a custom Python virtualenv to use')
blank=True, null=True, default=None, max_length=100, help_text=_('Local absolute file path containing a custom Python virtualenv to use')
)
def clean_custom_virtualenv(self):
value = self.custom_virtualenv
if value and os.path.join(value, '') not in get_custom_venv_choices():
raise ValidationError(
_('{} is not a valid virtualenv in {}').format(value, settings.BASE_VENV_PATH)
)
raise ValidationError(_('{} is not a valid virtualenv in {}').format(value, settings.BASE_VENV_PATH))
if value:
return os.path.join(value, '')
return None
@@ -504,12 +504,13 @@ class CustomVirtualEnvMixin(models.Model):
class RelatedJobsMixin(object):
'''
"""
This method is intended to be overwritten.
Called by get_active_jobs()
Returns a list of active jobs (i.e. running) associated with the calling
resource (self). Expected to return a QuerySet
'''
"""
def _get_related_jobs(self):
return self.objects.none()
@@ -519,6 +520,7 @@ class RelatedJobsMixin(object):
'''
Returns [{'id': 1, 'type': 'job'}, {'id': 2, 'type': 'project_update'}, ...]
'''
def get_active_jobs(self):
UnifiedJob = apps.get_model('main', 'UnifiedJob')
mapping = build_polymorphic_ctypes_map(UnifiedJob)
@@ -538,24 +540,15 @@ class WebhookTemplateMixin(models.Model):
('gitlab', "GitLab"),
]
webhook_service = models.CharField(
max_length=16,
choices=SERVICES,
blank=True,
help_text=_('Service that webhook requests will be accepted from')
)
webhook_key = prevent_search(models.CharField(
max_length=64,
blank=True,
help_text=_('Shared secret that the webhook service will use to sign requests')
))
webhook_service = models.CharField(max_length=16, choices=SERVICES, blank=True, help_text=_('Service that webhook requests will be accepted from'))
webhook_key = prevent_search(models.CharField(max_length=64, blank=True, help_text=_('Shared secret that the webhook service will use to sign requests')))
webhook_credential = models.ForeignKey(
'Credential',
blank=True,
null=True,
on_delete=models.SET_NULL,
related_name='%(class)ss',
help_text=_('Personal Access Token for posting back the status to the service API')
help_text=_('Personal Access Token for posting back the status to the service API'),
)
def rotate_webhook_key(self):
@@ -582,25 +575,16 @@ class WebhookMixin(models.Model):
SERVICES = WebhookTemplateMixin.SERVICES
webhook_service = models.CharField(
max_length=16,
choices=SERVICES,
blank=True,
help_text=_('Service that webhook requests will be accepted from')
)
webhook_service = models.CharField(max_length=16, choices=SERVICES, blank=True, help_text=_('Service that webhook requests will be accepted from'))
webhook_credential = models.ForeignKey(
'Credential',
blank=True,
null=True,
on_delete=models.SET_NULL,
related_name='%(class)ss',
help_text=_('Personal Access Token for posting back the status to the service API')
)
webhook_guid = models.CharField(
blank=True,
max_length=128,
help_text=_('Unique identifier of the event that triggered this webhook')
help_text=_('Personal Access Token for posting back the status to the service API'),
)
webhook_guid = models.CharField(blank=True, max_length=128, help_text=_('Unique identifier of the event that triggered this webhook'))
def update_webhook_status(self, status):
if not self.webhook_credential:
@@ -645,10 +629,7 @@ class WebhookMixin(models.Model):
'target_url': self.get_ui_url(),
}
k, v = service_header[self.webhook_service]
headers = {
k: v.format(self.webhook_credential.get_input('token')),
'Content-Type': 'application/json'
}
headers = {k: v.format(self.webhook_credential.get_input('token')), 'Content-Type': 'application/json'}
response = requests.post(status_api, data=json.dumps(data), headers=headers, timeout=30)
except Exception:
logger.exception("Posting webhook status caused an error.")
@@ -657,8 +638,4 @@ class WebhookMixin(models.Model):
if response.status_code < 400:
logger.debug("Webhook status update sent.")
else:
logger.error(
"Posting webhook status failed, code: {}\n"
"{}\n"
"Payload sent: {}".format(response.status_code, response.text, json.dumps(data))
)
logger.error("Posting webhook status failed, code: {}\n" "{}\n" "Payload sent: {}".format(response.status_code, response.text, json.dumps(data)))

327
awx/main/models/notifications.py
View File

@@ -38,15 +38,17 @@ __all__ = ['NotificationTemplate', 'Notification']
class NotificationTemplate(CommonModelNameNotUnique):
NOTIFICATION_TYPES = [('email', _('Email'), CustomEmailBackend),
('slack', _('Slack'), SlackBackend),
('twilio', _('Twilio'), TwilioBackend),
('pagerduty', _('Pagerduty'), PagerDutyBackend),
('grafana', _('Grafana'), GrafanaBackend),
('webhook', _('Webhook'), WebhookBackend),
('mattermost', _('Mattermost'), MattermostBackend),
('rocketchat', _('Rocket.Chat'), RocketChatBackend),
('irc', _('IRC'), IrcBackend)]
NOTIFICATION_TYPES = [
('email', _('Email'), CustomEmailBackend),
('slack', _('Slack'), SlackBackend),
('twilio', _('Twilio'), TwilioBackend),
('pagerduty', _('Pagerduty'), PagerDutyBackend),
('grafana', _('Grafana'), GrafanaBackend),
('webhook', _('Webhook'), WebhookBackend),
('mattermost', _('Mattermost'), MattermostBackend),
('rocketchat', _('Rocket.Chat'), RocketChatBackend),
('irc', _('IRC'), IrcBackend),
]
NOTIFICATION_TYPE_CHOICES = sorted([(x[0], x[1]) for x in NOTIFICATION_TYPES])
CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NOTIFICATION_TYPES])
@@ -64,7 +66,7 @@ class NotificationTemplate(CommonModelNameNotUnique):
)
notification_type = models.CharField(
max_length = 32,
max_length=32,
choices=NOTIFICATION_TYPE_CHOICES,
)
@@ -73,11 +75,7 @@ class NotificationTemplate(CommonModelNameNotUnique):
def default_messages():
return {'started': None, 'success': None, 'error': None, 'workflow_approval': None}
messages = JSONField(
null=True,
blank=True,
default=default_messages,
help_text=_('Optional custom messages for notification template.'))
messages = JSONField(null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.'))
def has_message(self, condition):
potential_template = self.messages.get(condition, {})
@@ -114,6 +112,7 @@ class NotificationTemplate(CommonModelNameNotUnique):
for msg_type in ['message', 'body']:
if msg_type not in local_new_event_msgs and local_old_event_msgs.get(msg_type, None):
local_new_event_msgs[msg_type] = local_old_event_msgs[msg_type]
if old_messages is not None and new_messages is not None:
for event in ('started', 'success', 'error', 'workflow_approval'):
if not new_messages.get(event, {}) and old_messages.get(event, {}):
@@ -134,9 +133,7 @@ class NotificationTemplate(CommonModelNameNotUnique):
merge_messages(old_messages, new_messages, event)
new_messages.setdefault(event, None)
for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password",
self.notification_class.init_parameters):
for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password", self.notification_class.init_parameters):
if self.notification_configuration[field].startswith("$encrypted$"):
continue
if new_instance:
@@ -151,8 +148,7 @@ class NotificationTemplate(CommonModelNameNotUnique):
super(NotificationTemplate, self).save(*args, **kwargs)
if new_instance:
update_fields = []
for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password",
self.notification_class.init_parameters):
for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password", self.notification_class.init_parameters):
saved_value = getattr(self, '_saved_{}_{}'.format("config", field), '')
self.notification_configuration[field] = saved_value
if 'notification_configuration' not in update_fields:
@@ -164,21 +160,16 @@ class NotificationTemplate(CommonModelNameNotUnique):
return self.notification_configuration[self.notification_class.recipient_parameter]
def generate_notification(self, msg, body):
notification = Notification(notification_template=self,
notification_type=self.notification_type,
recipients=smart_str(self.recipients),
subject=msg,
body=body)
notification = Notification(
notification_template=self, notification_type=self.notification_type, recipients=smart_str(self.recipients), subject=msg, body=body
)
notification.save()
return notification
def send(self, subject, body):
for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password",
self.notification_class.init_parameters):
for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password", self.notification_class.init_parameters):
if field in self.notification_configuration:
self.notification_configuration[field] = decrypt_field(self,
'notification_configuration',
subfield=field)
self.notification_configuration[field] = decrypt_field(self, 'notification_configuration', subfield=field)
recipients = self.notification_configuration.pop(self.notification_class.recipient_parameter)
if not isinstance(recipients, list):
recipients = [recipients]
@@ -202,9 +193,9 @@ class NotificationTemplate(CommonModelNameNotUnique):
class Notification(CreatedModifiedModel):
'''
"""
A notification event emitted when a NotificationTemplate is run
'''
"""
NOTIFICATION_STATE_CHOICES = [
('pending', _('Pending')),
@@ -216,12 +207,7 @@ class Notification(CreatedModifiedModel):
app_label = 'main'
ordering = ('pk',)
notification_template = models.ForeignKey(
'NotificationTemplate',
related_name='notifications',
on_delete=models.CASCADE,
editable=False
)
notification_template = models.ForeignKey('NotificationTemplate', related_name='notifications', on_delete=models.CASCADE, editable=False)
status = models.CharField(
max_length=20,
choices=NOTIFICATION_STATE_CHOICES,
@@ -238,7 +224,7 @@ class Notification(CreatedModifiedModel):
editable=False,
)
notification_type = models.CharField(
max_length = 32,
max_length=32,
choices=NotificationTemplate.NOTIFICATION_TYPE_CHOICES,
)
recipients = models.TextField(
@@ -258,112 +244,160 @@ class Notification(CreatedModifiedModel):
class JobNotificationMixin(object):
STATUS_TO_TEMPLATE_TYPE = {'succeeded': 'success',
'running': 'started',
'failed': 'error'}
STATUS_TO_TEMPLATE_TYPE = {'succeeded': 'success', 'running': 'started', 'failed': 'error'}
# Tree of fields that can be safely referenced in a notification message
JOB_FIELDS_ALLOWED_LIST = ['id', 'type', 'url', 'created', 'modified', 'name', 'description', 'job_type', 'playbook',
'forks', 'limit', 'verbosity', 'job_tags', 'force_handlers', 'skip_tags', 'start_at_task',
'timeout', 'use_fact_cache', 'launch_type', 'status', 'failed', 'started', 'finished',
'elapsed', 'job_explanation', 'execution_node', 'controller_node', 'allow_simultaneous',
'scm_revision', 'diff_mode', 'job_slice_number', 'job_slice_count', 'custom_virtualenv',
'approval_status', 'approval_node_name', 'workflow_url', 'scm_branch', 'artifacts',
{'host_status_counts': ['skipped', 'ok', 'changed', 'failed', 'failures', 'dark'
'processed', 'rescued', 'ignored']},
{'summary_fields': [{'inventory': ['id', 'name', 'description', 'has_active_failures',
'total_hosts', 'hosts_with_active_failures', 'total_groups',
'has_inventory_sources',
'total_inventory_sources', 'inventory_sources_with_failures',
'organization_id', 'kind']},
{'project': ['id', 'name', 'description', 'status', 'scm_type']},
{'job_template': ['id', 'name', 'description']},
{'unified_job_template': ['id', 'name', 'description', 'unified_job_type']},
{'instance_group': ['name', 'id']},
{'created_by': ['id', 'username', 'first_name', 'last_name']},
{'schedule': ['id', 'name', 'description', 'next_run']},
{'labels': ['count', 'results']}]}]
JOB_FIELDS_ALLOWED_LIST = [
'id',
'type',
'url',
'created',
'modified',
'name',
'description',
'job_type',
'playbook',
'forks',
'limit',
'verbosity',
'job_tags',
'force_handlers',
'skip_tags',
'start_at_task',
'timeout',
'use_fact_cache',
'launch_type',
'status',
'failed',
'started',
'finished',
'elapsed',
'job_explanation',
'execution_node',
'controller_node',
'allow_simultaneous',
'scm_revision',
'diff_mode',
'job_slice_number',
'job_slice_count',
'custom_virtualenv',
'approval_status',
'approval_node_name',
'workflow_url',
'scm_branch',
'artifacts',
{'host_status_counts': ['skipped', 'ok', 'changed', 'failed', 'failures', 'dark' 'processed', 'rescued', 'ignored']},
{
'summary_fields': [
{
'inventory': [
'id',
'name',
'description',
'has_active_failures',
'total_hosts',
'hosts_with_active_failures',
'total_groups',
'has_inventory_sources',
'total_inventory_sources',
'inventory_sources_with_failures',
'organization_id',
'kind',
]
},
{'project': ['id', 'name', 'description', 'status', 'scm_type']},
{'job_template': ['id', 'name', 'description']},
{'unified_job_template': ['id', 'name', 'description', 'unified_job_type']},
{'instance_group': ['name', 'id']},
{'created_by': ['id', 'username', 'first_name', 'last_name']},
{'schedule': ['id', 'name', 'description', 'next_run']},
{'labels': ['count', 'results']},
]
},
]
@classmethod
def context_stub(cls):
"""Returns a stub context that can be used for validating notification messages.
Context has the same structure as the context that will actually be used to render
a notification message."""
context = {'job': {'allow_simultaneous': False,
'artifacts': {},
'controller_node': 'foo_controller',
'created': datetime.datetime(2018, 11, 13, 6, 4, 0, 0, tzinfo=datetime.timezone.utc),
'custom_virtualenv': 'my_venv',
'description': 'Sample job description',
'diff_mode': False,
'elapsed': 0.403018,
'execution_node': 'awx',
'failed': False,
'finished': False,
'force_handlers': False,
'forks': 0,
'host_status_counts': {'skipped': 1, 'ok': 5, 'changed': 3, 'failures': 0, 'dark': 0, 'failed': False, 'processed': 0, 'rescued': 0},
'id': 42,
'job_explanation': 'Sample job explanation',
'job_slice_count': 1,
'job_slice_number': 0,
'job_tags': '',
'job_type': 'run',
'launch_type': 'workflow',
'limit': 'bar_limit',
'modified': datetime.datetime(2018, 12, 13, 6, 4, 0, 0, tzinfo=datetime.timezone.utc),
'name': 'Stub JobTemplate',
'playbook': 'ping.yml',
'scm_branch': '',
'scm_revision': '',
'skip_tags': '',
'start_at_task': '',
'started': '2019-07-29T17:38:14.137461Z',
'status': 'running',
'summary_fields': {'created_by': {'first_name': '',
'id': 1,
'last_name': '',
'username': 'admin'},
'instance_group': {'id': 1, 'name': 'tower'},
'inventory': {'description': 'Sample inventory description',
'has_active_failures': False,
'has_inventory_sources': False,
'hosts_with_active_failures': 0,
'id': 17,
'inventory_sources_with_failures': 0,
'kind': '',
'name': 'Stub Inventory',
'organization_id': 121,
'total_groups': 0,
'total_hosts': 1,
'total_inventory_sources': 0},
'job_template': {'description': 'Sample job template description',
'id': 39,
'name': 'Stub JobTemplate'},
'labels': {'count': 0, 'results': []},
'project': {'description': 'Sample project description',
'id': 38,
'name': 'Stub project',
'scm_type': 'git',
'status': 'successful'},
'schedule': {'description': 'Sample schedule',
'id': 42,
'name': 'Stub schedule',
'next_run': datetime.datetime(2038, 1, 1, 0, 0, 0, 0, tzinfo=datetime.timezone.utc)},
'unified_job_template': {'description': 'Sample unified job template description',
'id': 39,
'name': 'Stub Job Template',
'unified_job_type': 'job'}},
'timeout': 0,
'type': 'job',
'url': '/api/v2/jobs/13/',
'use_fact_cache': False,
'verbosity': 0},
'job_friendly_name': 'Job',
'url': 'https://towerhost/#/jobs/playbook/1010',
'approval_status': 'approved',
'approval_node_name': 'Approve Me',
'workflow_url': 'https://towerhost/#/jobs/workflow/1010',
'job_metadata': """{'url': 'https://towerhost/$/jobs/playbook/13',
context = {
'job': {
'allow_simultaneous': False,
'artifacts': {},
'controller_node': 'foo_controller',
'created': datetime.datetime(2018, 11, 13, 6, 4, 0, 0, tzinfo=datetime.timezone.utc),
'custom_virtualenv': 'my_venv',
'description': 'Sample job description',
'diff_mode': False,
'elapsed': 0.403018,
'execution_node': 'awx',
'failed': False,
'finished': False,
'force_handlers': False,
'forks': 0,
'host_status_counts': {'skipped': 1, 'ok': 5, 'changed': 3, 'failures': 0, 'dark': 0, 'failed': False, 'processed': 0, 'rescued': 0},
'id': 42,
'job_explanation': 'Sample job explanation',
'job_slice_count': 1,
'job_slice_number': 0,
'job_tags': '',
'job_type': 'run',
'launch_type': 'workflow',
'limit': 'bar_limit',
'modified': datetime.datetime(2018, 12, 13, 6, 4, 0, 0, tzinfo=datetime.timezone.utc),
'name': 'Stub JobTemplate',
'playbook': 'ping.yml',
'scm_branch': '',
'scm_revision': '',
'skip_tags': '',
'start_at_task': '',
'started': '2019-07-29T17:38:14.137461Z',
'status': 'running',
'summary_fields': {
'created_by': {'first_name': '', 'id': 1, 'last_name': '', 'username': 'admin'},
'instance_group': {'id': 1, 'name': 'tower'},
'inventory': {
'description': 'Sample inventory description',
'has_active_failures': False,
'has_inventory_sources': False,
'hosts_with_active_failures': 0,
'id': 17,
'inventory_sources_with_failures': 0,
'kind': '',
'name': 'Stub Inventory',
'organization_id': 121,
'total_groups': 0,
'total_hosts': 1,
'total_inventory_sources': 0,
},
'job_template': {'description': 'Sample job template description', 'id': 39, 'name': 'Stub JobTemplate'},
'labels': {'count': 0, 'results': []},
'project': {'description': 'Sample project description', 'id': 38, 'name': 'Stub project', 'scm_type': 'git', 'status': 'successful'},
'schedule': {
'description': 'Sample schedule',
'id': 42,
'name': 'Stub schedule',
'next_run': datetime.datetime(2038, 1, 1, 0, 0, 0, 0, tzinfo=datetime.timezone.utc),
},
'unified_job_template': {
'description': 'Sample unified job template description',
'id': 39,
'name': 'Stub Job Template',
'unified_job_type': 'job',
},
},
'timeout': 0,
'type': 'job',
'url': '/api/v2/jobs/13/',
'use_fact_cache': False,
'verbosity': 0,
},
'job_friendly_name': 'Job',
'url': 'https://towerhost/#/jobs/playbook/1010',
'approval_status': 'approved',
'approval_node_name': 'Approve Me',
'workflow_url': 'https://towerhost/#/jobs/workflow/1010',
'job_metadata': """{'url': 'https://towerhost/$/jobs/playbook/13',
'traceback': '',
'status': 'running',
'started': '2019-08-07T21:46:38.362630+00:00',
@@ -377,7 +411,8 @@ class JobNotificationMixin(object):
'friendly_name': 'Job',
'finished': False,
'credential': 'Stub credential',
'created_by': 'admin'}"""}
'created_by': 'admin'}""",
}
return context
@@ -403,11 +438,7 @@ class JobNotificationMixin(object):
'job': job_context,
'job_friendly_name': self.get_notification_friendly_name(),
'url': self.get_ui_url(),
'job_metadata': json.dumps(
self.notification_data(),
ensure_ascii=False,
indent=4
)
'job_metadata': json.dumps(self.notification_data(), ensure_ascii=False, indent=4),
}
def build_context(node, fields, allowed_fields):
@@ -425,6 +456,7 @@ class JobNotificationMixin(object):
if safe_field not in fields:
continue
node[safe_field] = fields[safe_field]
build_context(context['job'], serialized_job, self.JOB_FIELDS_ALLOWED_LIST)
return context
@@ -442,6 +474,7 @@ class JobNotificationMixin(object):
env = sandbox.ImmutableSandboxedEnvironment(undefined=ChainableUndefined)
from awx.api.serializers import UnifiedJobSerializer
job_serialization = UnifiedJobSerializer(self).to_representation(self)
context = self.context(job_serialization)
@@ -476,6 +509,7 @@ class JobNotificationMixin(object):
def send_notification_templates(self, status):
from awx.main.tasks import send_notifications # avoid circular import
if status not in ['running', 'succeeded', 'failed']:
raise ValueError(_("status must be either running, succeeded or failed"))
try:
@@ -494,7 +528,8 @@ class JobNotificationMixin(object):
# https://stackoverflow.com/a/3431699/10669572
def send_it(local_nt=nt, local_msg=msg, local_body=body):
def _func():
send_notifications.delay([local_nt.generate_notification(local_msg, local_body).id],
job_id=self.id)
send_notifications.delay([local_nt.generate_notification(local_msg, local_body).id], job_id=self.id)
return _func
connection.on_commit(send_it())

49
awx/main/models/oauth.py
View File

@@ -27,13 +27,12 @@ logger = logging.getLogger('awx.main.models.oauth')
class OAuth2Application(AbstractApplication):
class Meta:
app_label = 'main'
verbose_name = _('application')
unique_together = (("name", "organization"),)
ordering = ('organization', 'name')
CLIENT_CONFIDENTIAL = "confidential"
CLIENT_PUBLIC = "public"
CLIENT_TYPES = (
@@ -65,42 +64,34 @@ class OAuth2Application(AbstractApplication):
null=True,
)
client_secret = OAuth2ClientSecretField(
max_length=1024,
blank=True,
default=generate_client_secret,
max_length=1024,
blank=True,
default=generate_client_secret,
db_index=True,
help_text=_('Used for more stringent verification of access to an application when creating a token.')
help_text=_('Used for more stringent verification of access to an application when creating a token.'),
)
client_type = models.CharField(
max_length=32,
choices=CLIENT_TYPES,
help_text=_('Set to Public or Confidential depending on how secure the client device is.')
)
skip_authorization = models.BooleanField(
default=False,
help_text=_('Set True to skip authorization step for completely trusted applications.')
max_length=32, choices=CLIENT_TYPES, help_text=_('Set to Public or Confidential depending on how secure the client device is.')
)
skip_authorization = models.BooleanField(default=False, help_text=_('Set True to skip authorization step for completely trusted applications.'))
authorization_grant_type = models.CharField(
max_length=32,
choices=GRANT_TYPES,
help_text=_('The Grant type the user must use for acquire tokens for this application.')
max_length=32, choices=GRANT_TYPES, help_text=_('The Grant type the user must use for acquire tokens for this application.')
)
class OAuth2AccessToken(AbstractAccessToken):
class Meta:
app_label = 'main'
verbose_name = _('access token')
ordering = ('id',)
user = models.ForeignKey(
settings.AUTH_USER_MODEL,
on_delete=models.CASCADE,
blank=True,
settings.AUTH_USER_MODEL,
on_delete=models.CASCADE,
blank=True,
null=True,
related_name="%(app_label)s_%(class)s",
help_text=_('The user representing the token owner')
help_text=_('The user representing the token owner'),
)
description = models.TextField(
default='',
@@ -114,12 +105,11 @@ class OAuth2AccessToken(AbstractAccessToken):
scope = models.TextField(
blank=True,
default='write',
help_text=_('Allowed scopes, further restricts user\'s permissions. Must be a simple space-separated string with allowed scopes [\'read\', \'write\'].')
)
modified = models.DateTimeField(
editable=False,
auto_now=True
help_text=_(
'Allowed scopes, further restricts user\'s permissions. Must be a simple space-separated string with allowed scopes [\'read\', \'write\'].'
),
)
modified = models.DateTimeField(editable=False, auto_now=True)
def is_valid(self, scopes=None):
valid = super(OAuth2AccessToken, self).is_valid(scopes)
@@ -129,6 +119,7 @@ class OAuth2AccessToken(AbstractAccessToken):
def _update_last_used():
if OAuth2AccessToken.objects.filter(pk=self.pk).exists():
self.save(update_fields=['last_used'])
connection.on_commit(_update_last_used)
return valid
@@ -136,9 +127,9 @@ class OAuth2AccessToken(AbstractAccessToken):
if self.user and settings.ALLOW_OAUTH2_FOR_EXTERNAL_USERS is False:
external_account = get_external_account(self.user)
if external_account is not None:
raise oauth2.AccessDeniedError(_(
'OAuth2 Tokens cannot be created by users associated with an external authentication provider ({})'
).format(external_account))
raise oauth2.AccessDeniedError(
_('OAuth2 Tokens cannot be created by users associated with an external authentication provider ({})').format(external_account)
)
def save(self, *args, **kwargs):
if not self.pk:

102
awx/main/models/organization.py
View File

@@ -2,7 +2,6 @@
# All Rights Reserved.
# Django
from django.conf import settings
from django.db import models
@@ -14,13 +13,8 @@ from django.utils.translation import ugettext_lazy as _
# AWX
from awx.api.versioning import reverse
from awx.main.fields import (
AutoOneToOneField, ImplicitRoleField, OrderedManyToManyField
)
from awx.main.models.base import (
BaseModel, CommonModel, CommonModelNameNotUnique, CreatedModifiedModel,
NotificationFieldsModel
)
from awx.main.fields import AutoOneToOneField, ImplicitRoleField, OrderedManyToManyField
from awx.main.models.base import BaseModel, CommonModel, CommonModelNameNotUnique, CreatedModifiedModel, NotificationFieldsModel
from awx.main.models.rbac import (
ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
ROLE_SINGLETON_SYSTEM_AUDITOR,
@@ -32,35 +26,24 @@ __all__ = ['Organization', 'Team', 'Profile', 'UserSessionMembership']
class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin):
'''
"""
An organization is the basic unit of multi-tenancy divisions
'''
"""
class Meta:
app_label = 'main'
ordering = ('name',)
instance_groups = OrderedManyToManyField(
'InstanceGroup',
blank=True,
through='OrganizationInstanceGroupMembership'
)
instance_groups = OrderedManyToManyField('InstanceGroup', blank=True, through='OrganizationInstanceGroupMembership')
galaxy_credentials = OrderedManyToManyField(
'Credential',
blank=True,
through='OrganizationGalaxyCredentialMembership',
related_name='%(class)s_galaxy_credentials'
'Credential', blank=True, through='OrganizationGalaxyCredentialMembership', related_name='%(class)s_galaxy_credentials'
)
max_hosts = models.PositiveIntegerField(
blank=True,
default=0,
help_text=_('Maximum number of hosts allowed to be managed by this organization.'),
)
notification_templates_approvals = models.ManyToManyField(
"NotificationTemplate",
blank=True,
related_name='%(class)s_notification_templates_for_approvals'
)
notification_templates_approvals = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_approvals')
default_environment = models.ForeignKey(
'ExecutionEnvironment',
null=True,
@@ -101,42 +84,41 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi
auditor_role = ImplicitRoleField(
parent_role='singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR,
)
member_role = ImplicitRoleField(
parent_role=['admin_role']
)
member_role = ImplicitRoleField(parent_role=['admin_role'])
read_role = ImplicitRoleField(
parent_role=['member_role', 'auditor_role',
'execute_role', 'project_admin_role',
'inventory_admin_role', 'workflow_admin_role',
'notification_admin_role', 'credential_admin_role',
'job_template_admin_role', 'approval_role',
'execution_environment_admin_role',],
parent_role=[
'member_role',
'auditor_role',
'execute_role',
'project_admin_role',
'inventory_admin_role',
'workflow_admin_role',
'notification_admin_role',
'credential_admin_role',
'job_template_admin_role',
'approval_role',
'execution_environment_admin_role',
],
)
approval_role = ImplicitRoleField(
parent_role='admin_role',
)
def get_absolute_url(self, request=None):
return reverse('api:organization_detail', kwargs={'pk': self.pk}, request=request)
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return UnifiedJob.objects.non_polymorphic().filter(organization=self)
class OrganizationGalaxyCredentialMembership(models.Model):
organization = models.ForeignKey(
'Organization',
on_delete=models.CASCADE
)
credential = models.ForeignKey(
'Credential',
on_delete=models.CASCADE
)
organization = models.ForeignKey('Organization', on_delete=models.CASCADE)
credential = models.ForeignKey('Credential', on_delete=models.CASCADE)
position = models.PositiveIntegerField(
null=True,
default=None,
@@ -145,9 +127,9 @@ class OrganizationGalaxyCredentialMembership(models.Model):
class Team(CommonModelNameNotUnique, ResourceMixin):
'''
"""
A team is a group of users that work on common projects.
'''
"""
class Meta:
app_label = 'main'
@@ -176,20 +158,15 @@ class Team(CommonModelNameNotUnique, ResourceMixin):
class Profile(CreatedModifiedModel):
'''
"""
Profile model related to User object. Currently stores LDAP DN for users
loaded from LDAP.
'''
"""
class Meta:
app_label = 'main'
user = AutoOneToOneField(
'auth.User',
related_name='profile',
editable=False,
on_delete=models.CASCADE
)
user = AutoOneToOneField('auth.User', related_name='profile', editable=False, on_delete=models.CASCADE)
ldap_dn = models.CharField(
max_length=1024,
default='',
@@ -197,21 +174,17 @@ class Profile(CreatedModifiedModel):
class UserSessionMembership(BaseModel):
'''
"""
A lookup table for API session membership given user. Note, there is a
different session created by channels for websockets using the same
underlying model.
'''
"""
class Meta:
app_label = 'main'
user = models.ForeignKey(
'auth.User', related_name='+', blank=False, null=False, on_delete=models.CASCADE
)
session = models.OneToOneField(
Session, related_name='+', blank=False, null=False, on_delete=models.CASCADE
)
user = models.ForeignKey('auth.User', related_name='+', blank=False, null=False, on_delete=models.CASCADE)
session = models.OneToOneField(Session, related_name='+', blank=False, null=False, on_delete=models.CASCADE)
created = models.DateTimeField(default=None, editable=False)
@staticmethod
@@ -220,16 +193,15 @@ class UserSessionMembership(BaseModel):
return []
if now is None:
now = tz_now()
query_set = UserSessionMembership.objects\
.select_related('session')\
.filter(user_id=user_id)\
.order_by('-created')
query_set = UserSessionMembership.objects.select_related('session').filter(user_id=user_id).order_by('-created')
non_expire_memberships = [x for x in query_set if x.session.expire_date > now]
return non_expire_memberships[settings.SESSIONS_PER_USER:]
return non_expire_memberships[settings.SESSIONS_PER_USER :]
# Add get_absolute_url method to User model if not present.
if not hasattr(User, 'get_absolute_url'):
def user_get_absolute_url(user, request=None):
return reverse('api:user_detail', kwargs={'pk': user.pk}, request=request)
User.add_to_class('get_absolute_url', user_get_absolute_url)

124
awx/main/models/projects.py
View File

@@ -29,12 +29,7 @@ from awx.main.models.unified_jobs import (
UnifiedJobTemplate,
)
from awx.main.models.jobs import Job
from awx.main.models.mixins import (
ResourceMixin,
TaskManagerProjectUpdateMixin,
CustomVirtualEnvMixin,
RelatedJobsMixin
)
from awx.main.models.mixins import ResourceMixin, TaskManagerProjectUpdateMixin, CustomVirtualEnvMixin, RelatedJobsMixin
from awx.main.utils import update_scm_url, polymorphic
from awx.main.utils.ansible import skip_directory, could_be_inventory, could_be_playbook
from awx.main.fields import ImplicitRoleField
@@ -68,9 +63,11 @@ class ProjectOptions(models.Model):
@classmethod
def get_local_path_choices(cls):
if os.path.exists(settings.PROJECTS_ROOT):
paths = [x for x in os.listdir(settings.PROJECTS_ROOT)
if (os.path.isdir(os.path.join(settings.PROJECTS_ROOT, x)) and
not x.startswith('.') and not x.startswith('_'))]
paths = [
x
for x in os.listdir(settings.PROJECTS_ROOT)
if (os.path.isdir(os.path.join(settings.PROJECTS_ROOT, x)) and not x.startswith('.') and not x.startswith('_'))
]
qs = Project.objects
used_paths = qs.values_list('local_path', flat=True)
return [x for x in paths if x not in used_paths]
@@ -78,10 +75,7 @@ class ProjectOptions(models.Model):
return []
local_path = models.CharField(
max_length=1024,
blank=True,
help_text=_('Local path (relative to PROJECTS_ROOT) containing '
'playbooks and related files for this project.')
max_length=1024, blank=True, help_text=_('Local path (relative to PROJECTS_ROOT) containing ' 'playbooks and related files for this project.')
)
scm_type = models.CharField(
@@ -145,8 +139,7 @@ class ProjectOptions(models.Model):
if not self.scm_type:
return ''
try:
scm_url = update_scm_url(self.scm_type, scm_url,
check_special_cases=False)
scm_url = update_scm_url(self.scm_type, scm_url, check_special_cases=False)
except ValueError as e:
raise ValidationError((e.args or (_('Invalid SCM URL.'),))[0])
scm_url_parts = urlparse.urlsplit(scm_url)
@@ -169,8 +162,7 @@ class ProjectOptions(models.Model):
try:
if self.scm_type == 'insights':
self.scm_url = settings.INSIGHTS_URL_BASE
scm_url = update_scm_url(self.scm_type, self.scm_url,
check_special_cases=False)
scm_url = update_scm_url(self.scm_type, self.scm_url, check_special_cases=False)
scm_url_parts = urlparse.urlsplit(scm_url)
# Prefer the username/password in the URL, if provided.
scm_username = scm_url_parts.username or cred.get_input('username', default='')
@@ -179,8 +171,7 @@ class ProjectOptions(models.Model):
else:
scm_password = ''
try:
update_scm_url(self.scm_type, self.scm_url, scm_username,
scm_password)
update_scm_url(self.scm_type, self.scm_url, scm_username, scm_password)
except ValueError as e:
raise ValidationError((e.args or (_('Invalid credential.'),))[0])
except ValueError:
@@ -221,7 +212,6 @@ class ProjectOptions(models.Model):
results.append(smart_text(playbook))
return sorted(results, key=lambda x: smart_str(x).lower())
@property
def inventories(self):
results = []
@@ -243,10 +233,10 @@ class ProjectOptions(models.Model):
return sorted(results, key=lambda x: smart_str(x).lower())
def get_lock_file(self):
'''
"""
We want the project path in name only, we don't care if it exists or
not. This method will just append .lock onto the full directory path.
'''
"""
proj_path = self.get_project_path(check_if_exists=False)
if not proj_path:
return None
@@ -254,9 +244,9 @@ class ProjectOptions(models.Model):
class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin):
'''
"""
A project represents a playbook git repo that can access a set of inventories
'''
"""
SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')]
FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'instance_groups', 'credentials']
@@ -283,13 +273,11 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
scm_update_cache_timeout = models.PositiveIntegerField(
default=0,
blank=True,
help_text=_('The number of seconds after the last project update ran that a new '
'project update will be launched as a job dependency.'),
help_text=_('The number of seconds after the last project update ran that a new ' 'project update will be launched as a job dependency.'),
)
allow_override = models.BooleanField(
default=False,
help_text=_('Allow changing the SCM branch or revision in a job template '
'that uses this project.'),
help_text=_('Allow changing the SCM branch or revision in a job template ' 'that uses this project.'),
)
scm_revision = models.CharField(
@@ -317,10 +305,12 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
help_text=_('Suggested list of content that could be Ansible inventory in the project'),
)
admin_role = ImplicitRoleField(parent_role=[
'organization.project_admin_role',
'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
])
admin_role = ImplicitRoleField(
parent_role=[
'organization.project_admin_role',
'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
]
)
use_role = ImplicitRoleField(
parent_role='admin_role',
@@ -330,12 +320,14 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
parent_role='admin_role',
)
read_role = ImplicitRoleField(parent_role=[
'organization.auditor_role',
'singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR,
'use_role',
'update_role',
])
read_role = ImplicitRoleField(
parent_role=[
'organization.auditor_role',
'singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR,
'use_role',
'update_role',
]
)
@classmethod
def _get_unified_job_class(cls):
@@ -343,9 +335,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
@classmethod
def _get_unified_job_field_names(cls):
return set(f.name for f in ProjectOptions._meta.fields) | set(
['name', 'description', 'organization']
)
return set(f.name for f in ProjectOptions._meta.fields) | set(['name', 'description', 'organization'])
def clean_organization(self):
if self.pk:
@@ -370,20 +360,18 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
# Do the actual save.
super(Project, self).save(*args, **kwargs)
if new_instance:
update_fields=[]
update_fields = []
# Generate local_path for SCM after initial save (so we have a PK).
if self.scm_type and not self.local_path.startswith('_'):
update_fields.append('local_path')
if update_fields:
from awx.main.signals import disable_activity_stream
with disable_activity_stream():
self.save(update_fields=update_fields)
# If we just created a new project with SCM, start the initial update.
# also update if certain fields have changed
relevant_change = any(
pre_save_vals.get(fd_name, None) != self._prior_values_store.get(fd_name, None)
for fd_name in self.FIELDS_TRIGGER_UPDATE
)
relevant_change = any(pre_save_vals.get(fd_name, None) != self._prior_values_store.get(fd_name, None) for fd_name in self.FIELDS_TRIGGER_UPDATE)
if (relevant_change or new_instance) and (not skip_update) and self.scm_type:
self.update()
@@ -447,26 +435,21 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
@property
def notification_templates(self):
base_notification_templates = NotificationTemplate.objects
error_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_errors=self))
started_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_started=self))
success_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_success=self))
error_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_errors=self))
started_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_started=self))
success_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_success=self))
# Get Organization NotificationTemplates
if self.organization is not None:
error_notification_templates = set(error_notification_templates +
list(base_notification_templates
.filter(organization_notification_templates_for_errors=self.organization)))
started_notification_templates = set(started_notification_templates +
list(base_notification_templates
.filter(organization_notification_templates_for_started=self.organization)))
success_notification_templates = set(success_notification_templates +
list(base_notification_templates
.filter(organization_notification_templates_for_success=self.organization)))
return dict(error=list(error_notification_templates),
started=list(started_notification_templates),
success=list(success_notification_templates))
error_notification_templates = set(
error_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_errors=self.organization))
)
started_notification_templates = set(
started_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_started=self.organization))
)
success_notification_templates = set(
success_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_success=self.organization))
)
return dict(error=list(error_notification_templates), started=list(started_notification_templates), success=list(success_notification_templates))
def get_absolute_url(self, request=None):
return reverse('api:project_detail', kwargs={'pk': self.pk}, request=request)
@@ -474,11 +457,9 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return UnifiedJob.objects.non_polymorphic().filter(
models.Q(job__project=self) |
models.Q(projectupdate__project=self)
)
return UnifiedJob.objects.non_polymorphic().filter(models.Q(job__project=self) | models.Q(projectupdate__project=self))
def delete(self, *args, **kwargs):
paths_to_delete = (self.get_project_path(check_if_exists=False), self.get_cache_path())
@@ -486,14 +467,15 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
for path_to_delete in paths_to_delete:
if self.scm_type and path_to_delete: # non-manual, concrete path
from awx.main.tasks import delete_project_files
delete_project_files.delay(path_to_delete)
return r
class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManagerProjectUpdateMixin):
'''
"""
Internal job for tracking project updates from SCM.
'''
"""
class Meta:
app_label = 'main'
@@ -546,6 +528,7 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
@classmethod
def _get_task_class(cls):
from awx.main.tasks import RunProjectUpdate
return RunProjectUpdate
def _global_timeout_setting(self):
@@ -618,6 +601,7 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
'''
JobNotificationMixin
'''
def get_notification_templates(self):
return self.project.notification_templates

108
awx/main/models/rbac.py
View File

@@ -15,7 +15,7 @@ from django.utils.translation import ugettext_lazy as _
# AWX
from awx.api.versioning import reverse
from django.contrib.auth.models import User # noqa
from django.contrib.auth.models import User # noqa
__all__ = [
'Role',
@@ -23,13 +23,13 @@ __all__ = [
'get_roles_on_resource',
'ROLE_SINGLETON_SYSTEM_ADMINISTRATOR',
'ROLE_SINGLETON_SYSTEM_AUDITOR',
'role_summary_fields_generator'
'role_summary_fields_generator',
]
logger = logging.getLogger('awx.main.models.rbac')
ROLE_SINGLETON_SYSTEM_ADMINISTRATOR='system_administrator'
ROLE_SINGLETON_SYSTEM_AUDITOR='system_auditor'
ROLE_SINGLETON_SYSTEM_ADMINISTRATOR = 'system_administrator'
ROLE_SINGLETON_SYSTEM_AUDITOR = 'system_auditor'
role_names = {
'system_administrator': _('System Administrator'),
@@ -77,15 +77,16 @@ role_descriptions = {
}
tls = threading.local() # thread local storage
tls = threading.local() # thread local storage
def check_singleton(func):
'''
"""
check_singleton is a decorator that checks if a user given
to a `visible_roles` method is in either of our singleton roles (Admin, Auditor)
and if so, returns their full list of roles without filtering.
'''
"""
def wrapper(*args, **kwargs):
sys_admin = Role.singleton(ROLE_SINGLETON_SYSTEM_ADMINISTRATOR)
sys_audit = Role.singleton(ROLE_SINGLETON_SYSTEM_AUDITOR)
@@ -95,12 +96,13 @@ def check_singleton(func):
return args[1]
return Role.objects.all()
return func(*args, **kwargs)
return wrapper
@contextlib.contextmanager
def batch_role_ancestor_rebuilding(allow_nesting=False):
'''
"""
Batches the role ancestor rebuild work necessary whenever role-role
relations change. This can result in a big speedup when performing
any bulk manipulation.
@@ -108,7 +110,7 @@ def batch_role_ancestor_rebuilding(allow_nesting=False):
WARNING: Calls to anything related to checking access/permissions
while within the context of the batch_role_ancestor_rebuilding will
likely not work.
'''
"""
batch_role_rebuilding = getattr(tls, 'batch_role_rebuilding', False)
@@ -131,17 +133,15 @@ def batch_role_ancestor_rebuilding(allow_nesting=False):
class Role(models.Model):
'''
"""
Role model
'''
"""
class Meta:
app_label = 'main'
verbose_name_plural = _('roles')
db_table = 'main_rbac_roles'
index_together = [
("content_type", "object_id")
]
index_together = [("content_type", "object_id")]
ordering = ("content_type", "object_id")
role_field = models.TextField(null=False)
@@ -149,11 +149,8 @@ class Role(models.Model):
parents = models.ManyToManyField('Role', related_name='children')
implicit_parents = models.TextField(null=False, default='[]')
ancestors = models.ManyToManyField(
'Role',
through='RoleAncestorEntry',
through_fields=('descendent', 'ancestor'),
related_name='descendents'
) # auto-generated by `rebuild_role_ancestor_list`
'Role', through='RoleAncestorEntry', through_fields=('descendent', 'ancestor'), related_name='descendents'
) # auto-generated by `rebuild_role_ancestor_list`
members = models.ManyToManyField('auth.User', related_name='roles')
content_type = models.ForeignKey(ContentType, null=True, default=None, on_delete=models.CASCADE)
object_id = models.PositiveIntegerField(null=True, default=None)
@@ -181,8 +178,7 @@ class Role(models.Model):
return self.ancestors.filter(pk=accessor.pk).exists()
else:
accessor_type = ContentType.objects.get_for_model(accessor)
roles = Role.objects.filter(content_type__pk=accessor_type.id,
object_id=accessor.id)
roles = Role.objects.filter(content_type__pk=accessor_type.id, object_id=accessor.id)
return self.ancestors.filter(pk__in=roles).exists()
@property
@@ -214,12 +210,12 @@ class Role(models.Model):
@staticmethod
def rebuild_role_ancestor_list(additions, removals):
'''
"""
Updates our `ancestors` map to accurately reflect all of the ancestors for a role
You should never need to call this. Signal handlers should be calling
this method when the role hierachy changes automatically.
'''
"""
# The ancestry table
# =================================================
#
@@ -320,8 +316,7 @@ class Role(models.Model):
# to the magic number of 41496, or 40000 for a nice round number
def split_ids_for_sqlite(role_ids):
for i in range(0, len(role_ids), 40000):
yield role_ids[i:i + 40000]
yield role_ids[i : i + 40000]
with transaction.atomic():
while len(additions) > 0 or len(removals) > 0:
@@ -333,7 +328,8 @@ class Role(models.Model):
if len(removals) > 0:
for ids in split_ids_for_sqlite(removals):
sql_params['ids'] = ','.join(str(x) for x in ids)
cursor.execute('''
cursor.execute(
'''
DELETE FROM %(ancestors_table)s
WHERE descendent_id IN (%(ids)s)
AND descendent_id != ancestor_id
@@ -345,7 +341,9 @@ class Role(models.Model):
WHERE parents.from_role_id = %(ancestors_table)s.descendent_id
AND %(ancestors_table)s.ancestor_id = inner_ancestors.ancestor_id
)
''' % sql_params)
'''
% sql_params
)
delete_ct += cursor.rowcount
@@ -353,7 +351,8 @@ class Role(models.Model):
if len(additions) > 0:
for ids in split_ids_for_sqlite(additions):
sql_params['ids'] = ','.join(str(x) for x in ids)
cursor.execute('''
cursor.execute(
'''
INSERT INTO %(ancestors_table)s (descendent_id, ancestor_id, role_field, content_type_id, object_id)
SELECT from_id, to_id, new_ancestry_list.role_field, new_ancestry_list.content_type_id, new_ancestry_list.object_id FROM (
SELECT roles.id from_id,
@@ -383,7 +382,9 @@ class Role(models.Model):
AND %(ancestors_table)s.ancestor_id = new_ancestry_list.to_id
)
''' % sql_params)
'''
% sql_params
)
insert_ct += cursor.rowcount
if insert_ct == 0 and delete_ct == 0:
@@ -405,7 +406,6 @@ class Role(models.Model):
new_removals.update([row[0] for row in cursor.fetchall()])
removals = list(new_removals)
@staticmethod
def visible_roles(user):
return Role.filter_visible_roles(user, Role.objects.all())
@@ -413,19 +413,21 @@ class Role(models.Model):
@staticmethod
@check_singleton
def filter_visible_roles(user, roles_qs):
'''
"""
Visible roles include all roles that are ancestors of any
roles that the user has access to.
Case in point - organization auditor_role must see all roles
in their organization, but some of those roles descend from
organization admin_role, but not auditor_role.
'''
"""
return roles_qs.filter(
id__in=RoleAncestorEntry.objects.filter(
descendent__in=RoleAncestorEntry.objects.filter(
ancestor_id__in=list(user.roles.values_list('id', flat=True))
).values_list('descendent', flat=True)
).distinct().values_list('ancestor', flat=True)
descendent__in=RoleAncestorEntry.objects.filter(ancestor_id__in=list(user.roles.values_list('id', flat=True))).values_list(
'descendent', flat=True
)
)
.distinct()
.values_list('ancestor', flat=True)
)
@staticmethod
@@ -441,30 +443,29 @@ class Role(models.Model):
class RoleAncestorEntry(models.Model):
class Meta:
app_label = 'main'
verbose_name_plural = _('role_ancestors')
db_table = 'main_rbac_role_ancestors'
index_together = [
("ancestor", "content_type_id", "object_id"), # used by get_roles_on_resource
("ancestor", "content_type_id", "role_field"), # used by accessible_objects
("ancestor", "descendent"), # used by rebuild_role_ancestor_list in the NOT EXISTS clauses.
("ancestor", "content_type_id", "object_id"), # used by get_roles_on_resource
("ancestor", "content_type_id", "role_field"), # used by accessible_objects
("ancestor", "descendent"), # used by rebuild_role_ancestor_list in the NOT EXISTS clauses.
]
descendent = models.ForeignKey(Role, null=False, on_delete=models.CASCADE, related_name='+')
ancestor = models.ForeignKey(Role, null=False, on_delete=models.CASCADE, related_name='+')
role_field = models.TextField(null=False)
descendent = models.ForeignKey(Role, null=False, on_delete=models.CASCADE, related_name='+')
ancestor = models.ForeignKey(Role, null=False, on_delete=models.CASCADE, related_name='+')
role_field = models.TextField(null=False)
content_type_id = models.PositiveIntegerField(null=False)
object_id = models.PositiveIntegerField(null=False)
object_id = models.PositiveIntegerField(null=False)
def get_roles_on_resource(resource, accessor):
'''
"""
Returns a string list of the roles a accessor has for a given resource.
An accessor can be either a User, Role, or an arbitrary resource that
contains one or more Roles associated with it.
'''
"""
if type(accessor) == User:
roles = accessor.roles.all()
@@ -472,16 +473,15 @@ def get_roles_on_resource(resource, accessor):
roles = [accessor]
else:
accessor_type = ContentType.objects.get_for_model(accessor)
roles = Role.objects.filter(content_type__pk=accessor_type.id,
object_id=accessor.id)
roles = Role.objects.filter(content_type__pk=accessor_type.id, object_id=accessor.id)
return [
role_field for role_field in
RoleAncestorEntry.objects.filter(
ancestor__in=roles,
content_type_id=ContentType.objects.get_for_model(resource).id,
object_id=resource.id
).values_list('role_field', flat=True).distinct()
role_field
for role_field in RoleAncestorEntry.objects.filter(
ancestor__in=roles, content_type_id=ContentType.objects.get_for_model(resource).id, object_id=resource.id
)
.values_list('role_field', flat=True)
.distinct()
]

53
awx/main/models/schedules.py
View File

@@ -35,7 +35,6 @@ UTC_TIMEZONES = {x: tzutc() for x in dateutil.parser.parserinfo().UTCZONE}
class ScheduleFilterMethods(object):
def enabled(self, enabled=True):
return self.filter(enabled=enabled)
@@ -62,7 +61,6 @@ class ScheduleManager(ScheduleFilterMethods, models.Manager):
class Schedule(PrimordialModel, LaunchTimeConfig):
class Meta:
app_label = 'main'
ordering = ['-next_run']
@@ -78,32 +76,13 @@ class Schedule(PrimordialModel, LaunchTimeConfig):
name = models.CharField(
max_length=512,
)
enabled = models.BooleanField(
default=True,
help_text=_("Enables processing of this schedule.")
)
dtstart = models.DateTimeField(
null=True,
default=None,
editable=False,
help_text=_("The first occurrence of the schedule occurs on or after this time.")
)
enabled = models.BooleanField(default=True, help_text=_("Enables processing of this schedule."))
dtstart = models.DateTimeField(null=True, default=None, editable=False, help_text=_("The first occurrence of the schedule occurs on or after this time."))
dtend = models.DateTimeField(
null=True,
default=None,
editable=False,
help_text=_("The last occurrence of the schedule occurs before this time, aftewards the schedule expires.")
)
rrule = models.CharField(
max_length=255,
help_text=_("A value representing the schedules iCal recurrence rule.")
)
next_run = models.DateTimeField(
null=True,
default=None,
editable=False,
help_text=_("The next time that the scheduled action will run.")
null=True, default=None, editable=False, help_text=_("The last occurrence of the schedule occurs before this time, aftewards the schedule expires.")
)
rrule = models.CharField(max_length=255, help_text=_("A value representing the schedules iCal recurrence rule."))
next_run = models.DateTimeField(null=True, default=None, editable=False, help_text=_("The next time that the scheduled action will run."))
@classmethod
def get_zoneinfo(self):
@@ -113,7 +92,7 @@ class Schedule(PrimordialModel, LaunchTimeConfig):
def timezone(self):
utc = tzutc()
all_zones = Schedule.get_zoneinfo()
all_zones.sort(key = lambda x: -len(x))
all_zones.sort(key=lambda x: -len(x))
for r in Schedule.rrulestr(self.rrule)._rrule:
if r._dtstart:
tzinfo = r._dtstart.tzinfo
@@ -169,17 +148,11 @@ class Schedule(PrimordialModel, LaunchTimeConfig):
# What is the DTSTART timezone for:
# DTSTART;TZID=America/New_York:20200601T120000 RRULE:...;UNTIL=20200601T170000Z
# local_tz = tzfile('/usr/share/zoneinfo/America/New_York')
local_tz = dateutil.rrule.rrulestr(
rrule.replace(naive_until, naive_until + 'Z'),
tzinfos=UTC_TIMEZONES
)._dtstart.tzinfo
local_tz = dateutil.rrule.rrulestr(rrule.replace(naive_until, naive_until + 'Z'), tzinfos=UTC_TIMEZONES)._dtstart.tzinfo
# Make a datetime object with tzinfo=<the DTSTART timezone>
# localized_until = datetime.datetime(2020, 6, 1, 17, 0, tzinfo=tzfile('/usr/share/zoneinfo/America/New_York'))
localized_until = make_aware(
datetime.datetime.strptime(re.sub('^UNTIL=', '', naive_until), "%Y%m%dT%H%M%S"),
local_tz
)
localized_until = make_aware(datetime.datetime.strptime(re.sub('^UNTIL=', '', naive_until), "%Y%m%dT%H%M%S"), local_tz)
# Coerce the datetime to UTC and format it as a string w/ Zulu format
# utc_until = UNTIL=20200601T220000Z
@@ -201,15 +174,9 @@ class Schedule(PrimordialModel, LaunchTimeConfig):
for r in x._rrule:
if r._dtstart and r._dtstart.tzinfo is None:
raise ValueError(
'A valid TZID must be provided (e.g., America/New_York)'
)
raise ValueError('A valid TZID must be provided (e.g., America/New_York)')
if (
fast_forward and
('MINUTELY' in rrule or 'HOURLY' in rrule) and
'COUNT=' not in rrule
):
if fast_forward and ('MINUTELY' in rrule or 'HOURLY' in rrule) and 'COUNT=' not in rrule:
try:
first_event = x[0]
# If the first event was over a week ago...

336
awx/main/models/unified_jobs.py
View File

@@ -30,22 +30,24 @@ from rest_framework.exceptions import ParseError
from polymorphic.models import PolymorphicModel
# AWX
from awx.main.models.base import (
CommonModelNameNotUnique,
PasswordFieldsModel,
NotificationFieldsModel,
prevent_search
)
from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel, prevent_search
from awx.main.dispatch import get_local_queuename
from awx.main.dispatch.control import Control as ControlDispatcher
from awx.main.registrar import activity_stream_registrar
from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin, ExecutionEnvironmentMixin
from awx.main.utils import (
camelcase_to_underscore, get_model_for_type,
encrypt_dict, decrypt_field, _inventory_updates,
copy_model_by_class, copy_m2m_relationships,
get_type_for_model, parse_yaml_or_json, getattr_dne,
polymorphic, schedule_task_manager
camelcase_to_underscore,
get_model_for_type,
encrypt_dict,
decrypt_field,
_inventory_updates,
copy_model_by_class,
copy_m2m_relationships,
get_type_for_model,
parse_yaml_or_json,
getattr_dne,
polymorphic,
schedule_task_manager,
)
from awx.main.constants import ACTIVE_STATES, CAN_CANCEL
from awx.main.redact import UriCleaner, REPLACE_STR
@@ -60,40 +62,40 @@ logger_job_lifecycle = logging.getLogger('awx.analytics.job_lifecycle')
class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEnvironmentMixin, NotificationFieldsModel):
'''
"""
Concrete base class for unified job templates.
'''
"""
# status inherits from related jobs. Thus, status must be able to be set to any status that a job status is settable to.
JOB_STATUS_CHOICES = [
('new', _('New')), # Job has been created, but not started.
('pending', _('Pending')), # Job is pending Task Manager processing (blocked by dependency req, capacity or a concurrent job)
('waiting', _('Waiting')), # Job has been assigned to run on a specific node (and is about to run).
('running', _('Running')), # Job is currently running.
('successful', _('Successful')), # Job completed successfully.
('failed', _('Failed')), # Job completed, but with failures.
('error', _('Error')), # The job was unable to run.
('canceled', _('Canceled')), # The job was canceled before completion.
('new', _('New')), # Job has been created, but not started.
('pending', _('Pending')), # Job is pending Task Manager processing (blocked by dependency req, capacity or a concurrent job)
('waiting', _('Waiting')), # Job has been assigned to run on a specific node (and is about to run).
('running', _('Running')), # Job is currently running.
('successful', _('Successful')), # Job completed successfully.
('failed', _('Failed')), # Job completed, but with failures.
('error', _('Error')), # The job was unable to run.
('canceled', _('Canceled')), # The job was canceled before completion.
]
COMMON_STATUS_CHOICES = JOB_STATUS_CHOICES + [
('never updated', _('Never Updated')), # A job has never been run using this template.
('never updated', _('Never Updated')), # A job has never been run using this template.
]
PROJECT_STATUS_CHOICES = COMMON_STATUS_CHOICES + [
('ok', _('OK')), # Project is not configured for SCM and path exists.
('missing', _('Missing')), # Project path does not exist.
('ok', _('OK')), # Project is not configured for SCM and path exists.
('missing', _('Missing')), # Project path does not exist.
]
INVENTORY_SOURCE_STATUS_CHOICES = COMMON_STATUS_CHOICES + [
('none', _('No External Source')), # Inventory source is not configured to update from an external source.
('none', _('No External Source')), # Inventory source is not configured to update from an external source.
]
JOB_TEMPLATE_STATUS_CHOICES = COMMON_STATUS_CHOICES
DEPRECATED_STATUS_CHOICES = [
# No longer used for Project / Inventory Source:
('updating', _('Updating')), # Same as running.
('updating', _('Updating')), # Same as running.
]
ALL_STATUS_CHOICES = OrderedDict(PROJECT_STATUS_CHOICES + INVENTORY_SOURCE_STATUS_CHOICES + JOB_TEMPLATE_STATUS_CHOICES + DEPRECATED_STATUS_CHOICES).items()
@@ -103,7 +105,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
ordering = ('name',)
# unique_together here is intentionally commented out. Please make sure sub-classes of this model
# contain at least this uniqueness restriction: SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name')]
#unique_together = [('polymorphic_ctype', 'name', 'organization')]
# unique_together = [('polymorphic_ctype', 'name', 'organization')]
old_pk = models.PositiveIntegerField(
null=True,
@@ -135,16 +137,16 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
default=None,
editable=False,
)
#on_missed_schedule = models.CharField(
# on_missed_schedule = models.CharField(
# max_length=32,
# choices=[],
#)
# )
next_job_run = models.DateTimeField(
null=True,
default=None,
editable=False,
)
next_schedule = models.ForeignKey( # Schedule entry responsible for next_job_run.
next_schedule = models.ForeignKey( # Schedule entry responsible for next_job_run.
'Schedule',
null=True,
default=None,
@@ -170,16 +172,8 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
'Credential',
related_name='%(class)ss',
)
labels = models.ManyToManyField(
"Label",
blank=True,
related_name='%(class)s_labels'
)
instance_groups = OrderedManyToManyField(
'InstanceGroup',
blank=True,
through='UnifiedJobTemplateInstanceGroupMembership'
)
labels = models.ManyToManyField("Label", blank=True, related_name='%(class)s_labels')
instance_groups = OrderedManyToManyField('InstanceGroup', blank=True, through='UnifiedJobTemplateInstanceGroupMembership')
def get_absolute_url(self, request=None):
real_instance = self.get_real_instance()
@@ -198,23 +192,21 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
@classmethod
def _submodels_with_roles(cls):
ujt_classes = [c for c in cls.__subclasses__()
if c._meta.model_name not in ['inventorysource', 'systemjobtemplate']]
ujt_classes = [c for c in cls.__subclasses__() if c._meta.model_name not in ['inventorysource', 'systemjobtemplate']]
ct_dict = ContentType.objects.get_for_models(*ujt_classes)
return [ct.id for ct in ct_dict.values()]
@classmethod
def accessible_pk_qs(cls, accessor, role_field):
'''
"""
A re-implementation of accessible pk queryset for the "normal" unified JTs.
Does not return inventory sources or system JTs, these should
be handled inside of get_queryset where it is utilized.
'''
"""
# do not use this if in a subclass
if cls != UnifiedJobTemplate:
return super(UnifiedJobTemplate, cls).accessible_pk_qs(accessor, role_field)
return ResourceMixin._accessible_pk_qs(
cls, accessor, role_field, content_types=cls._submodels_with_roles())
return ResourceMixin._accessible_pk_qs(cls, accessor, role_field, content_types=cls._submodels_with_roles())
def _perform_unique_checks(self, unique_checks):
# Handle the list of unique fields returned above. Replace with an
@@ -246,19 +238,19 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
exclude = [x for x in exclude if x != 'polymorphic_ctype']
return super(UnifiedJobTemplate, self).validate_unique(exclude)
@property # Alias for backwards compatibility.
@property # Alias for backwards compatibility.
def current_update(self):
return self.current_job
@property # Alias for backwards compatibility.
@property # Alias for backwards compatibility.
def last_update(self):
return self.last_job
@property # Alias for backwards compatibility.
@property # Alias for backwards compatibility.
def last_update_failed(self):
return self.last_job_failed
@property # Alias for backwards compatibility.
@property # Alias for backwards compatibility.
def last_updated(self):
return self.last_job_run
@@ -285,7 +277,6 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
# Do the actual save.
super(UnifiedJobTemplate, self).save(*args, **kwargs)
def _get_current_status(self):
# Override in subclasses as needed.
if self.current_job and self.current_job.status:
@@ -305,8 +296,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
def _set_status_and_last_job_run(self, save=True):
status = self._get_current_status()
last_job_run = self._get_last_job_run()
return self.update_fields(status=status, last_job_run=last_job_run,
save=save)
return self.update_fields(status=status, last_job_run=last_job_run, save=save)
def _can_update(self):
# Override in subclasses as needed.
@@ -324,24 +314,25 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
@classmethod
def _get_unified_job_class(cls):
'''
"""
Return subclass of UnifiedJob that is created from this template.
'''
raise NotImplementedError # Implement in subclass.
"""
raise NotImplementedError # Implement in subclass.
@property
def notification_templates(self):
'''
"""
Return notification_templates relevant to this Unified Job Template
'''
"""
# NOTE: Derived classes should implement
from awx.main.models.notifications import NotificationTemplate
return NotificationTemplate.objects.none()
def create_unified_job(self, **kwargs):
'''
"""
Create a new unified job based on this unified job template.
'''
"""
new_job_passwords = kwargs.pop('survey_passwords', {})
eager_fields = kwargs.pop('_eager_fields', None)
@@ -364,9 +355,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
validated_kwargs = kwargs.copy()
if unallowed_fields:
if parent_field_name is None:
logger.warn('Fields {} are not allowed as overrides to spawn from {}.'.format(
', '.join(unallowed_fields), self
))
logger.warn('Fields {} are not allowed as overrides to spawn from {}.'.format(', '.join(unallowed_fields), self))
for f in unallowed_fields:
validated_kwargs.pop(f)
@@ -393,6 +382,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
kwargs['survey_passwords'] = new_job_passwords # saved in config object for relaunch
from awx.main.signals import disable_activity_stream, activity_stream_create
with disable_activity_stream():
# Don't emit the activity stream record here for creation,
# because we haven't attached important M2M relations yet, like
@@ -427,10 +417,10 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
@classmethod
def get_ask_mapping(cls):
'''
"""
Creates dictionary that maps the unified job field (keys)
to the field that enables prompting for the field (values)
'''
"""
mapping = {}
for field in cls._meta.fields:
if isinstance(field, AskForField):
@@ -442,10 +432,10 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
return cls._get_unified_job_field_names()
def copy_unified_jt(self):
'''
"""
Returns saved object, including related fields.
Create a copy of this unified job template.
'''
"""
unified_jt_class = self.__class__
fields = self._get_unified_jt_copy_names()
unified_jt = copy_model_by_class(self, unified_jt_class, fields, {})
@@ -458,9 +448,9 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
return unified_jt
def _accept_or_ignore_job_kwargs(self, _exclude_errors=(), **kwargs):
'''
"""
Override in subclass if template accepts _any_ prompted params
'''
"""
errors = {}
if kwargs:
for field_name in kwargs.keys():
@@ -468,11 +458,11 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
return ({}, kwargs, errors)
def accept_or_ignore_variables(self, data, errors=None, _exclude_errors=(), extra_passwords=None):
'''
"""
If subclasses accept any `variables` or `extra_vars`, they should
define _accept_or_ignore_variables to place those variables in the accepted dict,
according to the acceptance rules of the template.
'''
"""
if errors is None:
errors = {}
if not isinstance(data, dict):
@@ -486,14 +476,13 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
# resolution order, forced by how metaclass processes fields,
# thus the need for hasattr check
if extra_passwords:
return self._accept_or_ignore_variables(
data, errors, _exclude_errors=_exclude_errors, extra_passwords=extra_passwords)
return self._accept_or_ignore_variables(data, errors, _exclude_errors=_exclude_errors, extra_passwords=extra_passwords)
else:
return self._accept_or_ignore_variables(data, errors, _exclude_errors=_exclude_errors)
elif data:
errors['extra_vars'] = [
_('Variables {list_of_keys} provided, but this template cannot accept variables.'.format(
list_of_keys=', '.join(data.keys())))]
_('Variables {list_of_keys} provided, but this template cannot accept variables.'.format(list_of_keys=', '.join(data.keys())))
]
return ({}, data, errors)
@@ -510,7 +499,6 @@ class UnifiedJobTypeStringMixin(object):
class UnifiedJobDeprecatedStdout(models.Model):
class Meta:
managed = False
db_table = 'main_unifiedjob'
@@ -522,30 +510,30 @@ class UnifiedJobDeprecatedStdout(models.Model):
class StdoutMaxBytesExceeded(Exception):
def __init__(self, total, supported):
self.total = total
self.supported = supported
class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique,
UnifiedJobTypeStringMixin, TaskManagerUnifiedJobMixin, ExecutionEnvironmentMixin):
'''
class UnifiedJob(
PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique, UnifiedJobTypeStringMixin, TaskManagerUnifiedJobMixin, ExecutionEnvironmentMixin
):
"""
Concrete base class for unified job run by the task engine.
'''
"""
STATUS_CHOICES = UnifiedJobTemplate.JOB_STATUS_CHOICES
LAUNCH_TYPE_CHOICES = [
('manual', _('Manual')), # Job was started manually by a user.
('relaunch', _('Relaunch')), # Job was started via relaunch.
('callback', _('Callback')), # Job was started via host callback.
('scheduled', _('Scheduled')), # Job was started from a schedule.
('dependency', _('Dependency')), # Job was started as a dependency of another job.
('workflow', _('Workflow')), # Job was started from a workflow job.
('webhook', _('Webhook')), # Job was started from a webhook event.
('sync', _('Sync')), # Job was started from a project sync.
('scm', _('SCM Update')) # Job was created as an Inventory SCM sync.
('manual', _('Manual')), # Job was started manually by a user.
('relaunch', _('Relaunch')), # Job was started via relaunch.
('callback', _('Callback')), # Job was started via host callback.
('scheduled', _('Scheduled')), # Job was started from a schedule.
('dependency', _('Dependency')), # Job was started as a dependency of another job.
('workflow', _('Workflow')), # Job was started from a workflow job.
('webhook', _('Webhook')), # Job was started from a webhook event.
('sync', _('Sync')), # Job was started from a project sync.
('scm', _('SCM Update')), # Job was created as an Inventory SCM sync.
]
PASSWORD_FIELDS = ('start_args',)
@@ -565,7 +553,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
)
unified_job_template = models.ForeignKey(
'UnifiedJobTemplate',
null=True, # Some jobs can be run without a template.
null=True, # Some jobs can be run without a template.
default=None,
editable=False,
related_name='%(class)s_unified_jobs',
@@ -576,14 +564,8 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
editable=False,
db_index=True, # add an index, this is a commonly queried field
)
launch_type = models.CharField(
max_length=20,
choices=LAUNCH_TYPE_CHOICES,
default='manual',
editable=False,
db_index=True
)
schedule = models.ForeignKey( # Which schedule entry was responsible for starting this job.
launch_type = models.CharField(max_length=20, choices=LAUNCH_TYPE_CHOICES, default='manual', editable=False, db_index=True)
schedule = models.ForeignKey( # Which schedule entry was responsible for starting this job.
'Schedule',
null=True,
default=None,
@@ -635,9 +617,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
help_text=_("The date and time the job was queued for starting."),
)
dependencies_processed = models.BooleanField(
default=False,
editable=False,
help_text=_("If True, the task manager has already processed potential dependencies for this job.")
default=False, editable=False, help_text=_("If True, the task manager has already processed potential dependencies for this job.")
)
finished = models.DateTimeField(
null=True,
@@ -659,33 +639,39 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
editable=False,
help_text=_("Elapsed time in seconds that the job ran."),
)
job_args = prevent_search(models.TextField(
blank=True,
default='',
editable=False,
))
job_args = prevent_search(
models.TextField(
blank=True,
default='',
editable=False,
)
)
job_cwd = models.CharField(
max_length=1024,
blank=True,
default='',
editable=False,
)
job_env = prevent_search(JSONField(
blank=True,
default=dict,
editable=False,
))
job_env = prevent_search(
JSONField(
blank=True,
default=dict,
editable=False,
)
)
job_explanation = models.TextField(
blank=True,
default='',
editable=False,
help_text=_("A status field to indicate the state of the job if it wasn't able to run and capture stdout"),
)
start_args = prevent_search(models.TextField(
blank=True,
default='',
editable=False,
))
start_args = prevent_search(
models.TextField(
blank=True,
default='',
editable=False,
)
)
result_traceback = models.TextField(
blank=True,
default='',
@@ -697,11 +683,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
default='',
editable=False,
)
labels = models.ManyToManyField(
"Label",
blank=True,
related_name='%(class)s_labels'
)
labels = models.ManyToManyField("Label", blank=True, related_name='%(class)s_labels')
instance_group = models.ForeignKey(
'InstanceGroup',
blank=True,
@@ -752,7 +734,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
@classmethod
def _get_task_class(cls):
raise NotImplementedError # Implement in subclasses.
raise NotImplementedError # Implement in subclasses.
@classmethod
def supports_isolation(cls):
@@ -763,14 +745,14 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
return False
def _get_parent_field_name(self):
return 'unified_job_template' # Override in subclasses.
return 'unified_job_template' # Override in subclasses.
@classmethod
def _get_unified_job_template_class(cls):
'''
"""
Return subclass of UnifiedJobTemplate that applies to this unified job.
'''
raise NotImplementedError # Implement in subclass.
"""
raise NotImplementedError # Implement in subclass.
def _global_timeout_setting(self):
"Override in child classes, None value indicates this is not configurable"
@@ -900,10 +882,10 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
return result
def copy_unified_job(self, _eager_fields=None, **new_prompts):
'''
"""
Returns saved object, including related fields.
Create a copy of this unified job for the purpose of relaunch
'''
"""
unified_job_class = self.__class__
unified_jt_class = self._get_unified_job_template_class()
parent_field_name = self._get_parent_field_name()
@@ -927,16 +909,17 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
# Labels copied here
from awx.main.signals import disable_activity_stream
with disable_activity_stream():
copy_m2m_relationships(self, unified_job, fields)
return unified_job
def launch_prompts(self):
'''
"""
Return dictionary of prompts job was launched with
returns None if unknown
'''
"""
JobLaunchConfig = self._meta.get_field('launch_config').related_model
try:
config = self.launch_config
@@ -945,10 +928,10 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
return None
def create_config_from_prompts(self, kwargs, parent=None):
'''
"""
Create a launch configuration entry for this job, given prompts
returns None if it can not be created
'''
"""
JobLaunchConfig = self._meta.get_field('launch_config').related_model
config = JobLaunchConfig(job=self)
if parent is None:
@@ -1016,9 +999,9 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
@property
def event_processing_finished(self):
'''
"""
Returns True / False, whether all events from job have been saved
'''
"""
if self.status in ACTIVE_STATES:
return False # tally of events is only available at end of run
try:
@@ -1051,13 +1034,10 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
if not os.path.exists(settings.JOBOUTPUT_ROOT):
os.makedirs(settings.JOBOUTPUT_ROOT)
fd = tempfile.NamedTemporaryFile(
mode='w',
prefix='{}-{}-'.format(self.model_to_str(), self.pk),
suffix='.out',
dir=settings.JOBOUTPUT_ROOT,
encoding='utf-8'
mode='w', prefix='{}-{}-'.format(self.model_to_str(), self.pk), suffix='.out', dir=settings.JOBOUTPUT_ROOT, encoding='utf-8'
)
from awx.main.tasks import purge_old_stdout_files # circular import
purge_old_stdout_files.apply_async()
# Before the addition of event-based stdout, older versions of
@@ -1092,9 +1072,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
# detect the length of all stdout for this UnifiedJob, and
# if it exceeds settings.STDOUT_MAX_BYTES_DISPLAY bytes,
# don't bother actually fetching the data
total = self.get_event_queryset().aggregate(
total=models.Sum(models.Func(models.F('stdout'), function='LENGTH'))
)['total'] or 0
total = self.get_event_queryset().aggregate(total=models.Sum(models.Func(models.F('stdout'), function='LENGTH')))['total'] or 0
if total > max_supported:
raise StdoutMaxBytesExceeded(total, max_supported)
@@ -1106,11 +1084,9 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
cursor.copy_expert(
"copy (select stdout from {} where {}={} and stdout != '' order by start_line) to stdout".format(
self._meta.db_table + 'event',
self.event_parent_key,
self.id
self._meta.db_table + 'event', self.event_parent_key, self.id
),
fd
fd,
)
if hasattr(fd, 'name'):
@@ -1154,7 +1130,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
end_line = int(end_line)
stdout_lines = self.result_stdout_raw_handle().readlines()
absolute_end = len(stdout_lines)
for line in stdout_lines[int(start_line):end_line]:
for line in stdout_lines[int(start_line) : end_line]:
return_buffer.write(line)
if int(start_line) < 0:
start_actual = len(stdout_lines) + int(start_line)
@@ -1243,14 +1219,13 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
@property
def task_impact(self):
raise NotImplementedError # Implement in subclass.
raise NotImplementedError # Implement in subclass.
def websocket_emit_data(self):
''' Return extra data that should be included when submitting data to the browser over the websocket connection '''
websocket_data = dict(type=self.job_type_name)
if self.spawned_by_workflow:
websocket_data.update(dict(workflow_job_id=self.workflow_job_id,
workflow_node_id=self.workflow_node_id))
websocket_data.update(dict(workflow_job_id=self.workflow_job_id, workflow_node_id=self.workflow_node_id))
return websocket_data
def _websocket_emit_status(self, status):
@@ -1282,14 +1257,16 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
connection.on_commit(lambda: self.update_webhook_status(status))
def notification_data(self):
return dict(id=self.id,
name=self.name,
url=self.get_ui_url(),
created_by=smart_text(self.created_by),
started=self.started.isoformat() if self.started is not None else None,
finished=self.finished.isoformat() if self.finished is not None else None,
status=self.status,
traceback=self.result_traceback)
return dict(
id=self.id,
name=self.name,
url=self.get_ui_url(),
created_by=smart_text(self.created_by),
started=self.started.isoformat() if self.started is not None else None,
finished=self.finished.isoformat() if self.finished is not None else None,
status=self.status,
traceback=self.result_traceback,
)
def pre_start(self, **kwargs):
if not self.can_start:
@@ -1307,9 +1284,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
if missing_credential_inputs:
self.job_explanation = '{} cannot start because Credential {} does not provide one or more required fields ({}).'.format(
self._meta.verbose_name.title(),
credential.name,
', '.join(sorted(missing_credential_inputs))
self._meta.verbose_name.title(), credential.name, ', '.join(sorted(missing_credential_inputs))
)
self.save(update_fields=['job_explanation'])
return (False, None)
@@ -1326,7 +1301,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
opts = dict([(field, start_args.get(field, '')) for field in needed])
if not all(opts.values()):
missing_fields = ', '.join([k for k,v in opts.items() if not v])
missing_fields = ', '.join([k for k, v in opts.items() if not v])
self.job_explanation = u'Missing needed fields: %s.' % missing_fields
self.save(update_fields=['job_explanation'])
return (False, None)
@@ -1367,35 +1342,26 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
# Actually tell the task runner to run this task.
# FIXME: This will deadlock the task runner
#from awx.main.tasks import notify_task_runner
#notify_task_runner.delay({'id': self.id, 'metadata': kwargs,
# from awx.main.tasks import notify_task_runner
# notify_task_runner.delay({'id': self.id, 'metadata': kwargs,
# 'task_type': task_type})
# Done!
return True
@property
def actually_running(self):
# returns True if the job is running in the appropriate dispatcher process
running = False
if all([
self.status == 'running',
self.celery_task_id,
self.execution_node
]):
if all([self.status == 'running', self.celery_task_id, self.execution_node]):
# If the job is marked as running, but the dispatcher
# doesn't know about it (or the dispatcher doesn't reply),
# then cancel the job
timeout = 5
try:
running = self.celery_task_id in ControlDispatcher(
'dispatcher', self.controller_node or self.execution_node
).running(timeout=timeout)
running = self.celery_task_id in ControlDispatcher('dispatcher', self.controller_node or self.execution_node).running(timeout=timeout)
except (socket.timeout, RuntimeError):
logger.error('could not reach dispatcher on {} within {}s'.format(
self.execution_node, timeout
))
logger.error('could not reach dispatcher on {} within {}s'.format(self.execution_node, timeout))
running = False
return running
@@ -1405,8 +1371,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
def _build_job_explanation(self):
if not self.job_explanation:
return 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \
(self.model_to_str(), self.name, self.id)
return 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (self.model_to_str(), self.name, self.id)
return None
def cancel(self, job_explanation=None, is_chain=False):
@@ -1434,9 +1399,9 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
@property
def preferred_instance_groups(self):
'''
"""
Return Instance/Rampart Groups preferred by this unified job templates
'''
"""
if not self.unified_job_template:
return []
template_groups = [x for x in self.unified_job_template.instance_groups.all()]
@@ -1445,16 +1410,17 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
@property
def global_instance_groups(self):
from awx.main.models.ha import InstanceGroup
default_instance_group = InstanceGroup.objects.filter(name='tower')
if default_instance_group.exists():
return [default_instance_group.first()]
return []
def awx_meta_vars(self):
'''
"""
The result of this method is used as extra_vars of a job launched
by AWX, for purposes of client playbook hooks
'''
"""
r = {}
for name in ('awx', 'tower'):
r['{}_job_id'.format(name)] = self.pk
@@ -1507,9 +1473,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
return False
def log_lifecycle(self, state, blocked_by=None):
extra={'type': self._meta.model_name,
'task_id': self.id,
'state': state}
extra = {'type': self._meta.model_name, 'task_id': self.id, 'state': state}
if self.unified_job_template:
extra["template_name"] = self.unified_job_template.name
if state == "blocked" and blocked_by:

265
awx/main/models/workflow.py
View File

@@ -13,7 +13,8 @@ from django.db import connection, models
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import ObjectDoesNotExist
#from django import settings as tower_settings
# from django import settings as tower_settings
# Django-CRUM
from crum import get_current_user
@@ -23,17 +24,10 @@ from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
# AWX
from awx.api.versioning import reverse
from awx.main.models import (prevent_search, accepts_json, UnifiedJobTemplate,
UnifiedJob)
from awx.main.models.notifications import (
NotificationTemplate,
JobNotificationMixin
)
from awx.main.models import prevent_search, accepts_json, UnifiedJobTemplate, UnifiedJob
from awx.main.models.notifications import NotificationTemplate, JobNotificationMixin
from awx.main.models.base import CreatedModifiedModel, VarsDictProperty
from awx.main.models.rbac import (
ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
ROLE_SINGLETON_SYSTEM_AUDITOR
)
from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR
from awx.main.fields import ImplicitRoleField, AskForField
from awx.main.models.mixins import (
ResourceMixin,
@@ -50,8 +44,15 @@ from awx.main.fields import JSONField
from awx.main.utils import schedule_task_manager
__all__ = ['WorkflowJobTemplate', 'WorkflowJob', 'WorkflowJobOptions', 'WorkflowJobNode',
'WorkflowJobTemplateNode', 'WorkflowApprovalTemplate', 'WorkflowApproval']
__all__ = [
'WorkflowJobTemplate',
'WorkflowJob',
'WorkflowJobOptions',
'WorkflowJobNode',
'WorkflowJobTemplateNode',
'WorkflowApprovalTemplate',
'WorkflowApproval',
]
logger = logging.getLogger('awx.main.models.workflow')
@@ -81,9 +82,7 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig):
related_name='%(class)ss_always',
)
all_parents_must_converge = models.BooleanField(
default=False,
help_text=_("If enabled then the node will only run if all of the parent nodes "
"have met the criteria to reach this node")
default=False, help_text=_("If enabled then the node will only run if all of the parent nodes " "have met the criteria to reach this node")
)
unified_job_template = models.ForeignKey(
'UnifiedJobTemplate',
@@ -103,17 +102,24 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig):
@classmethod
def _get_workflow_job_field_names(cls):
'''
"""
Return field names that should be copied from template node to job node.
'''
return ['workflow_job', 'unified_job_template',
'extra_data', 'survey_passwords',
'inventory', 'credentials', 'char_prompts', 'all_parents_must_converge']
"""
return [
'workflow_job',
'unified_job_template',
'extra_data',
'survey_passwords',
'inventory',
'credentials',
'char_prompts',
'all_parents_must_converge',
]
def create_workflow_job_node(self, **kwargs):
'''
"""
Create a new workflow job node based on this workflow node.
'''
"""
create_kwargs = {}
for field_name in self._get_workflow_job_field_names():
if field_name == 'credentials':
@@ -135,9 +141,18 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig):
class WorkflowJobTemplateNode(WorkflowNodeBase):
FIELDS_TO_PRESERVE_AT_COPY = [
'unified_job_template', 'workflow_job_template', 'success_nodes', 'failure_nodes',
'always_nodes', 'credentials', 'inventory', 'extra_data', 'survey_passwords',
'char_prompts', 'all_parents_must_converge', 'identifier'
'unified_job_template',
'workflow_job_template',
'success_nodes',
'failure_nodes',
'always_nodes',
'credentials',
'inventory',
'extra_data',
'survey_passwords',
'char_prompts',
'all_parents_must_converge',
'identifier',
]
REENCRYPTION_BLOCKLIST_AT_COPY = ['extra_data', 'survey_passwords']
@@ -150,9 +165,7 @@ class WorkflowJobTemplateNode(WorkflowNodeBase):
max_length=512,
default=uuid4,
blank=False,
help_text=_(
'An identifier for this node that is unique within its workflow. '
'It is copied to workflow job nodes corresponding to this node.'),
help_text=_('An identifier for this node that is unique within its workflow. ' 'It is copied to workflow job nodes corresponding to this node.'),
)
class Meta:
@@ -166,10 +179,10 @@ class WorkflowJobTemplateNode(WorkflowNodeBase):
return reverse('api:workflow_job_template_node_detail', kwargs={'pk': self.pk}, request=request)
def create_wfjt_node_copy(self, user, workflow_job_template=None):
'''
"""
Copy this node to a new WFJT, leaving out related fields the user
is not allowed to access
'''
"""
create_kwargs = {}
allowed_creds = []
for field_name in self._get_workflow_job_field_names():
@@ -226,9 +239,11 @@ class WorkflowJobNode(WorkflowNodeBase):
)
do_not_run = models.BooleanField(
default=False,
help_text=_("Indicates that a job will not be created when True. Workflow runtime "
"semantics will mark this True if the node is in a path that will "
"decidedly not be ran. A value of False means the node may not run."),
help_text=_(
"Indicates that a job will not be created when True. Workflow runtime "
"semantics will mark this True if the node is in a path that will "
"decidedly not be ran. A value of False means the node may not run."
),
)
identifier = models.CharField(
max_length=512,
@@ -260,12 +275,12 @@ class WorkflowJobNode(WorkflowNodeBase):
return r
def get_job_kwargs(self):
'''
"""
In advance of creating a new unified job as part of a workflow,
this method builds the attributes to use
It alters the node by saving its updated version of
ancestor_artifacts, making it available to subsequent nodes.
'''
"""
# reject/accept prompted fields
data = {}
ujt_obj = self.unified_job_template
@@ -279,11 +294,11 @@ class WorkflowJobNode(WorkflowNodeBase):
prompts_data['extra_vars'].update(self.workflow_job.extra_vars_dict)
accepted_fields, ignored_fields, errors = ujt_obj._accept_or_ignore_job_kwargs(**prompts_data)
if errors:
logger.info(_('Bad launch configuration starting template {template_pk} as part of '
'workflow {workflow_pk}. Errors:\n{error_text}').format(
template_pk=ujt_obj.pk,
workflow_pk=self.pk,
error_text=errors))
logger.info(
_('Bad launch configuration starting template {template_pk} as part of ' 'workflow {workflow_pk}. Errors:\n{error_text}').format(
template_pk=ujt_obj.pk, workflow_pk=self.pk, error_text=errors
)
)
data.update(accepted_fields) # missing fields are handled in the scheduler
try:
# config saved on the workflow job itself
@@ -347,13 +362,15 @@ class WorkflowJobOptions(LaunchTimeConfigBase):
class Meta:
abstract = True
extra_vars = accepts_json(prevent_search(models.TextField(
blank=True,
default='',
)))
allow_simultaneous = models.BooleanField(
default=False
extra_vars = accepts_json(
prevent_search(
models.TextField(
blank=True,
default='',
)
)
)
allow_simultaneous = models.BooleanField(default=False)
extra_vars_dict = VarsDictProperty('extra_vars', True)
@@ -404,9 +421,7 @@ class WorkflowJobOptions(LaunchTimeConfigBase):
class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTemplateMixin, ResourceMixin, RelatedJobsMixin, WebhookTemplateMixin):
SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')]
FIELDS_TO_PRESERVE_AT_COPY = [
'labels', 'organization', 'instance_groups', 'workflow_job_template_nodes', 'credentials', 'survey_spec'
]
FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'organization', 'instance_groups', 'workflow_job_template_nodes', 'credentials', 'survey_spec']
class Meta:
app_label = 'main'
@@ -423,28 +438,30 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
blank=True,
default=False,
)
notification_templates_approvals = models.ManyToManyField(
"NotificationTemplate",
blank=True,
related_name='%(class)s_notification_templates_for_approvals'
)
notification_templates_approvals = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_approvals')
admin_role = ImplicitRoleField(parent_role=[
'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
'organization.workflow_admin_role'
])
execute_role = ImplicitRoleField(parent_role=[
'admin_role',
'organization.execute_role',
])
read_role = ImplicitRoleField(parent_role=[
'singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR,
'organization.auditor_role', 'execute_role', 'admin_role',
'approval_role',
])
approval_role = ImplicitRoleField(parent_role=[
'organization.approval_role', 'admin_role',
])
admin_role = ImplicitRoleField(parent_role=['singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, 'organization.workflow_admin_role'])
execute_role = ImplicitRoleField(
parent_role=[
'admin_role',
'organization.execute_role',
]
)
read_role = ImplicitRoleField(
parent_role=[
'singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR,
'organization.auditor_role',
'execute_role',
'admin_role',
'approval_role',
]
)
approval_role = ImplicitRoleField(
parent_role=[
'organization.approval_role',
'admin_role',
]
)
@property
def workflow_nodes(self):
@@ -458,46 +475,50 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
def _get_unified_jt_copy_names(cls):
base_list = super(WorkflowJobTemplate, cls)._get_unified_jt_copy_names()
base_list.remove('labels')
return (base_list |
set(['survey_spec', 'survey_enabled', 'ask_variables_on_launch', 'organization']))
return base_list | set(['survey_spec', 'survey_enabled', 'ask_variables_on_launch', 'organization'])
def get_absolute_url(self, request=None):
return reverse('api:workflow_job_template_detail', kwargs={'pk': self.pk}, request=request)
@property
def cache_timeout_blocked(self):
if WorkflowJob.objects.filter(workflow_job_template=self,
status__in=['pending', 'waiting', 'running']).count() >= getattr(settings, 'SCHEDULE_MAX_JOBS', 10):
logger.error("Workflow Job template %s could not be started because there are more than %s other jobs from that template waiting to run" %
(self.name, getattr(settings, 'SCHEDULE_MAX_JOBS', 10)))
if WorkflowJob.objects.filter(workflow_job_template=self, status__in=['pending', 'waiting', 'running']).count() >= getattr(
settings, 'SCHEDULE_MAX_JOBS', 10
):
logger.error(
"Workflow Job template %s could not be started because there are more than %s other jobs from that template waiting to run"
% (self.name, getattr(settings, 'SCHEDULE_MAX_JOBS', 10))
)
return True
return False
@property
def notification_templates(self):
base_notification_templates = NotificationTemplate.objects.all()
error_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
started_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
success_notification_templates = list(base_notification_templates
.filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
approval_notification_templates = list(base_notification_templates
.filter(workflowjobtemplate_notification_templates_for_approvals__in=[self]))
error_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
started_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
success_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
approval_notification_templates = list(base_notification_templates.filter(workflowjobtemplate_notification_templates_for_approvals__in=[self]))
# Get Organization NotificationTemplates
if self.organization is not None:
error_notification_templates = set(error_notification_templates + list(base_notification_templates.filter(
organization_notification_templates_for_errors=self.organization)))
started_notification_templates = set(started_notification_templates + list(base_notification_templates.filter(
organization_notification_templates_for_started=self.organization)))
success_notification_templates = set(success_notification_templates + list(base_notification_templates.filter(
organization_notification_templates_for_success=self.organization)))
approval_notification_templates = set(approval_notification_templates + list(base_notification_templates.filter(
organization_notification_templates_for_approvals=self.organization)))
return dict(error=list(error_notification_templates),
started=list(started_notification_templates),
success=list(success_notification_templates),
approvals=list(approval_notification_templates))
error_notification_templates = set(
error_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_errors=self.organization))
)
started_notification_templates = set(
started_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_started=self.organization))
)
success_notification_templates = set(
success_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_success=self.organization))
)
approval_notification_templates = set(
approval_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_approvals=self.organization))
)
return dict(
error=list(error_notification_templates),
started=list(started_notification_templates),
success=list(success_notification_templates),
approvals=list(approval_notification_templates),
)
def create_unified_job(self, **kwargs):
workflow_job = super(WorkflowJobTemplate, self).create_unified_job(**kwargs)
@@ -516,9 +537,8 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
if field_name == 'extra_vars':
accepted_vars, rejected_vars, vars_errors = self.accept_or_ignore_variables(
kwargs.get('extra_vars', {}),
_exclude_errors=exclude_errors,
extra_passwords=kwargs.get('survey_passwords', {}))
kwargs.get('extra_vars', {}), _exclude_errors=exclude_errors, extra_passwords=kwargs.get('survey_passwords', {})
)
if accepted_vars:
prompted_data['extra_vars'] = accepted_vars
if rejected_vars:
@@ -550,8 +570,7 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
return not bool(self.variables_needed_to_start)
def node_templates_missing(self):
return [node.pk for node in self.workflow_job_template_nodes.filter(
unified_job_template__isnull=True).all()]
return [node.pk for node in self.workflow_job_template_nodes.filter(unified_job_template__isnull=True).all()]
def node_prompts_rejected(self):
node_list = []
@@ -568,6 +587,7 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return WorkflowJob.objects.filter(workflow_job_template=self)
@@ -592,12 +612,9 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
null=True,
default=None,
on_delete=models.SET_NULL,
help_text=_("If automatically created for a sliced job run, the job template "
"the workflow job was created from."),
)
is_sliced_job = models.BooleanField(
default=False
help_text=_("If automatically created for a sliced job run, the job template " "the workflow job was created from."),
)
is_sliced_job = models.BooleanField(default=False)
@property
def workflow_nodes(self):
@@ -629,8 +646,7 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
if node.job is None:
node_job_description = 'no job.'
else:
node_job_description = ('job #{0}, "{1}", which finished with status {2}.'
.format(node.job.id, node.job.name, node.job.status))
node_job_description = 'job #{0}, "{1}", which finished with status {2}.'.format(node.job.id, node.job.name, node.job.status)
str_arr.append("- node #{0} spawns {1}".format(node.id, node_job_description))
result['body'] = '\n'.join(str_arr)
return result
@@ -649,8 +665,7 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
wj = self.get_workflow_job()
while wj and wj.workflow_job_template_id:
if wj.pk in wj_ids:
logger.critical('Cycles detected in the workflow jobs graph, '
'this is not normal and suggests task manager degeneracy.')
logger.critical('Cycles detected in the workflow jobs graph, ' 'this is not normal and suggests task manager degeneracy.')
break
wj_ids.add(wj.pk)
ancestors.append(wj.workflow_job_template)
@@ -676,7 +691,10 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
class WorkflowApprovalTemplate(UnifiedJobTemplate, RelatedJobsMixin):
FIELDS_TO_PRESERVE_AT_COPY = ['description', 'timeout',]
FIELDS_TO_PRESERVE_AT_COPY = [
'description',
'timeout',
]
class Meta:
app_label = 'main'
@@ -705,6 +723,7 @@ class WorkflowApprovalTemplate(UnifiedJobTemplate, RelatedJobsMixin):
'''
RelatedJobsMixin
'''
def _get_related_jobs(self):
return UnifiedJob.objects.filter(unified_job_template=self)
@@ -726,10 +745,7 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin):
default=0,
help_text=_("The amount of time (in seconds) before the approval node expires and fails."),
)
timed_out = models.BooleanField(
default=False,
help_text=_("Shows when an approval node (with a timeout assigned to it) has timed out.")
)
timed_out = models.BooleanField(default=False, help_text=_("Shows when an approval node (with a timeout assigned to it) has timed out."))
approved_or_denied_by = models.ForeignKey(
'auth.User',
related_name='%s(class)s_approved+',
@@ -739,7 +755,6 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin):
on_delete=models.SET_NULL,
)
@classmethod
def _get_unified_job_template_class(cls):
return WorkflowApprovalTemplate
@@ -788,6 +803,7 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin):
def send_approval_notification(self, approval_status):
from awx.main.tasks import send_notifications # avoid circular import
if self.workflow_job_template is None:
return
for nt in self.workflow_job_template.notification_templates["approvals"]:
@@ -800,9 +816,10 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin):
# https://stackoverflow.com/a/3431699/10669572
def send_it(local_nt=nt, local_subject=notification_subject, local_body=notification_body):
def _func():
send_notifications.delay([local_nt.generate_notification(local_subject, local_body).id],
job_id=self.id)
send_notifications.delay([local_nt.generate_notification(local_subject, local_body).id], job_id=self.id)
return _func
connection.on_commit(send_it())
def build_approval_notification_message(self, nt, approval_status):
@@ -841,10 +858,12 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin):
def context(self, approval_status):
workflow_url = urljoin(settings.TOWER_URL_BASE, '/#/jobs/workflow/{}'.format(self.workflow_job.id))
return {'approval_status': approval_status,
'approval_node_name': self.workflow_approval_template.name,
'workflow_url': workflow_url,
'job_metadata': json.dumps(self.notification_data(), indent=4)}
return {
'approval_status': approval_status,
'approval_node_name': self.workflow_approval_template.name,
'workflow_url': workflow_url,
'job_metadata': json.dumps(self.notification_data(), indent=4),
}
@property
def workflow_job_template(self):