diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 42516b9a15..bd3da38b51 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,7 +80,7 @@ For Linux platforms, refer to the following from Docker: If you're not using Docker for Mac, or Docker for Windows, you may need, or choose to, install the Docker compose Python module separately, in which case you'll need to run the following: ```bash -(host)$ pip install docker-compose +(host)$ pip3 install docker-compose ``` #### Frontend Development diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index 45cc247bf4..ba27905092 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -273,7 +273,7 @@ def copy_tables(since, full_path, subset=None): main_unifiedjob.organization_id, main_organization.name as organization_name, main_job.inventory_id, - main_inventory.name, + main_inventory.name as inventory_name, main_unifiedjob.created, main_unifiedjob.name, main_unifiedjob.unified_job_template_id, diff --git a/awx/main/consumers.py b/awx/main/consumers.py index d32219b3ac..b6d8872ebd 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -1,5 +1,3 @@ -import collections -import functools import json import logging import time @@ -14,40 +12,12 @@ from django.contrib.auth.models import User from channels.generic.websocket import AsyncJsonWebsocketConsumer from channels.layers import get_channel_layer from channels.db import database_sync_to_async -from channels_redis.core import RedisChannelLayer logger = logging.getLogger('awx.main.consumers') XRF_KEY = '_auth_user_xrf' -class BoundedQueue(asyncio.Queue): - - def put_nowait(self, item): - if self.full(): - # dispose the oldest item - # if we actually get into this code block, it likely means that - # this specific consumer has stopped reading - # unfortunately, channels_redis will just happily continue to - # queue messages specific to their channel until the heat death - # of the sun: https://github.com/django/channels_redis/issues/212 - # this isn't a huge deal for browser clients that disconnect, - # but it *does* cause a problem for our global broadcast topic - # that's used to broadcast messages to peers in a cluster - # if we get into this code block, it's better to drop messages - # than to continue to malloc() forever - self.get_nowait() - return super(BoundedQueue, self).put_nowait(item) - - -class ExpiringRedisChannelLayer(RedisChannelLayer): - def __init__(self, *args, **kw): - super(ExpiringRedisChannelLayer, self).__init__(*args, **kw) - self.receive_buffer = collections.defaultdict( - functools.partial(BoundedQueue, self.capacity) - ) - - class WebsocketSecretAuthHelper: """ Middlewareish for websockets to verify node websocket broadcast interconnect. diff --git a/awx/main/models/ha.py b/awx/main/models/ha.py index ac7df97e10..4b6d8926e9 100644 --- a/awx/main/models/ha.py +++ b/awx/main/models/ha.py @@ -12,6 +12,7 @@ from django.utils.translation import ugettext_lazy as _ from django.conf import settings from django.utils.timezone import now, timedelta +import redis from solo.models import SingletonModel from awx import __version__ as awx_application_version @@ -152,6 +153,14 @@ class Instance(HasPolicyEditsMixin, BaseModel): self.capacity = get_system_task_capacity(self.capacity_adjustment) else: self.capacity = 0 + + try: + # if redis is down for some reason, that means we can't persist + # playbook event data; we should consider this a zero capacity event + redis.Redis.from_url(settings.BROKER_URL).ping() + except redis.ConnectionError: + self.capacity = 0 + self.cpu = cpu[0] self.memory = mem[0] self.cpu_capacity = cpu[1] diff --git a/awx/main/scheduler/task_manager.py b/awx/main/scheduler/task_manager.py index cba6161d83..9f4818bd37 100644 --- a/awx/main/scheduler/task_manager.py +++ b/awx/main/scheduler/task_manager.py @@ -12,6 +12,7 @@ import random from django.db import transaction, connection from django.utils.translation import ugettext_lazy as _, gettext_noop from django.utils.timezone import now as tz_now +from django.conf import settings # AWX from awx.main.dispatch.reaper import reap_job @@ -45,6 +46,12 @@ class TaskManager(): def __init__(self): self.graph = dict() + # start task limit indicates how many pending jobs can be started on this + # .schedule() run. Starting jobs is expensive, and there is code in place to reap + # the task manager after 5 minutes. At scale, the task manager can easily take more than + # 5 minutes to start pending jobs. If this limit is reached, pending jobs + # will no longer be started and will be started on the next task manager cycle. + self.start_task_limit = settings.START_TASK_LIMIT for rampart_group in InstanceGroup.objects.prefetch_related('instances'): self.graph[rampart_group.name] = dict(graph=DependencyGraph(rampart_group.name), capacity_total=rampart_group.capacity, @@ -189,6 +196,10 @@ class TaskManager(): return result def start_task(self, task, rampart_group, dependent_tasks=None, instance=None): + self.start_task_limit -= 1 + if self.start_task_limit == 0: + # schedule another run immediately after this task manager + schedule_task_manager() from awx.main.tasks import handle_work_error, handle_work_success dependent_tasks = dependent_tasks or [] @@ -448,6 +459,8 @@ class TaskManager(): def process_pending_tasks(self, pending_tasks): running_workflow_templates = set([wf.unified_job_template_id for wf in self.get_running_workflow_jobs()]) for task in pending_tasks: + if self.start_task_limit <= 0: + break if self.is_job_blocked(task): logger.debug("{} is blocked from running".format(task.log_format)) continue diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 0c7b219371..2857b00c57 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1632,11 +1632,6 @@ class RunJob(BaseTask): # callbacks to work. env['JOB_ID'] = str(job.pk) env['INVENTORY_ID'] = str(job.inventory.pk) - if job.use_fact_cache: - library_source = self.get_path_to('..', 'plugins', 'library') - library_dest = os.path.join(private_data_dir, 'library') - copy_tree(library_source, library_dest) - env['ANSIBLE_LIBRARY'] = library_dest if job.project: env['PROJECT_REVISION'] = job.project.scm_revision env['ANSIBLE_RETRY_FILES_ENABLED'] = "False" diff --git a/awx/main/tests/factories/README.md b/awx/main/tests/factories/README.md index c451c02598..916c996cfa 100644 --- a/awx/main/tests/factories/README.md +++ b/awx/main/tests/factories/README.md @@ -52,11 +52,11 @@ patterns -------- `mk` functions are single object fixtures. They should create only a single object with the minimum deps. -They should also accept a `persited` flag, if they must be persisted to work, they raise an error if persisted=False +They should also accept a `persisted` flag, if they must be persisted to work, they raise an error if persisted=False `generate` and `apply` functions are helpers that build up the various parts of a `create` functions objects. These should be useful for more than one create function to use and should explicitly accept all of the values needed -to execute. These functions should also be robust and have very speciifc error reporting about constraints and/or +to execute. These functions should also be robust and have very specific error reporting about constraints and/or bad values. `create` functions compose many of the `mk` and `generate` functions to make different object diff --git a/awx/main/tests/functional/test_jobs.py b/awx/main/tests/functional/test_jobs.py index 2bc10fa0df..b4754a6803 100644 --- a/awx/main/tests/functional/test_jobs.py +++ b/awx/main/tests/functional/test_jobs.py @@ -1,3 +1,4 @@ +import redis import pytest from unittest import mock import json @@ -25,7 +26,8 @@ def test_orphan_unified_job_creation(instance, inventory): @mock.patch('awx.main.utils.common.get_mem_capacity', lambda: (8000,62)) def test_job_capacity_and_with_inactive_node(): i = Instance.objects.create(hostname='test-1') - i.refresh_capacity() + with mock.patch.object(redis.client.Redis, 'ping', lambda self: True): + i.refresh_capacity() assert i.capacity == 62 i.enabled = False i.save() @@ -35,6 +37,19 @@ def test_job_capacity_and_with_inactive_node(): assert i.capacity == 0 +@pytest.mark.django_db +@mock.patch('awx.main.utils.common.get_cpu_capacity', lambda: (2,8)) +@mock.patch('awx.main.utils.common.get_mem_capacity', lambda: (8000,62)) +def test_job_capacity_with_redis_disabled(): + i = Instance.objects.create(hostname='test-1') + + def _raise(self): + raise redis.ConnectionError() + with mock.patch.object(redis.client.Redis, 'ping', _raise): + i.refresh_capacity() + assert i.capacity == 0 + + @pytest.mark.django_db def test_job_type_name(): job = Job.objects.create() diff --git a/awx/playbooks/scan_facts.yml b/awx/playbooks/scan_facts.yml deleted file mode 100644 index 884760f717..0000000000 --- a/awx/playbooks/scan_facts.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -- hosts: all - vars: - scan_use_checksum: false - scan_use_recursive: false - tasks: - - - name: "Scan packages (Unix/Linux)" - scan_packages: - os_family: '{{ ansible_os_family }}' - when: ansible_os_family != "Windows" - - name: "Scan services (Unix/Linux)" - scan_services: - when: ansible_os_family != "Windows" - - name: "Scan files (Unix/Linux)" - scan_files: - paths: '{{ scan_file_paths }}' - get_checksum: '{{ scan_use_checksum }}' - recursive: '{{ scan_use_recursive }}' - when: scan_file_paths is defined and ansible_os_family != "Windows" - - name: "Scan Insights for Machine ID (Unix/Linux)" - scan_insights: - when: ansible_os_family != "Windows" - - - name: "Scan packages (Windows)" - win_scan_packages: - when: ansible_os_family == "Windows" - - name: "Scan services (Windows)" - win_scan_services: - when: ansible_os_family == "Windows" - - name: "Scan files (Windows)" - win_scan_files: - paths: '{{ scan_file_paths }}' - get_checksum: '{{ scan_use_checksum }}' - recursive: '{{ scan_use_recursive }}' - when: scan_file_paths is defined and ansible_os_family == "Windows" diff --git a/awx/plugins/library/scan_files.py b/awx/plugins/library/scan_files.py deleted file mode 100644 index b3b07ad64a..0000000000 --- a/awx/plugins/library/scan_files.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python - -import os -import stat -from ansible.module_utils.basic import * # noqa - -DOCUMENTATION = ''' ---- -module: scan_files -short_description: Return file state information as fact data for a directory tree -description: - - Return file state information recursively for a directory tree on the filesystem -version_added: "1.9" -options: - path: - description: The path containing files to be analyzed - required: true - default: null - recursive: - description: scan this directory and all subdirectories - required: false - default: no - get_checksum: - description: Checksum files that you can access - required: false - default: false -requirements: [ ] -author: Matthew Jones -''' - -EXAMPLES = ''' -# Example fact output: -# host | success >> { -# "ansible_facts": { -# "files": [ -# { -# "atime": 1427313854.0755742, -# "checksum": "cf7566e6149ad9af91e7589e0ea096a08de9c1e5", -# "ctime": 1427129299.22948, -# "dev": 51713, -# "gid": 0, -# "inode": 149601, -# "isblk": false, -# "ischr": false, -# "isdir": false, -# "isfifo": false, -# "isgid": false, -# "islnk": false, -# "isreg": true, -# "issock": false, -# "isuid": false, -# "mode": "0644", -# "mtime": 1427112663.0321455, -# "nlink": 1, -# "path": "/var/log/dmesg.1.gz", -# "rgrp": true, -# "roth": true, -# "rusr": true, -# "size": 28, -# "uid": 0, -# "wgrp": false, -# "woth": false, -# "wusr": true, -# "xgrp": false, -# "xoth": false, -# "xusr": false -# }, -# { -# "atime": 1427314385.1155744, -# "checksum": "16fac7be61a6e4591a33ef4b729c5c3302307523", -# "ctime": 1427384148.5755742, -# "dev": 51713, -# "gid": 43, -# "inode": 149564, -# "isblk": false, -# "ischr": false, -# "isdir": false, -# "isfifo": false, -# "isgid": false, -# "islnk": false, -# "isreg": true, -# "issock": false, -# "isuid": false, -# "mode": "0664", -# "mtime": 1427384148.5755742, -# "nlink": 1, -# "path": "/var/log/wtmp", -# "rgrp": true, -# "roth": true, -# "rusr": true, -# "size": 48768, -# "uid": 0, -# "wgrp": true, -# "woth": false, -# "wusr": true, -# "xgrp": false, -# "xoth": false, -# "xusr": false -# }, -''' - - -def main(): - module = AnsibleModule( # noqa - argument_spec = dict(paths=dict(required=True, type='list'), - recursive=dict(required=False, default='no', type='bool'), - get_checksum=dict(required=False, default='no', type='bool'))) - files = [] - paths = module.params.get('paths') - for path in paths: - path = os.path.expanduser(path) - if not os.path.exists(path) or not os.path.isdir(path): - module.fail_json(msg = "Given path must exist and be a directory") - - get_checksum = module.params.get('get_checksum') - should_recurse = module.params.get('recursive') - if not should_recurse: - path_list = [os.path.join(path, subpath) for subpath in os.listdir(path)] - else: - path_list = [os.path.join(w_path, f) for w_path, w_names, w_file in os.walk(path) for f in w_file] - for filepath in path_list: - try: - st = os.stat(filepath) - except OSError: - continue - - mode = st.st_mode - d = { - 'path' : filepath, - 'mode' : "%04o" % stat.S_IMODE(mode), - 'isdir' : stat.S_ISDIR(mode), - 'ischr' : stat.S_ISCHR(mode), - 'isblk' : stat.S_ISBLK(mode), - 'isreg' : stat.S_ISREG(mode), - 'isfifo' : stat.S_ISFIFO(mode), - 'islnk' : stat.S_ISLNK(mode), - 'issock' : stat.S_ISSOCK(mode), - 'uid' : st.st_uid, - 'gid' : st.st_gid, - 'size' : st.st_size, - 'inode' : st.st_ino, - 'dev' : st.st_dev, - 'nlink' : st.st_nlink, - 'atime' : st.st_atime, - 'mtime' : st.st_mtime, - 'ctime' : st.st_ctime, - 'wusr' : bool(mode & stat.S_IWUSR), - 'rusr' : bool(mode & stat.S_IRUSR), - 'xusr' : bool(mode & stat.S_IXUSR), - 'wgrp' : bool(mode & stat.S_IWGRP), - 'rgrp' : bool(mode & stat.S_IRGRP), - 'xgrp' : bool(mode & stat.S_IXGRP), - 'woth' : bool(mode & stat.S_IWOTH), - 'roth' : bool(mode & stat.S_IROTH), - 'xoth' : bool(mode & stat.S_IXOTH), - 'isuid' : bool(mode & stat.S_ISUID), - 'isgid' : bool(mode & stat.S_ISGID), - } - if get_checksum and stat.S_ISREG(mode) and os.access(filepath, os.R_OK): - d['checksum'] = module.sha1(filepath) - files.append(d) - results = dict(ansible_facts=dict(files=files)) - module.exit_json(**results) - - -main() diff --git a/awx/plugins/library/scan_insights.py b/awx/plugins/library/scan_insights.py deleted file mode 100755 index f7b7919bca..0000000000 --- a/awx/plugins/library/scan_insights.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -from ansible.module_utils.basic import * # noqa - -DOCUMENTATION = ''' ---- -module: scan_insights -short_description: Return insights id as fact data -description: - - Inspects the /etc/redhat-access-insights/machine-id file for insights id and returns the found id as fact data -version_added: "2.3" -options: -requirements: [ ] -author: Chris Meyers -''' - -EXAMPLES = ''' -# Example fact output: -# host | success >> { -# "ansible_facts": { -# "insights": { -# "system_id": "4da7d1f8-14f3-4cdc-acd5-a3465a41f25d" -# }, ... } -''' - - -INSIGHTS_SYSTEM_ID_FILE='/etc/redhat-access-insights/machine-id' - - -def get_system_id(filname): - system_id = None - try: - f = open(INSIGHTS_SYSTEM_ID_FILE, "r") - except IOError: - return None - else: - try: - data = f.readline() - system_id = str(data) - except (IOError, ValueError): - pass - finally: - f.close() - if system_id: - system_id = system_id.strip() - return system_id - - -def main(): - module = AnsibleModule( # noqa - argument_spec = dict() - ) - - system_id = get_system_id(INSIGHTS_SYSTEM_ID_FILE) - - results = { - 'ansible_facts': { - 'insights': { - 'system_id': system_id - } - } - } - module.exit_json(**results) - - -main() diff --git a/awx/plugins/library/scan_packages.py b/awx/plugins/library/scan_packages.py deleted file mode 100755 index d0b544bb9f..0000000000 --- a/awx/plugins/library/scan_packages.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python - -from ansible.module_utils.basic import * # noqa - -DOCUMENTATION = ''' ---- -module: scan_packages -short_description: Return installed packages information as fact data -description: - - Return information about installed packages as fact data -version_added: "1.9" -options: -requirements: [ ] -author: Matthew Jones -''' - -EXAMPLES = ''' -# Example fact output: -# host | success >> { -# "ansible_facts": { -# "packages": { -# "libbz2-1.0": [ -# { -# "version": "1.0.6-5", -# "source": "apt", -# "arch": "amd64", -# "name": "libbz2-1.0" -# } -# ], -# "patch": [ -# { -# "version": "2.7.1-4ubuntu1", -# "source": "apt", -# "arch": "amd64", -# "name": "patch" -# } -# ], -# "gcc-4.8-base": [ -# { -# "version": "4.8.2-19ubuntu1", -# "source": "apt", -# "arch": "amd64", -# "name": "gcc-4.8-base" -# }, -# { -# "version": "4.9.2-19ubuntu1", -# "source": "apt", -# "arch": "amd64", -# "name": "gcc-4.8-base" -# } -# ] -# } -''' - - -def rpm_package_list(): - import rpm - trans_set = rpm.TransactionSet() - installed_packages = {} - for package in trans_set.dbMatch(): - package_details = dict(name=package[rpm.RPMTAG_NAME], - version=package[rpm.RPMTAG_VERSION], - release=package[rpm.RPMTAG_RELEASE], - epoch=package[rpm.RPMTAG_EPOCH], - arch=package[rpm.RPMTAG_ARCH], - source='rpm') - if package_details['name'] not in installed_packages: - installed_packages[package_details['name']] = [package_details] - else: - installed_packages[package_details['name']].append(package_details) - return installed_packages - - -def deb_package_list(): - import apt - apt_cache = apt.Cache() - installed_packages = {} - apt_installed_packages = [pk for pk in apt_cache.keys() if apt_cache[pk].is_installed] - for package in apt_installed_packages: - ac_pkg = apt_cache[package].installed - package_details = dict(name=package, - version=ac_pkg.version, - arch=ac_pkg.architecture, - source='apt') - if package_details['name'] not in installed_packages: - installed_packages[package_details['name']] = [package_details] - else: - installed_packages[package_details['name']].append(package_details) - return installed_packages - - -def main(): - module = AnsibleModule( # noqa - argument_spec = dict(os_family=dict(required=True)) - ) - ans_os = module.params['os_family'] - if ans_os in ('RedHat', 'Suse', 'openSUSE Leap'): - packages = rpm_package_list() - elif ans_os == 'Debian': - packages = deb_package_list() - else: - packages = None - - if packages is not None: - results = dict(ansible_facts=dict(packages=packages)) - else: - results = dict(skipped=True, msg="Unsupported Distribution") - module.exit_json(**results) - - -main() diff --git a/awx/plugins/library/scan_services.py b/awx/plugins/library/scan_services.py deleted file mode 100644 index 5d8ccdbb74..0000000000 --- a/awx/plugins/library/scan_services.py +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python - -import re -from ansible.module_utils.basic import * # noqa - -DOCUMENTATION = ''' ---- -module: scan_services -short_description: Return service state information as fact data -description: - - Return service state information as fact data for various service management utilities -version_added: "1.9" -options: -requirements: [ ] -author: Matthew Jones -''' - -EXAMPLES = ''' -- monit: scan_services -# Example fact output: -# host | success >> { -# "ansible_facts": { -# "services": { -# "network": { -# "source": "sysv", -# "state": "running", -# "name": "network" -# }, -# "arp-ethers.service": { -# "source": "systemd", -# "state": "stopped", -# "name": "arp-ethers.service" -# } -# } -# } -''' - - -class BaseService(object): - - def __init__(self, module): - self.module = module - self.incomplete_warning = False - - -class ServiceScanService(BaseService): - - def gather_services(self): - services = {} - service_path = self.module.get_bin_path("service") - if service_path is None: - return None - initctl_path = self.module.get_bin_path("initctl") - chkconfig_path = self.module.get_bin_path("chkconfig") - - # sysvinit - if service_path is not None and chkconfig_path is None: - rc, stdout, stderr = self.module.run_command("%s --status-all 2>&1 | grep -E \"\\[ (\\+|\\-) \\]\"" % service_path, use_unsafe_shell=True) - for line in stdout.split("\n"): - line_data = line.split() - if len(line_data) < 4: - continue # Skipping because we expected more data - service_name = " ".join(line_data[3:]) - if line_data[1] == "+": - service_state = "running" - else: - service_state = "stopped" - services[service_name] = {"name": service_name, "state": service_state, "source": "sysv"} - - # Upstart - if initctl_path is not None and chkconfig_path is None: - p = re.compile(r'^\s?(?P.*)\s(?P\w+)\/(?P\w+)(\,\sprocess\s(?P[0-9]+))?\s*$') - rc, stdout, stderr = self.module.run_command("%s list" % initctl_path) - real_stdout = stdout.replace("\r","") - for line in real_stdout.split("\n"): - m = p.match(line) - if not m: - continue - service_name = m.group('name') - service_goal = m.group('goal') - service_state = m.group('state') - if m.group('pid'): - pid = m.group('pid') - else: - pid = None # NOQA - payload = {"name": service_name, "state": service_state, "goal": service_goal, "source": "upstart"} - services[service_name] = payload - - # RH sysvinit - elif chkconfig_path is not None: - #print '%s --status-all | grep -E "is (running|stopped)"' % service_path - p = re.compile( - r'(?P.*?)\s+[0-9]:(?Pon|off)\s+[0-9]:(?Pon|off)\s+[0-9]:(?Pon|off)\s+' - r'[0-9]:(?Pon|off)\s+[0-9]:(?Pon|off)\s+[0-9]:(?Pon|off)\s+[0-9]:(?Pon|off)') - rc, stdout, stderr = self.module.run_command('%s' % chkconfig_path, use_unsafe_shell=True) - # Check for special cases where stdout does not fit pattern - match_any = False - for line in stdout.split('\n'): - if p.match(line): - match_any = True - if not match_any: - p_simple = re.compile(r'(?P.*?)\s+(?Pon|off)') - match_any = False - for line in stdout.split('\n'): - if p_simple.match(line): - match_any = True - if match_any: - # Try extra flags " -l --allservices" needed for SLES11 - rc, stdout, stderr = self.module.run_command('%s -l --allservices' % chkconfig_path, use_unsafe_shell=True) - elif '--list' in stderr: - # Extra flag needed for RHEL5 - rc, stdout, stderr = self.module.run_command('%s --list' % chkconfig_path, use_unsafe_shell=True) - for line in stdout.split('\n'): - m = p.match(line) - if m: - service_name = m.group('service') - service_state = 'stopped' - if m.group('rl3') == 'on': - rc, stdout, stderr = self.module.run_command('%s %s status' % (service_path, service_name), use_unsafe_shell=True) - service_state = rc - if rc in (0,): - service_state = 'running' - #elif rc in (1,3): - else: - if 'root' in stderr or 'permission' in stderr.lower() or 'not in sudoers' in stderr.lower(): - self.incomplete_warning = True - continue - else: - service_state = 'stopped' - service_data = {"name": service_name, "state": service_state, "source": "sysv"} - services[service_name] = service_data - return services - - -class SystemctlScanService(BaseService): - - def systemd_enabled(self): - # Check if init is the systemd command, using comm as cmdline could be symlink - try: - f = open('/proc/1/comm', 'r') - except IOError: - # If comm doesn't exist, old kernel, no systemd - return False - for line in f: - if 'systemd' in line: - return True - return False - - def gather_services(self): - services = {} - if not self.systemd_enabled(): - return None - systemctl_path = self.module.get_bin_path("systemctl", opt_dirs=["/usr/bin", "/usr/local/bin"]) - if systemctl_path is None: - return None - rc, stdout, stderr = self.module.run_command("%s list-unit-files --type=service | tail -n +2 | head -n -2" % systemctl_path, use_unsafe_shell=True) - for line in stdout.split("\n"): - line_data = line.split() - if len(line_data) != 2: - continue - if line_data[1] == "enabled": - state_val = "running" - else: - state_val = "stopped" - services[line_data[0]] = {"name": line_data[0], "state": state_val, "source": "systemd"} - return services - - -def main(): - module = AnsibleModule(argument_spec = dict()) # noqa - service_modules = (ServiceScanService, SystemctlScanService) - all_services = {} - incomplete_warning = False - for svc_module in service_modules: - svcmod = svc_module(module) - svc = svcmod.gather_services() - if svc is not None: - all_services.update(svc) - if svcmod.incomplete_warning: - incomplete_warning = True - if len(all_services) == 0: - results = dict(skipped=True, msg="Failed to find any services. Sometimes this is due to insufficient privileges.") - else: - results = dict(ansible_facts=dict(services=all_services)) - if incomplete_warning: - results['msg'] = "WARNING: Could not find status for all services. Sometimes this is due to insufficient privileges." - module.exit_json(**results) - - -main() diff --git a/awx/plugins/library/win_scan_files.ps1 b/awx/plugins/library/win_scan_files.ps1 deleted file mode 100644 index 6d114dfcc8..0000000000 --- a/awx/plugins/library/win_scan_files.ps1 +++ /dev/null @@ -1,102 +0,0 @@ -#!powershell -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -# WANT_JSON -# POWERSHELL_COMMON - -$params = Parse-Args $args $true; - -$paths = Get-Attr $params "paths" $FALSE; -If ($paths -eq $FALSE) -{ - Fail-Json (New-Object psobject) "missing required argument: paths"; -} - -$get_checksum = Get-Attr $params "get_checksum" $false | ConvertTo-Bool; -$recursive = Get-Attr $params "recursive" $false | ConvertTo-Bool; - -function Date_To_Timestamp($start_date, $end_date) -{ - If($start_date -and $end_date) - { - Write-Output (New-TimeSpan -Start $start_date -End $end_date).TotalSeconds - } -} - -$files = @() - -ForEach ($path In $paths) -{ - "Path: " + $path - ForEach ($file in Get-ChildItem $path -Recurse: $recursive) - { - "File: " + $file.FullName - $fileinfo = New-Object psobject - Set-Attr $fileinfo "path" $file.FullName - $info = Get-Item $file.FullName; - $iscontainer = Get-Attr $info "PSIsContainer" $null; - $length = Get-Attr $info "Length" $null; - $extension = Get-Attr $info "Extension" $null; - $attributes = Get-Attr $info "Attributes" ""; - If ($info) - { - $accesscontrol = $info.GetAccessControl(); - } - Else - { - $accesscontrol = $null; - } - $owner = Get-Attr $accesscontrol "Owner" $null; - $creationtime = Get-Attr $info "CreationTime" $null; - $lastaccesstime = Get-Attr $info "LastAccessTime" $null; - $lastwritetime = Get-Attr $info "LastWriteTime" $null; - - $epoch_date = Get-Date -Date "01/01/1970" - If ($iscontainer) - { - Set-Attr $fileinfo "isdir" $TRUE; - } - Else - { - Set-Attr $fileinfo "isdir" $FALSE; - Set-Attr $fileinfo "size" $length; - } - Set-Attr $fileinfo "extension" $extension; - Set-Attr $fileinfo "attributes" $attributes.ToString(); - # Set-Attr $fileinfo "owner" $getaccesscontrol.Owner; - # Set-Attr $fileinfo "owner" $info.GetAccessControl().Owner; - Set-Attr $fileinfo "owner" $owner; - Set-Attr $fileinfo "creationtime" (Date_To_Timestamp $epoch_date $creationtime); - Set-Attr $fileinfo "lastaccesstime" (Date_To_Timestamp $epoch_date $lastaccesstime); - Set-Attr $fileinfo "lastwritetime" (Date_To_Timestamp $epoch_date $lastwritetime); - - If (($get_checksum) -and -not $fileinfo.isdir) - { - $hash = Get-FileChecksum($file.FullName); - Set-Attr $fileinfo "checksum" $hash; - } - - $files += $fileinfo - } -} - -$result = New-Object psobject @{ - ansible_facts = New-Object psobject @{ - files = $files - } -} - -Exit-Json $result; diff --git a/awx/plugins/library/win_scan_packages.ps1 b/awx/plugins/library/win_scan_packages.ps1 deleted file mode 100644 index 2ab3fdbec6..0000000000 --- a/awx/plugins/library/win_scan_packages.ps1 +++ /dev/null @@ -1,66 +0,0 @@ -#!powershell -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -# WANT_JSON -# POWERSHELL_COMMON - -$uninstall_native_path = "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall" -$uninstall_wow6432_path = "HKLM:\SOFTWARE\Wow6432Node\Microsoft\Windows\CurrentVersion\Uninstall" - -if ([System.IntPtr]::Size -eq 4) { - - # This is a 32-bit Windows system, so we only check for 32-bit programs, which will be - # at the native registry location. - - [PSObject []]$packages = Get-ChildItem -Path $uninstall_native_path | - Get-ItemProperty | - Select-Object -Property @{Name="name"; Expression={$_."DisplayName"}}, - @{Name="version"; Expression={$_."DisplayVersion"}}, - @{Name="publisher"; Expression={$_."Publisher"}}, - @{Name="arch"; Expression={ "Win32" }} | - Where-Object { $_.name } - -} else { - - # This is a 64-bit Windows system, so we check for 64-bit programs in the native - # registry location, and also for 32-bit programs under Wow6432Node. - - [PSObject []]$packages = Get-ChildItem -Path $uninstall_native_path | - Get-ItemProperty | - Select-Object -Property @{Name="name"; Expression={$_."DisplayName"}}, - @{Name="version"; Expression={$_."DisplayVersion"}}, - @{Name="publisher"; Expression={$_."Publisher"}}, - @{Name="arch"; Expression={ "Win64" }} | - Where-Object { $_.name } - - $packages += Get-ChildItem -Path $uninstall_wow6432_path | - Get-ItemProperty | - Select-Object -Property @{Name="name"; Expression={$_."DisplayName"}}, - @{Name="version"; Expression={$_."DisplayVersion"}}, - @{Name="publisher"; Expression={$_."Publisher"}}, - @{Name="arch"; Expression={ "Win32" }} | - Where-Object { $_.name } - -} - -$result = New-Object psobject @{ - ansible_facts = New-Object psobject @{ - packages = $packages - } - changed = $false -} - -Exit-Json $result; diff --git a/awx/plugins/library/win_scan_services.ps1 b/awx/plugins/library/win_scan_services.ps1 deleted file mode 100644 index 3de8ac4c9b..0000000000 --- a/awx/plugins/library/win_scan_services.ps1 +++ /dev/null @@ -1,30 +0,0 @@ -#!powershell -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -# WANT_JSON -# POWERSHELL_COMMON - -$result = New-Object psobject @{ - ansible_facts = New-Object psobject @{ - services = Get-Service | - Select-Object -Property @{Name="name"; Expression={$_."DisplayName"}}, - @{Name="win_svc_name"; Expression={$_."Name"}}, - @{Name="state"; Expression={$_."Status".ToString().ToLower()}} - } - changed = $false -} - -Exit-Json $result; diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 5aa0b834ea..025e0c2c9a 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -203,6 +203,9 @@ JOB_EVENT_MAX_QUEUE_SIZE = 10000 # The number of job events to migrate per-transaction when moving from int -> bigint JOB_EVENT_MIGRATION_CHUNK_SIZE = 1000000 +# The maximum allowed jobs to start on a given task manager cycle +START_TASK_LIMIT = 100 + # Disallow sending session cookies over insecure connections SESSION_COOKIE_SECURE = True @@ -477,6 +480,7 @@ SOCIAL_AUTH_SAML_PIPELINE = _SOCIAL_AUTH_PIPELINE_BASE + ( 'awx.sso.pipeline.update_user_orgs', 'awx.sso.pipeline.update_user_teams', ) +SAML_AUTO_CREATE_OBJECTS = True SOCIAL_AUTH_LOGIN_URL = '/' SOCIAL_AUTH_LOGIN_REDIRECT_URL = '/sso/complete/' @@ -789,7 +793,7 @@ ASGI_APPLICATION = "awx.main.routing.application" CHANNEL_LAYERS = { "default": { - "BACKEND": "awx.main.consumers.ExpiringRedisChannelLayer", + "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { "hosts": [BROKER_URL], "capacity": 10000, diff --git a/awx/sso/conf.py b/awx/sso/conf.py index c408d72b40..3406787f6b 100644 --- a/awx/sso/conf.py +++ b/awx/sso/conf.py @@ -919,6 +919,17 @@ def get_saml_entity_id(): return settings.TOWER_URL_BASE +register( + 'SAML_AUTO_CREATE_OBJECTS', + field_class=fields.BooleanField, + default=True, + label=_('Automatically Create Organizations and Teams on SAML Login'), + help_text=_('When enabled (the default), mapped Organizations and Teams ' + 'will be created automatically on successful SAML login.'), + category=_('SAML'), + category_slug='saml', +) + register( 'SOCIAL_AUTH_SAML_CALLBACK_URL', field_class=fields.CharField, diff --git a/awx/sso/pipeline.py b/awx/sso/pipeline.py index 6d7e05da90..3e73974474 100644 --- a/awx/sso/pipeline.py +++ b/awx/sso/pipeline.py @@ -10,6 +10,7 @@ import logging from social_core.exceptions import AuthException # Django +from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import ugettext_lazy as _ from django.db.models import Q @@ -80,11 +81,18 @@ def _update_m2m_from_expression(user, related, expr, remove=True): def _update_org_from_attr(user, related, attr, remove, remove_admins, remove_auditors): from awx.main.models import Organization + from django.conf import settings org_ids = [] for org_name in attr: - org = Organization.objects.get_or_create(name=org_name)[0] + try: + if settings.SAML_AUTO_CREATE_OBJECTS: + org = Organization.objects.get_or_create(name=org_name)[0] + else: + org = Organization.objects.get(name=org_name) + except ObjectDoesNotExist: + continue org_ids.append(org.id) getattr(org, related).members.add(user) @@ -199,11 +207,24 @@ def update_user_teams_by_saml_attr(backend, details, user=None, *args, **kwargs) if organization_alias: organization_name = organization_alias - org = Organization.objects.get_or_create(name=organization_name)[0] + + try: + if settings.SAML_AUTO_CREATE_OBJECTS: + org = Organization.objects.get_or_create(name=organization_name)[0] + else: + org = Organization.objects.get(name=organization_name) + except ObjectDoesNotExist: + continue if team_alias: team_name = team_alias - team = Team.objects.get_or_create(name=team_name, organization=org)[0] + try: + if settings.SAML_AUTO_CREATE_OBJECTS: + team = Team.objects.get_or_create(name=team_name, organization=org)[0] + else: + team = Team.objects.get(name=team_name, organization=org) + except ObjectDoesNotExist: + continue team_ids.append(team.id) team.member_role.members.add(user) diff --git a/awx/sso/tests/functional/test_pipeline.py b/awx/sso/tests/functional/test_pipeline.py index 06d5503db8..e691939752 100644 --- a/awx/sso/tests/functional/test_pipeline.py +++ b/awx/sso/tests/functional/test_pipeline.py @@ -174,8 +174,15 @@ class TestSAMLAttr(): return (o1, o2, o3) @pytest.fixture - def mock_settings(self): + def mock_settings(self, request): + fixture_args = request.node.get_closest_marker('fixture_args') + if fixture_args and 'autocreate' in fixture_args.kwargs: + autocreate = fixture_args.kwargs['autocreate'] + else: + autocreate = True + class MockSettings(): + SAML_AUTO_CREATE_OBJECTS = autocreate SOCIAL_AUTH_SAML_ORGANIZATION_ATTR = { 'saml_attr': 'memberOf', 'saml_admin_attr': 'admins', @@ -304,3 +311,41 @@ class TestSAMLAttr(): assert Team.objects.get( name='Yellow_Alias', organization__name='Default4_Alias').member_role.members.count() == 1 + @pytest.mark.fixture_args(autocreate=False) + def test_autocreate_disabled(self, users, kwargs, mock_settings): + kwargs['response']['attributes']['memberOf'] = ['Default1', 'Default2', 'Default3'] + kwargs['response']['attributes']['groups'] = ['Blue', 'Red', 'Green'] + with mock.patch('django.conf.settings', mock_settings): + for u in users: + update_user_orgs_by_saml_attr(None, None, u, **kwargs) + update_user_teams_by_saml_attr(None, None, u, **kwargs) + assert Organization.objects.count() == 0 + assert Team.objects.count() == 0 + + # precreate everything + o1 = Organization.objects.create(name='Default1') + o2 = Organization.objects.create(name='Default2') + o3 = Organization.objects.create(name='Default3') + Team.objects.create(name='Blue', organization_id=o1.id) + Team.objects.create(name='Blue', organization_id=o2.id) + Team.objects.create(name='Blue', organization_id=o3.id) + Team.objects.create(name='Red', organization_id=o1.id) + Team.objects.create(name='Green', organization_id=o1.id) + Team.objects.create(name='Green', organization_id=o3.id) + + for u in users: + update_user_orgs_by_saml_attr(None, None, u, **kwargs) + update_user_teams_by_saml_attr(None, None, u, **kwargs) + + assert o1.member_role.members.count() == 3 + assert o2.member_role.members.count() == 3 + assert o3.member_role.members.count() == 3 + + assert Team.objects.get(name='Blue', organization__name='Default1').member_role.members.count() == 3 + assert Team.objects.get(name='Blue', organization__name='Default2').member_role.members.count() == 3 + assert Team.objects.get(name='Blue', organization__name='Default3').member_role.members.count() == 3 + + assert Team.objects.get(name='Red', organization__name='Default1').member_role.members.count() == 3 + + assert Team.objects.get(name='Green', organization__name='Default1').member_role.members.count() == 3 + assert Team.objects.get(name='Green', organization__name='Default3').member_role.members.count() == 3 diff --git a/awx/ui_next/src/api/models/JobTemplates.js b/awx/ui_next/src/api/models/JobTemplates.js index 4f631cec2a..23f2716dda 100644 --- a/awx/ui_next/src/api/models/JobTemplates.js +++ b/awx/ui_next/src/api/models/JobTemplates.js @@ -24,6 +24,10 @@ class JobTemplates extends SchedulesMixin( return this.http.post(`${this.baseUrl}${id}/launch/`, data); } + readTemplateOptions(id) { + return this.http.options(`${this.baseUrl}/${id}/`); + } + readLaunch(id) { return this.http.get(`${this.baseUrl}${id}/launch/`); } diff --git a/awx/ui_next/src/api/models/WorkflowJobTemplates.js b/awx/ui_next/src/api/models/WorkflowJobTemplates.js index 7c582cd57f..e52e050b1d 100644 --- a/awx/ui_next/src/api/models/WorkflowJobTemplates.js +++ b/awx/ui_next/src/api/models/WorkflowJobTemplates.js @@ -12,6 +12,10 @@ class WorkflowJobTemplates extends SchedulesMixin(NotificationsMixin(Base)) { return this.http.get(`${this.baseUrl}${id}/webhook_key/`); } + readWorkflowJobTemplateOptions(id) { + return this.http.options(`${this.baseUrl}/${id}/`); + } + updateWebhookKey(id) { return this.http.post(`${this.baseUrl}${id}/webhook_key/`); } diff --git a/awx/ui_next/src/components/CodeMirrorInput/CodeMirrorField.jsx b/awx/ui_next/src/components/CodeMirrorInput/CodeMirrorField.jsx new file mode 100644 index 0000000000..d08ab12674 --- /dev/null +++ b/awx/ui_next/src/components/CodeMirrorInput/CodeMirrorField.jsx @@ -0,0 +1,74 @@ +import React from 'react'; +import { + string, + oneOfType, + object, + func, + bool, + node, + oneOf, + number, +} from 'prop-types'; +import { useField } from 'formik'; +import { FormGroup } from '@patternfly/react-core'; +import CodeMirrorInput from './CodeMirrorInput'; +import { FieldTooltip } from '../FormField'; + +function CodeMirrorField({ + id, + name, + label, + tooltip, + helperText, + validate, + isRequired, + mode, + ...rest +}) { + const [field, meta, helpers] = useField({ name, validate }); + const isValid = !(meta.touched && meta.error); + + return ( + } + > + { + helpers.setValue(value); + }} + mode={mode} + /> + + ); +} +CodeMirrorField.propTypes = { + helperText: string, + id: string.isRequired, + name: string.isRequired, + label: oneOfType([object, string]).isRequired, + validate: func, + isRequired: bool, + tooltip: node, + mode: oneOf(['javascript', 'yaml', 'jinja2']).isRequired, + rows: number, +}; + +CodeMirrorField.defaultProps = { + helperText: '', + validate: () => {}, + isRequired: false, + tooltip: null, + rows: 5, +}; + +export default CodeMirrorField; diff --git a/awx/ui_next/src/components/CodeMirrorInput/CodeMirrorInput.jsx b/awx/ui_next/src/components/CodeMirrorInput/CodeMirrorInput.jsx index d945462987..92a9071332 100644 --- a/awx/ui_next/src/components/CodeMirrorInput/CodeMirrorInput.jsx +++ b/awx/ui_next/src/components/CodeMirrorInput/CodeMirrorInput.jsx @@ -107,6 +107,7 @@ CodeMirrorInput.propTypes = { hasErrors: bool, fullHeight: bool, rows: number, + className: string, }; CodeMirrorInput.defaultProps = { readOnly: false, @@ -114,6 +115,7 @@ CodeMirrorInput.defaultProps = { rows: 6, fullHeight: false, hasErrors: false, + className: '', }; export default CodeMirrorInput; diff --git a/awx/ui_next/src/components/CodeMirrorInput/index.js b/awx/ui_next/src/components/CodeMirrorInput/index.js index 9cad016228..2c60b806f5 100644 --- a/awx/ui_next/src/components/CodeMirrorInput/index.js +++ b/awx/ui_next/src/components/CodeMirrorInput/index.js @@ -1,6 +1,7 @@ import CodeMirrorInput from './CodeMirrorInput'; export default CodeMirrorInput; +export { default as CodeMirrorField } from './CodeMirrorField'; export { default as VariablesDetail } from './VariablesDetail'; export { default as VariablesInput } from './VariablesInput'; export { default as VariablesField } from './VariablesField'; diff --git a/awx/ui_next/src/components/FieldWithPrompt/FieldWithPrompt.jsx b/awx/ui_next/src/components/FieldWithPrompt/FieldWithPrompt.jsx index b0d27ccc6e..d505049cf0 100644 --- a/awx/ui_next/src/components/FieldWithPrompt/FieldWithPrompt.jsx +++ b/awx/ui_next/src/components/FieldWithPrompt/FieldWithPrompt.jsx @@ -23,6 +23,7 @@ function FieldWithPrompt({ promptId, promptName, tooltip, + isDisabled, }) { return (
@@ -39,6 +40,7 @@ function FieldWithPrompt({ {tooltip && }
} + > +