diff --git a/MANIFEST.in b/MANIFEST.in index ff4d8ccddb..3c687ce2da 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ recursive-include awx *.py +recursive-include awx *.po +recursive-include awx *.mo recursive-include awx/static * recursive-include awx/templates *.html recursive-include awx/api/templates *.md *.html diff --git a/awx/api/serializers.py b/awx/api/serializers.py index edd1b0d079..1aa929eaf4 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -42,7 +42,7 @@ from awx.main.fields import ImplicitRoleField from awx.main.utils import ( get_type_for_model, get_model_for_type, timestamp_apiformat, camelcase_to_underscore, getattrd, parse_yaml_or_json, - has_model_field_prefetched) + has_model_field_prefetched, extract_ansible_vars) from awx.main.utils.filters import SmartFilter from awx.main.validators import vars_validate_or_raise @@ -2749,6 +2749,14 @@ class AdHocCommandSerializer(UnifiedJobSerializer): ret['name'] = obj.module_name return ret + def validate_extra_vars(self, value): + redacted_extra_vars, removed_vars = extract_ansible_vars(value) + if removed_vars: + raise serializers.ValidationError(_( + "{} are prohibited from use in ad hoc commands." + ).format(", ".join(removed_vars))) + return vars_validate_or_raise(value) + class AdHocCommandCancelSerializer(AdHocCommandSerializer): diff --git a/awx/api/views.py b/awx/api/views.py index c0cb75ffe8..ade26dfacd 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -68,7 +68,7 @@ from awx.conf.license import get_license, feature_enabled, feature_exists, Licen from awx.main.models import * # noqa from awx.main.utils import * # noqa from awx.main.utils import ( - callback_filter_out_ansible_extra_vars, + extract_ansible_vars, decrypt_field, ) from awx.main.utils.filters import SmartFilter @@ -3112,7 +3112,8 @@ class JobTemplateCallback(GenericAPIView): # Everything is fine; actually create the job. kv = {"limit": limit, "launch_type": 'callback'} if extra_vars is not None and job_template.ask_variables_on_launch: - kv['extra_vars'] = callback_filter_out_ansible_extra_vars(extra_vars) + extra_vars_redacted, removed = extract_ansible_vars(extra_vars) + kv['extra_vars'] = extra_vars_redacted with transaction.atomic(): job = job_template.create_job(**kv) diff --git a/awx/main/expect/isolated_manager.py b/awx/main/expect/isolated_manager.py index 48fd0e7b5c..265531443f 100644 --- a/awx/main/expect/isolated_manager.py +++ b/awx/main/expect/isolated_manager.py @@ -9,6 +9,7 @@ import stat import tempfile import time import logging +from distutils.version import LooseVersion as Version from django.conf import settings @@ -370,7 +371,24 @@ class IsolatedManager(object): logger.warning('Isolated job {} cleanup error, output:\n{}'.format(self.instance.id, output)) @classmethod - def health_check(cls, instance_qs): + def update_capacity(cls, instance, task_result, awx_application_version): + instance.version = task_result['version'] + + isolated_version = instance.version.split("-", 1)[0] + cluster_version = awx_application_version.split("-", 1)[0] + + if Version(cluster_version) > Version(isolated_version): + err_template = "Isolated instance {} reports version {}, cluster node is at {}, setting capacity to zero." + logger.error(err_template.format(instance.hostname, instance.version, awx_application_version)) + instance.capacity = 0 + else: + if instance.capacity == 0 and task_result['capacity']: + logger.warning('Isolated instance {} has re-joined.'.format(instance.hostname)) + instance.capacity = int(task_result['capacity']) + instance.save(update_fields=['capacity', 'version', 'modified']) + + @classmethod + def health_check(cls, instance_qs, awx_application_version): ''' :param instance_qs: List of Django objects representing the isolated instances to manage @@ -412,11 +430,7 @@ class IsolatedManager(object): except (KeyError, IndexError): task_result = {} if 'capacity' in task_result: - instance.version = task_result['version'] - if instance.capacity == 0 and task_result['capacity']: - logger.warning('Isolated instance {} has re-joined.'.format(instance.hostname)) - instance.capacity = int(task_result['capacity']) - instance.save(update_fields=['capacity', 'version', 'modified']) + cls.update_capacity(instance, task_result, awx_application_version) elif instance.capacity == 0: logger.debug('Isolated instance {} previously marked as lost, could not re-join.'.format( instance.hostname)) diff --git a/awx/main/migrations/0005_squashed_v310_v313_updates.py b/awx/main/migrations/0005_squashed_v310_v313_updates.py index 24a7f2d207..85dac8bbaa 100644 --- a/awx/main/migrations/0005_squashed_v310_v313_updates.py +++ b/awx/main/migrations/0005_squashed_v310_v313_updates.py @@ -12,8 +12,6 @@ class Migration(migrations.Migration): replaces = [ (b'main', '0035_v310_remove_tower_settings'), - (b'main', '0036_v311_insights'), - (b'main', '0037_v313_instance_version'), ] operations = [ @@ -36,11 +34,4 @@ class Migration(migrations.Migration): name='scm_type', field=models.CharField(default=b'', choices=[(b'', 'Manual'), (b'git', 'Git'), (b'hg', 'Mercurial'), (b'svn', 'Subversion'), (b'insights', 'Red Hat Insights')], max_length=8, blank=True, help_text='Specifies the source control system used to store the project.', verbose_name='SCM Type'), ), - - migrations.AddField( - model_name='instance', - name='version', - field=models.CharField(max_length=24, blank=True), - ), - ] diff --git a/awx/main/migrations/0005a_squashed_v310_v313_updates.py b/awx/main/migrations/0005a_squashed_v310_v313_updates.py new file mode 100644 index 0000000000..6599268ce1 --- /dev/null +++ b/awx/main/migrations/0005a_squashed_v310_v313_updates.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0005_squashed_v310_v313_updates'), + ] + + replaces = [ + (b'main', '0036_v311_insights'), + ] + + operations = [ + migrations.AlterField( + model_name='project', + name='scm_type', + field=models.CharField(default=b'', choices=[(b'', 'Manual'), (b'git', 'Git'), (b'hg', 'Mercurial'), (b'svn', 'Subversion'), (b'insights', 'Red Hat Insights')], max_length=8, blank=True, help_text='Specifies the source control system used to store the project.', verbose_name='SCM Type'), + ), + migrations.AlterField( + model_name='projectupdate', + name='scm_type', + field=models.CharField(default=b'', choices=[(b'', 'Manual'), (b'git', 'Git'), (b'hg', 'Mercurial'), (b'svn', 'Subversion'), (b'insights', 'Red Hat Insights')], max_length=8, blank=True, help_text='Specifies the source control system used to store the project.', verbose_name='SCM Type'), + ), + ] \ No newline at end of file diff --git a/awx/main/migrations/0005b_squashed_v310_v313_updates.py b/awx/main/migrations/0005b_squashed_v310_v313_updates.py new file mode 100644 index 0000000000..ea9d26d2bc --- /dev/null +++ b/awx/main/migrations/0005b_squashed_v310_v313_updates.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0005a_squashed_v310_v313_updates'), + ] + + replaces = [ + (b'main', '0037_v313_instance_version'), + ] + + operations = [ + # Remove Tower settings, these settings are now in separate awx.conf app. + migrations.AddField( + model_name='instance', + name='version', + field=models.CharField(max_length=24, blank=True), + ), + ] diff --git a/awx/main/migrations/0006_v320_release.py b/awx/main/migrations/0006_v320_release.py index 45eabbebc5..4cb87f4fba 100644 --- a/awx/main/migrations/0006_v320_release.py +++ b/awx/main/migrations/0006_v320_release.py @@ -18,7 +18,7 @@ from awx.main.models import Host class Migration(migrations.Migration): dependencies = [ - ('main', '0005_squashed_v310_v313_updates'), + ('main', '0005b_squashed_v310_v313_updates'), ] operations = [ diff --git a/awx/main/migrations/_scan_jobs.py b/awx/main/migrations/_scan_jobs.py index 4dfc7cf972..ffeb8007e3 100644 --- a/awx/main/migrations/_scan_jobs.py +++ b/awx/main/migrations/_scan_jobs.py @@ -12,7 +12,7 @@ logger = logging.getLogger('awx.main.migrations') def _create_fact_scan_project(ContentType, Project, org): ct = ContentType.objects.get_for_model(Project) - name = "Tower Fact Scan - {}".format(org.name if org else "No Organization") + name = u"Tower Fact Scan - {}".format(org.name if org else "No Organization") proj = Project(name=name, scm_url='https://github.com/ansible/awx-facts-playbooks', scm_type='git', diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index e148d45862..4e4af4c8ef 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -34,7 +34,7 @@ from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin from awx.main.utils import ( decrypt_field, _inventory_updates, copy_model_by_class, copy_m2m_relationships, - get_type_for_model + get_type_for_model, parse_yaml_or_json ) from awx.main.redact import UriCleaner, REPLACE_STR from awx.main.consumers import emit_channel_notification @@ -878,21 +878,14 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique return [] def handle_extra_data(self, extra_data): - if hasattr(self, 'extra_vars'): - extra_vars = {} - if isinstance(extra_data, dict): - extra_vars = extra_data - elif extra_data is None: - return - else: - if extra_data == "": - return - try: - extra_vars = json.loads(extra_data) - except Exception as e: - logger.warn("Exception deserializing extra vars: " + str(e)) + if hasattr(self, 'extra_vars') and extra_data: + extra_data_dict = {} + try: + extra_data_dict = parse_yaml_or_json(extra_data, silent_failure=False) + except Exception as e: + logger.warn("Exception deserializing extra vars: " + str(e)) evars = self.extra_vars_dict - evars.update(extra_vars) + evars.update(extra_data_dict) self.update_fields(extra_vars=json.dumps(evars)) @property diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 5fb7920c4d..3f38961b6f 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -56,7 +56,7 @@ from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, check_proot_installed, build_proot_temp_dir, get_licenser, wrap_args_with_proot, get_system_task_capacity, OutputEventFilter, parse_yaml_or_json, ignore_inventory_computed_fields, ignore_inventory_group_removal, - get_type_for_model) + get_type_for_model, extract_ansible_vars) from awx.main.utils.reload import restart_local_services, stop_local_services from awx.main.utils.handlers import configure_external_logger from awx.main.consumers import emit_channel_notification @@ -283,7 +283,7 @@ def awx_isolated_heartbeat(self): # Slow pass looping over isolated IGs and their isolated instances if len(isolated_instance_qs) > 0: logger.debug("Managing isolated instances {}.".format(','.join([inst.hostname for inst in isolated_instance_qs]))) - isolated_manager.IsolatedManager.health_check(isolated_instance_qs) + isolated_manager.IsolatedManager.health_check(isolated_instance_qs, awx_application_version) @task(bind=True, queue='tower', base=LogErrorsTask) @@ -683,7 +683,14 @@ class BaseTask(LogErrorsTask): json_data = json.dumps(instance.inventory.get_script_data(hostvars=True)) f.write('#! /usr/bin/env python\n# -*- coding: utf-8 -*-\nprint """%s"""\n' % json_data) os.chmod(path, stat.S_IRUSR | stat.S_IXUSR) - return path + return path + else: + # work around an inventory caching bug in Ansible 2.4.0 + # see: https://github.com/ansible/ansible/pull/30817 + # see: https://github.com/ansible/awx/issues/246 + inventory_script = tempfile.mktemp(suffix='.awxrest.py', dir=kwargs['private_data_dir']) + shutil.copy(plugin, inventory_script) + return inventory_script def build_args(self, instance, **kwargs): raise NotImplementedError @@ -2108,6 +2115,12 @@ class RunAdHocCommand(BaseTask): args.append('-%s' % ('v' * min(5, ad_hoc_command.verbosity))) if ad_hoc_command.extra_vars_dict: + redacted_extra_vars, removed_vars = extract_ansible_vars(ad_hoc_command.extra_vars_dict) + if removed_vars: + raise ValueError(_( + "{} are prohibited from use in ad hoc commands." + ).format(", ".join(removed_vars))) + args.extend(['-e', json.dumps(ad_hoc_command.extra_vars_dict)]) args.extend(['-m', ad_hoc_command.module_name]) diff --git a/awx/main/tests/functional/test_scan_jobs_migration.py b/awx/main/tests/functional/test_scan_jobs_migration.py index f7bc08364a..8582bd10c6 100644 --- a/awx/main/tests/functional/test_scan_jobs_migration.py +++ b/awx/main/tests/functional/test_scan_jobs_migration.py @@ -19,46 +19,46 @@ from awx.main.migrations._scan_jobs import _migrate_scan_job_templates @pytest.fixture def organizations(): - return [Organization.objects.create(name="org-{}".format(x)) for x in range(3)] + return [Organization.objects.create(name=u"org-\xe9-{}".format(x)) for x in range(3)] @pytest.fixture def inventories(organizations): - return [Inventory.objects.create(name="inv-{}".format(x), + return [Inventory.objects.create(name=u"inv-\xe9-{}".format(x), organization=organizations[x]) for x in range(3)] @pytest.fixture def job_templates_scan(inventories): - return [JobTemplate.objects.create(name="jt-scan-{}".format(x), + return [JobTemplate.objects.create(name=u"jt-\xe9-scan-{}".format(x), job_type=PERM_INVENTORY_SCAN, inventory=inventories[x]) for x in range(3)] @pytest.fixture def job_templates_deploy(inventories): - return [JobTemplate.objects.create(name="jt-deploy-{}".format(x), + return [JobTemplate.objects.create(name=u"jt-\xe9-deploy-{}".format(x), job_type=PERM_INVENTORY_DEPLOY, inventory=inventories[x]) for x in range(3)] @pytest.fixture def project_custom(organizations): - return Project.objects.create(name="proj-scan_custom", + return Project.objects.create(name=u"proj-\xe9-scan_custom", scm_url='https://giggity.com', organization=organizations[0]) @pytest.fixture def job_templates_custom_scan_project(project_custom): - return [JobTemplate.objects.create(name="jt-scan-custom-{}".format(x), + return [JobTemplate.objects.create(name=u"jt-\xe9-scan-custom-{}".format(x), project=project_custom, job_type=PERM_INVENTORY_SCAN) for x in range(3)] @pytest.fixture def job_template_scan_no_org(): - return JobTemplate.objects.create(name="jt-scan-no-org", + return JobTemplate.objects.create(name=u"jt-\xe9-scan-no-org", job_type=PERM_INVENTORY_SCAN) diff --git a/awx/main/tests/functional/test_tasks.py b/awx/main/tests/functional/test_tasks.py index 24451abbdd..2cfdcae849 100644 --- a/awx/main/tests/functional/test_tasks.py +++ b/awx/main/tests/functional/test_tasks.py @@ -117,6 +117,28 @@ class TestIsolatedManagementTask: inst.save() return inst + @pytest.fixture + def old_version(self, control_group): + ig = InstanceGroup.objects.create(name='thepentagon', controller=control_group) + inst = ig.instances.create(hostname='isolated-old', capacity=103) + inst.save() + return inst + + def test_old_version(self, control_instance, old_version): + update_capacity = isolated_manager.IsolatedManager.update_capacity + + assert old_version.capacity == 103 + with mock.patch('awx.main.tasks.settings', MockSettings()): + # Isolated node is reporting an older version than the cluster + # instance that issued the health check, set capacity to zero. + update_capacity(old_version, {'version': '1.0.0'}, '3.0.0') + assert old_version.capacity == 0 + + # Upgrade was completed, health check playbook now reports matching + # version, make sure capacity is set. + update_capacity(old_version, {'version': '5.0.0-things', 'capacity':103}, '5.0.0-stuff') + assert old_version.capacity == 103 + def test_takes_action(self, control_instance, needs_updating): original_isolated_instance = needs_updating.instances.all().first() with mock.patch('awx.main.tasks.settings', MockSettings()): diff --git a/awx/main/tests/unit/models/test_survey_models.py b/awx/main/tests/unit/models/test_survey_models.py index 43494cef7f..ff04ac2b68 100644 --- a/awx/main/tests/unit/models/test_survey_models.py +++ b/awx/main/tests/unit/models/test_survey_models.py @@ -1,3 +1,5 @@ +import tempfile +import pytest import json import tempfile @@ -73,7 +75,7 @@ def test_job_safe_args_redacted_passwords(job): assert extra_vars['secret_key'] == '$encrypted$' -def test_job_args_unredacted_passwords(job): +def test_job_args_unredacted_passwords(job, tmpdir_factory): kwargs = {'ansible_version': '2.1', 'private_data_dir': tempfile.mkdtemp()} run_job = RunJob() args = run_job.build_args(job, **kwargs) diff --git a/awx/main/tests/unit/utils/test_common.py b/awx/main/tests/unit/utils/test_common.py index d31f2b7472..45ed62068b 100644 --- a/awx/main/tests/unit/utils/test_common.py +++ b/awx/main/tests/unit/utils/test_common.py @@ -155,3 +155,12 @@ def test_memoize_parameter_error(): with pytest.raises(common.IllegalArgumentError): fn() + +def test_extract_ansible_vars(): + my_dict = { + "foobar": "baz", + "ansible_connetion_setting": "1928" + } + redacted, var_list = common.extract_ansible_vars(json.dumps(my_dict)) + assert var_list == set(['ansible_connetion_setting']) + assert redacted == {"foobar": "baz"} diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 902621ca62..c37da71436 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -42,7 +42,7 @@ __all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore', 'ignore_inventory_computed_fields', 'ignore_inventory_group_removal', '_inventory_updates', 'get_pk_from_dict', 'getattrd', 'NoDefaultProvided', 'get_current_apps', 'set_current_apps', 'OutputEventFilter', - 'callback_filter_out_ansible_extra_vars', 'get_search_fields', 'get_system_task_capacity', + 'extract_ansible_vars', 'get_search_fields', 'get_system_task_capacity', 'wrap_args_with_proot', 'build_proot_temp_dir', 'check_proot_installed', 'model_to_dict', 'model_instance_diff', 'timestamp_apiformat', 'parse_yaml_or_json', 'RequireDebugTrueOrTest', 'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError',] @@ -904,13 +904,18 @@ class OutputEventFilter(object): self._current_event_data = None -def callback_filter_out_ansible_extra_vars(extra_vars): - extra_vars_redacted = {} +def is_ansible_variable(key): + return key.startswith('ansible_') + + +def extract_ansible_vars(extra_vars): extra_vars = parse_yaml_or_json(extra_vars) - for key, value in extra_vars.iteritems(): - if not key.startswith('ansible_'): - extra_vars_redacted[key] = value - return extra_vars_redacted + ansible_vars = set([]) + for key in extra_vars.keys(): + if is_ansible_variable(key): + extra_vars.pop(key) + ansible_vars.add(key) + return (extra_vars, ansible_vars) def get_search_fields(model): diff --git a/setup.py b/setup.py index aa530a6d10..51c992e5e0 100755 --- a/setup.py +++ b/setup.py @@ -155,9 +155,7 @@ setup( }, data_files = proc_data_files([ ("%s" % homedir, ["config/wsgi.py", - "awx/static/favicon.ico", - "awx/locale/*/LC_MESSAGES/*.po", - "awx/locale/*/LC_MESSAGES/*.mo"]), + "awx/static/favicon.ico"]), ("%s" % siteconfig, ["config/awx-nginx.conf"]), # ("%s" % webconfig, ["config/uwsgi_params"]), ("%s" % sharedir, ["tools/scripts/request_tower_configuration.sh","tools/scripts/request_tower_configuration.ps1"]),