Inventory plugins transition dev finishing work

Bump keystone auth to resolve problem with openstack script

Clarify code path, routing to template vs. managed injector
  behavior is also now reflected in test data files

Refactor test data layout for inventory injector logic

Add developer docs for inventory plugins transition

Memoize only get_ansible_version with no parameters

Make inventory plugin injector enablement a separate
  concept from the initial_version
  switch tests to look for plugin_name as well

Add plugin injectors for tower and foreman.

Add jinja2 native types compat feature

move tower source license compare logic to management command

introduce inventory source compat mode

pin jinja2 for native Ansible types

Add parent group keys, and additional translations

manual dash sanitization for un-region-like ec2 groups

nest zones under regions using Ansible core feature just merged
  implement conditionally only with BOTH group_by options

Make compat mode default be true
  in API models, UI add and edit controllers

Add several additional hostvars to translation
Add Azure tags null case translation

Make Azure group_by key off source_vars
  to be consistent with the script

support top-level ec2 boto_profile setting
This commit is contained in:
AlanCoding
2019-02-08 10:27:26 -05:00
parent bc5881ad21
commit cd7e358b73
64 changed files with 1248 additions and 476 deletions
@@ -1,4 +0,0 @@
plugin: azure_rm
regions:
- southcentralus
- westus
@@ -0,0 +1,8 @@
{
"AZURE_SUBSCRIPTION_ID": "fooo",
"AZURE_CLIENT_ID": "fooo",
"AZURE_TENANT": "fooo",
"AZURE_SECRET": "fooo",
"AZURE_CLOUD_ENVIRONMENT": "fooo",
"ANSIBLE_JINJA2_NATIVE": "True"
}
@@ -0,0 +1,35 @@
conditional_groups:
azure: true
default_host_filters: []
exclude_host_filters:
- resource_group not in ['foo_resources', 'bar_resources']
- location not in ['southcentralus', 'westus']
hostvar_expressions:
ansible_host: private_ipv4_addresses | json_query("[0]")
computer_name: name
private_ip: private_ipv4_addresses | json_query("[0]")
provisioning_state: provisioning_state | title
public_ip: public_ipv4_addresses | json_query("[0]")
tags: tags if tags else None
type: resource_type
keyed_groups:
- key: location
prefix: ''
separator: ''
- key: tags.keys() if tags else []
prefix: ''
separator: ''
- key: security_group
prefix: ''
separator: ''
- key: resource_group
prefix: ''
separator: ''
- key: os_disk.operating_system_type
prefix: ''
separator: ''
- key: dict(tags.keys() | map("regex_replace", "^(.*)$", "\1_") | list | zip(tags.values() | list)) if tags else []
prefix: ''
separator: ''
plugin: azure_rm
use_contrib_script_compatible_sanitization: true
@@ -1,4 +0,0 @@
plugin: aws_ec2
regions:
- us-east-2
- ap-south-1
@@ -0,0 +1,6 @@
{
"AWS_ACCESS_KEY_ID": "fooo",
"AWS_SECRET_ACCESS_KEY": "fooo",
"AWS_SECURITY_TOKEN": "fooo",
"ANSIBLE_JINJA2_NATIVE": "True"
}
@@ -0,0 +1,82 @@
boto_profile: /tmp/my_boto_stuff
compose:
ansible_host: public_ip_address
ec2_account_id: network_interfaces | json_query("[0].owner_id")
ec2_ami_launch_index: ami_launch_index | string
ec2_architecture: architecture
ec2_block_devices: dict(block_device_mappings | map(attribute='device_name') | list | zip(block_device_mappings | map(attribute='ebs.volume_id') | list))
ec2_client_token: client_token
ec2_dns_name: public_dns_name
ec2_ebs_optimized: ebs_optimized
ec2_eventsSet: events | default("")
ec2_group_name: placement.group_name
ec2_hypervisor: hypervisor
ec2_id: instance_id
ec2_image_id: image_id
ec2_instance_profile: iam_instance_profile | default("")
ec2_instance_type: instance_type
ec2_ip_address: public_ip_address
ec2_kernel: kernel_id | default("")
ec2_key_name: key_name
ec2_launch_time: launch_time | regex_replace(" ", "T") | regex_replace("(\+)(\d\d):(\d)(\d)$", ".\g<2>\g<3>Z")
ec2_monitored: monitoring.state in ['enabled', 'pending']
ec2_monitoring_state: monitoring.state
ec2_persistent: persistent | default(false)
ec2_placement: placement.availability_zone
ec2_platform: platform | default("")
ec2_private_dns_name: private_dns_name
ec2_private_ip_address: private_ip_address
ec2_public_dns_name: public_dns_name
ec2_ramdisk: ramdisk_id | default("")
ec2_reason: state_transition_reason
ec2_region: placement.region
ec2_requester_id: requester_id | default("")
ec2_root_device_name: root_device_name
ec2_root_device_type: root_device_type
ec2_security_group_ids: security_groups | map(attribute='group_id') | list | join(',')
ec2_security_group_names: security_groups | map(attribute='group_name') | list | join(',')
ec2_sourceDestCheck: source_dest_check | default(false) | lower | string
ec2_spot_instance_request_id: spot_instance_request_id | default("")
ec2_state: state.name
ec2_state_code: state.code
ec2_state_reason: state_reason.message if state_reason is defined else ""
ec2_subnet_id: subnet_id | default("")
ec2_tag_Name: tags.Name
ec2_virtualization_type: virtualization_type
ec2_vpc_id: vpc_id | default("")
filters:
instance-state-name:
- running
groups:
ec2: true
hostnames:
- network-interface.addresses.association.public-ip
- dns-name
- private-dns-name
keyed_groups:
- key: placement.availability_zone
parent_group: zones
prefix: ''
separator: ''
- key: instance_type | regex_replace("[^A-Za-z0-9\_]", "_")
parent_group: types
prefix: type
- key: placement.region
parent_group: regions
prefix: ''
separator: ''
- key: dict(tags.keys() | map("regex_replace", "[^A-Za-z0-9\_]", "_") | list | zip(tags.values() | map("regex_replace", "[^A-Za-z0-9\_]", "_") | list))
parent_group: tags
prefix: tag
- key: tags.keys() | map("regex_replace", "[^A-Za-z0-9\_]", "_") | list
parent_group: tags
prefix: tag
- key: placement.availability_zone
parent_group: '{{ placement.region }}'
prefix: ''
separator: ''
plugin: aws_ec2
regions:
- us-east-2
- ap-south-1
use_contrib_script_compatible_sanitization: true
@@ -0,0 +1,46 @@
auth_kind: serviceaccount
compose:
ansible_ssh_host: networkInterfaces | json_query("[0].accessConfigs[0].natIP")
gce_description: description if description else None
gce_id: id
gce_machine_type: machineType
gce_metadata: metadata.get("items", []) | items2dict(key_name="key", value_name="value")
gce_name: name
gce_network: networkInterfaces | json_query("[0].network.name")
gce_private_ip: networkInterfaces | json_query("[0].networkIP")
gce_public_ip: networkInterfaces | json_query("[0].accessConfigs[0].natIP")
gce_status: status
gce_subnetwork: networkInterfaces | json_query("[0].subnetwork.name")
gce_tags: tags | json_query("items")
gce_zone: zone
hostnames:
- name
- public_ip
- private_ip
keyed_groups:
- key: networkInterfaces | json_query("[0].subnetwork.name")
prefix: network
- key: networkInterfaces | json_query("[0].networkIP")
prefix: ''
separator: ''
- key: networkInterfaces | json_query("[0].accessConfigs[0].natIP")
prefix: ''
separator: ''
- key: machineType
prefix: ''
separator: ''
- key: zone
prefix: ''
separator: ''
- key: tags | json_query("items")
prefix: tag
- key: status | lower
prefix: status
plugin: gcp_compute
projects:
- fooo
service_account_file: {{ file_reference }}
use_contrib_script_compatible_sanitization: true
zones:
- us-east4-a
- us-west1-b
@@ -1,9 +0,0 @@
auth_kind: serviceaccount
filters: null
plugin: gcp_compute
projects:
- fooo
service_account_file: {{ file_reference }}
zones:
- us-east4-a
- us-west1-b
@@ -11,3 +11,4 @@ clouds:
project_name: fooo
username: fooo
private: false
verify: false
@@ -2,5 +2,5 @@ clouds_yaml_path:
- {{ file_reference }}
expand_hostvars: true
fail_on_errors: true
inventory_hostname: name
inventory_hostname: uuid
plugin: openstack
@@ -0,0 +1,5 @@
{
"FOREMAN_SERVER": "https://foo.invalid",
"FOREMAN_USER": "fooo",
"FOREMAN_PASSWORD": "fooo"
}
@@ -0,0 +1 @@
plugin: foreman
@@ -0,0 +1,6 @@
{
"TOWER_HOST": "https://foo.invalid",
"TOWER_USERNAME": "fooo",
"TOWER_PASSWORD": "fooo",
"TOWER_VERIFY_SSL": "False"
}
@@ -0,0 +1,3 @@
include_metadata: true
inventory_id: 42
plugin: tower
@@ -0,0 +1,8 @@
{
"AZURE_SUBSCRIPTION_ID": "fooo",
"AZURE_CLIENT_ID": "fooo",
"AZURE_TENANT": "fooo",
"AZURE_SECRET": "fooo",
"AZURE_CLOUD_ENVIRONMENT": "fooo",
"AZURE_INI_PATH": "{{ file_reference }}"
}
@@ -5,4 +5,6 @@ group_by_location = yes
group_by_tag = yes
locations = southcentralus,westus
base_source_var = value_of_var
use_private_ip = True
resource_groups = foo_resources,bar_resources
@@ -0,0 +1,3 @@
{
"CLOUDFORMS_INI_PATH": "{{ file_reference }}"
}
@@ -0,0 +1,6 @@
{
"AWS_ACCESS_KEY_ID": "fooo",
"AWS_SECRET_ACCESS_KEY": "fooo",
"AWS_SECURITY_TOKEN": "fooo",
"EC2_INI_PATH": "{{ file_reference }}"
}
@@ -1,5 +1,6 @@
[ec2]
base_source_var = value_of_var
boto_profile = /tmp/my_boto_stuff
regions = us-east-2,ap-south-1
regions_exclude = us-gov-west-1,cn-north-1
destination_variable = public_dns_name
@@ -14,16 +15,16 @@ elasticache = False
stack_filters = False
instance_filters = foobaa
group_by_ami_id = False
group_by_availability_zone = False
group_by_availability_zone = True
group_by_aws_account = False
group_by_instance_id = False
group_by_instance_state = False
group_by_platform = False
group_by_instance_type = False
group_by_instance_type = True
group_by_key_pair = False
group_by_region = False
group_by_region = True
group_by_security_group = False
group_by_tag_keys = False
group_by_tag_keys = True
group_by_tag_none = False
group_by_vpc_id = False
cache_path = {{ cache_dir }}
@@ -0,0 +1,7 @@
{
"GCE_EMAIL": "fooo",
"GCE_PROJECT": "fooo",
"GCE_CREDENTIALS_FILE_PATH": "{{ file_reference }}",
"GCE_ZONE": "us-east4-a,us-west1-b",
"GCE_INI_PATH": "{{ file_reference }}"
}
@@ -0,0 +1,3 @@
{
"OS_CLIENT_CONFIG_FILE": "{{ file_reference }}"
}
@@ -13,3 +13,4 @@ clouds:
project_name: fooo
username: fooo
private: false
verify: false
@@ -0,0 +1,6 @@
{
"OVIRT_INI_PATH": "{{ file_reference }}",
"OVIRT_URL": "https://foo.invalid",
"OVIRT_USERNAME": "fooo",
"OVIRT_PASSWORD": "fooo"
}
@@ -0,0 +1,3 @@
{
"FOREMAN_INI_PATH": "{{ file_reference }}"
}
@@ -0,0 +1,8 @@
{
"TOWER_HOST": "https://foo.invalid",
"TOWER_USERNAME": "fooo",
"TOWER_PASSWORD": "fooo",
"TOWER_VERIFY_SSL": "False",
"TOWER_INVENTORY": "42",
"TOWER_LICENSE_TYPE": "open"
}
@@ -0,0 +1,7 @@
{
"VMWARE_USER": "fooo",
"VMWARE_PASSWORD": "fooo",
"VMWARE_HOST": "https://foo.invalid",
"VMWARE_VALIDATE_CERTS": "False",
"VMWARE_INI_PATH": "{{ file_reference }}"
}
@@ -11,7 +11,8 @@ from django.core.management.base import CommandError
# AWX
from awx.main.management.commands import inventory_import
from awx.main.models import Inventory, Host, Group
from awx.main.models import Inventory, Host, Group, InventorySource
from awx.main.utils.mem_inventory import MemGroup
TEST_INVENTORY_CONTENT = {
@@ -306,3 +307,21 @@ class TestEnabledVar:
def test_enabled_var_is_enabled_value(self, cmd):
assert cmd._get_enabled({'foo': {'bar': 'barfoo'}}) is True
def test_tower_version_compare():
cmd = inventory_import.Command()
cmd.inventory_source = InventorySource(source='tower')
cmd.all_group = MemGroup('all')
# mimic example from https://github.com/ansible/ansible/pull/52747
# until that is merged, this is the best testing we can do
cmd.all_group.variables = {
'tower_metadata': {
"ansible_version": "2.7.5",
"license_type": "open",
"version": "2.0.1-1068-g09684e2c41"
}
}
with pytest.raises(CommandError):
cmd.remote_tower_license_compare('very_supported')
cmd.remote_tower_license_compare('open')
+2 -1
View File
@@ -464,8 +464,9 @@ def group(inventory):
@pytest.fixture
def inventory_source(inventory):
# by making it ec2, the credential is not required
return InventorySource.objects.create(name='single-inv-src',
inventory=inventory, source='gce')
inventory=inventory, source='ec2')
@pytest.fixture
@@ -2,6 +2,7 @@
import pytest
from unittest import mock
import json
from django.core.exceptions import ValidationError
@@ -13,6 +14,8 @@ from awx.main.models import (
InventoryUpdate,
Job
)
from awx.main.constants import CLOUD_PROVIDERS
from awx.main.models.inventory import PluginFileInjector
from awx.main.utils.filters import SmartFilter
@@ -206,6 +209,103 @@ class TestSCMClean:
inv_src2.clean_update_on_project_update()
@pytest.mark.django_db
class TestInventorySourceInjectors:
def test_should_use_plugin(self):
class foo(PluginFileInjector):
plugin_name = 'foo_compute'
initial_version = '2.7.8'
assert not foo('2.7.7').should_use_plugin()
assert foo('2.8').should_use_plugin()
def test_extra_credentials(self, project, credential):
inventory_source = InventorySource.objects.create(
name='foo', source='custom', source_project=project
)
inventory_source.credentials.add(credential)
assert inventory_source.get_cloud_credential() is None
assert inventory_source.get_extra_credentials() == [credential]
inventory_source.source = 'ec2'
assert inventory_source.get_cloud_credential() == credential
assert inventory_source.get_extra_credentials() == []
def test_all_cloud_sources_covered(self):
"""Code in several places relies on the fact that the older
CLOUD_PROVIDERS constant contains the same names as what are
defined within the injectors
"""
assert set(CLOUD_PROVIDERS) == set(InventorySource.injectors.keys())
@pytest.mark.parametrize('source,filename', [
('ec2', 'aws_ec2.yml'),
('openstack', 'openstack.yml'),
('gce', 'gcp_compute.yml')
])
def test_plugin_filenames(self, source, filename):
"""It is important that the filenames for inventory plugin files
are named correctly, because Ansible will reject files that do
not have these exact names
"""
injector = InventorySource.injectors[source]('2.7.7')
assert injector.filename == filename
@pytest.mark.parametrize('source,script_name', [
('ec2', 'ec2.py'),
('rhv', 'ovirt4.py'),
('satellite6', 'foreman.py'),
('openstack', 'openstack_inventory.py')
], ids=['ec2', 'rhv', 'satellite6', 'openstack'])
def test_script_filenames(self, source, script_name):
"""Ansible has several exceptions in naming of scripts
"""
injector = InventorySource.injectors[source]('2.7.7')
assert injector.script_name == script_name
def test_group_by_azure(self):
injector = InventorySource.injectors['azure_rm']('2.9')
inv_src = InventorySource(
name='azure source', source='azure_rm',
compatibility_mode=True,
source_vars={'group_by_os_family': True}
)
group_by_on = injector.inventory_as_dict(inv_src, '/tmp/foo')
# suspicious, yes, that is just what the script did
expected_groups = 6
assert len(group_by_on['keyed_groups']) == expected_groups
inv_src.source_vars = json.dumps({'group_by_os_family': False})
group_by_off = injector.inventory_as_dict(inv_src, '/tmp/foo')
# much better, everyone should turn off the flag and live in the future
assert len(group_by_off['keyed_groups']) == expected_groups - 1
@pytest.mark.parametrize('source', ['ec2', 'azure_rm'])
def test_default_groupings_same(self, source):
"""Just a sanity check, the number of groupings should be the same
with or without compatibility mode turned on.
This was a change made during feature development.
"""
injector = InventorySource.injectors[source]('2.9')
inv_src = InventorySource(
name='test source', source=source, compatibility_mode=True)
compat_on = injector.inventory_as_dict(inv_src, '/tmp/foo')
inv_src = InventorySource(
name='test source', source=source, compatibility_mode=False)
compat_off = injector.inventory_as_dict(inv_src, '/tmp/foo')
# Both default uses should give the same number of groups
assert len(compat_on['keyed_groups']) > 0
assert len(compat_on['keyed_groups']) == len(compat_off['keyed_groups'])
def test_tower_plugin_named_url(self):
injector = InventorySource.injectors['tower']('2.9')
inv_src = InventorySource(
name='my tower source', source='tower',
# named URL pattern "inventory++organization"
instance_filters='Designer hair 읰++Cosmetic_products䵆'
)
result = injector.inventory_as_dict(inv_src, '/tmp/foo')
assert result['inventory_id'] == 'Designer%20hair%20%EC%9D%B0++Cosmetic_products%E4%B5%86'
@pytest.fixture
def setup_ec2_gce(organization):
ec2_inv = Inventory.objects.create(name='test_ec2', organization=organization)
@@ -6,9 +6,11 @@ import re
from awx.main.tasks import RunInventoryUpdate
from awx.main.models import InventorySource, Credential, CredentialType, UnifiedJob
from awx.main.constants import CLOUD_PROVIDERS
from awx.main.constants import CLOUD_PROVIDERS, STANDARD_INVENTORY_UPDATE_ENV
from awx.main.tests import data
from django.conf import settings
DATA = os.path.join(os.path.dirname(data.__file__), 'inventory')
TEST_SOURCE_FIELDS = {
@@ -18,7 +20,8 @@ TEST_SOURCE_FIELDS = {
},
'ec2': {
'instance_filters': 'foobaa',
'group_by': 'fouo',
# group_by selected to capture some non-trivial cross-interactions
'group_by': 'availability_zone,instance_type,tag_keys,region',
'source_regions': 'us-east-2,ap-south-1'
},
'gce': {
@@ -27,10 +30,15 @@ TEST_SOURCE_FIELDS = {
'azure_rm': {
'source_regions': 'southcentralus,westus'
},
'tower': {
'instance_filters': '42'
}
}
INI_TEST_VARS = {
'ec2': {},
'ec2': {
'boto_profile': '/tmp/my_boto_stuff'
},
'gce': {},
'openstack': {
'private': False,
@@ -43,7 +51,10 @@ INI_TEST_VARS = {
'vmware': {
# setting VMWARE_VALIDATE_CERTS is duplicated with env var
},
'azure_rm': {}, # there are none
'azure_rm': {
'use_private_ip': True,
'resource_groups': 'foo_resources,bar_resources'
},
'satellite6': {
'satellite6_group_patterns': 'foo_group_patterns',
'satellite6_group_prefix': 'foo_group_prefix',
@@ -111,12 +122,23 @@ def fake_credential_factory(source):
)
def read_content(private_data_dir, env, inventory_update):
def read_content(private_data_dir, raw_env, inventory_update):
"""Read the environmental data laid down by the task system
template out private and secret data so they will be readable and predictable
return a dictionary `content` with file contents, keyed off environment variable
that references the file
"""
# Filter out environment variables which come from runtime environment
env = {}
exclude_keys = set(('PATH', 'INVENTORY_SOURCE_ID', 'INVENTORY_UPDATE_ID'))
for key in dir(settings):
if key.startswith('ANSIBLE_'):
exclude_keys.add(key)
for k, v in raw_env.items():
if k in STANDARD_INVENTORY_UPDATE_ENV or k in exclude_keys:
continue
if k not in os.environ or v != os.environ[k]:
env[k] = v
inverse_env = {}
for key, value in env.items():
inverse_env[value] = key
@@ -131,7 +153,9 @@ def read_content(private_data_dir, env, inventory_update):
for filename in os.listdir(private_data_dir):
abs_file_path = os.path.join(private_data_dir, filename)
if abs_file_path in inverse_env:
references[abs_file_path] = inverse_env[abs_file_path]
env_key = inverse_env[abs_file_path]
references[abs_file_path] = env_key
env[env_key] = '{{ file_reference }}'
try:
with open(abs_file_path, 'r') as f:
dir_contents[abs_file_path] = f.read()
@@ -181,21 +205,28 @@ def read_content(private_data_dir, env, inventory_update):
file_content = private_key_regex.sub('{{private_key}}', file_content)
content[reference_key] = file_content
return content
return (env, content)
def create_reference_data(ref_dir, content):
if not os.path.exists(ref_dir):
os.mkdir(ref_dir)
for env_name, content in content.items():
with open(os.path.join(ref_dir, env_name), 'w') as f:
f.write(content)
def create_reference_data(source_dir, env, content):
if not os.path.exists(source_dir):
os.mkdir(source_dir)
if content:
files_dir = os.path.join(source_dir, 'files')
if not os.path.exists(files_dir):
os.mkdir(files_dir)
for env_name, content in content.items():
with open(os.path.join(files_dir, env_name), 'w') as f:
f.write(content)
if env:
with open(os.path.join(source_dir, 'env.json'), 'w') as f:
f.write(json.dumps(env, indent=4))
@pytest.mark.django_db
@pytest.mark.parametrize('this_kind', CLOUD_PROVIDERS)
@pytest.mark.parametrize('script_or_plugin', ['scripts', 'plugins'])
def test_inventory_script_structure(this_kind, script_or_plugin, inventory):
def test_inventory_update_injected_content(this_kind, script_or_plugin, inventory):
src_vars = dict(base_source_var='value_of_var')
if this_kind in INI_TEST_VARS:
src_vars.update(INI_TEST_VARS[this_kind])
@@ -206,6 +237,7 @@ def test_inventory_script_structure(this_kind, script_or_plugin, inventory):
inventory=inventory,
source=this_kind,
source_vars=src_vars,
compatibility_mode=True,
**extra_kwargs
)
inventory_source.credentials.add(fake_credential_factory(this_kind))
@@ -213,44 +245,57 @@ def test_inventory_script_structure(this_kind, script_or_plugin, inventory):
task = RunInventoryUpdate()
use_plugin = bool(script_or_plugin == 'plugins')
if use_plugin:
if this_kind not in InventorySource.injectors:
pytest.skip('Injector class for this source is not written yet')
elif InventorySource.injectors[this_kind].initial_version is None:
pytest.skip('Use of inventory plugin is not enabled for this source')
if use_plugin and InventorySource.injectors[this_kind].plugin_name is None:
pytest.skip('Use of inventory plugin is not enabled for this source')
def substitute_run(args, cwd, env, stdout_handle, **_kw):
def substitute_run(args, cwd, call_env, stdout_handle, **_kw):
"""This method will replace run_pexpect
instead of running, it will read the private data directory contents
It will make assertions that the contents are correct
If MAKE_INVENTORY_REFERENCE_FILES is set, it will produce reference files
"""
private_data_dir = env['AWX_PRIVATE_DATA_DIR']
private_data_dir = call_env.pop('AWX_PRIVATE_DATA_DIR')
assert call_env.pop('ANSIBLE_INVENTORY_ENABLED') == ('auto' if use_plugin else 'script')
assert call_env.pop('ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS') == 'never'
set_files = bool(os.getenv("MAKE_INVENTORY_REFERENCE_FILES", 'false').lower()[0] not in ['f', '0'])
content = read_content(private_data_dir, env, inventory_update)
env, content = read_content(private_data_dir, call_env, inventory_update)
base_dir = os.path.join(DATA, script_or_plugin)
if not os.path.exists(base_dir):
os.mkdir(base_dir)
ref_dir = os.path.join(base_dir, this_kind) # this_kind is a global
source_dir = os.path.join(base_dir, this_kind) # this_kind is a global
if set_files:
create_reference_data(ref_dir, content)
create_reference_data(source_dir, env, content)
pytest.skip('You set MAKE_INVENTORY_REFERENCE_FILES, so this created files, unset to run actual test.')
else:
try:
expected_file_list = os.listdir(ref_dir)
except FileNotFoundError as e:
if not os.path.exists(source_dir):
raise FileNotFoundError(
'Maybe you never made reference files? '
'MAKE_INVENTORY_REFERENCE_FILES=true py.test ...\noriginal: {}'.format(e))
'MAKE_INVENTORY_REFERENCE_FILES=true py.test ...\noriginal: {}')
files_dir = os.path.join(source_dir, 'files')
try:
expected_file_list = os.listdir(files_dir)
except FileNotFoundError:
expected_file_list = []
assert set(expected_file_list) == set(content.keys()), (
'Inventory update runtime environment does not have expected files'
)
for f_name in expected_file_list:
with open(os.path.join(ref_dir, f_name), 'r') as f:
with open(os.path.join(files_dir, f_name), 'r') as f:
ref_content = f.read()
assert content[f_name] == ref_content
assert ref_content == content[f_name]
try:
with open(os.path.join(source_dir, 'env.json'), 'r') as f:
ref_env_text = f.read()
ref_env = json.loads(ref_env_text)
except FileNotFoundError:
ref_env = {}
assert ref_env == env
return ('successful', 0)
mock_licenser = mock.Mock(return_value=mock.Mock(
validate=mock.Mock(return_value={'license_type': 'open'})
))
# Mock this so that it will not send events to the callback receiver
# because doing so in pytest land creates large explosions
with mock.patch('awx.main.queue.CallbackQueueDispatcher.dispatch', lambda self, obj: None):
@@ -260,5 +305,7 @@ def test_inventory_script_structure(this_kind, script_or_plugin, inventory):
with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()):
# The point of this test is that we replace run_pexpect with assertions
with mock.patch('awx.main.expect.run.run_pexpect', substitute_run):
# so this sets up everything for a run and then yields control over to substitute_run
task.run(inventory_update.pk)
# mocking the licenser is necessary for the tower source
with mock.patch('awx.main.models.inventory.get_licenser', mock_licenser):
# so this sets up everything for a run and then yields control over to substitute_run
task.run(inventory_update.pk)
+32 -17
View File
@@ -167,7 +167,8 @@ def test_openstack_client_config_generation(mocker, source, expected, private_da
inventory_update = mocker.Mock(**{
'source': 'openstack',
'source_vars_dict': {},
'get_cloud_credential': cred_method
'get_cloud_credential': cred_method,
'get_extra_credentials': lambda x: []
})
cloud_config = update.build_private_data(inventory_update, private_data_dir)
cloud_credential = yaml.load(
@@ -208,7 +209,8 @@ def test_openstack_client_config_generation_with_private_source_vars(mocker, sou
inventory_update = mocker.Mock(**{
'source': 'openstack',
'source_vars_dict': {'private': source},
'get_cloud_credential': cred_method
'get_cloud_credential': cred_method,
'get_extra_credentials': lambda x: []
})
cloud_config = update.build_private_data(inventory_update, private_data_dir)
cloud_credential = yaml.load(
@@ -1759,6 +1761,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
task = tasks.RunInventoryUpdate()
inventory_update.source = 'ec2'
inventory_update.get_cloud_credential = mocker.Mock(return_value=None)
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
private_data_files = task.build_private_data_files(inventory_update, private_data_dir)
env = task.build_env(inventory_update, private_data_dir, False, private_data_files)
@@ -1781,7 +1784,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
if with_credential:
azure_rm = CredentialType.defaults['azure_rm']()
def get_cred():
def get_creds():
cred = Credential(
pk=1,
credential_type=azure_rm,
@@ -1792,10 +1795,11 @@ class TestInventoryUpdateCredentials(TestJobExecution):
'subscription': 'some-subscription',
}
)
return cred
inventory_update.get_cloud_credential = get_cred
return [cred]
inventory_update.get_extra_credentials = get_creds
else:
inventory_update.get_cloud_credential = mocker.Mock(return_value=None)
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
inventory_update.get_cloud_credential = mocker.Mock(return_value=None)
env = task.build_env(inventory_update, private_data_dir, False)
args = task.build_args(inventory_update, private_data_dir, {})
@@ -1818,7 +1822,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
assert env['AZURE_TENANT'] == 'some-tenant'
assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription'
def test_ec2_source(self, private_data_dir, inventory_update):
def test_ec2_source(self, private_data_dir, inventory_update, mocker):
task = tasks.RunInventoryUpdate()
aws = CredentialType.defaults['aws']()
inventory_update.source = 'ec2'
@@ -1832,6 +1836,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
cred.inputs['password'] = encrypt_field(cred, 'password')
return cred
inventory_update.get_cloud_credential = get_cred
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
private_data_files = task.build_private_data_files(inventory_update, private_data_dir)
env = task.build_env(inventory_update, private_data_dir, False, private_data_files)
@@ -1854,7 +1859,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
assert safe_env['AWS_SECRET_ACCESS_KEY'] == tasks.HIDDEN_PASSWORD
def test_vmware_source(self, inventory_update, private_data_dir):
def test_vmware_source(self, inventory_update, private_data_dir, mocker):
task = tasks.RunInventoryUpdate()
vmware = CredentialType.defaults['vmware']()
inventory_update.source = 'vmware'
@@ -1868,6 +1873,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
cred.inputs['password'] = encrypt_field(cred, 'password')
return cred
inventory_update.get_cloud_credential = get_cred
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
private_data_files = task.build_private_data_files(inventory_update, private_data_dir)
env = task.build_env(inventory_update, private_data_dir, False, private_data_files)
@@ -1886,7 +1892,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
assert config.get('vmware', 'password') == 'secret'
assert config.get('vmware', 'server') == 'https://example.org'
def test_azure_rm_source_with_tenant(self, private_data_dir, inventory_update):
def test_azure_rm_source_with_tenant(self, private_data_dir, inventory_update, mocker):
task = tasks.RunInventoryUpdate()
azure_rm = CredentialType.defaults['azure_rm']()
inventory_update.source = 'azure_rm'
@@ -1906,6 +1912,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
)
return cred
inventory_update.get_cloud_credential = get_cred
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
inventory_update.source_vars = {
'include_powerstate': 'yes',
'group_by_resource_group': 'no'
@@ -1939,7 +1946,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
assert safe_env['AZURE_SECRET'] == tasks.HIDDEN_PASSWORD
def test_azure_rm_source_with_password(self, private_data_dir, inventory_update):
def test_azure_rm_source_with_password(self, private_data_dir, inventory_update, mocker):
task = tasks.RunInventoryUpdate()
azure_rm = CredentialType.defaults['azure_rm']()
inventory_update.source = 'azure_rm'
@@ -1958,6 +1965,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
)
return cred
inventory_update.get_cloud_credential = get_cred
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
inventory_update.source_vars = {
'include_powerstate': 'yes',
'group_by_resource_group': 'no',
@@ -1990,7 +1998,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
assert 'locations' not in config.items('azure')
assert safe_env['AZURE_PASSWORD'] == tasks.HIDDEN_PASSWORD
def test_gce_source(self, inventory_update, private_data_dir):
def test_gce_source(self, inventory_update, private_data_dir, mocker):
task = tasks.RunInventoryUpdate()
gce = CredentialType.defaults['gce']()
inventory_update.source = 'gce'
@@ -2011,6 +2019,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
)
return cred
inventory_update.get_cloud_credential = get_cred
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
def run(expected_gce_zone):
private_data_files = task.build_private_data_files(inventory_update, private_data_dir)
@@ -2042,7 +2051,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
self.instance.source_regions = 'us-east-4'
run('us-east-4')
def test_openstack_source(self, inventory_update, private_data_dir):
def test_openstack_source(self, inventory_update, private_data_dir, mocker):
task = tasks.RunInventoryUpdate()
openstack = CredentialType.defaults['openstack']()
inventory_update.source = 'openstack'
@@ -2064,6 +2073,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
)
return cred
inventory_update.get_cloud_credential = get_cred
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
private_data_files = task.build_private_data_files(inventory_update, private_data_dir)
env = task.build_env(inventory_update, private_data_dir, False, private_data_files)
@@ -2080,7 +2090,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
''
]) in shade_config
def test_satellite6_source(self, inventory_update, private_data_dir):
def test_satellite6_source(self, inventory_update, private_data_dir, mocker):
task = tasks.RunInventoryUpdate()
satellite6 = CredentialType.defaults['satellite6']()
inventory_update.source = 'satellite6'
@@ -2100,6 +2110,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
)
return cred
inventory_update.get_cloud_credential = get_cred
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
inventory_update.source_vars = '{"satellite6_group_patterns": "[a,b,c]", "satellite6_group_prefix": "hey_", "satellite6_want_hostcollections": True}'
@@ -2115,7 +2126,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
assert config.get('ansible', 'group_prefix') == 'hey_'
assert config.get('ansible', 'want_hostcollections') == 'True'
def test_cloudforms_source(self, inventory_update, private_data_dir):
def test_cloudforms_source(self, inventory_update, private_data_dir, mocker):
task = tasks.RunInventoryUpdate()
cloudforms = CredentialType.defaults['cloudforms']()
inventory_update.source = 'cloudforms'
@@ -2135,6 +2146,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
)
return cred
inventory_update.get_cloud_credential = get_cred
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
inventory_update.source_vars = '{"prefer_ipv4": True}'
@@ -2154,7 +2166,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
assert os.path.isdir(cache_path)
@pytest.mark.parametrize('verify', [True, False])
def test_tower_source(self, verify, inventory_update, private_data_dir):
def test_tower_source(self, verify, inventory_update, private_data_dir, mocker):
task = tasks.RunInventoryUpdate()
tower = CredentialType.defaults['tower']()
inventory_update.source = 'tower'
@@ -2171,6 +2183,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
cred.inputs['password'] = encrypt_field(cred, 'password')
return cred
inventory_update.get_cloud_credential = get_cred
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
env = task.build_env(inventory_update, private_data_dir, False)
@@ -2192,7 +2205,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
assert env['TOWER_VERIFY_SSL'] == 'False'
assert safe_env['TOWER_PASSWORD'] == tasks.HIDDEN_PASSWORD
def test_tower_source_ssl_verify_empty(self, inventory_update, private_data_dir):
def test_tower_source_ssl_verify_empty(self, inventory_update, private_data_dir, mocker):
task = tasks.RunInventoryUpdate()
tower = CredentialType.defaults['tower']()
inventory_update.source = 'tower'
@@ -2208,6 +2221,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
cred.inputs['password'] = encrypt_field(cred, 'password')
return cred
inventory_update.get_cloud_credential = get_cred
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
env = task.build_env(inventory_update, private_data_dir, False)
safe_env = {}
@@ -2220,7 +2234,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
assert env['TOWER_VERIFY_SSL'] == 'False'
def test_awx_task_env(self, inventory_update, private_data_dir, settings):
def test_awx_task_env(self, inventory_update, private_data_dir, settings, mocker):
task = tasks.RunInventoryUpdate()
gce = CredentialType.defaults['gce']()
inventory_update.source = 'gce'
@@ -2236,6 +2250,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
)
return cred
inventory_update.get_cloud_credential = get_cred
inventory_update.get_extra_credentials = mocker.Mock(return_value=[])
settings.AWX_TASK_ENV = {'FOO': 'BAR'}
env = task.build_env(inventory_update, private_data_dir, False)
@@ -1,6 +1,5 @@
import os
import os.path
import json
import pytest
@@ -32,10 +31,3 @@ def test_could_be_inventory(filename):
def test_is_not_inventory(filename):
path = os.path.join(DATA, 'inventories', 'invalid')
assert could_be_inventory(DATA, path, filename) is None
def test_filter_non_json_lines():
data = {'foo': 'bar', 'bar': 'foo'}
dumped_data = json.dumps(data, indent=2)
output = 'Openstack does this\nOh why oh why\n{}\ntrailing lines\nneed testing too'.format(dumped_data)
assert filter_non_json_lines(output) == dumped_data