zware/awx
1
0
Fork 0
mirror of https://github.com/ZwareBear/awx.git synced 2026-05-05 16:01:50 -05:00
Code Issues Packages Projects Releases Wiki Activity

AC-1235 Improvements to inventory import and computed field performance.

Browse Source
This commit is contained in:
Chris Church
2014-05-18 22:10:06 -04:00
parent 0fcfe6114a
commit b14aa0b55d
5 changed files with 717 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
awx/main/tests/commands.py
View File

@@ -445,7 +445,7 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest):
contact_name='AWX Admin',
contact_email='awx@example.com',
license_date=int(time.time() + 3600),
instance_count=500,
instance_count=10000,
)
handle, license_path = tempfile.mkstemp(suffix='.json')
os.close(handle)
@@ -565,7 +565,7 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest):
result, stdout, stderr = self.run_command('inventory_import',
inventory_id=new_inv.pk,
source=inv_src)
self.assertEqual(result, None)
self.assertEqual(result, None, stdout + stderr)
# Check that inventory is populated as expected.
new_inv = Inventory.objects.get(pk=new_inv.pk)
expected_group_names = set(['servers', 'dbservers', 'webservers'])
@@ -637,7 +637,7 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest):
source=self.ini_path,
overwrite=overwrite,
overwrite_vars=overwrite_vars)
self.assertEqual(result, None)
self.assertEqual(result, None, stdout + stderr)
# Check that inventory is populated as expected.
new_inv = Inventory.objects.get(pk=new_inv.pk)
expected_group_names = set(['servers', 'dbservers', 'webservers',
@@ -828,7 +828,7 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest):
result, stdout, stderr = self.run_command('inventory_import',
inventory_id=new_inv.pk,
source=source)
self.assertEqual(result, None)
self.assertEqual(result, None, stdout + stderr)
# Check that inventory is populated as expected.
new_inv = Inventory.objects.get(pk=new_inv.pk)
self.assertEqual(old_inv.variables_dict, new_inv.variables_dict)
@@ -860,14 +860,13 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest):
new_inv = self.organizations[0].inventories.create(name='newec2')
self.assertEqual(new_inv.hosts.count(), 0)
self.assertEqual(new_inv.groups.count(), 0)
#inv_file = os.path.join(os.path.dirname(__file__), 'data',
# 'large_ec2_inventory.py')
os.chdir(os.path.join(os.path.dirname(__file__), 'data'))
inv_file = 'large_ec2_inventory.py'
settings.DEBUG = True
result, stdout, stderr = self.run_command('inventory_import',
inventory_id=new_inv.pk,
source=inv_file)
self.assertEqual(result, None, stdout+stderr)
self.assertEqual(result, None, stdout + stderr)
# Check that inventory is populated as expected within a reasonable
# amount of time. Computed fields should also be updated.
new_inv = Inventory.objects.get(pk=new_inv.pk)
@@ -875,5 +874,45 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest):
self.assertNotEqual(new_inv.groups.count(), 0)
self.assertNotEqual(new_inv.total_hosts, 0)
self.assertNotEqual(new_inv.total_groups, 0)
self.assertElapsedLessThan(60)
self.assertElapsedLessThan(30)
def _get_ngroups_for_nhosts(self, n):
if n > 0:
return min(n, 10) + ((n - 1) / 10 + 1) + ((n - 1) / 100 + 1) + ((n - 1) / 1000 + 1)
else:
return 0
def _check_largeinv_import(self, new_inv, nhosts, nhosts_inactive=0):
self._start_time = time.time()
inv_file = os.path.join(os.path.dirname(__file__), 'data', 'largeinv.py')
ngroups = self._get_ngroups_for_nhosts(nhosts)
os.environ['NHOSTS'] = str(nhosts)
result, stdout, stderr = self.run_command('inventory_import',
inventory_id=new_inv.pk,
source=inv_file,
overwrite=True, verbosity=0)
self.assertEqual(result, None, stdout + stderr)
# Check that inventory is populated as expected within a reasonable
# amount of time. Computed fields should also be updated.
new_inv = Inventory.objects.get(pk=new_inv.pk)
self.assertEqual(new_inv.hosts.filter(active=True).count(), nhosts)
self.assertEqual(new_inv.groups.filter(active=True).count(), ngroups)
self.assertEqual(new_inv.hosts.filter(active=False).count(), nhosts_inactive)
self.assertEqual(new_inv.total_hosts, nhosts)
self.assertEqual(new_inv.total_groups, ngroups)
self.assertElapsedLessThan(30)
def test_large_inventory_file(self):
new_inv = self.organizations[0].inventories.create(name='largeinv')
self.assertEqual(new_inv.hosts.count(), 0)
self.assertEqual(new_inv.groups.count(), 0)
settings.DEBUG = True
nhosts = 2000
# Test initial import into empty inventory.
self._check_largeinv_import(new_inv, nhosts, 0)
# Test re-importing and overwriting.
self._check_largeinv_import(new_inv, nhosts, 0)
# Test re-importing with only half as many hosts.
self._check_largeinv_import(new_inv, nhosts / 2, nhosts / 2)
# Test re-importing that clears all hosts.
self._check_largeinv_import(new_inv, 0, nhosts)

63
awx/main/tests/data/largeinv.py Executable file
View File

@@ -0,0 +1,63 @@
#!/usr/bin/env python
# Python
import json
import optparse
import os
nhosts = int(os.environ.get('NHOSTS', 100))
inv_list = {
'_meta': {
'hostvars': {},
},
}
for n in xrange(nhosts):
hostname = 'host-%08d.example.com' % n
group_evens_odds = 'evens.example.com' if n % 2 == 0 else 'odds.example.com'
group_threes = 'threes.example.com' if n % 3 == 0 else ''
group_fours = 'fours.example.com' if n % 4 == 0 else ''
group_fives = 'fives.example.com' if n % 5 == 0 else ''
group_sixes = 'sixes.example.com' if n % 6 == 0 else ''
group_sevens = 'sevens.example.com' if n % 7 == 0 else ''
group_eights = 'eights.example.com' if n % 8 == 0 else ''
group_nines = 'nines.example.com' if n % 9 == 0 else ''
group_tens = 'tens.example.com' if n % 10 == 0 else ''
group_by_10s = 'group-%07dX.example.com' % (n / 10)
group_by_100s = 'group-%06dXX.example.com' % (n / 100)
group_by_1000s = 'group-%05dXXX.example.com' % (n / 1000)
for group in [group_evens_odds, group_threes, group_fours, group_fives, group_sixes, group_sevens, group_eights, group_nines, group_tens, group_by_10s]:
if not group:
continue
if group in inv_list:
inv_list[group]['hosts'].append(hostname)
else:
inv_list[group] = {'hosts': [hostname], 'children': [], 'vars': {'group_prefix': group.split('.')[0]}}
if group_by_1000s not in inv_list:
inv_list[group_by_1000s] = {'hosts': [], 'children': [], 'vars': {'group_prefix': group_by_1000s.split('.')[0]}}
if group_by_100s not in inv_list:
inv_list[group_by_100s] = {'hosts': [], 'children': [], 'vars': {'group_prefix': group_by_100s.split('.')[0]}}
if group_by_100s not in inv_list[group_by_1000s]['children']:
inv_list[group_by_1000s]['children'].append(group_by_100s)
if group_by_10s not in inv_list[group_by_100s]['children']:
inv_list[group_by_100s]['children'].append(group_by_10s)
inv_list['_meta']['hostvars'][hostname] = {
'ansible_ssh_user': 'example',
'ansible_connection': 'local',
'host_prefix': hostname.split('.')[0],
'host_id': n,
}
if __name__ == '__main__':
parser = optparse.OptionParser()
parser.add_option('--list', action='store_true', dest='list')
parser.add_option('--host', dest='hostname', default='')
options, args = parser.parse_args()
if options.list:
print json.dumps(inv_list, indent=4)
elif options.hostname:
print json.dumps(inv_list['_meta']['hostvars'][options.hostname], indent=4)
else:
print json.dumps({}, indent=4)

2
awx/main/tests/inventory.py
View File

@@ -614,7 +614,7 @@ class InventoryTest(BaseTest):
# data used for testing listing all hosts that are transitive members of a group
g2 = Group.objects.get(name='web4')
nh = Host.objects.create(name='newhost.example.com', inventory=inva,
nh = Host.objects.create(name='newhost.example.com', inventory=g2.inventory,
created_by=self.super_django_user)
g2.hosts.add(nh)
g2.save()