zware/awx
1
0
Fork 0
mirror of https://github.com/ZwareBear/awx.git synced 2026-04-29 11:21:49 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'devel' of github.com:ansible/ansible-tower into rbac

Browse Source
This commit is contained in:
Akita Noek
2016-03-04 14:02:01 -05:00
parent db6117a56d e5b0e2aeb3
commit e9c3d98a44
82 changed files with 21118 additions and 1959 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
awx/main/management/commands/cleanup_facts.py
View File

@@ -12,7 +12,7 @@ from django.db import transaction
from django.utils.timezone import now
# AWX
from awx.fact.models.fact import * # noqa
from awx.main.models.fact import Fact
from awx.api.license import feature_enabled
OLDER_THAN = 'older_than'
@@ -31,7 +31,7 @@ class CleanupFacts(object):
# pivot -= granularity
# group by host
def cleanup(self, older_than_abs, granularity, module=None):
fact_oldest = FactVersion.objects.all().order_by('timestamp').first()
fact_oldest = Fact.objects.all().order_by('timestamp').first()
if not fact_oldest:
return 0
@@ -44,7 +44,10 @@ class CleanupFacts(object):
# Special case, granularity=0x where x is d, w, or y
# The intent is to delete all facts < older_than_abs
if granularity == relativedelta():
return FactVersion.objects.filter(**kv).order_by('-timestamp').delete()
qs = Fact.objects.filter(**kv)
count = qs.count()
qs.delete()
return count
total = 0
@@ -61,18 +64,17 @@ class CleanupFacts(object):
kv['module'] = module
fact_version_objs = FactVersion.objects.filter(**kv).order_by('-timestamp').limit(1)
if fact_version_objs:
fact_version_obj = fact_version_objs[0]
fact_version_obj = Fact.objects.filter(**kv).order_by('-timestamp').first()
if fact_version_obj:
kv = {
'timestamp__lt': fact_version_obj.timestamp,
'timestamp__gt': date_pivot_next
}
if module:
kv['module'] = module
count = FactVersion.objects.filter(**kv).delete()
# FIXME: These two deletes should be a transaction
count = Fact.objects.filter(**kv).delete()
qs = Fact.objects.filter(**kv)
count = qs.count()
qs.delete()
total += count
date_pivot = date_pivot_next

2
awx/main/management/commands/run_fact_cache_receiver.py
View File

@@ -67,7 +67,7 @@ class FactCacheReceiver(object):
self.timestamp = datetime.fromtimestamp(date_key, None)
# Update existing Fact entry
fact_obj = Fact.get_host_fact(host_obj.id, module_name, self.timestamp)
fact_obj = Fact.objects.filter(host__id=host_obj.id, module=module_name, timestamp=self.timestamp)
if fact_obj:
fact_obj.facts = facts
fact_obj.save()

6
awx/main/tasks.py
View File

@@ -53,7 +53,6 @@ from awx.main.task_engine import TaskSerializer, TASK_TIMEOUT_INTERVAL
from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, update_scm_url,
ignore_inventory_computed_fields, emit_websocket_notification,
check_proot_installed, build_proot_temp_dir, wrap_args_with_proot)
from awx.fact.utils.connection import test_mongo_connection
__all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate',
'RunAdHocCommand', 'handle_work_error', 'handle_work_success',
@@ -959,11 +958,6 @@ class RunJob(BaseTask):
'''
return getattr(tower_settings, 'AWX_PROOT_ENABLED', False)
def pre_run_hook(self, job, **kwargs):
if job.job_type == PERM_INVENTORY_SCAN:
if not test_mongo_connection():
raise RuntimeError("Fact Scan Database is offline")
def post_run_hook(self, job, **kwargs):
'''
Hook for actions to run after job/task has completed.

51
awx/main/tests/functional/api/test_fact_versions.py
View File

@@ -16,6 +16,9 @@ from django.utils import timezone
def mock_feature_enabled(feature, bypass_database=None):
return True
def mock_feature_disabled(feature, bypass_database=None):
return False
def setup_common(hosts, fact_scans, get, user, epoch=timezone.now(), get_params={}, host_count=1):
hosts = hosts(host_count=host_count)
fact_scans(fact_scans=3, timestamp_epoch=epoch)
@@ -42,8 +45,33 @@ def check_response_facts(facts_known, response):
assert timestamp_apiformat(fact_known.timestamp) == response.data['results'][i]['timestamp']
check_url(response.data['results'][i]['related']['fact_view'], fact_known, fact_known.module)
def check_system_tracking_feature_forbidden(response):
assert 402 == response.status_code
assert 'Your license does not permit use of system tracking.' == response.data['detail']
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_disabled)
@pytest.mark.django_db
@pytest.mark.license_feature
def test_system_tracking_license_get(hosts, get, user):
hosts = hosts(host_count=1)
url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
response = get(url, user('admin', True))
check_system_tracking_feature_forbidden(response)
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_disabled)
@pytest.mark.django_db
@pytest.mark.license_feature
def test_system_tracking_license_options(hosts, options, user):
hosts = hosts(host_count=1)
url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
response = options(url, None, user('admin', True))
check_system_tracking_feature_forbidden(response)
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
@pytest.mark.license_feature
def test_no_facts_db(hosts, get, user):
hosts = hosts(host_count=1)
url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
@@ -72,28 +100,19 @@ def test_basic_fields(hosts, fact_scans, get, user):
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
@pytest.mark.skipif(True, reason="Options fix landed in devel but not here. Enable this after this pr gets merged.")
@pytest.mark.license_feature
def test_basic_options_fields(hosts, fact_scans, options, user):
hosts = hosts(host_count=1)
fact_scans(fact_scans=1)
url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
response = options(url, user('admin', True), pk=hosts[0].id)
response = options(url, None, user('admin', True), pk=hosts[0].id)
#import json
#print(json.dumps(response.data))
assert 'related' in response.data
assert 'id' in response.data
assert 'facts' in response.data
assert 'module' in response.data
assert 'host' in response.data
assert isinstance(response.data['host'], int)
assert 'summary_fields' in response.data
assert 'host' in response.data['summary_fields']
assert 'name' in response.data['summary_fields']['host']
assert 'description' in response.data['summary_fields']['host']
assert 'host' in response.data['related']
assert reverse('api:host_detail', args=(hosts[0].pk,)) == response.data['related']['host']
assert 'related' in response.data['actions']['GET']
assert 'module' in response.data['actions']['GET']
assert ("ansible", "Ansible") in response.data['actions']['GET']['module']['choices']
assert ("services", "Services") in response.data['actions']['GET']['module']['choices']
assert ("packages", "Packages") in response.data['actions']['GET']['module']['choices']
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db

28
awx/main/tests/functional/api/test_fact_view.py
View File

@@ -1,7 +1,6 @@
import mock
import pytest
import json
import urllib
from awx.main.utils import timestamp_apiformat
from django.core.urlresolvers import reverse
@@ -10,6 +9,9 @@ from django.utils import timezone
def mock_feature_enabled(feature, bypass_database=None):
return True
def mock_feature_disabled(feature, bypass_database=None):
return False
# TODO: Consider making the fact_scan() fixture a Class, instead of a function, and move this method into it
def find_fact(facts, host_id, module_name, timestamp):
for f in facts:
@@ -27,6 +29,30 @@ def setup_common(hosts, fact_scans, get, user, epoch=timezone.now(), module_name
fact_known = find_fact(facts, hosts[0].id, module_name, epoch)
return (fact_known, response)
def check_system_tracking_feature_forbidden(response):
assert 402 == response.status_code
assert 'Your license does not permit use of system tracking.' == response.data['detail']
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_disabled)
@pytest.mark.django_db
@pytest.mark.license_feature
def test_system_tracking_license_get(hosts, get, user):
hosts = hosts(host_count=1)
url = reverse('api:host_fact_compare_view', args=(hosts[0].pk,))
response = get(url, user('admin', True))
check_system_tracking_feature_forbidden(response)
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_disabled)
@pytest.mark.django_db
@pytest.mark.license_feature
def test_system_tracking_license_options(hosts, options, user):
hosts = hosts(host_count=1)
url = reverse('api:host_fact_compare_view', args=(hosts[0].pk,))
response = options(url, None, user('admin', True))
check_system_tracking_feature_forbidden(response)
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_no_fact_found(hosts, get, user):

200
awx/main/tests/functional/commands/test_cleanup_facts.py Normal file
View File

@@ -0,0 +1,200 @@
# Copyright (c) 2016 Ansible, Inc.
# All Rights Reserved
# Python
import pytest
import mock
from dateutil.relativedelta import relativedelta
from datetime import timedelta
# Django
from django.utils import timezone
from django.core.management.base import CommandError
# AWX
from awx.main.management.commands.cleanup_facts import CleanupFacts, Command
from awx.main.models.fact import Fact
from awx.main.models.inventory import Host
def mock_feature_enabled(feature, bypass_database=None):
return True
def mock_feature_disabled(feature, bypass_database=None):
return False
@pytest.mark.django_db
def test_cleanup_granularity(fact_scans, hosts):
epoch = timezone.now()
hosts(5)
fact_scans(10, timestamp_epoch=epoch)
fact_newest = Fact.objects.all().order_by('-timestamp').first()
timestamp_future = fact_newest.timestamp + timedelta(days=365)
granularity = relativedelta(days=2)
cleanup_facts = CleanupFacts()
deleted_count = cleanup_facts.cleanup(timestamp_future, granularity)
assert 60 == deleted_count
'''
Delete half of the scans
'''
@pytest.mark.django_db
def test_cleanup_older_than(fact_scans, hosts):
epoch = timezone.now()
hosts(5)
fact_scans(28, timestamp_epoch=epoch)
qs = Fact.objects.all().order_by('-timestamp')
fact_middle = qs[qs.count() / 2]
granularity = relativedelta()
cleanup_facts = CleanupFacts()
deleted_count = cleanup_facts.cleanup(fact_middle.timestamp, granularity)
assert 210 == deleted_count
@pytest.mark.django_db
def test_cleanup_older_than_granularity_module(fact_scans, hosts):
epoch = timezone.now()
hosts(5)
fact_scans(10, timestamp_epoch=epoch)
fact_newest = Fact.objects.all().order_by('-timestamp').first()
timestamp_future = fact_newest.timestamp + timedelta(days=365)
granularity = relativedelta(days=2)
cleanup_facts = CleanupFacts()
deleted_count = cleanup_facts.cleanup(timestamp_future, granularity, module='ansible')
assert 20 == deleted_count
'''
Reduce the granularity of half of the facts scans, by half.
'''
@pytest.mark.django_db
def test_cleanup_logic(fact_scans, hosts):
epoch = timezone.now()
hosts = hosts(5)
fact_scans(60, timestamp_epoch=epoch)
timestamp_middle = epoch + timedelta(days=30)
granularity = relativedelta(days=2)
module = 'ansible'
cleanup_facts = CleanupFacts()
cleanup_facts.cleanup(timestamp_middle, granularity, module=module)
host_ids = Host.objects.all().values_list('id', flat=True)
host_facts = {}
for host_id in host_ids:
facts = Fact.objects.filter(host__id=host_id, module=module, timestamp__lt=timestamp_middle).order_by('-timestamp')
host_facts[host_id] = facts
for host_id, facts in host_facts.iteritems():
assert 15 == len(facts)
timestamp_pivot = timestamp_middle
for fact in facts:
timestamp_pivot -= granularity
assert fact.timestamp == timestamp_pivot
@mock.patch('awx.main.management.commands.cleanup_facts.feature_enabled', new=mock_feature_disabled)
@pytest.mark.django_db
@pytest.mark.license_feature
def test_system_tracking_feature_disabled(mocker):
cmd = Command()
with pytest.raises(CommandError) as err:
cmd.handle(None)
assert 'The System Tracking feature is not enabled for your Tower instance' in err.value
@mock.patch('awx.main.management.commands.cleanup_facts.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_parameters_ok(mocker):
run = mocker.patch('awx.main.management.commands.cleanup_facts.CleanupFacts.run')
kv = {
'older_than': '1d',
'granularity': '1d',
'module': None,
}
cmd = Command()
cmd.handle(None, **kv)
run.assert_called_once_with(relativedelta(days=1), relativedelta(days=1), module=None)
@pytest.mark.django_db
def test_string_time_to_timestamp_ok():
kvs = [
{
'time': '2w',
'timestamp': relativedelta(weeks=2),
'msg': '2 weeks',
},
{
'time': '23d',
'timestamp': relativedelta(days=23),
'msg': '23 days',
},
{
'time': '11m',
'timestamp': relativedelta(months=11),
'msg': '11 months',
},
{
'time': '14y',
'timestamp': relativedelta(years=14),
'msg': '14 years',
},
]
for kv in kvs:
cmd = Command()
res = cmd.string_time_to_timestamp(kv['time'])
assert kv['timestamp'] == res
@pytest.mark.django_db
def test_string_time_to_timestamp_invalid():
kvs = [
{
'time': '2weeks',
'msg': 'weeks instead of w',
},
{
'time': '2days',
'msg': 'days instead of d',
},
{
'time': '23',
'msg': 'no unit specified',
},
{
'time': None,
'msg': 'no value specified',
},
{
'time': 'zigzag',
'msg': 'random string specified',
},
]
for kv in kvs:
cmd = Command()
res = cmd.string_time_to_timestamp(kv['time'])
assert res is None
@mock.patch('awx.main.management.commands.cleanup_facts.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_parameters_fail(mocker):
# Mock run() just in case, but it should never get called because an error should be thrown
mocker.patch('awx.main.management.commands.cleanup_facts.CleanupFacts.run')
kvs = [
{
'older_than': '1week',
'granularity': '1d',
'msg': '--older_than invalid value "1week"',
},
{
'older_than': '1d',
'granularity': '1year',
'msg': '--granularity invalid value "1year"',
}
]
for kv in kvs:
cmd = Command()
with pytest.raises(CommandError) as err:
cmd.handle(None, older_than=kv['older_than'], granularity=kv['granularity'])
assert kv['msg'] in err.value

238
awx/main/tests/old/commands/cleanup_facts.py
View File

@@ -1,238 +0,0 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
# Python
from datetime import datetime
from dateutil.relativedelta import relativedelta
import mock
#Django
from django.core.management.base import CommandError
# AWX
from awx.main.tests.base import BaseTest
from awx.fact.tests.base import MongoDBRequired, FactScanBuilder, TEST_FACT_PACKAGES, TEST_FACT_ANSIBLE, TEST_FACT_SERVICES
from command_base import BaseCommandMixin
from awx.main.management.commands.cleanup_facts import Command, CleanupFacts
from awx.fact.models.fact import * # noqa
__all__ = ['CommandTest','CleanupFactsUnitTest', 'CleanupFactsCommandFunctionalTest']
class CleanupFactsCommandFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequired):
def setUp(self):
super(CleanupFactsCommandFunctionalTest, self).setUp()
self.create_test_license_file()
self.builder = FactScanBuilder()
self.builder.add_fact('ansible', TEST_FACT_ANSIBLE)
def test_invoke_zero_ok(self):
self.builder.set_epoch(datetime(year=2015, day=2, month=1, microsecond=0))
self.builder.build(scan_count=20, host_count=10)
result, stdout, stderr = self.run_command('cleanup_facts', granularity='2y', older_than='1d')
self.assertEqual(stdout, 'Deleted %s facts.\n' % ((200 / 2)))
def test_invoke_zero_deleted(self):
result, stdout, stderr = self.run_command('cleanup_facts', granularity='1w',older_than='5d')
self.assertEqual(stdout, 'Deleted 0 facts.\n')
def test_invoke_all_deleted(self):
self.builder.build(scan_count=20, host_count=10)
result, stdout, stderr = self.run_command('cleanup_facts', granularity='0d', older_than='0d')
self.assertEqual(stdout, 'Deleted 200 facts.\n')
def test_invoke_params_required(self):
result, stdout, stderr = self.run_command('cleanup_facts')
self.assertIsInstance(result, CommandError)
self.assertEqual(str(result), 'Both --granularity and --older_than are required.')
def test_module(self):
self.builder.add_fact('packages', TEST_FACT_PACKAGES)
self.builder.add_fact('services', TEST_FACT_SERVICES)
self.builder.build(scan_count=5, host_count=5)
result, stdout, stderr = self.run_command('cleanup_facts', granularity='0d', older_than='0d', module='packages')
self.assertEqual(stdout, 'Deleted 25 facts.\n')
class CommandTest(BaseTest):
def setUp(self):
super(CommandTest, self).setUp()
self.create_test_license_file()
@mock.patch('awx.main.management.commands.cleanup_facts.CleanupFacts.run')
def test_parameters_ok(self, run):
kv = {
'older_than': '1d',
'granularity': '1d',
'module': None,
}
cmd = Command()
cmd.handle(None, **kv)
run.assert_called_once_with(relativedelta(days=1), relativedelta(days=1), module=None)
def test_string_time_to_timestamp_ok(self):
kvs = [
{
'time': '2w',
'timestamp': relativedelta(weeks=2),
'msg': '2 weeks',
},
{
'time': '23d',
'timestamp': relativedelta(days=23),
'msg': '23 days',
},
{
'time': '11m',
'timestamp': relativedelta(months=11),
'msg': '11 months',
},
{
'time': '14y',
'timestamp': relativedelta(years=14),
'msg': '14 years',
},
]
for kv in kvs:
cmd = Command()
res = cmd.string_time_to_timestamp(kv['time'])
self.assertEqual(kv['timestamp'], res, "%s should convert to %s" % (kv['time'], kv['msg']))
def test_string_time_to_timestamp_invalid(self):
kvs = [
{
'time': '2weeks',
'msg': 'weeks instead of w',
},
{
'time': '2days',
'msg': 'days instead of d',
},
{
'time': '23',
'msg': 'no unit specified',
},
{
'time': None,
'msg': 'no value specified',
},
{
'time': 'zigzag',
'msg': 'random string specified',
},
]
for kv in kvs:
cmd = Command()
res = cmd.string_time_to_timestamp(kv['time'])
self.assertIsNone(res, kv['msg'])
# Mock run() just in case, but it should never get called because an error should be thrown
@mock.patch('awx.main.management.commands.cleanup_facts.CleanupFacts.run')
def test_parameters_fail(self, run):
kvs = [
{
'older_than': '1week',
'granularity': '1d',
'msg': 'Invalid older_than param value',
},
{
'older_than': '1d',
'granularity': '1year',
'msg': 'Invalid granularity param value',
}
]
for kv in kvs:
cmd = Command()
with self.assertRaises(CommandError):
cmd.handle(None, older_than=kv['older_than'], granularity=kv['granularity'])
class CleanupFactsUnitTest(BaseCommandMixin, BaseTest, MongoDBRequired):
def setUp(self):
super(CleanupFactsUnitTest, self).setUp()
self.builder = FactScanBuilder()
self.builder.add_fact('ansible', TEST_FACT_ANSIBLE)
self.builder.add_fact('packages', TEST_FACT_PACKAGES)
self.builder.build(scan_count=20, host_count=10)
'''
Create 10 hosts with 40 facts each. After cleanup, there should be 20 facts for each host.
Then ensure the correct facts are deleted.
'''
def test_cleanup_logic(self):
cleanup_facts = CleanupFacts()
fact_oldest = FactVersion.objects.all().order_by('timestamp').first()
granularity = relativedelta(years=2)
deleted_count = cleanup_facts.cleanup(self.builder.get_timestamp(0), granularity)
self.assertEqual(deleted_count, 2 * (self.builder.get_scan_count() * self.builder.get_host_count()) / 2)
# Check the number of facts per host
for host in self.builder.get_hosts():
count = FactVersion.objects.filter(host=host).count()
scan_count = (2 * self.builder.get_scan_count()) / 2
self.assertEqual(count, scan_count)
count = Fact.objects.filter(host=host).count()
self.assertEqual(count, scan_count)
# Ensure that only 2 facts (ansible and packages) exists per granularity time
date_pivot = self.builder.get_timestamp(0)
for host in self.builder.get_hosts():
while date_pivot > fact_oldest.timestamp:
date_pivot_next = date_pivot - granularity
kv = {
'timestamp__lte': date_pivot,
'timestamp__gt': date_pivot_next,
'host': host,
}
count = FactVersion.objects.filter(**kv).count()
self.assertEqual(count, 2, "should only be 2 FactVersion per the 2 year granularity")
count = Fact.objects.filter(**kv).count()
self.assertEqual(count, 2, "should only be 2 Fact per the 2 year granularity")
date_pivot = date_pivot_next
'''
Create 10 hosts with 40 facts each. After cleanup, there should be 30 facts for each host.
Then ensure the correct facts are deleted.
'''
def test_cleanup_module(self):
cleanup_facts = CleanupFacts()
fact_oldest = FactVersion.objects.all().order_by('timestamp').first()
granularity = relativedelta(years=2)
deleted_count = cleanup_facts.cleanup(self.builder.get_timestamp(0), granularity, module='ansible')
self.assertEqual(deleted_count, (self.builder.get_scan_count() * self.builder.get_host_count()) / 2)
# Check the number of facts per host
for host in self.builder.get_hosts():
count = FactVersion.objects.filter(host=host).count()
self.assertEqual(count, 30)
count = Fact.objects.filter(host=host).count()
self.assertEqual(count, 30)
# Ensure that only 1 ansible fact exists per granularity time
date_pivot = self.builder.get_timestamp(0)
for host in self.builder.get_hosts():
while date_pivot > fact_oldest.timestamp:
date_pivot_next = date_pivot - granularity
kv = {
'timestamp__lte': date_pivot,
'timestamp__gt': date_pivot_next,
'host': host,
'module': 'ansible',
}
count = FactVersion.objects.filter(**kv).count()
self.assertEqual(count, 1)
count = Fact.objects.filter(**kv).count()
self.assertEqual(count, 1)
date_pivot = date_pivot_next