migrate data from mongo to postgres

awx/main/tests/functional/migrations/conftest.py

@@ -0,0 +1,84 @@
+# Python
+import pytest
+from datetime import timedelta
+
+# Django
+from django.utils import timezone
+from django.conf import settings
+
+# AWX
+from awx.fact.models.fact import Fact, FactHost
+
+# MongoEngine
+from mongoengine.connection import ConnectionError
+
+@pytest.fixture(autouse=True)
+def mongo_db(request):
+    marker = request.keywords.get('mongo_db', None)
+    if marker:
+        # Drop mongo database
+        try:
+            db = Fact._get_db()
+            db.connection.drop_database(settings.MONGO_DB)
+        except ConnectionError:
+            raise
+
+@pytest.fixture
+def inventories(organization):
+    def rf(inventory_count=1):
+        invs = []
+        for i in xrange(0, inventory_count):
+            inv = organization.inventories.create(name="test-inv-%d" % i, description="test-inv-desc")
+            invs.append(inv)
+        return invs
+    return rf
+
+'''
+hosts naming convension should align with hosts_mongo 
+'''
+@pytest.fixture
+def hosts(organization):
+    def rf(host_count=1, inventories=[]):
+        hosts = []
+        for inv in inventories:
+            for i in xrange(0, host_count):
+                name = '%s-host-%s' % (inv.name, i)
+                host = inv.hosts.create(name=name)
+                hosts.append(host)
+        return hosts
+    return rf
+
+@pytest.fixture
+def hosts_mongo(organization):
+    def rf(host_count=1, inventories=[]):
+        hosts = []
+        for inv in inventories:
+            for i in xrange(0, host_count):
+                name = '%s-host-%s' % (inv.name, i)
+                (host, created) = FactHost.objects.get_or_create(hostname=name, inventory_id=inv.id)
+                hosts.append(host)
+        return hosts
+    return rf
+
+@pytest.fixture
+def fact_scans(organization, fact_ansible_json, fact_packages_json, fact_services_json):
+    def rf(fact_scans=1, inventories=[], timestamp_epoch=timezone.now()):
+        facts_json = {}
+        facts = []
+        module_names = ['ansible', 'services', 'packages']
+
+        facts_json['ansible'] = fact_ansible_json
+        facts_json['packages'] = fact_packages_json
+        facts_json['services'] = fact_services_json
+
+        for inv in inventories:
+            for host_obj in FactHost.objects.filter(inventory_id=inv.id):
+                timestamp_current = timestamp_epoch
+                for i in xrange(0, fact_scans):
+                    for module_name in module_names:
+                        facts.append(Fact.add_fact(timestamp_current, facts_json[module_name],  host_obj, module_name))
+                    timestamp_current += timedelta(days=1)
+        return facts
+    return rf
+
+

awx/main/tests/functional/migrations/test_fact.py

@@ -0,0 +1,60 @@
+import pytest
+import datetime
+
+from django.apps import apps
+
+from awx.main.models.inventory import Host
+from awx.main.models.fact import Fact
+
+from awx.main.migrations import _system_tracking as system_tracking
+
+def micro_to_milli(micro):
+    return micro - (((int)(micro / 1000)) * 1000)
+
+@pytest.mark.django_db
+@pytest.mark.mongo_db
+def test_migrate_facts(inventories, hosts, hosts_mongo, fact_scans):
+    inventory_objs = inventories(2)
+    hosts(2, inventory_objs)
+    hosts_mongo(2, inventory_objs)
+    facts_known = fact_scans(2, inventory_objs)
+
+    (migrated_count, not_migrated_count) = system_tracking.migrate_facts(apps, None)
+    # 4 hosts w/ 2 fact scans each, 3 modules each scan
+    assert migrated_count == 24
+    assert not_migrated_count == 0
+
+    
+    for fact_mongo, fact_version in facts_known:
+        host = Host.objects.get(inventory_id=fact_mongo.host.inventory_id, name=fact_mongo.host.hostname)
+        t = fact_mongo.timestamp - datetime.timedelta(microseconds=micro_to_milli(fact_mongo.timestamp.microsecond))
+        fact = Fact.objects.filter(host_id=host.id, timestamp=t, module=fact_mongo.module)
+
+        assert len(fact) == 1
+        assert fact[0] is not None
+
+@pytest.mark.django_db
+@pytest.mark.mongo_db
+def test_migrate_facts_hostname_does_not_exist(inventories, hosts, hosts_mongo, fact_scans):
+    inventory_objs = inventories(2)
+    host_objs = hosts(1, inventory_objs)
+    hosts_mongo(2, inventory_objs)
+    facts_known = fact_scans(2, inventory_objs)
+
+    (migrated_count, not_migrated_count) = system_tracking.migrate_facts(apps, None)
+    assert migrated_count == 12
+    assert not_migrated_count == 12
+
+
+    for fact_mongo, fact_version in facts_known:
+        # Facts that don't match the only host will not be migrated
+        if fact_mongo.host.hostname != host_objs[0].name:
+            continue
+
+        host = Host.objects.get(inventory_id=fact_mongo.host.inventory_id, name=fact_mongo.host.hostname)
+        t = fact_mongo.timestamp - datetime.timedelta(microseconds=micro_to_milli(fact_mongo.timestamp.microsecond))
+        fact = Fact.objects.filter(host_id=host.id, timestamp=t, module=fact_mongo.module)
+
+        assert len(fact) == 1
+        assert fact[0] is not None
+