From 8e315ec393092d5464d008118bab083338ad3896 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Thu, 2 Feb 2023 15:17:14 +0100 Subject: [PATCH 1/8] Host Metrics List API --- awx/api/serializers.py | 9 +++++++++ awx/api/urls/host_metric.py | 10 ++++++++++ awx/api/urls/urls.py | 2 ++ awx/api/views/__init__.py | 7 +++++++ awx/main/access.py | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 61 insertions(+) create mode 100644 awx/api/urls/host_metric.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index be87a50a82..85ba959530 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -55,6 +55,7 @@ from awx.main.models import ( ExecutionEnvironment, Group, Host, + HostMetric, Instance, InstanceGroup, InstanceLink, @@ -5002,6 +5003,14 @@ class InstanceHealthCheckSerializer(BaseSerializer): fields = read_only_fields +class HostMetricSerializer(BaseSerializer): + show_capabilities = ['delete'] + + class Meta: + model = HostMetric + fields = ("hostname", "first_automation", "last_automation") + + class InstanceGroupSerializer(BaseSerializer): show_capabilities = ['edit', 'delete'] capacity = serializers.SerializerMethodField() diff --git a/awx/api/urls/host_metric.py b/awx/api/urls/host_metric.py new file mode 100644 index 0000000000..547c995c10 --- /dev/null +++ b/awx/api/urls/host_metric.py @@ -0,0 +1,10 @@ +# Copyright (c) 2017 Ansible, Inc. +# All Rights Reserved. + +from django.urls import re_path + +from awx.api.views import HostMetricList + +urls = [re_path(r'$^', HostMetricList.as_view(), name='host_metric_list')] + +__all__ = ['urls'] diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index 5d0818b191..26857f1106 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -43,6 +43,7 @@ from .inventory import urls as inventory_urls from .execution_environments import urls as execution_environment_urls from .team import urls as team_urls from .host import urls as host_urls +from .host_metric import urls as host_metric_urls from .group import urls as group_urls from .inventory_source import urls as inventory_source_urls from .inventory_update import urls as inventory_update_urls @@ -111,6 +112,7 @@ v2_urls = [ re_path(r'^teams/', include(team_urls)), re_path(r'^inventories/', include(inventory_urls)), re_path(r'^hosts/', include(host_urls)), + re_path(r'^host_metrics/', include(host_metric_urls)), re_path(r'^groups/', include(group_urls)), re_path(r'^inventory_sources/', include(inventory_source_urls)), re_path(r'^inventory_updates/', include(inventory_update_urls)), diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index e81a6ebbde..b5cf1f88cc 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -1531,6 +1531,13 @@ class HostRelatedSearchMixin(object): return ret +class HostMetricList(ListAPIView): + always_allow_superuser = False + name = _("Host Metrics List") + model = models.HostMetric + serializer_class = serializers.HostMetricSerializer + + class HostList(HostRelatedSearchMixin, ListCreateAPIView): always_allow_superuser = False model = models.Host diff --git a/awx/main/access.py b/awx/main/access.py index 4d6bdf2c55..26ea9c4d2e 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -37,6 +37,7 @@ from awx.main.models import ( ExecutionEnvironment, Group, Host, + HostMetric, Instance, InstanceGroup, Inventory, @@ -861,6 +862,38 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess): return super(OrganizationAccess, self).can_attach(obj, sub_obj, relationship, *args, **kwargs) +class HostMetricAccess(BaseAccess): + """ + - I can see host metrics when a super user or system auditor. + - I can delete host metrics when a super user. + """ + + model = HostMetric + + def get_queryset(self): + # if self.user.is_superuser or self.user.is_system_auditor: + # return self.model.objects.filter(Q(user__isnull=True) | Q(user=self.user)) + # else: + # return self.model.objects.filter(user=self.user) + if self.user.is_superuser or self.user.is_system_auditor: + qs = self.model.objects.all() + else: + qs = self.filtered_queryset() + return qs + + def can_read(self, obj): + return bool(self.user.is_superuser or self.user.is_system_auditor or (obj and obj.user == self.user)) + + def can_add(self, data): + return False # There is no API endpoint to POST new settings. + + def can_change(self, obj, data): + return False + + def can_delete(self, obj): + return bool(self.user.is_superuser or (obj and obj.user == self.user)) + + class InventoryAccess(BaseAccess): """ I can see inventory when: From 068c9a572fa689e8f56ccbc48ffd0db8162513a6 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Fri, 3 Feb 2023 14:22:21 +0100 Subject: [PATCH 2/8] HostMetrics migration --- awx/api/serializers.py | 13 +++++- awx/api/urls/host_metric.py | 4 +- awx/api/views/__init__.py | 10 ++++- awx/main/access.py | 8 +--- .../migrations/0175_add_hostmetric_fields.py | 43 +++++++++++++++++++ awx/main/models/inventory.py | 12 +++++- 6 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 awx/main/migrations/0175_add_hostmetric_fields.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 85ba959530..d44744c44f 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -5008,7 +5008,18 @@ class HostMetricSerializer(BaseSerializer): class Meta: model = HostMetric - fields = ("hostname", "first_automation", "last_automation") + fields = ( + "id", + "hostname", + "url", + "first_automation", + "last_automation", + "last_deleted", + "automated_counter", + "deleted_counter", + "deleted", + "used_in_inventories", + ) class InstanceGroupSerializer(BaseSerializer): diff --git a/awx/api/urls/host_metric.py b/awx/api/urls/host_metric.py index 547c995c10..d464fb82c5 100644 --- a/awx/api/urls/host_metric.py +++ b/awx/api/urls/host_metric.py @@ -3,8 +3,8 @@ from django.urls import re_path -from awx.api.views import HostMetricList +from awx.api.views import HostMetricList, HostMetricDetail -urls = [re_path(r'$^', HostMetricList.as_view(), name='host_metric_list')] +urls = [re_path(r'$^', HostMetricList.as_view(), name='host_metric_list'), re_path(r'^(?P[0-9]+)/$', HostMetricDetail.as_view(), name='host_metric_detail')] __all__ = ['urls'] diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index b5cf1f88cc..f3170867f3 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -1532,10 +1532,18 @@ class HostRelatedSearchMixin(object): class HostMetricList(ListAPIView): - always_allow_superuser = False name = _("Host Metrics List") model = models.HostMetric serializer_class = serializers.HostMetricSerializer + permission_classes = (IsSystemAdminOrAuditor,) + search_fields = ('hostname', 'deleted') + + +class HostMetricDetail(RetrieveDestroyAPIView): + name = _("Host Metric Detail") + model = models.HostMetric + serializer_class = serializers.HostMetricSerializer + permission_classes = (IsSystemAdminOrAuditor,) class HostList(HostRelatedSearchMixin, ListCreateAPIView): diff --git a/awx/main/access.py b/awx/main/access.py index 26ea9c4d2e..3610079ae4 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -864,17 +864,13 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess): class HostMetricAccess(BaseAccess): """ - - I can see host metrics when a super user or system auditor. - - I can delete host metrics when a super user. + - I can see host metrics when I'm a super user or system auditor. + - I can delete host metrics when I'm a super user. """ model = HostMetric def get_queryset(self): - # if self.user.is_superuser or self.user.is_system_auditor: - # return self.model.objects.filter(Q(user__isnull=True) | Q(user=self.user)) - # else: - # return self.model.objects.filter(user=self.user) if self.user.is_superuser or self.user.is_system_auditor: qs = self.model.objects.all() else: diff --git a/awx/main/migrations/0175_add_hostmetric_fields.py b/awx/main/migrations/0175_add_hostmetric_fields.py new file mode 100644 index 0000000000..d273a6b6ea --- /dev/null +++ b/awx/main/migrations/0175_add_hostmetric_fields.py @@ -0,0 +1,43 @@ +# Generated by Django 3.2.16 on 2023-02-03 09:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('main', '0174_ensure_org_ee_admin_roles'), + ] + + operations = [ + migrations.AlterField(model_name='hostmetric', name='hostname', field=models.CharField(max_length=512, primary_key=False, serialize=True, unique=True)), + migrations.AddField( + model_name='hostmetric', + name='last_deleted', + field=models.DateTimeField(db_index=True, null=True, help_text='When the host was last deleted'), + ), + migrations.AddField( + model_name='hostmetric', + name='automated_counter', + field=models.BigIntegerField(default=0, help_text='How many times was the host automated'), + ), + migrations.AddField( + model_name='hostmetric', + name='deleted_counter', + field=models.BigIntegerField(default=0, help_text='How many times was the host deleted'), + ), + migrations.AddField( + model_name='hostmetric', + name='deleted', + field=models.BooleanField( + default=False, help_text='Boolean flag saying whether the host is deleted and therefore not counted into the subscription consumption' + ), + ), + migrations.AddField( + model_name='hostmetric', + name='used_in_inventories', + field=models.BigIntegerField(null=True, help_text='How many inventories contain this host'), + ), + migrations.AddField( + model_name='hostmetric', name='id', field=models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID') + ), + ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index a0df4dfde4..3334ba5f17 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -820,9 +820,19 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin): class HostMetric(models.Model): - hostname = models.CharField(primary_key=True, max_length=512) + hostname = models.CharField(unique=True, max_length=512) first_automation = models.DateTimeField(auto_now_add=True, null=False, db_index=True, help_text=_('When the host was first automated against')) last_automation = models.DateTimeField(db_index=True, help_text=_('When the host was last automated against')) + last_deleted = models.DateTimeField(null=True, db_index=True, help_text=_('When the host was last deleted')) + automated_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host automated')) + deleted_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host deleted')) + deleted = models.BooleanField( + default=False, help_text=_('Boolean flag saying whether the host is deleted and therefore not counted into the subscription consumption') + ) + used_in_inventories = models.BigIntegerField(null=True, help_text=_('How many inventories contain this host')) + + def get_absolute_url(self, request=None): + return reverse('api:host_metric_detail', kwargs={'pk': self.pk}, request=request) class InventorySourceOptions(BaseModel): From e62795fdc2363e71524a755bb1fa3b68625780bb Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Thu, 9 Feb 2023 16:39:39 +0100 Subject: [PATCH 3/8] Host Metrics update/soft delete --- awx/api/views/__init__.py | 5 + awx/main/models/events.py | 28 +- awx/main/models/inventory.py | 12 + .../tests/functional/models/test_events.py | 357 ++++++++++-------- .../functional/models/test_host_metric.py | 50 +++ 5 files changed, 282 insertions(+), 170 deletions(-) diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index f3170867f3..4c817f1255 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -1545,6 +1545,11 @@ class HostMetricDetail(RetrieveDestroyAPIView): serializer_class = serializers.HostMetricSerializer permission_classes = (IsSystemAdminOrAuditor,) + def delete(self, request, *args, **kwargs): + self.get_object().soft_delete() + + return Response(status=status.HTTP_204_NO_CONTENT) + class HostList(HostRelatedSearchMixin, ListCreateAPIView): always_allow_superuser = False diff --git a/awx/main/models/events.py b/awx/main/models/events.py index 1827802812..0ba07185f4 100644 --- a/awx/main/models/events.py +++ b/awx/main/models/events.py @@ -6,7 +6,7 @@ from collections import defaultdict from django.conf import settings from django.core.exceptions import ObjectDoesNotExist -from django.db import models, DatabaseError +from django.db import connection, models, DatabaseError from django.utils.dateparse import parse_datetime from django.utils.text import Truncator from django.utils.timezone import utc, now @@ -536,7 +536,7 @@ class JobEvent(BasePlaybookEvent): return job = self.job - from awx.main.models import Host, JobHostSummary, HostMetric # circular import + from awx.main.models import Host, JobHostSummary # circular import all_hosts = Host.objects.filter(pk__in=self.host_map.values()).only('id', 'name') existing_host_ids = set(h.id for h in all_hosts) @@ -575,12 +575,26 @@ class JobEvent(BasePlaybookEvent): Host.objects.bulk_update(list(updated_hosts), ['last_job_id', 'last_job_host_summary_id'], batch_size=100) - # bulk-create - current_time = now() - HostMetric.objects.bulk_create( - [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=100 + # Create/update Host Metrics + self._update_host_metrics(updated_hosts_list) + + @staticmethod + def _update_host_metrics(updated_hosts_list): + from awx.main.models import HostMetric # circular import + + # bulk-create + current_time = now() + HostMetric.objects.bulk_create( + [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=1000 + ) + # bulk-update + batch_start, batch_size = 0, 1000 + while batch_start <= len(updated_hosts_list): + batched_host_list = updated_hosts_list[batch_start : (batch_start + batch_size)] + HostMetric.objects.filter(hostname__in=batched_host_list).update( + last_automation=current_time, automated_counter=models.F('automated_counter') + 1, deleted=False ) - HostMetric.objects.filter(hostname__in=updated_hosts_list).update(last_automation=current_time) + batch_start += batch_size @property def job_verbosity(self): diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 3334ba5f17..2611f83a6a 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -834,6 +834,18 @@ class HostMetric(models.Model): def get_absolute_url(self, request=None): return reverse('api:host_metric_detail', kwargs={'pk': self.pk}, request=request) + def soft_delete(self): + if not self.deleted: + self.deleted_counter = (self.deleted_counter or 0) + 1 + self.last_deleted = now() + self.deleted = True + self.save() + + def soft_restore(self): + if self.deleted: + self.deleted = False + self.save() + class InventorySourceOptions(BaseModel): """ diff --git a/awx/main/tests/functional/models/test_events.py b/awx/main/tests/functional/models/test_events.py index 758e69b641..48adc781e7 100644 --- a/awx/main/tests/functional/models/test_events.py +++ b/awx/main/tests/functional/models/test_events.py @@ -3,178 +3,209 @@ import pytest from django.utils.timezone import now -from awx.main.models import Job, JobEvent, Inventory, Host, JobHostSummary +from django.db.models import Q + +from awx.main.models import Job, JobEvent, Inventory, Host, JobHostSummary, HostMetric @pytest.mark.django_db -@mock.patch('awx.main.models.events.emit_event_detail') -def test_parent_changed(emit): - j = Job() - j.save() - JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save() - assert JobEvent.objects.count() == 1 - for e in JobEvent.objects.all(): - assert e.changed is False +class TestEvents: + def setup_method(self): + self.hostnames = [] + self.host_map = dict() + self.inventory = None + self.job = None - JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='runner_on_ok', event_data={'res': {'changed': ['localhost']}}).save() - # the `playbook_on_stats` event is where we update the parent changed linkage - JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save() - events = JobEvent.objects.filter(event__in=['playbook_on_task_start', 'runner_on_ok']) - assert events.count() == 2 - for e in events.all(): - assert e.changed is True + @mock.patch('awx.main.models.events.emit_event_detail') + def test_parent_changed(self, emit): + j = Job() + j.save() + JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save() + assert JobEvent.objects.count() == 1 + for e in JobEvent.objects.all(): + assert e.changed is False + JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='runner_on_ok', event_data={'res': {'changed': ['localhost']}}).save() + # the `playbook_on_stats` event is where we update the parent changed linkage + JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save() + events = JobEvent.objects.filter(event__in=['playbook_on_task_start', 'runner_on_ok']) + assert events.count() == 2 + for e in events.all(): + assert e.changed is True -@pytest.mark.django_db -@pytest.mark.parametrize('event', JobEvent.FAILED_EVENTS) -@mock.patch('awx.main.models.events.emit_event_detail') -def test_parent_failed(emit, event): - j = Job() - j.save() - JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save() - assert JobEvent.objects.count() == 1 - for e in JobEvent.objects.all(): - assert e.failed is False + @pytest.mark.parametrize('event', JobEvent.FAILED_EVENTS) + @mock.patch('awx.main.models.events.emit_event_detail') + def test_parent_failed(self, emit, event): + j = Job() + j.save() + JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save() + assert JobEvent.objects.count() == 1 + for e in JobEvent.objects.all(): + assert e.failed is False - JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event=event).save() + JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event=event).save() - # the `playbook_on_stats` event is where we update the parent failed linkage - JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save() - events = JobEvent.objects.filter(event__in=['playbook_on_task_start', event]) - assert events.count() == 2 - for e in events.all(): - assert e.failed is True + # the `playbook_on_stats` event is where we update the parent failed linkage + JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save() + events = JobEvent.objects.filter(event__in=['playbook_on_task_start', event]) + assert events.count() == 2 + for e in events.all(): + assert e.failed is True + def test_host_summary_generation(self): + self._generate_hosts(100) + self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames)) -@pytest.mark.django_db -def test_host_summary_generation(): - hostnames = [f'Host {i}' for i in range(100)] - inv = Inventory() - inv.save() - Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames]) - j = Job(inventory=inv) - j.save() - host_map = dict((host.name, host.id) for host in inv.hosts.all()) - JobEvent.create_from_data( - job_id=j.pk, + assert self.job.job_host_summaries.count() == len(self.hostnames) + assert sorted([s.host_name for s in self.job.job_host_summaries.all()]) == sorted(self.hostnames) + + for s in self.job.job_host_summaries.all(): + assert self.host_map[s.host_name] == s.host_id + assert s.ok == len(s.host_name) + assert s.changed == 0 + assert s.dark == 0 + assert s.failures == 0 + assert s.ignored == 0 + assert s.processed == 0 + assert s.rescued == 0 + assert s.skipped == 0 + + for host in Host.objects.all(): + assert host.last_job_id == self.job.id + assert host.last_job_host_summary.host == host + + def test_host_summary_generation_with_deleted_hosts(self): + self._generate_hosts(10) + + # delete half of the hosts during the playbook run + for h in self.inventory.hosts.all()[:5]: + h.delete() + + self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames)) + + ids = sorted([s.host_id or -1 for s in self.job.job_host_summaries.order_by('id').all()]) + names = sorted([s.host_name for s in self.job.job_host_summaries.all()]) + assert ids == [-1, -1, -1, -1, -1, 6, 7, 8, 9, 10] + assert names == ['Host 0', 'Host 1', 'Host 2', 'Host 3', 'Host 4', 'Host 5', 'Host 6', 'Host 7', 'Host 8', 'Host 9'] + + def test_host_summary_generation_with_limit(self): + # Make an inventory with 10 hosts, run a playbook with a --limit + # pointed at *one* host, + # Verify that *only* that host has an associated JobHostSummary and that + # *only* that host has an updated value for .last_job. + self._generate_hosts(10) + + # by making the playbook_on_stats *only* include Host 1, we're emulating + # the behavior of a `--limit=Host 1` + matching_host = Host.objects.get(name='Host 1') + self._create_job_event(ok={matching_host.name: len(matching_host.name)}) # effectively, limit=Host 1 + + # since the playbook_on_stats only references one host, + # there should *only* be on JobHostSummary record (and it should + # be related to the appropriate Host) + assert JobHostSummary.objects.count() == 1 + for h in Host.objects.all(): + if h.name == 'Host 1': + assert h.last_job_id == self.job.id + assert h.last_job_host_summary_id == JobHostSummary.objects.first().id + else: + # all other hosts in the inventory should remain untouched + assert h.last_job_id is None + assert h.last_job_host_summary_id is None + + def test_host_metrics_insert(self): + self._generate_hosts(10) + + self._create_job_event( + ok=dict((hostname, len(hostname)) for hostname in self.hostnames[0:3]), + failures=dict((hostname, len(hostname)) for hostname in self.hostnames[3:6]), + processed=dict((hostname, len(hostname)) for hostname in self.hostnames[6:9]), + skipped=dict((hostname, len(hostname)) for hostname in [self.hostnames[9]]), + ) + + metrics = HostMetric.objects.all() + assert len(metrics) == 10 + for hm in metrics: + assert hm.automated_counter == 1 + assert hm.last_automation is not None + assert hm.deleted is False + + def test_host_metrics_update(self): + self._generate_hosts(12) + + self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames)) + + # Soft delete 6 host metrics + for hm in HostMetric.objects.filter(id__in=[1, 3, 5, 7, 9, 11]): + hm.soft_delete() + + assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6 + assert len(HostMetric.objects.filter(Q(deleted=True) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6 + + # hostnames in 'ignored' and 'rescued' stats are ignored + self.job = Job(inventory=self.inventory) + self.job.save() + self._create_job_event( + ignored=dict((hostname, len(hostname)) for hostname in self.hostnames[0:6]), + rescued=dict((hostname, len(hostname)) for hostname in self.hostnames[6:11]), + ) + + assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6 + assert len(HostMetric.objects.filter(Q(deleted=True) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6 + + # hostnames in 'changed', 'dark', 'failures', 'ok', 'processed', 'skipped' are processed + self.job = Job(inventory=self.inventory) + self.job.save() + self._create_job_event( + changed=dict((hostname, len(hostname)) for hostname in self.hostnames[0:2]), + dark=dict((hostname, len(hostname)) for hostname in self.hostnames[2:4]), + failures=dict((hostname, len(hostname)) for hostname in self.hostnames[4:6]), + ok=dict((hostname, len(hostname)) for hostname in self.hostnames[6:8]), + processed=dict((hostname, len(hostname)) for hostname in self.hostnames[8:10]), + skipped=dict((hostname, len(hostname)) for hostname in self.hostnames[10:12]), + ) + assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6 + assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6 + + def _generate_hosts(self, cnt, id_from=0): + self.hostnames = [f'Host {i}' for i in range(id_from, id_from + cnt)] + self.inventory = Inventory() + self.inventory.save() + Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=self.inventory.id) for h in self.hostnames]) + self.job = Job(inventory=self.inventory) + self.job.save() + + # host map is a data structure that tracks a mapping of host name --> ID + # for the inventory, _regardless_ of whether or not there's a limit + # applied to the actual playbook run + self.host_map = dict((host.name, host.id) for host in self.inventory.hosts.all()) + + def _create_job_event( + self, parent_uuid='abc123', event='playbook_on_stats', - event_data={ - 'ok': dict((hostname, len(hostname)) for hostname in hostnames), - 'changed': {}, - 'dark': {}, - 'failures': {}, - 'ignored': {}, - 'processed': {}, - 'rescued': {}, - 'skipped': {}, - }, - host_map=host_map, - ).save() - - assert j.job_host_summaries.count() == len(hostnames) - assert sorted([s.host_name for s in j.job_host_summaries.all()]) == sorted(hostnames) - - for s in j.job_host_summaries.all(): - assert host_map[s.host_name] == s.host_id - assert s.ok == len(s.host_name) - assert s.changed == 0 - assert s.dark == 0 - assert s.failures == 0 - assert s.ignored == 0 - assert s.processed == 0 - assert s.rescued == 0 - assert s.skipped == 0 - - for host in Host.objects.all(): - assert host.last_job_id == j.id - assert host.last_job_host_summary.host == host - - -@pytest.mark.django_db -def test_host_summary_generation_with_deleted_hosts(): - hostnames = [f'Host {i}' for i in range(10)] - inv = Inventory() - inv.save() - Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames]) - j = Job(inventory=inv) - j.save() - host_map = dict((host.name, host.id) for host in inv.hosts.all()) - - # delete half of the hosts during the playbook run - for h in inv.hosts.all()[:5]: - h.delete() - - JobEvent.create_from_data( - job_id=j.pk, - parent_uuid='abc123', - event='playbook_on_stats', - event_data={ - 'ok': dict((hostname, len(hostname)) for hostname in hostnames), - 'changed': {}, - 'dark': {}, - 'failures': {}, - 'ignored': {}, - 'processed': {}, - 'rescued': {}, - 'skipped': {}, - }, - host_map=host_map, - ).save() - - ids = sorted([s.host_id or -1 for s in j.job_host_summaries.order_by('id').all()]) - names = sorted([s.host_name for s in j.job_host_summaries.all()]) - assert ids == [-1, -1, -1, -1, -1, 6, 7, 8, 9, 10] - assert names == ['Host 0', 'Host 1', 'Host 2', 'Host 3', 'Host 4', 'Host 5', 'Host 6', 'Host 7', 'Host 8', 'Host 9'] - - -@pytest.mark.django_db -def test_host_summary_generation_with_limit(): - # Make an inventory with 10 hosts, run a playbook with a --limit - # pointed at *one* host, - # Verify that *only* that host has an associated JobHostSummary and that - # *only* that host has an updated value for .last_job. - hostnames = [f'Host {i}' for i in range(10)] - inv = Inventory() - inv.save() - Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames]) - j = Job(inventory=inv) - j.save() - - # host map is a data structure that tracks a mapping of host name --> ID - # for the inventory, _regardless_ of whether or not there's a limit - # applied to the actual playbook run - host_map = dict((host.name, host.id) for host in inv.hosts.all()) - - # by making the playbook_on_stats *only* include Host 1, we're emulating - # the behavior of a `--limit=Host 1` - matching_host = Host.objects.get(name='Host 1') - JobEvent.create_from_data( - job_id=j.pk, - parent_uuid='abc123', - event='playbook_on_stats', - event_data={ - 'ok': {matching_host.name: len(matching_host.name)}, # effectively, limit=Host 1 - 'changed': {}, - 'dark': {}, - 'failures': {}, - 'ignored': {}, - 'processed': {}, - 'rescued': {}, - 'skipped': {}, - }, - host_map=host_map, - ).save() - - # since the playbook_on_stats only references one host, - # there should *only* be on JobHostSummary record (and it should - # be related to the appropriate Host) - assert JobHostSummary.objects.count() == 1 - for h in Host.objects.all(): - if h.name == 'Host 1': - assert h.last_job_id == j.id - assert h.last_job_host_summary_id == JobHostSummary.objects.first().id - else: - # all other hosts in the inventory should remain untouched - assert h.last_job_id is None - assert h.last_job_host_summary_id is None + ok=None, + changed=None, + dark=None, + failures=None, + ignored=None, + processed=None, + rescued=None, + skipped=None, + ): + JobEvent.create_from_data( + job_id=self.job.pk, + parent_uuid=parent_uuid, + event=event, + event_data={ + 'ok': ok or {}, + 'changed': changed or {}, + 'dark': dark or {}, + 'failures': failures or {}, + 'ignored': ignored or {}, + 'processed': processed or {}, + 'rescued': rescued or {}, + 'skipped': skipped or {}, + }, + host_map=self.host_map, + ).save() diff --git a/awx/main/tests/functional/models/test_host_metric.py b/awx/main/tests/functional/models/test_host_metric.py index 1f560e474f..dad8295435 100644 --- a/awx/main/tests/functional/models/test_host_metric.py +++ b/awx/main/tests/functional/models/test_host_metric.py @@ -20,3 +20,53 @@ def test_host_metrics_generation(): date_today = now().strftime('%Y-%m-%d') result = HostMetric.objects.filter(first_automation__startswith=date_today).count() assert result == len(hostnames) + + +@pytest.mark.django_db +def test_soft_delete(): + hostnames = [f'Host to delete {i}' for i in range(2)] + current_time = now() + HostMetric.objects.bulk_create([HostMetric(hostname=h, last_automation=current_time, automated_counter=42) for h in hostnames]) + + hm = HostMetric.objects.get(hostname="Host to delete 0") + assert hm.last_deleted is None + + last_deleted = None + for _ in range(3): + # soft delete 1st + # 2nd/3rd delete don't have an effect + hm.soft_delete() + if last_deleted is None: + last_deleted = hm.last_deleted + + assert hm.deleted is True + assert hm.deleted_counter == 1 + assert hm.last_deleted == last_deleted + assert hm.automated_counter == 42 + + # 2nd record is not touched + hm = HostMetric.objects.get(hostname="Host to delete 1") + assert hm.deleted is False + assert hm.deleted_counter == 0 + assert hm.last_deleted is None + assert hm.automated_counter == 42 + + +@pytest.mark.django_db +def test_soft_restore(): + current_time = now() + HostMetric.objects.create(hostname="Host 1", last_automation=current_time, deleted=True) + HostMetric.objects.create(hostname="Host 2", last_automation=current_time, deleted=True, last_deleted=current_time) + HostMetric.objects.create(hostname="Host 3", last_automation=current_time, deleted=False, last_deleted=current_time) + HostMetric.objects.all().update(automated_counter=42, deleted_counter=10) + + # 1. deleted, last_deleted not null + for hm in HostMetric.objects.all(): + for _ in range(3): + hm.soft_restore() + assert hm.deleted is False + assert hm.automated_counter == 42 and hm.deleted_counter == 10 + if hm.hostname == "Host 1": + assert hm.last_deleted is None + else: + assert hm.last_deleted == current_time From bfb3044f29251c2623b7f9df705844129ed2ecc8 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Fri, 10 Feb 2023 10:29:28 +0100 Subject: [PATCH 4/8] HostMetric compliance computation --- awx/main/managers.py | 5 +++++ awx/main/models/inventory.py | 5 ++++- awx/main/utils/licensing.py | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/awx/main/managers.py b/awx/main/managers.py index 32d6ed7f5b..b674049b0e 100644 --- a/awx/main/managers.py +++ b/awx/main/managers.py @@ -79,6 +79,11 @@ class HostManager(models.Manager): return qs +class HostMetricActiveManager(models.Manager): + def get_queryset(self): + return super().get_queryset().filter(deleted=False) + + def get_ig_ig_mapping(ig_instance_mapping, instance_ig_mapping): # Create IG mapping by union of all groups their instances are members of ig_ig_mapping = {} diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 2611f83a6a..2d1f07529a 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -32,7 +32,7 @@ from awx.main.fields import ( SmartFilterField, OrderedManyToManyField, ) -from awx.main.managers import HostManager +from awx.main.managers import HostManager, HostMetricActiveManager from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, prevent_search, accepts_json from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate @@ -831,6 +831,9 @@ class HostMetric(models.Model): ) used_in_inventories = models.BigIntegerField(null=True, help_text=_('How many inventories contain this host')) + objects = models.Manager() + active_objects = HostMetricActiveManager() + def get_absolute_url(self, request=None): return reverse('api:host_metric_detail', kwargs={'pk': self.pk}, request=request) diff --git a/awx/main/utils/licensing.py b/awx/main/utils/licensing.py index bec953f822..3bc5e174e5 100644 --- a/awx/main/utils/licensing.py +++ b/awx/main/utils/licensing.py @@ -382,8 +382,8 @@ class Licenser(object): current_instances = Host.objects.active_count() license_date = int(attrs.get('license_date', 0) or 0) - automated_instances = HostMetric.objects.count() - first_host = HostMetric.objects.only('first_automation').order_by('first_automation').first() + automated_instances = HostMetric.active_objects.count() + first_host = HostMetric.active_objects.only('first_automation').order_by('first_automation').first() if first_host: automated_since = int(first_host.first_automation.timestamp()) else: From c9ced6988a095f793baafd4eedc01598064704e6 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Mon, 13 Feb 2023 13:54:11 +0100 Subject: [PATCH 5/8] HostMetricSummaryMonthly API and Migrations --- awx/api/filters.py | 4 +++ awx/api/serializers.py | 8 +++++ awx/api/urls/urls.py | 2 ++ awx/api/views/__init__.py | 34 ++++++++++++++++++- awx/main/access.py | 28 +++++++++++++++ .../0176_hostmetricsummarymonthly.py | 33 ++++++++++++++++++ awx/main/models/__init__.py | 1 + awx/main/models/inventory.py | 15 +++++++- 8 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 awx/main/migrations/0176_hostmetricsummarymonthly.py diff --git a/awx/api/filters.py b/awx/api/filters.py index a40006d670..e2d0d0ba71 100644 --- a/awx/api/filters.py +++ b/awx/api/filters.py @@ -385,6 +385,10 @@ class FieldLookupBackend(BaseFilterBackend): raise ParseError(json.dumps(e.messages, ensure_ascii=False)) +class HostMetricSummaryMonthlyFieldLookupBackend(FieldLookupBackend): + RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by', 'search', 'type', 'past_months', 'count_disabled', 'no_truncate', 'limit') + + class OrderByBackend(BaseFilterBackend): """ Filter to apply ordering based on query string parameters. diff --git a/awx/api/serializers.py b/awx/api/serializers.py index d44744c44f..b0bcdeea63 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -56,6 +56,7 @@ from awx.main.models import ( Group, Host, HostMetric, + HostMetricSummaryMonthly, Instance, InstanceGroup, InstanceLink, @@ -5022,6 +5023,13 @@ class HostMetricSerializer(BaseSerializer): ) +class HostMetricSummaryMonthlySerializer(BaseSerializer): + class Meta: + model = HostMetricSummaryMonthly + read_only_fields = ("id", "date", "license_consumed", "license_capacity", "hosts_added", "hosts_deleted", "indirectly_managed_hosts") + fields = read_only_fields + + class InstanceGroupSerializer(BaseSerializer): show_capabilities = ['edit', 'delete'] capacity = serializers.SerializerMethodField() diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index 26857f1106..db912b71cb 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -30,6 +30,7 @@ from awx.api.views import ( OAuth2TokenList, ApplicationOAuth2TokenList, OAuth2ApplicationDetail, + HostMetricSummaryMonthlyList, ) from awx.api.views.mesh_visualizer import MeshVisualizer @@ -113,6 +114,7 @@ v2_urls = [ re_path(r'^inventories/', include(inventory_urls)), re_path(r'^hosts/', include(host_urls)), re_path(r'^host_metrics/', include(host_metric_urls)), + re_path(r'^host_metric_summary_monthly/$', HostMetricSummaryMonthlyList.as_view(), name='host_metric_summary_monthly_list'), re_path(r'^groups/', include(group_urls)), re_path(r'^inventory_sources/', include(inventory_source_urls)), re_path(r'^inventory_updates/', include(inventory_update_urls)), diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 4c817f1255..b31ffa4808 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -3,6 +3,7 @@ # Python import dateutil +import datetime import functools import html import itertools @@ -17,7 +18,6 @@ from collections import OrderedDict from urllib3.exceptions import ConnectTimeoutError - # Django from django.conf import settings from django.core.exceptions import FieldError, ObjectDoesNotExist @@ -122,6 +122,7 @@ from awx.api.views.mixin import ( UnifiedJobDeletionMixin, NoTruncateMixin, ) +from awx.api.filters import HostMetricSummaryMonthlyFieldLookupBackend from awx.api.pagination import UnifiedJobEventPagination from awx.main.utils import set_environ @@ -1551,6 +1552,37 @@ class HostMetricDetail(RetrieveDestroyAPIView): return Response(status=status.HTTP_204_NO_CONTENT) +class HostMetricSummaryMonthlyList(ListAPIView): + name = _("Host Metrics Summary Monthly") + model = models.HostMetricSummaryMonthly + permission_classes = (IsSystemAdminOrAuditor,) + serializer_class = serializers.HostMetricSummaryMonthlySerializer + search_fields = ('date',) + filter_backends = [HostMetricSummaryMonthlyFieldLookupBackend] + + def get_queryset(self): + queryset = super().get_queryset() + past_months = self.request.query_params.get('past_months', None) + date_from = self._get_date_from(past_months) + + queryset = queryset.filter(date__gte=date_from) + return queryset + + @staticmethod + def _get_date_from(past_months, default=12, maximum=36): + try: + months_ago = int(past_months or default) + except ValueError: + months_ago = default + months_ago = min(months_ago, maximum) + months_ago = max(months_ago, 1) + + date_from = datetime.date.today() - dateutil.relativedelta.relativedelta(months=months_ago) + # Set to beginning of the month + date_from = date_from.replace(day=1).isoformat() + return date_from + + class HostList(HostRelatedSearchMixin, ListCreateAPIView): always_allow_superuser = False model = models.Host diff --git a/awx/main/access.py b/awx/main/access.py index 3610079ae4..246e590cd2 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -38,6 +38,7 @@ from awx.main.models import ( Group, Host, HostMetric, + HostMetricSummaryMonthly, Instance, InstanceGroup, Inventory, @@ -890,6 +891,33 @@ class HostMetricAccess(BaseAccess): return bool(self.user.is_superuser or (obj and obj.user == self.user)) +class HostMetricSummaryMonthlyAccess(BaseAccess): + """ + - I can see host metrics when I'm a super user or system auditor. + """ + + model = HostMetricSummaryMonthly + + def get_queryset(self): + if self.user.is_superuser or self.user.is_system_auditor: + qs = self.model.objects.all() + else: + qs = self.filtered_queryset() + return qs + + def can_read(self, obj): + return bool(self.user.is_superuser or self.user.is_system_auditor or (obj and obj.user == self.user)) + + def can_add(self, data): + return False # There is no API endpoint to POST new settings. + + def can_change(self, obj, data): + return False + + def can_delete(self, obj): + return False + + class InventoryAccess(BaseAccess): """ I can see inventory when: diff --git a/awx/main/migrations/0176_hostmetricsummarymonthly.py b/awx/main/migrations/0176_hostmetricsummarymonthly.py new file mode 100644 index 0000000000..735f46f0d6 --- /dev/null +++ b/awx/main/migrations/0176_hostmetricsummarymonthly.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.16 on 2023-02-10 12:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('main', '0175_add_hostmetric_fields'), + ] + + operations = [ + migrations.CreateModel( + name='HostMetricSummaryMonthly', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date', models.DateField(unique=True)), + ('license_consumed', models.BigIntegerField(default=0, help_text='How much unique hosts are consumed from the license')), + ('license_capacity', models.BigIntegerField(default=0, help_text="'License capacity as max. number of unique hosts")), + ( + 'hosts_added', + models.BigIntegerField(default=0, help_text='How many hosts were added in the associated month, consuming more license capacity'), + ), + ( + 'hosts_deleted', + models.BigIntegerField(default=0, help_text='How many hosts were deleted in the associated month, freeing the license capacity'), + ), + ( + 'indirectly_managed_hosts', + models.BigIntegerField(default=0, help_text='Manually entered number indirectly managed hosts for a certain month'), + ), + ], + ), + ] diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index ed49b98083..8a608aeead 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -16,6 +16,7 @@ from awx.main.models.inventory import ( # noqa Group, Host, HostMetric, + HostMetricSummaryMonthly, Inventory, InventorySource, InventoryUpdate, diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 2d1f07529a..00c5cb576d 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -53,7 +53,7 @@ from awx.main.utils.execution_environments import to_container_path from awx.main.utils.licensing import server_product_name -__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'SmartInventoryMembership'] +__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'SmartInventoryMembership', 'HostMetric', 'HostMetricSummaryMonthly'] logger = logging.getLogger('awx.main.models.inventory') @@ -850,6 +850,19 @@ class HostMetric(models.Model): self.save() +class HostMetricSummaryMonthly(models.Model): + """ + HostMetric summaries computed by scheduled task monthly + """ + + date = models.DateField(unique=True) + license_consumed = models.BigIntegerField(default=0, help_text=_("How much unique hosts are consumed from the license")) + license_capacity = models.BigIntegerField(default=0, help_text=_("'License capacity as max. number of unique hosts")) + hosts_added = models.BigIntegerField(default=0, help_text=_("How many hosts were added in the associated month, consuming more license capacity")) + hosts_deleted = models.BigIntegerField(default=0, help_text=_("How many hosts were deleted in the associated month, freeing the license capacity")) + indirectly_managed_hosts = models.BigIntegerField(default=0, help_text=("Manually entered number indirectly managed hosts for a certain month")) + + class InventorySourceOptions(BaseModel): """ Common fields for InventorySource and InventoryUpdate. From 18e312c884877804f16dd8870ebf42c5cd81086a Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Wed, 15 Feb 2023 16:49:43 +0100 Subject: [PATCH 6/8] HostMetric review,migration,permissions --- awx/api/urls/host_metric.py | 2 +- awx/api/views/__init__.py | 7 ++- awx/api/views/root.py | 2 + awx/main/access.py | 57 ------------------- .../migrations/0175_add_hostmetric_fields.py | 6 +- .../0176_hostmetricsummarymonthly.py | 6 +- awx/main/models/events.py | 4 +- awx/main/models/inventory.py | 16 +++--- 8 files changed, 24 insertions(+), 76 deletions(-) diff --git a/awx/api/urls/host_metric.py b/awx/api/urls/host_metric.py index d464fb82c5..a5e43fefbc 100644 --- a/awx/api/urls/host_metric.py +++ b/awx/api/urls/host_metric.py @@ -5,6 +5,6 @@ from django.urls import re_path from awx.api.views import HostMetricList, HostMetricDetail -urls = [re_path(r'$^', HostMetricList.as_view(), name='host_metric_list'), re_path(r'^(?P[0-9]+)/$', HostMetricDetail.as_view(), name='host_metric_detail')] +urls = [re_path(r'^$', HostMetricList.as_view(), name='host_metric_list'), re_path(r'^(?P[0-9]+)/$', HostMetricDetail.as_view(), name='host_metric_detail')] __all__ = ['urls'] diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index b31ffa4808..09e7eaf37b 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -1539,6 +1539,9 @@ class HostMetricList(ListAPIView): permission_classes = (IsSystemAdminOrAuditor,) search_fields = ('hostname', 'deleted') + def get_queryset(self): + return self.model.objects.all() + class HostMetricDetail(RetrieveDestroyAPIView): name = _("Host Metric Detail") @@ -1555,13 +1558,13 @@ class HostMetricDetail(RetrieveDestroyAPIView): class HostMetricSummaryMonthlyList(ListAPIView): name = _("Host Metrics Summary Monthly") model = models.HostMetricSummaryMonthly - permission_classes = (IsSystemAdminOrAuditor,) serializer_class = serializers.HostMetricSummaryMonthlySerializer + permission_classes = (IsSystemAdminOrAuditor,) search_fields = ('date',) filter_backends = [HostMetricSummaryMonthlyFieldLookupBackend] def get_queryset(self): - queryset = super().get_queryset() + queryset = self.model.objects.all() past_months = self.request.query_params.get('past_months', None) date_from = self._get_date_from(past_months) diff --git a/awx/api/views/root.py b/awx/api/views/root.py index 01a742bf74..f5d093d2d2 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -102,6 +102,8 @@ class ApiVersionRootView(APIView): data['inventory_updates'] = reverse('api:inventory_update_list', request=request) data['groups'] = reverse('api:group_list', request=request) data['hosts'] = reverse('api:host_list', request=request) + data['host_metrics'] = reverse('api:host_metric_list', request=request) + data['host_metric_summary_monthly'] = reverse('api:host_metric_summary_monthly_list', request=request) data['job_templates'] = reverse('api:job_template_list', request=request) data['jobs'] = reverse('api:job_list', request=request) data['ad_hoc_commands'] = reverse('api:ad_hoc_command_list', request=request) diff --git a/awx/main/access.py b/awx/main/access.py index 246e590cd2..4d6bdf2c55 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -37,8 +37,6 @@ from awx.main.models import ( ExecutionEnvironment, Group, Host, - HostMetric, - HostMetricSummaryMonthly, Instance, InstanceGroup, Inventory, @@ -863,61 +861,6 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess): return super(OrganizationAccess, self).can_attach(obj, sub_obj, relationship, *args, **kwargs) -class HostMetricAccess(BaseAccess): - """ - - I can see host metrics when I'm a super user or system auditor. - - I can delete host metrics when I'm a super user. - """ - - model = HostMetric - - def get_queryset(self): - if self.user.is_superuser or self.user.is_system_auditor: - qs = self.model.objects.all() - else: - qs = self.filtered_queryset() - return qs - - def can_read(self, obj): - return bool(self.user.is_superuser or self.user.is_system_auditor or (obj and obj.user == self.user)) - - def can_add(self, data): - return False # There is no API endpoint to POST new settings. - - def can_change(self, obj, data): - return False - - def can_delete(self, obj): - return bool(self.user.is_superuser or (obj and obj.user == self.user)) - - -class HostMetricSummaryMonthlyAccess(BaseAccess): - """ - - I can see host metrics when I'm a super user or system auditor. - """ - - model = HostMetricSummaryMonthly - - def get_queryset(self): - if self.user.is_superuser or self.user.is_system_auditor: - qs = self.model.objects.all() - else: - qs = self.filtered_queryset() - return qs - - def can_read(self, obj): - return bool(self.user.is_superuser or self.user.is_system_auditor or (obj and obj.user == self.user)) - - def can_add(self, data): - return False # There is no API endpoint to POST new settings. - - def can_change(self, obj, data): - return False - - def can_delete(self, obj): - return False - - class InventoryAccess(BaseAccess): """ I can see inventory when: diff --git a/awx/main/migrations/0175_add_hostmetric_fields.py b/awx/main/migrations/0175_add_hostmetric_fields.py index d273a6b6ea..ee91b01fbb 100644 --- a/awx/main/migrations/0175_add_hostmetric_fields.py +++ b/awx/main/migrations/0175_add_hostmetric_fields.py @@ -18,12 +18,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='hostmetric', name='automated_counter', - field=models.BigIntegerField(default=0, help_text='How many times was the host automated'), + field=models.IntegerField(default=0, help_text='How many times was the host automated'), ), migrations.AddField( model_name='hostmetric', name='deleted_counter', - field=models.BigIntegerField(default=0, help_text='How many times was the host deleted'), + field=models.IntegerField(default=0, help_text='How many times was the host deleted'), ), migrations.AddField( model_name='hostmetric', @@ -35,7 +35,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='hostmetric', name='used_in_inventories', - field=models.BigIntegerField(null=True, help_text='How many inventories contain this host'), + field=models.IntegerField(null=True, help_text='How many inventories contain this host'), ), migrations.AddField( model_name='hostmetric', name='id', field=models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID') diff --git a/awx/main/migrations/0176_hostmetricsummarymonthly.py b/awx/main/migrations/0176_hostmetricsummarymonthly.py index 735f46f0d6..7631363a4c 100644 --- a/awx/main/migrations/0176_hostmetricsummarymonthly.py +++ b/awx/main/migrations/0176_hostmetricsummarymonthly.py @@ -18,15 +18,15 @@ class Migration(migrations.Migration): ('license_capacity', models.BigIntegerField(default=0, help_text="'License capacity as max. number of unique hosts")), ( 'hosts_added', - models.BigIntegerField(default=0, help_text='How many hosts were added in the associated month, consuming more license capacity'), + models.IntegerField(default=0, help_text='How many hosts were added in the associated month, consuming more license capacity'), ), ( 'hosts_deleted', - models.BigIntegerField(default=0, help_text='How many hosts were deleted in the associated month, freeing the license capacity'), + models.IntegerField(default=0, help_text='How many hosts were deleted in the associated month, freeing the license capacity'), ), ( 'indirectly_managed_hosts', - models.BigIntegerField(default=0, help_text='Manually entered number indirectly managed hosts for a certain month'), + models.IntegerField(default=0, help_text='Manually entered number indirectly managed hosts for a certain month'), ), ], ), diff --git a/awx/main/models/events.py b/awx/main/models/events.py index 0ba07185f4..2d6dee6f61 100644 --- a/awx/main/models/events.py +++ b/awx/main/models/events.py @@ -6,7 +6,7 @@ from collections import defaultdict from django.conf import settings from django.core.exceptions import ObjectDoesNotExist -from django.db import connection, models, DatabaseError +from django.db import models, DatabaseError from django.utils.dateparse import parse_datetime from django.utils.text import Truncator from django.utils.timezone import utc, now @@ -585,7 +585,7 @@ class JobEvent(BasePlaybookEvent): # bulk-create current_time = now() HostMetric.objects.bulk_create( - [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=1000 + [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=100 ) # bulk-update batch_start, batch_size = 0, 1000 diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 00c5cb576d..a4e1aac4ae 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -824,12 +824,12 @@ class HostMetric(models.Model): first_automation = models.DateTimeField(auto_now_add=True, null=False, db_index=True, help_text=_('When the host was first automated against')) last_automation = models.DateTimeField(db_index=True, help_text=_('When the host was last automated against')) last_deleted = models.DateTimeField(null=True, db_index=True, help_text=_('When the host was last deleted')) - automated_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host automated')) - deleted_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host deleted')) + automated_counter = models.IntegerField(default=0, help_text=_('How many times was the host automated')) + deleted_counter = models.IntegerField(default=0, help_text=_('How many times was the host deleted')) deleted = models.BooleanField( default=False, help_text=_('Boolean flag saying whether the host is deleted and therefore not counted into the subscription consumption') ) - used_in_inventories = models.BigIntegerField(null=True, help_text=_('How many inventories contain this host')) + used_in_inventories = models.IntegerField(null=True, help_text=_('How many inventories contain this host')) objects = models.Manager() active_objects = HostMetricActiveManager() @@ -842,12 +842,12 @@ class HostMetric(models.Model): self.deleted_counter = (self.deleted_counter or 0) + 1 self.last_deleted = now() self.deleted = True - self.save() + self.save(update_fields=['deleted', 'deleted_counter', 'last_deleted']) def soft_restore(self): if self.deleted: self.deleted = False - self.save() + self.save(update_fields=['deleted']) class HostMetricSummaryMonthly(models.Model): @@ -858,9 +858,9 @@ class HostMetricSummaryMonthly(models.Model): date = models.DateField(unique=True) license_consumed = models.BigIntegerField(default=0, help_text=_("How much unique hosts are consumed from the license")) license_capacity = models.BigIntegerField(default=0, help_text=_("'License capacity as max. number of unique hosts")) - hosts_added = models.BigIntegerField(default=0, help_text=_("How many hosts were added in the associated month, consuming more license capacity")) - hosts_deleted = models.BigIntegerField(default=0, help_text=_("How many hosts were deleted in the associated month, freeing the license capacity")) - indirectly_managed_hosts = models.BigIntegerField(default=0, help_text=("Manually entered number indirectly managed hosts for a certain month")) + hosts_added = models.IntegerField(default=0, help_text=_("How many hosts were added in the associated month, consuming more license capacity")) + hosts_deleted = models.IntegerField(default=0, help_text=_("How many hosts were deleted in the associated month, freeing the license capacity")) + indirectly_managed_hosts = models.IntegerField(default=0, help_text=("Manually entered number indirectly managed hosts for a certain month")) class InventorySourceOptions(BaseModel): From c308c90d323069e22a32ca45d591aa289f59cfa2 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 23 Feb 2023 14:41:53 -0500 Subject: [PATCH 7/8] Remove custom API filters and suggest solution via templates --- awx/api/filters.py | 4 ---- awx/api/templates/api/host_metric_detail.md | 18 +++++++++++++++ .../api/host_metric_summary_monthly_list.md | 12 ++++++++++ awx/api/views/__init__.py | 23 +------------------ 4 files changed, 31 insertions(+), 26 deletions(-) create mode 100644 awx/api/templates/api/host_metric_detail.md create mode 100644 awx/api/templates/api/host_metric_summary_monthly_list.md diff --git a/awx/api/filters.py b/awx/api/filters.py index e2d0d0ba71..a40006d670 100644 --- a/awx/api/filters.py +++ b/awx/api/filters.py @@ -385,10 +385,6 @@ class FieldLookupBackend(BaseFilterBackend): raise ParseError(json.dumps(e.messages, ensure_ascii=False)) -class HostMetricSummaryMonthlyFieldLookupBackend(FieldLookupBackend): - RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by', 'search', 'type', 'past_months', 'count_disabled', 'no_truncate', 'limit') - - class OrderByBackend(BaseFilterBackend): """ Filter to apply ordering based on query string parameters. diff --git a/awx/api/templates/api/host_metric_detail.md b/awx/api/templates/api/host_metric_detail.md new file mode 100644 index 0000000000..0a59a1b410 --- /dev/null +++ b/awx/api/templates/api/host_metric_detail.md @@ -0,0 +1,18 @@ +{% ifmeth GET %} +# Retrieve {{ model_verbose_name|title|anora }}: + +Make GET request to this resource to retrieve a single {{ model_verbose_name }} +record containing the following fields: + +{% include "api/_result_fields_common.md" %} +{% endifmeth %} + +{% ifmeth DELETE %} +# Delete {{ model_verbose_name|title|anora }}: + +Make a DELETE request to this resource to soft-delete this {{ model_verbose_name }}. + +A soft deletion will mark the `deleted` field as true and exclude the host +metric from license calculations. +This may be undone later if the same hostname is automated again afterwards. +{% endifmeth %} diff --git a/awx/api/templates/api/host_metric_summary_monthly_list.md b/awx/api/templates/api/host_metric_summary_monthly_list.md new file mode 100644 index 0000000000..953b1827a6 --- /dev/null +++ b/awx/api/templates/api/host_metric_summary_monthly_list.md @@ -0,0 +1,12 @@ +# Intended Use Case + +To get summaries from a certain day or earlier, you can filter this +endpoint in the following way. + + ?date__gte=2023-01-01 + +This will return summaries that were produced on that date or later. +These host metric monthly summaries should be automatically produced +by a background task that runs once each month. + +{% include "api/list_api_view.md" %} diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 09e7eaf37b..b12a1777aa 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -122,7 +122,6 @@ from awx.api.views.mixin import ( UnifiedJobDeletionMixin, NoTruncateMixin, ) -from awx.api.filters import HostMetricSummaryMonthlyFieldLookupBackend from awx.api.pagination import UnifiedJobEventPagination from awx.main.utils import set_environ @@ -1561,29 +1560,9 @@ class HostMetricSummaryMonthlyList(ListAPIView): serializer_class = serializers.HostMetricSummaryMonthlySerializer permission_classes = (IsSystemAdminOrAuditor,) search_fields = ('date',) - filter_backends = [HostMetricSummaryMonthlyFieldLookupBackend] def get_queryset(self): - queryset = self.model.objects.all() - past_months = self.request.query_params.get('past_months', None) - date_from = self._get_date_from(past_months) - - queryset = queryset.filter(date__gte=date_from) - return queryset - - @staticmethod - def _get_date_from(past_months, default=12, maximum=36): - try: - months_ago = int(past_months or default) - except ValueError: - months_ago = default - months_ago = min(months_ago, maximum) - months_ago = max(months_ago, 1) - - date_from = datetime.date.today() - dateutil.relativedelta.relativedelta(months=months_ago) - # Set to beginning of the month - date_from = date_from.replace(day=1).isoformat() - return date_from + return self.model.objects.all() class HostList(HostRelatedSearchMixin, ListCreateAPIView): From 581460e85410e37c2d8f8ada394f8faf6eebc792 Mon Sep 17 00:00:00 2001 From: Martin Slemr Date: Mon, 27 Feb 2023 11:25:09 +0100 Subject: [PATCH 8/8] HostMetric migration --- awx/api/views/__init__.py | 1 - awx/main/migrations/0175_add_hostmetric_fields.py | 2 +- awx/main/migrations/0176_hostmetricsummarymonthly.py | 2 +- awx/main/models/inventory.py | 4 ++-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index b12a1777aa..bd6bcfba04 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -3,7 +3,6 @@ # Python import dateutil -import datetime import functools import html import itertools diff --git a/awx/main/migrations/0175_add_hostmetric_fields.py b/awx/main/migrations/0175_add_hostmetric_fields.py index ee91b01fbb..75090bd678 100644 --- a/awx/main/migrations/0175_add_hostmetric_fields.py +++ b/awx/main/migrations/0175_add_hostmetric_fields.py @@ -18,7 +18,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='hostmetric', name='automated_counter', - field=models.IntegerField(default=0, help_text='How many times was the host automated'), + field=models.BigIntegerField(default=0, help_text='How many times was the host automated'), ), migrations.AddField( model_name='hostmetric', diff --git a/awx/main/migrations/0176_hostmetricsummarymonthly.py b/awx/main/migrations/0176_hostmetricsummarymonthly.py index 7631363a4c..fe482aa416 100644 --- a/awx/main/migrations/0176_hostmetricsummarymonthly.py +++ b/awx/main/migrations/0176_hostmetricsummarymonthly.py @@ -14,7 +14,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('date', models.DateField(unique=True)), - ('license_consumed', models.BigIntegerField(default=0, help_text='How much unique hosts are consumed from the license')), + ('license_consumed', models.BigIntegerField(default=0, help_text='How many unique hosts are consumed from the license')), ('license_capacity', models.BigIntegerField(default=0, help_text="'License capacity as max. number of unique hosts")), ( 'hosts_added', diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index a4e1aac4ae..89905b8411 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -824,7 +824,7 @@ class HostMetric(models.Model): first_automation = models.DateTimeField(auto_now_add=True, null=False, db_index=True, help_text=_('When the host was first automated against')) last_automation = models.DateTimeField(db_index=True, help_text=_('When the host was last automated against')) last_deleted = models.DateTimeField(null=True, db_index=True, help_text=_('When the host was last deleted')) - automated_counter = models.IntegerField(default=0, help_text=_('How many times was the host automated')) + automated_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host automated')) deleted_counter = models.IntegerField(default=0, help_text=_('How many times was the host deleted')) deleted = models.BooleanField( default=False, help_text=_('Boolean flag saying whether the host is deleted and therefore not counted into the subscription consumption') @@ -856,7 +856,7 @@ class HostMetricSummaryMonthly(models.Model): """ date = models.DateField(unique=True) - license_consumed = models.BigIntegerField(default=0, help_text=_("How much unique hosts are consumed from the license")) + license_consumed = models.BigIntegerField(default=0, help_text=_("How many unique hosts are consumed from the license")) license_capacity = models.BigIntegerField(default=0, help_text=_("'License capacity as max. number of unique hosts")) hosts_added = models.IntegerField(default=0, help_text=_("How many hosts were added in the associated month, consuming more license capacity")) hosts_deleted = models.IntegerField(default=0, help_text=_("How many hosts were deleted in the associated month, freeing the license capacity"))