decide the node a job will run early

* Deciding the Instance that a Job runs on at celery task run-time makes
it hard to evenly distribute tasks among Instnaces. Instead, the task
manager will look at the world of running jobs and choose an instance
node to run on; applying a deterministic job distribution algo.

awx/main/models/ha.py

@@ -92,6 +92,10 @@ class Instance(BaseModel):
         return sum(x.task_impact for x in UnifiedJob.objects.filter(execution_node=self.hostname,
                                                                     status__in=('running', 'waiting')))
 
+    @property
+    def remaining_capacity(self):
+        return self.capacity - self.consumed_capacity
+
     @property
     def role(self):
         # NOTE: TODO: Likely to repurpose this once standalone ramparts are a thing
@@ -187,6 +191,25 @@ class InstanceGroup(BaseModel, RelatedJobsMixin):
         validate_queuename(self.name)
         return self.name
 
+    def fit_task_to_most_remaining_capacity_instance(self, task):
+        instance_most_capacity = None
+        for i in self.instances.order_by('hostname'):
+            if i.remaining_capacity >= task.task_impact and \
+                    (instance_most_capacity is None or
+                     i.remaining_capacity > instance_most_capacity.remaining_capacity):
+                instance_most_capacity = i
+        return instance_most_capacity
+
+    def find_largest_idle_instance(self):
+        largest_instance = None
+        for i in self.instances.order_by('hostname'):
+            if i.jobs_running == 0:
+                if largest_instance is None:
+                    largest_instance = i
+                elif i.capacity > largest_instance.capacity:
+                    largest_instance = i
+        return largest_instance
+
 
 class TowerScheduleState(SingletonModel):
     schedule_last_run = models.DateTimeField(auto_now_add=True)

awx/main/models/unified_jobs.py

@@ -1228,9 +1228,9 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
             raise RuntimeError("Expected celery_task_id to be set on model.")
         kwargs['task_id'] = self.celery_task_id
         task_class = self._get_task_class()
+        args = [self.pk]
         from awx.main.models.ha import InstanceGroup
         ig = InstanceGroup.objects.get(name=queue)
-        args = [self.pk]
         if ig.controller_id:
             if self.supports_isolation():  # case of jobs and ad hoc commands
                 isolated_instance = ig.instances.order_by('-capacity').first()