Swap in-memory logger to write stdout in post_save_hook

This commit makes the needed changes to inventory update
post_save_hook logic so that the historic log lines that
inventory updates write will be written to stdout,
but this hack bypasses the ansible-runner verbose event
logic and dispatches verbose events directly.

Fix the venv application with the ansible-inventory system
(note: much of this is undone in a later commit)

Deal with some minor test updates for
the ansible-inventory interface changes

awx/main/tasks.py

@@ -2485,15 +2485,11 @@ class RunInventoryUpdate(BaseTask):
             return injector.build_private_data(inventory_update, private_data_dir)
 
     def build_env(self, inventory_update, private_data_dir, isolated, private_data_files=None):
-        """Build environment dictionary for inventory import.
+        """Build environment dictionary for ansible-inventory.
 
-        This used to be the mechanism by which any data that needs to be passed
-        to the inventory update script is set up. In particular, this is how
-        inventory update is aware of its proper credentials.
-
-        Most environment injection is now accomplished by the credential
-        injectors. The primary purpose this still serves is to
-        still point to the inventory update INI or config file.
+        Most environment variables related to credentials or configuration
+        are accomplished by the inventory source injectors (in this method)
+        or custom credential type injectors (in main run method).
         """
         env = super(RunInventoryUpdate, self).build_env(inventory_update,
                                                         private_data_dir,
@@ -2501,8 +2497,10 @@ class RunInventoryUpdate(BaseTask):
                                                         private_data_files=private_data_files)
         if private_data_files is None:
             private_data_files = {}
-        self.add_awx_venv(env)
-        # Pass inventory source ID to inventory script.
+        self.add_ansible_venv(settings.ANSIBLE_VENV_PATH, env)
+
+        # Legacy environment variables, were used as signal to awx-manage command
+        # now they are provided in case some scripts may be relying on them
         env['INVENTORY_SOURCE_ID'] = str(inventory_update.inventory_source_id)
         env['INVENTORY_UPDATE_ID'] = str(inventory_update.pk)
         env.update(STANDARD_INVENTORY_UPDATE_ENV)
@@ -2568,14 +2566,21 @@ class RunInventoryUpdate(BaseTask):
         args = ['ansible-inventory', '--list', '--export']
 
         # Add arguments for the source inventory file/script/thing
+        source_location = self.pseudo_build_inventory(inventory_update, private_data_dir)
         args.append('-i')
-        args.append(self.pseudo_build_inventory(inventory_update, private_data_dir))
+        args.append(source_location)
 
         args.append('--output')
         args.append(os.path.join(private_data_dir, 'artifacts', 'output.json'))
 
+        if os.path.isdir(source_location):
+            playbook_dir = source_location
+        else:
+            playbook_dir = os.path.dirname(source_location)
+        args.extend(['--playbook-dir', playbook_dir])
+
         if inventory_update.verbosity:
-            args.append('-' + 'v' * inventory_update.verbosity)
+            args.append('-' + 'v' * min(5, inventory_update.verbosity * 2 + 1))
 
         return args
 
@@ -2713,19 +2718,82 @@ class RunInventoryUpdate(BaseTask):
         if inventory_update.verbosity:
             options['verbosity'] = inventory_update.verbosity
 
+        # Mock ansible-runner events
+        class CallbackHandler(logging.Handler):
+            def __init__(self, event_handler, cancel_callback, job_timeout, verbosity,
+                         counter=0, **kwargs):
+                self.event_handler = event_handler
+                self.cancel_callback = cancel_callback
+                self.job_timeout = job_timeout
+                self.job_start = time.time()
+                self.last_check = self.job_start
+                # TODO: we do not have events from the ansible-inventory process
+                # so there is no way to know initial counter of start line
+                self.counter = counter
+                self.skip_level = [logging.WARNING, logging.INFO, logging.DEBUG, 0][verbosity]
+                self._start_line = 0
+                super(CallbackHandler, self).__init__(**kwargs)
+
+            def emit(self, record):
+                this_time = time.time()
+                if this_time - self.last_check > 0.5:
+                    self.last_check = this_time
+                    if self.cancel_callback():
+                        raise RuntimeError('Inventory update has been canceled')
+                if self.job_timeout and ((this_time - self.job_start) > self.job_timeout):
+                    raise RuntimeError('Inventory update has timed out')
+
+                # skip logging for low severity logs
+                if record.levelno < self.skip_level:
+                    return
+
+                self.counter += 1
+                msg = self.format(record)
+                n_lines = msg.count('\n')
+                dispatch_data = dict(
+                    created=now().isoformat(),
+                    event='verbose',
+                    counter=self.counter,
+                    stdout=msg + '\n',
+                    start_line=self._start_line,
+                    end_line=self._start_line + n_lines
+                )
+                self._start_line += n_lines + 1
+
+                self.event_handler(dispatch_data)
+
+        handler = CallbackHandler(
+            self.event_handler, self.cancel_callback,
+            verbosity=inventory_update.verbosity,
+            job_timeout=self.get_instance_timeout(self.instance),
+            counter=self.event_ct
+        )
+        inv_logger = logging.getLogger('awx.main.commands.inventory_import')
+        handler.formatter = inv_logger.handlers[0].formatter
+        inv_logger.handlers[0] = handler
+
         from awx.main.management.commands.inventory_import import Command as InventoryImportCommand
         cmd = InventoryImportCommand()
-        save_status, tb, exc = cmd.perform_update(options, data, inventory_update)
+        try:
+            save_status, tb, exc = cmd.perform_update(options, data, inventory_update)
+        except Exception as raw_exc:
+            # Ignore license errors specifically
+            if 'Host limit for organization' not in str(exc) and 'License' not in str(exc):
+                raise raw_exc
 
         model_updates = {}
-        if save_status != inventory_update.status:
+        if save_status != status:
             model_updates['status'] = save_status
         if tb:
             model_updates['result_traceback'] = tb
 
         if model_updates:
-            logger.debug('{} saw problems saving to database.'.format(inventory_update.log_format))
-            model_updates['job_explanation'] = 'Update failed to save all changes to database properly'
+            logger.info('{} had problems saving to database with {}'.format(
+                inventory_update.log_format, ', '.join(list(model_updates.keys()))
+            ))
+            model_updates['job_explanation'] = 'Update failed to save all changes to database properly.'
+            if exc:
+                model_updates['job_explanation'] += ' {}'.format(exc)
             self.update_model(inventory_update.pk, **model_updates)