Protect cluster nodes after an upgrade

* Modify instance model to container a version number for the node
* Update that version number during the heartbeat
* If during a heartbeat any of the nodes are of a newer version then
  shutdown the current node.

The idea behind this is that if all nodes were upgraded at the same
time then at the moment of the healthcheck they should all be at the
newer version. Otherwise we put the system in a state where it can
receive the upgrade but stay down until that happens. During setup
playbook run the services will be fully restarted.

awx/main/utils/reload.py

@@ -14,12 +14,12 @@ from celery import current_app
 logger = logging.getLogger('awx.main.utils.reload')
 
 
-def _uwsgi_reload():
+def _uwsgi_fifo_command(uwsgi_command):
     # http://uwsgi-docs.readthedocs.io/en/latest/MasterFIFO.html#available-commands
     logger.warn('Initiating uWSGI chain reload of server')
-    TRIGGER_CHAIN_RELOAD = 'c'
+    TRIGGER_COMMAND = uwsgi_command
     with open(settings.UWSGI_FIFO_LOCATION, 'w') as awxfifo:
-        awxfifo.write(TRIGGER_CHAIN_RELOAD)
+        awxfifo.write(TRIGGER_COMMAND)
 
 
 def _reset_celery_thread_pool():
@@ -29,7 +29,7 @@ def _reset_celery_thread_pool():
                           destination=['celery@{}'.format(settings.CLUSTER_HOST_ID)], reply=False)
 
 
-def _supervisor_service_restart(service_internal_names):
+def _supervisor_service_command(service_internal_names, command):
     '''
     Service internal name options:
      - beat - celery - callback - channels - uwsgi - daphne
@@ -46,7 +46,7 @@ def _supervisor_service_restart(service_internal_names):
     for n in service_internal_names:
         if n in name_translation_dict:
             programs.append('{}:{}'.format(group_name, name_translation_dict[n]))
-    args.extend(['restart'])
+    args.extend([command])
     args.extend(programs)
     logger.debug('Issuing command to restart services, args={}'.format(args))
     subprocess.Popen(args)
@@ -55,14 +55,18 @@ def _supervisor_service_restart(service_internal_names):
 def restart_local_services(service_internal_names):
     logger.warn('Restarting services {} on this node in response to user action'.format(service_internal_names))
     if 'uwsgi' in service_internal_names:
-        _uwsgi_reload()
+        _uwsgi_fifo_command(uwsgi_command='c')
         service_internal_names.remove('uwsgi')
     restart_celery = False
     if 'celery' in service_internal_names:
         restart_celery = True
         service_internal_names.remove('celery')
-    _supervisor_service_restart(service_internal_names)
+    _supervisor_service_command(service_internal_names, command='restart')
     if restart_celery:
         # Celery restarted last because this probably includes current process
         _reset_celery_thread_pool()
 
+
+def stop_local_services(service_internal_names):
+    logger.warn('Stopping services {} on this node in response to user action'.format(service_internal_names))
+    _supervisor_service_command(service_internal_names, command='stop')