zware/awx
1
0
Fork 0
mirror of https://github.com/ZwareBear/awx.git synced 2026-05-15 07:48:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1372 from chrismeyersfsu/old-celery3

Browse Source 
celery 4.x to 3.x roll back
This commit is contained in:
Chris Meyers
2018-02-27 15:26:46 -05:00
committed by GitHub
parent f9129aefba 6606a29f57
commit d551566b4d
20 changed files with 70 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
awx/__init__.py
+1 -1
View File
@@ -7,7 +7,7 @@ import sys
import warnings
from pkg_resources import get_distribution
from .celery import app as celery_app
from .celery import app as celery_app # noqa
__version__ = get_distribution('awx').version
awx/celery.py
+4 -2
View File
@@ -1,3 +1,4 @@
# Copyright (c) 2017 Ansible, Inc.
# All Rights Reserved.
@@ -5,6 +6,7 @@ from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
from django.conf import settings # noqa
try:
@@ -16,8 +18,8 @@ except ImportError: # pragma: no cover
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
app = Celery('awx')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
if __name__ == '__main__':
app.start()
awx/main/management/commands/run_callback_receiver.py
+1 -1
View File
@@ -208,7 +208,7 @@ class Command(BaseCommand):
help = 'Launch the job callback receiver'
def handle(self, *arg, **options):
with Connection(settings.CELERY_BROKER_URL) as conn:
with Connection(settings.BROKER_URL) as conn:
try:
worker = CallbackBrokerWorker(conn)
worker.run()
awx/main/models/unified_jobs.py
+4 -4
View File
@@ -28,7 +28,7 @@ from rest_framework.exceptions import ParseError
from polymorphic.models import PolymorphicModel
# Django-Celery
from django_celery_results.models import TaskResult
from djcelery.models import TaskMeta
# AWX
from awx.main.models.base import * # noqa
@@ -1093,8 +1093,8 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
def celery_task(self):
try:
if self.celery_task_id:
return TaskResult.objects.get(task_id=self.celery_task_id)
except TaskResult.DoesNotExist:
return TaskMeta.objects.get(task_id=self.celery_task_id)
except TaskMeta.DoesNotExist:
pass
def get_passwords_needed_to_start(self):
@@ -1335,7 +1335,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
cancel_fields.append('job_explanation')
self.save(update_fields=cancel_fields)
self.websocket_emit_status("canceled")
if settings.CELERY_BROKER_URL.startswith('amqp://'):
if settings.BROKER_URL.startswith('amqp://'):
self._force_cancel()
return self.cancel_flag
awx/main/queue.py
+1 -1
View File
@@ -19,7 +19,7 @@ __all__ = ['CallbackQueueDispatcher']
class CallbackQueueDispatcher(object):
def __init__(self):
self.callback_connection = getattr(settings, 'CELERY_BROKER_URL', None)
self.callback_connection = getattr(settings, 'BROKER_URL', None)
self.connection_queue = getattr(settings, 'CALLBACK_QUEUE', '')
self.connection = None
self.exchange = None
awx/main/scheduler/task_manager.py
+1 -1
View File
@@ -133,7 +133,7 @@ class TaskManager():
def get_active_tasks(self):
if not hasattr(settings, 'IGNORE_CELERY_INSPECTOR'):
app = Celery('awx')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.config_from_object('django.conf:settings')
inspector = Inspect(app=app)
active_task_queues = inspector.active()
else:
awx/main/tasks.py
+3 -11
View File
@@ -47,7 +47,6 @@ from crum import impersonate
# AWX
from awx import __version__ as awx_application_version
from awx import celery_app
from awx.main.constants import CLOUD_PROVIDERS, PRIVILEGE_ESCALATION_METHODS
from awx.main.models import * # noqa
from awx.main.models.unified_jobs import ACTIVE_STATES
@@ -208,14 +207,14 @@ def handle_ha_toplogy_changes(self):
instance = Instance.objects.me()
logger.debug("Reconfigure celeryd queues task on host {}".format(self.request.hostname))
awx_app = Celery('awx')
awx_app.config_from_object('django.conf:settings', namespace='CELERY')
awx_app.config_from_object('django.conf:settings')
instances, removed_queues, added_queues = register_celery_worker_queues(awx_app, self.request.hostname)
for instance in instances:
logger.info("Workers on tower node '{}' removed from queues {} and added to queues {}"
.format(instance.hostname, removed_queues, added_queues))
updated_routes = update_celery_worker_routes(instance, settings)
logger.info("Worker on tower node '{}' updated celery routes {} all routes are now {}"
.format(instance.hostname, updated_routes, self.app.conf.CELERY_TASK_ROUTES))
.format(instance.hostname, updated_routes, self.app.conf.CELERY_ROUTES))
@worker_ready.connect
@@ -234,7 +233,7 @@ def handle_update_celery_routes(sender=None, conf=None, **kwargs):
instance = Instance.objects.me()
added_routes = update_celery_worker_routes(instance, conf)
logger.info("Workers on tower node '{}' added routes {} all routes are now {}"
.format(instance.hostname, added_routes, conf.CELERY_TASK_ROUTES))
.format(instance.hostname, added_routes, conf.CELERY_ROUTES))
@celeryd_after_setup.connect
@@ -2359,10 +2358,3 @@ def deep_copy_model_obj(
importlib.import_module(permission_check_func[0]), permission_check_func[1]
), permission_check_func[2])
permission_check_func(creater, copy_mapping.values())
celery_app.register_task(RunJob())
celery_app.register_task(RunProjectUpdate())
celery_app.register_task(RunInventoryUpdate())
celery_app.register_task(RunAdHocCommand())
celery_app.register_task(RunSystemJob())
awx/main/tests/functional/conftest.py
+1 -1
View File
@@ -73,7 +73,7 @@ def celery_memory_broker():
Allows django signal code to execute without the need for redis
'''
settings.CELERY_BROKER_URL='memory://localhost/'
settings.BROKER_URL='memory://localhost/'
@pytest.fixture
awx/main/tests/unit/settings/test_defaults.py
+5 -5
View File
@@ -8,11 +8,11 @@ from datetime import timedelta
('admin_checks', 'awx.main.tasks.run_administrative_checks'),
('tower_scheduler', 'awx.main.tasks.awx_periodic_scheduler'),
])
def test_CELERY_BEAT_SCHEDULE(mocker, job_name, function_path):
assert job_name in settings.CELERY_BEAT_SCHEDULE
assert 'schedule' in settings.CELERY_BEAT_SCHEDULE[job_name]
assert type(settings.CELERY_BEAT_SCHEDULE[job_name]['schedule']) is timedelta
assert settings.CELERY_BEAT_SCHEDULE[job_name]['task'] == function_path
def test_CELERYBEAT_SCHEDULE(mocker, job_name, function_path):
assert job_name in settings.CELERYBEAT_SCHEDULE
assert 'schedule' in settings.CELERYBEAT_SCHEDULE[job_name]
assert type(settings.CELERYBEAT_SCHEDULE[job_name]['schedule']) is timedelta
assert settings.CELERYBEAT_SCHEDULE[job_name]['task'] == function_path
# Ensures that the function exists
mocker.patch(function_path)
awx/main/tests/unit/utils/test_ha.py
+4 -4
View File
@@ -17,7 +17,7 @@ from awx.main.utils.ha import (
@pytest.fixture
def conf():
class Conf():
CELERY_TASK_ROUTES = dict()
CELERY_ROUTES = dict()
CELERYBEAT_SCHEDULE = dict()
return Conf()
@@ -88,14 +88,14 @@ class TestUpdateCeleryWorkerRoutes():
instance.is_controller = mocker.MagicMock(return_value=is_controller)
assert update_celery_worker_routes(instance, conf) == expected_routes
assert conf.CELERY_TASK_ROUTES == expected_routes
assert conf.CELERY_ROUTES == expected_routes
def test_update_celery_worker_routes_deleted(self, mocker, conf):
instance = mocker.MagicMock()
instance.hostname = 'east-1'
instance.is_controller = mocker.MagicMock(return_value=False)
conf.CELERY_TASK_ROUTES = {'awx.main.tasks.awx_isolated_heartbeat': 'foobar'}
conf.CELERY_ROUTES = {'awx.main.tasks.awx_isolated_heartbeat': 'foobar'}
update_celery_worker_routes(instance, conf)
assert 'awx.main.tasks.awx_isolated_heartbeat' not in conf.CELERY_TASK_ROUTES
assert 'awx.main.tasks.awx_isolated_heartbeat' not in conf.CELERY_ROUTES
awx/main/utils/ha.py
+4 -4
View File
@@ -48,12 +48,12 @@ def update_celery_worker_routes(instance, conf):
if instance.is_controller():
tasks.append('awx.main.tasks.awx_isolated_heartbeat')
else:
if 'awx.main.tasks.awx_isolated_heartbeat' in conf.CELERY_TASK_ROUTES:
del conf.CELERY_TASK_ROUTES['awx.main.tasks.awx_isolated_heartbeat']
if 'awx.main.tasks.awx_isolated_heartbeat' in conf.CELERY_ROUTES:
del conf.CELERY_ROUTES['awx.main.tasks.awx_isolated_heartbeat']
for t in tasks:
conf.CELERY_TASK_ROUTES[t] = {'queue': instance.hostname, 'routing_key': instance.hostname}
routes_updated[t] = conf.CELERY_TASK_ROUTES[t]
conf.CELERY_ROUTES[t] = {'queue': instance.hostname, 'routing_key': instance.hostname}
routes_updated[t] = conf.CELERY_ROUTES[t]
return routes_updated
awx/settings/defaults.py
+16 -13
View File
@@ -5,6 +5,7 @@ import os
import re # noqa
import sys
import ldap
import djcelery
from datetime import timedelta
from kombu import Queue, Exchange
@@ -266,7 +267,7 @@ INSTALLED_APPS = (
'oauth2_provider',
'rest_framework',
'django_extensions',
'django_celery_results',
'djcelery',
'channels',
'polymorphic',
'taggit',
@@ -452,28 +453,30 @@ DEVSERVER_DEFAULT_PORT = '8013'
# Set default ports for live server tests.
os.environ.setdefault('DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:9013-9199')
djcelery.setup_loader()
BROKER_POOL_LIMIT = None
CELERY_BROKER_URL = 'amqp://guest:guest@localhost:5672//'
BROKER_URL = 'amqp://guest:guest@localhost:5672//'
CELERY_EVENT_QUEUE_TTL = 5
CELERY_TASK_DEFAULT_QUEUE = 'tower'
CELERY_DEFAULT_QUEUE = 'tower'
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_TRACK_STARTED = True
CELERY_TASK_TIME_LIMIT = None
CELERY_TASK_SOFT_TIME_LIMIT = None
CELERY_WORKER_POOL_RESTARTS = True
CELERY_BEAT_SCHEDULER = 'celery.beat.PersistentScheduler'
CELERY_BEAT_MAX_LOOP_INTERVAL = 60
CELERY_RESULT_BACKEND = 'django-db'
CELERY_TRACK_STARTED = True
CELERYD_TASK_TIME_LIMIT = None
CELERYD_TASK_SOFT_TIME_LIMIT = None
CELERYD_POOL_RESTARTS = True
CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'
CELERY_IMPORTS = ('awx.main.scheduler.tasks',)
CELERY_TASK_QUEUES = (
CELERY_QUEUES = (
Queue('tower', Exchange('tower'), routing_key='tower'),
Broadcast('tower_broadcast_all')
)
CELERY_TASK_ROUTES = {}
CELERY_ROUTES = {}
CELERY_BEAT_SCHEDULE = {
CELERYBEAT_SCHEDULER = 'celery.beat.PersistentScheduler'
CELERYBEAT_MAX_LOOP_INTERVAL = 60
CELERYBEAT_SCHEDULE = {
'tower_scheduler': {
'task': 'awx.main.tasks.awx_periodic_scheduler',
'schedule': timedelta(seconds=30),
awx/settings/local_settings.py.docker_compose
+5 -5
View File
@@ -79,15 +79,15 @@ if is_testing(sys.argv):
}
# Celery AMQP configuration.
CELERY_BROKER_URL = "amqp://{}:{}@{}/{}".format(os.environ.get("RABBITMQ_USER"),
os.environ.get("RABBITMQ_PASS"),
os.environ.get("RABBITMQ_HOST"),
urllib.quote(os.environ.get("RABBITMQ_VHOST", "/"), safe=''))
BROKER_URL = "amqp://{}:{}@{}/{}".format(os.environ.get("RABBITMQ_USER"),
os.environ.get("RABBITMQ_PASS"),
os.environ.get("RABBITMQ_HOST"),
urllib.quote(os.environ.get("RABBITMQ_VHOST", "/"), safe=''))
CHANNEL_LAYERS = {
'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer',
'ROUTING': 'awx.main.routing.channel_routing',
'CONFIG': {'url': CELERY_BROKER_URL}}
'CONFIG': {'url': BROKER_URL}}
}
# Set True to enable additional logging from the job_event_callback plugin