zware/awx
1
0
Fork 0
mirror of https://github.com/ZwareBear/awx.git synced 2026-05-06 16:32:01 -05:00
Code Issues Packages Projects Releases Wiki Activity

optimize the callback receiver to buffer writes on high throughput

Browse Source 
additionaly, optimize away several per-event host lookups and
changed/failed propagation lookups

we've always performed these (fairly expensive) queries *on every event
save* - if you're processing tens of thousands of events in short
bursts, this is way too slow

this commit also introduces a new command for profiling the insertion
rate of events, `awx-manage callback_stats`

see: https://github.com/ansible/awx/issues/5514
This commit is contained in:
Ryan Petrello
2020-01-08 16:14:47 -05:00
parent 862fafab86
commit 306f504fb7
17 changed files with 370 additions and 495 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
awx/main/tasks.py
View File

@@ -703,6 +703,7 @@ class BaseTask(object):
def __init__(self):
self.cleanup_paths = []
self.parent_workflow_job_id = None
self.host_map = {}
def update_model(self, pk, _attempt=0, **updates):
"""Reload the model instance from the database and update the
@@ -1001,11 +1002,17 @@ class BaseTask(object):
return False
def build_inventory(self, instance, private_data_dir):
script_params = dict(hostvars=True)
script_params = dict(hostvars=True, towervars=True)
if hasattr(instance, 'job_slice_number'):
script_params['slice_number'] = instance.job_slice_number
script_params['slice_count'] = instance.job_slice_count
script_data = instance.inventory.get_script_data(**script_params)
# maintain a list of host_name --> host_id
# so we can associate emitted events to Host objects
self.host_map = {
hostname: hv.pop('remote_tower_id', '')
for hostname, hv in script_data.get('_meta', {}).get('hostvars', {}).items()
}
json_data = json.dumps(script_data)
handle, path = tempfile.mkstemp(dir=private_data_dir)
f = os.fdopen(handle, 'w')
@@ -1114,6 +1121,15 @@ class BaseTask(object):
event_data.pop('parent_uuid', None)
if self.parent_workflow_job_id:
event_data['workflow_job_id'] = self.parent_workflow_job_id
if self.host_map:
host = event_data.get('event_data', {}).get('host', '').strip()
if host:
event_data['host_name'] = host
if host in self.host_map:
event_data['host_id'] = self.host_map[host]
else:
event_data['host_name'] = ''
event_data['host_id'] = ''
should_write_event = False
event_data.setdefault(self.event_data_key, self.instance.id)
self.dispatcher.dispatch(event_data)