Merge pull request #4436 from jbradberry/webhook-receivers

Webhook receivers

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul
This commit is contained in:
softwarefactory-project-zuul[bot]
2019-10-03 22:23:22 +00:00
committed by GitHub
51 changed files with 2161 additions and 118 deletions

View File

@@ -249,3 +249,8 @@ class InstanceGroupTowerPermission(ModelAccessPermission):
if request.method == 'DELETE' and obj.name == "tower":
return False
return super(InstanceGroupTowerPermission, self).has_object_permission(request, view, obj)
class WebhookKeyPermission(permissions.BasePermission):
def has_object_permission(self, request, view, obj):
return request.user.can_access(view.model, 'admin', obj, request.data)

View File

@@ -139,6 +139,7 @@ SUMMARIZABLE_FK_FIELDS = {
'insights_credential': DEFAULT_SUMMARY_FIELDS,
'source_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'),
'target_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'),
'webhook_credential': DEFAULT_SUMMARY_FIELDS,
}
@@ -2826,6 +2827,25 @@ class JobTemplateMixin(object):
d['recent_jobs'] = self._recent_jobs(obj)
return d
def validate(self, attrs):
webhook_service = attrs.get('webhook_service', getattr(self.instance, 'webhook_service', None))
webhook_credential = attrs.get('webhook_credential', getattr(self.instance, 'webhook_credential', None))
if webhook_credential:
if webhook_credential.credential_type.kind != 'token':
raise serializers.ValidationError({
'webhook_credential': _("Must be a Personal Access Token."),
})
msg = {'webhook_credential': _("Must match the selected webhook service.")}
if webhook_service:
if webhook_credential.credential_type.namespace != '{}_token'.format(webhook_service):
raise serializers.ValidationError(msg)
else:
raise serializers.ValidationError(msg)
return super().validate(attrs)
class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobOptionsSerializer):
show_capabilities = ['start', 'schedule', 'copy', 'edit', 'delete']
@@ -2838,30 +2858,39 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
class Meta:
model = JobTemplate
fields = ('*', 'host_config_key', 'ask_scm_branch_on_launch', 'ask_diff_mode_on_launch', 'ask_variables_on_launch',
'ask_limit_on_launch', 'ask_tags_on_launch',
'ask_skip_tags_on_launch', 'ask_job_type_on_launch', 'ask_verbosity_on_launch', 'ask_inventory_on_launch',
'ask_credential_on_launch', 'survey_enabled', 'become_enabled', 'diff_mode',
'allow_simultaneous', 'custom_virtualenv', 'job_slice_count')
fields = (
'*', 'host_config_key', 'ask_scm_branch_on_launch', 'ask_diff_mode_on_launch',
'ask_variables_on_launch', 'ask_limit_on_launch', 'ask_tags_on_launch',
'ask_skip_tags_on_launch', 'ask_job_type_on_launch', 'ask_verbosity_on_launch',
'ask_inventory_on_launch', 'ask_credential_on_launch', 'survey_enabled',
'become_enabled', 'diff_mode', 'allow_simultaneous', 'custom_virtualenv',
'job_slice_count', 'webhook_service', 'webhook_credential',
)
def get_related(self, obj):
res = super(JobTemplateSerializer, self).get_related(obj)
res.update(dict(
jobs = self.reverse('api:job_template_jobs_list', kwargs={'pk': obj.pk}),
schedules = self.reverse('api:job_template_schedules_list', kwargs={'pk': obj.pk}),
activity_stream = self.reverse('api:job_template_activity_stream_list', kwargs={'pk': obj.pk}),
launch = self.reverse('api:job_template_launch', kwargs={'pk': obj.pk}),
notification_templates_started = self.reverse('api:job_template_notification_templates_started_list', kwargs={'pk': obj.pk}),
notification_templates_success = self.reverse('api:job_template_notification_templates_success_list', kwargs={'pk': obj.pk}),
notification_templates_error = self.reverse('api:job_template_notification_templates_error_list', kwargs={'pk': obj.pk}),
access_list = self.reverse('api:job_template_access_list', kwargs={'pk': obj.pk}),
survey_spec = self.reverse('api:job_template_survey_spec', kwargs={'pk': obj.pk}),
labels = self.reverse('api:job_template_label_list', kwargs={'pk': obj.pk}),
object_roles = self.reverse('api:job_template_object_roles_list', kwargs={'pk': obj.pk}),
instance_groups = self.reverse('api:job_template_instance_groups_list', kwargs={'pk': obj.pk}),
slice_workflow_jobs = self.reverse('api:job_template_slice_workflow_jobs_list', kwargs={'pk': obj.pk}),
copy = self.reverse('api:job_template_copy', kwargs={'pk': obj.pk}),
))
res.update(
jobs=self.reverse('api:job_template_jobs_list', kwargs={'pk': obj.pk}),
schedules=self.reverse('api:job_template_schedules_list', kwargs={'pk': obj.pk}),
activity_stream=self.reverse('api:job_template_activity_stream_list', kwargs={'pk': obj.pk}),
launch=self.reverse('api:job_template_launch', kwargs={'pk': obj.pk}),
webhook_key=self.reverse('api:webhook_key', kwargs={'model_kwarg': 'job_templates', 'pk': obj.pk}),
webhook_receiver=(
self.reverse('api:webhook_receiver_{}'.format(obj.webhook_service),
kwargs={'model_kwarg': 'job_templates', 'pk': obj.pk})
if obj.webhook_service else ''
),
notification_templates_started=self.reverse('api:job_template_notification_templates_started_list', kwargs={'pk': obj.pk}),
notification_templates_success=self.reverse('api:job_template_notification_templates_success_list', kwargs={'pk': obj.pk}),
notification_templates_error=self.reverse('api:job_template_notification_templates_error_list', kwargs={'pk': obj.pk}),
access_list=self.reverse('api:job_template_access_list', kwargs={'pk': obj.pk}),
survey_spec=self.reverse('api:job_template_survey_spec', kwargs={'pk': obj.pk}),
labels=self.reverse('api:job_template_label_list', kwargs={'pk': obj.pk}),
object_roles=self.reverse('api:job_template_object_roles_list', kwargs={'pk': obj.pk}),
instance_groups=self.reverse('api:job_template_instance_groups_list', kwargs={'pk': obj.pk}),
slice_workflow_jobs=self.reverse('api:job_template_slice_workflow_jobs_list', kwargs={'pk': obj.pk}),
copy=self.reverse('api:job_template_copy', kwargs={'pk': obj.pk}),
)
if obj.host_config_key:
res['callback'] = self.reverse('api:job_template_callback', kwargs={'pk': obj.pk})
return res
@@ -2889,7 +2918,6 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
def validate_extra_vars(self, value):
return vars_validate_or_raise(value)
def get_summary_fields(self, obj):
summary_fields = super(JobTemplateSerializer, self).get_summary_fields(obj)
all_creds = []
@@ -2930,9 +2958,11 @@ class JobSerializer(UnifiedJobSerializer, JobOptionsSerializer):
class Meta:
model = Job
fields = ('*', 'job_template', 'passwords_needed_to_start',
'allow_simultaneous', 'artifacts', 'scm_revision',
'instance_group', 'diff_mode', 'job_slice_number', 'job_slice_count')
fields = (
'*', 'job_template', 'passwords_needed_to_start', 'allow_simultaneous',
'artifacts', 'scm_revision', 'instance_group', 'diff_mode', 'job_slice_number',
'job_slice_count', 'webhook_service', 'webhook_credential', 'webhook_guid',
)
def get_related(self, obj):
res = super(JobSerializer, self).get_related(obj)
@@ -3320,16 +3350,25 @@ class WorkflowJobTemplateSerializer(JobTemplateMixin, LabelsListMixin, UnifiedJo
class Meta:
model = WorkflowJobTemplate
fields = ('*', 'extra_vars', 'organization', 'survey_enabled', 'allow_simultaneous',
'ask_variables_on_launch', 'inventory', 'limit', 'scm_branch',
'ask_inventory_on_launch', 'ask_scm_branch_on_launch', 'ask_limit_on_launch',)
fields = (
'*', 'extra_vars', 'organization', 'survey_enabled', 'allow_simultaneous',
'ask_variables_on_launch', 'inventory', 'limit', 'scm_branch',
'ask_inventory_on_launch', 'ask_scm_branch_on_launch', 'ask_limit_on_launch',
'webhook_service', 'webhook_credential',
)
def get_related(self, obj):
res = super(WorkflowJobTemplateSerializer, self).get_related(obj)
res.update(dict(
res.update(
workflow_jobs = self.reverse('api:workflow_job_template_jobs_list', kwargs={'pk': obj.pk}),
schedules = self.reverse('api:workflow_job_template_schedules_list', kwargs={'pk': obj.pk}),
launch = self.reverse('api:workflow_job_template_launch', kwargs={'pk': obj.pk}),
webhook_key=self.reverse('api:webhook_key', kwargs={'model_kwarg': 'workflow_job_templates', 'pk': obj.pk}),
webhook_receiver=(
self.reverse('api:webhook_receiver_{}'.format(obj.webhook_service),
kwargs={'model_kwarg': 'workflow_job_templates', 'pk': obj.pk})
if obj.webhook_service else ''
),
workflow_nodes = self.reverse('api:workflow_job_template_workflow_nodes_list', kwargs={'pk': obj.pk}),
labels = self.reverse('api:workflow_job_template_label_list', kwargs={'pk': obj.pk}),
activity_stream = self.reverse('api:workflow_job_template_activity_stream_list', kwargs={'pk': obj.pk}),
@@ -3341,7 +3380,7 @@ class WorkflowJobTemplateSerializer(JobTemplateMixin, LabelsListMixin, UnifiedJo
object_roles = self.reverse('api:workflow_job_template_object_roles_list', kwargs={'pk': obj.pk}),
survey_spec = self.reverse('api:workflow_job_template_survey_spec', kwargs={'pk': obj.pk}),
copy = self.reverse('api:workflow_job_template_copy', kwargs={'pk': obj.pk}),
))
)
if obj.organization:
res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk})
return res
@@ -3382,10 +3421,11 @@ class WorkflowJobSerializer(LabelsListMixin, UnifiedJobSerializer):
class Meta:
model = WorkflowJob
fields = ('*', 'workflow_job_template', 'extra_vars', 'allow_simultaneous',
'job_template', 'is_sliced_job',
'-execution_node', '-event_processing_finished', '-controller_node',
'inventory', 'limit', 'scm_branch',)
fields = (
'*', 'workflow_job_template', 'extra_vars', 'allow_simultaneous', 'job_template',
'is_sliced_job', '-execution_node', '-event_processing_finished', '-controller_node',
'inventory', 'limit', 'scm_branch', 'webhook_service', 'webhook_credential', 'webhook_guid',
)
def get_related(self, obj):
res = super(WorkflowJobSerializer, self).get_related(obj)

View File

@@ -0,0 +1,12 @@
Webhook Secret Key:
Make a GET request to this resource to obtain the secret key for a job
template or workflow job template configured to be triggered by
webhook events. The response will include the following fields:
* `webhook_key`: Secret key that needs to be copied and added to the
webhook configuration of the service this template will be receiving
webhook events from (string, read-only)
Make an empty POST request to this resource to generate a new
replacement `webhook_key`.

View File

@@ -1,7 +1,7 @@
# Copyright (c) 2017 Ansible, Inc.
# All Rights Reserved.
from django.conf.urls import url
from django.conf.urls import include, url
from awx.api.views import (
JobTemplateList,
@@ -45,6 +45,7 @@ urls = [
url(r'^(?P<pk>[0-9]+)/object_roles/$', JobTemplateObjectRolesList.as_view(), name='job_template_object_roles_list'),
url(r'^(?P<pk>[0-9]+)/labels/$', JobTemplateLabelList.as_view(), name='job_template_label_list'),
url(r'^(?P<pk>[0-9]+)/copy/$', JobTemplateCopy.as_view(), name='job_template_copy'),
url(r'^(?P<pk>[0-9]+)/', include('awx.api.urls.webhooks'), {'model_kwarg': 'job_templates'}),
]
__all__ = ['urls']

14
awx/api/urls/webhooks.py Normal file
View File

@@ -0,0 +1,14 @@
from django.conf.urls import url
from awx.api.views import (
WebhookKeyView,
GithubWebhookReceiver,
GitlabWebhookReceiver,
)
urlpatterns = [
url(r'^webhook_key/$', WebhookKeyView.as_view(), name='webhook_key'),
url(r'^github/$', GithubWebhookReceiver.as_view(), name='webhook_receiver_github'),
url(r'^gitlab/$', GitlabWebhookReceiver.as_view(), name='webhook_receiver_gitlab'),
]

View File

@@ -1,7 +1,7 @@
# Copyright (c) 2017 Ansible, Inc.
# All Rights Reserved.
from django.conf.urls import url
from django.conf.urls import include, url
from awx.api.views import (
WorkflowJobTemplateList,
@@ -44,6 +44,7 @@ urls = [
url(r'^(?P<pk>[0-9]+)/access_list/$', WorkflowJobTemplateAccessList.as_view(), name='workflow_job_template_access_list'),
url(r'^(?P<pk>[0-9]+)/object_roles/$', WorkflowJobTemplateObjectRolesList.as_view(), name='workflow_job_template_object_roles_list'),
url(r'^(?P<pk>[0-9]+)/labels/$', WorkflowJobTemplateLabelList.as_view(), name='workflow_job_template_label_list'),
url(r'^(?P<pk>[0-9]+)/', include('awx.api.urls.webhooks'), {'model_kwarg': 'workflow_job_templates'}),
]
__all__ = ['urls']

View File

@@ -150,6 +150,11 @@ from awx.api.views.root import ( # noqa
ApiV2ConfigView,
ApiV2SubscriptionView,
)
from awx.api.views.webhooks import ( # noqa
WebhookKeyView,
GithubWebhookReceiver,
GitlabWebhookReceiver,
)
logger = logging.getLogger('awx.api.views')

247
awx/api/views/webhooks.py Normal file
View File

@@ -0,0 +1,247 @@
from hashlib import sha1
import hmac
import json
import logging
import urllib.parse
from django.utils.encoding import force_bytes
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.csrf import csrf_exempt
from rest_framework import status
from rest_framework.exceptions import PermissionDenied
from rest_framework.permissions import AllowAny
from rest_framework.response import Response
from awx.api import serializers
from awx.api.generics import APIView, GenericAPIView
from awx.api.permissions import WebhookKeyPermission
from awx.main.models import Job, JobTemplate, WorkflowJob, WorkflowJobTemplate
logger = logging.getLogger('awx.api.views.webhooks')
class WebhookKeyView(GenericAPIView):
serializer_class = serializers.EmptySerializer
permission_classes = (WebhookKeyPermission,)
def get_queryset(self):
qs_models = {
'job_templates': JobTemplate,
'workflow_job_templates': WorkflowJobTemplate,
}
self.model = qs_models.get(self.kwargs['model_kwarg'])
return super().get_queryset()
def get(self, request, *args, **kwargs):
obj = self.get_object()
return Response({'webhook_key': obj.webhook_key})
def post(self, request, *args, **kwargs):
obj = self.get_object()
obj.rotate_webhook_key()
obj.save(update_fields=['webhook_key'])
return Response({'webhook_key': obj.webhook_key}, status=status.HTTP_201_CREATED)
class WebhookReceiverBase(APIView):
lookup_url_kwarg = None
lookup_field = 'pk'
permission_classes = (AllowAny,)
authentication_classes = ()
ref_keys = {}
def get_queryset(self):
qs_models = {
'job_templates': JobTemplate,
'workflow_job_templates': WorkflowJobTemplate,
}
model = qs_models.get(self.kwargs['model_kwarg'])
if model is None:
raise PermissionDenied
return model.objects.filter(webhook_service=self.service).exclude(webhook_key='')
def get_object(self):
queryset = self.get_queryset()
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
filter_kwargs = {self.lookup_field: self.kwargs[lookup_url_kwarg]}
obj = queryset.filter(**filter_kwargs).first()
if obj is None:
raise PermissionDenied
return obj
def get_event_type(self):
raise NotImplementedError
def get_event_guid(self):
raise NotImplementedError
def get_event_status_api(self):
raise NotImplementedError
def get_event_ref(self):
key = self.ref_keys.get(self.get_event_type(), '')
value = self.request.data
for element in key.split('.'):
try:
if element.isdigit():
value = value[int(element)]
else:
value = (value or {}).get(element)
except Exception:
value = None
if value == '0000000000000000000000000000000000000000': # a deleted ref
value = None
return value
def get_signature(self):
raise NotImplementedError
def check_signature(self, obj):
if not obj.webhook_key:
raise PermissionDenied
mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha1)
logger.debug("header signature: %s", self.get_signature())
logger.debug("calculated signature: %s", force_bytes(mac.hexdigest()))
if not hmac.compare_digest(force_bytes(mac.hexdigest()), self.get_signature()):
raise PermissionDenied
@csrf_exempt
def post(self, request, *args, **kwargs):
# Ensure that the full contents of the request are captured for multiple uses.
request.body
logger.debug(
"headers: {}\n"
"data: {}\n".format(request.headers, request.data)
)
obj = self.get_object()
self.check_signature(obj)
event_type = self.get_event_type()
event_guid = self.get_event_guid()
event_ref = self.get_event_ref()
status_api = self.get_event_status_api()
kwargs = {
'unified_job_template_id': obj.id,
'webhook_service': obj.webhook_service,
'webhook_guid': event_guid,
}
if WorkflowJob.objects.filter(**kwargs).exists() or Job.objects.filter(**kwargs).exists():
# Short circuit if this webhook has already been received and acted upon.
logger.debug("Webhook previously received, returning without action.")
return Response({'message': _("Webhook previously received, aborting.")},
status=status.HTTP_202_ACCEPTED)
kwargs = {
'_eager_fields': {
'launch_type': 'webhook',
'webhook_service': obj.webhook_service,
'webhook_credential': obj.webhook_credential,
'webhook_guid': event_guid,
},
'extra_vars': json.dumps({
'tower_webhook_event_type': event_type,
'tower_webhook_event_guid': event_guid,
'tower_webhook_event_ref': event_ref,
'tower_webhook_status_api': status_api,
'tower_webhook_payload': request.data,
})
}
new_job = obj.create_unified_job(**kwargs)
new_job.signal_start()
return Response({'message': "Job queued."}, status=status.HTTP_202_ACCEPTED)
class GithubWebhookReceiver(WebhookReceiverBase):
service = 'github'
ref_keys = {
'pull_request': 'pull_request.head.sha',
'pull_request_review': 'pull_request.head.sha',
'pull_request_review_comment': 'pull_request.head.sha',
'push': 'after',
'release': 'release.tag_name',
'commit_comment': 'comment.commit_id',
'create': 'ref',
'page_build': 'build.commit',
}
def get_event_type(self):
return self.request.META.get('HTTP_X_GITHUB_EVENT')
def get_event_guid(self):
return self.request.META.get('HTTP_X_GITHUB_DELIVERY')
def get_event_status_api(self):
if self.get_event_type() != 'pull_request':
return
return self.request.data.get('pull_request', {}).get('statuses_url')
def get_signature(self):
header_sig = self.request.META.get('HTTP_X_HUB_SIGNATURE')
if not header_sig:
logger.debug("Expected signature missing from header key HTTP_X_HUB_SIGNATURE")
raise PermissionDenied
hash_alg, signature = header_sig.split('=')
if hash_alg != 'sha1':
logger.debug("Unsupported signature type, expected: sha1, received: {}".format(hash_alg))
raise PermissionDenied
return force_bytes(signature)
class GitlabWebhookReceiver(WebhookReceiverBase):
service = 'gitlab'
ref_keys = {
'Push Hook': 'checkout_sha',
'Tag Push Hook': 'checkout_sha',
'Merge Request Hook': 'object_attributes.last_commit.id',
}
def get_event_type(self):
return self.request.META.get('HTTP_X_GITLAB_EVENT')
def get_event_guid(self):
# GitLab does not provide a unique identifier on events, so construct one.
h = sha1()
h.update(force_bytes(self.request.body))
return h.hexdigest()
def get_event_status_api(self):
if self.get_event_type() != 'Merge Request Hook':
return
project = self.request.data.get('project', {})
repo_url = project.get('web_url')
if not repo_url:
return
parsed = urllib.parse.urlparse(repo_url)
return "{}://{}/api/v4/projects/{}/statuses/{}".format(
parsed.scheme, parsed.netloc, project['id'], self.get_event_ref())
def get_signature(self):
return force_bytes(self.request.META.get('HTTP_X_GITLAB_TOKEN') or '')
def check_signature(self, obj):
if not obj.webhook_key:
raise PermissionDenied
# GitLab only returns the secret token, not an hmac hash. Use
# the hmac `compare_digest` helper function to prevent timing
# analysis by attackers.
if not hmac.compare_digest(force_bytes(obj.webhook_key), self.get_signature()):
raise PermissionDenied