This is an automated email from the ASF dual-hosted git repository. abaker pushed a commit to branch develop in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push: new 983fa53 [GEODE-4688] Add metrics pipeline. 983fa53 is described below commit 983fa53868acfcd9616692cc0b4921e92148c557 Author: Sean Goller <sgol...@pivotal.io> AuthorDate: Tue Feb 13 16:08:07 2018 -0800 [GEODE-4688] Add metrics pipeline. * Create new pipeline that provides success metrics on develop pipeline jobs. * For each job, display percentage success over last 50 jobs, as well as enumerate test failures and the builds they occurred in. Co-authored-by: Vince Ford <vf...@pivotal.io> --- ci/bin/concourse_job_performance.py | 120 ++++++++++++++++++++++++ ci/pipelines/meta.yml | 19 ++++ ci/pipelines/metrics.yml | 159 ++++++++++++++++++++++++++++++++ ci/scripts/concourse_job_performance.sh | 37 ++++++++ 4 files changed, 335 insertions(+) diff --git a/ci/bin/concourse_job_performance.py b/ci/bin/concourse_job_performance.py new file mode 100755 index 0000000..6b530bf --- /dev/null +++ b/ci/bin/concourse_job_performance.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import requests +import json +import sseclient +import time +from colors import * +from urllib.parse import urlparse +from operator import itemgetter + +if len(sys.argv) != 5: + print("Usage: {} <concourse url> <pipeline> <job> <count>".format(os.path.basename(sys.argv[0]))) + exit(1) + +url = sys.argv[1] +pipeline = sys.argv[2] +job = sys.argv[3] +build_count = int(sys.argv[4]) +concourse_url = urlparse(url) + +completed_build_states = ['succeeded', 'failed'] +if not concourse_url.scheme or not concourse_url.netloc or concourse_url.path != '': + print(color("Url {} seems to be invalid. Please check your arguments.".format(url), fg='red')) + exit(1) + +session = requests.Session() +# print("Login status is {}".format(login_response.status_code)) +builds_url = '{}/api/v1/teams/main/pipelines/{}/jobs/{}/builds'.format(url, pipeline, job) +# build_params = {'limit': build_count} +# build_response = session.get(builds_url, params=build_params) +build_response = session.get(builds_url) + +builds = build_response.json() +sorted_builds = sorted(builds, key=itemgetter('id'), reverse=True) +failures = {} +# build = sorted_build[0] +completed_build_count = 0 +failed_build_count = 0 +for build in sorted_builds: + # print(color("Checking build id {} ({}/{})".format(build['id'], job, build['name']), fg='yellow')) + if build['status'] not in completed_build_states: + continue + completed_build_count += 1 + build_url = '{}{}'.format(url, build['api_url']) + event_url = '{}/events'.format(build_url) + + event_response = session.get(event_url, stream=True, timeout=60) + + # print("Event Status is {}".format(event_response.status_code)) + event_output = '' + build_status = 'unknown' + + event_client = sseclient.SSEClient(event_response) + for event in event_client.events(): + if event.event == 'end': + break + if event.data: + event_json = json.loads(event.data) + event_data = event_json['data'] + if event_json['event'] == 'status': + build_status = event_data['status'] + if build_status == 'succeeded': + break + if event_json['event'] == 'log': + event_output += event_data['payload'] + + # print("***************************************************") + # pprint.pprint("Event *{}* - {}".format(event.event,json.loads(event.data))) + + for line in event_output.splitlines(): + buildfailmatcher = re.search('BUILD FAILED|Test Failed!', line) + if buildfailmatcher: + failed_build_count += 1 + + matcher = re.search('(\S+)\s*>\s*(\S+).*FAILED', line) + if matcher: + test_name = "{}.{}".format(matcher.group(1), matcher.group(2)) + if not failures.get(test_name): + failures[test_name] = [build] + else: + failures[test_name].append(build) + # print("Failure information: {} - {}".format(matcher.group(1), matcher.group(2))) + + # print("Results: Job status is {}".format(build_status)) + if completed_build_count == build_count: + break + time.sleep(2) + +# pprint.pprint(failures) +if failed_build_count > 0: + print(color("***********************************************************************************", fg='yellow')) + print(" Overall build success rate: ", color("{}%".format((completed_build_count - failed_build_count)*100/completed_build_count), fg='blue')) + print(color("***********************************************************************************", fg='yellow')) + if failures: + for failure in failures.keys(): + count = len(failures[failure]) + print(color("{}: ".format(failure), fg='cyan'), + color("{} failures".format(count), fg='red'), + color("({}% success rate)".format(((completed_build_count - count)/completed_build_count) * 100), fg='blue')) + for build in failures[failure]: + print(color(" Failed build {} ".format(build['name']), fg='red'), color("at {}{}".format(url, build['url']), fg='magenta', style='bold')) +else: + print(color("No failures! 100% success rate", fg='green',style='bold')) diff --git a/ci/pipelines/meta.yml b/ci/pipelines/meta.yml index 91d86f4..3926f5f 100644 --- a/ci/pipelines/meta.yml +++ b/ci/pipelines/meta.yml @@ -46,6 +46,13 @@ resources: branch: develop paths: - ci/pipelines/docker-images.yml +- name: geode-metrics-pipeline + type: git + source: + uri: https://github.com/smgoller/geode.git + branch: smgoller-ci + paths: + - ci/pipelines/metrics.yml jobs: - name: set-develop-pipeline @@ -72,3 +79,15 @@ jobs: - name: docker-images team: main config_file: geode-docker-images-pipeline/ci/pipelines/docker-images.yml +- name: set-metrics-pipeline + serial: true + public: true + plan: + - get: geode-metrics-pipeline + trigger: true + - put: apachegeode-concourse + params: + pipelines: + - name: staging-metrics + team: staging + config_file: geode-metrics-pipeline/ci/pipelines/metrics.yml diff --git a/ci/pipelines/metrics.yml b/ci/pipelines/metrics.yml new file mode 100644 index 0000000..e1fe4f9 --- /dev/null +++ b/ci/pipelines/metrics.yml @@ -0,0 +1,159 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +image_resource: &docker-geode-build-image + type: docker-image + source: + username: ((!docker-username)) + password: ((!docker-password)) + repository: gcr.io/apachegeode-ci/((!docker-image-name)) + tag: latest + +resources: +- name: once-a-day + type: time + source: + interval: 24h +- name: geode-ci + type: git + source: + depth: 1 + uri: https://github.com/apache/geode.git + branch: develop + paths: + - ci/* + +jobs: +- name: GeodeBuildMetrics + serial: true + public: true + plan: + - get: geode-ci + - get: once-a-day + trigger: true + - task: get_metrics + config: + inputs: + - name: geode-ci + outputs: + - name: workspace + platform: linux + image_resource: *docker-geode-build-image + params: + CONCOURSE_URL: https://concourse.apachegeode-ci.info + CONCOURSE_PIPELINE: develop + CONCOURSE_JOB: Build + COUNT: 50 + run: + args: [workspace] + path: geode-ci/ci/scripts/concourse_job_performance.sh + +- name: GeodeAcceptanceTestMetrics + serial: true + public: true + plan: + - get: geode-ci + - get: once-a-day + trigger: true + - task: get_metrics + config: + inputs: + - name: geode-ci + outputs: + - name: workspace + platform: linux + image_resource: *docker-geode-build-image + params: + CONCOURSE_URL: https://concourse.apachegeode-ci.info + CONCOURSE_PIPELINE: develop + CONCOURSE_JOB: AcceptanceTest + COUNT: 50 + run: + args: [workspace] + path: geode-ci/ci/scripts/concourse_job_performance.sh + +- name: GeodeDistributedTestMetrics + serial: true + public: true + plan: + - get: geode-ci + - get: once-a-day + trigger: true + - task: get_metrics + config: + inputs: + - name: geode-ci + outputs: + - name: workspace + platform: linux + image_resource: *docker-geode-build-image + params: + CONCOURSE_URL: https://concourse.apachegeode-ci.info + CONCOURSE_PIPELINE: develop + CONCOURSE_JOB: DistributedTest + COUNT: 50 + run: + args: [workspace] + path: geode-ci/ci/scripts/concourse_job_performance.sh + +- name: GeodeIntegrationTestMetrics + serial: true + public: true + plan: + - get: geode-ci + - get: once-a-day + trigger: true + - task: get_metrics + config: + inputs: + - name: geode-ci + outputs: + - name: workspace + platform: linux + image_resource: *docker-geode-build-image + params: + CONCOURSE_URL: https://concourse.apachegeode-ci.info + CONCOURSE_PIPELINE: develop + CONCOURSE_JOB: IntegrationTest + COUNT: 50 + run: + args: [workspace] + path: geode-ci/ci/scripts/concourse_job_performance.sh + +- name: GeodeFlakyTestMetrics + serial: true + public: true + plan: + - get: geode-ci + - get: once-a-day + trigger: true + - task: get_metrics + config: + inputs: + - name: geode-ci + outputs: + - name: workspace + platform: linux + image_resource: *docker-geode-build-image + params: + CONCOURSE_URL: https://concourse.apachegeode-ci.info + CONCOURSE_PIPELINE: develop + CONCOURSE_JOB: FlakyTest + COUNT: 50 + run: + args: [workspace] + path: geode-ci/ci/scripts/concourse_job_performance.sh diff --git a/ci/scripts/concourse_job_performance.sh b/ci/scripts/concourse_job_performance.sh new file mode 100755 index 0000000..a674cae --- /dev/null +++ b/ci/scripts/concourse_job_performance.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e +WORKDIR=${1} + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +apt-get install -y python3 python3-pip +pip3 install requests +pip3 install ansicolors +pip3 install sseclient-py +pip3 install urllib3 +set -x +python3 geode-ci/ci/bin/concourse_job_performance.py ${CONCOURSE_URL} ${CONCOURSE_PIPELINE} ${CONCOURSE_JOB} ${COUNT} +set +x -- To stop receiving notification emails like this one, please contact aba...@apache.org.