This is an automated email from the ASF dual-hosted git repository. humbedooh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kibble.git
commit 2fd2c89040320f8216864e3f21e09d20c5bbe159 Author: Daniel Gruno <humbed...@apache.org> AuthorDate: Fri Feb 16 19:55:40 2018 +0100 Add preliminary CI status pages --- api/pages/ci/queue.py | 182 ++++++++++++++++++++++++++++++++++++++++ api/pages/ci/status.py | 180 ++++++++++++++++++++++++++++++++++++++++ api/pages/ci/top-buildcount.py | 183 +++++++++++++++++++++++++++++++++++++++++ api/pages/ci/top-buildtime.py | 183 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 728 insertions(+) diff --git a/api/pages/ci/queue.py b/api/pages/ci/queue.py new file mode 100644 index 0000000..06d495d --- /dev/null +++ b/api/pages/ci/queue.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +######################################################################## +# OPENAPI-URI: /api/ci/queue +######################################################################## +# get: +# responses: +# '200': +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Timeseries' +# description: 200 Response +# default: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Error' +# description: unexpected error +# security: +# - cookieAuth: [] +# summary: Shows email sent over time +# post: +# requestBody: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/defaultWidgetArgs' +# responses: +# '200': +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Timeseries' +# description: 200 Response +# default: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Error' +# description: unexpected error +# security: +# - cookieAuth: [] +# summary: Shows CI queue over time +# +######################################################################## + + + +""" +This is the CI queue timeseries renderer for Kibble +""" + +import json +import time +import hashlib + +def run(API, environ, indata, session): + + # We need to be logged in for this! + if not session.user: + raise API.exception(403, "You must be logged in to use this API endpoint! %s") + + now = time.time() + + # First, fetch the view if we have such a thing enabled + viewList = [] + if indata.get('view'): + viewList = session.getView(indata.get('view')) + if indata.get('subfilter'): + viewList = session.subFilter(indata.get('subfilter'), view = viewList) + + + dateTo = indata.get('to', int(time.time())) + dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span + + interval = indata.get('interval', 'month') + + + #################################################################### + #################################################################### + dOrg = session.user['defaultOrganisation'] or "apache" + query = { + 'query': { + 'bool': { + 'must': [ + {'range': + { + 'time': { + 'from': dateFrom, + 'to': dateTo + } + } + }, + { + 'term': { + 'organisation': dOrg + } + } + ] + } + } + } + # Source-specific or view-specific?? + if indata.get('source'): + query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}}) + elif viewList: + query['query']['bool']['must'].append({'terms': {'sourceID': viewList}}) + + # Get queue stats + query['aggs'] = { + 'timeseries': { + 'date_histogram': { + 'field': 'date', + 'interval': interval + }, + 'aggs': { + 'size': { + 'avg': { + 'field': 'size' + } + }, + 'blocked': { + 'avg': { + 'field': 'blocked' + } + }, + 'stuck': { + 'avg': { + 'field': 'stuck' + } + }, + 'wait': { + 'avg': { + 'field': 'avgwait' + } + } + } + } + } + res = session.DB.ES.search( + index=session.DB.dbname, + doc_type="ci_queue", + size = 0, + body = query + ) + + timeseries = [] + for bucket in res['aggregations']['timeseries']['buckets']: + ts = int(bucket['key'] / 1000) + timeseries.append({ + 'date': ts, + 'queue size': bucket['size']['value'], +# 'builds blocked': bucket['blocked']['value'], +# 'builds stuck': bucket['stuck']['value'], + 'average wait (hours)': int(bucket['wait']['value']/3600), + }) + + JSON_OUT = { + 'widgetType': { + 'chartType': 'line' # Recommendation for the UI + }, + 'timeseries': timeseries, + 'interval': interval, + 'okay': True, + 'responseTime': time.time() - now + } + yield json.dumps(JSON_OUT) diff --git a/api/pages/ci/status.py b/api/pages/ci/status.py new file mode 100644 index 0000000..e3678a2 --- /dev/null +++ b/api/pages/ci/status.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +######################################################################## +# OPENAPI-URI: /api/ci/status +######################################################################## +# get: +# responses: +# '200': +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Timeseries' +# description: 200 Response +# default: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Error' +# description: unexpected error +# security: +# - cookieAuth: [] +# summary: Shows email sent over time +# post: +# requestBody: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/defaultWidgetArgs' +# responses: +# '200': +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Timeseries' +# description: 200 Response +# default: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Error' +# description: unexpected error +# security: +# - cookieAuth: [] +# summary: Shows CI queue over time +# +######################################################################## + + + +""" +This is the CI queue status (blocked/stuck) timeseries renderer for Kibble +""" + +import json +import time +import hashlib + +def run(API, environ, indata, session): + + # We need to be logged in for this! + if not session.user: + raise API.exception(403, "You must be logged in to use this API endpoint! %s") + + now = time.time() + + # First, fetch the view if we have such a thing enabled + viewList = [] + if indata.get('view'): + viewList = session.getView(indata.get('view')) + if indata.get('subfilter'): + viewList = session.subFilter(indata.get('subfilter'), view = viewList) + + + dateTo = indata.get('to', int(time.time())) + dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span + + interval = indata.get('interval', 'month') + + + #################################################################### + #################################################################### + dOrg = session.user['defaultOrganisation'] or "apache" + query = { + 'query': { + 'bool': { + 'must': [ + {'range': + { + 'time': { + 'from': dateFrom, + 'to': dateTo + } + } + }, + { + 'term': { + 'organisation': dOrg + } + } + ] + } + } + } + # Source-specific or view-specific?? + if indata.get('source'): + query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}}) + elif viewList: + query['query']['bool']['must'].append({'terms': {'sourceID': viewList}}) + + # Get queue stats + query['aggs'] = { + 'timeseries': { + 'date_histogram': { + 'field': 'date', + 'interval': interval + }, + 'aggs': { + 'size': { + 'avg': { + 'field': 'size' + } + }, + 'blocked': { + 'avg': { + 'field': 'blocked' + } + }, + 'stuck': { + 'avg': { + 'field': 'stuck' + } + }, + 'wait': { + 'avg': { + 'field': 'avgwait' + } + } + } + } + } + res = session.DB.ES.search( + index=session.DB.dbname, + doc_type="ci_queue", + size = 0, + body = query + ) + + timeseries = [] + for bucket in res['aggregations']['timeseries']['buckets']: + ts = int(bucket['key'] / 1000) + timeseries.append({ + 'date': ts, + 'builds blocked': bucket['blocked']['value'], + 'builds stuck': bucket['stuck']['value'] + }) + + JSON_OUT = { + 'widgetType': { + 'chartType': 'bar' # Recommendation for the UI + }, + 'timeseries': timeseries, + 'interval': interval, + 'okay': True, + 'responseTime': time.time() - now + } + yield json.dumps(JSON_OUT) diff --git a/api/pages/ci/top-buildcount.py b/api/pages/ci/top-buildcount.py new file mode 100644 index 0000000..31ef5c3 --- /dev/null +++ b/api/pages/ci/top-buildcount.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +######################################################################## +# OPENAPI-URI: /api/ci/top-buildcount +######################################################################## +# get: +# responses: +# '200': +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Timeseries' +# description: 200 Response +# default: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Error' +# description: unexpected error +# security: +# - cookieAuth: [] +# summary: Shows top 25 repos by lines of code +# post: +# requestBody: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/defaultWidgetArgs' +# responses: +# '200': +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Timeseries' +# description: 200 Response +# default: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Error' +# description: unexpected error +# security: +# - cookieAuth: [] +# summary: Shows top 25 jobs by total builds done. Essentially buildtime, tweaked +# +######################################################################## + + + + + +""" +This is the TopN CI jobs by total build time renderer for Kibble +""" + +import json +import time +import re + +def run(API, environ, indata, session): + + # We need to be logged in for this! + if not session.user: + raise API.exception(403, "You must be logged in to use this API endpoint! %s") + + now = time.time() + + # First, fetch the view if we have such a thing enabled + viewList = [] + if indata.get('view'): + viewList = session.getView(indata.get('view')) + if indata.get('subfilter'): + viewList = session.subFilter(indata.get('subfilter'), view = viewList) + + dateTo = indata.get('to', int(time.time())) + dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span + + #################################################################### + #################################################################### + dOrg = session.user['defaultOrganisation'] or "apache" + query = { + 'query': { + 'bool': { + 'must': [ + {'range': + { + 'date': { + 'from': time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(dateFrom)), + 'to': time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(dateTo)) + } + } + }, + { + 'term': { + 'organisation': dOrg + } + } + ] + } + } + } + # Source-specific or view-specific?? + if indata.get('source'): + query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}}) + elif viewList: + query['query']['bool']['must'].append({'terms': {'sourceID': viewList}}) + + query['aggs'] = { + 'by_job': { + 'terms': { + 'field': 'jobURL.keyword', + 'size': 5000, + }, + 'aggs': { + 'duration': { + 'sum': { + 'field': 'duration' + } + }, + 'ci': { + 'terms': { + 'field': 'ci.keyword', + 'size': 1 + } + }, + 'name': { + 'terms': { + 'field': 'job.keyword', + 'size': 1 + } + } + } + } + } + + res = session.DB.ES.search( + index=session.DB.dbname, + doc_type="ci_build", + size = 0, + body = query + ) + + jobs = [] + for doc in res['aggregations']['by_job']['buckets']: + job = doc['key'] + builds = doc['doc_count'] + duration = doc['duration']['value'] + ci = doc['ci']['buckets'][0]['key'] + jobname = doc['name']['buckets'][0]['key'] + jobs.append([builds, duration, jobname, ci]) + + topjobs = sorted(jobs, key = lambda x: int(x[0]), reverse = True) + top = topjobs[0:24] + if len(topjobs) > 25: + count = 0 + for repo in topjobs[25:]: + count += repo[1] + top.append(["Other jobs", 1, count, '??']) + + tophash = {} + for v in top: + tophash["%s (%s)" % (v[2], v[3])] = v[0] + + JSON_OUT = { + 'counts': tophash, + 'okay': True, + 'responseTime': time.time() - now, + } + yield json.dumps(JSON_OUT) diff --git a/api/pages/ci/top-buildtime.py b/api/pages/ci/top-buildtime.py new file mode 100644 index 0000000..995bd2c --- /dev/null +++ b/api/pages/ci/top-buildtime.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +######################################################################## +# OPENAPI-URI: /api/ci/top-buildtime +######################################################################## +# get: +# responses: +# '200': +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Timeseries' +# description: 200 Response +# default: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Error' +# description: unexpected error +# security: +# - cookieAuth: [] +# summary: Shows top 25 repos by lines of code +# post: +# requestBody: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/defaultWidgetArgs' +# responses: +# '200': +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Timeseries' +# description: 200 Response +# default: +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Error' +# description: unexpected error +# security: +# - cookieAuth: [] +# summary: Shows top 25 jobs by total build time spent +# +######################################################################## + + + + + +""" +This is the TopN CI jobs by total build time renderer for Kibble +""" + +import json +import time +import re + +def run(API, environ, indata, session): + + # We need to be logged in for this! + if not session.user: + raise API.exception(403, "You must be logged in to use this API endpoint! %s") + + now = time.time() + + # First, fetch the view if we have such a thing enabled + viewList = [] + if indata.get('view'): + viewList = session.getView(indata.get('view')) + if indata.get('subfilter'): + viewList = session.subFilter(indata.get('subfilter'), view = viewList) + + dateTo = indata.get('to', int(time.time())) + dateFrom = indata.get('from', dateTo - (86400*30*6)) # Default to a 6 month span + + #################################################################### + #################################################################### + dOrg = session.user['defaultOrganisation'] or "apache" + query = { + 'query': { + 'bool': { + 'must': [ + {'range': + { + 'date': { + 'from': time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(dateFrom)), + 'to': time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(dateTo)) + } + } + }, + { + 'term': { + 'organisation': dOrg + } + } + ] + } + } + } + # Source-specific or view-specific?? + if indata.get('source'): + query['query']['bool']['must'].append({'term': {'sourceID': indata.get('source')}}) + elif viewList: + query['query']['bool']['must'].append({'terms': {'sourceID': viewList}}) + + query['aggs'] = { + 'by_job': { + 'terms': { + 'field': 'jobURL.keyword', + 'size': 5000, + }, + 'aggs': { + 'duration': { + 'sum': { + 'field': 'duration' + } + }, + 'ci': { + 'terms': { + 'field': 'ci.keyword', + 'size': 1 + } + }, + 'name': { + 'terms': { + 'field': 'job.keyword', + 'size': 1 + } + } + } + } + } + + res = session.DB.ES.search( + index=session.DB.dbname, + doc_type="ci_build", + size = 0, + body = query + ) + + jobs = [] + for doc in res['aggregations']['by_job']['buckets']: + job = doc['key'] + builds = doc['doc_count'] + duration = doc['duration']['value'] + ci = doc['ci']['buckets'][0]['key'] + jobname = doc['name']['buckets'][0]['key'] + jobs.append([builds, duration, jobname, ci]) + + topjobs = sorted(jobs, key = lambda x: int(x[1]), reverse = True) + top = topjobs[0:24] + if len(topjobs) > 25: + count = 0 + for repo in topjobs[25:]: + count += repo[1] + top.append(["Other jobs", 1, count, '??']) + + tophash = {} + for v in top: + tophash["%s (%s)" % (v[2], v[3])] = int((v[1]/86400)) + + JSON_OUT = { + 'counts': tophash, + 'okay': True, + 'responseTime': time.time() - now, + } + yield json.dumps(JSON_OUT) -- To stop receiving notification emails like this one, please contact humbed...@apache.org.