This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 9bc9a36829 KYLIN-5465 add dashboard Ui and api (#2097)
9bc9a36829 is described below
commit 9bc9a36829f272d2049a3a4691be9c696306f9e0
Author: Syleechan <[email protected]>
AuthorDate: Fri Jun 30 14:55:35 2023 +0800
KYLIN-5465 add dashboard Ui and api (#2097)
* [KYLIN-5465] add dashboard Ui and api
* [KYLIN-5465] fix dependency
* [KYLIN-5465] fix job metrics not refresh issue
* [KYLIN-5465] fix code smell
* [KYLIN-5465] fix code smell 1
* [KYLIN-5465] fix code smell 1
* [KYLIN-5465] modify method usage of UI
---------
Co-authored-by: cli2 <[email protected]>
---
kystudio/package.json | 4 +-
kystudio/src/components/dashboard/BarEcharts.vue | 71 +++
kystudio/src/components/dashboard/LineEcharts.vue | 71 +++
kystudio/src/components/dashboard/chartOption.js | 99 +++
kystudio/src/components/dashboard/dashboard.vue | 702 +++++++++++++++++++++
kystudio/src/config/index.js | 5 +
kystudio/src/directive/index.js | 2 +-
kystudio/src/locale/en.js | 1 +
kystudio/src/router/index.js | 5 +
kystudio/src/service/api.js | 4 +-
kystudio/src/service/dashboard.js | 23 +
kystudio/src/store/dashboard.js | 20 +
kystudio/src/store/index.js | 2 +
kystudio/src/store/types.js | 7 +
src/common-server/pom.xml | 8 +
.../kylin/rest/controller/DashboardController.java | 92 +++
.../apache/kylin/rest/service/BasicService.java | 4 +
.../kylin/common/response/MetricsResponse.java | 34 +
.../src/main/resources/metadata-jdbc-h2.properties | 8 +-
.../main/resources/metadata-jdbc-mysql.properties | 6 +
.../metadata/query/JdbcQueryHistoryStore.java | 115 +++-
.../kylin/metadata/query/QueryHistoryDAO.java | 8 +
.../query/QueryHistoryRealizationTable.java | 5 +
.../apache/kylin/metadata/query/QueryMetrics.java | 6 +
.../kylin/metadata/query/QueryMetricsContext.java | 3 +
.../kylin/metadata/query/RDBMSQueryHistoryDAO.java | 18 +
.../metadata/query/util/QueryHisStoreUtil.java | 20 +
.../metadata/query/RDBMSQueryHistoryDaoTest.java | 43 ++
.../kylin/rest/service/DashboardService.java | 221 +++++++
.../kylin/rest/service/QueryHistoryService.java | 59 +-
.../kylin/rest/service/DashboardServiceTest.java | 394 ++++++++++++
31 files changed, 2046 insertions(+), 14 deletions(-)
diff --git a/kystudio/package.json b/kystudio/package.json
index d7ed33f2e8..952b2aa7b2 100644
--- a/kystudio/package.json
+++ b/kystudio/package.json
@@ -18,8 +18,9 @@
"@tweenjs/tween.js": "17.1.1",
"brace": "0.10.0",
"d3": "3.5.17",
+ "daterangepicker": "^3.1.0",
"dayjs": "1.7.7",
- "echarts": "4.2.0-rc.1",
+ "echarts": "^4.9.0",
"express-http-proxy": "0.11.0",
"jquery": "3.5.0",
"js-base64": "2.1.9",
@@ -44,6 +45,7 @@
"vue-router": "2.8.1",
"vue-virtual-scroller": "^1.0.10",
"vue2-ace-editor": "0.0.3",
+ "vue2-daterange-picker": "^0.6.8",
"vuex": "2.5.0"
},
"devDependencies": {
diff --git a/kystudio/src/components/dashboard/BarEcharts.vue
b/kystudio/src/components/dashboard/BarEcharts.vue
new file mode 100644
index 0000000000..70d1fa4409
--- /dev/null
+++ b/kystudio/src/components/dashboard/BarEcharts.vue
@@ -0,0 +1,71 @@
+<template>
+ <div :id="uuid" :style="style"></div>
+</template>
+
+<script>
+import * as echarts from 'echarts'
+import {Component, Watch} from 'vue-property-decorator'
+import Vue from 'vue'
+
+const idGen = () => {
+ return new Date().getTime() + 1
+}
+
+@Component({
+ props: {
+ height: {
+ type: String,
+ default: '300px'
+ },
+ width: {
+ type: String,
+ default: '450px'
+ },
+
+ options: {
+ type: Object,
+ default: null
+ }
+ }
+})
+
+export default class BarEcharts extends Vue {
+ @Watch('width')
+ onWithChange () {
+ if (this.myChart) {
+ setTimeout(() => {
+ this.myChart.resize({
+ animation: {
+ duration: 300
+ }
+ })
+ }, 0)
+ }
+ }
+
+ @Watch('options', {deep: true})
+ onOptionChange () {
+ if (this.myChart) {
+ this.myChart.dispose()
+ this.myChart = echarts.init(document.getElementById(this.uuid))
+ this.myChart.setOption(this.options, {notMerge: true})
+ }
+ }
+
+ get style () {
+ return {
+ height: this.height,
+ width: this.width
+ }
+ }
+
+ created () {
+ this.uuid = idGen()
+ }
+
+ mounted () {
+ this.myChart = echarts.init(document.getElementById(this.uuid))
+ this.myChart.setOption(this.options)
+ }
+}
+</script>
diff --git a/kystudio/src/components/dashboard/LineEcharts.vue
b/kystudio/src/components/dashboard/LineEcharts.vue
new file mode 100644
index 0000000000..478f34123d
--- /dev/null
+++ b/kystudio/src/components/dashboard/LineEcharts.vue
@@ -0,0 +1,71 @@
+<template>
+ <div :id="uuid" :style="style"></div>
+</template>
+
+<script>
+import * as echarts from 'echarts'
+import {Component, Watch} from 'vue-property-decorator'
+import Vue from 'vue'
+
+const idGen = () => {
+ return new Date().getTime() - 1
+}
+
+@Component({
+ props: {
+ height: {
+ type: String,
+ default: '300px'
+ },
+ width: {
+ type: String,
+ default: '450px'
+ },
+
+ options: {
+ type: Object,
+ default: null
+ }
+ }
+})
+
+export default class LineEcharts extends Vue {
+ @Watch('width')
+ onWithChange () {
+ if (this.myChart) {
+ setTimeout(() => {
+ this.myChart.resize({
+ animation: {
+ duration: 300
+ }
+ })
+ }, 0)
+ }
+ }
+
+ @Watch('options', {deep: true})
+ onOptionChange () {
+ if (this.myChart) {
+ this.myChart.dispose()
+ this.myChart = echarts.init(document.getElementById(this.uuid))
+ this.myChart.setOption(this.options, {notMerge: true})
+ }
+ }
+
+ get style () {
+ return {
+ height: this.height,
+ width: this.width
+ }
+ }
+
+ created () {
+ this.uuid = idGen()
+ }
+
+ mounted () {
+ this.myChart = echarts.init(document.getElementById(this.uuid))
+ this.myChart.setOption(this.options)
+ }
+}
+</script>
diff --git a/kystudio/src/components/dashboard/chartOption.js
b/kystudio/src/components/dashboard/chartOption.js
new file mode 100644
index 0000000000..936c2b0441
--- /dev/null
+++ b/kystudio/src/components/dashboard/chartOption.js
@@ -0,0 +1,99 @@
+export default {
+ barChartOptions: (xData, yData) => {
+ return {
+ title: {
+ text: '',
+ left: 'center',
+ top: -5
+ },
+ height: 255,
+ grid: {
+ top: 25,
+ left: 45,
+ right: 15,
+ bottom: 0
+ },
+ tooltip: {
+ trigger: 'axis',
+ axisPointer: {
+ type: 'shadow'
+ }
+ },
+ xAxis: [
+ {
+ type: 'category',
+ data: xData || [],
+ axisTick: {
+ alignWithLabel: true
+ }
+ }
+ ],
+ yAxis: [
+ {
+ type: 'value',
+ axisTick: false
+ }
+ ],
+ series: [
+ {
+ name: 'value',
+ type: 'bar',
+ barWidth: '60%',
+ data: yData || [],
+ label: {
+ show: false
+ },
+ itemStyle: {
+ normal: {
+ color: function (params) {
+ const colorList = ['#5470c6', '#91cc75', '#fac858', '#ee6666',
'#73c0de', '#3ba272', '#fc8452', '#9a60b4', '#ea7ccc']
+ return colorList[params.dataIndex]
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ lineChartOptions: (xData, yData) => {
+ return {
+ title: {
+ text: '',
+ left: 'center',
+ top: 15
+ },
+ grid: {
+ top: 45,
+ left: 45,
+ right: 15,
+ height: 235
+ },
+ tooltip: {
+ trigger: 'axis'
+ },
+ xAxis: [
+ {
+ type: 'category',
+ data: xData || [],
+ axisTick: {
+ alignWithLabel: true
+ }
+ }
+ ],
+ yAxis: [
+ {
+ type: 'value',
+ axisTick: false
+ }
+ ],
+ series: [
+ {
+ name: 'value',
+ type: 'line',
+ color: '#5470c6',
+ data: yData || []
+ }
+ ]
+ }
+ }
+}
diff --git a/kystudio/src/components/dashboard/dashboard.vue
b/kystudio/src/components/dashboard/dashboard.vue
new file mode 100644
index 0000000000..986bd635a8
--- /dev/null
+++ b/kystudio/src/components/dashboard/dashboard.vue
@@ -0,0 +1,702 @@
+<template>
+ <div class="dashboard" v-loading="isLoading">
+ <header class="dashboard-header">
+ <h1 class="ksd-title-label"
v-if="projectSettings">{{projectSettings.project || projectSettings.alias}}</h1>
+ <div id="datepicker">
+ <date-range-picker v-model="pickerDates" :locale-data="{format:
'yyyy-mm-dd'}" @update="changeDateRange">
+ <template v-slot:input="picker" style="min-width: 350px;" >
+ {{ pickerDates.startDate }} - {{ pickerDates.endDate }}
+ </template>
+ </date-range-picker>
+ </div>
+ </header>
+
+ <section class="dashboard-body" v-if="projectSettings">
+ <!-- Model Metrics -->
+ <div class = "row">
+ <div class="el-col-sm-1">
+ <el-tooltip placement="bottom" :content="'As of ' + currentTime">
+ <div class="square-big" tool-palcement="bottom">
+ <div class = "title">
+ TOTAL MODEL COUNT
+ </div>
+ <div class="metric" v-if="totalModel || totalModel === 0">
+ {{totalModel}}
+ </div>
+ <div class="metric" v-if="!totalModel && (totalModel !== 0)">
+ --
+ </div>
+ <a class="description" @click="toModel()">
+ More Details
+ </a>
+ </div>
+ </el-tooltip>
+ <el-tooltip placement="bottom">
+ <div slot="content">{{"Max:" + maxExpansionRate + " | Min:" +
minExpansionRate}}<br/>{{"As of " + currentTime}}</div>
+ <div class="square-big" tool-palcement="bottom">
+ <div class = "title">
+ AVG MODEL EXPANSION
+ </div>
+ <div class="metric" v-if="avgExpansionRate || avgExpansionRate
=== 0">
+ {{avgExpansionRate}}
+ </div>
+ <div class="metric" v-if="!avgExpansionRate && (avgExpansionRate
!== 0)">
+ --
+ </div>
+ <a class="description" @click="toModel()">
+ More Details
+ </a>
+ </div>
+ </el-tooltip>
+ </div>
+ </div>
+ <!-- Query Metrics -->
+ <div class = "col-sm-10">
+ <div class = "row">
+ <div class = "col-sm-2" style="width: 20%">
+ <div class = "square" :class="{'square-active': currentSquare ===
'queryCount'}" @click="queryCountChart()">
+ <div class = "title">
+ QUERY<br/>COUNT
+ </div>
+ <div class = "metric" v-if="queryCount || queryCount === 0">
+ {{queryCount}}
+ </div>
+ <div class="metric" v-if="!queryCount && (queryCount !== 0)">
+ --
+ </div>
+ <a class="description" @click="toQuery()">
+ More Details
+ </a>
+ </div>
+ </div>
+ <div class="col-sm-2" style="width: 20%">
+ <div class="square1" :class="{'square-active': currentSquare ===
'queryAvg'}" @click="queryAvgChart()" >
+ <div class="title">
+ AVG QUERY LATENCY
+ </div>
+ <div class="metric" v-if="avgQueryLatency || avgQueryLatency ===
0">
+ {{numFilter(avgQueryLatency / 1000)}}<span class="unit">
sec</span>
+ </div>
+ <div class="metric" v-if="!avgQueryLatency && (avgQueryLatency
!== 0)">
+ --
+ </div>
+ <a class="description" @click="toQuery()">
+ More Details
+ </a>
+ </div>
+ </div>
+ <!-- Job Metrics -->
+ <div class="col-sm-2" style="width: 20%">
+ <div class="square2" :class="{'square-active': currentSquare ===
'jobCount'}" @click="jobCountChart()" >
+ <div class="title">
+ JOB<br/>COUNT
+ </div>
+ <div class="metric" v-if="jobCount || jobCount === 0">
+ {{jobCount}}
+ </div>
+ <div class="metric" v-if="!avgQueryLatency && (avgQueryLatency
!== 0)">
+ --
+ </div>
+ <a class="description" @click="toJob()">
+ More Details
+ </a>
+ </div>
+ </div>
+ <div class="col-sm-2" style="width: 20%">
+ <div class="square3" :class="{'square-active': currentSquare ===
'jobAvgBuild'}" @click="jobAvgBuildChart()" >
+ <div class="title">
+ AVG BUILD TIME PER {{jobUnit}}
+ </div>
+ <div class="metric" v-if="jobAvgBuildTime || jobAvgBuildTime ===
0">
+ {{jobAvgBuildTime}}<span class="unit"> {{jobTimeUnit}}</span>
+ </div>
+ <div class="metric" v-if="!avgQueryLatency && (avgQueryLatency
!== 0)">
+ --
+ </div>
+ <a class="description" @click="toJob()">
+ More Details
+ </a>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!-- charts -->
+ <div class="row">
+ <div class="col-sm-2" v-if="barChartOptions">
+ <div class="barChartSquare">
+ <div class="form-control">
+ Show Value: <input type="checkbox"
v-model="barChartOptions.series[0].label.show" @click="showBarChartValue()">
+ </div>
+ <BarEcharts :options="barChartOptions"/>
+ </div>
+ </div>
+ <div class="col-sm-2" v-if="lineChartOptions">
+ <div class="lineChartSquare">
+ <select class="select-control"
+ v-model="currentSelectFilter"
+ @change="getCurrentSelectedLineChart">
+ <option v-for="filter in chartFilter" :value="filter.name">
+ {{filter.name}}
+ </option>
+ </select>
+ <LineEcharts :options="lineChartOptions"/>
+ </div>
+ </div>
+ </div>
+ </section>
+ <kylin-empty-data v-else>
+ </kylin-empty-data>
+ </div>
+</template>
+
+<script>
+import Vue from 'vue'
+import {mapActions, mapGetters} from 'vuex'
+import { Component } from 'vue-property-decorator'
+import {handleError, handleSuccessAsync} from '../../util/index'
+import DateRangePicker from 'vue2-daterange-picker'
+import 'vue2-daterange-picker/dist/vue2-daterange-picker.css'
+import moment from 'moment'
+import baseOptions from './chartOption'
+import BarEcharts from './BarEcharts'
+import LineEcharts from './LineEcharts'
+Vue.component('date-range-picker', DateRangePicker)
+
+@Component({
+ methods: {
+ ...mapActions({
+ getModelStatistics: 'GET_MODEL_STATISTICS',
+ getQueryStatistics: 'GET_QUERY_STATISTICS',
+ fetchProjectSettings: 'FETCH_PROJECT_SETTINGS',
+ getChartData: 'GET_CHART_DATA',
+ getJobStatistics: 'GET_JOB_STATISTICS'
+ })
+ },
+ computed: {
+ ...mapGetters([
+ 'currentSelectedProject',
+ 'currentProjectData'
+ ])
+ },
+ components: {DateRangePicker, BarEcharts, LineEcharts},
+ watch: {
+ pickerDates: {
+ handler (newValue, oldValue) {
+ },
+ deep: true
+ },
+ barChartOptions: {
+ handler (newValue, oldValue) {
+ },
+ deep: true
+ },
+ lineChartOptions: {
+ handler (newValue, oldValue) {
+ },
+ deep: true
+ },
+ currentSelectFilter: {
+ handler (newValue, oldValue) {
+ },
+ deep: true
+ }
+ }
+})
+
+export default class Dashboard extends Vue {
+ // name = 'Dashboard'
+ isLoading = false
+ projectSettings = null;
+ totalModel = 0;
+ avgExpansionRate = 0;
+ minExpansionRate = 0;
+ maxExpansionRate = 0;
+ currentTime = moment().format('YYYY-MM-DD');
+ queryCount = 0;
+ avgQueryLatency = 0;
+ currentSquare = '';
+ pickerDates = {
+ startDate: '',
+ endDate: ''
+ };
+ options = null
+ // category: JOB, QUERY
+ // metric: QUERY:{count, avg_query_latency}
+ // dimension : filter by model or date(day, week)
+ barChart = null;
+ lineChart = null;
+ barChartOptions = null;
+ lineChartOptions = null;
+ category = ['QUERY', 'JOB'];
+ metric = [
+ {name: 'query count', value: 'QUERY_COUNT'},
+ {name: 'avg query latency', value: 'AVG_QUERY_LATENCY'},
+ {name: 'job count', value: 'JOB_COUNT'},
+ {name: 'avg build time', value: 'AVG_JOB_BUILD_TIME'}
+ ];
+ dimension = [
+ {name: 'project', value: 'PROJECT'},
+ {name: 'model', value: 'MODEL'},
+ {name: 'day', value: 'DAY'},
+ {name: 'week', value: 'WEEK'},
+ {name: 'month', value: 'MONTH'}
+ ];
+ chartFilter = [
+ {name: 'Daily', value: 'day'},
+ {name: 'Weekly', value: 'week'},
+ {name: 'Monthly', value: 'month'}
+ ];
+ currentSelectFilter = this.chartFilter[1].name
+ barChartCategory = null;
+ barChartMetric = null;
+ lineChartCategory = null;
+ lineChartMetric = null;
+ chartType = '';
+ barChartDimension = this.dimension[1].value;
+ lineChartDimension = this.dimension[3].value;
+ jobUnit = 'MB';
+ jobTimeUnit = 'sec';
+ jobCount = 0;
+ jobTotalByteSize = 0;
+ jobTotalLatency = 0;
+ jobAvgBuildTime = 0;
+ toModel () {
+ this.$router.push('/studio/model')
+ }
+ toQuery () {
+ this.$router.push('/query/queryhistory')
+ }
+ toJob () {
+ this.$router.push('/monitor/job')
+ }
+ showLoading () {
+ this.isLoading = true
+ }
+ changeDateRange (val) {
+ this.pickerDates = val
+ this.pickerDates.startDate =
moment(this.pickerDates.startDate).format('YYYY-MM-DD')
+ this.pickerDates.endDate =
moment(this.pickerDates.endDate).format('YYYY-MM-DD')
+ this.refresh()
+ }
+ refresh () {
+ this.getModelInfo()
+ this.getQueryMetrics()
+ this.createCharts()
+ this.getJobMetrics()
+ }
+ hideLoading () {
+ this.isLoading = false
+ }
+ numFilter (value) {
+ const realVal = parseFloat(value).toFixed(2)
+ return realVal
+ }
+ async getCurrentSettings () {
+ this.showLoading()
+ try {
+ const projectName = this.currentProjectData.name
+ const response = await this.fetchProjectSettings({ projectName })
+ const result = await handleSuccessAsync(response)
+ const quotaSize = result.storage_quota_size / 1024 / 1024 / 1024 / 1024
+ this.projectSettings = {...result, storage_quota_tb_size:
quotaSize.toFixed(2)}
+ } catch (e) {
+ handleError(e)
+ }
+ this.hideLoading()
+ }
+ getModelInfo () {
+ this.getModelStatistics({projectName: this.currentProjectData.name,
modelName: null}).then((res) => {
+ const data = res.data
+ if (data) {
+ this.totalModel = data.totalModel
+ this.avgExpansionRate = this.numFilter(data.avgModelExpansion)
+ this.minExpansionRate = this.numFilter(data.minModelExpansion)
+ this.maxExpansionRate = this.numFilter(data.maxModelExpansion)
+ }
+ })
+ }
+ getQueryMetrics () {
+ this.getQueryStatistics({projectName: this.currentProjectData.name,
+ startTime: String(this.pickerDates.startDate),
+ endTime: String(this.pickerDates.endDate),
+ modelName: null}).then((res) => {
+ const data = res.data
+ if (data) {
+ this.queryCount = data.queryCount
+ this.avgQueryLatency = data.avgQueryLatency
+ }
+ })
+ }
+ getJobMetrics () {
+ this.getJobStatistics({projectName: this.currentProjectData.name,
+ startTime: String(this.pickerDates.startDate),
+ endTime: String(this.pickerDates.endDate),
+ modelName: null}).then((res) => {
+ const data = res.data
+ if (data) {
+ this.jobCount = data.jobCount
+ this.jobTotalByteSize = data.jobTotalByteSize
+ this.jobTotalLatency = data.jobTotalLatency
+ if (this.jobTotalByteSize > 0) {
+ this.jobAvgBuildTime =
this.numFilter(this.determineAvgJobBuildTimeUnit(
+ this.jobTotalLatency / this.jobTotalByteSize))
+ } else {
+ this.jobAvgBuildTime = 0
+ }
+ }
+ })
+ }
+ data () {
+ const startDate = moment().subtract('days', 5).format('YYYY-MM-DD')
+ const endDate = moment().subtract('days', -1).format('YYYY-MM-DD')
+ return {
+ pickerDates: {
+ startDate,
+ endDate
+ }
+ }
+ }
+ mounted () {
+ this.getCurrentSettings()
+ this.getModelInfo()
+ this.getQueryMetrics()
+ this.queryCountChart()
+ this.getJobMetrics()
+ }
+ // create chart
+ queryCountChart () {
+ this.currentSquare = 'queryCount'
+ this.barChartCategory = this.category[0]
+ this.barChartMetric = this.metric[0].value
+ this.lineChartCategory = this.category[0]
+ this.lineChartMetric = this.metric[0].value
+ this.createCharts()
+ }
+ queryAvgChart () {
+ this.currentSquare = 'queryAvg'
+ this.barChartCategory = this.category[0]
+ this.barChartMetric = this.metric[1].value
+ this.lineChartCategory = this.category[0]
+ this.lineChartMetric = this.metric[1].value
+ this.createCharts()
+ }
+ jobCountChart () {
+ this.currentSquare = 'jobCount'
+ this.barChartCategory = this.category[1]
+ this.barChartMetric = this.metric[2].value
+ this.lineChartCategory = this.category[1]
+ this.lineChartMetric = this.metric[2].value
+ this.createCharts()
+ }
+ jobAvgBuildChart () {
+ this.currentSquare = 'jobAvgBuild'
+ this.barChartCategory = this.category[1]
+ this.barChartMetric = this.metric[3].value
+ this.lineChartCategory = this.category[1]
+ this.lineChartMetric = this.metric[3].value
+ this.createCharts()
+ }
+ createCharts () {
+ this.createChart(this.barChartDimension, this.barChartCategory,
this.barChartMetric, 'bar')
+ this.createChart(this.lineChartDimension, this.lineChartCategory,
this.lineChartMetric, 'line')
+ }
+ async createChart (dimension, category, metric, chartType) {
+ const startTime = String(this.pickerDates.startDate)
+ const endTime = String(this.pickerDates.endDate)
+ if (chartType === 'bar') {
+ let xdata = []
+ let ydata = []
+ this.barChartOptions = baseOptions.barChartOptions(xdata, ydata)
+ this.barChartOptions.series.type = chartType
+ const data = await this.getChartDataResult(category, dimension, metric,
+ this.currentProjectData.name, null, startTime, endTime)
+ for (let key in data) {
+ xdata.push(key)
+ if (this.barChartMetric === this.metric[1].value) {
+ ydata.push(this.numFilter(data[key] / 1000))
+ } else if (this.barChartMetric === this.metric[3].value) {
+ ydata.push(this.numFilter(this.determineAvgJobBuildTime(data[key])))
+ } else {
+ ydata.push(data[key])
+ }
+ }
+ this.barChartOptions.xAxis.data = xdata
+ this.barChartOptions.series.data = ydata
+ this.barChartOptions.title.text = metric + ' BY ' + dimension
+ console.log(this.barChartOptions)
+ } else if (chartType === 'line') {
+ let xdata = []
+ let ydata = []
+ this.lineChartOptions = baseOptions.lineChartOptions(xdata, ydata)
+ this.lineChartOptions.series.type = chartType
+ const data = await this.getChartDataResult(category, dimension, metric,
+ this.currentProjectData.name, null, startTime, endTime)
+ for (let key in data) {
+ xdata.push(key)
+ if (this.lineChartMetric === this.metric[1].value) {
+ ydata.push(this.numFilter(data[key] / 1000))
+ } else if (this.lineChartMetric === this.metric[3].value) {
+ ydata.push(this.numFilter(this.determineAvgJobBuildTime(data[key])))
+ } else {
+ ydata.push(data[key])
+ }
+ }
+ this.lineChartOptions.title.text = metric + ' BY ' + dimension
+ this.lineChartOptions.xAxis.data = xdata
+ this.lineChartOptions.series.data = ydata
+ }
+ }
+ // if data >= 100 ms/byte then transform display unit sec/MB to min/MB
+ determineAvgJobBuildTimeUnit (data) {
+ let avgTime = data < 100 ? data * 1024 * 1024 / 1000 : data * 1024 * 1024
/ 1000 / 60
+ this.jobTimeUnit = data < 100 ? 'sec' : 'min'
+ return avgTime
+ }
+ determineAvgJobBuildTime (data) {
+ return this.jobTimeUnit === 'sec' ? data * 1024 * 1024 / 1000 : data *
1024 * 1024 / 1000 / 60
+ }
+ async getChartDataResult (category, dimension, metric, projectName,
modelName, startTime, endTime) {
+ const res = await this.getChartData({
+ category: category,
+ dimension: dimension,
+ metric: metric,
+ projectName: projectName,
+ modelName: modelName,
+ startTime: startTime,
+ endTime: endTime
+ })
+ if (res) return res.data
+ return null
+ }
+ // refresh line chart by dimension
+ getCurrentSelectedLineChart () {
+ let temp = this.currentSelectFilter
+ for (let k in this.chartFilter) {
+ if (this.chartFilter[k].name === temp) {
+ temp = this.chartFilter[k].value
+ break
+ }
+ }
+ for (let k in this.dimension) {
+ if (temp === this.dimension[k].name) {
+ this.lineChartDimension = this.dimension[k].value
+ break
+ }
+ }
+ this.createChart(this.lineChartDimension, this.lineChartCategory,
this.lineChartMetric, 'line')
+ }
+ // checkbox display
+ showBarChartValue () {
+ this.barChartOptions.series[0].label.show =
this.barChartOptions.series[0].label.show === false
+ }
+}
+</script>
+
+<style lang="less">
+.dashboard {
+ height: 100%;
+ padding: 20px;
+ .dashboard-header {
+ margin-bottom: 20px;
+ }
+ .dashboard-header h1 {
+ font-size: 18px;
+ }
+ .dashboard-body {
+ height: 100%;
+ }
+ .square-big {
+ border: 5px solid #ddd;
+ text-align: center;
+ width: 185px;
+ height: 225px;
+ padding: 25px 0;
+ margin-bottom: 30px;
+ .title {
+ font-size: 24px;
+ height: 55px;
+ }
+ .metric {
+ padding: 15px 0;
+ font-size: 50px ;
+ font-weight: bolder;
+ height: 100px;
+ color: #06d;
+ .unit {
+ font-size: 35px;
+ }
+ }
+ .description {
+ font-size: 18px;
+ color: #6a6a6a;
+ }
+ }
+ .square {
+ border: 2px solid #ddd;
+ text-align: center;
+ width: 170px;
+ height: 165px;
+ cursor: zoom-in;
+ padding-top: 20px;
+ padding-bottom: 15px;
+ margin-left: 230px;
+ .title {
+ font-size: 20px;
+ height: 65px;
+ }
+ .metric {
+ font-size: 35px ;
+ font-weight: bolder;
+ color: #8b1;
+ display: inline-block;
+ white-space: nowrap;
+ .unit {
+ font-size: 16px;
+ }
+ }
+ .description {
+ font-size: 15px;
+ color: #6a6a6a;
+ display: block;
+ }
+ }
+ .square-active {
+ border: 2px solid #6a6a6a;
+ }
+ .square1 {
+ margin-top: -204px;
+ border: 2px solid #ddd;
+ text-align: center;
+ width: 170px;
+ height: 165px;
+ cursor: zoom-in;
+ padding-top: 20px;
+ padding-bottom: 15px;
+ margin-left: 440px;
+ .title {
+ font-size: 20px;
+ height: 65px;
+ }
+ .metric {
+ font-size: 35px ;
+ font-weight: bolder;
+ color: #8b1;
+ display: inline-block;
+ white-space: nowrap;
+ .unit {
+ font-size: 16px;
+ }
+ }
+ .description {
+ font-size: 15px;
+ color: #6a6a6a;
+ display: block;
+ }
+ }
+ .square2 {
+ margin-top: -204px;
+ border: 2px solid #ddd;
+ text-align: center;
+ width: 170px;
+ height: 165px;
+ cursor: zoom-in;
+ padding-top: 20px;
+ padding-bottom: 15px;
+ margin-left: 650px;
+ .title {
+ font-size: 20px;
+ height: 65px;
+ }
+ .metric {
+ font-size: 35px ;
+ font-weight: bolder;
+ color: #8b1;
+ display: inline-block;
+ white-space: nowrap;
+ .unit {
+ font-size: 16px;
+ }
+ }
+ .description {
+ font-size: 15px;
+ color: #6a6a6a;
+ display: block;
+ }
+ }
+ .square3 {
+ margin-top: -204px;
+ border: 2px solid #ddd;
+ text-align: center;
+ width: 170px;
+ height: 165px;
+ cursor: zoom-in;
+ padding-top: 20px;
+ padding-bottom: 15px;
+ margin-left: 860px;
+ .title {
+ font-size: 20px;
+ height: 65px;
+ }
+ .metric {
+ font-size: 35px ;
+ font-weight: bolder;
+ color: #8b1;
+ display: inline-block;
+ white-space: nowrap;
+ .unit {
+ font-size: 16px;
+ }
+ }
+ .description {
+ font-size: 15px;
+ color: #6a6a6a;
+ display: block;
+ }
+ }
+ .barChartSquare {
+ margin-top: 30px;
+ border: 2px solid #ddd;
+ text-align: center;
+ width: 450px;
+ height: 335px;
+ cursor: zoom-in;
+ padding-top: 10px;
+ padding-bottom: 15px;
+ margin-left: 230px;
+ }
+ .lineChartSquare {
+ margin-top: -365px;
+ border: 2px solid #ddd;
+ text-align: right;
+ width: 450px;
+ height: 360px;
+ cursor: zoom-in;
+ margin-left: 710px;
+ }
+ .form-control {
+ border-radius: 0px !important;
+ box-shadow: none;
+ border-color: #5470c6;
+ padding-top: -25px;
+ padding-bottom: 15px;
+ width: 105px;
+ }
+ .select-control {
+ border-radius: 0px !important;
+ box-shadow: none;
+ border-color: #5470c6;
+ width: 80px;
+ padding: 4px;
+ }
+ #datepicker {
+ font-family: "Avenir", Helvetica, Arial, sans-serif;
+ -webkit-font-smoothing: antialiased;
+ -moz-osx-font-smoothing: grayscale;
+ text-align: right;
+ color: #2c3e50;
+ margin-right: 280px;
+ height: 24px;
+ margin-top: -25px;
+ }
+}
+</style>
diff --git a/kystudio/src/config/index.js b/kystudio/src/config/index.js
index b8ecc8af02..f35e572b7d 100644
--- a/kystudio/src/config/index.js
+++ b/kystudio/src/config/index.js
@@ -89,6 +89,11 @@ export const menusData = [
name: 'group',
path: '/admin/group',
icon: 'el-ksd-icon-nav_user_group_24'
+ },
+ {
+ name: 'dashboard',
+ path: '/dashboard',
+ icon: 'el-ksd-icon-nav_dashboard_24'
}
]
diff --git a/kystudio/src/directive/index.js b/kystudio/src/directive/index.js
index 86b2c3cecb..f66286e96d 100644
--- a/kystudio/src/directive/index.js
+++ b/kystudio/src/directive/index.js
@@ -4,7 +4,6 @@ import Scrollbar from 'smooth-scrollbar'
import store from '../store'
import { stopPropagation, on } from 'util/event'
import { closestElm } from 'util'
-// import commonTip from 'components/common/common_tip'
import ElementUI from 'kyligence-kylin-ui'
const nodeList = []
const ctx = '@@clickoutsideContext'
@@ -707,6 +706,7 @@ Vue.directive('custom-tooltip', {
}
})
+
// MutationObserver 方式监听 dom attrs 的改变
function licenseDom (id) {
if (!parentList[id]) return
diff --git a/kystudio/src/locale/en.js b/kystudio/src/locale/en.js
index 22357326e7..25a367e66d 100644
--- a/kystudio/src/locale/en.js
+++ b/kystudio/src/locale/en.js
@@ -469,6 +469,7 @@ exports.default = {
insight: 'Insight',
monitor: 'Monitor',
system: 'System',
+ dashboard: 'Dashboard',
setting: 'Setting',
project: 'Project',
auto: 'Auto-Modeling',
diff --git a/kystudio/src/router/index.js b/kystudio/src/router/index.js
index 4c75e0a7e7..29e93ab054 100644
--- a/kystudio/src/router/index.js
+++ b/kystudio/src/router/index.js
@@ -10,6 +10,7 @@ import Insight from 'components/query/insight'
import queryHistory from 'components/query/query_history'
import jobs from 'components/monitor/batchJobs/jobs'
import streamingJobs from 'components/monitor/streamingJobs/streamingJobs'
+import dashboard from 'components/dashboard/dashboard'
import { bindRouterGuard } from './routerGuard.js'
Vue.use(Router)
@@ -54,6 +55,10 @@ let routerOptions = {
name: 'Setting',
path: '/setting',
component: () => import('../components/setting/setting.vue')
+ }, {
+ name: 'Dashboard',
+ path: '/dashboard',
+ component: dashboard
}, {
name: 'Source',
path: 'studio/source',
diff --git a/kystudio/src/service/api.js b/kystudio/src/service/api.js
index 7a22502816..ffaa073188 100644
--- a/kystudio/src/service/api.js
+++ b/kystudio/src/service/api.js
@@ -8,6 +8,7 @@ import userApi from './user'
import systemApi from './system'
import datasourceApi from './datasource'
import monitorApi from './monitor'
+import dashboardApi from './dashboard'
// console.log(base64)
Vue.use(VueResource)
export default {
@@ -18,5 +19,6 @@ export default {
user: userApi,
system: systemApi,
datasource: datasourceApi,
- monitor: monitorApi
+ monitor: monitorApi,
+ dashboard: dashboardApi
}
diff --git a/kystudio/src/service/dashboard.js
b/kystudio/src/service/dashboard.js
new file mode 100644
index 0000000000..a1102a2da7
--- /dev/null
+++ b/kystudio/src/service/dashboard.js
@@ -0,0 +1,23 @@
+import Vue from 'vue'
+import VueResource from 'vue-resource'
+import { apiUrl } from '../config'
+
+Vue.use(VueResource)
+
+export default {
+ getModelStatistics: (para) => {
+ return Vue.resource(apiUrl + 'dashboard/metric/model').get(para)
+ },
+ getQueryStatistics: (para) => {
+ return Vue.resource(apiUrl + 'dashboard/metric/query').get(para)
+ },
+ // category: JOB, QUERY
+ // dimension: QUERY:{count, avg_query_latency}
+ // metric: filter by model or date(day, week)
+ getChartData: (category, dimension, metric, projectName, modelName,
startTime, endTime) => {
+ return Vue.resource(apiUrl +
`dashboard/chart/${category}/${dimension}/${metric}?projectName=${projectName}&modelName=${modelName}&startTime=${startTime}&endTime=${endTime}`).get()
+ },
+ getJobStatistics: (para) => {
+ return Vue.resource(apiUrl + 'dashboard/metric/job').get(para)
+ }
+}
diff --git a/kystudio/src/store/dashboard.js b/kystudio/src/store/dashboard.js
new file mode 100644
index 0000000000..ca49dcba6a
--- /dev/null
+++ b/kystudio/src/store/dashboard.js
@@ -0,0 +1,20 @@
+import * as types from './types'
+import api from '../service/api'
+
+export default {
+ state: {},
+ actions: {
+ [types.GET_MODEL_STATISTICS]: function ({commit, state}, param) {
+ return api.dashboard.getModelStatistics(param)
+ },
+ [types.GET_QUERY_STATISTICS]: function ({commit, state}, param) {
+ return api.dashboard.getQueryStatistics(param)
+ },
+ [types.GET_CHART_DATA]: function ({commit, state}, param) {
+ return api.dashboard.getChartData(param.category, param.metric,
param.dimension, param.projectName, param.modelName, param.startTime,
param.endTime)
+ },
+ [types.GET_JOB_STATISTICS]: function ({commit, state}, param) {
+ return api.dashboard.getJobStatistics(param)
+ }
+ }
+}
diff --git a/kystudio/src/store/index.js b/kystudio/src/store/index.js
index ec0956b973..dc51ad0d43 100644
--- a/kystudio/src/store/index.js
+++ b/kystudio/src/store/index.js
@@ -12,6 +12,7 @@ import system from './system'
import monitor from './monitor'
import capacity from './capacity'
import * as actionTypes from './types'
+import dashboard from './dashboard'
export default new Vuex.Store({
modules: {
@@ -24,6 +25,7 @@ export default new Vuex.Store({
system: system,
monitor: monitor,
capacity: capacity,
+ dashboard: dashboard,
modals: {}
}
})
diff --git a/kystudio/src/store/types.js b/kystudio/src/store/types.js
index 7ce238b2e7..d852acf269 100644
--- a/kystudio/src/store/types.js
+++ b/kystudio/src/store/types.js
@@ -439,3 +439,10 @@ export const RESET_CAPACITY_DATA = 'RESET_CAPACITY_DATA'
export const SET_NODES_LIST = 'SET_NODES_LIST'
export const SET_NODES_INFOS = 'SET_NODES_INFOS'
export const SET_SYSTEM_CAPACITY_INFO = 'SET_SYSTEM_CAPACITY_INFO'
+
+// dashboard
+export const GET_MODEL_STATISTICS = 'GET_MODEL_STATISTICS'
+export const GET_QUERY_STATISTICS = 'GET_QUERY_STATISTICS'
+export const GET_CHART_DATA = 'GET_CHART_DATA'
+export const GET_JOB_STATISTICS = 'GET_JOB_STATISTICS'
+
diff --git a/src/common-server/pom.xml b/src/common-server/pom.xml
index d85c09ad2c..b3dc4ed315 100644
--- a/src/common-server/pom.xml
+++ b/src/common-server/pom.xml
@@ -142,5 +142,13 @@
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-metrics</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-modeling-service</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git
a/src/common-server/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
b/src/common-server/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
new file mode 100644
index 0000000000..0c2f2648d7
--- /dev/null
+++
b/src/common-server/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.controller;
+
+import org.apache.kylin.common.response.MetricsResponse;
+import org.apache.kylin.query.exception.UnsupportedQueryException;
+import org.apache.kylin.rest.service.DashboardService;
+import org.apache.kylin.rest.service.ModelService;
+import org.apache.kylin.rest.service.QueryService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+@Controller
+@RequestMapping(value = "/api/dashboard")
+public class DashboardController extends NBasicController {
+ public static final Logger logger =
LoggerFactory.getLogger(DashboardController.class);
+
+ @Autowired
+ DashboardService dashboardService;
+
+ @Autowired
+ ModelService modelService;
+
+ @Autowired
+ QueryService queryService;
+
+
+ @RequestMapping(value = "/metric/model", method = { RequestMethod.GET })
+ @ResponseBody
+ public MetricsResponse getCubeMetrics(@RequestParam(value = "projectName")
String projectName,
+ @RequestParam(value = "modelName", required = false) String
modelName) {
+ dashboardService.checkAuthorization(projectName);
+ return dashboardService.getModelMetrics(projectName, modelName);
+ }
+
+ @RequestMapping(value = "/metric/query", method = RequestMethod.GET)
+ @ResponseBody
+ public MetricsResponse getQueryMetrics(@RequestParam(value =
"projectName", required = false) String projectName,
+ @RequestParam(value = "modelName", required = false) String
cubeName,
+ @RequestParam(value = "startTime") String startTime,
@RequestParam(value = "endTime") String endTime) {
+ dashboardService.checkAuthorization(projectName);
+ return dashboardService.getQueryMetrics(projectName, startTime,
endTime);
+ }
+
+ @RequestMapping(value = "/metric/job", method = RequestMethod.GET)
+ @ResponseBody
+ public MetricsResponse getJobMetrics(@RequestParam(value = "projectName",
required = false) String projectName,
+ @RequestParam(value = "modelName", required = false) String
cubeName,
+ @RequestParam(value = "startTime") String startTime,
@RequestParam(value = "endTime") String endTime) {
+ dashboardService.checkAuthorization(projectName);
+ return dashboardService.getJobMetrics(projectName, startTime, endTime);
+ }
+
+ @RequestMapping(value = "/chart/{category}/{metric}/{dimension}", method =
RequestMethod.GET)
+ @ResponseBody
+ public MetricsResponse getChartData(@PathVariable String dimension,
@PathVariable String metric,
+ @PathVariable String category, @RequestParam(value =
"projectName", required = false) String projectName,
+ @RequestParam(value = "modelName", required = false) String
cubeName,
+ @RequestParam(value = "startTime") String startTime,
@RequestParam(value = "endTime") String endTime) {
+ dashboardService.checkAuthorization(projectName);
+ try {
+ return dashboardService.getChartData(category, projectName,
startTime, endTime, dimension, metric);
+ } catch (Exception e) {
+ throw new UnsupportedQueryException("Category or Metric is not
right: { " + e.getMessage() + " }");
+ }
+ }
+
+
+}
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java
b/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java
index f157021be1..c08995edc6 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -161,4 +161,8 @@ public abstract class BasicService {
return null;
}, project);
}
+
+ public NProjectManager getProjectManager(){
+ return NProjectManager.getInstance(getConfig());
+ }
}
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/response/MetricsResponse.java
b/src/core-common/src/main/java/org/apache/kylin/common/response/MetricsResponse.java
new file mode 100644
index 0000000000..1eda0e22b5
--- /dev/null
+++
b/src/core-common/src/main/java/org/apache/kylin/common/response/MetricsResponse.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.response;
+
+import java.util.TreeMap;
+
+public class MetricsResponse extends TreeMap<String, Float> {
+
+ public static final long serialVersionUID = 1L;
+
+ public void increase(String key, Float increased) {
+ if (this.containsKey(key)) {
+ this.put(key, (this.get(key) + increased));
+ } else {
+ this.put(key, increased);
+ }
+ }
+}
diff --git a/src/core-common/src/main/resources/metadata-jdbc-h2.properties
b/src/core-common/src/main/resources/metadata-jdbc-h2.properties
index 77328950c1..21a9ddd38d 100644
--- a/src/core-common/src/main/resources/metadata-jdbc-h2.properties
+++ b/src/core-common/src/main/resources/metadata-jdbc-h2.properties
@@ -65,12 +65,16 @@ create.queryhistoryrealization.store.table=CREATE TABLE IF
NOT EXISTS `%s` ( \
`duration` BIGINT, \
`query_time` BIGINT, \
`project_name` VARCHAR(255), \
+ `query_first_day_of_month` BIGINT, \
+ `query_first_day_of_week` BIGINT, \
+ `query_day` BIGINT, \
primary key (`id`,`project_name`) \
);
-
create.queryhistoryrealization.store.tableindex1=CREATE INDEX %s_ix1 ON %s (
query_time );
create.queryhistoryrealization.store.tableindex2=CREATE INDEX %s_ix2 ON %s (
model );
-
+create.queryhistoryrealization.store.tableindex3=CREATE INDEX %s_ix3 ON %s (
query_first_day_of_month );
+create.queryhistoryrealization.store.tableindex4=CREATE INDEX %s_ix4 ON %s (
query_first_day_of_week );
+create.queryhistoryrealization.store.tableindex5=CREATE INDEX %s_ix5 ON %s (
query_day );
# RAW RECOMMENDATION STORE
create.rawrecommendation.store.table=CREATE TABLE IF NOT EXISTS `%s` ( \
`id` int not null auto_increment, \
diff --git a/src/core-common/src/main/resources/metadata-jdbc-mysql.properties
b/src/core-common/src/main/resources/metadata-jdbc-mysql.properties
index 6a2df4c3b1..48299736ce 100644
--- a/src/core-common/src/main/resources/metadata-jdbc-mysql.properties
+++ b/src/core-common/src/main/resources/metadata-jdbc-mysql.properties
@@ -87,11 +87,17 @@ create.queryhistoryrealization.store.table=CREATE TABLE IF
NOT EXISTS `%s` ( \
`duration` BIGINT, \
`query_time` BIGINT, \
`project_name` VARCHAR(255), \
+ `query_first_day_of_month` BIGINT, \
+ `query_first_day_of_week` BIGINT, \
+ `query_day` BIGINT, \
primary key (`id`,`project_name`) \
) DEFAULT CHARSET=utf8;
create.queryhistoryrealization.store.tableindex1=ALTER table %s ADD INDEX
%s_ix1(`query_time`);
create.queryhistoryrealization.store.tableindex2=ALTER table %s ADD INDEX
%s_ix2(`model`);
+create.queryhistoryrealization.store.tableindex3=ALTER table %s ADD INDEX
%s_ix3(`query_first_day_of_month`);
+create.queryhistoryrealization.store.tableindex4=ALTER table %s ADD INDEX
%s_ix4(`query_first_day_of_week`);
+create.queryhistoryrealization.store.tableindex5=ALTER table %s ADD INDEX
%s_ix5(`query_day`);
#### JDBC STREAMING JOB STATS STORE
create.streamingjobstats.store.table=CREATE TABLE IF NOT EXISTS `%s` ( \
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
index 76bf56aedf..6a696c0fdd 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
@@ -91,6 +91,8 @@ public class JdbcQueryHistoryStore {
public static final String ID_TABLE_ALIAS = "idTable";
public static final String DELETE_REALIZATION_LOG = "Delete {} row query
history realization takes {} ms";
+ public static final String UNSUPPORTED_MESSAGE = "Unsupported time
window!";
+
private final QueryHistoryTable queryHistoryTable;
private final QueryHistoryRealizationTable queryHistoryRealizationTable;
@@ -348,6 +350,19 @@ public class JdbcQueryHistoryStore {
}
}
+ public List<QueryStatistics> queryCountAndAvgDurationRealization(long
startTime, long endTime, String project) {
+ try (SqlSession session = sqlSessionFactory.openSession()) {
+ QueryStatisticsMapper mapper =
session.getMapper(QueryStatisticsMapper.class);
+ SelectStatementProvider statementProvider =
select(count(queryHistoryRealizationTable.queryId).as(COUNT),
+
avg(queryHistoryRealizationTable.duration).as("mean")).from(queryHistoryRealizationTable)
+ .where(queryHistoryRealizationTable.queryTime,
isGreaterThanOrEqualTo(startTime))
+ .and(queryHistoryRealizationTable.queryTime,
isLessThan(endTime))
+ .and(queryHistoryRealizationTable.projectName,
isEqualTo(project)).build()
+ .render(RenderingStrategies.MYBATIS3);
+ return mapper.selectMany(statementProvider);
+ }
+ }
+
public List<QueryStatistics> queryCountByModel(long startTime, long
endTime, String project) {
try (SqlSession session = sqlSessionFactory.openSession()) {
QueryStatisticsMapper mapper =
session.getMapper(QueryStatisticsMapper.class);
@@ -429,6 +444,26 @@ public class JdbcQueryHistoryStore {
}
}
+ public List<QueryStatistics> queryAvgDurationRealizationByTime(long
startTime, long endTime, String timeDimension,
+ String project) {
+ try (SqlSession session = sqlSessionFactory.openSession()) {
+ QueryStatisticsMapper mapper =
session.getMapper(QueryStatisticsMapper.class);
+ SelectStatementProvider statementProvider =
queryAvgDurationRealizationByTimeProvider(startTime, endTime,
+ timeDimension, project);
+ return mapper.selectMany(statementProvider);
+ }
+ }
+
+ public List<QueryStatistics> queryCountRealizationByTime(long startTime,
long endTime, String timeDimension,
+ String project) {
+ try (SqlSession session = sqlSessionFactory.openSession()) {
+ QueryStatisticsMapper mapper =
session.getMapper(QueryStatisticsMapper.class);
+ SelectStatementProvider statementProvider =
queryCountRealizationByTimeProvider(startTime, endTime,
+ timeDimension, project);
+ return mapper.selectMany(statementProvider);
+ }
+ }
+
public void deleteQueryHistory() {
long startTime = System.currentTimeMillis();
try (SqlSession session = sqlSessionFactory.openSession()) {
@@ -605,7 +640,13 @@ public class JdbcQueryHistoryStore {
.map(queryHistoryRealizationTable.queryTime)
.toPropertyWhenPresent("queryTime",
realizationMetrics::getQueryTime)
.map(queryHistoryRealizationTable.projectName)
- .toPropertyWhenPresent("projectName",
realizationMetrics::getProjectName).build()
+ .toPropertyWhenPresent("projectName",
realizationMetrics::getProjectName)
+ .map(queryHistoryRealizationTable.queryDay)
+ .toPropertyWhenPresent("queryDay",
realizationMetrics::getQueryDay)
+ .map(queryHistoryRealizationTable.queryFirstDayOfWeek)
+ .toPropertyWhenPresent("queryFirstDayOfWeek",
realizationMetrics::getQueryFirstDayOfWeek)
+ .map(queryHistoryRealizationTable.queryFirstDayOfMonth)
+ .toPropertyWhenPresent("queryFirstDayOfMonth",
realizationMetrics::getQueryFirstDayOfMonth).build()
.render(RenderingStrategies.MYBATIS3);
}
@@ -768,7 +809,7 @@ public class JdbcQueryHistoryStore {
.groupBy(queryHistoryTable.queryDay) //
.build().render(RenderingStrategies.MYBATIS3);
} else {
- throw new IllegalStateException("Unsupported time window!");
+ throw new IllegalStateException(UNSUPPORTED_MESSAGE);
}
}
@@ -799,7 +840,75 @@ public class JdbcQueryHistoryStore {
.groupBy(queryHistoryTable.queryDay) //
.build().render(RenderingStrategies.MYBATIS3);
} else {
- throw new IllegalStateException("Unsupported time window!");
+ throw new IllegalStateException(UNSUPPORTED_MESSAGE);
+ }
+ }
+
+ private SelectStatementProvider
queryAvgDurationRealizationByTimeProvider(long startTime, long endTime,
+ String timeDimension, String project) {
+ if (timeDimension.equalsIgnoreCase(MONTH)) {
+ return
select(queryHistoryRealizationTable.queryFirstDayOfMonth.as("time"),
+ avg(queryHistoryRealizationTable.duration).as("mean")) //
+ .from(queryHistoryRealizationTable) //
+ .where(queryHistoryRealizationTable.queryTime,
isGreaterThanOrEqualTo(startTime)) //
+ .and(queryHistoryRealizationTable.queryTime,
isLessThan(endTime)) //
+ .and(queryHistoryRealizationTable.projectName,
isEqualTo(project)) //
+
.groupBy(queryHistoryRealizationTable.queryFirstDayOfMonth) //
+ .build().render(RenderingStrategies.MYBATIS3);
+ } else if (timeDimension.equalsIgnoreCase(WEEK)) {
+ return
select(queryHistoryRealizationTable.queryFirstDayOfWeek.as("time"),
+ avg(queryHistoryRealizationTable.duration).as("mean")) //
+ .from(queryHistoryRealizationTable) //
+ .where(queryHistoryRealizationTable.queryTime,
isGreaterThanOrEqualTo(startTime)) //
+ .and(queryHistoryRealizationTable.queryTime,
isLessThan(endTime)) //
+ .and(queryHistoryRealizationTable.projectName,
isEqualTo(project)) //
+
.groupBy(queryHistoryRealizationTable.queryFirstDayOfWeek) //
+ .build().render(RenderingStrategies.MYBATIS3);
+ } else if (timeDimension.equalsIgnoreCase(DAY)) {
+ return select(queryHistoryRealizationTable.queryDay.as("time"),
+ avg(queryHistoryRealizationTable.duration).as("mean")) //
+ .from(queryHistoryRealizationTable) //
+ .where(queryHistoryRealizationTable.queryTime,
isGreaterThanOrEqualTo(startTime)) //
+ .and(queryHistoryRealizationTable.queryTime,
isLessThan(endTime)) //
+ .and(queryHistoryRealizationTable.projectName,
isEqualTo(project)) //
+ .groupBy(queryHistoryRealizationTable.queryDay) //
+ .build().render(RenderingStrategies.MYBATIS3);
+ } else {
+ throw new IllegalStateException(UNSUPPORTED_MESSAGE);
+ }
+ }
+
+ private SelectStatementProvider queryCountRealizationByTimeProvider(long
startTime, long endTime,
+ String timeDimension, String project) {
+ if (timeDimension.equalsIgnoreCase(MONTH)) {
+ return
select(queryHistoryRealizationTable.queryFirstDayOfMonth.as("time"),
+ count(queryHistoryRealizationTable.id).as(COUNT)) //
+ .from(queryHistoryRealizationTable) //
+ .where(queryHistoryRealizationTable.queryTime,
isGreaterThanOrEqualTo(startTime)) //
+ .and(queryHistoryRealizationTable.queryTime,
isLessThan(endTime)) //
+ .and(queryHistoryRealizationTable.projectName,
isEqualTo(project)) //
+
.groupBy(queryHistoryRealizationTable.queryFirstDayOfMonth) //
+ .build().render(RenderingStrategies.MYBATIS3);
+ } else if (timeDimension.equalsIgnoreCase(WEEK)) {
+ return
select(queryHistoryRealizationTable.queryFirstDayOfWeek.as("time"),
+ count(queryHistoryRealizationTable.id).as(COUNT)) //
+ .from(queryHistoryRealizationTable) //
+ .where(queryHistoryRealizationTable.queryTime,
isGreaterThanOrEqualTo(startTime)) //
+ .and(queryHistoryRealizationTable.queryTime,
isLessThan(endTime)) //
+ .and(queryHistoryRealizationTable.projectName,
isEqualTo(project)) //
+
.groupBy(queryHistoryRealizationTable.queryFirstDayOfWeek) //
+ .build().render(RenderingStrategies.MYBATIS3);
+ } else if (timeDimension.equalsIgnoreCase(DAY)) {
+ return select(queryHistoryRealizationTable.queryDay.as("time"),
+ count(queryHistoryRealizationTable.id).as(COUNT)) //
+ .from(queryHistoryRealizationTable) //
+ .where(queryHistoryRealizationTable.queryTime,
isGreaterThanOrEqualTo(startTime)) //
+ .and(queryHistoryRealizationTable.queryTime,
isLessThan(endTime)) //
+ .and(queryHistoryRealizationTable.projectName,
isEqualTo(project)) //
+ .groupBy(queryHistoryRealizationTable.queryDay) //
+ .build().render(RenderingStrategies.MYBATIS3);
+ } else {
+ throw new IllegalStateException(UNSUPPORTED_MESSAGE);
}
}
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java
index d954368f35..882dc78477 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java
@@ -29,6 +29,8 @@ public interface QueryHistoryDAO {
QueryStatistics getQueryCountAndAvgDuration(long startTime, long endTime,
String project);
+ QueryStatistics getQueryCountAndAvgDurationRealization(long startTime,
long endTime, String project);
+
QueryStatistics getQueryCountByRange(long startTime, long endTime, String
project);
long getQueryHistoryCountBeyondOffset(long offset, String project);
@@ -37,10 +39,16 @@ public interface QueryHistoryDAO {
List<QueryStatistics> getQueryCountByTime(long startTime, long endTime,
String timeDimension, String project);
+ List<QueryStatistics> getQueryCountRealizationByTime(long startTime, long
endTime, String timeDimension,
+ String project);
+
List<QueryStatistics> getAvgDurationByModel(long startTime, long endTime,
String project);
List<QueryStatistics> getAvgDurationByTime(long startTime, long endTime,
String timeDimension, String project);
+ List<QueryStatistics> getAvgDurationRealizationByTime(long startTime, long
endTime, String timeDimension,
+ String project);
+
String getQueryMetricMeasurement();
void deleteQueryHistoriesIfMaxSizeReached();
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRealizationTable.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRealizationTable.java
index 3bbc2f4505..754257f87d 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRealizationTable.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRealizationTable.java
@@ -32,6 +32,11 @@ public class QueryHistoryRealizationTable extends SqlTable {
public final SqlColumn<String> model = column("model", JDBCType.VARCHAR);
public final SqlColumn<String> layoutId = column("layout_id",
JDBCType.VARCHAR);
public final SqlColumn<String> indexType = column("index_type",
JDBCType.VARCHAR);
+ public final SqlColumn<Long> queryFirstDayOfMonth =
column("query_first_day_of_month", JDBCType.BIGINT);
+ public final SqlColumn<Long> queryFirstDayOfWeek =
column("query_first_day_of_week", JDBCType.BIGINT);
+ public final SqlColumn<Long> queryDay = column("query_day",
JDBCType.BIGINT);
+
+ public final SqlColumn<Long> id = column("id", JDBCType.BIGINT);
public QueryHistoryRealizationTable(String tableName) {
super(tableName);
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java
index 92323af54f..031bfc2f06 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java
@@ -143,6 +143,12 @@ public class QueryMetrics extends SchedulerEventNotifier {
protected List<String> snapshots;
+ protected long queryFirstDayOfMonth;
+
+ protected long queryFirstDayOfWeek;
+
+ protected long queryDay;
+
// For serialize
public RealizationMetrics() {
}
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java
index fc726dc804..341f7ecb48 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java
@@ -242,6 +242,9 @@ public class QueryMetricsContext extends QueryMetrics {
realizationMetrics.setDuration(queryDuration);
realizationMetrics.setQueryTime(queryTime);
realizationMetrics.setProjectName(projectName);
+ realizationMetrics.setQueryDay(queryDay);
+ realizationMetrics.setQueryFirstDayOfWeek(queryFirstDayOfWeek);
+ realizationMetrics.setQueryFirstDayOfMonth(queryFirstDayOfMonth);
realizationMetrics.setStreamingLayout(realization.isStreamingLayout());
realizationMetrics.setSnapshots(realization.getSnapshots());
realizationMetricList.add(realizationMetrics);
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
index 14fdf0c4a1..64d874d1e6 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
@@ -195,6 +195,14 @@ public class RDBMSQueryHistoryDAO implements
QueryHistoryDAO {
return result.get(0);
}
+ public QueryStatistics getQueryCountAndAvgDurationRealization(long
startTime, long endTime, String project) {
+ List<QueryStatistics> result =
jdbcQueryHisStore.queryCountAndAvgDurationRealization(startTime, endTime,
+ project);
+ if (CollectionUtils.isEmpty(result))
+ return new QueryStatistics();
+ return result.get(0);
+ }
+
public List<QueryStatistics> getQueryCountByModel(long startTime, long
endTime, String project) {
return jdbcQueryHisStore.queryCountByModel(startTime, endTime,
project);
}
@@ -216,6 +224,11 @@ public class RDBMSQueryHistoryDAO implements
QueryHistoryDAO {
return jdbcQueryHisStore.queryCountByTime(startTime, endTime,
timeDimension, project);
}
+ public List<QueryStatistics> getQueryCountRealizationByTime(long
startTime, long endTime, String timeDimension,
+ String project) {
+ return jdbcQueryHisStore.queryCountRealizationByTime(startTime,
endTime, timeDimension, project);
+ }
+
public List<QueryStatistics> getAvgDurationByModel(long startTime, long
endTime, String project) {
return jdbcQueryHisStore.queryAvgDurationByModel(startTime, endTime,
project);
}
@@ -225,6 +238,11 @@ public class RDBMSQueryHistoryDAO implements
QueryHistoryDAO {
return jdbcQueryHisStore.queryAvgDurationByTime(startTime, endTime,
timeDimension, project);
}
+ public List<QueryStatistics> getAvgDurationRealizationByTime(long
startTime, long endTime, String timeDimension,
+ String
project) {
+ return jdbcQueryHisStore.queryAvgDurationRealizationByTime(startTime,
endTime, timeDimension, project);
+ }
+
@Override
public Map<String, Long> getQueryCountByProject() {
return jdbcQueryHisStore.getCountGroupByProject();
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/util/QueryHisStoreUtil.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/util/QueryHisStoreUtil.java
index f6589ed5d0..d8b2c05613 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/util/QueryHisStoreUtil.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/util/QueryHisStoreUtil.java
@@ -47,6 +47,8 @@ import org.apache.kylin.common.Singletons;
import org.apache.kylin.common.logging.LogOutputStream;
import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil;
import org.apache.kylin.common.util.SetThreadName;
+import org.apache.kylin.common.util.SetThreadName;
+import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.query.QueryHistoryDAO;
@@ -74,6 +76,12 @@ public class QueryHisStoreUtil {
private static final String CREATE_QUERY_HISTORY_REALIZATION_INDEX1 =
"create.queryhistoryrealization.store.tableindex1";
private static final String CREATE_QUERY_HISTORY_REALIZATION_INDEX2 =
"create.queryhistoryrealization.store.tableindex2";
+ private static final String CREATE_QUERY_HISTORY_REALIZATION_INDEX3 =
"create.queryhistoryrealization.store.tableindex3";
+
+ private static final String CREATE_QUERY_HISTORY_REALIZATION_INDEX4 =
"create.queryhistoryrealization.store.tableindex4";
+
+ private static final String CREATE_QUERY_HISTORY_REALIZATION_INDEX5 =
"create.queryhistoryrealization.store.tableindex5";
+
private QueryHisStoreUtil() {
}
@@ -158,6 +166,18 @@ public class QueryHisStoreUtil {
String.format(Locale.ROOT,
properties.getProperty(CREATE_QUERY_HISTORY_REALIZATION_INDEX2),
qhRealizationTableName,
qhRealizationTableName).getBytes(Charset.defaultCharset())),
Charset.defaultCharset()));
+ sr.runScript(new InputStreamReader(new ByteArrayInputStream(//
+ String.format(Locale.ROOT,
properties.getProperty(CREATE_QUERY_HISTORY_REALIZATION_INDEX3),
+ qhRealizationTableName,
qhRealizationTableName).getBytes(Charset.defaultCharset())),
+ Charset.defaultCharset()));
+ sr.runScript(new InputStreamReader(new ByteArrayInputStream(//
+ String.format(Locale.ROOT,
properties.getProperty(CREATE_QUERY_HISTORY_REALIZATION_INDEX4),
+ qhRealizationTableName,
qhRealizationTableName).getBytes(Charset.defaultCharset())),
+ Charset.defaultCharset()));
+ sr.runScript(new InputStreamReader(new ByteArrayInputStream(//
+ String.format(Locale.ROOT,
properties.getProperty(CREATE_QUERY_HISTORY_REALIZATION_INDEX5),
+ qhRealizationTableName,
qhRealizationTableName).getBytes(Charset.defaultCharset())),
+ Charset.defaultCharset()));
}
}
diff --git
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
index a6bccb5038..b30e13d383 100644
---
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
+++
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
@@ -43,6 +43,7 @@ public class RDBMSQueryHistoryDaoTest extends
NLocalFileMetadataTestCase {
String PROJECT = "default";
public static final String WEEK = "week";
public static final String DAY = "day";
+ public static final String MONTH = "month";
public static final String NORMAL_USER = "normal_user";
public static final String ADMIN = "ADMIN";
@@ -283,6 +284,48 @@ public class RDBMSQueryHistoryDaoTest extends
NLocalFileMetadataTestCase {
Assert.assertEquals(3, monthQueryStatistics.get(0).getCount());
}
+ @Test
+ public void testGetQueryRealizationByTime() {
+ // 2020-01-29 23:25:12
+ queryHistoryDAO.insert(createQueryMetrics(1580311512000L, 1L, true,
PROJECT, true));
+ // 2020-01-30 23:25:12
+ queryHistoryDAO.insert(createQueryMetrics(1580397912000L, 1L, false,
PROJECT, true));
+ // 2020-01-31 23:25:12
+ queryHistoryDAO.insert(createQueryMetrics(1580484312000L, 1L, false,
PROJECT, true));
+ // 2021-01-29 23:25:12
+ queryHistoryDAO.insert(createQueryMetrics(1611933912000L, 1L, false,
PROJECT, true));
+
+ // filter from 2020-01-26 23:25:11 to 2020-01-31 23:25:13
+ List<QueryStatistics> dayQueryStatistics =
queryHistoryDAO.getQueryCountRealizationByTime(1580052311000L,
+ 1580484313000L, DAY, PROJECT);
+ Assert.assertEquals(0, dayQueryStatistics.size());
+
+ List<QueryStatistics> weekQueryStatistics =
queryHistoryDAO.getQueryCountRealizationByTime(1580052311000L,
+ 1580484313000L, WEEK, PROJECT);
+ Assert.assertEquals(0, weekQueryStatistics.size());
+
+ List<QueryStatistics> monthQueryStatistics =
queryHistoryDAO.getQueryCountRealizationByTime(1580052311000L,
+ 1580484313000L, MONTH, PROJECT);
+ Assert.assertEquals(0, monthQueryStatistics.size());
+
+
+ dayQueryStatistics =
queryHistoryDAO.getAvgDurationRealizationByTime(1580052311000L, 1580484313000L,
+ DAY, PROJECT);
+ Assert.assertEquals(0, dayQueryStatistics.size());
+ weekQueryStatistics =
queryHistoryDAO.getAvgDurationRealizationByTime(1580052311000L, 1580484313000L,
+ WEEK, PROJECT);
+ Assert.assertEquals(0, weekQueryStatistics.size());
+ monthQueryStatistics =
queryHistoryDAO.getAvgDurationRealizationByTime(1580052311000L, 1580484313000L,
+ WEEK, PROJECT);
+ Assert.assertEquals(0, monthQueryStatistics.size());
+
+ QueryStatistics statistics =
queryHistoryDAO.getQueryCountAndAvgDurationRealization(1580052311000L,
1580484313000L,
+ PROJECT);
+ Assert.assertEquals(0, statistics.getCount());
+ Assert.assertEquals(0, statistics.getMeanDuration(), 0.1);
+
+ }
+
@Test
public void testGetAvgDurationByTime() throws Exception {
// 2020-01-29 23:25:12
diff --git
a/src/query-service/src/main/java/org/apache/kylin/rest/service/DashboardService.java
b/src/query-service/src/main/java/org/apache/kylin/rest/service/DashboardService.java
new file mode 100644
index 0000000000..07c1511f93
--- /dev/null
+++
b/src/query-service/src/main/java/org/apache/kylin/rest/service/DashboardService.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.service;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kylin.common.response.MetricsResponse;
+import org.apache.kylin.metadata.cube.realization.HybridRealization;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.exception.UnsupportedQueryException;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.response.JobStatisticsResponse;
+import org.apache.kylin.rest.response.NDataModelOldParams;
+import org.apache.kylin.rest.response.NDataModelResponse;
+import org.apache.kylin.rest.response.QueryStatisticsResponse;
+import org.apache.kylin.rest.util.ModelUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.access.AccessDeniedException;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Component;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+@Slf4j
+@Component("dashboardService")
+public class DashboardService extends BasicService {
+
+ public static final Logger logger =
LoggerFactory.getLogger(DashboardService.class);
+ public static final String DAY = "day";
+ public static final String AVG_QUERY_LATENCY = "AVG_QUERY_LATENCY";
+ public static final String JOB = "JOB";
+ public static final String AVG_JOB_BUILD_TIME = "AVG_JOB_BUILD_TIME";
+ private static final String QUERY = "QUERY";
+ private static final String QUERY_COUNT = "QUERY_COUNT";
+ private static final String JOB_COUNT = "JOB_COUNT";
+ @Autowired
+ ModelService modelService;
+
+ @Autowired
+ QueryHistoryService queryHistoryService;
+
+ @Autowired
+ JobService jobService;
+
+ public MetricsResponse getModelMetrics(String projectName, String
modelName) {
+ MetricsResponse modelMetrics = new MetricsResponse();
+ long totalModelSize = 0;
+ long totalRecordSize = 0;
+ List<NDataModelResponse> models = modelService.getCubes0(modelName,
projectName);//5.0 cube is model
+ Integer totalModel = models.size();
+ ProjectInstance project = getProjectManager().getProject(projectName);
+ totalModel +=
project.getRealizationCount(HybridRealization.REALIZATION_TYPE);
+ modelMetrics.increase("totalModel", totalModel.floatValue());
+
+ Float minModelExpansion = Float.POSITIVE_INFINITY;
+ Float maxModelExpansion = Float.NEGATIVE_INFINITY;
+
+ for (NDataModelResponse dataModel : models) {
+ NDataModelOldParams params = dataModel.getOldParams();
+ if (params.getInputRecordSizeBytes() > 0) {
+ totalModelSize += params.getSizeKB() * 1024;
+ totalRecordSize += params.getInputRecordSizeBytes();//size /
1024 * 1024 * 1024 = x GB
+ Float modelExpansion =
Float.valueOf(dataModel.getExpansionrate());
+
+ if (modelExpansion > maxModelExpansion) {
+ maxModelExpansion = modelExpansion;
+ }
+ if (modelExpansion < minModelExpansion) {
+ minModelExpansion = modelExpansion;
+ }
+ }
+ }
+
+ Float avgModelExpansion = 0f;
+ if (totalRecordSize != 0) {
+ avgModelExpansion =
Float.valueOf(ModelUtils.computeExpansionRate(totalModelSize, totalRecordSize));
+ }
+
+ modelMetrics.increase("avgModelExpansion", avgModelExpansion);
+ modelMetrics.increase("maxModelExpansion", maxModelExpansion);
+ modelMetrics.increase("minModelExpansion", minModelExpansion);
+
+ return modelMetrics;
+ }
+
+ public MetricsResponse getQueryMetrics(String projectName, String
startTime, String endTime) {
+ MetricsResponse queryMetrics = new MetricsResponse();
+ QueryStatisticsResponse queryStatistics =
queryHistoryService.getQueryStatisticsByRealization(projectName,
+ convertToTimestamp(startTime), convertToTimestamp(endTime));
+ Float queryCount = (float) queryStatistics.getCount();
+ Float avgQueryLatency = (float) queryStatistics.getMean();
+ queryMetrics.increase("queryCount", queryCount);
+ queryMetrics.increase("avgQueryLatency", avgQueryLatency);
+ return queryMetrics;
+ }
+
+ public MetricsResponse getJobMetrics(String projectName, String startTime,
String endTime) {
+ MetricsResponse jobMetrics = new MetricsResponse();
+ JobStatisticsResponse jobStats = jobService.getJobStats(projectName,
convertToTimestamp(startTime),
+ convertToTimestamp(endTime));
+ Float jobCount = (float) jobStats.getCount();
+ Float jobTotalByteSize = (float) jobStats.getTotalByteSize();
+ Float jobTotalLatency = (float) jobStats.getTotalDuration();
+ jobMetrics.increase("jobCount", jobCount);
+ jobMetrics.increase("jobTotalByteSize", jobTotalByteSize);
+ jobMetrics.increase("jobTotalLatency", jobTotalLatency);
+ return jobMetrics;
+ }
+
+ public MetricsResponse getChartData(String category, String projectName,
String startTime, String endTime,
+ String dimension, String metric) {
+ long _startTime = convertToTimestamp(startTime);
+ long _endTime = convertToTimestamp(endTime);
+ switch (category) {
+ case QUERY: {
+ switch (metric) {
+ case QUERY_COUNT:
+ Map<String, Object> queryCounts =
queryHistoryService.getQueryCountByRealization(projectName,
+ _startTime, _endTime, dimension.toLowerCase());
+ return transformChartData(queryCounts);
+
+ case AVG_QUERY_LATENCY:
+ Map<String, Object> avgDurations =
queryHistoryService.getAvgDurationByRealization(projectName,
+ _startTime, _endTime, dimension.toLowerCase());
+ return transformChartData(avgDurations);
+ default:
+ throw new UnsupportedQueryException("Metric should be COUNT or
AVG_QUERY_LATENCY");
+ }
+ }
+ case JOB: {
+ switch (metric) {
+ case JOB_COUNT:
+ Map<String, Integer> jobCounts =
jobService.getJobCount(projectName, _startTime, _endTime,
+ dimension.toLowerCase());
+ MetricsResponse counts = new MetricsResponse();
+ jobCounts.forEach((k, v) -> counts.increase(k,
Float.valueOf(v)));
+ return counts;
+ case AVG_JOB_BUILD_TIME:
+ Map<String, Double> jobDurationPerByte =
jobService.getJobDurationPerByte(projectName, _startTime,
+ _endTime, dimension.toLowerCase());
+ MetricsResponse avgBuild = new MetricsResponse();
+ jobDurationPerByte.forEach((k, v) -> avgBuild.increase(k,
Float.valueOf(String.valueOf(v))));
+ return avgBuild;
+ default:
+ throw new UnsupportedQueryException("Metric should be
JOB_COUNT or AVG_JOB_BUILD_TIME");
+ }
+ }
+ default:
+ throw new UnsupportedQueryException("Category should either be
QUERY or JOB");
+ }
+ }
+
+ private MetricsResponse transformChartData(Map<String, Object> responses) {
+ MetricsResponse metrics = new MetricsResponse();
+ for (Map.Entry<String, Object> entry : responses.entrySet()) {
+ String metric = entry.getKey();
+ float value = Float.parseFloat(entry.getValue().toString());
+ metrics.increase(metric, value);
+ }
+ return metrics;
+ }
+
+ private long convertToTimestamp(String time) {
+ Date date;
+ try {
+ date = new SimpleDateFormat("yyyy-MM-dd",
Locale.getDefault(Locale.Category.FORMAT)).parse(time);
+ } catch (ParseException e) {
+ logger.error("parse time to timestamp error!");
+ return 0L;
+ }
+ return date.getTime();
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or
hasPermission(#project, 'ADMINISTRATION')")
+ private void checkAuthorization(ProjectInstance project) throws
AccessDeniedException {
+ //for selected project
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+ private void checkAuthorization() throws AccessDeniedException {
+ //for no selected project
+ }
+
+ public void checkAuthorization(String projectName) {
+ if (projectName != null && !projectName.isEmpty()) {
+ ProjectInstance project =
getProjectManager().getProject(projectName);
+ try {
+ checkAuthorization(project);
+ } catch (AccessDeniedException e) {
+ List<NDataModelResponse> models = modelService.getCubes0(null,
projectName);
+ if (models.isEmpty()) {
+ throw new AccessDeniedException("Access is denied");
+ }
+ }
+ } else {
+ checkAuthorization();
+ }
+ }
+}
diff --git
a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
index c6dc63bba1..339b742aa4 100644
---
a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
+++
b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
@@ -86,6 +86,9 @@ public class QueryHistoryService extends BasicService
implements AsyncTaskQueryH
// public static final String DELETED_MODEL = "Deleted Model";
// public static final byte[] CSV_UTF8_BOM = new byte[]{(byte)0xEF,
(byte)0xBB, (byte)0xBF};
public static final String DAY = "day";
+ public static final String MODEL = "model";
+ public static final String COUNT = "count";
+ public static final String MEAN_DURATION = "meanDuration";
private static final Logger logger = LoggerFactory.getLogger("query");
@Autowired
private AclEvaluate aclEvaluate;
@@ -299,6 +302,16 @@ public class QueryHistoryService extends BasicService
implements AsyncTaskQueryH
return new QueryStatisticsResponse(queryStatistics.getCount(),
queryStatistics.getMeanDuration());
}
+ public QueryStatisticsResponse getQueryStatisticsByRealization(String
project, long startTime, long endTime) {
+ Preconditions.checkArgument(StringUtils.isNotEmpty(project));
+ aclEvaluate.checkProjectReadPermission(project);
+
+ QueryHistoryDAO queryHistoryDao = getQueryHistoryDao();
+ QueryStatistics queryStatistics =
queryHistoryDao.getQueryCountAndAvgDurationRealization(startTime, endTime,
+ project);
+ return new QueryStatisticsResponse(queryStatistics.getCount(),
queryStatistics.getMeanDuration());
+ }
+
public long getLastWeekQueryCount(String project) {
Preconditions.checkArgument(StringUtils.isNotEmpty(project));
aclEvaluate.checkProjectReadPermission(project);
@@ -325,14 +338,14 @@ public class QueryHistoryService extends BasicService
implements AsyncTaskQueryH
QueryHistoryDAO queryHistoryDAO = getQueryHistoryDao();
List<QueryStatistics> queryStatistics;
- if (dimension.equals("model")) {
+ if (dimension.equals(MODEL)) {
queryStatistics = queryHistoryDAO.getQueryCountByModel(startTime,
endTime, project);
- return transformQueryStatisticsByModel(project, queryStatistics,
"count");
+ return transformQueryStatisticsByModel(project, queryStatistics,
COUNT);
}
queryStatistics = queryHistoryDAO.getQueryCountByTime(startTime,
endTime, dimension, project);
fillZeroForQueryStatistics(queryStatistics, startTime, endTime,
dimension);
- return transformQueryStatisticsByTime(queryStatistics, "count",
dimension);
+ return transformQueryStatisticsByTime(queryStatistics, COUNT,
dimension);
}
public Map<String, Object> getAvgDuration(String project, long startTime,
long endTime, String dimension) {
@@ -341,14 +354,48 @@ public class QueryHistoryService extends BasicService
implements AsyncTaskQueryH
QueryHistoryDAO queryHistoryDAO = getQueryHistoryDao();
List<QueryStatistics> queryStatistics;
- if (dimension.equals("model")) {
+ if (dimension.equals(MODEL)) {
queryStatistics = queryHistoryDAO.getAvgDurationByModel(startTime,
endTime, project);
- return transformQueryStatisticsByModel(project, queryStatistics,
"meanDuration");
+ return transformQueryStatisticsByModel(project, queryStatistics,
MEAN_DURATION);
}
queryStatistics = queryHistoryDAO.getAvgDurationByTime(startTime,
endTime, dimension, project);
fillZeroForQueryStatistics(queryStatistics, startTime, endTime,
dimension);
- return transformQueryStatisticsByTime(queryStatistics, "meanDuration",
dimension);
+ return transformQueryStatisticsByTime(queryStatistics, MEAN_DURATION,
dimension);
+ }
+
+ public Map<String, Object> getAvgDurationByRealization(String project,
long startTime, long endTime,
+ String dimension) {
+ Preconditions.checkArgument(StringUtils.isNotEmpty(project));
+ aclEvaluate.checkProjectReadPermission(project);
+ QueryHistoryDAO queryHistoryDAO = getQueryHistoryDao();
+ List<QueryStatistics> queryStatistics;
+
+ if (dimension.equals(MODEL)) {
+ queryStatistics = queryHistoryDAO.getAvgDurationByModel(startTime,
endTime, project);
+ return transformQueryStatisticsByModel(project, queryStatistics,
MEAN_DURATION);
+ }
+
+ queryStatistics =
queryHistoryDAO.getAvgDurationRealizationByTime(startTime, endTime, dimension,
project);
+ fillZeroForQueryStatistics(queryStatistics, startTime, endTime,
dimension);
+ return transformQueryStatisticsByTime(queryStatistics, MEAN_DURATION,
dimension);
+ }
+
+ public Map<String, Object> getQueryCountByRealization(String project, long
startTime, long endTime,
+ String dimension) {
+ Preconditions.checkArgument(StringUtils.isNotEmpty(project));
+ aclEvaluate.checkProjectReadPermission(project);
+ QueryHistoryDAO queryHistoryDAO = getQueryHistoryDao();
+ List<QueryStatistics> queryStatistics;
+
+ if (dimension.equals(MODEL)) {
+ queryStatistics = queryHistoryDAO.getQueryCountByModel(startTime,
endTime, project);
+ return transformQueryStatisticsByModel(project, queryStatistics,
COUNT);
+ }
+
+ queryStatistics =
queryHistoryDAO.getQueryCountRealizationByTime(startTime, endTime, dimension,
project);
+ fillZeroForQueryStatistics(queryStatistics, startTime, endTime,
dimension);
+ return transformQueryStatisticsByTime(queryStatistics, COUNT,
dimension);
}
private Map<String, Object> transformQueryStatisticsByModel(String
project, List<QueryStatistics> statistics,
diff --git
a/src/query-service/src/test/java/org/apache/kylin/rest/service/DashboardServiceTest.java
b/src/query-service/src/test/java/org/apache/kylin/rest/service/DashboardServiceTest.java
new file mode 100644
index 0000000000..3710ec21ae
--- /dev/null
+++
b/src/query-service/src/test/java/org/apache/kylin/rest/service/DashboardServiceTest.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.service;
+
+import com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.response.MetricsResponse;
+import org.apache.kylin.engine.spark.utils.ComputedColumnEvalUtil;
+import org.apache.kylin.metadata.model.util.ExpandableMeasureUtil;
+import org.apache.kylin.metadata.query.QueryStatistics;
+import org.apache.kylin.metadata.query.RDBMSQueryHistoryDAO;
+import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.response.JobStatisticsResponse;
+import org.apache.kylin.rest.util.AclEvaluate;
+import org.apache.kylin.rest.util.AclPermissionUtil;
+import org.apache.kylin.rest.util.AclUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.springframework.security.authentication.TestingAuthenticationToken;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.util.List;
+import java.util.Locale;
+import java.util.TimeZone;
+
+@Slf4j
+public class DashboardServiceTest extends SourceTestCase{
+
+ public static final String MODEL = "model";
+ public static final String DAY = "day";
+ public static final String AVG_QUERY_LATENCY = "AVG_QUERY_LATENCY";
+ public static final String JOB = "JOB";
+ public static final String AVG_JOB_BUILD_TIME = "AVG_JOB_BUILD_TIME";
+ private static final String WEEK = "week";
+ private static final String MONTH = "month";
+ private static final String QUERY = "QUERY";
+ private static final String QUERY_COUNT = "QUERY_COUNT";
+ private static final String JOB_COUNT = "JOB_COUNT";
+ @InjectMocks
+ private final DashboardService dashboardService = Mockito.spy(new
DashboardService());
+ @InjectMocks
+ private final ModelService modelService = Mockito.spy(new ModelService());
+ @InjectMocks
+ private final JobService jobService = Mockito.spy(new JobService());
+ @InjectMocks
+ private final QueryHistoryService queryHistoryService = Mockito.spy(new
QueryHistoryService());
+ @InjectMocks
+ private final ModelBuildService modelBuildService = Mockito.spy(new
ModelBuildService());
+ @InjectMocks
+ private final ModelSemanticHelper semanticService = Mockito.spy(new
ModelSemanticHelper());
+ @InjectMocks
+ private final ProjectService projectService = Mockito.spy(new
ProjectService());
+ @InjectMocks
+ private final MockModelQueryService modelQueryService = Mockito.spy(new
MockModelQueryService());
+ @InjectMocks
+ private final SegmentHelper segmentHelper = new SegmentHelper();
+
+
+ @Mock
+ private final AclEvaluate aclEvaluate = Mockito.spy(AclEvaluate.class);
+ @Mock
+ protected IUserGroupService userGroupService =
Mockito.spy(NUserGroupService.class);
+ @Mock
+ private final AccessService accessService =
Mockito.spy(AccessService.class);
+ @Mock
+ private final AclUtil aclUtil = Mockito.spy(AclUtil.class);
+
+
+ protected String getProject() {
+ return "default";
+ }
+
+ @Before
+ public void setup() {
+ super.setup();
+ ReflectionTestUtils.setField(aclEvaluate, "aclUtil", aclUtil);
+ ReflectionTestUtils.setField(modelService, "aclEvaluate", aclEvaluate);
+ ReflectionTestUtils.setField(modelService, "accessService",
accessService);
+ ReflectionTestUtils.setField(modelService, "userGroupService",
userGroupService);
+ ReflectionTestUtils.setField(semanticService, "userGroupService",
userGroupService);
+ ReflectionTestUtils.setField(modelBuildService, "userGroupService",
userGroupService);
+ ReflectionTestUtils.setField(semanticService, "expandableMeasureUtil",
+ new ExpandableMeasureUtil((model, ccDesc) -> {
+ String ccExpression =
QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
+ AclPermissionUtil.createAclInfo(model.getProject(),
+ semanticService.getCurrentUserGroups()));
+ ccDesc.setInnerExpression(ccExpression);
+ ComputedColumnEvalUtil.evaluateExprAndType(model, ccDesc);
+ }));
+ ReflectionTestUtils.setField(modelService, "projectService",
projectService);
+ ReflectionTestUtils.setField(modelService, "modelQuerySupporter",
modelQueryService);
+ ReflectionTestUtils.setField(modelService, "modelBuildService",
modelBuildService);
+
+ ReflectionTestUtils.setField(modelBuildService, "modelService",
modelService);
+ ReflectionTestUtils.setField(modelBuildService, "segmentHelper",
segmentHelper);
+ ReflectionTestUtils.setField(modelBuildService, "aclEvaluate",
aclEvaluate);
+ modelService.setSemanticUpdater(semanticService);
+ modelService.setSegmentHelper(segmentHelper);
+
+ ReflectionTestUtils.setField(jobService, "aclEvaluate", aclEvaluate);
+ ReflectionTestUtils.setField(jobService, "projectService",
projectService);
+ ReflectionTestUtils.setField(jobService, "modelService", modelService);
+
+
+ ReflectionTestUtils.setField(queryHistoryService, "aclEvaluate",
aclEvaluate);
+ ReflectionTestUtils.setField(queryHistoryService, "modelService",
modelService);
+ ReflectionTestUtils.setField(queryHistoryService, "userGroupService",
userGroupService);
+ ReflectionTestUtils.setField(queryHistoryService, "asyncTaskService",
new AsyncTaskService());
+
+ ReflectionTestUtils.setField(dashboardService, "jobService",
jobService);
+ ReflectionTestUtils.setField(dashboardService, "modelService",
modelService);
+ ReflectionTestUtils.setField(dashboardService, "queryHistoryService",
queryHistoryService);
+
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("ADMIN",
"ADMIN", Constant.ROLE_ADMIN));
+ }
+
+ @Test
+ public void testGetModelMetrics() {
+ MetricsResponse modelMetrics =
dashboardService.getModelMetrics(getProject(), null);
+ Assert.assertEquals(4, modelMetrics.size());
+ }
+
+ @Test
+ public void testGetQueryMetrics() {
+ QueryStatistics queryStatistics = new QueryStatistics();
+ queryStatistics.setCount(777);
+ queryStatistics.setMeanDuration(7070);
+
+ RDBMSQueryHistoryDAO queryHistoryDAO =
Mockito.mock(RDBMSQueryHistoryDAO.class);
+
Mockito.doReturn(queryStatistics).when(queryHistoryDAO).getQueryCountAndAvgDurationRealization(1262275200000L,
+ 1640966400000L, "default");
+
Mockito.doReturn(queryHistoryDAO).when(queryHistoryService).getQueryHistoryDao();
+
+ MetricsResponse queryMetrics =
dashboardService.getQueryMetrics(getProject(), "2010-01-01",
+ "2022-01-01");
+ Assert.assertEquals(2, queryMetrics.size());
+ Assert.assertEquals(777, (double) queryMetrics.get("queryCount"), 0.1);
+ Assert.assertEquals(7070, (double)
queryMetrics.get("avgQueryLatency"), 0.1);
+ }
+
+ @Test
+ public void testGetJobMetrics() {
+ JobStatisticsResponse jobStats = jobService.getJobStats("default",
Long.MIN_VALUE, Long.MAX_VALUE);
+ Assert.assertEquals(0, jobStats.getCount());
+ Assert.assertEquals(0, jobStats.getTotalByteSize());
+ Assert.assertEquals(0, jobStats.getTotalDuration());
+
+ String startTime = "2018-01-01";
+ String endTime = "2018-02-01";
+
+ MetricsResponse metricsResponse =
dashboardService.getJobMetrics(getProject(), startTime, endTime);
+ Assert.assertEquals(3, metricsResponse.size());
+ Assert.assertEquals((float) 0, metricsResponse.get("jobCount"), 0.1);
+ }
+
+ @Test
+ public void testGetChartDataOfQuery() throws ParseException {
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd",
Locale.getDefault(Locale.Category.FORMAT));
+ String _startTime = "2018-01-01";
+ String _endTime = "2018-01-03";
+
+ long startTime = format.parse("2018-01-01").getTime();
+ long endTime = format.parse("2018-01-03").getTime();
+
+
+ RDBMSQueryHistoryDAO queryHistoryDAO =
Mockito.mock(RDBMSQueryHistoryDAO.class);
+
Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getQueryCountByModel(startTime,
endTime, "default");
+
Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getQueryCountRealizationByTime(Mockito.anyLong(),
+ Mockito.anyLong(), Mockito.anyString(), Mockito.anyString());
+
Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getAvgDurationByModel(startTime,
endTime,
+ "default");
+
Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getAvgDurationRealizationByTime(Mockito.anyLong(),
+ Mockito.anyLong(), Mockito.anyString(), Mockito.anyString());
+
Mockito.doReturn(queryHistoryDAO).when(queryHistoryService).getQueryHistoryDao();
+
+ // QUERY_COUNT
+ // query count by model
+ MetricsResponse metricsResponse = dashboardService.getChartData(QUERY,
getProject(), _startTime, _endTime, MODEL, QUERY_COUNT);
+ Assert.assertEquals(3, metricsResponse.size());
+ Assert.assertEquals(10, (double)metricsResponse.get("nmodel_basic"),
0.1);
+ Assert.assertEquals(11,
(double)metricsResponse.get("all_fixed_length"), 0.1);
+
+ // query count by day
+ metricsResponse = dashboardService.getChartData(QUERY, getProject(),
_startTime, _endTime, DAY, QUERY_COUNT);
+ Assert.assertEquals(4, metricsResponse.size());
+ Assert.assertEquals(10, (double)metricsResponse.get("2018-01-01"),
0.1);
+ Assert.assertEquals(11, (double)metricsResponse.get("2018-01-02"),
0.1);
+
+ // query count by week
+ metricsResponse = dashboardService.getChartData(QUERY, getProject(),
_startTime, _endTime, WEEK, QUERY_COUNT);
+ Assert.assertEquals(5, metricsResponse.size());
+ Assert.assertEquals(10, (double)metricsResponse.get("2018-01-01"),
0.1);
+ Assert.assertEquals(11, (double)metricsResponse.get("2018-01-02"),
0.1);
+
+ // query count by month
+ metricsResponse = dashboardService.getChartData(QUERY, getProject(),
_startTime, _endTime, MONTH, QUERY_COUNT);
+ Assert.assertEquals(2, metricsResponse.size());
+ Assert.assertEquals(11, (double)metricsResponse.get("2018-01"), 0.1);
+
+
+
Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getAvgDurationByModel(startTime,
endTime,
+ "default");
+
Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getAvgDurationByTime(Mockito.anyLong(),
+ Mockito.anyLong(), Mockito.anyString(), Mockito.anyString());
+
Mockito.doReturn(queryHistoryDAO).when(queryHistoryService).getQueryHistoryDao();
+
+ // AVG_QUERY_LATENCY
+ // avg duration by model
+ metricsResponse = dashboardService.getChartData(QUERY, getProject(),
_startTime, _endTime, MODEL, AVG_QUERY_LATENCY);
+ Assert.assertEquals(3, metricsResponse.size());
+ Assert.assertEquals(500, (double) metricsResponse.get("nmodel_basic"),
0.1);
+ Assert.assertEquals(600, (double)
metricsResponse.get("all_fixed_length"), 0.1);
+
+ // avg duration by day
+ metricsResponse = dashboardService.getChartData(QUERY, getProject(),
_startTime, _endTime, DAY, AVG_QUERY_LATENCY);
+ Assert.assertEquals(4, metricsResponse.size());
+ Assert.assertEquals(500, (double) metricsResponse.get("2018-01-01"),
0.1);
+ Assert.assertEquals(600, (double) metricsResponse.get("2018-01-02"),
0.1);
+
+ // avg duration by week
+ metricsResponse = dashboardService.getChartData(QUERY, getProject(),
_startTime, _endTime, WEEK, AVG_QUERY_LATENCY);
+ Assert.assertEquals(5, metricsResponse.size());
+ Assert.assertEquals(500, (double) metricsResponse.get("2018-01-01"),
0.1);
+ Assert.assertEquals(600, (double) metricsResponse.get("2018-01-02"),
0.1);
+
+ // avg duration by month
+ metricsResponse = dashboardService.getChartData(QUERY, getProject(),
_startTime, _endTime, MONTH, AVG_QUERY_LATENCY);
+ Assert.assertEquals(2, metricsResponse.size());
+ Assert.assertEquals(600, (double) metricsResponse.get("2018-01"), 0.1);
+ }
+
+ @Test
+ public void testGetChartDataOfJob() {
+ JobStatisticsResponse jobStats = jobService.getJobStats("default",
Long.MIN_VALUE, Long.MAX_VALUE);
+ Assert.assertEquals(0, jobStats.getCount());
+ Assert.assertEquals(0, jobStats.getTotalByteSize());
+ Assert.assertEquals(0, jobStats.getTotalDuration());
+
+ String startTime = "2018-01-01";
+ String endTime = "2018-02-01";
+
+ //JOB_COUNT
+ //model
+ MetricsResponse metricsResponse = dashboardService.getChartData(JOB,
getProject(), startTime, endTime, MODEL, JOB_COUNT);
+ Assert.assertEquals(0, metricsResponse.size());
+
+ //day
+ metricsResponse = dashboardService.getChartData(JOB, getProject(),
startTime, endTime, DAY, JOB_COUNT);
+ Assert.assertEquals(32, metricsResponse.size());
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-01-01"), 0.1);
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-02-01"), 0.1);
+
+ //week
+ metricsResponse = dashboardService.getChartData(JOB, getProject(),
startTime, endTime, WEEK, JOB_COUNT);
+ Assert.assertEquals(5, metricsResponse.size());
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-01-08"), 0.1);
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-01-29"), 0.1);
+
+ //month
+ metricsResponse = dashboardService.getChartData(JOB, getProject(),
startTime, endTime, MONTH, JOB_COUNT);
+ Assert.assertEquals(2, metricsResponse.size());
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-01-01"), 0.1);
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-02-01"), 0.1);
+
+ //AVG_BUILD_TIME
+ //model
+ metricsResponse = dashboardService.getChartData(JOB, getProject(),
startTime, endTime, MODEL, AVG_JOB_BUILD_TIME);
+ Assert.assertEquals(0, metricsResponse.size());
+
+ //day
+ metricsResponse = dashboardService.getChartData(JOB, getProject(),
startTime, endTime, DAY, AVG_JOB_BUILD_TIME);
+ Assert.assertEquals(32, metricsResponse.size());
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-01-01"), 0.1);
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-02-01"), 0.1);
+
+ //week
+ metricsResponse = dashboardService.getChartData(JOB, getProject(),
startTime, endTime, WEEK, AVG_JOB_BUILD_TIME);
+ Assert.assertEquals(5, metricsResponse.size());
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-01-08"), 0.1);
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-01-29"), 0.1);
+
+ //month
+ metricsResponse = dashboardService.getChartData(JOB, getProject(),
startTime, endTime, MONTH, AVG_JOB_BUILD_TIME);
+ Assert.assertEquals(2, metricsResponse.size());
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-01-01"), 0.1);
+ Assert.assertEquals(0, (double)metricsResponse.get("2018-02-01"), 0.1);
+ }
+
+ @Test
+ public void testErrorCase() {
+ String errorMsg = "";
+ try {
+ dashboardService.getChartData("error", getProject(), "2018-01-01",
"2018-02-01", null, null);
+ } catch (Exception e) {
+ errorMsg = e.getMessage();
+ }
+ Assert.assertNotNull(errorMsg);
+
+ errorMsg = "";
+ try {
+ dashboardService.getChartData(QUERY, getProject(), "2018-01-01",
"2018-02-01", null, "error");
+ } catch (Exception e) {
+ errorMsg = e.getMessage();
+ }
+ Assert.assertNotNull(errorMsg);
+
+ errorMsg = "";
+ try {
+ dashboardService.getChartData(JOB, getProject(), "2018-01-01",
"2018-02-01", null, "error");
+ } catch (Exception e) {
+ errorMsg = e.getMessage();
+ }
+ Assert.assertNotNull(errorMsg);
+
+ }
+
+ @Test
+ public void testCheckAuthorization() {
+ dashboardService.checkAuthorization(null);
+ dashboardService.checkAuthorization(getProject());
+ }
+
+ private List<QueryStatistics> getTestStatistics() throws ParseException {
+ int rawOffsetTime =
TimeZone.getTimeZone(KylinConfig.getInstanceFromEnv().getTimeZone()).getRawOffset();
+ String date = "2018-01-01";
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd",
Locale.getDefault(Locale.Category.FORMAT));
+ long time = format.parse(date).getTime();
+
+ QueryStatistics queryStatistics1 = new QueryStatistics();
+ queryStatistics1.setCount(10);
+ queryStatistics1.setMeanDuration(500);
+ queryStatistics1.setModel("89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ queryStatistics1.setTime(Instant.ofEpochMilli(time + rawOffsetTime));
+ queryStatistics1.setMonth(date);
+
+ date = "2018-01-02";
+ time = format.parse(date).getTime();
+
+ QueryStatistics queryStatistics2 = new QueryStatistics();
+ queryStatistics2.setCount(11);
+ queryStatistics2.setMeanDuration(600);
+ queryStatistics2.setModel("abe3bf1a-c4bc-458d-8278-7ea8b00f5e96");
+ queryStatistics2.setTime(Instant.ofEpochMilli(time + rawOffsetTime));
+ queryStatistics2.setMonth(date);
+
+ date = "2018-01-03";
+ time = format.parse(date).getTime();
+
+ QueryStatistics queryStatistics3 = new QueryStatistics();
+ queryStatistics3.setCount(12);
+ queryStatistics3.setMeanDuration(700);
+ queryStatistics3.setModel("a8ba3ff1-83bd-4066-ad54-d2fb3d1f0e94");
+ queryStatistics3.setTime(Instant.ofEpochMilli(time + rawOffsetTime));
+ queryStatistics3.setMonth(date);
+
+ date = "2018-01-04";
+ time = format.parse(date).getTime();
+ QueryStatistics queryStatistics4 = new QueryStatistics();
+ queryStatistics4.setCount(11);
+ queryStatistics4.setMeanDuration(600);
+ queryStatistics4.setModel("not_existing_model");
+ queryStatistics4.setTime(Instant.ofEpochMilli(time + rawOffsetTime));
+ queryStatistics4.setMonth(date);
+
+ return Lists.newArrayList(queryStatistics1, queryStatistics2,
queryStatistics3, queryStatistics4);
+ }
+}
\ No newline at end of file
