This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4db6a37c7220c122cedb9fac1a2c735f10e27226
Author: kyotoYaho <nju_y...@apache.org>
AuthorDate: Wed Dec 19 10:40:33 2018 +0800
KYLIN-3540 refactor the interface of querying on SYSTEM project
---
.../kylin/rest/controller/DashboardController.java | 66 ++++++++++++----------
.../org/apache/kylin/rest/service/CubeService.java | 27 +++------
.../kylin/rest/service/DashboardService.java | 34 ++++++-----
.../apache/kylin/rest/service/QueryService.java | 14 ++++-
4 files changed, 73 insertions(+), 68 deletions(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
index ee9fdcd..846d6d3 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
@@ -16,15 +16,12 @@
* limitations under the License.
*/
-
package org.apache.kylin.rest.controller;
import java.util.List;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metrics.MetricsManager;
-import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.CubeService;
@@ -57,62 +54,69 @@ public class DashboardController extends BasicController {
@RequestMapping(value = "/metric/cube", method = { RequestMethod.GET })
@ResponseBody
- public MetricsResponse getCubeMetrics(@RequestParam(value = "projectName",
required = false) String projectName, @RequestParam(value = "cubeName",
required = false) String cubeName) {
+ public MetricsResponse getCubeMetrics(@RequestParam(value = "projectName",
required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String
cubeName) {
checkAuthorization(projectName);
return dashboardService.getCubeMetrics(projectName, cubeName);
}
@RequestMapping(value = "/metric/query", method = RequestMethod.GET)
@ResponseBody
- public MetricsResponse getQueryMetrics(@RequestParam(value =
"projectName", required = false) String projectName, @RequestParam(value =
"cubeName", required = false) String cubeName, @RequestParam(value =
"startTime") String startTime, @RequestParam(value = "endTime") String endTime)
{
+ public MetricsResponse getQueryMetrics(@RequestParam(value =
"projectName", required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String
cubeName,
+ @RequestParam(value = "startTime") String startTime,
@RequestParam(value = "endTime") String endTime) {
checkAuthorization(projectName);
MetricsResponse queryMetrics = new MetricsResponse();
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
String sql = dashboardService.getQueryMetricsSQL(startTime, endTime,
projectName, cubeName);
- sqlRequest.setSql(sql);
- SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
- if(!sqlResponse.getIsException()){
- queryMetrics.increase("queryCount",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
- queryMetrics.increase("avgQueryLatency",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
- queryMetrics.increase("maxQueryLatency",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
- queryMetrics.increase("minQueryLatency",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
+ SQLResponse sqlResponse = queryService.querySystemCube(sql);
+ if (!sqlResponse.getIsException()) {
+ queryMetrics.increase("queryCount",
+
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
+ queryMetrics.increase("avgQueryLatency",
+
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
+ queryMetrics.increase("maxQueryLatency",
+
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
+ queryMetrics.increase("minQueryLatency",
+
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
}
return queryMetrics;
}
@RequestMapping(value = "/metric/job", method = RequestMethod.GET)
@ResponseBody
- public MetricsResponse getJobMetrics(@RequestParam(value = "projectName",
required = false) String projectName, @RequestParam(value = "cubeName",
required = false) String cubeName, @RequestParam(value = "startTime") String
startTime, @RequestParam(value = "endTime") String endTime) {
+ public MetricsResponse getJobMetrics(@RequestParam(value = "projectName",
required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String
cubeName,
+ @RequestParam(value = "startTime") String startTime,
@RequestParam(value = "endTime") String endTime) {
checkAuthorization(projectName);
MetricsResponse jobMetrics = new MetricsResponse();
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
String sql = dashboardService.getJobMetricsSQL(startTime, endTime,
projectName, cubeName);
- sqlRequest.setSql(sql);
- SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
- if(!sqlResponse.getIsException()){
+ SQLResponse sqlResponse = queryService.querySystemCube(sql);
+ if (!sqlResponse.getIsException()) {
jobMetrics.increase("jobCount",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
- jobMetrics.increase("avgJobBuildTime",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
- jobMetrics.increase("maxJobBuildTime",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
- jobMetrics.increase("minJobBuildTime",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
+ jobMetrics.increase("avgJobBuildTime",
+
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
+ jobMetrics.increase("maxJobBuildTime",
+
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
+ jobMetrics.increase("minJobBuildTime",
+
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
}
return jobMetrics;
}
@RequestMapping(value = "/chart/{category}/{metric}/{dimension}", method =
RequestMethod.GET)
@ResponseBody
- public MetricsResponse getChartData(@PathVariable String dimension,
@PathVariable String metric, @PathVariable String category, @RequestParam(value
= "projectName", required = false) String projectName, @RequestParam(value =
"cubeName", required = false) String cubeName, @RequestParam(value =
"startTime") String startTime, @RequestParam(value = "endTime") String endTime)
{
+ public MetricsResponse getChartData(@PathVariable String dimension,
@PathVariable String metric,
+ @PathVariable String category, @RequestParam(value =
"projectName", required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String
cubeName,
+ @RequestParam(value = "startTime") String startTime,
@RequestParam(value = "endTime") String endTime) {
checkAuthorization(projectName);
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String sql = dashboardService.getChartSQL(startTime, endTime,
projectName, cubeName, dimension, metric, category);
- sqlRequest.setSql(sql);
- return
dashboardService.transformChartData(queryService.doQueryWithCache(sqlRequest));
+ String sql = dashboardService.getChartSQL(startTime, endTime,
projectName, cubeName, dimension, metric,
+ category);
+ return
dashboardService.transformChartData(queryService.querySystemCube(sql));
}
- private void checkAuthorization(String projectName){
- if (projectName!=null && !projectName.isEmpty()) {
+ private void checkAuthorization(String projectName) {
+ if (projectName != null && !projectName.isEmpty()) {
ProjectInstance project =
dashboardService.getProjectManager().getProject(projectName);
try {
dashboardService.checkAuthorization(project);
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index e24e9de..4944b58 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -65,7 +65,6 @@ import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.metrics.MetricsManager;
import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -74,7 +73,6 @@ import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.MetricsRequest;
-import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
@@ -919,42 +917,32 @@ public class CubeService extends BasicService implements
InitializingBean {
}
public Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean
isCuboidSource) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- if (!isCuboidSource) {
- cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString();
- }
+ String cuboidColumn = isCuboidSource ?
QueryCubePropertyEnum.CUBOID_SOURCE.toString()
+ : QueryCubePropertyEnum.CUBOID_TARGET.toString();
String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
String table =
getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ")" //
+ " from " + table//
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" +
cubeName + "' " //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" +
cubeName + "'" //
+ " group by " + cuboidColumn;
- sqlRequest.setSql(sql);
- List<List<String>> orgHitFrequency =
queryService.doQueryWithCache(sqlRequest).getResults();
+ List<List<String>> orgHitFrequency =
queryService.querySystemCube(sql).getResults();
return formatQueryCount(orgHitFrequency);
}
public Map<Long, Map<Long, Long>> getCuboidRollingUpStats(String cubeName)
{
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString();
String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
String table =
getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidSource + ", " + cuboidTarget + ", sum("
+ aggCount + ")/count(*)" //
+ String sql = "select " + cuboidSource + ", " + cuboidTarget + ", avg("
+ aggCount + ")" //
+ " from " + table //
+ " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" +
cubeName + "' " //
+ " group by " + cuboidSource + ", " + cuboidTarget;
- sqlRequest.setSql(sql);
- List<List<String>> orgRollingUpCount =
queryService.doQueryWithCache(sqlRequest).getResults();
+ List<List<String>> orgRollingUpCount =
queryService.querySystemCube(sql).getResults();
return formatRollingUpStats(orgRollingUpCount);
}
public Map<Long, Long> getCuboidQueryMatchCount(String cubeName) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
String table =
getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
@@ -963,8 +951,7 @@ public class CubeService extends BasicService implements
InitializingBean {
+ " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" +
cubeName + "'" //
+ " and " + QueryCubePropertyEnum.IF_MATCH.toString() + " =
true" //
+ " group by " + cuboidSource;
- sqlRequest.setSql(sql);
- List<List<String>> orgMatchHitFrequency =
queryService.doQueryWithCache(sqlRequest).getResults();
+ List<List<String>> orgMatchHitFrequency =
queryService.querySystemCube(sql).getResults();
return formatQueryCount(orgMatchHitFrequency);
}
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
index ec395e0..3910245 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
@@ -58,9 +58,12 @@ public class DashboardService extends BasicService {
}
private enum QueryDimensionEnum {
- PROJECT(QueryPropertyEnum.PROJECT.toString()),
CUBE(QueryPropertyEnum.REALIZATION.toString()), DAY(
- TimePropertyEnum.DAY_DATE.toString()), WEEK(
- TimePropertyEnum.WEEK_BEGIN_DATE.toString()),
MONTH(TimePropertyEnum.MONTH.toString());
+ PROJECT(QueryPropertyEnum.PROJECT.toString()), //
+ CUBE(QueryPropertyEnum.REALIZATION.toString()), //
+ DAY(TimePropertyEnum.DAY_DATE.toString()), //
+ WEEK(TimePropertyEnum.WEEK_BEGIN_DATE.toString()), //
+ MONTH(TimePropertyEnum.MONTH.toString());
+
private final String sql;
QueryDimensionEnum(String sql) {
@@ -73,9 +76,12 @@ public class DashboardService extends BasicService {
};
private enum JobDimensionEnum {
- PROJECT(JobPropertyEnum.PROJECT.toString()),
CUBE(JobPropertyEnum.CUBE.toString()), DAY(
- TimePropertyEnum.DAY_DATE.toString()), WEEK(
- TimePropertyEnum.WEEK_BEGIN_DATE.toString()),
MONTH(TimePropertyEnum.MONTH.toString());
+ PROJECT(JobPropertyEnum.PROJECT.toString()), //
+ CUBE(JobPropertyEnum.CUBE.toString()), //
+ DAY(TimePropertyEnum.DAY_DATE.toString()), //
+ WEEK(TimePropertyEnum.WEEK_BEGIN_DATE.toString()), //
+ MONTH(TimePropertyEnum.MONTH.toString());
+
private final String sql;
JobDimensionEnum(String sql) {
@@ -88,10 +94,10 @@ public class DashboardService extends BasicService {
};
private enum QueryMetricEnum {
- QUERY_COUNT("count(*)"), AVG_QUERY_LATENCY("sum(" +
QueryPropertyEnum.TIME_COST.toString() + ")/(count("
- + QueryPropertyEnum.TIME_COST.toString() + "))"),
MAX_QUERY_LATENCY(
- "max(" + QueryPropertyEnum.TIME_COST.toString() +
")"), MIN_QUERY_LATENCY(
- "min(" +
QueryPropertyEnum.TIME_COST.toString() + ")");
+ QUERY_COUNT("count(*)"), //
+ AVG_QUERY_LATENCY("avg(" + QueryPropertyEnum.TIME_COST.toString() +
")"), //
+ MAX_QUERY_LATENCY("max(" + QueryPropertyEnum.TIME_COST.toString() +
")"), //
+ MIN_QUERY_LATENCY("min(" + QueryPropertyEnum.TIME_COST.toString() +
")");
private final String sql;
@@ -105,10 +111,10 @@ public class DashboardService extends BasicService {
}
private enum JobMetricEnum {
- JOB_COUNT("count(*)"), AVG_JOB_BUILD_TIME("sum(" +
JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")/count("
- + JobPropertyEnum.PER_BYTES_TIME_COST + ")"),
MAX_JOB_BUILD_TIME(
- "max(" +
JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), MIN_JOB_BUILD_TIME(
- "min(" +
JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")");
+ JOB_COUNT("count(*)"), //
+ AVG_JOB_BUILD_TIME("avg(" +
JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
+ MAX_JOB_BUILD_TIME("max(" +
JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
+ MIN_JOB_BUILD_TIME("min(" +
JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")");
private final String sql;
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 02bb804..da4fcb6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -93,6 +93,7 @@ import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.metadata.querymeta.TableMetaWithType;
import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metrics.MetricsManager;
import org.apache.kylin.query.QueryConnection;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.PushDownUtil;
@@ -352,6 +353,13 @@ public class QueryService extends BasicService {
logger.info(stringBuilder.toString());
}
+ public SQLResponse querySystemCube(String sql) {
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ sqlRequest.setSql(sql);
+ return doQueryWithCache(sqlRequest, false);
+ }
+
public SQLResponse doQueryWithCache(SQLRequest sqlRequest) {
long t = System.currentTimeMillis();
aclEvaluate.checkProjectReadPermission(sqlRequest.getProject());
@@ -375,7 +383,8 @@ public class QueryService extends BasicService {
// project not found
ProjectManager mgr =
ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
if (mgr.getProject(sqlRequest.getProject()) == null) {
- throw new BadRequestException(String.format(Locale.ROOT,
msg.getPROJECT_NOT_FOUND(), sqlRequest.getProject()));
+ throw new BadRequestException(
+ String.format(Locale.ROOT, msg.getPROJECT_NOT_FOUND(),
sqlRequest.getProject()));
}
if (StringUtils.isBlank(sqlRequest.getSql())) {
throw new BadRequestException(msg.getNULL_EMPTY_SQL());
@@ -1020,8 +1029,7 @@ public class QueryService extends BasicService {
}
private SQLResponse getPrepareOnlySqlResponse(String projectName, String
correctedSql, Connection conn,
- Boolean isPushDown,
- List<List<String>> results, List<SelectedColumnMeta> columnMetas)
throws SQLException {
+ Boolean isPushDown, List<List<String>> results,
List<SelectedColumnMeta> columnMetas) throws SQLException {
CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(true);
