This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8cfe32cf4f61d218439e57e134ccc3413aa98f89 Author: kyotoYaho <nju_y...@apache.org> AuthorDate: Wed Dec 19 10:29:38 2018 +0800 KYLIN-3540 move queryService of CubeController to CubeService --- .../kylin/rest/controller/CubeController.java | 71 ++--------------- .../org/apache/kylin/rest/service/CubeService.java | 88 ++++++++++++++++++---- 2 files changed, 78 insertions(+), 81 deletions(-) diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index dcaa259..11753ec 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -56,8 +56,6 @@ import org.apache.kylin.metadata.model.SegmentRange; import org.apache.kylin.metadata.model.SegmentRange.TSRange; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.realization.RealizationStatusEnum; -import org.apache.kylin.metrics.MetricsManager; -import org.apache.kylin.metrics.property.QueryCubePropertyEnum; import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.exception.ForbiddenException; import org.apache.kylin.rest.exception.InternalErrorException; @@ -70,7 +68,6 @@ import org.apache.kylin.rest.request.JobBuildRequest; import org.apache.kylin.rest.request.JobBuildRequest2; import org.apache.kylin.rest.request.JobOptimizeRequest; import org.apache.kylin.rest.request.LookupSnapshotBuildRequest; -import org.apache.kylin.rest.request.SQLRequest; import org.apache.kylin.rest.response.CubeInstanceResponse; import org.apache.kylin.rest.response.CuboidTreeResponse; import org.apache.kylin.rest.response.EnvelopeResponse; @@ -80,7 +77,6 @@ import org.apache.kylin.rest.response.ResponseCode; import org.apache.kylin.rest.service.CubeService; import org.apache.kylin.rest.service.JobService; import org.apache.kylin.rest.service.ProjectService; -import org.apache.kylin.rest.service.QueryService; import org.apache.kylin.rest.util.ValidateUtil; import org.apache.kylin.source.kafka.util.KafkaClient; import org.slf4j.Logger; @@ -124,10 +120,6 @@ public class CubeController extends BasicController { @Qualifier("projectService") private ProjectService projectService; - @Autowired - @Qualifier("queryService") - private QueryService queryService; - @RequestMapping(value = "/validate/{cubeName}", method = RequestMethod.GET, produces = { "application/json" }) @ResponseBody public EnvelopeResponse<Boolean> validateModelName(@PathVariable String cubeName) { @@ -873,7 +865,7 @@ public class CubeController extends BasicController { Map<Long, Long> queryMatchMap = null; try { hitFrequencyMap = getTargetCuboidHitFrequency(cubeName); - queryMatchMap = getCuboidQueryMatchCount(cubeName); + queryMatchMap = cubeService.getCuboidQueryMatchCount(cubeName); } catch (Exception e) { logger.warn("Fail to query on system cube due to " + e); } @@ -899,7 +891,7 @@ public class CubeController extends BasicController { // Get cuboid target info for displaying heat map of cuboid hit Map<Long, Long> displayHitFrequencyMap = getTargetCuboidHitFrequency(cubeName); // Get exactly matched cuboid query count - Map<Long, Long> queryMatchMap = getCuboidQueryMatchCount(cubeName); + Map<Long, Long> queryMatchMap = cubeService.getCuboidQueryMatchCount(cubeName); Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds(); return cubeService.getCuboidTreeResponse(cuboidScheduler, recommendCuboidStatsMap, displayHitFrequencyMap, @@ -909,69 +901,16 @@ public class CubeController extends BasicController { private Map<Long, Long> getRecommendCuboidList(CubeInstance cube) throws IOException { // Get cuboid source info Map<Long, Long> optimizeHitFrequencyMap = getSourceCuboidHitFrequency(cube.getName()); - Map<Long, Map<Long, Long>> rollingUpCountSourceMap = getCuboidRollingUpCount(cube.getName()); + Map<Long, Map<Long, Long>> rollingUpCountSourceMap = cubeService.getCuboidRollingUpStats(cube.getName()); return cubeService.getRecommendCuboidStatistics(cube, optimizeHitFrequencyMap, rollingUpCountSourceMap); } private Map<Long, Long> getSourceCuboidHitFrequency(String cubeName) { - return getCuboidHitFrequency(cubeName, true); + return cubeService.getCuboidHitFrequency(cubeName, true); } private Map<Long, Long> getTargetCuboidHitFrequency(String cubeName) { - return getCuboidHitFrequency(cubeName, false); - } - - private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean isCuboidSource) { - SQLRequest sqlRequest = new SQLRequest(); - sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT); - String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString(); - if (!isCuboidSource) { - cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString(); - } - String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString(); - String table = cubeService.getMetricsManager() - .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube()); - String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ") " // - + "from " + table// - + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " // - + "group by " + cuboidColumn; - sqlRequest.setSql(sql); - List<List<String>> orgHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults(); - return cubeService.formatQueryCount(orgHitFrequency); - } - - private Map<Long, Map<Long, Long>> getCuboidRollingUpCount(String cubeName) { - SQLRequest sqlRequest = new SQLRequest(); - sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT); - String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString(); - String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString(); - String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString(); - String table = cubeService.getMetricsManager() - .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube()); - String sql = "select " + cuboidSource + ", " + cuboidTarget + ", sum(" + aggCount + ")/count(*) " // - + "from " + table // - + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " // - + "group by " + cuboidSource + ", " + cuboidTarget; - sqlRequest.setSql(sql); - List<List<String>> orgRollingUpCount = queryService.doQueryWithCache(sqlRequest).getResults(); - return cubeService.formatRollingUpCount(orgRollingUpCount); - } - - private Map<Long, Long> getCuboidQueryMatchCount(String cubeName) { - SQLRequest sqlRequest = new SQLRequest(); - sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT); - String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString(); - String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString(); - String table = cubeService.getMetricsManager() - .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube()); - String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ") " // - + "from " + table // - + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' and " - + QueryCubePropertyEnum.IF_MATCH.toString() + " = true " // - + "group by " + cuboidSource; - sqlRequest.setSql(sql); - List<List<String>> orgMatchHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults(); - return cubeService.formatQueryCount(orgMatchHitFrequency); + return cubeService.getCuboidHitFrequency(cubeName, false); } /** diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index e541c37..e24e9de 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -65,6 +65,8 @@ import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.project.RealizationEntry; import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.metrics.MetricsManager; +import org.apache.kylin.metrics.property.QueryCubePropertyEnum; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.exception.ForbiddenException; @@ -72,6 +74,7 @@ import org.apache.kylin.rest.exception.InternalErrorException; import org.apache.kylin.rest.msg.Message; import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.rest.request.MetricsRequest; +import org.apache.kylin.rest.request.SQLRequest; import org.apache.kylin.rest.response.CubeInstanceResponse; import org.apache.kylin.rest.response.CuboidTreeResponse; import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo; @@ -121,6 +124,10 @@ public class CubeService extends BasicService implements InitializingBean { private ModelService modelService; @Autowired + @Qualifier("queryService") + private QueryService queryService; + + @Autowired private AclEvaluate aclEvaluate; @Autowired @@ -472,7 +479,8 @@ public class CubeService extends BasicService implements InitializingBean { hr = new HBaseResponse(); CubeInstance cube = CubeManager.getInstance(getConfig()).getCube(cubeName); - if (cube.getStorageType() == IStorageAware.ID_HBASE || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE) { + if (cube.getStorageType() == IStorageAware.ID_HBASE + || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE) { try { logger.debug("Loading HTable info " + cubeName + ", " + tableName); @@ -547,9 +555,9 @@ public class CubeService extends BasicService implements InitializingBean { } public boolean isOrphonSegment(CubeInstance cube, String segId) { - List<JobInstance> jobInstances = jobService.searchJobsByCubeName(cube.getName(), - cube.getProject(), Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING, JobStatusEnum.RUNNING, - JobStatusEnum.ERROR, JobStatusEnum.STOPPED), + List<JobInstance> jobInstances = jobService.searchJobsByCubeName( + cube.getName(), cube.getProject(), Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING, + JobStatusEnum.RUNNING, JobStatusEnum.ERROR, JobStatusEnum.STOPPED), JobTimeFilterEnum.ALL, JobService.JobSearchMode.CUBING_ONLY); for (JobInstance jobInstance : jobInstances) { // if there are segment related jobs, can not delete this segment. @@ -886,6 +894,12 @@ public class CubeService extends BasicService implements InitializingBean { } /** cube planner services */ + public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap, + Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException { + aclEvaluate.checkProjectAdminPermission(cube.getProject()); + return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap); + } + public Map<Long, Long> formatQueryCount(List<List<String>> orgQueryCount) { Map<Long, Long> formattedQueryCount = Maps.newLinkedHashMap(); for (List<String> hit : orgQueryCount) { @@ -894,20 +908,64 @@ public class CubeService extends BasicService implements InitializingBean { return formattedQueryCount; } - public Map<Long, Map<Long, Long>> formatRollingUpCount(List<List<String>> orgRollingUpCount) { - Map<Long, Map<Long, Long>> formattedRollingUpCount = Maps.newLinkedHashMap(); + public Map<Long, Map<Long, Long>> formatRollingUpStats(List<List<String>> orgRollingUpCount) { + Map<Long, Map<Long, Long>> formattedRollingUpStats = Maps.newLinkedHashMap(); for (List<String> rollingUp : orgRollingUpCount) { Map<Long, Long> childMap = Maps.newLinkedHashMap(); childMap.put(Long.parseLong(rollingUp.get(1)), (long) Double.parseDouble(rollingUp.get(2))); - formattedRollingUpCount.put(Long.parseLong(rollingUp.get(0)), childMap); - } - return formattedRollingUpCount; - } - - public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap, - Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException { - aclEvaluate.checkProjectAdminPermission(cube.getProject()); - return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap); + formattedRollingUpStats.put(Long.parseLong(rollingUp.get(0)), childMap); + } + return formattedRollingUpStats; + } + + public Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean isCuboidSource) { + SQLRequest sqlRequest = new SQLRequest(); + sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT); + String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString(); + if (!isCuboidSource) { + cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString(); + } + String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString(); + String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube()); + String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ")" // + + " from " + table// + + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " // + + " group by " + cuboidColumn; + sqlRequest.setSql(sql); + List<List<String>> orgHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults(); + return formatQueryCount(orgHitFrequency); + } + + public Map<Long, Map<Long, Long>> getCuboidRollingUpStats(String cubeName) { + SQLRequest sqlRequest = new SQLRequest(); + sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT); + String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString(); + String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString(); + String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString(); + String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube()); + String sql = "select " + cuboidSource + ", " + cuboidTarget + ", sum(" + aggCount + ")/count(*)" // + + " from " + table // + + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " // + + " group by " + cuboidSource + ", " + cuboidTarget; + sqlRequest.setSql(sql); + List<List<String>> orgRollingUpCount = queryService.doQueryWithCache(sqlRequest).getResults(); + return formatRollingUpStats(orgRollingUpCount); + } + + public Map<Long, Long> getCuboidQueryMatchCount(String cubeName) { + SQLRequest sqlRequest = new SQLRequest(); + sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT); + String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString(); + String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString(); + String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube()); + String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ")" // + + " from " + table // + + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "'" // + + " and " + QueryCubePropertyEnum.IF_MATCH.toString() + " = true" // + + " group by " + cuboidSource; + sqlRequest.setSql(sql); + List<List<String>> orgMatchHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults(); + return formatQueryCount(orgMatchHitFrequency); } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN