Repository: kylin Updated Branches: refs/heads/yaho-cube-planner 8491905eb -> 199c198b4
APACHE-KYLIN-2723: add 'user' property for metrics subject query & job Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/199c198b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/199c198b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/199c198b Branch: refs/heads/yaho-cube-planner Commit: 199c198b44d7ed6d0923d504a66efcccf7f2c464 Parents: 8491905 Author: Zhong <nju_y...@apache.org> Authored: Fri Aug 11 10:24:12 2017 +0800 Committer: Zhong <nju_y...@apache.org> Committed: Fri Aug 11 10:24:12 2017 +0800 ---------------------------------------------------------------------- .../kylin/job/metrics/JobMetricsFacade.java | 21 +++++++++----- .../kylin/metrics/property/JobPropertyEnum.java | 4 +-- .../metrics/property/QueryPropertyEnum.java | 4 +-- .../org/apache/kylin/engine/mr/CubingJob.java | 30 +++++++------------- .../kylin/rest/metrics/QueryMetricsFacade.java | 10 +++++-- 5 files changed, 37 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/199c198b/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java b/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java index 9762b81..6171b54 100644 --- a/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java +++ b/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java @@ -36,19 +36,22 @@ public class JobMetricsFacade { RecordEvent metricsEvent; if (jobStats.throwable == null) { metricsEvent = new TimedRecordEvent(KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectJob()); - setJobWrapper(metricsEvent, jobStats.projectName, jobStats.cubeName, jobStats.jobId, jobStats.jobType, jobStats.cubingType); + setJobWrapper(metricsEvent, jobStats.user, jobStats.projectName, jobStats.cubeName, jobStats.jobId, + jobStats.jobType, jobStats.cubingType); setJobStats(metricsEvent, jobStats.tableSize, jobStats.cubeSize, jobStats.buildDuration, jobStats.waitResourceTime, jobStats.perBytesTimeCost, // jobStats.dColumnDistinct, jobStats.dDictBuilding, jobStats.dCubingInmem, jobStats.dHfileConvert); } else { metricsEvent = new TimedRecordEvent(KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectJobException()); - setJobExceptionWrapper(metricsEvent, jobStats.projectName, jobStats.cubeName, jobStats.jobId, jobStats.jobType, jobStats.cubingType, // + setJobExceptionWrapper(metricsEvent, jobStats.user, jobStats.projectName, jobStats.cubeName, jobStats.jobId, + jobStats.jobType, jobStats.cubingType, // jobStats.throwable.getClass()); } MetricsManager.getInstance().update(metricsEvent); } - private static void setJobWrapper(RecordEvent metricsEvent, String projectName, String cubeName, String jobId, - String jobType, String cubingType) { + private static void setJobWrapper(RecordEvent metricsEvent, String user, String projectName, String cubeName, + String jobId, String jobType, String cubingType) { + metricsEvent.put(JobPropertyEnum.USER.toString(), user); metricsEvent.put(JobPropertyEnum.PROJECT.toString(), projectName); metricsEvent.put(JobPropertyEnum.CUBE.toString(), cubeName); metricsEvent.put(JobPropertyEnum.ID_CODE.toString(), jobId); @@ -70,14 +73,16 @@ public class JobMetricsFacade { metricsEvent.put(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString(), dHfileConvert); } - private static <T extends Throwable> void setJobExceptionWrapper(RecordEvent metricsEvent, String projectName, + private static <T extends Throwable> void setJobExceptionWrapper(RecordEvent metricsEvent, String user, + String projectName, String cubeName, String jobId, String jobType, String cubingType, Class<T> throwableClass) { - setJobWrapper(metricsEvent, projectName, cubeName, jobId, jobType, cubingType); + setJobWrapper(metricsEvent, user, projectName, cubeName, jobId, jobType, cubingType); metricsEvent.put(JobPropertyEnum.EXCEPTION.toString(), throwableClass.getName()); } public static class JobStatisticsResult { // dimensions + private String user; private String projectName; private String cubeName; private String jobId; @@ -100,7 +105,9 @@ public class JobMetricsFacade { // exception private Throwable throwable; - public void setWrapper(String projectName, String cubeName, String jobId, String jobType, String cubingType) { + public void setWrapper(String user, String projectName, String cubeName, String jobId, String jobType, + String cubingType) { + this.user = user; this.projectName = projectName; this.cubeName = cubeName; this.jobId = jobId; http://git-wip-us.apache.org/repos/asf/kylin/blob/199c198b/core-metrics/src/main/java/org/apache/kylin/metrics/property/JobPropertyEnum.java ---------------------------------------------------------------------- diff --git a/core-metrics/src/main/java/org/apache/kylin/metrics/property/JobPropertyEnum.java b/core-metrics/src/main/java/org/apache/kylin/metrics/property/JobPropertyEnum.java index 0453221..bbe987a 100644 --- a/core-metrics/src/main/java/org/apache/kylin/metrics/property/JobPropertyEnum.java +++ b/core-metrics/src/main/java/org/apache/kylin/metrics/property/JobPropertyEnum.java @@ -21,8 +21,8 @@ package org.apache.kylin.metrics.property; import com.google.common.base.Strings; public enum JobPropertyEnum { - ID_CODE("JOB_ID"), PROJECT("PROJECT"), CUBE("CUBE_NAME"), TYPE("JOB_TYPE"), ALGORITHM("CUBING_TYPE"), STATUS( - "JOB_STATUS"), EXCEPTION("EXCEPTION"), // + ID_CODE("JOB_ID"), USER("USER"), PROJECT("PROJECT"), CUBE("CUBE_NAME"), TYPE("JOB_TYPE"), ALGORITHM( + "CUBING_TYPE"), STATUS("JOB_STATUS"), EXCEPTION("EXCEPTION"), // SOURCE_SIZE("TABLE_SIZE"), CUBE_SIZE("CUBE_SIZE"), BUILD_DURATION("DURATION"), WAIT_RESOURCE_TIME( "WAIT_RESOURCE_TIME"), PER_BYTES_TIME_COST("PER_BYTES_TIME_COST"), STEP_DURATION_DISTINCT_COLUMNS( "STEP_DURATION_DISTINCT_COLUMNS"), STEP_DURATION_DICTIONARY( http://git-wip-us.apache.org/repos/asf/kylin/blob/199c198b/core-metrics/src/main/java/org/apache/kylin/metrics/property/QueryPropertyEnum.java ---------------------------------------------------------------------- diff --git a/core-metrics/src/main/java/org/apache/kylin/metrics/property/QueryPropertyEnum.java b/core-metrics/src/main/java/org/apache/kylin/metrics/property/QueryPropertyEnum.java index d3fd3ce..6fe5b0f 100644 --- a/core-metrics/src/main/java/org/apache/kylin/metrics/property/QueryPropertyEnum.java +++ b/core-metrics/src/main/java/org/apache/kylin/metrics/property/QueryPropertyEnum.java @@ -21,8 +21,8 @@ package org.apache.kylin.metrics.property; import com.google.common.base.Strings; public enum QueryPropertyEnum { - ID_CODE("QUERY_HASH_CODE"), TYPE("QUERY_TYPE"), PROJECT("PROJECT"), REALIZATION("REALIZATION"), REALIZATION_TYPE( - "REALIZATION_TYPE"), EXCEPTION("EXCEPTION"), // + ID_CODE("QUERY_HASH_CODE"), TYPE("QUERY_TYPE"), USER("USER"), PROJECT("PROJECT"), REALIZATION( + "REALIZATION"), REALIZATION_TYPE("REALIZATION_TYPE"), EXCEPTION("EXCEPTION"), // TIME_COST("QUERY_TIME_COST"), CALCITE_RETURN_COUNT("CALCITE_COUNT_RETURN"), STORAGE_RETURN_COUNT( "STORAGE_COUNT_RETURN"), AGGR_FILTER_COUNT("CALCITE_COUNT_AGGREGATE_FILTER"); http://git-wip-us.apache.org/repos/asf/kylin/blob/199c198b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java index 9d493aa..47f6a87 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java @@ -18,7 +18,14 @@ package org.apache.kylin.engine.mr; -import com.google.common.base.Strings; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; + import org.apache.commons.lang3.tuple.Pair; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.StringUtil; @@ -29,29 +36,14 @@ import org.apache.kylin.engine.mr.common.MapReduceExecutable; import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; import org.apache.kylin.job.constant.ExecutableConstants; import org.apache.kylin.job.engine.JobEngineConfig; -import org.apache.kylin.job.execution.AbstractExecutable; -import org.apache.kylin.job.execution.DefaultChainedExecutable; -import org.apache.kylin.job.execution.ExecutableContext; -import org.apache.kylin.job.execution.ExecutableState; -import org.apache.kylin.job.execution.ExecuteResult; -import org.apache.kylin.job.execution.Output; +import org.apache.kylin.job.execution.*; import org.apache.kylin.job.metrics.JobMetricsFacade; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.TimeZone; -import java.util.regex.Matcher; +import com.google.common.base.Strings; /** */ @@ -246,7 +238,7 @@ public class CubingJob extends DefaultChainedExecutable { protected void updateMetrics(ExecutableContext context, ExecuteResult result, ExecutableState state) { JobMetricsFacade.JobStatisticsResult jobStats = new JobMetricsFacade.JobStatisticsResult(); - jobStats.setWrapper(ProjectInstance.getNormalizedProjectName(getProjectName()), + jobStats.setWrapper(getSubmitter(), ProjectInstance.getNormalizedProjectName(getProjectName()), CubingExecutableUtil.getCubeName(getParams()), getId(), getJobType(), getAlgorithm() == null ? "NULL" : getAlgorithm().toString()); http://git-wip-us.apache.org/repos/asf/kylin/blob/199c198b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java index 938488a..e670580 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java +++ b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java @@ -38,6 +38,7 @@ import org.apache.kylin.rest.request.SQLRequest; import org.apache.kylin.rest.response.SQLResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.security.core.context.SecurityContextHolder; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; @@ -83,6 +84,10 @@ public class QueryMetricsFacade { /** * report query related metrics */ + String user = SecurityContextHolder.getContext().getAuthentication().getName(); + if (user == null) { + user = "unknown"; + } final QueryContext.QueryStatisticsResult queryStatisticsResult = sqlResponse.getQueryStatistics(); for (QueryContext.RPCStatistics entry : queryStatisticsResult.getRpcStatisticsList()) { RecordEvent rpcMetricsEvent = new TimedRecordEvent( @@ -102,7 +107,7 @@ public class QueryMetricsFacade { RecordEvent queryMetricsEvent = new TimedRecordEvent( KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQuery()); setQueryWrapper(queryMetricsEvent, // - sqlHashCode, sqlResponse.isStorageCacheUsed() ? "CACHE" : contextEntry.getQueryType(), + user, sqlHashCode, sqlResponse.isStorageCacheUsed() ? "CACHE" : contextEntry.getQueryType(), sqlRequest.getProject(), contextEntry.getRealization(), contextEntry.getRealizationType(), sqlResponse.getThrowable()); @@ -180,8 +185,9 @@ public class QueryMetricsFacade { metricsEvent.put(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString(), weightPerHit); } - private static void setQueryWrapper(RecordEvent metricsEvent, long queryHashCode, String queryType, + private static void setQueryWrapper(RecordEvent metricsEvent, String user, long queryHashCode, String queryType, String projectName, String realizationName, int realizationType, Throwable throwable) { + metricsEvent.put(QueryPropertyEnum.USER.toString(), user); metricsEvent.put(QueryPropertyEnum.ID_CODE.toString(), queryHashCode); metricsEvent.put(QueryPropertyEnum.TYPE.toString(), queryType); metricsEvent.put(QueryPropertyEnum.PROJECT.toString(), projectName);
