Repository: kylin
Updated Branches:
  refs/heads/yaho-cube-planner 8491905eb -> 88ffadbcd


APACHE-KYLIN-2723: add 'user' property for metrics subject query & job


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/88ffadbc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/88ffadbc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/88ffadbc

Branch: refs/heads/yaho-cube-planner
Commit: 88ffadbcdc4d32d4c847b757290a487abe833355
Parents: 8491905
Author: Zhong <nju_y...@apache.org>
Authored: Fri Aug 11 10:41:35 2017 +0800
Committer: Zhong <nju_y...@apache.org>
Committed: Fri Aug 11 10:41:35 2017 +0800

----------------------------------------------------------------------
 .../kylin/job/metrics/JobMetricsFacade.java     | 33 +++++++++++++-------
 .../kylin/metrics/property/JobPropertyEnum.java |  4 +--
 .../metrics/property/QueryPropertyEnum.java     |  4 +--
 .../org/apache/kylin/engine/mr/CubingJob.java   | 27 ++++++++--------
 .../kylin/rest/metrics/QueryMetricsFacade.java  | 27 +++++++++-------
 5 files changed, 55 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/88ffadbc/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java
----------------------------------------------------------------------
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java 
b/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java
index 9762b81..07fcc49 100644
--- a/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java
+++ b/core-job/src/main/java/org/apache/kylin/job/metrics/JobMetricsFacade.java
@@ -36,19 +36,23 @@ public class JobMetricsFacade {
         RecordEvent metricsEvent;
         if (jobStats.throwable == null) {
             metricsEvent = new 
TimedRecordEvent(KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectJob());
-            setJobWrapper(metricsEvent, jobStats.projectName, 
jobStats.cubeName, jobStats.jobId, jobStats.jobType, jobStats.cubingType);
-            setJobStats(metricsEvent, jobStats.tableSize, jobStats.cubeSize, 
jobStats.buildDuration, jobStats.waitResourceTime, jobStats.perBytesTimeCost, //
+            setJobWrapper(metricsEvent, jobStats.user, jobStats.projectName, 
jobStats.cubeName, jobStats.jobId,
+                    jobStats.jobType, jobStats.cubingType);
+            setJobStats(metricsEvent, jobStats.tableSize, jobStats.cubeSize, 
jobStats.buildDuration,
+                    jobStats.waitResourceTime, jobStats.perBytesTimeCost, //
                     jobStats.dColumnDistinct, jobStats.dDictBuilding, 
jobStats.dCubingInmem, jobStats.dHfileConvert);
         } else {
             metricsEvent = new 
TimedRecordEvent(KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectJobException());
-            setJobExceptionWrapper(metricsEvent, jobStats.projectName, 
jobStats.cubeName, jobStats.jobId, jobStats.jobType, jobStats.cubingType, //
+            setJobExceptionWrapper(metricsEvent, jobStats.user, 
jobStats.projectName, jobStats.cubeName, jobStats.jobId,
+                    jobStats.jobType, jobStats.cubingType, //
                     jobStats.throwable.getClass());
         }
         MetricsManager.getInstance().update(metricsEvent);
     }
 
-    private static void setJobWrapper(RecordEvent metricsEvent, String 
projectName, String cubeName, String jobId,
-            String jobType, String cubingType) {
+    private static void setJobWrapper(RecordEvent metricsEvent, String user, 
String projectName, String cubeName,
+            String jobId, String jobType, String cubingType) {
+        metricsEvent.put(JobPropertyEnum.USER.toString(), user);
         metricsEvent.put(JobPropertyEnum.PROJECT.toString(), projectName);
         metricsEvent.put(JobPropertyEnum.CUBE.toString(), cubeName);
         metricsEvent.put(JobPropertyEnum.ID_CODE.toString(), jobId);
@@ -57,8 +61,8 @@ public class JobMetricsFacade {
     }
 
     private static void setJobStats(RecordEvent metricsEvent, long tableSize, 
long cubeSize, long buildDuration,
-                                    long waitResourceTime, double 
perBytesTimeCost, long dColumnDistinct, long dDictBuilding,
-                                    long dCubingInmem, long dHfileConvert) {
+            long waitResourceTime, double perBytesTimeCost, long 
dColumnDistinct, long dDictBuilding, long dCubingInmem,
+            long dHfileConvert) {
         metricsEvent.put(JobPropertyEnum.SOURCE_SIZE.toString(), tableSize);
         metricsEvent.put(JobPropertyEnum.CUBE_SIZE.toString(), cubeSize);
         metricsEvent.put(JobPropertyEnum.BUILD_DURATION.toString(), 
buildDuration);
@@ -70,14 +74,16 @@ public class JobMetricsFacade {
         
metricsEvent.put(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString(), 
dHfileConvert);
     }
 
-    private static <T extends Throwable> void 
setJobExceptionWrapper(RecordEvent metricsEvent, String projectName,
-                                                                     String 
cubeName, String jobId, String jobType, String cubingType, Class<T> 
throwableClass) {
-        setJobWrapper(metricsEvent, projectName, cubeName, jobId, jobType, 
cubingType);
+    private static <T extends Throwable> void 
setJobExceptionWrapper(RecordEvent metricsEvent, String user,
+            String projectName, String cubeName, String jobId, String jobType, 
String cubingType,
+            Class<T> throwableClass) {
+        setJobWrapper(metricsEvent, user, projectName, cubeName, jobId, 
jobType, cubingType);
         metricsEvent.put(JobPropertyEnum.EXCEPTION.toString(), 
throwableClass.getName());
     }
 
     public static class JobStatisticsResult {
         // dimensions
+        private String user;
         private String projectName;
         private String cubeName;
         private String jobId;
@@ -100,7 +106,9 @@ public class JobMetricsFacade {
         // exception
         private Throwable throwable;
 
-        public void setWrapper(String projectName, String cubeName, String 
jobId, String jobType, String cubingType) {
+        public void setWrapper(String user, String projectName, String 
cubeName, String jobId, String jobType,
+                String cubingType) {
+            this.user = user;
             this.projectName = projectName;
             this.cubeName = cubeName;
             this.jobId = jobId;
@@ -108,7 +116,8 @@ public class JobMetricsFacade {
             this.cubingType = cubingType;
         }
 
-        public void setJobStats(long tableSize, long cubeSize, long 
buildDuration, long waitResourceTime, double perBytesTimeCost) {
+        public void setJobStats(long tableSize, long cubeSize, long 
buildDuration, long waitResourceTime,
+                double perBytesTimeCost) {
             this.tableSize = tableSize;
             this.cubeSize = cubeSize;
             this.buildDuration = buildDuration;

http://git-wip-us.apache.org/repos/asf/kylin/blob/88ffadbc/core-metrics/src/main/java/org/apache/kylin/metrics/property/JobPropertyEnum.java
----------------------------------------------------------------------
diff --git 
a/core-metrics/src/main/java/org/apache/kylin/metrics/property/JobPropertyEnum.java
 
b/core-metrics/src/main/java/org/apache/kylin/metrics/property/JobPropertyEnum.java
index 0453221..bbe987a 100644
--- 
a/core-metrics/src/main/java/org/apache/kylin/metrics/property/JobPropertyEnum.java
+++ 
b/core-metrics/src/main/java/org/apache/kylin/metrics/property/JobPropertyEnum.java
@@ -21,8 +21,8 @@ package org.apache.kylin.metrics.property;
 import com.google.common.base.Strings;
 
 public enum JobPropertyEnum {
-    ID_CODE("JOB_ID"), PROJECT("PROJECT"), CUBE("CUBE_NAME"), 
TYPE("JOB_TYPE"), ALGORITHM("CUBING_TYPE"), STATUS(
-            "JOB_STATUS"), EXCEPTION("EXCEPTION"), //
+    ID_CODE("JOB_ID"), USER("USER"), PROJECT("PROJECT"), CUBE("CUBE_NAME"), 
TYPE("JOB_TYPE"), ALGORITHM(
+            "CUBING_TYPE"), STATUS("JOB_STATUS"), EXCEPTION("EXCEPTION"), //
     SOURCE_SIZE("TABLE_SIZE"), CUBE_SIZE("CUBE_SIZE"), 
BUILD_DURATION("DURATION"), WAIT_RESOURCE_TIME(
             "WAIT_RESOURCE_TIME"), PER_BYTES_TIME_COST("PER_BYTES_TIME_COST"), 
STEP_DURATION_DISTINCT_COLUMNS(
                     "STEP_DURATION_DISTINCT_COLUMNS"), 
STEP_DURATION_DICTIONARY(

http://git-wip-us.apache.org/repos/asf/kylin/blob/88ffadbc/core-metrics/src/main/java/org/apache/kylin/metrics/property/QueryPropertyEnum.java
----------------------------------------------------------------------
diff --git 
a/core-metrics/src/main/java/org/apache/kylin/metrics/property/QueryPropertyEnum.java
 
b/core-metrics/src/main/java/org/apache/kylin/metrics/property/QueryPropertyEnum.java
index d3fd3ce..6fe5b0f 100644
--- 
a/core-metrics/src/main/java/org/apache/kylin/metrics/property/QueryPropertyEnum.java
+++ 
b/core-metrics/src/main/java/org/apache/kylin/metrics/property/QueryPropertyEnum.java
@@ -21,8 +21,8 @@ package org.apache.kylin.metrics.property;
 import com.google.common.base.Strings;
 
 public enum QueryPropertyEnum {
-    ID_CODE("QUERY_HASH_CODE"), TYPE("QUERY_TYPE"), PROJECT("PROJECT"), 
REALIZATION("REALIZATION"), REALIZATION_TYPE(
-            "REALIZATION_TYPE"), EXCEPTION("EXCEPTION"), //
+    ID_CODE("QUERY_HASH_CODE"), TYPE("QUERY_TYPE"), USER("USER"), 
PROJECT("PROJECT"), REALIZATION(
+            "REALIZATION"), REALIZATION_TYPE("REALIZATION_TYPE"), 
EXCEPTION("EXCEPTION"), //
     TIME_COST("QUERY_TIME_COST"), 
CALCITE_RETURN_COUNT("CALCITE_COUNT_RETURN"), STORAGE_RETURN_COUNT(
             "STORAGE_COUNT_RETURN"), 
AGGR_FILTER_COUNT("CALCITE_COUNT_AGGREGATE_FILTER");
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/88ffadbc/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 9d493aa..3cd8931 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -18,7 +18,18 @@
 
 package org.apache.kylin.engine.mr;
 
-import com.google.common.base.Strings;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.StringUtil;
@@ -41,17 +52,7 @@ import org.apache.kylin.metadata.project.ProjectManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.regex.Matcher;
+import com.google.common.base.Strings;
 
 /**
  */
@@ -246,7 +247,7 @@ public class CubingJob extends DefaultChainedExecutable {
 
     protected void updateMetrics(ExecutableContext context, ExecuteResult 
result, ExecutableState state) {
         JobMetricsFacade.JobStatisticsResult jobStats = new 
JobMetricsFacade.JobStatisticsResult();
-        
jobStats.setWrapper(ProjectInstance.getNormalizedProjectName(getProjectName()),
+        jobStats.setWrapper(getSubmitter(), 
ProjectInstance.getNormalizedProjectName(getProjectName()),
                 CubingExecutableUtil.getCubeName(getParams()), getId(), 
getJobType(),
                 getAlgorithm() == null ? "NULL" : getAlgorithm().toString());
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/88ffadbc/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
----------------------------------------------------------------------
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
 
b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
index 938488a..3a5c664 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
@@ -38,6 +38,7 @@ import org.apache.kylin.rest.request.SQLRequest;
 import org.apache.kylin.rest.response.SQLResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.security.core.context.SecurityContextHolder;
 
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
@@ -83,16 +84,19 @@ public class QueryMetricsFacade {
         /**
          * report query related metrics
          */
+        String user = 
SecurityContextHolder.getContext().getAuthentication().getName();
+        if (user == null) {
+            user = "unknown";
+        }
         final QueryContext.QueryStatisticsResult queryStatisticsResult = 
sqlResponse.getQueryStatistics();
         for (QueryContext.RPCStatistics entry : 
queryStatisticsResult.getRpcStatisticsList()) {
             RecordEvent rpcMetricsEvent = new TimedRecordEvent(
                     
KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryRpcCall());
             setRPCWrapper(rpcMetricsEvent, //
-                    sqlRequest.getProject(), entry.getRealizationName(), 
entry.getRpcServer(),
-                    entry.getException());
+                    sqlRequest.getProject(), entry.getRealizationName(), 
entry.getRpcServer(), entry.getException());
             setRPCStats(rpcMetricsEvent, //
-                    entry.getCallTimeMs(), entry.getSkippedRows(), 
entry.getScannedRows(),
-                    entry.getReturnedRows(), entry.getAggregatedRows());
+                    entry.getCallTimeMs(), entry.getSkippedRows(), 
entry.getScannedRows(), entry.getReturnedRows(),
+                    entry.getAggregatedRows());
             //For update rpc level related metrics
             MetricsManager.getInstance().update(rpcMetricsEvent);
         }
@@ -102,7 +106,7 @@ public class QueryMetricsFacade {
             RecordEvent queryMetricsEvent = new TimedRecordEvent(
                     
KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQuery());
             setQueryWrapper(queryMetricsEvent, //
-                    sqlHashCode, sqlResponse.isStorageCacheUsed() ? "CACHE" : 
contextEntry.getQueryType(),
+                    user, sqlHashCode, sqlResponse.isStorageCacheUsed() ? 
"CACHE" : contextEntry.getQueryType(),
                     sqlRequest.getProject(), contextEntry.getRealization(), 
contextEntry.getRealizationType(),
                     sqlResponse.getThrowable());
 
@@ -114,9 +118,9 @@ public class QueryMetricsFacade {
                             
KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryCube());
 
                     setCubeWrapper(cubeSegmentMetricsEvent, //
-                            sqlRequest.getProject(), 
segmentEntry.getCubeName(),
-                            segmentEntry.getSegmentName(), 
segmentEntry.getSourceCuboidId(),
-                            segmentEntry.getTargetCuboidId(), 
segmentEntry.getFilterMask());
+                            sqlRequest.getProject(), 
segmentEntry.getCubeName(), segmentEntry.getSegmentName(),
+                            segmentEntry.getSourceCuboidId(), 
segmentEntry.getTargetCuboidId(),
+                            segmentEntry.getFilterMask());
 
                     setCubeStats(cubeSegmentMetricsEvent, //
                             segmentEntry.getCallCount(), 
segmentEntry.getCallTimeSum(), segmentEntry.getCallTimeMax(),
@@ -155,8 +159,8 @@ public class QueryMetricsFacade {
         metricsEvent.put(QueryRPCPropertyEnum.AGGR_COUNT.toString(), 
aggrCount); //Count aggregated by coprocessor
     }
 
-    private static void setCubeWrapper(RecordEvent metricsEvent, String 
projectName, String cubeName, String segmentName,
-            long sourceCuboidId, long targetCuboidId, long filterMask) {
+    private static void setCubeWrapper(RecordEvent metricsEvent, String 
projectName, String cubeName,
+            String segmentName, long sourceCuboidId, long targetCuboidId, long 
filterMask) {
         metricsEvent.put(QueryCubePropertyEnum.PROJECT.toString(), 
projectName);
         metricsEvent.put(QueryCubePropertyEnum.CUBE.toString(), cubeName);
         metricsEvent.put(QueryCubePropertyEnum.SEGMENT.toString(), 
segmentName);
@@ -180,8 +184,9 @@ public class QueryMetricsFacade {
         metricsEvent.put(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString(), 
weightPerHit);
     }
 
-    private static void setQueryWrapper(RecordEvent metricsEvent, long 
queryHashCode, String queryType,
+    private static void setQueryWrapper(RecordEvent metricsEvent, String user, 
long queryHashCode, String queryType,
             String projectName, String realizationName, int realizationType, 
Throwable throwable) {
+        metricsEvent.put(QueryPropertyEnum.USER.toString(), user);
         metricsEvent.put(QueryPropertyEnum.ID_CODE.toString(), queryHashCode);
         metricsEvent.put(QueryPropertyEnum.TYPE.toString(), queryType);
         metricsEvent.put(QueryPropertyEnum.PROJECT.toString(), projectName);

Reply via email to