[15/50] incubator-gobblin git commit: [GOBBLIN-402] Add more metrics for gobblin cluster and fix the getJobs slowness issue

2018-03-21 Thread abti 
[GOBBLIN-402] Add more metrics for gobblin cluster and fix the getJobs slowness 
issue

Closes #2276 from yukuai518/morem


Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/de83a3fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/de83a3fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/de83a3fb

Branch: refs/heads/0.12.0
Commit: de83a3fb5f2644e9657b546c7415d1e8f5336a2c
Parents: 8879cde
Author: Kuai Yu 
Authored: Fri Feb 2 13:25:53 2018 -0800
Committer: Hung Tran 
Committed: Fri Feb 2 13:25:53 2018 -0800

--
 .../gobblin/cluster/GobblinClusterManager.java  |  6 +-
 .../cluster/GobblinHelixJobScheduler.java   | 33 ++-
 .../StreamingJobConfigurationManager.java   |  3 +
 .../service/StreamingKafkaSpecConsumer.java | 92 +++-
 .../apache/gobblin/runtime/api/JobCatalog.java  |  7 +-
 .../runtime/api/JobExecutionLauncher.java   |  6 ++
 .../runtime/job_monitor/KafkaJobMonitor.java|  3 +
 7 files changed, 139 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/de83a3fb/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
--
diff --git 
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
 
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
index 77e511e..3393df6 100644
--- 
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
+++ 
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
@@ -153,6 +153,9 @@ public class GobblinClusterManager implements 
ApplicationLauncher, StandardMetri
   private MutableJobCatalog jobCatalog;
   @Getter
   private GobblinHelixJobScheduler jobScheduler;
+  @Getter
+  private JobConfigurationManager jobConfigurationManager;
+  
   private final String clusterName;
   private final Config config;
   private final MetricContext metricContext;
@@ -209,7 +212,8 @@ public class GobblinClusterManager implements 
ApplicationLauncher, StandardMetri
 this.jobScheduler = buildGobblinHelixJobScheduler(config, this.appWorkDir, 
getMetadataTags(clusterName, applicationId),
 schedulerService);
 this.applicationLauncher.addService(this.jobScheduler);
-this.applicationLauncher.addService(buildJobConfigurationManager(config));
+this.jobConfigurationManager = buildJobConfigurationManager(config);
+this.applicationLauncher.addService(this.jobConfigurationManager);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/de83a3fb/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java
--
diff --git 
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java
 
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java
index 36ba542..141e3d1 100644
--- 
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java
+++ 
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -34,6 +35,7 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.eventbus.EventBus;
 import com.google.common.eventbus.Subscribe;
@@ -157,7 +159,17 @@ public class GobblinHelixJobScheduler extends JobScheduler 
implements StandardMe
 private final ContextAwareTimer timeBeforeJobScheduling;
 private final ContextAwareTimer timeBeforeJobLaunching;
 
+private final ThreadPoolExecutor threadPoolExecutor;
+private final ContextAwareGauge executorActiveCount;
+private final ContextAwareGauge executorMaximumPoolSize;
+private final ContextAwareGauge executorPoolSize;
+private final ContextAwareGauge executorCorePoolSize;
+private final ContextAwareGauge executorQueueSize;
+
 public Metrics(final MetricContext metricContext) {
+  // Thread executor reference from job scheduler
+  this.threadPoolExecutor = 
(ThreadPoolExecutor)GobblinHelixJobScheduler.this.jobExecutor;
+
   // All historical counters
   this.totalJobsLaunched = new AtomicLong(0);
   this.totalJobsCompleted = new AtomicLong(0);
@@

incubator-gobblin git commit: [GOBBLIN-402] Add more metrics for gobblin cluster and fix the getJobs slowness issue

2018-02-02 Thread hutran 
Repository: incubator-gobblin
Updated Branches:
  refs/heads/master 8879cdec2 -> de83a3fb5


[GOBBLIN-402] Add more metrics for gobblin cluster and fix the getJobs slowness 
issue

Closes #2276 from yukuai518/morem


Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/de83a3fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/de83a3fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/de83a3fb

Branch: refs/heads/master
Commit: de83a3fb5f2644e9657b546c7415d1e8f5336a2c
Parents: 8879cde
Author: Kuai Yu 
Authored: Fri Feb 2 13:25:53 2018 -0800
Committer: Hung Tran 
Committed: Fri Feb 2 13:25:53 2018 -0800

--
 .../gobblin/cluster/GobblinClusterManager.java  |  6 +-
 .../cluster/GobblinHelixJobScheduler.java   | 33 ++-
 .../StreamingJobConfigurationManager.java   |  3 +
 .../service/StreamingKafkaSpecConsumer.java | 92 +++-
 .../apache/gobblin/runtime/api/JobCatalog.java  |  7 +-
 .../runtime/api/JobExecutionLauncher.java   |  6 ++
 .../runtime/job_monitor/KafkaJobMonitor.java|  3 +
 7 files changed, 139 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/de83a3fb/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
--
diff --git 
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
 
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
index 77e511e..3393df6 100644
--- 
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
+++ 
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
@@ -153,6 +153,9 @@ public class GobblinClusterManager implements 
ApplicationLauncher, StandardMetri
   private MutableJobCatalog jobCatalog;
   @Getter
   private GobblinHelixJobScheduler jobScheduler;
+  @Getter
+  private JobConfigurationManager jobConfigurationManager;
+  
   private final String clusterName;
   private final Config config;
   private final MetricContext metricContext;
@@ -209,7 +212,8 @@ public class GobblinClusterManager implements 
ApplicationLauncher, StandardMetri
 this.jobScheduler = buildGobblinHelixJobScheduler(config, this.appWorkDir, 
getMetadataTags(clusterName, applicationId),
 schedulerService);
 this.applicationLauncher.addService(this.jobScheduler);
-this.applicationLauncher.addService(buildJobConfigurationManager(config));
+this.jobConfigurationManager = buildJobConfigurationManager(config);
+this.applicationLauncher.addService(this.jobConfigurationManager);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/de83a3fb/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java
--
diff --git 
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java
 
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java
index 36ba542..141e3d1 100644
--- 
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java
+++ 
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -34,6 +35,7 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.eventbus.EventBus;
 import com.google.common.eventbus.Subscribe;
@@ -157,7 +159,17 @@ public class GobblinHelixJobScheduler extends JobScheduler 
implements StandardMe
 private final ContextAwareTimer timeBeforeJobScheduling;
 private final ContextAwareTimer timeBeforeJobLaunching;
 
+private final ThreadPoolExecutor threadPoolExecutor;
+private final ContextAwareGauge executorActiveCount;
+private final ContextAwareGauge executorMaximumPoolSize;
+private final ContextAwareGauge executorPoolSize;
+private final ContextAwareGauge executorCorePoolSize;
+private final ContextAwareGauge executorQueueSize;
+
 public Metrics(final MetricContext metricContext) {
+  // Thread executor reference from job scheduler
+  this.threadPoolExecutor = 
(ThreadPoolExecutor)GobblinHelixJobScheduler.this.jobExecutor;
+
   // All historical counters
   this.t