This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 2bbf42b [GOBBLIN-996] Add support for managed Helix clusters for
Gobblin-on-Ya…
2bbf42b is described below
commit 2bbf42bbc47ce76597180066007164d385f64565
Author: sv2000 <[email protected]>
AuthorDate: Thu Dec 5 14:42:16 2019 -0800
[GOBBLIN-996] Add support for managed Helix clusters for Gobblin-on-Ya…
Closes #2841 from sv2000/helixManagedCluster
---
.../cluster/GobblinClusterConfigurationKeys.java | 5 +++++
.../gobblin/cluster/GobblinHelixMultiManager.java | 4 +++-
.../gobblin/yarn/GobblinYarnAppLauncher.java | 24 ++++++++++++++--------
gradle/scripts/dependencyDefinitions.gradle | 2 +-
4 files changed, 24 insertions(+), 11 deletions(-)
diff --git
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
index 8ffb97c..fdf1eca 100644
---
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
+++
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
@@ -173,4 +173,9 @@ public class GobblinClusterConfigurationKeys {
//Config to enable/disable reuse of existing Helix Cluster
public static final String HELIX_CLUSTER_OVERWRITE_KEY =
GOBBLIN_CLUSTER_PREFIX + "helix.overwrite";
public static final boolean DEFAULT_HELIX_CLUSTER_OVERWRITE = true;
+
+ //Config to enable/disable cluster creation. Should set this config to false
if Helix-as-a-Service is used to manage
+ // the cluster
+ public static final String IS_HELIX_CLUSTER_MANAGED = GOBBLIN_CLUSTER_PREFIX
+ "isHelixClusterManaged";
+ public static final boolean DEFAULT_IS_HELIX_CLUSTER_MANAGED = false;
}
diff --git
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixMultiManager.java
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixMultiManager.java
index 57c672b..cdc720a 100644
---
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixMultiManager.java
+++
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixMultiManager.java
@@ -227,10 +227,12 @@ public class GobblinHelixMultiManager implements
StandardMetricsBridge {
} else {
log.info("We will use same cluster to manage GobblinClusterManager and
job distribution.");
// This will create and register a Helix controller in ZooKeeper
+ boolean isHelixClusterManaged = ConfigUtils.getBoolean(this.config,
GobblinClusterConfigurationKeys.IS_HELIX_CLUSTER_MANAGED,
+ GobblinClusterConfigurationKeys.DEFAULT_IS_HELIX_CLUSTER_MANAGED);
this.managerClusterHelixManager = buildHelixManager(this.config,
zkConnectionString,
GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY,
- InstanceType.CONTROLLER);
+ isHelixClusterManaged ? InstanceType.ADMINISTRATOR :
InstanceType.CONTROLLER);
this.jobClusterHelixManager = this.managerClusterHelixManager;
}
}
diff --git
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
index 673cc6c..6aee52b 100644
---
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
+++
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
@@ -37,7 +37,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang3.reflect.ConstructorUtils;
import org.apache.commons.mail.EmailException;
-import org.apache.gobblin.util.ClassAliasResolver;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -99,11 +98,12 @@ import org.apache.gobblin.cluster.HelixUtils;
import org.apache.gobblin.configuration.ConfigurationKeys;
import org.apache.gobblin.rest.JobExecutionInfoServer;
import org.apache.gobblin.runtime.app.ServiceBasedAppLauncher;
+import org.apache.gobblin.util.ClassAliasResolver;
import org.apache.gobblin.util.ConfigUtils;
import org.apache.gobblin.util.EmailUtils;
import org.apache.gobblin.util.ExecutorsUtils;
-import org.apache.gobblin.util.io.StreamUtils;
import org.apache.gobblin.util.JvmUtils;
+import org.apache.gobblin.util.io.StreamUtils;
import org.apache.gobblin.util.logs.LogCopier;
import org.apache.gobblin.yarn.event.ApplicationReportArrivalEvent;
import org.apache.gobblin.yarn.event.GetApplicationReportFailureEvent;
@@ -286,13 +286,19 @@ public class GobblinYarnAppLauncher {
public void launch() throws IOException, YarnException {
this.eventBus.register(this);
- String clusterName =
this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY);
- boolean overwriteExistingCluster = ConfigUtils.getBoolean(this.config,
GobblinClusterConfigurationKeys.HELIX_CLUSTER_OVERWRITE_KEY,
- GobblinClusterConfigurationKeys.DEFAULT_HELIX_CLUSTER_OVERWRITE);
- LOGGER.info("Creating Helix cluster {} with overwrite: {}", clusterName,
overwriteExistingCluster);
- HelixUtils.createGobblinHelixCluster(
-
this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY),
clusterName, overwriteExistingCluster);
- LOGGER.info("Created Helix cluster " + clusterName);
+ boolean isHelixClusterManaged = ConfigUtils.getBoolean(this.config,
GobblinClusterConfigurationKeys.IS_HELIX_CLUSTER_MANAGED,
+ GobblinClusterConfigurationKeys.DEFAULT_IS_HELIX_CLUSTER_MANAGED);
+ if (isHelixClusterManaged) {
+ LOGGER.info("Helix cluster is managed; skipping creation of Helix
cluster");
+ } else {
+ String clusterName =
this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY);
+ boolean overwriteExistingCluster = ConfigUtils.getBoolean(this.config,
GobblinClusterConfigurationKeys.HELIX_CLUSTER_OVERWRITE_KEY,
+ GobblinClusterConfigurationKeys.DEFAULT_HELIX_CLUSTER_OVERWRITE);
+ LOGGER.info("Creating Helix cluster {} with overwrite: {}", clusterName,
overwriteExistingCluster);
+
HelixUtils.createGobblinHelixCluster(this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY),
+ clusterName, overwriteExistingCluster);
+ LOGGER.info("Created Helix cluster " + clusterName);
+ }
connectHelixManager();
diff --git a/gradle/scripts/dependencyDefinitions.gradle
b/gradle/scripts/dependencyDefinitions.gradle
index 8ef4b3a..24c3a65 100644
--- a/gradle/scripts/dependencyDefinitions.gradle
+++ b/gradle/scripts/dependencyDefinitions.gradle
@@ -62,7 +62,7 @@ ext.externalDependency = [
"hadoopYarnMiniCluster": "org.apache.hadoop:hadoop-minicluster:" +
hadoopVersion,
"hadoopAnnotations": "org.apache.hadoop:hadoop-annotations:" +
hadoopVersion,
"hadoopAws": "org.apache.hadoop:hadoop-aws:2.6.0",
- "helix": "org.apache.helix:helix-core:0.8.2",
+ "helix": "org.apache.helix:helix-core:0.9.1",
"hiveCommon": "org.apache.hive:hive-common:" + hiveVersion,
"hiveService": "org.apache.hive:hive-service:" + hiveVersion,
"hiveJdbc": "org.apache.hive:hive-jdbc:" + hiveVersion,
