This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 10f5932 [GOBBLIN-1048] Provide an option to pass and set System
properties via…
10f5932 is described below
commit 10f593223902aaed18d045095e812a2e04da04db
Author: sv2000 <[email protected]>
AuthorDate: Wed Feb 12 14:45:06 2020 -0800
[GOBBLIN-1048] Provide an option to pass and set System properties via…
Closes #2888 from sv2000/systemProperties
---
.../cluster/GobblinClusterConfigurationKeys.java | 2 ++
.../apache/gobblin/cluster/GobblinClusterManager.java | 5 +++++
.../org/apache/gobblin/cluster/GobblinTaskRunner.java | 5 +++++
.../java/org/apache/gobblin/cluster/HelixUtils.java | 18 ++++++++++++++++++
.../org/apache/gobblin/cluster/HelixUtilsTest.java | 18 ++++++++++++++++++
5 files changed, 48 insertions(+)
diff --git
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
index 27e909d..539e7d1 100644
---
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
+++
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java
@@ -193,4 +193,6 @@ public class GobblinClusterConfigurationKeys {
public static final String HELIX_PARTITION_ID_KEY = GOBBLIN_HELIX_PREFIX +
"partitionId" ;
public static final String TASK_RUNNER_HOST_NAME_KEY = GOBBLIN_HELIX_PREFIX
+ "hostName";
public static final String CONTAINER_ID_KEY = GOBBLIN_HELIX_PREFIX +
"containerId";
+
+ public static final String GOBBLIN_CLUSTER_SYSTEM_PROPERTY_PREFIX =
GOBBLIN_CLUSTER_PREFIX + "sysProps";
}
diff --git
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
index e1e8fd6..5900f64 100644
---
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
+++
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java
@@ -150,6 +150,11 @@ public class GobblinClusterManager implements
ApplicationLauncher, StandardMetri
this.applicationId = applicationId;
+ //Set system properties passed in via application config. As an example,
Helix uses System#getProperty() for ZK configuration
+ // overrides such as sessionTimeout. In this case, the overrides specified
+ // in the application configuration have to be extracted and set before
initializing HelixManager.
+ HelixUtils.setSystemProperties(config);
+
initializeHelixManager();
this.fs = buildFileSystem(config);
diff --git
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinTaskRunner.java
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinTaskRunner.java
index 9693227..4504cb5 100644
---
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinTaskRunner.java
+++
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinTaskRunner.java
@@ -174,6 +174,11 @@ public class GobblinTaskRunner implements
StandardMetricsBridge {
this.config = saveConfigToFile(config);
this.clusterName =
this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY);
+ //Set system properties passed in via application config. As an example,
Helix uses System#getProperty() for ZK configuration
+ // overrides such as sessionTimeout. In this case, the overrides specified
+ // in the application configuration have to be extracted and set before
initializing HelixManager.
+ HelixUtils.setSystemProperties(config);
+
initHelixManager();
this.containerMetrics = buildContainerMetrics();
diff --git
a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java
b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java
index 340c9e7..3eddb57 100644
--- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java
+++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Optional;
+import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -39,11 +40,15 @@ import org.apache.helix.task.WorkflowConfig;
import org.apache.helix.task.WorkflowContext;
import org.apache.helix.tools.ClusterSetup;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
import lombok.extern.slf4j.Slf4j;
import org.apache.gobblin.configuration.ConfigurationKeys;
import org.apache.gobblin.runtime.JobException;
import org.apache.gobblin.runtime.listeners.JobListener;
+import org.apache.gobblin.util.ConfigUtils;
import static org.apache.helix.task.TaskState.STOPPED;
@@ -308,4 +313,17 @@ public class HelixUtils {
}
return jobNameToWorkflowId;
}
+
+ /**
+ * Return the system properties from the input {@link Config} instance
+ * @param config
+ */
+ public static void setSystemProperties(Config config) {
+ Properties properties =
ConfigUtils.configToProperties(ConfigUtils.getConfig(config,
GobblinClusterConfigurationKeys.GOBBLIN_CLUSTER_SYSTEM_PROPERTY_PREFIX,
+ ConfigFactory.empty()));
+
+ for (Map.Entry<Object, Object> entry: properties.entrySet()) {
+ System.setProperty(entry.getKey().toString(),
entry.getValue().toString());
+ }
+ }
}
\ No newline at end of file
diff --git
a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/HelixUtilsTest.java
b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/HelixUtilsTest.java
index d7841a8..398d4d2 100644
---
a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/HelixUtilsTest.java
+++
b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/HelixUtilsTest.java
@@ -33,6 +33,7 @@ import org.testng.annotations.Test;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigValueFactory;
import org.apache.gobblin.util.ConfigUtils;
@@ -80,6 +81,23 @@ public class HelixUtilsTest {
Assert.assertEquals(properties.getProperty("k5"), "10000");
}
+ @Test
+ public void testSetSystemProperties() {
+ //Set a dummy property before calling HelixUtils#setSystemProperties() and
assert that this property and value
+ //exists even after the call to the setSystemProperties() method.
+ System.setProperty("prop1", "val1");
+
+ Config config =
ConfigFactory.empty().withValue(GobblinClusterConfigurationKeys.GOBBLIN_CLUSTER_SYSTEM_PROPERTY_PREFIX
+ ".prop2",
+ ConfigValueFactory.fromAnyRef("val2"))
+
.withValue(GobblinClusterConfigurationKeys.GOBBLIN_CLUSTER_SYSTEM_PROPERTY_PREFIX
+ ".prop3", ConfigValueFactory.fromAnyRef("val3"));
+
+ HelixUtils.setSystemProperties(config);
+
+ Assert.assertEquals(System.getProperty("prop1"), "val1");
+ Assert.assertEquals(System.getProperty("prop2"), "val2");
+ Assert.assertEquals(System.getProperty("prop3"), "val3");
+ }
+
@AfterClass
public void tearDown() throws IOException {
if (this.fileSystem.exists(this.tokenFilePath.getParent())) {
