TEZ-2364. Resolve config parameter replacement on the client, before sending them to the AM. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2935ef41 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2935ef41 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2935ef41 Branch: refs/heads/TEZ-2003 Commit: 2935ef415642a9869166a1d5e1484a635e4401a2 Parents: bbbebb8 Author: Siddharth Seth <[email protected]> Authored: Fri Apr 24 13:42:01 2015 -0700 Committer: Siddharth Seth <[email protected]> Committed: Fri Apr 24 13:42:01 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/tez/client/TezClientUtils.java | 2 +- .../apache/tez/dag/api/TezConfiguration.java | 2 +- .../apache/tez/client/TestTezClientUtils.java | 36 ++++++++++++++++---- 4 files changed, 33 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index e21d178..92c1f6e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly ALL CHANGES: + TEZ-2364. Resolve config parameter replacement on the client, before sending them to the AM. TEZ-2298. Avoid logging full exception trace in TaskRunner when it's not the main error reason and is ignored. TEZ-2248. VertexImpl/DAGImpl.checkForCompletion have too many termination cause checks TEZ-2341. TestMockDAGAppMaster.testBasicCounters fails on windows http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index d12ece2..86ba0f9 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -743,7 +743,7 @@ public class TezClientUtils { for (Entry<String, String> entry : amConf) { PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder(); kvp.setKey(entry.getKey()); - kvp.setValue(entry.getValue()); + kvp.setValue(amConf.get(entry.getKey())); builder.addConfKeyValues(kvp); } if (additionalConfigs != null && !additionalConfigs.isEmpty()) { http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 7a49ee5..50b3e00 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -133,7 +133,7 @@ public class TezConfiguration extends Configuration { public static final String TEZ_AM_STAGING_DIR = TEZ_PREFIX + "staging-dir"; public static final String TEZ_AM_STAGING_DIR_DEFAULT = "/tmp/" + System.getProperty("user.name") + "/tez/staging"; - + /** * String value that is a file path. * Path to a credentials file (with serialized credentials) located on the local file system. http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java index ea73ab3..cfba917 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java @@ -46,7 +46,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.io.DataInputByteBuffer; -import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -68,7 +67,8 @@ import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezConstants; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.Vertex; -import org.apache.tez.dag.api.records.DAGProtos; +import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto; +import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair; import org.junit.Assert; import org.junit.Test; @@ -283,7 +283,8 @@ public class TestTezClientUtils { public void testAMLoggingOptsPerLogger() throws IOException, YarnException { TezConfiguration tezConf = new TezConfiguration(); - tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG"); + tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, + "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG"); ApplicationId appId = ApplicationId.newInstance(1000, 1); Credentials credentials = new Credentials(); @@ -301,7 +302,8 @@ public class TestTezClientUtils { List<String> expectedCommands = new LinkedList<String>(); expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"); - expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE); + expectedCommands.add( + "-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE); expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," + @@ -465,6 +467,28 @@ public class TestTezClientUtils { && javaOpts.contains("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator")); } + @Test (timeout = 5000) + public void testConfSerializationForAm() { + Configuration conf =new Configuration(false); + String val1 = "fixedProperty"; + String val2 = "parametrizedProperty/${user.name}"; + String expVal2 = "parametrizedProperty/" + System.getProperty("user.name"); + conf.set("property1", val1); + conf.set("property2", val2); + + Map<String, String> expected = new HashMap<String, String>(); + expected.put("property1", val1); + expected.put("property2", expVal2); + + ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(conf, null); + + for (PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) { + String v = expected.remove(kvPair.getKey()); + assertEquals(v, kvPair.getValue()); + } + assertTrue(expected.isEmpty()); + } + // To run this test case see TestTezCommonUtils::testLocalResourceVisibility // We do not have much control over the directory structure, cannot mock as the functions are // static and do not want to spin up a minitez cluster just for this. @@ -560,9 +584,9 @@ public class TestTezClientUtils { srcConf.set(entry.getKey(), entry.getValue()); } - DAGProtos.ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(srcConf); + ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(srcConf); - for (DAGProtos.PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) { + for (PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) { String val = confMap.remove(kvPair.getKey()); assertNotNull(val); assertEquals(val, kvPair.getValue());
