Repository: tez Updated Branches: refs/heads/master 539b8e710 -> e9fb41864
TEZ-3330. Propagate additional config parameters when running MR jobs via Tez. (sseth) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/e9fb4186 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/e9fb4186 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/e9fb4186 Branch: refs/heads/master Commit: e9fb418647dec7253956a1a62fbbcd22175144e4 Parents: 539b8e7 Author: Siddharth Seth <[email protected]> Authored: Mon Oct 10 10:56:16 2016 -0700 Committer: Siddharth Seth <[email protected]> Committed: Mon Oct 10 10:56:16 2016 -0700 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../org/apache/tez/mapreduce/client/YARNRunner.java | 3 ++- .../tez/runtime/library/common/ConfigUtils.java | 10 ++++++++++ .../tez/runtime/library/conf/BaseConfigBuilder.java | 2 ++ .../library/conf/OrderedGroupedKVInputConfig.java | 16 ++++++++++++++++ .../conf/OrderedPartitionedKVEdgeConfig.java | 7 +++++++ .../conf/OrderedPartitionedKVOutputConfig.java | 16 ++++++++++++++++ .../runtime/library/conf/UnorderedKVEdgeConfig.java | 7 +++++++ .../library/conf/UnorderedKVInputConfig.java | 15 +++++++++++++++ .../library/conf/UnorderedKVOutputConfig.java | 16 ++++++++++++++++ .../conf/UnorderedPartitionedKVEdgeConfig.java | 8 ++++++++ .../conf/UnorderedPartitionedKVOutputConfig.java | 16 ++++++++++++++++ .../conf/TestOrderedGroupedMergedKVInputConfig.java | 7 ++++++- .../conf/TestOrderedPartitionedKVEdgeConfig.java | 9 ++++++++- .../conf/TestOrderedPartitionedKVOutputConfig.java | 7 ++++++- .../library/conf/TestUnorderedKVEdgeConfig.java | 7 ++++++- .../library/conf/TestUnorderedKVInputConfig.java | 7 ++++++- .../library/conf/TestUnorderedKVOutputConfig.java | 7 ++++++- .../conf/TestUnorderedPartitionedKVEdgeConfig.java | 7 ++++++- .../TestUnorderedPartitionedKVOutputConfig.java | 7 ++++++- 20 files changed, 167 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index addd2c0..10fbd92 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3330. Propagate additional config parameters when running MR jobs via Tez. TEZ-3335. DAG client thinks app is still running when app status is null TEZ-3437. Improve synchronization and the progress report behavior for Inputs from TEZ-3317. TEZ-3460. Fix precommit release audit warning. @@ -119,6 +120,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3330. Propagate additional config parameters when running MR jobs via Tez. TEZ-3335. DAG client thinks app is still running when app status is null TEZ-3460. Fix precommit release audit warning. TEZ-3368. NPE in DelayedContainerManager http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java index 1bea781..820e2e4 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java @@ -515,8 +515,9 @@ public class YARNRunner implements ClientProtocol { TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS), stageConfs[i - 1].get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS), MRPartitioner.class.getName(), partitionerConf) + .setFromConfigurationUnfiltered(stageConfs[i-1]) .configureInput().useLegacyInput().done() - .setFromConfiguration(stageConfs[i - 1]).build(); + .build(); Edge edge = Edge.create(vertices[i - 1], vertices[i], edgeConf.createDefaultEdgeProperty()); dag.addEdge(edge); } http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java index c63f737..6aa797f 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java @@ -227,6 +227,16 @@ public class ConfigUtils { } } + public static void mergeConfs(Configuration destConf, Configuration srcConf) { + Preconditions.checkState(destConf != null, "Destination conf cannot be null"); + Preconditions.checkState(srcConf != null, "Source conf cannot be null"); + for (Map.Entry<String, String> entry : srcConf) { + // Explicit get to have parameter replacement work. + String val = srcConf.get(entry.getKey()); + destConf.set(entry.getKey(), val); + } + } + private static Map<String, String> extractConfigurationMapInternal( Iterable<Map.Entry<String, String>> iterable, List<Set<String>> validKeySets, List<String> allowedPrefixes) { Set<String> validKeys = new HashSet<String>(); http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java index d991e89..162addf 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java @@ -59,4 +59,6 @@ interface BaseConfigBuilder<T> { */ @InterfaceAudience.LimitedPrivate({"hive, pig"}) public T setFromConfiguration(Configuration conf); + + public T setFromConfigurationUnfiltered(Configuration conf); } http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java index 95d8e75..11a8d6f 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java @@ -212,6 +212,13 @@ public class OrderedGroupedKVInputConfig { return this; } + @Override + public SpecificBuilder setFromConfigurationUnfiltered( + Configuration conf) { + builder.setFromConfigurationUnfiltered(conf); + return this; + } + public E done() { return edgeBuilder; } @@ -448,6 +455,15 @@ public class OrderedGroupedKVInputConfig { return this; } + @SuppressWarnings("unchecked") + @Override + public Builder setFromConfigurationUnfiltered(Configuration conf) { + // Maybe ensure this is the first call ? Otherwise this can end up overriding other parameters + Preconditions.checkArgument(conf != null, "Configuration cannot be null"); + ConfigUtils.mergeConfs(this.conf, conf); + return this; + } + public Builder setCompression(boolean enabled, @Nullable String compressionCodec, @Nullable Map<String, String> codecConf) { this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled); http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java index e6cc2c2..3504203 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java @@ -284,6 +284,13 @@ public class OrderedPartitionedKVEdgeConfig return this; } + @Override + public Builder setFromConfigurationUnfiltered(Configuration conf) { + outputBuilder.setFromConfigurationUnfiltered(conf); + inputBuilder.setFromConfigurationUnfiltered(conf); + return this; + } + /** * Configure the specific output * @return a builder to configure the output http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java index 0687e79..0f37c66 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java @@ -177,6 +177,13 @@ public class OrderedPartitionedKVOutputConfig { return this; } + @Override + public SpecificBuilder setFromConfigurationUnfiltered( + Configuration conf) { + builder.setFromConfigurationUnfiltered(conf); + return this; + } + public E done() { return edgeBuilder; } @@ -374,6 +381,15 @@ public class OrderedPartitionedKVOutputConfig { return this; } + @SuppressWarnings("unchecked") + @Override + public Builder setFromConfigurationUnfiltered(Configuration conf) { + // Maybe ensure this is the first call ? Otherwise this can end up overriding other parameters + Preconditions.checkArgument(conf != null, "Configuration cannot be null"); + ConfigUtils.mergeConfs(this.conf, conf); + return this; + } + /** * Set the key comparator class * http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java index 28a7503..25a4823 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java @@ -219,6 +219,13 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig { return this; } + @Override + public Builder setFromConfigurationUnfiltered(Configuration conf) { + outputBuilder.setFromConfigurationUnfiltered(conf); + inputBuilder.setFromConfigurationUnfiltered(conf); + return this; + } + /** * Set serialization class responsible for providing serializer/deserializer for key/value and * the corresponding comparator class to be used as key comparator. http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java index f52c672..af7dbf6 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java @@ -139,6 +139,12 @@ public class UnorderedKVInputConfig { return this; } + @Override + public SpecificBuilder setFromConfigurationUnfiltered(Configuration conf) { + builder.setFromConfigurationUnfiltered(conf); + return this; + } + public E done() { return edgeBuilder; } @@ -296,6 +302,15 @@ public class UnorderedKVInputConfig { return this; } + @SuppressWarnings("unchecked") + @Override + public Builder setFromConfigurationUnfiltered(Configuration conf) { + // Maybe ensure this is the first call ? Otherwise this can end up overriding other parameters + Preconditions.checkArgument(conf != null, "Configuration cannot be null"); + ConfigUtils.mergeConfs(this.conf, conf); + return this; + } + public Builder setCompression(boolean enabled, @Nullable String compressionCodec, @Nullable Map<String, String> codecConf) { this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled); http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java index e3eeedc..a9a15a0 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java @@ -86,6 +86,13 @@ public class UnorderedKVOutputConfig { return this; } + @Override + public SpecificBuilder setFromConfigurationUnfiltered( + Configuration conf) { + builder.setFromConfigurationUnfiltered(conf); + return this; + } + public E done() { return edgeBuilder; } @@ -222,6 +229,15 @@ public class UnorderedKVOutputConfig { return this; } + @SuppressWarnings("unchecked") + @Override + public Builder setFromConfigurationUnfiltered(Configuration conf) { + // Maybe ensure this is the first call ? Otherwise this can end up overriding other parameters + Preconditions.checkArgument(conf != null, "Configuration cannot be null"); + ConfigUtils.mergeConfs(this.conf, conf); + return this; + } + /** * Set serialization class responsible for providing serializer/deserializer for keys. * http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java index 8ac7d65..52da491 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java @@ -224,6 +224,14 @@ public class UnorderedPartitionedKVEdgeConfig return this; } + @Override + public Builder setFromConfigurationUnfiltered( + Configuration conf) { + outputBuilder.setFromConfigurationUnfiltered(conf); + inputBuilder.setFromConfigurationUnfiltered(conf); + return this; + } + /** * Set serialization class responsible for providing serializer/deserializer for keys. * http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java index 1cf0924..3555e1c 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java @@ -100,6 +100,13 @@ public class UnorderedPartitionedKVOutputConfig { return this; } + @Override + public SpecificBuilder setFromConfigurationUnfiltered( + Configuration conf) { + builder.setFromConfigurationUnfiltered(conf); + return this; + } + public E done() { return edgeBuilder; } @@ -266,6 +273,15 @@ public class UnorderedPartitionedKVOutputConfig { return this; } + @SuppressWarnings("unchecked") + @Override + public Builder setFromConfigurationUnfiltered(Configuration conf) { + // Maybe ensure this is the first call ? Otherwise this can end up overriding other parameters + Preconditions.checkArgument(conf != null, "Configuration cannot be null"); + ConfigUtils.mergeConfs(this.conf, conf); + return this; + } + public Builder setCompression(boolean enabled, @Nullable String compressionCodec, @Nullable Map<String, String> codecConf) { this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled); http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java index 49b5490..70dbe14 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java @@ -75,6 +75,8 @@ public class TestOrderedGroupedMergedKVInputConfig { .TEZ_RUNTIME_SHUFFLE_MIN_REQUIRED_PROGRESS_FRACTION, 0.6f); fromConf.setBoolean(TezRuntimeConfiguration .TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION, false); + Configuration fromConfUnfiltered = new Configuration(false); + fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1"); Map<String, String> additionalConf = new HashMap<String, String>(); additionalConf.put("test.key.2", "key2"); additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, "3"); @@ -92,7 +94,8 @@ public class TestOrderedGroupedMergedKVInputConfig { .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, String.valueOf(false)) .setAdditionalConfiguration(additionalConf) - .setFromConfiguration(fromConf); + .setFromConfiguration(fromConf) + .setFromConfigurationUnfiltered(fromConfUnfiltered); OrderedGroupedKVInputConfig configuration = builder.build(); @@ -139,6 +142,8 @@ public class TestOrderedGroupedMergedKVInputConfig { assertNull(conf.get("test.conf.key.1")); assertNull(conf.get("test.key.1")); assertNull(conf.get("test.key.2")); + + assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1")); } @Test(timeout = 5000) http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java index 9d6ca50..9567770 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java @@ -128,6 +128,8 @@ public class TestOrderedPartitionedKVEdgeConfig { true); fromConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1000); + Configuration fromConfUnfiltered = new Configuration(false); + fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1"); Map<String, String> additionalConfs = new HashMap<String, String>(); additionalConfs.put("test.key.2", "key2"); additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); @@ -145,7 +147,8 @@ public class TestOrderedPartitionedKVEdgeConfig { .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, "3333") .setAdditionalConfiguration(additionalConfs) - .setFromConfiguration(fromConf); + .setFromConfiguration(fromConf) + .setFromConfigurationUnfiltered(fromConfUnfiltered); OrderedPartitionedKVEdgeConfig configuration = builder.build(); @@ -175,6 +178,9 @@ public class TestOrderedPartitionedKVEdgeConfig { assertEquals("io", outputConf.get("io.shouldExist")); assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); + + assertEquals("unfiltered1", outputConf.get("test.conf.unfiltered.1")); + ReportPartitionStats partitionStats = ReportPartitionStats.fromString(outputConf.get( TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS, @@ -198,6 +204,7 @@ public class TestOrderedPartitionedKVEdgeConfig { assertEquals("file", inputConf.get("file.shouldExist")); assertEquals("fs", inputConf.get("fs.shouldExist")); + assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1")); } @Test (timeout=2000) http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java index 891e807..2dca183 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java @@ -73,6 +73,8 @@ public class TestOrderedPartitionedKVOutputConfig { additionalConf.put("test.key.2", "key2"); additionalConf.put("io.shouldExist", "io"); additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "TestInternalSorter"); + Configuration fromConfUnfiltered = new Configuration(false); + fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1"); OrderedPartitionedKVOutputConfig.Builder builder = OrderedPartitionedKVOutputConfig.newBuilder("KEY", "VALUE", "PARTITIONER", null) .setKeyComparatorClass("KEY_COMPARATOR", null) @@ -83,7 +85,8 @@ public class TestOrderedPartitionedKVOutputConfig { .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, String.valueOf(false)) .setAdditionalConfiguration(additionalConf) - .setFromConfiguration(fromConf); + .setFromConfiguration(fromConf) + .setFromConfigurationUnfiltered(fromConfUnfiltered); OrderedPartitionedKVOutputConfig configuration = builder.build(); @@ -116,6 +119,8 @@ public class TestOrderedPartitionedKVOutputConfig { assertNull(conf.get("test.conf.key.1")); assertNull(conf.get("test.key.1")); assertNull(conf.get("test.key.2")); + + assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1")); } @Test(timeout = 5000) http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java index 0ef1cc5..3c97020 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java @@ -121,6 +121,8 @@ public class TestUnorderedKVEdgeConfig { fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, false); fromConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.11f); fromConf.set("io.shouldExist", "io"); + Configuration fromConfUnfiltered = new Configuration(false); + fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1"); Map<String, String> additionalConfs = new HashMap<String, String>(); additionalConfs.put("test.key.2", "key2"); additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); @@ -135,7 +137,8 @@ public class TestUnorderedKVEdgeConfig { .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333") .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") .setAdditionalConfiguration(additionalConfs) - .setFromConfiguration(fromConf); + .setFromConfiguration(fromConf) + .setFromConfigurationUnfiltered(fromConfUnfiltered); UnorderedKVEdgeConfig configuration = builder.build(); @@ -159,6 +162,7 @@ public class TestUnorderedKVEdgeConfig { assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); + assertEquals("unfiltered1", outputConf.get("test.conf.unfiltered.1")); assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true)); assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); @@ -173,6 +177,7 @@ public class TestUnorderedKVEdgeConfig { assertEquals("file", inputConf.get("file.shouldExist")); assertEquals("fs", inputConf.get("fs.shouldExist")); + assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1")); } private void checkHistoryText(String historyText) { http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java index 8c7dea4..bb75442 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java @@ -62,6 +62,8 @@ public class TestUnorderedKVInputConfig { additionalConf.put("test.key.2", "key2"); additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, "3"); additionalConf.put("file.shouldExist", "file"); + Configuration fromConfUnfiltered = new Configuration(false); + fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1"); UnorderedKVInputConfig.Builder builder = UnorderedKVInputConfig.newBuilder("KEY", "VALUE") .setCompression(true, "CustomCodec", null) @@ -73,7 +75,8 @@ public class TestUnorderedKVInputConfig { .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, String.valueOf(false)) .setAdditionalConfiguration(additionalConf) - .setFromConfiguration(fromConf); + .setFromConfiguration(fromConf) + .setFromConfigurationUnfiltered(fromConfUnfiltered); UnorderedKVInputConfig configuration = builder.build(); @@ -105,6 +108,8 @@ public class TestUnorderedKVInputConfig { assertNull(conf.get("test.conf.key.1")); assertNull(conf.get("test.key.1")); assertNull(conf.get("test.key.2")); + + assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1")); } @Test(timeout = 5000) http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java index 10ed199..f99b3ae 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java @@ -63,6 +63,8 @@ public class TestUnorderedKVOutputConfig { additionalConf.put("test.key.2", "key2"); additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222"); additionalConf.put("file.shouldExist", "file"); + Configuration fromConfUnfiltered = new Configuration(false); + fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1"); UnorderedKVOutputConfig.Builder builder = UnorderedKVOutputConfig.newBuilder("KEY", "VALUE") .setCompression(true, "CustomCodec", null) @@ -71,7 +73,8 @@ public class TestUnorderedKVOutputConfig { .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, String.valueOf(false)) .setAdditionalConfiguration(additionalConf) - .setFromConfiguration(fromConf); + .setFromConfiguration(fromConf) + .setFromConfigurationUnfiltered(fromConfUnfiltered); UnorderedKVOutputConfig configuration = builder.build(); @@ -100,6 +103,8 @@ public class TestUnorderedKVOutputConfig { assertNull(conf.get("test.conf.key.1")); assertNull(conf.get("test.key.1")); assertNull(conf.get("test.key.2")); + + assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1")); } @Test(timeout = 5000) http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java index fff8efd..684f773 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java @@ -136,6 +136,8 @@ public class TestUnorderedPartitionedKVEdgeConfig { additionalConfs .put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222"); additionalConfs.put("file.shouldExist", "file"); + Configuration fromConfUnfiltered = new Configuration(false); + fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1"); UnorderedPartitionedKVEdgeConfig.Builder builder = UnorderedPartitionedKVEdgeConfig .newBuilder("KEY", "VALUE", "PARTITIONER") @@ -144,7 +146,8 @@ public class TestUnorderedPartitionedKVEdgeConfig { .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333") .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") .setAdditionalConfiguration(additionalConfs) - .setFromConfiguration(fromConf); + .setFromConfiguration(fromConf) + .setFromConfigurationUnfiltered(fromConfUnfiltered); UnorderedPartitionedKVEdgeConfig configuration = builder.build(); @@ -172,6 +175,7 @@ public class TestUnorderedPartitionedKVEdgeConfig { assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); + assertEquals("unfiltered1", outputConf.get("test.conf.unfiltered.1")); assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true)); assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); @@ -188,6 +192,7 @@ public class TestUnorderedPartitionedKVEdgeConfig { assertEquals("file", inputConf.get("file.shouldExist")); assertEquals("fs", inputConf.get("fs.shouldExist")); + assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1")); } private void checkHistoryText(String historyText) { http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java index f509acb..a9e955e 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java @@ -73,6 +73,8 @@ public class TestUnorderedPartitionedKVOutputConfig { additionalConf.put("test.key.2", "key2"); additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222"); additionalConf.put("file.shouldExist", "file"); + Configuration fromConfUnfiltered = new Configuration(false); + fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1"); UnorderedPartitionedKVOutputConfig.Builder builder = UnorderedPartitionedKVOutputConfig.newBuilder("KEY", "VALUE", "PARTITIONER", null) @@ -87,7 +89,8 @@ public class TestUnorderedPartitionedKVOutputConfig { .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, String.valueOf(false)) .setAdditionalConfiguration(additionalConf) - .setFromConfiguration(fromConf); + .setFromConfiguration(fromConf) + .setFromConfigurationUnfiltered(fromConfUnfiltered); UnorderedPartitionedKVOutputConfig configuration = builder.build(); @@ -125,6 +128,8 @@ public class TestUnorderedPartitionedKVOutputConfig { assertNull(conf.get("test.conf.key.1")); assertNull(conf.get("test.key.1")); assertNull(conf.get("test.key.2")); + + assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1")); } @Test(timeout = 5000)
