Repository: tez Updated Branches: refs/heads/master 06bd38302 -> 2050f50e9
http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfigurer.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfigurer.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfigurer.java index f36c18d..eaef16f 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfigurer.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfigurer.java @@ -29,7 +29,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.tez.common.TezJobConfig; +import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import org.junit.Test; public class TestOrderedPartitionedKVEdgeConfigurer { @@ -74,16 +74,16 @@ public class TestOrderedPartitionedKVEdgeConfigurer { rebuiltInput.fromByteArray(inputBytes); Configuration outputConf = rebuiltOutput.conf; - assertEquals(true, outputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); + assertEquals(true, outputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); assertEquals("TestCodec", - outputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); + outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); Configuration inputConf = rebuiltInput.conf; - assertEquals(true, inputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); + assertEquals(true, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); assertEquals("TestCodec", - inputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); + inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); } @Test @@ -104,11 +104,11 @@ public class TestOrderedPartitionedKVEdgeConfigurer { Configuration outputConf = rebuiltOutput.conf; assertEquals("TestCodec", - outputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); + outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); Configuration inputConf = rebuiltInput.conf; assertEquals("TestCodec", - inputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); + inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); } @Test @@ -116,24 +116,24 @@ public class TestOrderedPartitionedKVEdgeConfigurer { Configuration fromConf = new Configuration(false); fromConf.set("test.conf.key.1", "confkey1"); - fromConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 3); - fromConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.11f); - fromConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 123); + fromConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 3); + fromConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.11f); + fromConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 123); fromConf.set("io.shouldExist", "io"); Map<String, String> additionalConfs = new HashMap<String, String>(); additionalConfs.put("test.key.2", "key2"); - additionalConfs.put(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); - additionalConfs.put(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); - additionalConfs.put(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "CustomSorter"); + additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); + additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); + additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "CustomSorter"); additionalConfs.put("file.shouldExist", "file"); OrderedPartitionedKVEdgeConfigurer.Builder builder = OrderedPartitionedKVEdgeConfigurer .newBuilder("KEY", "VALUE", "PARTITIONER", null) .setAdditionalConfiguration("fs.shouldExist", "fs") .setAdditionalConfiguration("test.key.1", "key1") - .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "2222") - .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") - .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, "3333") + .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "2222") + .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") + .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, "3333") .setAdditionalConfiguration(additionalConfs) .setFromConfiguration(fromConf); @@ -150,33 +150,33 @@ public class TestOrderedPartitionedKVEdgeConfigurer { Configuration outputConf = rebuiltOutput.conf; Configuration inputConf = rebuiltInput.conf; - assertEquals(3, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0)); - assertEquals(1111, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); - assertEquals(2222, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); - assertEquals(123, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 0)); - assertEquals("CustomSorter", outputConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS)); + assertEquals(3, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 0)); + assertEquals(1111, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); + assertEquals(2222, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); + assertEquals(123, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 0)); + assertEquals("CustomSorter", outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS)); assertEquals(3333, - outputConf.getInt(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 0)); + outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 0)); assertEquals("io", outputConf.get("io.shouldExist")); assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); - assertEquals(3, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0)); - assertEquals(1111, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); - assertEquals(2222, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); + assertEquals(3, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 0)); + assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); + assertEquals(2222, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertEquals(0.11f, - inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.001f); + inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.001f); assertEquals(0.22f, - inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); - assertEquals(0.33f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), + inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); + assertEquals(0.33f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); - assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB)); - assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS)); - assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES)); + assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB)); + assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS)); + assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES)); assertEquals("io", inputConf.get("io.shouldExist")); assertEquals("file", inputConf.get("file.shouldExist")); assertEquals("fs", inputConf.get("fs.shouldExist")); @@ -206,43 +206,43 @@ public class TestOrderedPartitionedKVEdgeConfigurer { Configuration outputConf = rebuiltOutput.conf; Configuration inputConf = rebuiltInput.conf; - assertEquals("KEY", outputConf.get(TezJobConfig.TEZ_RUNTIME_KEY_CLASS, "")); + assertEquals("KEY", outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, "")); assertEquals("VALUE", - outputConf.get(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS, "")); - assertEquals("PARTITIONER", outputConf.get(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS, "")); - assertEquals(1111, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 0)); + outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, "")); + assertEquals("PARTITIONER", outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, "")); + assertEquals(1111, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 0)); assertEquals("CustomCodec", - outputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); + outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); assertEquals(true, - outputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_COMPRESS, + outputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false)); assertEquals("KEY_COMPARATOR", - outputConf.get(TezJobConfig.TEZ_RUNTIME_KEY_COMPARATOR_CLASS)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_INPUT_BUFFER_PERCENT)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT)); + outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_BUFFER_PERCENT)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT)); - assertEquals("KEY_COMPARATOR", inputConf.get(TezJobConfig.TEZ_RUNTIME_KEY_COMPARATOR_CLASS)); - assertEquals("KEY", inputConf.get(TezJobConfig.TEZ_RUNTIME_KEY_CLASS, "")); + assertEquals("KEY_COMPARATOR", inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS)); + assertEquals("KEY", inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, "")); assertEquals("VALUE", - inputConf.get(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS, "")); + inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, "")); assertEquals("CustomCodec", - inputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); + inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); assertEquals(true, - inputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_COMPRESS, + inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false)); assertEquals(0.11f, - inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); - assertEquals(0.22f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), + inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); + assertEquals(0.22f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); - assertEquals(0.33f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 0.0f), + assertEquals(0.33f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 0.0f), 0.001f); assertEquals(0.44f, - inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.00f), 0.001f); - assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB)); + inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.00f), 0.001f); + assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB)); } http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledMergedInputConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledMergedInputConfiguration.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledMergedInputConfiguration.java index cd0f0b1..4145ae0 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledMergedInputConfiguration.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledMergedInputConfiguration.java @@ -29,7 +29,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.tez.common.TezJobConfig; +import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import org.junit.Test; public class TestShuffledMergedInputConfiguration { @@ -55,11 +55,11 @@ public class TestShuffledMergedInputConfiguration { public void testSetters() { Configuration fromConf = new Configuration(false); fromConf.set("test.conf.key.1", "confkey1"); - fromConf.setInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 1111); + fromConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 1111); fromConf.set("io.shouldExist", "io"); Map<String, String> additionalConf = new HashMap<String, String>(); additionalConf.put("test.key.2", "key2"); - additionalConf.put(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, "3"); + additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, "3"); additionalConf.put("file.shouldExist", "file"); ShuffledMergedInputConfiguration.Builder builder = ShuffledMergedInputConfiguration.newBuilder("KEY", "VALUE") @@ -71,7 +71,7 @@ public class TestShuffledMergedInputConfiguration { .setShuffleBufferFraction(0.44f) .setAdditionalConfiguration("fs.shouldExist", "fs") .setAdditionalConfiguration("test.key.1", "key1") - .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, + .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, String.valueOf(false)) .setAdditionalConfiguration(additionalConf) .setFromConfiguration(fromConf); @@ -86,23 +86,23 @@ public class TestShuffledMergedInputConfiguration { Configuration conf = rebuilt.conf; // Verify programmatic API usage - assertEquals("KEY", conf.get(TezJobConfig.TEZ_RUNTIME_KEY_CLASS, "")); - assertEquals("VALUE", conf.get(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS, "")); + assertEquals("KEY", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, "")); + assertEquals("VALUE", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, "")); assertEquals("CustomCodec", - conf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); - assertEquals(true, conf.getBoolean(TezJobConfig.TEZ_RUNTIME_COMPRESS, + conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); + assertEquals(true, conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false)); - assertEquals(0.11f, conf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); - assertEquals(0.22f, conf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); - assertEquals(0.33f, conf.getFloat(TezJobConfig.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 0.0f), 0.001f); - assertEquals(0.44f, conf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.00f), 0.001f); + assertEquals(0.11f, conf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); + assertEquals(0.22f, conf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); + assertEquals(0.33f, conf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 0.0f), 0.001f); + assertEquals(0.44f, conf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.00f), 0.001f); // Verify additional configs - assertEquals(false, conf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); - assertEquals(1111, conf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT)); - assertEquals(3, conf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, -1)); + assertEquals(false, conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); + assertEquals(1111, conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT)); + assertEquals(3, conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, -1)); assertEquals("io", conf.get("io.shouldExist")); assertEquals("file", conf.get("file.shouldExist")); assertEquals("fs", conf.get("fs.shouldExist")); @@ -123,16 +123,16 @@ public class TestShuffledMergedInputConfiguration { Configuration conf = rebuilt.conf; - assertEquals(true, conf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); + assertEquals(true, conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); // Default property present. assertEquals("TestCodec", - conf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); + conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); // Verify whatever was configured - assertEquals("KEY", conf.get(TezJobConfig.TEZ_RUNTIME_KEY_CLASS, "")); - assertEquals("VALUE", conf.get(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS, "")); + assertEquals("KEY", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, "")); + assertEquals("VALUE", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, "")); } @Test @@ -140,7 +140,7 @@ public class TestShuffledMergedInputConfiguration { Configuration combinerConf = new Configuration(false); combinerConf.set("combiner.test.key", "COMBINERKEY"); combinerConf - .set(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "InvalidKeyOverride"); + .set(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "InvalidKeyOverride"); ShuffledMergedInputConfiguration.Builder builder = ShuffledMergedInputConfiguration.newBuilder("KEY", "VALUE") .setCombiner("COMBINER", combinerConf); @@ -155,7 +155,7 @@ public class TestShuffledMergedInputConfiguration { // Default Output property should not be overridden based on partitioner config assertEquals("TestCodec", - conf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); + conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); assertEquals("COMBINERKEY", conf.get("combiner.test.key")); } http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledUnorderedKVInputConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledUnorderedKVInputConfiguration.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledUnorderedKVInputConfiguration.java index 70e71c9..2a6eb17 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledUnorderedKVInputConfiguration.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledUnorderedKVInputConfiguration.java @@ -29,7 +29,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.tez.common.TezJobConfig; +import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import org.junit.Test; public class TestShuffledUnorderedKVInputConfiguration { @@ -55,11 +55,11 @@ public class TestShuffledUnorderedKVInputConfiguration { public void testSetters() { Configuration fromConf = new Configuration(false); fromConf.set("test.conf.key.1", "confkey1"); - fromConf.setInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 1111); + fromConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 1111); fromConf.set("io.shouldExist", "io"); Map<String, String> additionalConf = new HashMap<String, String>(); additionalConf.put("test.key.2", "key2"); - additionalConf.put(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, "3"); + additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, "3"); additionalConf.put("file.shouldExist", "file"); ShuffledUnorderedKVInputConfiguration.Builder builder = ShuffledUnorderedKVInputConfiguration.newBuilder("KEY", "VALUE") @@ -69,7 +69,7 @@ public class TestShuffledUnorderedKVInputConfiguration { .setShuffleBufferFraction(0.33f) .setAdditionalConfiguration("fs.shouldExist", "fs") .setAdditionalConfiguration("test.key.1", "key1") - .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, + .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, String.valueOf(false)) .setAdditionalConfiguration(additionalConf) .setFromConfiguration(fromConf); @@ -84,22 +84,22 @@ public class TestShuffledUnorderedKVInputConfiguration { Configuration conf = rebuilt.conf; // Verify programmatic API usage - assertEquals("KEY", conf.get(TezJobConfig.TEZ_RUNTIME_KEY_CLASS, "")); - assertEquals("VALUE", conf.get(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS, "")); + assertEquals("KEY", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, "")); + assertEquals("VALUE", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, "")); assertEquals("CustomCodec", - conf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); - assertEquals(true, conf.getBoolean(TezJobConfig.TEZ_RUNTIME_COMPRESS, + conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); + assertEquals(true, conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false)); - assertEquals(0.11f, conf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); - assertEquals(0.22f, conf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); - assertEquals(0.33f, conf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.00f), 0.001f); + assertEquals(0.11f, conf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); + assertEquals(0.22f, conf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); + assertEquals(0.33f, conf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.00f), 0.001f); // Verify additional configs - assertEquals(false, conf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); - assertEquals(1111, conf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT)); - assertEquals(3, conf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, -1)); + assertEquals(false, conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); + assertEquals(1111, conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT)); + assertEquals(3, conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, -1)); assertEquals("io", conf.get("io.shouldExist")); assertEquals("file", conf.get("file.shouldExist")); assertEquals("fs", conf.get("fs.shouldExist")); @@ -120,15 +120,15 @@ public class TestShuffledUnorderedKVInputConfiguration { Configuration conf = rebuilt.conf; - assertEquals(true, conf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); + assertEquals(true, conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); // Default property present. assertEquals("TestCodec", - conf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); + conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); // Verify whatever was configured - assertEquals("KEY", conf.get(TezJobConfig.TEZ_RUNTIME_KEY_CLASS, "")); - assertEquals("VALUE", conf.get(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS, "")); + assertEquals("KEY", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, "")); + assertEquals("VALUE", conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, "")); } } http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfigurer.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfigurer.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfigurer.java index 898c84b..758434c 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfigurer.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfigurer.java @@ -29,7 +29,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.tez.common.TezJobConfig; +import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import org.junit.Test; public class TestUnorderedPartitionedKVEdgeConfigurer { @@ -76,16 +76,16 @@ public class TestUnorderedPartitionedKVEdgeConfigurer { rebuiltInput.fromByteArray(inputBytes); Configuration outputConf = rebuiltOutput.conf; - assertEquals(true, outputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); + assertEquals(true, outputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); assertEquals("TestCodec", - outputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); + outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); Configuration inputConf = rebuiltInput.conf; - assertEquals(true, inputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); + assertEquals(true, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); assertEquals("TestCodec", - inputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); + inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); } @Test @@ -108,11 +108,11 @@ public class TestUnorderedPartitionedKVEdgeConfigurer { Configuration outputConf = rebuiltOutput.conf; assertEquals("TestCodec", - outputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); + outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); Configuration inputConf = rebuiltInput.conf; assertEquals("TestCodec", - inputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); + inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); } @Test @@ -120,24 +120,24 @@ public class TestUnorderedPartitionedKVEdgeConfigurer { Configuration fromConf = new Configuration(false); fromConf.set("test.conf.key.1", "confkey1"); - fromConf.setBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, false); - fromConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.11f); - fromConf.setInt(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 123); + fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, false); + fromConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.11f); + fromConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 123); fromConf.set("io.shouldExist", "io"); Map<String, String> additionalConfs = new HashMap<String, String>(); additionalConfs.put("test.key.2", "key2"); - additionalConfs.put(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); - additionalConfs.put(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); + additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); + additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); additionalConfs - .put(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222"); + .put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222"); additionalConfs.put("file.shouldExist", "file"); UnorderedPartitionedKVEdgeConfigurer.Builder builder = UnorderedPartitionedKVEdgeConfigurer .newBuilder("KEY", "VALUE", "PARTITIONER", null) .setAdditionalConfiguration("fs.shouldExist", "fs") .setAdditionalConfiguration("test.key.1", "key1") - .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333") - .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") + .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333") + .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") .setAdditionalConfiguration(additionalConfs) .setFromConfiguration(fromConf); @@ -156,32 +156,32 @@ public class TestUnorderedPartitionedKVEdgeConfigurer { Configuration outputConf = rebuiltOutput.conf; Configuration inputConf = rebuiltInput.conf; - assertEquals(false, outputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, true)); - assertEquals(1111, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); - assertEquals(3333, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); + assertEquals(false, outputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true)); + assertEquals(1111, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); + assertEquals(3333, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); assertEquals(123, - outputConf.getInt(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 0)); + outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 0)); assertEquals(2222, - outputConf.getInt(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, 0)); + outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, 0)); assertEquals("io", outputConf.get("io.shouldExist")); assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); - assertEquals(false, inputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, true)); - assertEquals(1111, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); - assertEquals(3333, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); + assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true)); + assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); + assertEquals(3333, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertEquals(0.11f, - inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.001f); + inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.001f); assertEquals(0.22f, - inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); - assertEquals(0.33f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), + inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); + assertEquals(0.33f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); - assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB)); - assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES)); + assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB)); + assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES)); assertEquals("io", inputConf.get("io.shouldExist")); assertEquals("file", inputConf.get("file.shouldExist")); assertEquals("fs", inputConf.get("fs.shouldExist")); http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedUnpartitionedKVEdgeConfigurer.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedUnpartitionedKVEdgeConfigurer.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedUnpartitionedKVEdgeConfigurer.java index 691acef..c86542f 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedUnpartitionedKVEdgeConfigurer.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedUnpartitionedKVEdgeConfigurer.java @@ -29,7 +29,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.tez.common.TezJobConfig; +import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import org.junit.Test; public class TestUnorderedUnpartitionedKVEdgeConfigurer { @@ -69,16 +69,16 @@ public class TestUnorderedUnpartitionedKVEdgeConfigurer { rebuiltInput.fromByteArray(inputBytes); Configuration outputConf = rebuiltOutput.conf; - assertEquals(true, outputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); + assertEquals(true, outputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); assertEquals("TestCodec", - outputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); + outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); Configuration inputConf = rebuiltInput.conf; - assertEquals(true, inputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, - TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); + assertEquals(true, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, + TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT)); assertEquals("TestCodec", - inputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "")); + inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); } @Test @@ -101,11 +101,11 @@ public class TestUnorderedUnpartitionedKVEdgeConfigurer { Configuration outputConf = rebuiltOutput.conf; assertEquals("TestCodec", - outputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); + outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); Configuration inputConf = rebuiltInput.conf; assertEquals("TestCodec", - inputConf.get(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); + inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); } @Test @@ -113,13 +113,13 @@ public class TestUnorderedUnpartitionedKVEdgeConfigurer { Configuration fromConf = new Configuration(false); fromConf.set("test.conf.key.1", "confkey1"); - fromConf.setBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, false); - fromConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.11f); + fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, false); + fromConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.11f); fromConf.set("io.shouldExist", "io"); Map<String, String> additionalConfs = new HashMap<String, String>(); additionalConfs.put("test.key.2", "key2"); - additionalConfs.put(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); - additionalConfs.put(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); + additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); + additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); additionalConfs.put("file.shouldExist", "file"); UnorderedUnpartitionedKVEdgeConfigurer.Builder builder = UnorderedUnpartitionedKVEdgeConfigurer @@ -127,8 +127,8 @@ public class TestUnorderedUnpartitionedKVEdgeConfigurer { "VALUE") .setAdditionalConfiguration("fs.shouldExist", "fs") .setAdditionalConfiguration("test.key.1", "key1") - .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333") - .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") + .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333") + .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") .setAdditionalConfiguration(additionalConfs) .setFromConfiguration(fromConf); @@ -147,25 +147,25 @@ public class TestUnorderedUnpartitionedKVEdgeConfigurer { Configuration outputConf = rebuiltOutput.conf; Configuration inputConf = rebuiltInput.conf; - assertEquals(false, outputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, true)); - assertEquals(1111, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); - assertEquals(3333, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); - assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); + assertEquals(false, outputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true)); + assertEquals(1111, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); + assertEquals(3333, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); + assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); assertEquals("io", outputConf.get("io.shouldExist")); assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); - assertEquals(false, inputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, true)); - assertEquals(1111, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); - assertEquals(3333, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); + assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true)); + assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); + assertEquals(3333, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertEquals(0.11f, - inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.001f); + inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.001f); assertEquals(0.22f, - inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); - assertEquals(0.33f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), + inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); + assertEquals(0.33f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); assertEquals("io", inputConf.get("io.shouldExist")); assertEquals("file", inputConf.get("file.shouldExist")); http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java index e96c7c7..17ba5dc 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java @@ -38,7 +38,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper; -import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.TezUtils; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.OutputDescriptor; @@ -54,6 +53,7 @@ import org.apache.tez.runtime.api.impl.TezOutputContextImpl; import org.apache.tez.runtime.api.impl.TezUmbilical; import org.apache.tez.runtime.common.resources.MemoryDistributor; import org.apache.tez.runtime.library.api.KeyValueWriter; +import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import org.apache.tez.runtime.library.shuffle.common.ShuffleUtils; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovementEventPayloadProto; import org.apache.tez.runtime.library.testutils.KVDataGen; @@ -95,8 +95,8 @@ public class TestOnFileUnorderedKVOutput { OnFileUnorderedKVOutput kvOutput = new OnFileUnorderedKVOutputForTest(); Configuration conf = new Configuration(); - conf.set(TezJobConfig.TEZ_RUNTIME_KEY_CLASS, Text.class.getName()); - conf.set(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS, IntWritable.class.getName()); + conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName()); + conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, IntWritable.class.getName()); int appAttemptNumber = 1; TezUmbilical tezUmbilical = null; http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestSimpleFetchedInputAllocator.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestSimpleFetchedInputAllocator.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestSimpleFetchedInputAllocator.java index d64da35..f6c3772 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestSimpleFetchedInputAllocator.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestSimpleFetchedInputAllocator.java @@ -26,8 +26,8 @@ import java.util.UUID; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.TezRuntimeFrameworkConfigs; +import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import org.apache.tez.runtime.library.common.InputAttemptIdentifier; import org.apache.tez.runtime.library.shuffle.common.FetchedInput; import org.junit.Test; @@ -45,8 +45,8 @@ public class TestSimpleFetchedInputAllocator { LOG.info("jvmMax: " + jvmMax); float bufferPercent = 0.1f; - conf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, bufferPercent); - conf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 1.0f); + conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, bufferPercent); + conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 1.0f); conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDirs); long inMemThreshold = (long) (bufferPercent * jvmMax); http://git-wip-us.apache.org/repos/asf/tez/blob/2050f50e/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java ---------------------------------------------------------------------- diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java index 8cebb71..594a2f7 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java @@ -32,8 +32,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; -import org.apache.tez.common.TezJobConfig; import org.apache.tez.mapreduce.examples.OrderedWordCount; +import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -68,8 +68,8 @@ public class TestSecureShuffle { conf.set("fs.defaultFS", fs.getUri().toString()); // 15 seconds should be good enough in local machine - conf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 15 * 1000); - conf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 15 * 1000); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 15 * 1000); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 15 * 1000); miniTezCluster = new MiniTezCluster(TestSecureShuffle.class.getName(), 3, 3, 1); @@ -87,7 +87,7 @@ public class TestSecureShuffle { @Test(timeout = 60000) public void testSecureShuffle() throws Exception { miniTezCluster.getConfig().setBoolean( - TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, true); + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, true); OrderedWordCount wordCount = new OrderedWordCount(); wordCount.setConf(new Configuration(miniTezCluster.getConfig()));
