Repository: hadoop Updated Branches: refs/heads/trunk e864f8147 -> f29a0fc28
HDFS-12303. Change default EC cell size to 1MB for better performance. Contributed by Wei Zhou. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f29a0fc2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f29a0fc2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f29a0fc2 Branch: refs/heads/trunk Commit: f29a0fc288a625522ba910e61b63fd5f10418b3d Parents: e864f81 Author: Andrew Wang <[email protected]> Authored: Fri Aug 25 14:14:23 2017 -0700 Committer: Andrew Wang <[email protected]> Committed: Fri Aug 25 14:14:23 2017 -0700 ---------------------------------------------------------------------- .../hdfs/client/HdfsClientConfigKeys.java | 2 +- .../protocol/SystemErasureCodingPolicies.java | 4 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- .../src/main/resources/hdfs-default.xml | 2 +- .../src/site/markdown/HDFSErasureCoding.md | 8 +- .../apache/hadoop/cli/TestErasureCodingCLI.java | 2 +- .../hadoop/hdfs/TestErasureCodingPolicies.java | 14 ++- .../blockmanagement/TestBlockManager.java | 89 ++++++++++---------- .../test/resources/testErasureCodingConf.xml | 86 +++++++++---------- .../src/test/resources/test_ec_policies.xml | 2 +- .../hadoop/tools/util/TestDistCpUtils.java | 26 +++--- 11 files changed, 123 insertions(+), 114 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29a0fc2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 5667989..e99b099 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -389,7 +389,7 @@ public interface HdfsClientConfigKeys { String THREADPOOL_SIZE_KEY = PREFIX + "threadpool.size"; /** - * With default RS-6-3-64k erasure coding policy, each normal read could + * With default RS-6-3-1024k erasure coding policy, each normal read could * span 6 DNs, so this default value accommodates 3 read streams */ int THREADPOOL_SIZE_DEFAULT = 18; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29a0fc2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java index f0efe76..c079fea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java @@ -40,8 +40,8 @@ public final class SystemErasureCodingPolicies { // Private constructor, this is a utility class. private SystemErasureCodingPolicies() {} - // 64 KB - private static final int DEFAULT_CELLSIZE = 64 * 1024; + // 1 MB + private static final int DEFAULT_CELLSIZE = 1024 * 1024; public static final byte RS_6_3_POLICY_ID = 1; private static final ErasureCodingPolicy SYS_POLICY1 = http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29a0fc2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 7f60000..b836ff8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -567,7 +567,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY = "dfs.namenode.ec.system.default.policy"; public static final String DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT = - "RS-6-3-64k"; + "RS-6-3-1024k"; public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_KEY = "dfs.datanode.ec.reconstruction.stripedread.threads"; public static final int DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_DEFAULT = 20; public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.ec.reconstruction.stripedread.buffer.size"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29a0fc2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index aedc7e8..2a42b5c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -3029,7 +3029,7 @@ <property> <name>dfs.namenode.ec.system.default.policy</name> - <value>RS-6-3-64k</value> + <value>RS-6-3-1024k</value> <description>The default erasure coding policy name will be used on the path if no policy name is passed. </description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29a0fc2/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 786b512..a9c0f3d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -65,7 +65,7 @@ Architecture 2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work. - Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, six built-in policies are supported: `RS-3-2-64k`, `RS-6-3-64k`, `RS-10-4-64k`, `RS-LEGACY-6-3-64k`, `XOR-2-1-64k` and `REPLICATION`. + Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, six built-in policies are supported: `RS-3-2-1024k`, `RS-6-3-1024k`, `RS-10-4-1024k`, `RS-LEGACY-6-3-1024k`, `XOR-2-1-1024k` and `REPLICATION`. `REPLICATION` is a special policy. It can only be set on directory, to force the directory to adopt 3x replication scheme, instead of inheriting its ancestor's erasure coding policy. This policy makes it possible to interleave 3x replication scheme directory with erasure coding directory. @@ -115,14 +115,14 @@ Deployment By default, all built-in erasure coding policies are disabled. Typically, the cluster administrator will enable set of policies by including them in the `dfs.namenode.ec.policies.enabled` configuration based on the size of the cluster and the desired fault-tolerance properties. For instance, - for a cluster with 9 racks, a policy like `RS-10-4-64k` will not preserve rack-level fault-tolerance, and `RS-6-3-64k` or `RS-3-2-64k` might - be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as + for a cluster with 9 racks, a policy like `RS-10-4-1024k` will not preserve rack-level fault-tolerance, and `RS-6-3-1024k` or `RS-3-2-1024k` might + be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-1024k` would still be appropriate as long as there are at least 14 DataNodes in the cluster. A system default EC policy can be configured via 'dfs.namenode.ec.system.default.policy' configuration. With this configuration, the default EC policy will be used when no policy name is passed as an argument in the '-setPolicy' command. - By default, the 'dfs.namenode.ec.system.default.policy' is "RS-6-3-64k". + By default, the 'dfs.namenode.ec.system.default.policy' is "RS-6-3-1024k". The codec implementations for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: `io.erasurecode.codec.rs.rawcoders` for the default RS codec, http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29a0fc2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java index 7181968..60f4f56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java @@ -46,7 +46,7 @@ public class TestErasureCodingCLI extends CLITestHelper { super.setUp(); conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, - "RS-6-3-64k,RS-3-2-64k,XOR-2-1-64k"); + "RS-6-3-1024k,RS-3-2-1024k,XOR-2-1-1024k"); dfsCluster = new MiniDFSCluster.Builder(conf) .numDataNodes(NUM_OF_DATANODES).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29a0fc2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index 47cdf23..c0946f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -600,7 +600,17 @@ public class TestErasureCodingPolicies { fs.mkdirs(dirPath); fs.setErasureCodingPolicy(dirPath, ecPolicy.getName()); - final String ecPolicyName = "RS-10-4-64k"; + String ecPolicyName = null; + Collection<ErasureCodingPolicy> allPolicies = + fs.getAllErasureCodingPolicies(); + for (ErasureCodingPolicy policy : allPolicies) { + if (!ecPolicy.equals(policy)) { + ecPolicyName = policy.getName(); + break; + } + } + assertNotNull(ecPolicyName); + fs.createFile(filePath).build().close(); assertEquals(ecPolicy, fs.getErasureCodingPolicy(filePath)); fs.delete(filePath, true); @@ -704,7 +714,7 @@ public class TestErasureCodingPolicies { // Test add policy successfully newPolicy = - new ErasureCodingPolicy(toAddSchema, 1 * 1024 * 1024); + new ErasureCodingPolicy(toAddSchema, 4 * 1024 * 1024); policyArray = new ErasureCodingPolicy[]{newPolicy}; responses = fs.addErasureCodingPolicies(policyArray); assertEquals(1, responses.length); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29a0fc2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 42aeadf..4092e5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -17,45 +17,10 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION; -import static org.apache.hadoop.test.MetricsAsserts.getMetrics; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; - -import java.io.BufferedReader; -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStreamReader; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.EnumSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map.Entry; -import java.util.concurrent.BrokenBarrierException; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.FutureTask; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; - +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -80,8 +45,8 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils; import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica; +import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils; import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten; import org.apache.hadoop.hdfs.server.namenode.CacheManager; import org.apache.hadoop.hdfs.server.namenode.CachedBlock; @@ -115,10 +80,44 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.LinkedListMultimap; -import com.google.common.collect.Lists; +import java.io.BufferedReader; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; public class TestBlockManager { private DatanodeStorageInfo[] storages; @@ -1365,7 +1364,7 @@ public class TestBlockManager { .getNumDataUnits(); final int numParityBlocks = StripedFileTestUtil.getDefaultECPolicy() .getNumParityUnits(); - final long blockSize = 64 * 1024; + final long blockSize = 6 * 1024 * 1024; Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29a0fc2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index 1baf355..1613323 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -206,7 +206,7 @@ <description>setPolicy : set erasure coding policy on a directory to encode files</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command> </test-commands> <cleanup-commands> <command>-fs NAMENODE -rmdir /ecdir</command> @@ -214,7 +214,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output> + <expected-output>Set erasure coding policy RS-6-3-1024k on /ecdir</expected-output> </comparator> </comparators> </test> @@ -223,8 +223,8 @@ <description>setPolicy : set a policy twice</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command> </test-commands> <cleanup-commands> <command>-fs NAMENODE -rmdir /ecdir</command> @@ -232,7 +232,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output> + <expected-output>Set erasure coding policy RS-6-3-1024k on /ecdir</expected-output> </comparator> </comparators> </test> @@ -241,7 +241,7 @@ <description>setPolicy : set replication policy on a directory</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command> <command>-fs NAMENODE -mkdir /ecdir/replica</command> <ec-admin-command>-fs NAMENODE -setPolicy -replicate -path /ecdir/replica</ec-admin-command> <command>-fs NAMENODE -touchz /ecdir/replica/file</command> @@ -264,7 +264,7 @@ <description>unsetPolicy : unset policy and get</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command> </test-commands> @@ -283,8 +283,8 @@ <description>setPolicy : change different policy and get</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-3-2-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-3-2-1024k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command> </test-commands> <cleanup-commands> @@ -293,7 +293,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>RS-3-2-64k</expected-output> + <expected-output>RS-3-2-1024k</expected-output> </comparator> </comparators> </test> @@ -303,7 +303,7 @@ <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir/child</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir/child</ec-admin-command> <command>-fs NAMENODE -touchz /ecdir/child/ecfile</command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/child/ecfile</ec-admin-command> @@ -316,7 +316,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>RS-6-3-64k</expected-output> + <expected-output>RS-6-3-1024k</expected-output> </comparator> </comparators> </test> @@ -342,7 +342,7 @@ <description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command> </test-commands> <cleanup-commands> @@ -351,7 +351,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>RS-6-3-64k</expected-output> + <expected-output>RS-6-3-1024k</expected-output> </comparator> </comparators> </test> @@ -360,7 +360,7 @@ <description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command> <command>-fs NAMENODE -touchz /ecdir/ecfile</command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/ecfile</ec-admin-command> </test-commands> @@ -371,7 +371,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>RS-6-3-64k</expected-output> + <expected-output>RS-6-3-1024k</expected-output> </comparator> </comparators> </test> @@ -405,7 +405,7 @@ </comparator> <comparator> <type>SubstringComparator</type> - <expected-output>Add ErasureCodingPolicy RS-6-3-64k failed</expected-output> + <expected-output>Add ErasureCodingPolicy RS-6-3-1024k failed</expected-output> </comparator> </comparators> </test> @@ -413,14 +413,14 @@ <test> <description>enablePolicy : enable the erasure coding policy</description> <test-commands> - <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k</ec-admin-command> </test-commands> <cleanup-commands> </cleanup-commands> <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>Erasure coding policy RS-6-3-64k is enabled</expected-output> + <expected-output>Erasure coding policy RS-6-3-1024k is enabled</expected-output> </comparator> </comparators> </test> @@ -428,15 +428,15 @@ <test> <description>enablePolicy : enable the erasure coding policy twice</description> <test-commands> - <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k</ec-admin-command> - <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k</ec-admin-command> </test-commands> <cleanup-commands> </cleanup-commands> <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>Erasure coding policy RS-6-3-64k is enabled</expected-output> + <expected-output>Erasure coding policy RS-6-3-1024k is enabled</expected-output> </comparator> </comparators> </test> @@ -444,15 +444,15 @@ <test> <description>disablePolicy : disable the erasure coding policy</description> <test-commands> - <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-1024k</ec-admin-command> </test-commands> <cleanup-commands> - <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k</ec-admin-command> </cleanup-commands> <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>Erasure coding policy RS-6-3-64k is disabled</expected-output> + <expected-output>Erasure coding policy RS-6-3-1024k is disabled</expected-output> </comparator> </comparators> </test> @@ -460,16 +460,16 @@ <test> <description>disablePolicy : disable the erasure coding policy twice</description> <test-commands> - <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-64k</ec-admin-command> - <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-1024k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-1024k</ec-admin-command> </test-commands> <cleanup-commands> - <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k</ec-admin-command> </cleanup-commands> <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>Erasure coding policy RS-6-3-64k is disabled</expected-output> + <expected-output>Erasure coding policy RS-6-3-1024k is disabled</expected-output> </comparator> </comparators> </test> @@ -513,7 +513,7 @@ <description>setPolicy : illegal parameters - too many arguments case 1</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k /ecdir2</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-1024k /ecdir2</ec-admin-command> </test-commands> <cleanup-commands> <command>-fs NAMENODE -rmdir /ecdir</command> @@ -530,7 +530,7 @@ <description>setPolicy : illegal parameters - too many arguments case 2</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k -replicate /ecdir2</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-1024k -replicate /ecdir2</ec-admin-command> </test-commands> <cleanup-commands> <command>-fs NAMENODE -rmdir /ecdir</command> @@ -561,10 +561,10 @@ </test> <test> - <description>setPolicy : illegal parameters - RS-10-4-64k</description> + <description>setPolicy : illegal parameters - RS-10-4-1024k</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-10-4-64k -path /ecdir</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-10-4-1024k -path /ecdir</ec-admin-command> </test-commands> <cleanup-commands> <command>-fs NAMENODE -rmdir /ecdir</command> @@ -572,7 +572,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>Policy 'RS-10-4-64k' does not match any enabled erasure coding policies</expected-output> + <expected-output>Policy 'RS-10-4-1024k' does not match any enabled erasure coding policies</expected-output> </comparator> </comparators> </test> @@ -580,7 +580,7 @@ <test> <description>setPolicy : illegal parameters - no such file</description> <test-commands> - <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-1024k</ec-admin-command> </test-commands> <cleanup-commands> </cleanup-commands> @@ -610,7 +610,7 @@ <test> <description>setPolicy : illegal parameters - replicate and policy coexist</description> <test-commands> - <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k -replicate</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-1024k -replicate</ec-admin-command> </test-commands> <cleanup-commands> </cleanup-commands> @@ -652,7 +652,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>RS-6-3-64k</expected-output> + <expected-output>RS-6-3-1024k</expected-output> </comparator> </comparators> </test> @@ -751,7 +751,7 @@ <test> <description>enablePolicy : illegal parameters - policy is missing</description> <test-commands> - <ec-admin-command>-fs NAMENODE -enablePolicy RS-6-3-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -enablePolicy RS-6-3-1024k</ec-admin-command> </test-commands> <cleanup-commands> </cleanup-commands> @@ -766,7 +766,7 @@ <test> <description>enablePolicy : illegal parameters - too many parameters</description> <test-commands> - <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k RS-3-2-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k RS-3-2-1024k</ec-admin-command> </test-commands> <cleanup-commands> </cleanup-commands> @@ -781,7 +781,7 @@ <test> <description>disablePolicy : illegal parameters - policy is missing</description> <test-commands> - <ec-admin-command>-fs NAMENODE -disablePolicy RS-6-3-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -disablePolicy RS-6-3-1024k</ec-admin-command> </test-commands> <cleanup-commands> </cleanup-commands> @@ -796,7 +796,7 @@ <test> <description>disablePolicy : illegal parameters - too many parameters</description> <test-commands> - <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-64k RS-3-2-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-1024k RS-3-2-1024k</ec-admin-command> </test-commands> <cleanup-commands> </cleanup-commands> @@ -842,7 +842,7 @@ <description>count: file using absolute path with option -e to show erasurecoding policy of a directory</description> <test-commands> <command>-fs NAMENODE -mkdir /dir1</command> - <ec-admin-command>-fs NAMENODE -setPolicy -path /dir1 -policy RS-6-3-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -path /dir1 -policy RS-6-3-1024k</ec-admin-command> <command>-fs NAMENODE -touchz /dir1/file1</command> <command>-fs NAMENODE -touchz /dir1/file2</command> <command>-fs NAMENODE -count -e -v /dir1</command> @@ -883,7 +883,7 @@ <description>ls: file using absolute path and option -e to show erasure coding policy of a directory</description> <test-commands> <command>-fs NAMENODE -mkdir -p /ecdir</command> - <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-6-3-64k</ec-admin-command> + <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-6-3-1024k</ec-admin-command> <command>-fs NAMENODE -touchz /ecdir/file1</command> <command>-fs NAMENODE -touchz /ecdir/file2</command> <command>-fs NAMENODE -touchz /ecdir/file3</command> http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29a0fc2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/test_ec_policies.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/test_ec_policies.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/test_ec_policies.xml index b2416ac..6d5bb28 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/test_ec_policies.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/test_ec_policies.xml @@ -59,7 +59,7 @@ </policy> <policy> <schema>RSk6m3</schema> - <cellsize>65536</cellsize> + <cellsize>1048576</cellsize> </policy> </policies> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29a0fc2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java index c42e546..f626de4 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java @@ -18,16 +18,6 @@ package org.apache.hadoop.tools.util; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.EnumSet; -import java.util.Random; -import java.util.Stack; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -49,6 +39,16 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; +import java.io.OutputStream; +import java.util.EnumSet; +import java.util.Random; +import java.util.Stack; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + public class TestDistCpUtils { private static final Log LOG = LogFactory.getLog(TestDistCpUtils.class); @@ -61,7 +61,7 @@ public class TestDistCpUtils { @BeforeClass public static void create() throws IOException { config.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, - "XOR-2-1-64k"); + "XOR-2-1-1024k"); cluster = new MiniDFSCluster.Builder(config) .numDataNodes(2) .format(true) @@ -562,7 +562,7 @@ public class TestDistCpUtils { fs.mkdirs(srcECDir); fs.mkdirs(dstReplDir); String[] args = {"-setPolicy", "-path", "/tmp/srcECDir", - "-policy", "XOR-2-1-64k"}; + "-policy", "XOR-2-1-1024k"}; int res = ToolRunner.run(config, new ECAdmin(config), args); assertEquals("Setting EC policy should succeed!", 0, res); verifyReplFactorNotPreservedOnErasureCodedFile(srcECFile, true, @@ -577,7 +577,7 @@ public class TestDistCpUtils { fs.mkdirs(srcReplDir); fs.mkdirs(dstECDir); args = new String[]{"-setPolicy", "-path", "/tmp/dstECDir", - "-policy", "XOR-2-1-64k"}; + "-policy", "XOR-2-1-1024k"}; res = ToolRunner.run(config, new ECAdmin(config), args); assertEquals("Setting EC policy should succeed!", 0, res); verifyReplFactorNotPreservedOnErasureCodedFile(srcReplFile, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
