Revert "drop thread/dn counts in various tests to avoid OOM errors"
This reverts commit 48191274e03ce66e130706c092e91ed5e6732890. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8aa4c7d0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8aa4c7d0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8aa4c7d0 Branch: refs/heads/branch-2-jhung-test Commit: 8aa4c7d0a6060fd1c9af5168f854771b0c2a3e07 Parents: b2bd1c9 Author: Jonathan Hung <[email protected]> Authored: Fri Dec 14 16:00:00 2018 -0800 Committer: Jonathan Hung <[email protected]> Committed: Fri Dec 14 16:03:56 2018 -0800 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hdfs/TestDatanodeDeath.java | 2 +- .../java/org/apache/hadoop/hdfs/TestFileAppend2.java | 2 +- .../hadoop/hdfs/server/balancer/TestBalancer.java | 2 +- .../hadoop/hdfs/server/datanode/TestBatchIbr.java | 2 +- .../TestBlockPlacementPolicyRackFaultTolerant.java | 12 ++++++------ 5 files changed, 10 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aa4c7d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java index 36fe966..cd82b2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java @@ -52,7 +52,7 @@ public class TestDatanodeDeath { static final int blockSize = 8192; static final int numBlocks = 2; static final int fileSize = numBlocks * blockSize + 1; - static final int numDatanodes = 5; + static final int numDatanodes = 15; static final short replication = 3; final int numberOfFiles = 3; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aa4c7d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java index ff298a3..cd1b851 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java @@ -61,7 +61,7 @@ public class TestFileAppend2 { private byte[] fileContents = null; - final int numDatanodes = 5; + final int numDatanodes = 6; final int numberOfFiles = 50; final int numThreads = 10; final int numAppendsPerThread = 20; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aa4c7d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index f1fcfad..2979111 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -1966,7 +1966,7 @@ public class TestBalancer { initConf(conf); conf.setInt(DFSConfigKeys.DFS_BALANCER_DISPATCHERTHREADS_KEY, 30); - int numDNs = 5; + int numDNs = 20; long[] capacities = new long[numDNs]; String[] racks = new String[numDNs]; for(int i = 0; i < numDNs; i++) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aa4c7d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java index 94c8882..38c8a38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java @@ -59,7 +59,7 @@ public class TestBatchIbr { private static final int BLOCK_SIZE = 1024; private static final int MAX_BLOCK_NUM = 8; private static final int NUM_FILES = 1000; - private static final int NUM_THREADS = 64; + private static final int NUM_THREADS = 128; private static final ThreadLocalBuffer IO_BUF = new ThreadLocalBuffer(); private static final ThreadLocalBuffer VERIFY_BUF = new ThreadLocalBuffer(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aa4c7d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java index 1834014..f40c464 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java @@ -56,7 +56,7 @@ public class TestBlockPlacementPolicyRackFaultTolerant { Configuration conf = new HdfsConfiguration(); final ArrayList<String> rackList = new ArrayList<String>(); final ArrayList<String> hostList = new ArrayList<String>(); - for (int i = 0; i < 3; i++) { + for (int i = 0; i < 10; i++) { for (int j = 0; j < 2; j++) { rackList.add("/rack" + i); hostList.add("/host" + i + j); @@ -96,8 +96,8 @@ public class TestBlockPlacementPolicyRackFaultTolerant { private void doTestChooseTargetNormalCase() throws Exception { String clientMachine = "client.foo.com"; short[][] testSuite = { - {3, 2}//, {3, 7}, {3, 8}, {3, 10}, {9, 1}, {10, 1}, {10, 6}, {11, 6}, - //{11, 9} + {3, 2}, {3, 7}, {3, 8}, {3, 10}, {9, 1}, {10, 1}, {10, 6}, {11, 6}, + {11, 9} }; // Test 5 files int fileCount = 0; @@ -139,12 +139,12 @@ public class TestBlockPlacementPolicyRackFaultTolerant { // Create the file with client machine HdfsFileStatus fileStatus = namesystem.startFile(src, perm, clientMachine, clientMachine, EnumSet.of(CreateFlag.CREATE), true, - (short) 5, DEFAULT_BLOCK_SIZE, null, false); + (short) 20, DEFAULT_BLOCK_SIZE, null, false); //test chooseTarget for new file LocatedBlock locatedBlock = nameNodeRpc.addBlock(src, clientMachine, null, null, fileStatus.getFileId(), null, null); - doTestLocatedBlock(5, locatedBlock); + doTestLocatedBlock(20, locatedBlock); DatanodeInfo[] locs = locatedBlock.getLocations(); String[] storageIDs = locatedBlock.getStorageIDs(); @@ -156,7 +156,7 @@ public class TestBlockPlacementPolicyRackFaultTolerant { String[] partStorageIDs = new String[i]; System.arraycopy(locs, 0, partLocs, 0, i); System.arraycopy(storageIDs, 0, partStorageIDs, 0, i); - for (int j = 1; j < 5 - i; j++) { + for (int j = 1; j < 20 - i; j++) { LocatedBlock additionalLocatedBlock = nameNodeRpc.getAdditionalDatanode(src, fileStatus.getFileId(), locatedBlock.getBlock(), partLocs, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
