Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java Tue Aug 19 23:49:39 2014 @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.protocol.H import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.test.PathUtils; @@ -101,6 +102,7 @@ public class TestReplicationPolicyConsid } } + private final double EPSILON = 0.0001; /** * Tests that chooseTarget with considerLoad set to true correctly calculates * load with decommissioned nodes. @@ -109,14 +111,6 @@ public class TestReplicationPolicyConsid public void testChooseTargetWithDecomNodes() throws IOException { namenode.getNamesystem().writeLock(); try { - // Decommission DNs so BlockPlacementPolicyDefault.isGoodTarget() - // returns false - for (int i = 0; i < 3; i++) { - DatanodeInfo d = dnManager.getDatanodeByXferAddr( - dnrList.get(i).getIpAddr(), - dnrList.get(i).getXferPort()); - d.setDecommissioned(); - } String blockPoolId = namenode.getNamesystem().getBlockPoolId(); dnManager.handleHeartbeat(dnrList.get(3), BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[3]), @@ -133,6 +127,20 @@ public class TestReplicationPolicyConsid blockPoolId, dataNodes[5].getCacheCapacity(), dataNodes[5].getCacheRemaining(), 4, 0, 0); + // value in the above heartbeats + final int load = 2 + 4 + 4; + + FSNamesystem fsn = namenode.getNamesystem(); + assertEquals((double)load/6, fsn.getInServiceXceiverAverage(), EPSILON); + + // Decommission DNs so BlockPlacementPolicyDefault.isGoodTarget() + // returns false + for (int i = 0; i < 3; i++) { + DatanodeDescriptor d = dnManager.getDatanode(dnrList.get(i)); + dnManager.startDecommission(d); + d.setDecommissioned(); + } + assertEquals((double)load/3, fsn.getInServiceXceiverAverage(), EPSILON); // Call chooseTarget() DatanodeStorageInfo[] targets = namenode.getNamesystem().getBlockManager()
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java Tue Aug 19 23:49:39 2014 @@ -47,11 +47,13 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; + public class TestReplicationPolicyWithNodeGroup { private static final int BLOCK_SIZE = 1024; private static final int NUM_OF_DATANODES = 8; private static final int NUM_OF_DATANODES_BOUNDARY = 6; private static final int NUM_OF_DATANODES_MORE_TARGETS = 12; + private static final int NUM_OF_DATANODES_FOR_DEPENDENCIES = 6; private final Configuration CONF = new HdfsConfiguration(); private NetworkTopology cluster; private NameNode namenode; @@ -113,7 +115,33 @@ public class TestReplicationPolicyWithNo private final static DatanodeDescriptor NODE = new DatanodeDescriptor(DFSTestUtil.getDatanodeDescriptor("9.9.9.9", "/d2/r4/n7")); - + + private static final DatanodeStorageInfo[] storagesForDependencies; + private static final DatanodeDescriptor[] dataNodesForDependencies; + static { + final String[] racksForDependencies = { + "/d1/r1/n1", + "/d1/r1/n1", + "/d1/r1/n2", + "/d1/r1/n2", + "/d1/r1/n3", + "/d1/r1/n4" + }; + final String[] hostNamesForDependencies = { + "h1", + "h2", + "h3", + "h4", + "h5", + "h6" + }; + + storagesForDependencies = DFSTestUtil.createDatanodeStorageInfos( + racksForDependencies, hostNamesForDependencies); + dataNodesForDependencies = DFSTestUtil.toDatanodeDescriptor(storagesForDependencies); + + }; + @Before public void setUp() throws Exception { FileSystem.setDefaultUri(CONF, "hdfs://localhost:0"); @@ -563,51 +591,50 @@ public class TestReplicationPolicyWithNo */ @Test public void testChooseReplicaToDelete() throws Exception { - List<DatanodeDescriptor> replicaNodeList = - new ArrayList<DatanodeDescriptor>(); - final Map<String, List<DatanodeDescriptor>> rackMap = - new HashMap<String, List<DatanodeDescriptor>>(); + List<DatanodeStorageInfo> replicaList = new ArrayList<DatanodeStorageInfo>(); + final Map<String, List<DatanodeStorageInfo>> rackMap + = new HashMap<String, List<DatanodeStorageInfo>>(); dataNodes[0].setRemaining(4*1024*1024); - replicaNodeList.add(dataNodes[0]); + replicaList.add(storages[0]); dataNodes[1].setRemaining(3*1024*1024); - replicaNodeList.add(dataNodes[1]); + replicaList.add(storages[1]); dataNodes[2].setRemaining(2*1024*1024); - replicaNodeList.add(dataNodes[2]); + replicaList.add(storages[2]); dataNodes[5].setRemaining(1*1024*1024); - replicaNodeList.add(dataNodes[5]); + replicaList.add(storages[5]); - List<DatanodeDescriptor> first = new ArrayList<DatanodeDescriptor>(); - List<DatanodeDescriptor> second = new ArrayList<DatanodeDescriptor>(); + List<DatanodeStorageInfo> first = new ArrayList<DatanodeStorageInfo>(); + List<DatanodeStorageInfo> second = new ArrayList<DatanodeStorageInfo>(); replicator.splitNodesWithRack( - replicaNodeList, rackMap, first, second); + replicaList, rackMap, first, second); assertEquals(3, first.size()); assertEquals(1, second.size()); - DatanodeDescriptor chosenNode = replicator.chooseReplicaToDelete( + DatanodeStorageInfo chosen = replicator.chooseReplicaToDelete( null, null, (short)3, first, second); // Within first set {dataNodes[0], dataNodes[1], dataNodes[2]}, // dataNodes[0] and dataNodes[1] are in the same nodegroup, // but dataNodes[1] is chosen as less free space - assertEquals(chosenNode, dataNodes[1]); + assertEquals(chosen, storages[1]); - replicator.adjustSetsWithChosenReplica(rackMap, first, second, chosenNode); + replicator.adjustSetsWithChosenReplica(rackMap, first, second, chosen); assertEquals(2, first.size()); assertEquals(1, second.size()); // Within first set {dataNodes[0], dataNodes[2]}, dataNodes[2] is chosen // as less free space - chosenNode = replicator.chooseReplicaToDelete( + chosen = replicator.chooseReplicaToDelete( null, null, (short)2, first, second); - assertEquals(chosenNode, dataNodes[2]); + assertEquals(chosen, storages[2]); - replicator.adjustSetsWithChosenReplica(rackMap, first, second, chosenNode); + replicator.adjustSetsWithChosenReplica(rackMap, first, second, chosen); assertEquals(0, first.size()); assertEquals(2, second.size()); // Within second set, dataNodes[5] with less free space - chosenNode = replicator.chooseReplicaToDelete( + chosen = replicator.chooseReplicaToDelete( null, null, (short)1, first, second); - assertEquals(chosenNode, dataNodes[5]); + assertEquals(chosen, storages[5]); } /** @@ -720,5 +747,63 @@ public class TestReplicationPolicyWithNo assertEquals(targets.length, 6); } - + @Test + public void testChooseTargetWithDependencies() throws Exception { + for(int i=0; i<NUM_OF_DATANODES; i++) { + cluster.remove(dataNodes[i]); + } + + for(int i=0; i<NUM_OF_DATANODES_MORE_TARGETS; i++) { + DatanodeDescriptor node = dataNodesInMoreTargetsCase[i]; + if (cluster.contains(node)) { + cluster.remove(node); + } + } + + Host2NodesMap host2DatanodeMap = namenode.getNamesystem() + .getBlockManager() + .getDatanodeManager().getHost2DatanodeMap(); + for(int i=0; i<NUM_OF_DATANODES_FOR_DEPENDENCIES; i++) { + cluster.add(dataNodesForDependencies[i]); + host2DatanodeMap.add(dataNodesForDependencies[i]); + } + + //add dependencies (node1 <-> node2, and node3<->node4) + dataNodesForDependencies[1].addDependentHostName( + dataNodesForDependencies[2].getHostName()); + dataNodesForDependencies[2].addDependentHostName( + dataNodesForDependencies[1].getHostName()); + dataNodesForDependencies[3].addDependentHostName( + dataNodesForDependencies[4].getHostName()); + dataNodesForDependencies[4].addDependentHostName( + dataNodesForDependencies[3].getHostName()); + + //Update heartbeat + for(int i=0; i<NUM_OF_DATANODES_FOR_DEPENDENCIES; i++) { + updateHeartbeatWithUsage(dataNodesForDependencies[i], + 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, + 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0); + } + + List<DatanodeStorageInfo> chosenNodes = new ArrayList<DatanodeStorageInfo>(); + + DatanodeStorageInfo[] targets; + Set<Node> excludedNodes = new HashSet<Node>(); + excludedNodes.add(dataNodesForDependencies[5]); + + //try to select three targets as there are three node groups + targets = chooseTarget(3, dataNodesForDependencies[1], chosenNodes, excludedNodes); + + //Even there are three node groups, verify that + //only two targets are selected due to dependencies + assertEquals(targets.length, 2); + assertEquals(targets[0], storagesForDependencies[1]); + assertTrue(targets[1].equals(storagesForDependencies[3]) || targets[1].equals(storagesForDependencies[4])); + + //verify that all data nodes are in the excluded list + assertEquals(excludedNodes.size(), NUM_OF_DATANODES_FOR_DEPENDENCIES); + for(int i=0; i<NUM_OF_DATANODES_FOR_DEPENDENCIES; i++) { + assertTrue(excludedNodes.contains(dataNodesForDependencies[i])); + } + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java Tue Aug 19 23:49:39 2014 @@ -17,37 +17,11 @@ */ package org.apache.hadoop.hdfs.server.common; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.io.StringReader; -import java.net.InetSocketAddress; -import java.util.ArrayList; - -import javax.servlet.ServletContext; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.jsp.JspWriter; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer; -import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; -import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.web.resources.DoAsParam; import org.apache.hadoop.hdfs.web.resources.UserParam; import org.apache.hadoop.io.DataInputBuffer; @@ -56,6 +30,8 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; +import org.apache.hadoop.security.authorize.ProxyServers; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -63,20 +39,20 @@ import org.apache.hadoop.security.token. import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import com.google.common.base.Strings; +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.net.InetSocketAddress; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TestJspHelper { private final Configuration conf = new HdfsConfiguration(); - private String jspWriterOutput = ""; // allow user with TGT to run tests @BeforeClass @@ -158,25 +134,6 @@ public class TestJspHelper { .next(); Assert.assertEquals(expected, tokenInUgi.getService().toString()); } - - - @Test - public void testDelegationTokenUrlParam() { - conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - UserGroupInformation.setConfiguration(conf); - String tokenString = "xyzabc"; - String delegationTokenParam = JspHelper - .getDelegationTokenUrlParam(tokenString); - //Security is enabled - Assert.assertEquals(JspHelper.SET_DELEGATION + "xyzabc", - delegationTokenParam); - conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "simple"); - UserGroupInformation.setConfiguration(conf); - delegationTokenParam = JspHelper - .getDelegationTokenUrlParam(tokenString); - //Empty string must be returned because security is disabled. - Assert.assertEquals("", delegationTokenParam); - } @Test public void testGetUgiFromToken() throws IOException { @@ -328,8 +285,10 @@ public class TestJspHelper { String user = "TheNurse"; conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - conf.set(ProxyUsers.CONF_HADOOP_PROXYUSER+realUser+".groups", "*"); - conf.set(ProxyUsers.CONF_HADOOP_PROXYUSER+realUser+".hosts", "*"); + conf.set(DefaultImpersonationProvider.getTestProvider(). + getProxySuperuserGroupConfKey(realUser), "*"); + conf.set(DefaultImpersonationProvider.getTestProvider(). + getProxySuperuserIpConfKey(realUser), "*"); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); UserGroupInformation.setConfiguration(conf); UserGroupInformation ugi; @@ -403,32 +362,6 @@ public class TestJspHelper { } } - @Test - public void testPrintGotoFormWritesValidXML() throws IOException, - ParserConfigurationException, SAXException { - JspWriter mockJspWriter = mock(JspWriter.class); - ArgumentCaptor<String> arg = ArgumentCaptor.forClass(String.class); - doAnswer(new Answer<Object>() { - @Override - public Object answer(InvocationOnMock invok) { - Object[] args = invok.getArguments(); - jspWriterOutput += (String) args[0]; - return null; - } - }).when(mockJspWriter).print(arg.capture()); - - jspWriterOutput = ""; - - JspHelper.printGotoForm(mockJspWriter, 424242, "a token string", - "foobar/file", "0.0.0.0"); - - DocumentBuilder parser = - DocumentBuilderFactory.newInstance().newDocumentBuilder(); - InputSource is = new InputSource(); - is.setCharacterStream(new StringReader(jspWriterOutput)); - parser.parse(is); - } - private HttpServletRequest getMockRequest(String remoteUser, String user, String doAs) { HttpServletRequest request = mock(HttpServletRequest.class); when(request.getParameter(UserParam.NAME)).thenReturn(user); @@ -464,146 +397,6 @@ public class TestJspHelper { } @Test - public void testSortNodeByFields() throws Exception { - DatanodeID dnId1 = new DatanodeID("127.0.0.1", "localhost1", "datanode1", - 1234, 2345, 3456, 4567); - DatanodeID dnId2 = new DatanodeID("127.0.0.2", "localhost2", "datanode2", - 1235, 2346, 3457, 4568); - - // Setup DatanodeDescriptors with one storage each. - DatanodeDescriptor dnDesc1 = new DatanodeDescriptor(dnId1, "rack1"); - DatanodeDescriptor dnDesc2 = new DatanodeDescriptor(dnId2, "rack2"); - - // Update the DatanodeDescriptors with their attached storages. - BlockManagerTestUtil.updateStorage(dnDesc1, new DatanodeStorage("dnStorage1")); - BlockManagerTestUtil.updateStorage(dnDesc2, new DatanodeStorage("dnStorage2")); - - DatanodeStorage dns1 = new DatanodeStorage("dnStorage1"); - DatanodeStorage dns2 = new DatanodeStorage("dnStorage2"); - - StorageReport[] report1 = new StorageReport[] { - new StorageReport(dns1, false, 1024, 100, 924, 100) - }; - StorageReport[] report2 = new StorageReport[] { - new StorageReport(dns2, false, 2500, 200, 1848, 200) - }; - dnDesc1.updateHeartbeat(report1, 5l, 3l, 10, 2); - dnDesc2.updateHeartbeat(report2, 10l, 2l, 20, 1); - - ArrayList<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>(); - live.add(dnDesc1); - live.add(dnDesc2); - - // Test sorting by failed volumes - JspHelper.sortNodeList(live, "volfails", "ASC"); - Assert.assertEquals(dnDesc2, live.get(0)); - Assert.assertEquals(dnDesc1, live.get(1)); - JspHelper.sortNodeList(live, "volfails", "DSC"); - Assert.assertEquals(dnDesc1, live.get(0)); - Assert.assertEquals(dnDesc2, live.get(1)); - - // Test sorting by Blockpool used - JspHelper.sortNodeList(live, "bpused", "ASC"); - Assert.assertEquals(dnDesc1, live.get(0)); - Assert.assertEquals(dnDesc2, live.get(1)); - JspHelper.sortNodeList(live, "bpused", "DSC"); - Assert.assertEquals(dnDesc2, live.get(0)); - Assert.assertEquals(dnDesc1, live.get(1)); - - // Test sorting by Percentage Blockpool used - JspHelper.sortNodeList(live, "pcbpused", "ASC"); - Assert.assertEquals(dnDesc2, live.get(0)); - Assert.assertEquals(dnDesc1, live.get(1)); - JspHelper.sortNodeList(live, "pcbpused", "DSC"); - Assert.assertEquals(dnDesc1, live.get(0)); - Assert.assertEquals(dnDesc2, live.get(1)); - - //unexisted field comparition is d1.getHostName().compareTo(d2.getHostName()); - JspHelper.sortNodeList(live, "unexists", "ASC"); - Assert.assertEquals(dnDesc1, live.get(0)); - Assert.assertEquals(dnDesc2, live.get(1)); - - JspHelper.sortNodeList(live, "unexists", "DSC"); - Assert.assertEquals(dnDesc2, live.get(0)); - Assert.assertEquals(dnDesc1, live.get(1)); - - // test sorting by capacity - JspHelper.sortNodeList(live, "capacity", "ASC"); - Assert.assertEquals(dnDesc1, live.get(0)); - Assert.assertEquals(dnDesc2, live.get(1)); - - JspHelper.sortNodeList(live, "capacity", "DSC"); - Assert.assertEquals(dnDesc2, live.get(0)); - Assert.assertEquals(dnDesc1, live.get(1)); - - // test sorting by used - JspHelper.sortNodeList(live, "used", "ASC"); - Assert.assertEquals(dnDesc1, live.get(0)); - Assert.assertEquals(dnDesc2, live.get(1)); - - JspHelper.sortNodeList(live, "used", "DSC"); - Assert.assertEquals(dnDesc2, live.get(0)); - Assert.assertEquals(dnDesc1, live.get(1)); - - // test sorting by nondfsused - JspHelper.sortNodeList(live, "nondfsused", "ASC"); - Assert.assertEquals(dnDesc1, live.get(0)); - Assert.assertEquals(dnDesc2, live.get(1)); - - JspHelper.sortNodeList(live, "nondfsused", "DSC"); - Assert.assertEquals(dnDesc2, live.get(0)); - Assert.assertEquals(dnDesc1, live.get(1)); - - // test sorting by remaining - JspHelper.sortNodeList(live, "remaining", "ASC"); - Assert.assertEquals(dnDesc1, live.get(0)); - Assert.assertEquals(dnDesc2, live.get(1)); - - JspHelper.sortNodeList(live, "remaining", "DSC"); - Assert.assertEquals(dnDesc2, live.get(0)); - Assert.assertEquals(dnDesc1, live.get(1)); - } - - @Test - public void testPrintMethods() throws IOException { - JspWriter out = mock(JspWriter.class); - HttpServletRequest req = mock(HttpServletRequest.class); - - final StringBuffer buffer = new StringBuffer(); - - ArgumentCaptor<String> arg = ArgumentCaptor.forClass(String.class); - doAnswer(new Answer<Object>() { - @Override - public Object answer(InvocationOnMock invok) { - Object[] args = invok.getArguments(); - buffer.append((String)args[0]); - return null; - } - }).when(out).print(arg.capture()); - - - JspHelper.createTitle(out, req, "testfile.txt"); - Mockito.verify(out, Mockito.times(1)).print(Mockito.anyString()); - - JspHelper.addTableHeader(out); - Mockito.verify(out, Mockito.times(1 + 2)).print(Mockito.anyString()); - - JspHelper.addTableRow(out, new String[] {" row11", "row12 "}); - Mockito.verify(out, Mockito.times(1 + 2 + 4)).print(Mockito.anyString()); - - JspHelper.addTableRow(out, new String[] {" row11", "row12 "}, 3); - Mockito.verify(out, Mockito.times(1 + 2 + 4 + 4)).print(Mockito.anyString()); - - JspHelper.addTableRow(out, new String[] {" row21", "row22"}); - Mockito.verify(out, Mockito.times(1 + 2 + 4 + 4 + 4)).print(Mockito.anyString()); - - JspHelper.addTableFooter(out); - Mockito.verify(out, Mockito.times(1 + 2 + 4 + 4 + 4 + 1)).print(Mockito.anyString()); - - assertFalse(Strings.isNullOrEmpty(buffer.toString())); - } - - @Test public void testReadWriteReplicaState() { try { DataOutputBuffer out = new DataOutputBuffer(); @@ -622,21 +415,6 @@ public class TestJspHelper { fail("testReadWrite ex error ReplicaState"); } } - - @Test - public void testAuthority(){ - DatanodeID dnWithIp = new DatanodeID("127.0.0.1", "hostName", null, - 50020, 50075, 50076, 50010); - assertNotNull(JspHelper.Url.authority("http", dnWithIp)); - - DatanodeID dnWithNullIp = new DatanodeID(null, "hostName", null, - 50020, 50075, 50076, 50010); - assertNotNull(JspHelper.Url.authority("http", dnWithNullIp)); - - DatanodeID dnWithEmptyIp = new DatanodeID("", "hostName", null, - 50020, 50075, 50076, 50010); - assertNotNull(JspHelper.Url.authority("http", dnWithEmptyIp)); - } private static String clientAddr = "1.1.1.1"; private static String chainedClientAddr = clientAddr+", 2.2.2.2"; @@ -675,7 +453,7 @@ public class TestJspHelper { when(req.getRemoteAddr()).thenReturn(proxyAddr); when(req.getHeader("X-Forwarded-For")).thenReturn(clientAddr); if (trusted) { - conf.set(ProxyUsers.CONF_HADOOP_PROXYSERVERS, proxyAddr); + conf.set(ProxyServers.CONF_HADOOP_PROXYSERVERS, proxyAddr); } } ProxyUsers.refreshSuperUserGroupsConfiguration(conf); Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java Tue Aug 19 23:49:39 2014 @@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -171,9 +172,6 @@ public abstract class BlockReportTestBas * Utility routine to send block reports to the NN, either in a single call * or reporting one storage per call. * - * @param dnR - * @param poolId - * @param reports * @throws IOException */ protected abstract void sendBlockReports(DatanodeRegistration dnR, String poolId, @@ -327,7 +325,7 @@ public abstract class BlockReportTestBas public void blockReport_03() throws IOException { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); - ArrayList<Block> blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath); + writeFile(METHOD_NAME, FILE_SIZE, filePath); // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N0); @@ -366,7 +364,7 @@ public abstract class BlockReportTestBas // Create a bogus new block which will not be present on the namenode. ExtendedBlock b = new ExtendedBlock( poolId, rand.nextLong(), 1024L, rand.nextLong()); - dn.getFSDataset().createRbw(b); + dn.getFSDataset().createRbw(StorageType.DEFAULT, b); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); @@ -413,6 +411,7 @@ public abstract class BlockReportTestBas * The second datanode is started in the cluster. * As soon as the replication process is completed test finds a block from * the second DN and sets its GS to be < of original one. + * this is the markBlockAsCorrupt case 3 so we expect one pending deletion * Block report is forced and the check for # of currupted blocks is performed. * Another block is chosen and its length is set to a lesser than original. * A check for another corrupted block is performed after yet another @@ -439,20 +438,20 @@ public abstract class BlockReportTestBas printStats(); assertThat("Wrong number of corrupt blocks", - cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L)); + cluster.getNamesystem().getCorruptReplicaBlocks(), is(0L)); assertThat("Wrong number of PendingDeletion blocks", - cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); + cluster.getNamesystem().getPendingDeletionBlocks(), is(1L)); assertThat("Wrong number of PendingReplication blocks", cluster.getNamesystem().getPendingReplicationBlocks(), is(0L)); - reports = getBlockReports(dn, poolId, true, true); + reports = getBlockReports(dn, poolId, false, true); sendBlockReports(dnR, poolId, reports); printStats(); assertThat("Wrong number of corrupt blocks", - cluster.getNamesystem().getCorruptReplicaBlocks(), is(2L)); + cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L)); assertThat("Wrong number of PendingDeletion blocks", - cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); + cluster.getNamesystem().getPendingDeletionBlocks(), is(1L)); assertThat("Wrong number of PendingReplication blocks", cluster.getNamesystem().getPendingReplicationBlocks(), is(0L)); Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java Tue Aug 19 23:49:39 2014 @@ -116,7 +116,8 @@ public class DataNodeTestUtils { public static void runBlockScannerForBlock(DataNode dn, ExtendedBlock b) { BlockPoolSliceScanner bpScanner = getBlockPoolScanner(dn, b); - bpScanner.verifyBlock(b); + bpScanner.verifyBlock(new ExtendedBlock(b.getBlockPoolId(), + new BlockPoolSliceScanner.BlockScanInfo(b.getLocalBlock()))); } private static BlockPoolSliceScanner getBlockPoolScanner(DataNode dn, Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Tue Aug 19 23:49:39 2014 @@ -18,10 +18,12 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.File; +import java.io.FileDescriptor; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -743,14 +745,14 @@ public class SimulatedFSDataset implemen } @Override // FsDatasetSpi - public synchronized ReplicaInPipelineInterface createRbw(ExtendedBlock b) - throws IOException { - return createTemporary(b); + public synchronized ReplicaInPipelineInterface createRbw( + StorageType storageType, ExtendedBlock b) throws IOException { + return createTemporary(storageType, b); } @Override // FsDatasetSpi - public synchronized ReplicaInPipelineInterface createTemporary(ExtendedBlock b) - throws IOException { + public synchronized ReplicaInPipelineInterface createTemporary( + StorageType storageType, ExtendedBlock b) throws IOException { if (isValidBlock(b)) { throw new ReplicaAlreadyExistsException("Block " + b + " is valid, and cannot be written to."); @@ -833,8 +835,8 @@ public class SimulatedFSDataset implemen /** * An input stream of size l with repeated bytes - * @param l - * @param iRepeatedData + * @param l size of the stream + * @param iRepeatedData byte that is repeated in the stream */ SimulatedInputStream(long l, byte iRepeatedData) { length = l; @@ -843,17 +845,14 @@ public class SimulatedFSDataset implemen /** * An input stream of of the supplied data - * - * @param iData + * @param iData data to construct the stream */ SimulatedInputStream(byte[] iData) { data = iData; length = data.length; - } /** - * * @return the lenght of the input stream */ long getLength() { @@ -1085,6 +1084,11 @@ public class SimulatedFSDataset implemen } @Override + public void addVolumes(Collection<StorageLocation> volumes) { + throw new UnsupportedOperationException(); + } + + @Override public DatanodeStorage getStorage(final String storageUuid) { return storageUuid.equals(storage.getStorageUuid()) ? storage.dnStorage : @@ -1115,5 +1119,11 @@ public class SimulatedFSDataset implemen public FsVolumeSpi getVolume(ExtendedBlock b) { throw new UnsupportedOperationException(); } + + @Override + public void submitBackgroundSyncFileRangeRequest(ExtendedBlock block, + FileDescriptor fd, long offset, long nbytes, int flags) { + throw new UnsupportedOperationException(); + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java Tue Aug 19 23:49:39 2014 @@ -325,13 +325,14 @@ public class TestBPOfferService { } }).when(mockDn).initBlockPool(Mockito.any(BPOfferService.class)); BPOfferService bpos = setupBPOSForNNs(mockDn, mockNN1, mockNN2); + List<BPServiceActor> actors = bpos.getBPServiceActors(); + assertEquals(2, actors.size()); bpos.start(); try { waitForInitialization(bpos); - List<BPServiceActor> actors = bpos.getBPServiceActors(); - assertEquals(1, actors.size()); - BPServiceActor actor = actors.get(0); - waitForBlockReport(actor.getNameNodeProxy()); + // even if one of the actor initialization fails, the other one will be + // finish block report. + waitForBlockReport(mockNN1, mockNN2); } finally { bpos.stop(); } @@ -342,7 +343,14 @@ public class TestBPOfferService { GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { - return bpos.countNameNodes() == 1; + List<BPServiceActor> actors = bpos.getBPServiceActors(); + int failedcount = 0; + for (BPServiceActor actor : actors) { + if (!actor.isAlive()) { + failedcount++; + } + } + return failedcount == 1; } }, 100, 10000); } @@ -400,10 +408,36 @@ public class TestBPOfferService { } }, 500, 10000); } - + + private void waitForBlockReport( + final DatanodeProtocolClientSideTranslatorPB mockNN1, + final DatanodeProtocolClientSideTranslatorPB mockNN2) + throws Exception { + GenericTestUtils.waitFor(new Supplier<Boolean>() { + @Override + public Boolean get() { + return get(mockNN1) || get(mockNN2); + } + + private Boolean get(DatanodeProtocolClientSideTranslatorPB mockNN) { + try { + Mockito.verify(mockNN).blockReport( + Mockito.<DatanodeRegistration>anyObject(), + Mockito.eq(FAKE_BPID), + Mockito.<StorageBlockReport[]>anyObject()); + return true; + } catch (Throwable t) { + LOG.info("waiting on block report: " + t.getMessage()); + return false; + } + } + }, 500, 10000); + } + private ReceivedDeletedBlockInfo[] waitForBlockReceived( - ExtendedBlock fakeBlock, - DatanodeProtocolClientSideTranslatorPB mockNN) throws Exception { + final ExtendedBlock fakeBlock, + final DatanodeProtocolClientSideTranslatorPB mockNN) throws Exception { + final String fakeBlockPoolId = fakeBlock.getBlockPoolId(); final ArgumentCaptor<StorageReceivedDeletedBlocks[]> captor = ArgumentCaptor.forClass(StorageReceivedDeletedBlocks[].class); GenericTestUtils.waitFor(new Supplier<Boolean>() { @@ -411,9 +445,9 @@ public class TestBPOfferService { @Override public Boolean get() { try { - Mockito.verify(mockNN1).blockReceivedAndDeleted( + Mockito.verify(mockNN).blockReceivedAndDeleted( Mockito.<DatanodeRegistration>anyObject(), - Mockito.eq(FAKE_BPID), + Mockito.eq(fakeBlockPoolId), captor.capture()); return true; } catch (Throwable t) { Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java Tue Aug 19 23:49:39 2014 @@ -57,6 +57,7 @@ import org.apache.hadoop.hdfs.Distribute import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -531,7 +532,7 @@ public class TestBlockRecovery { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); } - dn.data.createRbw(block); + dn.data.createRbw(StorageType.DEFAULT, block); try { dn.syncBlock(rBlock, initBlockRecords(dn)); fail("Sync should fail"); @@ -554,7 +555,8 @@ public class TestBlockRecovery { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); } - ReplicaInPipelineInterface replicaInfo = dn.data.createRbw(block); + ReplicaInPipelineInterface replicaInfo = dn.data.createRbw( + StorageType.DEFAULT, block); ReplicaOutputStreams streams = null; try { streams = replicaInfo.createStreams(true, @@ -588,7 +590,6 @@ public class TestBlockRecovery { Configuration conf = new HdfsConfiguration(); conf.set(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY, "1000"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(MiniDFSNNTopology.simpleSingleNN(8020, 50070)) .numDataNodes(1).build(); try { cluster.waitClusterUp(); Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java Tue Aug 19 23:49:39 2014 @@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; @@ -264,14 +265,17 @@ public class TestBlockReplacement { sock.setKeepAlive(true); // sendRequest DataOutputStream out = new DataOutputStream(sock.getOutputStream()); - new Sender(out).replaceBlock(block, BlockTokenSecretManager.DUMMY_TOKEN, + new Sender(out).replaceBlock(block, StorageType.DEFAULT, + BlockTokenSecretManager.DUMMY_TOKEN, source.getDatanodeUuid(), sourceProxy); out.flush(); // receiveResponse DataInputStream reply = new DataInputStream(sock.getInputStream()); - BlockOpResponseProto proto = - BlockOpResponseProto.parseDelimitedFrom(reply); + BlockOpResponseProto proto = BlockOpResponseProto.parseDelimitedFrom(reply); + while (proto.getStatus() == Status.IN_PROGRESS) { + proto = BlockOpResponseProto.parseDelimitedFrom(reply); + } return proto.getStatus() == Status.SUCCESS; } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java Tue Aug 19 23:49:39 2014 @@ -19,8 +19,10 @@ package org.apache.hadoop.hdfs.server.datanode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.net.InetSocketAddress; @@ -30,11 +32,14 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.util.StringUtils; import org.junit.Assert; @@ -189,7 +194,7 @@ public class TestDataNodeMultipleRegistr } @Test - public void testClusterIdMismatch() throws IOException { + public void testClusterIdMismatch() throws Exception { MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)) .build(); @@ -203,6 +208,7 @@ public class TestDataNodeMultipleRegistr // add another namenode cluster.addNameNode(conf, 9938); + Thread.sleep(500);// lets wait for the registration to happen bposs = dn.getAllBpOs(); LOG.info("dn bpos len (should be 3):" + bposs.length); Assert.assertEquals("should've registered with three namenodes", bposs.length,3); @@ -212,16 +218,90 @@ public class TestDataNodeMultipleRegistr cluster.addNameNode(conf, 9948); NameNode nn4 = cluster.getNameNode(3); assertNotNull("cannot create nn4", nn4); - + + Thread.sleep(500);// lets wait for the registration to happen bposs = dn.getAllBpOs(); LOG.info("dn bpos len (still should be 3):" + bposs.length); Assert.assertEquals("should've registered with three namenodes", 3, bposs.length); } finally { + cluster.shutdown(); + } + } + + @Test(timeout = 20000) + public void testClusterIdMismatchAtStartupWithHA() throws Exception { + MiniDFSNNTopology top = new MiniDFSNNTopology() + .addNameservice(new MiniDFSNNTopology.NSConf("ns1") + .addNN(new MiniDFSNNTopology.NNConf("nn0")) + .addNN(new MiniDFSNNTopology.NNConf("nn1"))) + .addNameservice(new MiniDFSNNTopology.NSConf("ns2") + .addNN(new MiniDFSNNTopology.NNConf("nn2").setClusterId("bad-cid")) + .addNN(new MiniDFSNNTopology.NNConf("nn3").setClusterId("bad-cid"))); + + top.setFederation(true); + + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(top) + .numDataNodes(0).build(); + + try { + cluster.startDataNodes(conf, 1, true, null, null); + // let the initialization be complete + Thread.sleep(10000); + DataNode dn = cluster.getDataNodes().get(0); + assertTrue("Datanode should be running", dn.isDatanodeUp()); + assertEquals("Only one BPOfferService should be running", 1, + dn.getAllBpOs().length); + } finally { cluster.shutdown(); } } @Test + public void testDNWithInvalidStorageWithHA() throws Exception { + MiniDFSNNTopology top = new MiniDFSNNTopology() + .addNameservice(new MiniDFSNNTopology.NSConf("ns1") + .addNN(new MiniDFSNNTopology.NNConf("nn0").setClusterId("cluster-1")) + .addNN(new MiniDFSNNTopology.NNConf("nn1").setClusterId("cluster-1"))); + + top.setFederation(true); + + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(top) + .numDataNodes(0).build(); + try { + cluster.startDataNodes(conf, 1, true, null, null); + // let the initialization be complete + Thread.sleep(10000); + DataNode dn = cluster.getDataNodes().get(0); + assertTrue("Datanode should be running", dn.isDatanodeUp()); + assertEquals("BPOfferService should be running", 1, + dn.getAllBpOs().length); + DataNodeProperties dnProp = cluster.stopDataNode(0); + + cluster.getNameNode(0).stop(); + cluster.getNameNode(1).stop(); + Configuration nn1 = cluster.getConfiguration(0); + Configuration nn2 = cluster.getConfiguration(1); + // setting up invalid cluster + StartupOption.FORMAT.setClusterId("cluster-2"); + DFSTestUtil.formatNameNode(nn1); + MiniDFSCluster.copyNameDirs(FSNamesystem.getNamespaceDirs(nn1), + FSNamesystem.getNamespaceDirs(nn2), nn2); + cluster.restartNameNode(0, false); + cluster.restartNameNode(1, false); + cluster.restartDataNode(dnProp); + + // let the initialization be complete + Thread.sleep(10000); + dn = cluster.getDataNodes().get(0); + assertFalse("Datanode should have shutdown as only service failed", + dn.isDatanodeUp()); + } finally { + cluster.shutdown(); + } + } + + + @Test public void testMiniDFSClusterWithMultipleNN() throws IOException { Configuration conf = new HdfsConfiguration(); // start Federated cluster and add a node. @@ -231,7 +311,6 @@ public class TestDataNodeMultipleRegistr // add a node try { - Assert.assertNotNull(cluster); cluster.waitActive(); Assert.assertEquals("(1)Should be 2 namenodes", 2, cluster.getNumNameNodes()); Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java Tue Aug 19 23:49:39 2014 @@ -27,11 +27,14 @@ import static org.junit.Assert.assertTru import java.io.File; import java.io.IOException; +import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -67,6 +70,7 @@ public class TestDataNodeRollingUpgrade private void startCluster() throws IOException { conf = new HdfsConfiguration(); + conf.setInt("dfs.blocksize", 1024*1024); cluster = new Builder(conf).numDataNodes(REPL_FACTOR).build(); cluster.waitActive(); fs = cluster.getFileSystem(); @@ -243,4 +247,48 @@ public class TestDataNodeRollingUpgrade shutdownCluster(); } } + + @Test (timeout=600000) + // Test DatanodeXceiver has correct peer-dataxceiver pairs for sending OOB message + public void testDatanodePeersXceiver() throws Exception { + try { + startCluster(); + + // Create files in DFS. + String testFile1 = "/TestDataNodeXceiver1.dat"; + String testFile2 = "/TestDataNodeXceiver2.dat"; + String testFile3 = "/TestDataNodeXceiver3.dat"; + + DFSClient client1 = new DFSClient(NameNode.getAddress(conf), conf); + DFSClient client2 = new DFSClient(NameNode.getAddress(conf), conf); + DFSClient client3 = new DFSClient(NameNode.getAddress(conf), conf); + + DFSOutputStream s1 = (DFSOutputStream) client1.create(testFile1, true); + DFSOutputStream s2 = (DFSOutputStream) client2.create(testFile2, true); + DFSOutputStream s3 = (DFSOutputStream) client3.create(testFile3, true); + + byte[] toWrite = new byte[1024*1024*8]; + Random rb = new Random(1111); + rb.nextBytes(toWrite); + s1.write(toWrite, 0, 1024*1024*8); + s1.flush(); + s2.write(toWrite, 0, 1024*1024*8); + s2.flush(); + s3.write(toWrite, 0, 1024*1024*8); + s3.flush(); + + assertTrue(dn.getXferServer().getNumPeersXceiver() == dn.getXferServer() + .getNumPeersXceiver()); + s1.close(); + s2.close(); + s3.close(); + assertTrue(dn.getXferServer().getNumPeersXceiver() == dn.getXferServer() + .getNumPeersXceiver()); + client1.close(); + client2.close(); + client3.close(); + } finally { + shutdownCluster(); + } + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java Tue Aug 19 23:49:39 2014 @@ -25,6 +25,7 @@ import java.io.FilenameFilter; import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,9 +47,11 @@ import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.net.TcpPeerServer; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -58,6 +61,7 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.token.Token; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -307,7 +311,8 @@ public class TestDataNodeVolumeFailure { setConfiguration(conf). setRemotePeerFactory(new RemotePeerFactory() { @Override - public Peer newConnectedPeer(InetSocketAddress addr) + public Peer newConnectedPeer(InetSocketAddress addr, + Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId) throws IOException { Peer peer = null; Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket(); @@ -380,7 +385,7 @@ public class TestDataNodeVolumeFailure { continue; } - String [] res = metaFilesInDir(dir); + List<File> res = MiniDFSCluster.getAllBlockMetadataFiles(dir); if(res == null) { System.out.println("res is null for dir = " + dir + " i=" + i + " and j=" + j); continue; @@ -388,7 +393,8 @@ public class TestDataNodeVolumeFailure { //System.out.println("for dn" + i + "." + j + ": " + dir + "=" + res.length+ " files"); //int ii = 0; - for(String s: res) { + for(File f: res) { + String s = f.getName(); // cut off "blk_-" at the beginning and ".meta" at the end assertNotNull("Block file name should not be null", s); String bid = s.substring(s.indexOf("_")+1, s.lastIndexOf("_")); @@ -404,25 +410,9 @@ public class TestDataNodeVolumeFailure { //System.out.println("dir1="+dir.getPath() + "blocks=" + res.length); //System.out.println("dir2="+dir2.getPath() + "blocks=" + res2.length); - total += res.length; + total += res.size(); } } return total; } - - /* - * count how many files *.meta are in the dir - */ - private String [] metaFilesInDir(File dir) { - String [] res = dir.list( - new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.startsWith("blk_") && - name.endsWith(Block.METADATA_EXTENSION); - } - } - ); - return res; - } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java Tue Aug 19 23:49:39 2014 @@ -103,9 +103,10 @@ public class TestDeleteBlockPool { fs1.delete(new Path("/alpha"), true); // Wait till all blocks are deleted from the dn2 for bpid1. - while ((MiniDFSCluster.getFinalizedDir(dn2StorageDir1, - bpid1).list().length != 0) || (MiniDFSCluster.getFinalizedDir( - dn2StorageDir2, bpid1).list().length != 0)) { + File finalDir1 = MiniDFSCluster.getFinalizedDir(dn2StorageDir1, bpid1); + File finalDir2 = MiniDFSCluster.getFinalizedDir(dn2StorageDir1, bpid2); + while ((!DatanodeUtil.dirNoFilesRecursive(finalDir1)) || + (!DatanodeUtil.dirNoFilesRecursive(finalDir2))) { try { Thread.sleep(3000); } catch (Exception ignored) { @@ -160,7 +161,8 @@ public class TestDeleteBlockPool { conf.set(DFSConfigKeys.DFS_NAMESERVICES, "namesServerId1,namesServerId2"); cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)) + .nnTopology(MiniDFSNNTopology.simpleFederatedTopology( + conf.get(DFSConfigKeys.DFS_NAMESERVICES))) .numDataNodes(1).build(); cluster.waitActive(); Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java Tue Aug 19 23:49:39 2014 @@ -18,16 +18,13 @@ package org.apache.hadoop.hdfs.server.datanode; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.DataOutputStream; import java.io.File; +import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; -import java.net.SocketException; -import java.net.SocketTimeoutException; -import java.nio.channels.ClosedChannelException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -38,6 +35,7 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -47,6 +45,7 @@ import org.apache.hadoop.hdfs.security.t import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.util.DataChecksum; +import org.apache.hadoop.util.Time; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -149,9 +148,9 @@ public class TestDiskError { DataChecksum checksum = DataChecksum.newDataChecksum( DataChecksum.Type.CRC32, 512); - new Sender(out).writeBlock(block.getBlock(), + new Sender(out).writeBlock(block.getBlock(), StorageType.DEFAULT, BlockTokenSecretManager.DUMMY_TOKEN, "", - new DatanodeInfo[0], null, + new DatanodeInfo[0], new StorageType[0], null, BlockConstructionStage.PIPELINE_SETUP_CREATE, 1, 0L, 0L, 0L, checksum, CachingStrategy.newDefaultStrategy()); out.flush(); @@ -201,15 +200,23 @@ public class TestDiskError { } } + /** + * Checks whether {@link DataNode#checkDiskErrorAsync()} is being called or not. + * Before refactoring the code the above function was not getting called + * @throws IOException, InterruptedException + */ @Test - public void testNetworkErrorsIgnored() { - DataNode dn = cluster.getDataNodes().iterator().next(); - - assertTrue(dn.isNetworkRelatedException(new SocketException())); - assertTrue(dn.isNetworkRelatedException(new SocketTimeoutException())); - assertTrue(dn.isNetworkRelatedException(new ClosedChannelException())); - assertTrue(dn.isNetworkRelatedException(new Exception("Broken pipe foo bar"))); - assertFalse(dn.isNetworkRelatedException(new Exception())); - assertFalse(dn.isNetworkRelatedException(new Exception("random problem"))); + public void testcheckDiskError() throws IOException, InterruptedException { + if(cluster.getDataNodes().size() <= 0) { + cluster.startDataNodes(conf, 1, true, null, null); + cluster.waitActive(); + } + DataNode dataNode = cluster.getDataNodes().get(0); + long slackTime = dataNode.checkDiskErrorInterval/2; + //checking for disk error + dataNode.checkDiskErrorAsync(); + Thread.sleep(dataNode.checkDiskErrorInterval); + long lastDiskErrorCheck = dataNode.getLastDiskErrorCheck(); + assertTrue("Disk Error check is not performed within " + dataNode.checkDiskErrorInterval + " ms", ((Time.monotonicNow()-lastDiskErrorCheck) < (dataNode.checkDiskErrorInterval + slackTime))); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java Tue Aug 19 23:49:39 2014 @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.da import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeTrue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyLong; @@ -68,6 +67,7 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse; import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat; import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.io.nativeio.NativeIO.POSIX.CacheManipulator; import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator; @@ -114,7 +114,6 @@ public class TestFsDatasetCache { @Before public void setUp() throws Exception { - assumeTrue(!Path.WINDOWS); conf = new HdfsConfiguration(); conf.setLong( DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS, 100); @@ -143,6 +142,9 @@ public class TestFsDatasetCache { @After public void tearDown() throws Exception { + // Verify that each test uncached whatever it cached. This cleanup is + // required so that file descriptors are not leaked across tests. + DFSTestUtil.verifyExpectedCacheUsage(0, 0, fsd); if (fs != null) { fs.close(); } @@ -205,9 +207,16 @@ public class TestFsDatasetCache { String bpid = loc.getLocatedBlock().getBlock().getBlockPoolId(); Block block = loc.getLocatedBlock().getBlock().getLocalBlock(); ExtendedBlock extBlock = new ExtendedBlock(bpid, block); - FileChannel blockChannel = - ((FileInputStream)fsd.getBlockInputStream(extBlock, 0)).getChannel(); - sizes[i] = blockChannel.size(); + FileInputStream blockInputStream = null; + FileChannel blockChannel = null; + try { + blockInputStream = + (FileInputStream)fsd.getBlockInputStream(extBlock, 0); + blockChannel = blockInputStream.getChannel(); + sizes[i] = blockChannel.size(); + } finally { + IOUtils.cleanup(LOG, blockChannel, blockInputStream); + } } return sizes; } @@ -571,5 +580,7 @@ public class TestFsDatasetCache { return true; } }, 1000, 30000); + + dfs.removeCacheDirective(shortCacheDirectiveId); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHdfsServerConstants.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHdfsServerConstants.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHdfsServerConstants.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHdfsServerConstants.java Tue Aug 19 23:49:39 2014 @@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.server.da import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.*; import org.junit.Test; -import static junit.framework.Assert.fail; +import static org.junit.Assert.fail; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java Tue Aug 19 23:49:39 2014 @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import static junit.framework.Assert.assertFalse; +import static org.junit.Assert.assertFalse; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.atLeastOnce; Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java Tue Aug 19 23:49:39 2014 @@ -29,6 +29,7 @@ import java.io.OutputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -65,7 +66,8 @@ public class TestSimulatedFSDataset { ExtendedBlock b = new ExtendedBlock(bpid, i, 0, 0); // we pass expected len as zero, - fsdataset should use the sizeof actual // data written - ReplicaInPipelineInterface bInfo = fsdataset.createRbw(b); + ReplicaInPipelineInterface bInfo = fsdataset.createRbw( + StorageType.DEFAULT, b); ReplicaOutputStreams out = bInfo.createStreams(true, DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512)); try { Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestAvailableSpaceVolumeChoosingPolicy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestAvailableSpaceVolumeChoosingPolicy.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestAvailableSpaceVolumeChoosingPolicy.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestAvailableSpaceVolumeChoosingPolicy.java Tue Aug 19 23:49:39 2014 @@ -22,6 +22,7 @@ import static org.apache.hadoop.hdfs.DFS import java.util.ArrayList; import java.util.List; +import java.util.Random; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -251,10 +252,10 @@ public class TestAvailableSpaceVolumeCho */ public void doRandomizedTest(float preferencePercent, int lowSpaceVolumes, int highSpaceVolumes) throws Exception { - @SuppressWarnings("unchecked") - final AvailableSpaceVolumeChoosingPolicy<FsVolumeSpi> policy = - ReflectionUtils.newInstance(AvailableSpaceVolumeChoosingPolicy.class, null); - + Random random = new Random(123L); + final AvailableSpaceVolumeChoosingPolicy<FsVolumeSpi> policy = + new AvailableSpaceVolumeChoosingPolicy<FsVolumeSpi>(random); + List<FsVolumeSpi> volumes = new ArrayList<FsVolumeSpi>(); // Volumes with 1MB free space Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java Tue Aug 19 23:49:39 2014 @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; @@ -147,7 +148,7 @@ public class TestWriteToReplica { }; ReplicaMap replicasMap = dataSet.volumeMap; - FsVolumeImpl vol = dataSet.volumes.getNextVolume(0); + FsVolumeImpl vol = dataSet.volumes.getNextVolume(StorageType.DEFAULT, 0); ReplicaInfo replicaInfo = new FinalizedReplica( blocks[FINALIZED].getLocalBlock(), vol, vol.getCurrentDir().getParentFile()); replicasMap.add(bpid, replicaInfo); @@ -357,7 +358,7 @@ public class TestWriteToReplica { } try { - dataSet.createRbw(blocks[FINALIZED]); + dataSet.createRbw(StorageType.DEFAULT, blocks[FINALIZED]); Assert.fail("Should not have created a replica that's already " + "finalized " + blocks[FINALIZED]); } catch (ReplicaAlreadyExistsException e) { @@ -375,7 +376,7 @@ public class TestWriteToReplica { } try { - dataSet.createRbw(blocks[TEMPORARY]); + dataSet.createRbw(StorageType.DEFAULT, blocks[TEMPORARY]); Assert.fail("Should not have created a replica that had created as " + "temporary " + blocks[TEMPORARY]); } catch (ReplicaAlreadyExistsException e) { @@ -385,7 +386,7 @@ public class TestWriteToReplica { 0L, blocks[RBW].getNumBytes()); // expect to be successful try { - dataSet.createRbw(blocks[RBW]); + dataSet.createRbw(StorageType.DEFAULT, blocks[RBW]); Assert.fail("Should not have created a replica that had created as RBW " + blocks[RBW]); } catch (ReplicaAlreadyExistsException e) { @@ -401,7 +402,7 @@ public class TestWriteToReplica { } try { - dataSet.createRbw(blocks[RWR]); + dataSet.createRbw(StorageType.DEFAULT, blocks[RWR]); Assert.fail("Should not have created a replica that was waiting to be " + "recovered " + blocks[RWR]); } catch (ReplicaAlreadyExistsException e) { @@ -417,7 +418,7 @@ public class TestWriteToReplica { } try { - dataSet.createRbw(blocks[RUR]); + dataSet.createRbw(StorageType.DEFAULT, blocks[RUR]); Assert.fail("Should not have created a replica that was under recovery " + blocks[RUR]); } catch (ReplicaAlreadyExistsException e) { @@ -434,45 +435,45 @@ public class TestWriteToReplica { e.getMessage().contains(ReplicaNotFoundException.NON_EXISTENT_REPLICA)); } - dataSet.createRbw(blocks[NON_EXISTENT]); + dataSet.createRbw(StorageType.DEFAULT, blocks[NON_EXISTENT]); } private void testWriteToTemporary(FsDatasetImpl dataSet, ExtendedBlock[] blocks) throws IOException { try { - dataSet.createTemporary(blocks[FINALIZED]); + dataSet.createTemporary(StorageType.DEFAULT, blocks[FINALIZED]); Assert.fail("Should not have created a temporary replica that was " + "finalized " + blocks[FINALIZED]); } catch (ReplicaAlreadyExistsException e) { } try { - dataSet.createTemporary(blocks[TEMPORARY]); + dataSet.createTemporary(StorageType.DEFAULT, blocks[TEMPORARY]); Assert.fail("Should not have created a replica that had created as" + "temporary " + blocks[TEMPORARY]); } catch (ReplicaAlreadyExistsException e) { } try { - dataSet.createTemporary(blocks[RBW]); + dataSet.createTemporary(StorageType.DEFAULT, blocks[RBW]); Assert.fail("Should not have created a replica that had created as RBW " + blocks[RBW]); } catch (ReplicaAlreadyExistsException e) { } try { - dataSet.createTemporary(blocks[RWR]); + dataSet.createTemporary(StorageType.DEFAULT, blocks[RWR]); Assert.fail("Should not have created a replica that was waiting to be " + "recovered " + blocks[RWR]); } catch (ReplicaAlreadyExistsException e) { } try { - dataSet.createTemporary(blocks[RUR]); + dataSet.createTemporary(StorageType.DEFAULT, blocks[RUR]); Assert.fail("Should not have created a replica that was under recovery " + blocks[RUR]); } catch (ReplicaAlreadyExistsException e) { } - dataSet.createTemporary(blocks[NON_EXISTENT]); + dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/AclTestHelpers.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/AclTestHelpers.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/AclTestHelpers.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/AclTestHelpers.java Tue Aug 19 23:49:39 2014 @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.permission.A import org.apache.hadoop.fs.permission.AclEntryScope; import org.apache.hadoop.fs.permission.AclEntryType; import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; @@ -150,6 +151,9 @@ public final class AclTestHelpers { */ public static void assertPermission(FileSystem fs, Path pathToCheck, short perm) throws IOException { - assertEquals(perm, fs.getFileStatus(pathToCheck).getPermission().toShort()); + short filteredPerm = (short)(perm & 01777); + FsPermission fsPermission = fs.getFileStatus(pathToCheck).getPermission(); + assertEquals(filteredPerm, fsPermission.toShort()); + assertEquals(((perm & (1 << 12)) != 0), fsPermission.getAclBit()); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java Tue Aug 19 23:49:39 2014 @@ -83,7 +83,7 @@ public class CreateEditsLog { final INodeFile inode = new INodeFile(inodeId.nextValue(), null, p, 0L, 0L, blocks, replication, blockSize); - inode.toUnderConstruction("", "", null); + inode.toUnderConstruction("", ""); // Append path to filename with information about blockIDs String path = "_" + iF + "_B" + blocks[0].getBlockId() + @@ -98,7 +98,7 @@ public class CreateEditsLog { } INodeFile fileUc = new INodeFile(inodeId.nextValue(), null, p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize); - fileUc.toUnderConstruction("", "", null); + fileUc.toUnderConstruction("", ""); editLog.logOpenFile(filePath, fileUc, false); editLog.logCloseFile(filePath, inode); @@ -131,14 +131,10 @@ public class CreateEditsLog { printUsageExit(); } /** - * @param args + * @param args arguments * @throws IOException */ - public static void main(String[] args) - throws IOException { - - - + public static void main(String[] args) throws IOException { long startingBlockId = 1; int numFiles = 0; short replication = 1;
