Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java Wed Sep 5 04:57:47 2012 @@ -19,13 +19,11 @@ package org.apache.hadoop.hdfs; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; - +import static org.junit.Assert.*; import java.io.IOException; import java.lang.reflect.Field; +import java.net.ServerSocket; +import java.net.Socket; import java.net.URI; import java.security.PrivilegedExceptionAction; @@ -55,7 +53,7 @@ public class TestHftpDelegationToken { new Text("127.0.0.1:8020")); user.addToken(token); Token<?> token2 = new Token<TokenIdentifier> - (null, null, new Text("other token"), new Text("127.0.0.1:8020")); + (null, null, new Text("other token"), new Text("127.0.0.1:8021")); user.addToken(token2); assertEquals("wrong tokens in user", 2, user.getTokens().size()); FileSystem fs = @@ -138,6 +136,53 @@ public class TestHftpDelegationToken { conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, 5); } + + @Test + public void testInsecureRemoteCluster() throws Exception { + final ServerSocket socket = new ServerSocket(0); // just reserve a port + socket.close(); + Configuration conf = new Configuration(); + URI fsUri = URI.create("hsftp://localhost:"+socket.getLocalPort()); + assertNull(FileSystem.newInstance(fsUri, conf).getDelegationToken(null)); + } + + @Test + public void testSecureClusterError() throws Exception { + final ServerSocket socket = new ServerSocket(0); + Thread t = new Thread() { + @Override + public void run() { + while (true) { // fetching does a few retries + try { + Socket s = socket.accept(); + s.getOutputStream().write(1234); + s.shutdownOutput(); + } catch (Exception e) { + break; + } + } + } + }; + t.start(); + + try { + Configuration conf = new Configuration(); + URI fsUri = URI.create("hsftp://localhost:"+socket.getLocalPort()); + Exception ex = null; + try { + FileSystem.newInstance(fsUri, conf).getDelegationToken(null); + } catch (Exception e) { + ex = e; + } + assertNotNull(ex); + assertNotNull(ex.getCause()); + assertEquals("Unexpected end of file from server", + ex.getCause().getMessage()); + } finally { + t.interrupt(); + } + } + private void checkTokenSelection(HftpFileSystem fs, int port, Configuration conf) throws IOException {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java Wed Sep 5 04:57:47 2012 @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEqu import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.io.OutputStream; import java.net.InetSocketAddress; import java.util.Random; @@ -32,12 +33,14 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.util.ThreadUtil; import org.junit.Test; /** * This class tests the decommissioning of nodes. */ public class TestModTime { + static final long seed = 0xDEADBEEFL; static final int blockSize = 8192; static final int fileSize = 16384; @@ -186,6 +189,46 @@ public class TestModTime { cluster.shutdown(); } } + + /** + * Regression test for HDFS-3864 - NN does not update internal file mtime for + * OP_CLOSE when reading from the edit log. + */ + @Test + public void testModTimePersistsAfterRestart() throws IOException { + final long sleepTime = 10; // 10 milliseconds + MiniDFSCluster cluster = null; + FileSystem fs = null; + Configuration conf = new HdfsConfiguration(); + try { + cluster = new MiniDFSCluster.Builder(conf).build(); + fs = cluster.getFileSystem(); + Path testPath = new Path("/test"); + + // Open a file, and get its initial modification time. + OutputStream out = fs.create(testPath); + long initialModTime = fs.getFileStatus(testPath).getModificationTime(); + assertTrue(initialModTime > 0); + + // Wait and then close the file. Ensure that the mod time goes up. + ThreadUtil.sleepAtLeastIgnoreInterrupts(sleepTime); + out.close(); + long modTimeAfterClose = fs.getFileStatus(testPath).getModificationTime(); + assertTrue(modTimeAfterClose >= initialModTime + sleepTime); + + // Restart the NN, and make sure that the later mod time is still used. + cluster.restartNameNode(); + long modTimeAfterRestart = fs.getFileStatus(testPath).getModificationTime(); + assertEquals(modTimeAfterClose, modTimeAfterRestart); + } finally { + if (fs != null) { + fs.close(); + } + if (cluster != null) { + cluster.shutdown(); + } + } + } public static void main(String[] args) throws Exception { new TestModTime().testModTime(); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java Wed Sep 5 04:57:47 2012 @@ -80,9 +80,11 @@ public class TestClientProtocolWithDeleg DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT, 3600000, mockNameSys); sm.startThreads(); - final Server server = RPC.getServer(ClientProtocol.class, mockNN, ADDRESS, - 0, 5, true, conf, sm); - + final Server server = new RPC.Builder(conf) + .setProtocol(ClientProtocol.class).setInstance(mockNN) + .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true) + .setSecretManager(sm).build(); + server.start(); final UserGroupInformation current = UserGroupInformation.getCurrentUser(); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java Wed Sep 5 04:57:47 2012 @@ -231,8 +231,9 @@ public class TestBlockToken { ProtobufRpcEngine.class); BlockingService service = ClientDatanodeProtocolService .newReflectiveBlockingService(mockDN); - return RPC.getServer(ClientDatanodeProtocolPB.class, service, ADDRESS, 0, 5, - true, conf, sm); + return new RPC.Builder(conf).setProtocol(ClientDatanodeProtocolPB.class) + .setInstance(service).setBindAddress(ADDRESS).setPort(0) + .setNumHandlers(5).setVerbose(true).setSecretManager(sm).build(); } @Test Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java Wed Sep 5 04:57:47 2012 @@ -111,30 +111,30 @@ public class TestReplicationPolicy { HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 4, 0); // overloaded DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, - 0, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 0, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, - 1, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 1, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 1); assertEquals(targets[0], dataNodes[0]); targets = replicator.chooseTarget(filename, - 2, dataNodes[0], BLOCK_SIZE); + 2, dataNodes[0], new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 2); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, - 3, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 3, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 3); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); - targets = replicator.chooseTarget(filename, - 4, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 4, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[0]); assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || @@ -249,30 +249,30 @@ public class TestReplicationPolicy { (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); // no space DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, - 0, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 0, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, - 1, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 1, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 1); assertEquals(targets[0], dataNodes[1]); - targets = replicator.chooseTarget(filename, - 2, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 2, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 2); assertEquals(targets[0], dataNodes[1]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, - 3, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 3, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 3); assertEquals(targets[0], dataNodes[1]); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, - 4, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 4, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[1]); for(int i=1; i<4; i++) { @@ -305,23 +305,23 @@ public class TestReplicationPolicy { } DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, - 0, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 0, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, - 1, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 1, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); - targets = replicator.chooseTarget(filename, - 2, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 2, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, - 3, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 3, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 3); for(int i=0; i<3; i++) { assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0])); @@ -350,21 +350,21 @@ public class TestReplicationPolicy { DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r4"); DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, - 0, writerDesc, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 0, writerDesc, + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 0); - - targets = replicator.chooseTarget(filename, - 1, writerDesc, BLOCK_SIZE); + + targets = replicator.chooseTarget(filename, 1, writerDesc, + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 1); - - targets = replicator.chooseTarget(filename, - 2, writerDesc, BLOCK_SIZE); + + targets = replicator.chooseTarget(filename, 2, writerDesc, + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - - targets = replicator.chooseTarget(filename, - 3, writerDesc, BLOCK_SIZE); + + targets = replicator.chooseTarget(filename, 3, writerDesc, + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 3); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java Wed Sep 5 04:57:47 2012 @@ -114,31 +114,31 @@ public class TestReplicationPolicyWithNo HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 4, 0); // overloaded DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, - 0, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 0, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, - 1, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 1, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 1); assertEquals(targets[0], dataNodes[0]); - targets = replicator.chooseTarget(filename, - 2, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 2, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 2); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, - 3, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 3, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 3); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameNodeGroup(targets[1], targets[2])); - targets = replicator.chooseTarget(filename, - 4, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 4, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[0]); assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || @@ -220,30 +220,30 @@ public class TestReplicationPolicyWithNo (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); // no space DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, - 0, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 0, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, - 1, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 1, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 1); assertEquals(targets[0], dataNodes[1]); - targets = replicator.chooseTarget(filename, - 2, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 2, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 2); assertEquals(targets[0], dataNodes[1]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, - 3, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 3, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 3); assertEquals(targets[0], dataNodes[1]); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, - 4, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 4, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[1]); assertTrue(cluster.isNodeGroupAware()); @@ -275,23 +275,23 @@ public class TestReplicationPolicyWithNo } DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, - 0, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 0, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, - 1, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 1, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); - targets = replicator.chooseTarget(filename, - 2, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 2, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, - 3, dataNodes[0], BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 3, dataNodes[0], + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 3); for(int i=0; i<3; i++) { assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0])); @@ -313,21 +313,21 @@ public class TestReplicationPolicyWithNo public void testChooseTarget5() throws Exception { setupDataNodeCapacity(); DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, - 0, NODE, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 0, NODE, + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 0); - - targets = replicator.chooseTarget(filename, - 1, NODE, BLOCK_SIZE); + + targets = replicator.chooseTarget(filename, 1, NODE, + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 1); - - targets = replicator.chooseTarget(filename, - 2, NODE, BLOCK_SIZE); + + targets = replicator.chooseTarget(filename, 2, NODE, + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - - targets = replicator.chooseTarget(filename, - 3, NODE, BLOCK_SIZE); + + targets = replicator.chooseTarget(filename, 3, NODE, + new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); assertEquals(targets.length, 3); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Wed Sep 5 04:57:47 2012 @@ -66,7 +66,7 @@ public class NameNodeAdapter { public static HdfsFileStatus getFileInfo(NameNode namenode, String src, boolean resolveLink) throws AccessControlException, UnresolvedLinkException, - StandbyException { + StandbyException, IOException { return namenode.getNamesystem().getFileInfo(src, resolveLink); } Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java Wed Sep 5 04:57:47 2012 @@ -32,13 +32,17 @@ import java.util.regex.Pattern; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.HftpFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; +import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.Level; @@ -83,6 +87,7 @@ public class TestAuditLogs { final long precision = 1L; conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); + conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); util = new DFSTestUtil.Builder().setName("TestAuditAllowed"). setNumFiles(20).build(); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); @@ -115,6 +120,18 @@ public class TestAuditLogs { assertTrue("failed to read from file", val > 0); } + /** test that allowed stat puts proper entry in audit log */ + @Test + public void testAuditAllowedStat() throws Exception { + final Path file = new Path(fnames[0]); + FileSystem userfs = DFSTestUtil.getFileSystemAs(userGroupInfo, conf); + + setupAuditLogs(); + FileStatus st = userfs.getFileStatus(file); + verifyAuditLogs(true); + assertTrue("failed to stat file", st != null && st.isFile()); + } + /** test that denied operation puts proper entry in audit log */ @Test public void testAuditDenied() throws Exception { @@ -135,6 +152,85 @@ public class TestAuditLogs { verifyAuditLogs(false); } + /** test that access via webhdfs puts proper entry in audit log */ + @Test + public void testAuditWebHdfs() throws Exception { + final Path file = new Path(fnames[0]); + + fs.setPermission(file, new FsPermission((short)0644)); + fs.setOwner(file, "root", null); + + setupAuditLogs(); + + WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf); + InputStream istream = webfs.open(file); + int val = istream.read(); + istream.close(); + + verifyAuditLogsRepeat(true, 3); + assertTrue("failed to read from file", val > 0); + } + + /** test that stat via webhdfs puts proper entry in audit log */ + @Test + public void testAuditWebHdfsStat() throws Exception { + final Path file = new Path(fnames[0]); + + fs.setPermission(file, new FsPermission((short)0644)); + fs.setOwner(file, "root", null); + + setupAuditLogs(); + + WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf); + FileStatus st = webfs.getFileStatus(file); + + verifyAuditLogs(true); + assertTrue("failed to stat file", st != null && st.isFile()); + } + + /** test that access via Hftp puts proper entry in audit log */ + @Test + public void testAuditHftp() throws Exception { + final Path file = new Path(fnames[0]); + + final String hftpUri = + "hftp://"; + conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY); + + HftpFileSystem hftpFs = null; + + setupAuditLogs(); + try { + hftpFs = (HftpFileSystem) new Path(hftpUri).getFileSystem(conf); + InputStream istream = hftpFs.open(file); + int val = istream.read(); + istream.close(); + + verifyAuditLogs(true); + } finally { + if (hftpFs != null) hftpFs.close(); + } + } + + /** test that denied access via webhdfs puts proper entry in audit log */ + @Test + public void testAuditWebHdfsDenied() throws Exception { + final Path file = new Path(fnames[0]); + + fs.setPermission(file, new FsPermission((short)0600)); + fs.setOwner(file, "root", null); + + setupAuditLogs(); + try { + WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf); + InputStream istream = webfs.open(file); + int val = istream.read(); + fail("open+read must not succeed, got " + val); + } catch(AccessControlException E) { + System.out.println("got access denied, as expected."); + } + verifyAuditLogsRepeat(false, 2); + } + /** Sets up log4j logger for auditlogs */ private void setupAuditLogs() throws IOException { File file = new File(auditLogFile); @@ -148,19 +244,34 @@ public class TestAuditLogs { logger.addAppender(appender); } + // Ensure audit log has only one entry private void verifyAuditLogs(boolean expectSuccess) throws IOException { + verifyAuditLogsRepeat(expectSuccess, 1); + } + + // Ensure audit log has exactly N entries + private void verifyAuditLogsRepeat(boolean expectSuccess, int ndupe) + throws IOException { // Turn off the logs Logger logger = ((Log4JLogger) FSNamesystem.auditLog).getLogger(); logger.setLevel(Level.OFF); - // Ensure audit log has only one entry BufferedReader reader = new BufferedReader(new FileReader(auditLogFile)); - String line = reader.readLine(); - assertNotNull(line); - assertTrue("Expected audit event not found in audit log", - auditPattern.matcher(line).matches()); - assertTrue("Expected success=" + expectSuccess, - successPattern.matcher(line).matches() == expectSuccess); - assertNull("Unexpected event in audit log", reader.readLine()); + String line = null; + boolean ret = true; + + try { + for (int i = 0; i < ndupe; i++) { + line = reader.readLine(); + assertNotNull(line); + assertTrue("Expected audit event not found in audit log", + auditPattern.matcher(line).matches()); + ret &= successPattern.matcher(line).matches(); + } + assertNull("Unexpected event in audit log", reader.readLine()); + assertTrue("Expected success=" + expectSuccess, ret == expectSuccess); + } finally { + reader.close(); + } } } Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Wed Sep 5 04:57:47 2012 @@ -1924,6 +1924,59 @@ public class TestCheckpoint { } } } + + /** + * Regression test for HDFS-3849. This makes sure that when we re-load the + * FSImage in the 2NN, we clear the existing leases. + */ + @Test + public void testSecondaryNameNodeWithSavedLeases() throws IOException { + MiniDFSCluster cluster = null; + SecondaryNameNode secondary = null; + FSDataOutputStream fos = null; + Configuration conf = new HdfsConfiguration(); + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes) + .format(true).build(); + FileSystem fs = cluster.getFileSystem(); + fos = fs.create(new Path("tmpfile")); + fos.write(new byte[] { 0, 1, 2, 3 }); + fos.hflush(); + assertEquals(1, cluster.getNamesystem().getLeaseManager().countLease()); + + secondary = startSecondaryNameNode(conf); + assertEquals(0, secondary.getFSNamesystem().getLeaseManager().countLease()); + + // Checkpoint once, so the 2NN loads the lease into its in-memory sate. + secondary.doCheckpoint(); + assertEquals(1, secondary.getFSNamesystem().getLeaseManager().countLease()); + fos.close(); + fos = null; + + // Perform a saveNamespace, so that the NN has a new fsimage, and the 2NN + // therefore needs to download a new fsimage the next time it performs a + // checkpoint. + cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_ENTER); + cluster.getNameNodeRpc().saveNamespace(); + cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + // Ensure that the 2NN can still perform a checkpoint. + secondary.doCheckpoint(); + + // And the leases have been cleared... + assertEquals(0, secondary.getFSNamesystem().getLeaseManager().countLease()); + } finally { + if (fos != null) { + fos.close(); + } + if (secondary != null) { + secondary.shutdown(); + } + if (cluster != null) { + cluster.shutdown(); + } + } + } @Test public void testCommandLineParsing() throws ParseException { Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java Wed Sep 5 04:57:47 2012 @@ -26,6 +26,9 @@ import java.net.URI; import java.util.Collection; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.junit.Test; public class TestFSNamesystem { @@ -45,4 +48,20 @@ public class TestFSNamesystem { assertEquals(2, editsDirs.size()); } + /** + * Test that FSNamesystem#clear clears all leases. + */ + @Test + public void testFSNamespaceClearLeases() throws Exception { + Configuration conf = new HdfsConfiguration(); + NameNode.initMetrics(conf, NamenodeRole.NAMENODE); + DFSTestUtil.formatNameNode(conf); + FSNamesystem fsn = FSNamesystem.loadFromDisk(conf); + LeaseManager leaseMan = fsn.getLeaseManager(); + leaseMan.addLease("client1", "importantFile"); + assertEquals(1, leaseMan.countLease()); + fsn.clear(); + leaseMan = fsn.getLeaseManager(); + assertEquals(0, leaseMan.countLease()); + } } Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java Wed Sep 5 04:57:47 2012 @@ -95,6 +95,12 @@ public class TestFsck { "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + "cmd=fsck\\ssrc=\\/\\sdst=null\\s" + "perm=null"); + static final Pattern getfileinfoPattern = Pattern.compile( + "allowed=.*?\\s" + + "ugi=.*?\\s" + + "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + + "cmd=getfileinfo\\ssrc=\\/\\sdst=null\\s" + + "perm=null"); static final Pattern numCorruptBlocksPattern = Pattern.compile( ".*Corrupt blocks:\t\t([0123456789]*).*"); @@ -180,10 +186,14 @@ public class TestFsck { Logger logger = ((Log4JLogger) FSNamesystem.auditLog).getLogger(); logger.setLevel(Level.OFF); - // Ensure audit log has only one for FSCK + // Audit log should contain one getfileinfo and one fsck BufferedReader reader = new BufferedReader(new FileReader(auditLogFile)); String line = reader.readLine(); assertNotNull(line); + assertTrue("Expected getfileinfo event not found in audit log", + getfileinfoPattern.matcher(line).matches()); + line = reader.readLine(); + assertNotNull(line); assertTrue("Expected fsck event not found in audit log", fsckPattern.matcher(line).matches()); assertNull("Unexpected event in audit log", reader.readLine()); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java Wed Sep 5 04:57:47 2012 @@ -27,7 +27,6 @@ import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.server.common.Storage.FormatConfirmable; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.junit.Test; Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java Wed Sep 5 04:57:47 2012 @@ -23,10 +23,7 @@ import java.util.List; import org.junit.Test; import org.junit.Before; -import org.junit.After; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileUtil; @@ -35,9 +32,6 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import java.util.Properties; -import java.io.FileReader; -import java.io.FileWriter; import org.junit.Assert; import org.apache.hadoop.test.GenericTestUtils; Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java Wed Sep 5 04:57:47 2012 @@ -89,7 +89,6 @@ public class TestWebHdfsDataLocality { //set client address to a particular datanode final DataNode dn = cluster.getDataNodes().get(i); final String ipAddr = dm.getDatanode(dn.getDatanodeId()).getIpAddr(); - NamenodeWebHdfsMethods.setRemoteAddress(ipAddr); //The chosen datanode must be the same as the client address final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml?rev=1380990&r1=1380989&r2=1380990&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml Wed Sep 5 04:57:47 2012 @@ -15885,6 +15885,23 @@ </comparators> </test> + <!-- Test for rollEdits --> + <test> <!-- TESTED --> + <description>rollEdits: test rollEdits admin command</description> + <test-commands> + <dfs-admin-command>-fs NAMENODE -rollEdits</dfs-admin-command> + </test-commands> + <cleanup-commands> + <!-- no cleanup --> + </cleanup-commands> + <comparators> + <comparator> + <type>RegexpComparator</type> + <expected-output>New segment starts at txid \d+</expected-output> + </comparator> + </comparators> + </test> + <!-- Test for refreshNodes --> <test> <!-- TESTED --> <description>refreshNodes: to refresh the nodes</description>
