Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java Fri Feb 14 18:32:37 2014 @@ -38,10 +38,16 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.BlockReader; import org.apache.hadoop.hdfs.BlockReaderFactory; +import org.apache.hadoop.hdfs.ClientContext; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.DFSClient.Conf; +import org.apache.hadoop.hdfs.RemotePeerFactory; +import org.apache.hadoop.hdfs.client.ShortCircuitCache; +import org.apache.hadoop.hdfs.client.ShortCircuitReplica; +import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.net.TcpPeerServer; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -55,10 +61,13 @@ import org.apache.hadoop.hdfs.server.com import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Level; +import org.junit.Assert; import org.junit.Test; public class TestBlockTokenWithDFS { @@ -131,50 +140,70 @@ public class TestBlockTokenWithDFS { } // try reading a block using a BlockReader directly - private static void tryRead(Configuration conf, LocatedBlock lblock, + private static void tryRead(final Configuration conf, LocatedBlock lblock, boolean shouldSucceed) { InetSocketAddress targetAddr = null; - Socket s = null; + IOException ioe = null; BlockReader blockReader = null; ExtendedBlock block = lblock.getBlock(); try { DatanodeInfo[] nodes = lblock.getLocations(); targetAddr = NetUtils.createSocketAddr(nodes[0].getXferAddr()); - s = NetUtils.getDefaultSocketFactory(conf).createSocket(); - s.connect(targetAddr, HdfsServerConstants.READ_TIMEOUT); - s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); - - String file = BlockReaderFactory.getFileName(targetAddr, - "test-blockpoolid", block.getBlockId()); - blockReader = BlockReaderFactory.newBlockReader( - new DFSClient.Conf(conf), file, block, lblock.getBlockToken(), 0, -1, - true, "TestBlockTokenWithDFS", TcpPeerServer.peerFromSocket(s), - nodes[0], null, null, null, false, - CachingStrategy.newDefaultStrategy()); + blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)). + setFileName(BlockReaderFactory.getFileName(targetAddr, + "test-blockpoolid", block.getBlockId())). + setBlock(block). + setBlockToken(lblock.getBlockToken()). + setInetSocketAddress(targetAddr). + setStartOffset(0). + setLength(-1). + setVerifyChecksum(true). + setClientName("TestBlockTokenWithDFS"). + setDatanodeInfo(nodes[0]). + setCachingStrategy(CachingStrategy.newDefaultStrategy()). + setClientCacheContext(ClientContext.getFromConf(conf)). + setConfiguration(conf). + setRemotePeerFactory(new RemotePeerFactory() { + @Override + public Peer newConnectedPeer(InetSocketAddress addr) + throws IOException { + Peer peer = null; + Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket(); + try { + sock.connect(addr, HdfsServerConstants.READ_TIMEOUT); + sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); + peer = TcpPeerServer.peerFromSocket(sock); + } finally { + if (peer == null) { + IOUtils.closeSocket(sock); + } + } + return peer; + } + }). + build(); } catch (IOException ex) { - if (ex instanceof InvalidBlockTokenException) { - assertFalse("OP_READ_BLOCK: access token is invalid, " - + "when it is expected to be valid", shouldSucceed); - return; - } - fail("OP_READ_BLOCK failed due to reasons other than access token: " - + StringUtils.stringifyException(ex)); + ioe = ex; } finally { - if (s != null) { + if (blockReader != null) { try { - s.close(); - } catch (IOException iex) { - } finally { - s = null; + blockReader.close(); + } catch (IOException e) { + throw new RuntimeException(e); } } } - if (blockReader == null) { - fail("OP_READ_BLOCK failed due to reasons other than access token"); + if (shouldSucceed) { + Assert.assertNotNull("OP_READ_BLOCK: access token is invalid, " + + "when it is expected to be valid", blockReader); + } else { + Assert.assertNotNull("OP_READ_BLOCK: access token is valid, " + + "when it is expected to be invalid", ioe); + Assert.assertTrue( + "OP_READ_BLOCK failed due to reasons other than access token: ", + ioe instanceof InvalidBlockTokenException); } - assertTrue("OP_READ_BLOCK: access token is valid, " - + "when it is expected to be invalid", shouldSucceed); } // get a conf for testing @@ -347,9 +376,13 @@ public class TestBlockTokenWithDFS { /* * testing READ interface on DN using a BlockReader */ - - new DFSClient(new InetSocketAddress("localhost", + DFSClient client = null; + try { + client = new DFSClient(new InetSocketAddress("localhost", cluster.getNameNodePort()), conf); + } finally { + if (client != null) client.close(); + } List<LocatedBlock> locatedBlocks = nnProto.getBlockLocations( FILE_TO_READ, 0, FILE_SIZE).getLocatedBlocks(); LocatedBlock lblock = locatedBlocks.get(0); // first block
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java Fri Feb 14 18:32:37 2014 @@ -35,11 +35,14 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.BlockReader; import org.apache.hadoop.hdfs.BlockReaderFactory; +import org.apache.hadoop.hdfs.ClientContext; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.RemotePeerFactory; +import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.net.TcpPeerServer; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; @@ -48,13 +51,14 @@ import org.apache.hadoop.hdfs.protocol.E import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -284,23 +288,43 @@ public class TestDataNodeVolumeFailure { private void accessBlock(DatanodeInfo datanode, LocatedBlock lblock) throws IOException { InetSocketAddress targetAddr = null; - Socket s = null; ExtendedBlock block = lblock.getBlock(); targetAddr = NetUtils.createSocketAddr(datanode.getXferAddr()); - - s = NetUtils.getDefaultSocketFactory(conf).createSocket(); - s.connect(targetAddr, HdfsServerConstants.READ_TIMEOUT); - s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); - - String file = BlockReaderFactory.getFileName(targetAddr, - "test-blockpoolid", - block.getBlockId()); - BlockReader blockReader = - BlockReaderFactory.newBlockReader(new DFSClient.Conf(conf), file, block, - lblock.getBlockToken(), 0, -1, true, "TestDataNodeVolumeFailure", - TcpPeerServer.peerFromSocket(s), datanode, null, null, null, false, - CachingStrategy.newDefaultStrategy()); + + BlockReader blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)). + setInetSocketAddress(targetAddr). + setBlock(block). + setFileName(BlockReaderFactory.getFileName(targetAddr, + "test-blockpoolid", block.getBlockId())). + setBlockToken(lblock.getBlockToken()). + setStartOffset(0). + setLength(-1). + setVerifyChecksum(true). + setClientName("TestDataNodeVolumeFailure"). + setDatanodeInfo(datanode). + setCachingStrategy(CachingStrategy.newDefaultStrategy()). + setClientCacheContext(ClientContext.getFromConf(conf)). + setConfiguration(conf). + setRemotePeerFactory(new RemotePeerFactory() { + @Override + public Peer newConnectedPeer(InetSocketAddress addr) + throws IOException { + Peer peer = null; + Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket(); + try { + sock.connect(addr, HdfsServerConstants.READ_TIMEOUT); + sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); + peer = TcpPeerServer.peerFromSocket(sock); + } finally { + if (peer == null) { + IOUtils.closeSocket(sock); + } + } + return peer; + } + }). + build(); blockReader.close(); } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java Fri Feb 14 18:32:37 2014 @@ -27,17 +27,12 @@ import static org.junit.Assert.assertTru import java.io.File; import java.io.IOException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.junit.Test; public class TestFSImageStorageInspector { - private static final Log LOG = LogFactory.getLog( - TestFSImageStorageInspector.class); - /** * Simple test with image, edits, and inprogress edits */ Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java Fri Feb 14 18:32:37 2014 @@ -140,7 +140,7 @@ public class TestFSImageWithSnapshot { private File saveFSImageToTempFile() throws IOException { SaveNamespaceContext context = new SaveNamespaceContext(fsn, txid, new Canceler()); - FSImageFormat.Saver saver = new FSImageFormat.Saver(context); + FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context); FSImageCompression compression = FSImageCompression.createCompression(conf); File imageFile = getImageFile(testDir, txid); fsn.readLock(); @@ -154,7 +154,7 @@ public class TestFSImageWithSnapshot { /** Load the fsimage from a temp file */ private void loadFSImageFromTempFile(File imageFile) throws IOException { - FSImageFormat.Loader loader = new FSImageFormat.Loader(conf, fsn); + FSImageFormat.LoaderDelegator loader = FSImageFormat.newLoader(conf, fsn); fsn.writeLock(); fsn.getFSDirectory().writeLock(); try { Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java Fri Feb 14 18:32:37 2014 @@ -288,7 +288,6 @@ public class TestStandbyCheckpoints { doEdits(0, 1000); nn0.getRpcServer().rollEditLog(); answerer.waitForCall(); - answerer.proceed(); assertTrue("SBN is not performing checkpoint but it should be.", answerer.getFireCount() == 1 && answerer.getResultCount() == 0); @@ -307,6 +306,7 @@ public class TestStandbyCheckpoints { // RPC to the SBN happened during the checkpoint. assertTrue("SBN should have still been checkpointing.", answerer.getFireCount() == 1 && answerer.getResultCount() == 0); + answerer.proceed(); answerer.waitForResult(); assertTrue("SBN should have finished checkpointing.", answerer.getFireCount() == 1 && answerer.getResultCount() == 1); Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java Fri Feb 14 18:32:37 2014 @@ -73,7 +73,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -; /** Testing rename with snapshots. */ public class TestRenameWithSnapshots { Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java Fri Feb 14 18:32:37 2014 @@ -25,6 +25,9 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -53,8 +56,7 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper.TestDirectoryTree; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper.TestDirectoryTree.Node; -import org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer; -import org.apache.hadoop.hdfs.tools.offlineImageViewer.XmlImageVisitor; +import org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; @@ -245,8 +247,8 @@ public class TestSnapshot { * snapshots */ @Test - public void testOfflineImageViewer() throws Throwable { - runTestSnapshot(SNAPSHOT_ITERATION_NUMBER); + public void testOfflineImageViewer() throws Exception { + runTestSnapshot(1); // retrieve the fsimage. Note that we already save namespace to fsimage at // the end of each iteration of runTestSnapshot. @@ -254,31 +256,10 @@ public class TestSnapshot { FSImageTestUtil.getFSImage( cluster.getNameNode()).getStorage().getStorageDir(0)); assertNotNull("Didn't generate or can't find fsimage", originalFsimage); - - String ROOT = System.getProperty("test.build.data", "build/test/data"); - File testFile = new File(ROOT, "/image"); - String xmlImage = ROOT + "/image_xml"; - boolean success = false; - - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - XmlImageVisitor v = new XmlImageVisitor(xmlImage, true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, - true); - oiv.go(); - success = true; - } finally { - if (testFile.exists()) { - testFile.delete(); - } - // delete the xml file if the parsing is successful - if (success) { - File xmlImageFile = new File(xmlImage); - if (xmlImageFile.exists()) { - xmlImageFile.delete(); - } - } - } + StringWriter output = new StringWriter(); + PrintWriter o = new PrintWriter(output); + PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o); + v.visit(new RandomAccessFile(originalFsimage, "r")); } private void runTestSnapshot(int iteration) throws Exception { Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java Fri Feb 14 18:32:37 2014 @@ -92,7 +92,7 @@ public class TestWebHdfsDataLocality { //The chosen datanode must be the same as the client address final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, PutOpParam.Op.CREATE, -1L, blocksize, conf); + namenode, f, PutOpParam.Op.CREATE, -1L, blocksize); Assert.assertEquals(ipAddr, chosen.getIpAddr()); } } @@ -117,19 +117,19 @@ public class TestWebHdfsDataLocality { { //test GETFILECHECKSUM final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize, conf); + namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize); Assert.assertEquals(expected, chosen); } { //test OPEN final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, GetOpParam.Op.OPEN, 0, blocksize, conf); + namenode, f, GetOpParam.Op.OPEN, 0, blocksize); Assert.assertEquals(expected, chosen); } { //test APPEND final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, PostOpParam.Op.APPEND, -1L, blocksize, conf); + namenode, f, PostOpParam.Op.APPEND, -1L, blocksize); Assert.assertEquals(expected, chosen); } } finally { Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Fri Feb 14 18:32:37 2014 @@ -20,23 +20,20 @@ package org.apache.hadoop.hdfs.tools.off import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import java.io.BufferedReader; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.io.StringWriter; import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -46,27 +43,29 @@ import org.apache.hadoop.fs.FSDataOutput import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.test.PathUtils; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; - +import org.junit.rules.TemporaryFolder; /** - * Test function of OfflineImageViewer by: - * * confirming it can correctly process a valid fsimage file and that - * the processing generates a correct representation of the namespace - * * confirming it correctly fails to process an fsimage file with a layout - * version it shouldn't be able to handle - * * confirm it correctly bails on malformed image files, in particular, a - * file that ends suddenly. + * Test function of OfflineImageViewer by: * confirming it can correctly process + * a valid fsimage file and that the processing generates a correct + * representation of the namespace * confirming it correctly fails to process an + * fsimage file with a layout version it shouldn't be able to handle * confirm + * it correctly bails on malformed image files, in particular, a file that ends + * suddenly. */ public class TestOfflineImageViewer { private static final Log LOG = LogFactory.getLog(OfflineImageViewer.class); @@ -76,22 +75,22 @@ public class TestOfflineImageViewer { private static File originalFsimage = null; // Elements of lines of ls-file output to be compared to FileStatus instance - private static class LsElements { - public String perms; - public int replication; - public String username; - public String groupname; - public long filesize; - public char dir; // d if dir, - otherwise + private static final class LsElements { + private String perms; + private int replication; + private String username; + private String groupname; + private long filesize; + private boolean isDir; } - + // namespace as written to dfs, to be compared with viewer's output - final static HashMap<String, FileStatus> writtenFiles = - new HashMap<String, FileStatus>(); - - private static String ROOT = PathUtils.getTestDirName(TestOfflineImageViewer.class); - - // Create a populated namespace for later testing. Save its contents to a + final static HashMap<String, FileStatus> writtenFiles = new HashMap<String, FileStatus>(); + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + // Create a populated namespace for later testing. Save its contents to a // data structure and store its fsimage location. // We only want to generate the fsimage file once and use it for // multiple tests. @@ -100,35 +99,39 @@ public class TestOfflineImageViewer { MiniDFSCluster cluster = null; try { Configuration conf = new HdfsConfiguration(); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000); + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL, "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT"); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); cluster.waitActive(); FileSystem hdfs = cluster.getFileSystem(); - + int filesize = 256; - - // Create a reasonable namespace - for(int i = 0; i < NUM_DIRS; i++) { + + // Create a reasonable namespace + for (int i = 0; i < NUM_DIRS; i++) { Path dir = new Path("/dir" + i); hdfs.mkdirs(dir); writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString())); - for(int j = 0; j < FILES_PER_DIR; j++) { + for (int j = 0; j < FILES_PER_DIR; j++) { Path file = new Path(dir, "file" + j); FSDataOutputStream o = hdfs.create(file); - o.write(new byte[ filesize++ ]); + o.write(new byte[filesize++]); o.close(); - - writtenFiles.put(file.toString(), pathToFileEntry(hdfs, file.toString())); + + writtenFiles.put(file.toString(), + pathToFileEntry(hdfs, file.toString())); } } // Get delegation tokens so we log the delegation token op - Token<?>[] delegationTokens = - hdfs.addDelegationTokens(TEST_RENEWER, null); + Token<?>[] delegationTokens = hdfs + .addDelegationTokens(TEST_RENEWER, null); for (Token<?> t : delegationTokens) { LOG.debug("got token " + t); } @@ -137,329 +140,113 @@ public class TestOfflineImageViewer { cluster.getNameNodeRpc() .setSafeMode(SafeModeAction.SAFEMODE_ENTER, false); cluster.getNameNodeRpc().saveNamespace(); - + // Determine location of fsimage file - originalFsimage = FSImageTestUtil.findLatestImageFile( - FSImageTestUtil.getFSImage( - cluster.getNameNode()).getStorage().getStorageDir(0)); + originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil + .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0)); if (originalFsimage == null) { throw new RuntimeException("Didn't generate or can't find fsimage"); } LOG.debug("original FS image file is " + originalFsimage); } finally { - if(cluster != null) + if (cluster != null) cluster.shutdown(); } } - + @AfterClass public static void deleteOriginalFSImage() throws IOException { - if(originalFsimage != null && originalFsimage.exists()) { + if (originalFsimage != null && originalFsimage.exists()) { originalFsimage.delete(); } } - - // Convenience method to generate a file status from file system for + + // Convenience method to generate a file status from file system for // later comparison - private static FileStatus pathToFileEntry(FileSystem hdfs, String file) - throws IOException { + private static FileStatus pathToFileEntry(FileSystem hdfs, String file) + throws IOException { return hdfs.getFileStatus(new Path(file)); } - - // Verify that we can correctly generate an ls-style output for a valid + + // Verify that we can correctly generate an ls-style output for a valid // fsimage @Test public void outputOfLSVisitor() throws IOException { - File testFile = new File(ROOT, "/basicCheck"); - File outputFile = new File(ROOT, "/basicCheckOutput"); - - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - - ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); - - oiv.go(); - - HashMap<String, LsElements> fileOutput = readLsfile(outputFile); - - compareNamespaces(writtenFiles, fileOutput); - } finally { - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); - } - LOG.debug("Correctly generated ls-style output."); - } - - // Confirm that attempting to read an fsimage file with an unsupported - // layout results in an error - @Test - public void unsupportedFSLayoutVersion() throws IOException { - File testFile = new File(ROOT, "/invalidLayoutVersion"); - File outputFile = new File(ROOT, "invalidLayoutVersionOutput"); - - try { - int badVersionNum = -432; - changeLayoutVersion(originalFsimage, testFile, badVersionNum); - ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); - - try { - oiv.go(); - fail("Shouldn't be able to read invalid laytout version"); - } catch(IOException e) { - if(!e.getMessage().contains(Integer.toString(badVersionNum))) - throw e; // wasn't error we were expecting - LOG.debug("Correctly failed at reading bad image version."); - } - } finally { - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); - } + StringWriter output = new StringWriter(); + PrintWriter out = new PrintWriter(output); + LsrPBImage v = new LsrPBImage(new Configuration(), out); + v.visit(new RandomAccessFile(originalFsimage, "r")); + out.close(); + Pattern pattern = Pattern + .compile("([d\\-])([rwx\\-]{9})\\s*(-|\\d+)\\s*(\\w+)\\s*(\\w+)\\s*(\\d+)\\s*(\\d+)\\s*([\b/]+)"); + int count = 0; + for (String s : output.toString().split("\n")) { + Matcher m = pattern.matcher(s); + assertTrue(m.find()); + LsElements e = new LsElements(); + e.isDir = m.group(1).equals("d"); + e.perms = m.group(2); + e.replication = m.group(3).equals("-") ? 0 : Integer.parseInt(m.group(3)); + e.username = m.group(4); + e.groupname = m.group(5); + e.filesize = Long.parseLong(m.group(7)); + String path = m.group(8); + if (!path.equals("/")) { + compareFiles(writtenFiles.get(path), e); + } + ++count; + } + assertEquals(writtenFiles.size() + 1, count); + } + + @Test(expected = IOException.class) + public void testTruncatedFSImage() throws IOException { + File truncatedFile = folder.newFile(); + StringWriter output = new StringWriter(); + copyPartOfFile(originalFsimage, truncatedFile); + new FileDistributionCalculator(new Configuration(), 0, 0, new PrintWriter( + output)).visit(new RandomAccessFile(truncatedFile, "r")); } - - // Verify that image viewer will bail on a file that ends unexpectedly - @Test - public void truncatedFSImage() throws IOException { - File testFile = new File(ROOT, "/truncatedFSImage"); - File outputFile = new File(ROOT, "/trucnatedFSImageOutput"); - try { - copyPartOfFile(originalFsimage, testFile); - assertTrue("Created truncated fsimage", testFile.exists()); - - ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); - - try { - oiv.go(); - fail("Managed to process a truncated fsimage file"); - } catch (EOFException e) { - LOG.debug("Correctly handled EOF"); - } - } finally { - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); - } - } - - // Test that our ls file has all the same compenents of the original namespace - private void compareNamespaces(HashMap<String, FileStatus> written, - HashMap<String, LsElements> fileOutput) { - assertEquals( "Should be the same number of files in both, plus one for root" - + " in fileoutput", fileOutput.keySet().size(), - written.keySet().size() + 1); - Set<String> inFile = fileOutput.keySet(); - - // For each line in the output file, verify that the namespace had a - // filestatus counterpart - for (String path : inFile) { - if (path.equals("/")) // root's not included in output from system call - continue; - - assertTrue("Path in file (" + path + ") was written to fs", written - .containsKey(path)); - - compareFiles(written.get(path), fileOutput.get(path)); - - written.remove(path); - } - - assertEquals("No more files were written to fs", 0, written.size()); - } - // Compare two files as listed in the original namespace FileStatus and // the output of the ls file from the image processor private void compareFiles(FileStatus fs, LsElements elements) { - assertEquals("directory listed as such", - fs.isDirectory() ? 'd' : '-', elements.dir); - assertEquals("perms string equal", - fs.getPermission().toString(), elements.perms); + assertEquals("directory listed as such", fs.isDirectory(), elements.isDir); + assertEquals("perms string equal", fs.getPermission().toString(), + elements.perms); assertEquals("replication equal", fs.getReplication(), elements.replication); assertEquals("owner equal", fs.getOwner(), elements.username); assertEquals("group equal", fs.getGroup(), elements.groupname); assertEquals("lengths equal", fs.getLen(), elements.filesize); } - // Read the contents of the file created by the Ls processor - private HashMap<String, LsElements> readLsfile(File lsFile) throws IOException { - BufferedReader br = new BufferedReader(new FileReader(lsFile)); - String line = null; - HashMap<String, LsElements> fileContents = new HashMap<String, LsElements>(); - - while((line = br.readLine()) != null) - readLsLine(line, fileContents); - - br.close(); - return fileContents; - } - - // Parse a line from the ls output. Store permissions, replication, - // username, groupname and filesize in hashmap keyed to the path name - private void readLsLine(String line, HashMap<String, LsElements> fileContents) { - String elements [] = line.split("\\s+"); - - assertEquals("Not enough elements in ls output", 8, elements.length); - - LsElements lsLine = new LsElements(); - - lsLine.dir = elements[0].charAt(0); - lsLine.perms = elements[0].substring(1); - lsLine.replication = elements[1].equals("-") - ? 0 : Integer.valueOf(elements[1]); - lsLine.username = elements[2]; - lsLine.groupname = elements[3]; - lsLine.filesize = Long.valueOf(elements[4]); - // skipping date and time - - String path = elements[7]; - - // Check that each file in the ls output was listed once - assertFalse("LS file had duplicate file entries", - fileContents.containsKey(path)); - - fileContents.put(path, lsLine); - } - - // Copy one fsimage to another, changing the layout version in the process - private void changeLayoutVersion(File src, File dest, int newVersion) - throws IOException { - DataInputStream in = null; - DataOutputStream out = null; - - try { - in = new DataInputStream(new FileInputStream(src)); - out = new DataOutputStream(new FileOutputStream(dest)); - - in.readInt(); - out.writeInt(newVersion); - - byte [] b = new byte[1024]; - while( in.read(b) > 0 ) { - out.write(b); - } - } finally { - if(in != null) in.close(); - if(out != null) out.close(); - } - } - - // Only copy part of file into the other. Used for testing truncated fsimage private void copyPartOfFile(File src, File dest) throws IOException { - InputStream in = null; - OutputStream out = null; - - byte [] b = new byte[256]; - int bytesWritten = 0; - int count; - int maxBytes = 700; - + FileInputStream in = null; + FileOutputStream out = null; + final int MAX_BYTES = 700; try { in = new FileInputStream(src); out = new FileOutputStream(dest); - - while( (count = in.read(b)) > 0 && bytesWritten < maxBytes ) { - out.write(b); - bytesWritten += count; - } + in.getChannel().transferTo(0, MAX_BYTES, out.getChannel()); } finally { - if(in != null) in.close(); - if(out != null) out.close(); + IOUtils.cleanup(null, in); + IOUtils.cleanup(null, out); } } @Test - public void outputOfFileDistributionVisitor() throws IOException { - File testFile = new File(ROOT, "/basicCheck"); - File outputFile = new File(ROOT, "/fileDistributionCheckOutput"); + public void testFileDistributionVisitor() throws IOException { + StringWriter output = new StringWriter(); + PrintWriter o = new PrintWriter(output); + new FileDistributionCalculator(new Configuration(), 0, 0, o) + .visit(new RandomAccessFile(originalFsimage, "r")); + o.close(); - int totalFiles = 0; - BufferedReader reader = null; - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - ImageVisitor v = new FileDistributionVisitor(outputFile.getPath(), 0, 0); - OfflineImageViewer oiv = - new OfflineImageViewer(testFile.getPath(), v, false); - - oiv.go(); - - reader = new BufferedReader(new FileReader(outputFile)); - String line = reader.readLine(); - assertEquals(line, "Size\tNumFiles"); - while((line = reader.readLine()) != null) { - String[] row = line.split("\t"); - assertEquals(row.length, 2); - totalFiles += Integer.parseInt(row[1]); - } - } finally { - if (reader != null) { - reader.close(); - } - if(testFile.exists()) testFile.delete(); - if(outputFile.exists()) outputFile.delete(); - } - assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR); - } - - private static class TestImageVisitor extends ImageVisitor { - private List<String> delegationTokenRenewers = new LinkedList<String>(); - TestImageVisitor() { - } - - List<String> getDelegationTokenRenewers() { - return delegationTokenRenewers; - } - - @Override - void start() throws IOException { - } - - @Override - void finish() throws IOException { - } - - @Override - void finishAbnormally() throws IOException { - } + Pattern p = Pattern.compile("totalFiles = (\\d+)\n"); + Matcher matcher = p.matcher(output.getBuffer()); - @Override - void visit(ImageElement element, String value) throws IOException { - if (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER) { - delegationTokenRenewers.add(value); - } - } - - @Override - void visitEnclosingElement(ImageElement element) throws IOException { - } - - @Override - void visitEnclosingElement(ImageElement element, ImageElement key, - String value) throws IOException { - } - - @Override - void leaveEnclosingElement() throws IOException { - } - } - - @Test - public void outputOfTestVisitor() throws IOException { - File testFile = new File(ROOT, "/basicCheck"); - - try { - DFSTestUtil.copyFile(originalFsimage, testFile); - TestImageVisitor v = new TestImageVisitor(); - OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, true); - oiv.go(); - - // Validated stored delegation token identifiers. - List<String> dtrs = v.getDelegationTokenRenewers(); - assertEquals(1, dtrs.size()); - assertEquals(TEST_RENEWER, dtrs.get(0)); - } finally { - if(testFile.exists()) testFile.delete(); - } - LOG.debug("Passed TestVisitor validation."); + assertTrue(matcher.find() && matcher.groupCount() == 1); + int totalFiles = Integer.parseInt(matcher.group(1)); + assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR); } } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpFileSystem.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpFileSystem.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpFileSystem.java Fri Feb 14 18:32:37 2014 @@ -136,6 +136,7 @@ public class TestHftpFileSystem { out.close(); FSDataInputStream in = hftpFs.open(p); assertEquals('0', in.read()); + in.close(); // Check the file status matches the path. Hftp returns a FileStatus // with the entire URI, extract the path part. @@ -250,6 +251,7 @@ public class TestHftpFileSystem { FSDataInputStream in = hftpFs.open(testFile); in.seek(7); assertEquals('7', in.read()); + in.close(); } @Test Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java?rev=1568437&r1=1568436&r2=1568437&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java Fri Feb 14 18:32:37 2014 @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.web; import java.io.File; +import java.io.InputStream; import java.net.InetSocketAddress; import java.net.URI; @@ -92,6 +93,9 @@ public class TestHttpsFileSystem { os.write(23); os.close(); Assert.assertTrue(fs.exists(f)); + InputStream is = fs.open(f); + Assert.assertEquals(23, is.read()); + is.close(); fs.close(); } }
