Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java Thu Oct 11 06:14:26 2012 @@ -22,6 +22,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.ArrayList; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; @@ -29,7 +30,9 @@ import javax.servlet.http.HttpServletReq import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer; import org.apache.hadoop.hdfs.web.resources.DoAsParam; import org.apache.hadoop.hdfs.web.resources.UserParam; @@ -399,4 +402,43 @@ public class TestJspHelper { ugi.getAuthenticationMethod()); } } + + @Test + public void testSortNodeByFields() throws Exception { + DatanodeID dnId1 = new DatanodeID("127.0.0.1", "localhost1", "storage1", + 1234, 2345, 3456); + DatanodeID dnId2 = new DatanodeID("127.0.0.2", "localhost2", "storage2", + 1235, 2346, 3457); + DatanodeDescriptor dnDesc1 = new DatanodeDescriptor(dnId1, "rack1", 1024, + 100, 924, 100, 10, 2); + DatanodeDescriptor dnDesc2 = new DatanodeDescriptor(dnId2, "rack2", 2500, + 200, 1848, 200, 20, 1); + ArrayList<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>(); + live.add(dnDesc1); + live.add(dnDesc2); + + // Test sorting by failed volumes + JspHelper.sortNodeList(live, "volfails", "ASC"); + Assert.assertEquals(dnDesc2, live.get(0)); + Assert.assertEquals(dnDesc1, live.get(1)); + JspHelper.sortNodeList(live, "volfails", "DSC"); + Assert.assertEquals(dnDesc1, live.get(0)); + Assert.assertEquals(dnDesc2, live.get(1)); + + // Test sorting by Blockpool used + JspHelper.sortNodeList(live, "bpused", "ASC"); + Assert.assertEquals(dnDesc1, live.get(0)); + Assert.assertEquals(dnDesc2, live.get(1)); + JspHelper.sortNodeList(live, "bpused", "DSC"); + Assert.assertEquals(dnDesc2, live.get(0)); + Assert.assertEquals(dnDesc1, live.get(1)); + + // Test sorting by Percentage Blockpool used + JspHelper.sortNodeList(live, "pcbpused", "ASC"); + Assert.assertEquals(dnDesc2, live.get(0)); + Assert.assertEquals(dnDesc1, live.get(1)); + JspHelper.sortNodeList(live, "pcbpused", "DSC"); + Assert.assertEquals(dnDesc1, live.get(0)); + Assert.assertEquals(dnDesc2, live.get(1)); + } }
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java Thu Oct 11 06:14:26 2012 @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNT import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.util.StringUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -216,48 +217,62 @@ public class TestDataNodeMultipleRegistr LOG.info("dn bpos len (still should be 3):" + bposs.length); Assert.assertEquals("should've registered with three namenodes", 3, bposs.length); } finally { - if(cluster != null) - cluster.shutdown(); + cluster.shutdown(); } } @Test public void testMiniDFSClusterWithMultipleNN() throws IOException { - Configuration conf = new HdfsConfiguration(); // start Federated cluster and add a node. MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)) .build(); - Assert.assertNotNull(cluster); - Assert.assertEquals("(1)Should be 2 namenodes", 2, cluster.getNumNameNodes()); // add a node - cluster.addNameNode(conf, 0); - Assert.assertEquals("(1)Should be 3 namenodes", 3, cluster.getNumNameNodes()); - cluster.shutdown(); + try { + Assert.assertNotNull(cluster); + cluster.waitActive(); + Assert.assertEquals("(1)Should be 2 namenodes", 2, cluster.getNumNameNodes()); + + cluster.addNameNode(conf, 0); + Assert.assertEquals("(1)Should be 3 namenodes", 3, cluster.getNumNameNodes()); + } catch (IOException ioe) { + Assert.fail("Failed to add NN to cluster:" + StringUtils.stringifyException(ioe)); + } finally { + cluster.shutdown(); + } // 2. start with Federation flag set conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(1)) .build(); - Assert.assertNotNull(cluster); - Assert.assertEquals("(2)Should be 1 namenodes", 1, cluster.getNumNameNodes()); - // add a node - cluster.addNameNode(conf, 0); - Assert.assertEquals("(2)Should be 2 namenodes", 2, cluster.getNumNameNodes()); - cluster.shutdown(); + try { + Assert.assertNotNull(cluster); + cluster.waitActive(); + Assert.assertEquals("(2)Should be 1 namenodes", 1, cluster.getNumNameNodes()); + + // add a node + cluster.addNameNode(conf, 0); + Assert.assertEquals("(2)Should be 2 namenodes", 2, cluster.getNumNameNodes()); + } catch (IOException ioe) { + Assert.fail("Failed to add NN to cluster:" + StringUtils.stringifyException(ioe)); + } finally { + cluster.shutdown(); + } // 3. start non-federated conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).build(); - Assert.assertNotNull(cluster); - Assert.assertEquals("(2)Should be 1 namenodes", 1, cluster.getNumNameNodes()); // add a node try { + cluster.waitActive(); + Assert.assertNotNull(cluster); + Assert.assertEquals("(2)Should be 1 namenodes", 1, cluster.getNumNameNodes()); + cluster.addNameNode(conf, 9929); Assert.fail("shouldn't be able to add another NN to non federated cluster"); } catch (IOException e) { @@ -268,6 +283,4 @@ public class TestDataNodeMultipleRegistr cluster.shutdown(); } } - - } Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Thu Oct 11 06:14:26 2012 @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -64,6 +65,10 @@ public class TestBackupNode { } static final String BASE_DIR = MiniDFSCluster.getBaseDirectory(); + + static final long seed = 0xDEADBEEFL; + static final int blockSize = 4096; + static final int fileSize = 8192; @Before public void setUp() throws Exception { @@ -350,14 +355,17 @@ public class TestBackupNode { + NetUtils.getHostPortString(add)).toUri(), conf); boolean canWrite = true; try { - TestCheckpoint.writeFile(bnFS, file3, replication); + DFSTestUtil.createFile(bnFS, file3, fileSize, fileSize, blockSize, + replication, seed); } catch (IOException eio) { LOG.info("Write to BN failed as expected: ", eio); canWrite = false; } assertFalse("Write to BackupNode must be prohibited.", canWrite); - TestCheckpoint.writeFile(fileSys, file3, replication); + DFSTestUtil.createFile(fileSys, file3, fileSize, fileSize, blockSize, + replication, seed); + TestCheckpoint.checkFile(fileSys, file3, replication); // should also be on BN right away assertTrue("file3 does not exist on BackupNode", Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Thu Oct 11 06:14:26 2012 @@ -28,22 +28,21 @@ import static org.junit.Assert.assertTru import static org.junit.Assert.fail; import java.io.File; -import java.io.FilenameFilter; import java.io.IOException; import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Random; import org.apache.commons.cli.ParseException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; @@ -51,6 +50,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -76,6 +76,7 @@ import org.apache.hadoop.test.GenericTes import org.apache.hadoop.test.GenericTestUtils.LogCapturer; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Level; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatcher; @@ -116,19 +117,22 @@ public class TestCheckpoint { faultInjector = Mockito.mock(CheckpointFaultInjector.class); CheckpointFaultInjector.instance = faultInjector; } - - static void writeFile(FileSystem fileSys, Path name, int repl) - throws IOException { - FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf() - .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), - (short) repl, blockSize); - byte[] buffer = new byte[TestCheckpoint.fileSize]; - Random rand = new Random(TestCheckpoint.seed); - rand.nextBytes(buffer); - stm.write(buffer); - stm.close(); - } + @After + public void checkForSNNThreads() { + ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + + ThreadInfo[] infos = threadBean.getThreadInfo(threadBean.getAllThreadIds(), 20); + for (ThreadInfo info : infos) { + if (info == null) continue; + LOG.info("Check thread: " + info.getThreadName()); + if (info.getThreadName().contains("SecondaryNameNode")) { + fail("Leaked thread: " + info + "\n" + + Joiner.on("\n").join(info.getStackTrace())); + } + } + LOG.info("--------"); + } static void checkFile(FileSystem fileSys, Path name, int repl) throws IOException { @@ -259,7 +263,8 @@ public class TestCheckpoint { // // Create a new file // - writeFile(fileSys, file1, replication); + DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize, + replication, seed); checkFile(fileSys, file1, replication); } finally { fileSys.close(); @@ -323,7 +328,8 @@ public class TestCheckpoint { // // Create a new file // - writeFile(fileSys, file1, replication); + DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize, + replication, seed); checkFile(fileSys, file1, replication); } finally { fileSys.close(); @@ -394,7 +400,8 @@ public class TestCheckpoint { // // Create a new file // - writeFile(fileSys, file1, replication); + DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize, + replication, seed); checkFile(fileSys, file1, replication); } finally { fileSys.close(); @@ -580,7 +587,8 @@ public class TestCheckpoint { // // Create a new file // - writeFile(fileSys, file1, replication); + DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize, + replication, seed); checkFile(fileSys, file1, replication); } finally { fileSys.close(); @@ -906,7 +914,8 @@ public class TestCheckpoint { // // Create file1 // - writeFile(fileSys, file1, replication); + DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize, + replication, seed); checkFile(fileSys, file1, replication); // @@ -933,7 +942,8 @@ public class TestCheckpoint { cleanupFile(fileSys, file1); // create new file file2 - writeFile(fileSys, file2, replication); + DFSTestUtil.createFile(fileSys, file2, fileSize, fileSize, blockSize, + replication, seed); checkFile(fileSys, file2, replication); // @@ -999,7 +1009,8 @@ public class TestCheckpoint { } // create new file Path file = new Path("namespace.dat"); - writeFile(fs, file, replication); + DFSTestUtil.createFile(fs, file, fileSize, fileSize, blockSize, + replication, seed); checkFile(fs, file, replication); // create new link @@ -1746,7 +1757,7 @@ public class TestCheckpoint { /** * Test that the 2NN triggers a checkpoint after the configurable interval */ - @Test + @Test(timeout=30000) public void testCheckpointTriggerOnTxnCount() throws Exception { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; @@ -1760,8 +1771,7 @@ public class TestCheckpoint { .format(true).build(); FileSystem fs = cluster.getFileSystem(); secondary = startSecondaryNameNode(conf); - Thread t = new Thread(secondary); - t.start(); + secondary.startCheckpointThread(); final NNStorage storage = secondary.getFSImage().getStorage(); // 2NN should checkpoint at startup Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java Thu Oct 11 06:14:26 2012 @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -116,19 +117,6 @@ public class TestDecommissioningStatus { stm.close(); } - private void writeFile(FileSystem fileSys, Path name, short repl) - throws IOException { - // create and write a file that contains three blocks of data - FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf() - .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), repl, - blockSize); - byte[] buffer = new byte[fileSize]; - Random rand = new Random(seed); - rand.nextBytes(buffer); - stm.write(buffer); - stm.close(); - } - private FSDataOutputStream writeIncompleteFile(FileSystem fileSys, Path name, short repl) throws IOException { // create and write a file that contains three blocks of data @@ -198,7 +186,8 @@ public class TestDecommissioningStatus { // Decommission one node. Verify the decommission status // Path file1 = new Path("decommission.dat"); - writeFile(fileSys, file1, replicas); + DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize, + replicas, seed); Path file2 = new Path("decommission1.dat"); FSDataOutputStream st1 = writeIncompleteFile(fileSys, file2, replicas); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java Thu Oct 11 06:14:26 2012 @@ -20,14 +20,12 @@ package org.apache.hadoop.hdfs.server.na import static org.junit.Assert.assertTrue; import java.io.IOException; -import java.util.Random; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; @@ -43,19 +41,6 @@ public class TestFileLimit { static final int blockSize = 8192; boolean simulatedStorage = false; - // creates a zero file. - private void createFile(FileSystem fileSys, Path name) - throws IOException { - FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf() - .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), - (short) 1, blockSize); - byte[] buffer = new byte[1024]; - Random rand = new Random(seed); - rand.nextBytes(buffer); - stm.write(buffer); - stm.close(); - } - private void waitForLimit(FSNamesystem namesys, long num) { // wait for number of blocks to decrease @@ -106,7 +91,7 @@ public class TestFileLimit { // for (int i = 0; i < maxObjects/2; i++) { Path file = new Path("/filestatus" + i); - createFile(fs, file); + DFSTestUtil.createFile(fs, file, 1024, 1024, blockSize, (short) 1, seed); System.out.println("Created file " + file); currentNodes += 2; // two more objects for this creation. } @@ -115,7 +100,7 @@ public class TestFileLimit { boolean hitException = false; try { Path file = new Path("/filestatus"); - createFile(fs, file); + DFSTestUtil.createFile(fs, file, 1024, 1024, blockSize, (short) 1, seed); System.out.println("Created file " + file); } catch (IOException e) { hitException = true; @@ -132,7 +117,7 @@ public class TestFileLimit { waitForLimit(namesys, currentNodes); // now, we shud be able to create a new file - createFile(fs, file0); + DFSTestUtil.createFile(fs, file0, 1024, 1024, blockSize, (short) 1, seed); System.out.println("Created file " + file0 + " again."); currentNodes += 2; Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java Thu Oct 11 06:14:26 2012 @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FSDataOutput import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.AfterClass; @@ -48,17 +49,6 @@ public class TestMetaSave { private static MiniDFSCluster cluster = null; private static FileSystem fileSys = null; - private void createFile(FileSystem fileSys, Path name) throws IOException { - FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf() - .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), - (short) 2, blockSize); - byte[] buffer = new byte[1024]; - Random rand = new Random(seed); - rand.nextBytes(buffer); - stm.write(buffer); - stm.close(); - } - @BeforeClass public static void setUp() throws IOException { // start a cluster @@ -84,7 +74,8 @@ public class TestMetaSave { for (int i = 0; i < 2; i++) { Path file = new Path("/filestatus" + i); - createFile(fileSys, file); + DFSTestUtil.createFile(fileSys, file, 1024, 1024, blockSize, (short) 2, + seed); } cluster.stopDataNode(1); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameEditsConfigs.java Thu Oct 11 06:14:26 2012 @@ -25,17 +25,15 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; import java.util.List; -import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; @@ -71,18 +69,6 @@ public class TestNameEditsConfigs { } } - private void writeFile(FileSystem fileSys, Path name, int repl) - throws IOException { - FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf() - .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), - (short) repl, BLOCK_SIZE); - byte[] buffer = new byte[FILE_SIZE]; - Random rand = new Random(SEED); - rand.nextBytes(buffer); - stm.write(buffer); - stm.close(); - } - void checkImageAndEditsFilesExistence(File dir, boolean shouldHaveImages, boolean shouldHaveEdits) @@ -187,7 +173,8 @@ public class TestNameEditsConfigs { try { assertTrue(!fileSys.exists(file1)); - writeFile(fileSys, file1, replication); + DFSTestUtil.createFile(fileSys, file1, FILE_SIZE, FILE_SIZE, BLOCK_SIZE, + (short) replication, SEED); checkFile(fileSys, file1, replication); secondary.doCheckpoint(); } finally { @@ -224,7 +211,8 @@ public class TestNameEditsConfigs { assertTrue(fileSys.exists(file1)); checkFile(fileSys, file1, replication); cleanupFile(fileSys, file1); - writeFile(fileSys, file2, replication); + DFSTestUtil.createFile(fileSys, file2, FILE_SIZE, FILE_SIZE, BLOCK_SIZE, + (short) replication, SEED); checkFile(fileSys, file2, replication); secondary.doCheckpoint(); } finally { @@ -260,7 +248,8 @@ public class TestNameEditsConfigs { assertTrue(fileSys.exists(file2)); checkFile(fileSys, file2, replication); cleanupFile(fileSys, file2); - writeFile(fileSys, file3, replication); + DFSTestUtil.createFile(fileSys, file3, FILE_SIZE, FILE_SIZE, BLOCK_SIZE, + (short) replication, SEED); checkFile(fileSys, file3, replication); secondary.doCheckpoint(); } finally { @@ -364,7 +353,8 @@ public class TestNameEditsConfigs { fileSys = cluster.getFileSystem(); assertTrue(!fileSys.exists(file1)); - writeFile(fileSys, file1, replication); + DFSTestUtil.createFile(fileSys, file1, FILE_SIZE, FILE_SIZE, BLOCK_SIZE, + (short) replication, SEED); checkFile(fileSys, file1, replication); } finally { fileSys.close(); @@ -402,7 +392,8 @@ public class TestNameEditsConfigs { assertTrue(fileSys.exists(file1)); checkFile(fileSys, file1, replication); cleanupFile(fileSys, file1); - writeFile(fileSys, file2, replication); + DFSTestUtil.createFile(fileSys, file2, FILE_SIZE, FILE_SIZE, BLOCK_SIZE, + (short) replication, SEED); checkFile(fileSys, file2, replication); } finally { fileSys.close(); @@ -429,7 +420,8 @@ public class TestNameEditsConfigs { assertTrue(fileSys.exists(file2)); checkFile(fileSys, file2, replication); cleanupFile(fileSys, file2); - writeFile(fileSys, file3, replication); + DFSTestUtil.createFile(fileSys, file3, FILE_SIZE, FILE_SIZE, BLOCK_SIZE, + (short) replication, SEED); checkFile(fileSys, file3, replication); } finally { fileSys.close(); @@ -483,7 +475,8 @@ public class TestNameEditsConfigs { assertTrue(fileSys.exists(file3)); checkFile(fileSys, file3, replication); cleanupFile(fileSys, file3); - writeFile(fileSys, file3, replication); + DFSTestUtil.createFile(fileSys, file3, FILE_SIZE, FILE_SIZE, BLOCK_SIZE, + (short) replication, SEED); checkFile(fileSys, file3, replication); } finally { fileSys.close(); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java Thu Oct 11 06:14:26 2012 @@ -77,20 +77,6 @@ public class TestStartup { static final int fileSize = 8192; private long editsLength=0, fsimageLength=0; - - private void writeFile(FileSystem fileSys, Path name, int repl) - throws IOException { - FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf() - .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), - (short) repl, blockSize); - byte[] buffer = new byte[fileSize]; - Random rand = new Random(seed); - rand.nextBytes(buffer); - stm.write(buffer); - stm.close(); - } - - @Before public void setUp() throws Exception { config = new HdfsConfiguration(); @@ -150,7 +136,8 @@ public class TestStartup { // create a file FileSystem fileSys = cluster.getFileSystem(); Path file1 = new Path("t1"); - this.writeFile(fileSys, file1, 1); + DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize, + (short) 1, seed); LOG.info("--doing checkpoint"); sn.doCheckpoint(); // this shouldn't fail Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java Thu Oct 11 06:14:26 2012 @@ -166,7 +166,7 @@ public class TestPermission { } @Test - public void testFilePermision() throws Exception { + public void testFilePermission() throws Exception { final Configuration conf = new HdfsConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); @@ -244,6 +244,10 @@ public class TestPermission { fs.mkdirs(p); return true; } catch(AccessControlException e) { + // We check that AccessControlExceptions contain absolute paths. + Path parent = p.getParent(); + assertTrue(parent.isUriPathAbsolute()); + assertTrue(e.getMessage().contains(parent.toString())); return false; } } @@ -253,6 +257,9 @@ public class TestPermission { fs.create(p); return true; } catch(AccessControlException e) { + Path parent = p.getParent(); + assertTrue(parent.isUriPathAbsolute()); + assertTrue(e.getMessage().contains(parent.toString())); return false; } } @@ -262,6 +269,8 @@ public class TestPermission { fs.open(p); return true; } catch(AccessControlException e) { + assertTrue(p.isUriPathAbsolute()); + assertTrue(e.getMessage().contains(p.toString())); return false; } } @@ -272,6 +281,9 @@ public class TestPermission { fs.rename(src, dst); return true; } catch(AccessControlException e) { + Path parent = dst.getParent(); + assertTrue(parent.isUriPathAbsolute()); + assertTrue(e.getMessage().contains(parent.toString())); return false; } } Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java Thu Oct 11 06:14:26 2012 @@ -36,9 +36,7 @@ import org.apache.hadoop.hdfs.security.t import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenRenewer; import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; @@ -50,7 +48,6 @@ public class TestDelegationTokenFetcher private Configuration conf; private URI uri; private static final String SERVICE_VALUE = "localhost:2005"; - private static final Text KIND = new Text("TESTING-TOKEN-KIND"); private static String tokenFile = "file.dta"; @Before @@ -61,37 +58,6 @@ public class TestDelegationTokenFetcher FileSystemTestHelper.addFileSystemForTesting(uri, conf, dfs); } - public static class FakeRenewer extends TokenRenewer { - static Token<?> lastRenewed = null; - static Token<?> lastCanceled = null; - - @Override - public boolean handleKind(Text kind) { - return KIND.equals(kind); - } - - @Override - public boolean isManaged(Token<?> token) throws IOException { - return true; - } - - @Override - public long renew(Token<?> token, Configuration conf) { - lastRenewed = token; - return 0; - } - - @Override - public void cancel(Token<?> token, Configuration conf) { - lastCanceled = token; - } - - public static void reset() { - lastRenewed = null; - lastCanceled = null; - } - } - /** * Verify that when the DelegationTokenFetcher runs, it talks to the Namenode, * pulls out the correct user's token and successfully serializes it to disk. @@ -103,13 +69,11 @@ public class TestDelegationTokenFetcher new Text("renewer"), new Text("realuser")).getBytes(); final byte[] pw = new byte[] { 42 }; final Text service = new Text(uri.toString()); - final String user = - UserGroupInformation.getCurrentUser().getShortUserName(); // Create a token for the fetcher to fetch, wire NN to return it when asked // for this particular user. final Token<DelegationTokenIdentifier> t = - new Token<DelegationTokenIdentifier>(ident, pw, KIND, service); + new Token<DelegationTokenIdentifier>(ident, pw, FakeRenewer.KIND, service); when(dfs.addDelegationTokens(eq((String) null), any(Credentials.class))).thenAnswer( new Answer<Token<?>[]>() { @Override Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java Thu Oct 11 06:14:26 2012 @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FSDataOutput import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.tools.JMXGet; @@ -53,19 +54,6 @@ public class TestJMXGet { static final int blockSize = 4096; static final int fileSize = 8192; - private void writeFile(FileSystem fileSys, Path name, int repl) - throws IOException { - FSDataOutputStream stm = fileSys.create(name, true, - fileSys.getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), - (short)repl, blockSize); - byte[] buffer = new byte[fileSize]; - Random rand = new Random(seed); - rand.nextBytes(buffer); - stm.write(buffer); - stm.close(); - } - - @Before public void setUp() throws Exception { config = new HdfsConfiguration(); @@ -96,7 +84,8 @@ public class TestJMXGet { cluster = new MiniDFSCluster.Builder(config).numDataNodes(numDatanodes).build(); cluster.waitActive(); - writeFile(cluster.getFileSystem(), new Path("/test1"), 2); + DFSTestUtil.createFile(cluster.getFileSystem(), new Path("/test1"), + fileSize, fileSize, blockSize, (short) 2, seed); JMXGet jmx = new JMXGet(); //jmx.setService("*"); // list all hadoop services @@ -125,7 +114,8 @@ public class TestJMXGet { cluster = new MiniDFSCluster.Builder(config).numDataNodes(numDatanodes).build(); cluster.waitActive(); - writeFile(cluster.getFileSystem(), new Path("/test"), 2); + DFSTestUtil.createFile(cluster.getFileSystem(), new Path("/test"), + fileSize, fileSize, blockSize, (short) 2, seed); JMXGet jmx = new JMXGet(); //jmx.setService("*"); // list all hadoop services Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer?rev=1396918&r1=1396917&r2=1396918&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer Thu Oct 11 06:14:26 2012 @@ -1 +1 @@ -org.apache.hadoop.tools.TestDelegationTokenFetcher$FakeRenewer +org.apache.hadoop.tools.FakeRenewer
