Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java Sun Nov 10 20:09:09 2013 @@ -203,6 +203,9 @@ public class SnapshotFSImageFormat { // useful, but set the parent here to be consistent with the original // fsdir tree. deleted.setParent(parent); + if (deleted.isFile()) { + loader.updateBlocksMap(deleted.asFile()); + } } return deletedList; }
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgress.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgress.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgress.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgress.java Sun Nov 10 20:09:09 2013 @@ -149,8 +149,8 @@ public class StartupProgress { * @return Counter associated with phase and step */ public Counter getCounter(Phase phase, Step step) { - final StepTracking tracking = lazyInitStep(phase, step); if (!isComplete()) { + final StepTracking tracking = lazyInitStep(phase, step); return new Counter() { @Override public void increment() { Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java Sun Nov 10 20:09:09 2013 @@ -21,9 +21,12 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; /** * A tool used to list all snapshottable directories that are owned by the @@ -31,23 +34,23 @@ import org.apache.hadoop.hdfs.protocol.S * is a super user. */ @InterfaceAudience.Private -public class LsSnapshottableDir { - public static void main(String[] argv) throws IOException { +public class LsSnapshottableDir extends Configured implements Tool { + @Override + public int run(String[] argv) throws Exception { String description = "LsSnapshottableDir: \n" + "\tGet the list of snapshottable directories that are owned by the current user.\n" + "\tReturn all the snapshottable directories if the current user is a super user.\n"; if(argv.length != 0) { System.err.println("Usage: \n" + description); - System.exit(1); + return 1; } - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.get(conf); + FileSystem fs = FileSystem.get(getConf()); if (! (fs instanceof DistributedFileSystem)) { System.err.println( "LsSnapshottableDir can only be used in DistributedFileSystem"); - System.exit(1); + return 1; } DistributedFileSystem dfs = (DistributedFileSystem) fs; @@ -57,7 +60,12 @@ public class LsSnapshottableDir { } catch (IOException e) { String[] content = e.getLocalizedMessage().split("\n"); System.err.println("lsSnapshottableDir: " + content[0]); + return 1; } + return 0; + } + public static void main(String[] argv) throws Exception { + int rc = ToolRunner.run(new LsSnapshottableDir(), argv); + System.exit(rc); } - } Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Sun Nov 10 20:09:09 2013 @@ -684,19 +684,6 @@ public class WebHdfsFileSystem extends F } } - @VisibleForTesting - final class ConnRunner extends AbstractRunner { - protected ConnRunner(final HttpOpParam.Op op, HttpURLConnection conn) { - super(op, false); - this.conn = conn; - } - - @Override - protected URL getUrl() { - return null; - } - } - private FsPermission applyUMask(FsPermission permission) { if (permission == null) { permission = FsPermission.getDefault(); Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Sun Nov 10 20:09:09 2013 @@ -1459,4 +1459,29 @@ </description> </property> +<property> + <name>dfs.namenode.edit.log.autoroll.multiplier.threshold</name> + <value>2.0</value> + <description> + Determines when an active namenode will roll its own edit log. + The actual threshold (in number of edits) is determined by multiplying + this value by dfs.namenode.checkpoint.txns. + + This prevents extremely large edit files from accumulating on the active + namenode, which can cause timeouts during namenode startup and pose an + administrative hassle. This behavior is intended as a failsafe for when + the standby or secondary namenode fail to roll the edit log by the normal + checkpoint threshold. + </description> +</property> + +<property> + <name>dfs.namenode.edit.log.autoroll.check.interval.ms</name> + <value>300000</value> + <description> + How often an active namenode will check if it needs to roll its edit log, + in milliseconds. + </description> +</property> + </configuration> Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientExcludedNodes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientExcludedNodes.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientExcludedNodes.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientExcludedNodes.java Sun Nov 10 20:09:09 2013 @@ -32,6 +32,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.util.ThreadUtil; +import org.junit.After; +import org.junit.Before; import org.junit.Test; @@ -41,10 +43,25 @@ import org.junit.Test; */ public class TestDFSClientExcludedNodes { - @Test(timeout=10000) + private MiniDFSCluster cluster; + private Configuration conf; + + @Before + public void setUp() { + cluster = null; + conf = new HdfsConfiguration(); + } + + @After + public void tearDown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test(timeout=60000) public void testExcludedNodes() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); FileSystem fs = cluster.getFileSystem(); Path filePath = new Path("/testExcludedNodes"); @@ -67,17 +84,16 @@ public class TestDFSClientExcludedNodes } } - @Test(timeout=10000) + @Test(timeout=60000) public void testExcludedNodesForgiveness() throws IOException { - Configuration conf = new HdfsConfiguration(); - // Forgive nodes in under 1s for this test case. + // Forgive nodes in under 2.5s for this test case. conf.setLong( DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL, - 1000); + 2500); // We'll be using a 512 bytes block size just for tests // so making sure the checksum bytes too match it. conf.setInt("io.bytes.per.checksum", 512); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); List<DataNodeProperties> props = cluster.dataNodes; FileSystem fs = cluster.getFileSystem(); Path filePath = new Path("/testForgivingExcludedNodes"); @@ -112,11 +128,11 @@ public class TestDFSClientExcludedNodes Assert.assertEquals(true, cluster.restartDataNode(two, true)); cluster.waitActive(); - // Sleep for 2s, to let the excluded nodes be expired + // Sleep for 5s, to let the excluded nodes be expired // from the excludes list (i.e. forgiven after the configured wait period). - // [Sleeping just in case the restart of the DNs completed < 2s cause + // [Sleeping just in case the restart of the DNs completed < 5s cause // otherwise, we'll end up quickly excluding those again.] - ThreadUtil.sleepAtLeastIgnoreInterrupts(2000); + ThreadUtil.sleepAtLeastIgnoreInterrupts(5000); // Terminate the last good DN, to assert that there's no // single-DN-available scenario, caused by not forgiving the other Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java Sun Nov 10 20:09:09 2013 @@ -23,20 +23,14 @@ package org.apache.hadoop.hdfs.security; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; -import java.net.HttpURLConnection; import java.net.InetAddress; import java.net.NetworkInterface; -import java.net.URL; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Enumeration; -import java.util.Map; - -import javax.servlet.http.HttpServletResponse; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -47,23 +41,17 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; -import org.apache.hadoop.hdfs.web.resources.DoAsParam; -import org.apache.hadoop.hdfs.web.resources.ExceptionHandler; -import org.apache.hadoop.hdfs.web.resources.GetOpParam; -import org.apache.hadoop.hdfs.web.resources.PostOpParam; -import org.apache.hadoop.hdfs.web.resources.PutOpParam; import org.apache.hadoop.security.TestDoAsEffectiveUser; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.Token; -import org.apache.log4j.Level; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; public class TestDelegationTokenForProxyUser { private static MiniDFSCluster cluster; @@ -155,56 +143,26 @@ public class TestDelegationTokenForProxy } } - @Test(timeout=20000) + @Test(timeout=5000) public void testWebHdfsDoAs() throws Exception { WebHdfsTestUtil.LOG.info("START: testWebHdfsDoAs()"); - ((Log4JLogger)NamenodeWebHdfsMethods.LOG).getLogger().setLevel(Level.ALL); - ((Log4JLogger)ExceptionHandler.LOG).getLogger().setLevel(Level.ALL); WebHdfsTestUtil.LOG.info("ugi.getShortUserName()=" + ugi.getShortUserName()); final WebHdfsFileSystem webhdfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(ugi, config); final Path root = new Path("/"); cluster.getFileSystem().setPermission(root, new FsPermission((short)0777)); - { - //test GETHOMEDIRECTORY with doAs - final URL url = WebHdfsTestUtil.toUrl(webhdfs, - GetOpParam.Op.GETHOMEDIRECTORY, root, new DoAsParam(PROXY_USER)); - final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - final Map<?, ?> m = WebHdfsTestUtil.connectAndGetJson(conn, HttpServletResponse.SC_OK); - conn.disconnect(); - - final Object responsePath = m.get(Path.class.getSimpleName()); - WebHdfsTestUtil.LOG.info("responsePath=" + responsePath); - Assert.assertEquals("/user/" + PROXY_USER, responsePath); - } + Whitebox.setInternalState(webhdfs, "ugi", proxyUgi); { - //test GETHOMEDIRECTORY with DOas - final URL url = WebHdfsTestUtil.toUrl(webhdfs, - GetOpParam.Op.GETHOMEDIRECTORY, root, new DoAsParam(PROXY_USER) { - @Override - public String getName() { - return "DOas"; - } - }); - final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - final Map<?, ?> m = WebHdfsTestUtil.connectAndGetJson(conn, HttpServletResponse.SC_OK); - conn.disconnect(); - - final Object responsePath = m.get(Path.class.getSimpleName()); + Path responsePath = webhdfs.getHomeDirectory(); WebHdfsTestUtil.LOG.info("responsePath=" + responsePath); - Assert.assertEquals("/user/" + PROXY_USER, responsePath); + Assert.assertEquals(webhdfs.getUri() + "/user/" + PROXY_USER, responsePath.toString()); } final Path f = new Path("/testWebHdfsDoAs/a.txt"); { - //test create file with doAs - final PutOpParam.Op op = PutOpParam.Op.CREATE; - final URL url = WebHdfsTestUtil.toUrl(webhdfs, op, f, new DoAsParam(PROXY_USER)); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn = WebHdfsTestUtil.twoStepWrite(webhdfs, op, conn); - final FSDataOutputStream out = WebHdfsTestUtil.write(webhdfs, op, conn, 4096); + FSDataOutputStream out = webhdfs.create(f); out.write("Hello, webhdfs user!".getBytes()); out.close(); @@ -214,12 +172,7 @@ public class TestDelegationTokenForProxy } { - //test append file with doAs - final PostOpParam.Op op = PostOpParam.Op.APPEND; - final URL url = WebHdfsTestUtil.toUrl(webhdfs, op, f, new DoAsParam(PROXY_USER)); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn = WebHdfsTestUtil.twoStepWrite(webhdfs, op, conn); - final FSDataOutputStream out = WebHdfsTestUtil.write(webhdfs, op, conn, 4096); + final FSDataOutputStream out = webhdfs.append(f); out.write("\nHello again!".getBytes()); out.close(); Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java Sun Nov 10 20:09:09 2013 @@ -2243,4 +2243,50 @@ public class TestRenameWithSnapshots { restartClusterAndCheckImage(true); } + + /** + * Make sure we clean the whole subtree under a DstReference node after + * deleting a snapshot. + * see HDFS-5476. + */ + @Test + public void testCleanDstReference() throws Exception { + final Path test = new Path("/test"); + final Path foo = new Path(test, "foo"); + final Path bar = new Path(foo, "bar"); + hdfs.mkdirs(bar); + SnapshotTestHelper.createSnapshot(hdfs, test, "s0"); + + // create file after s0 so that the file should not be included in s0 + final Path fileInBar = new Path(bar, "file"); + DFSTestUtil.createFile(hdfs, fileInBar, BLOCKSIZE, REPL, SEED); + // rename foo --> foo2 + final Path foo2 = new Path(test, "foo2"); + hdfs.rename(foo, foo2); + // create snapshot s1, note the file is included in s1 + hdfs.createSnapshot(test, "s1"); + // delete bar and foo2 + hdfs.delete(new Path(foo2, "bar"), true); + hdfs.delete(foo2, true); + + final Path sfileInBar = SnapshotTestHelper.getSnapshotPath(test, "s1", + "foo2/bar/file"); + assertTrue(hdfs.exists(sfileInBar)); + + hdfs.deleteSnapshot(test, "s1"); + assertFalse(hdfs.exists(sfileInBar)); + + restartClusterAndCheckImage(true); + // make sure the file under bar is deleted + final Path barInS0 = SnapshotTestHelper.getSnapshotPath(test, "s0", + "foo/bar"); + INodeDirectoryWithSnapshot barNode = (INodeDirectoryWithSnapshot) fsdir + .getINode(barInS0.toString()); + assertEquals(0, barNode.getChildrenList(null).size()); + List<DirectoryDiff> diffList = barNode.getDiffs().asList(); + assertEquals(1, diffList.size()); + DirectoryDiff diff = diffList.get(0); + assertEquals(0, diff.getChildrenDiff().getList(ListType.DELETED).size()); + assertEquals(0, diff.getChildrenDiff().getList(ListType.CREATED).size()); + } } Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java Sun Nov 10 20:09:09 2013 @@ -21,6 +21,7 @@ import static org.apache.hadoop.test.Gen import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; @@ -31,11 +32,14 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -205,4 +209,187 @@ public class TestSnapshotBlocksMap { assertExceptionContains("File does not exist: " + s1f0, e); } } + + /* + * Try to read the files inside snapshot but deleted in original place after + * restarting post checkpoint. refer HDFS-5427 + */ + @Test(timeout = 30000) + public void testReadSnapshotFileWithCheckpoint() throws Exception { + Path foo = new Path("/foo"); + hdfs.mkdirs(foo); + hdfs.allowSnapshot(foo); + Path bar = new Path("/foo/bar"); + DFSTestUtil.createFile(hdfs, bar, 100, (short) 2, 100024L); + hdfs.createSnapshot(foo, "s1"); + assertTrue(hdfs.delete(bar, true)); + + // checkpoint + NameNode nameNode = cluster.getNameNode(); + NameNodeAdapter.enterSafeMode(nameNode, false); + NameNodeAdapter.saveNamespace(nameNode); + NameNodeAdapter.leaveSafeMode(nameNode); + + // restart namenode to load snapshot files from fsimage + cluster.restartNameNode(true); + String snapshotPath = Snapshot.getSnapshotPath(foo.toString(), "s1/bar"); + DFSTestUtil.readFile(hdfs, new Path(snapshotPath)); + } + + /* + * Try to read the files inside snapshot but renamed to different file and + * deleted after restarting post checkpoint. refer HDFS-5427 + */ + @Test(timeout = 30000) + public void testReadRenamedSnapshotFileWithCheckpoint() throws Exception { + final Path foo = new Path("/foo"); + final Path foo2 = new Path("/foo2"); + hdfs.mkdirs(foo); + hdfs.mkdirs(foo2); + + hdfs.allowSnapshot(foo); + hdfs.allowSnapshot(foo2); + final Path bar = new Path(foo, "bar"); + final Path bar2 = new Path(foo2, "bar"); + DFSTestUtil.createFile(hdfs, bar, 100, (short) 2, 100024L); + hdfs.createSnapshot(foo, "s1"); + // rename to another snapshottable directory and take snapshot + assertTrue(hdfs.rename(bar, bar2)); + hdfs.createSnapshot(foo2, "s2"); + // delete the original renamed file to make sure blocks are not updated by + // the original file + assertTrue(hdfs.delete(bar2, true)); + + // checkpoint + NameNode nameNode = cluster.getNameNode(); + NameNodeAdapter.enterSafeMode(nameNode, false); + NameNodeAdapter.saveNamespace(nameNode); + NameNodeAdapter.leaveSafeMode(nameNode); + // restart namenode to load snapshot files from fsimage + cluster.restartNameNode(true); + // file in first snapshot + String barSnapshotPath = Snapshot.getSnapshotPath(foo.toString(), "s1/bar"); + DFSTestUtil.readFile(hdfs, new Path(barSnapshotPath)); + // file in second snapshot after rename+delete + String bar2SnapshotPath = Snapshot.getSnapshotPath(foo2.toString(), + "s2/bar"); + DFSTestUtil.readFile(hdfs, new Path(bar2SnapshotPath)); + } + + /** + * Make sure we delete 0-sized block when deleting an INodeFileUCWithSnapshot + */ + @Test + public void testDeletionWithZeroSizeBlock() throws Exception { + final Path foo = new Path("/foo"); + final Path bar = new Path(foo, "bar"); + DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPLICATION, 0L); + + SnapshotTestHelper.createSnapshot(hdfs, foo, "s0"); + hdfs.append(bar); + + INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile(); + BlockInfo[] blks = barNode.getBlocks(); + assertEquals(1, blks.length); + assertEquals(BLOCKSIZE, blks[0].getNumBytes()); + ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]); + cluster.getNameNodeRpc() + .addBlock(bar.toString(), hdfs.getClient().getClientName(), previous, + null, barNode.getId(), null); + + SnapshotTestHelper.createSnapshot(hdfs, foo, "s1"); + + barNode = fsdir.getINode4Write(bar.toString()).asFile(); + blks = barNode.getBlocks(); + assertEquals(2, blks.length); + assertEquals(BLOCKSIZE, blks[0].getNumBytes()); + assertEquals(0, blks[1].getNumBytes()); + + hdfs.delete(bar, true); + final Path sbar = SnapshotTestHelper.getSnapshotPath(foo, "s1", + bar.getName()); + barNode = fsdir.getINode(sbar.toString()).asFile(); + blks = barNode.getBlocks(); + assertEquals(1, blks.length); + assertEquals(BLOCKSIZE, blks[0].getNumBytes()); + } + + /** Make sure we delete 0-sized block when deleting an INodeFileUC */ + @Test + public void testDeletionWithZeroSizeBlock2() throws Exception { + final Path foo = new Path("/foo"); + final Path subDir = new Path(foo, "sub"); + final Path bar = new Path(subDir, "bar"); + DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPLICATION, 0L); + + hdfs.append(bar); + + INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile(); + BlockInfo[] blks = barNode.getBlocks(); + assertEquals(1, blks.length); + ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]); + cluster.getNameNodeRpc() + .addBlock(bar.toString(), hdfs.getClient().getClientName(), previous, + null, barNode.getId(), null); + + SnapshotTestHelper.createSnapshot(hdfs, foo, "s1"); + + barNode = fsdir.getINode4Write(bar.toString()).asFile(); + blks = barNode.getBlocks(); + assertEquals(2, blks.length); + assertEquals(BLOCKSIZE, blks[0].getNumBytes()); + assertEquals(0, blks[1].getNumBytes()); + + hdfs.delete(subDir, true); + final Path sbar = SnapshotTestHelper.getSnapshotPath(foo, "s1", "sub/bar"); + barNode = fsdir.getINode(sbar.toString()).asFile(); + blks = barNode.getBlocks(); + assertEquals(1, blks.length); + assertEquals(BLOCKSIZE, blks[0].getNumBytes()); + } + + /** + * 1. rename under-construction file with 0-sized blocks after snapshot. + * 2. delete the renamed directory. + * make sure we delete the 0-sized block. + * see HDFS-5476. + */ + @Test + public void testDeletionWithZeroSizeBlock3() throws Exception { + final Path foo = new Path("/foo"); + final Path subDir = new Path(foo, "sub"); + final Path bar = new Path(subDir, "bar"); + DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPLICATION, 0L); + + hdfs.append(bar); + + INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile(); + BlockInfo[] blks = barNode.getBlocks(); + assertEquals(1, blks.length); + ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]); + cluster.getNameNodeRpc() + .addBlock(bar.toString(), hdfs.getClient().getClientName(), previous, + null, barNode.getId(), null); + + SnapshotTestHelper.createSnapshot(hdfs, foo, "s1"); + + // rename bar + final Path bar2 = new Path(subDir, "bar2"); + hdfs.rename(bar, bar2); + + INodeFile bar2Node = fsdir.getINode4Write(bar2.toString()).asFile(); + blks = bar2Node.getBlocks(); + assertEquals(2, blks.length); + assertEquals(BLOCKSIZE, blks[0].getNumBytes()); + assertEquals(0, blks[1].getNumBytes()); + + // delete subDir + hdfs.delete(subDir, true); + + final Path sbar = SnapshotTestHelper.getSnapshotPath(foo, "s1", "sub/bar"); + barNode = fsdir.getINode(sbar.toString()).asFile(); + blks = barNode.getBlocks(); + assertEquals(1, blks.length); + assertEquals(BLOCKSIZE, blks[0].getNumBytes()); + } } Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgress.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgress.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgress.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgress.java Sun Nov 10 20:09:09 2013 @@ -179,6 +179,14 @@ public class TestStartupProgress { startupProgress.endStep(LOADING_FSIMAGE, step); startupProgress.endPhase(LOADING_FSIMAGE); + // Also attempt a whole new step that wasn't used last time. + startupProgress.beginPhase(LOADING_EDITS); + Step newStep = new Step("file1"); + startupProgress.beginStep(LOADING_EDITS, newStep); + incrementCounter(startupProgress, LOADING_EDITS, newStep, 100L); + startupProgress.endStep(LOADING_EDITS, newStep); + startupProgress.endPhase(LOADING_EDITS); + StartupProgressView after = startupProgress.createView(); // Expect that data was frozen after completion of entire startup process, so @@ -200,6 +208,7 @@ public class TestStartupProgress { after.getTotal(LOADING_FSIMAGE)); assertEquals(before.getTotal(LOADING_FSIMAGE, step), after.getTotal(LOADING_FSIMAGE, step)); + assertFalse(after.getSteps(LOADING_EDITS).iterator().hasNext()); } @Test(timeout=10000) Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java (original) +++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java Sun Nov 10 20:09:09 2013 @@ -78,11 +78,6 @@ public class WebHdfsTestUtil { Assert.assertEquals(expectedResponseCode, conn.getResponseCode()); return WebHdfsFileSystem.jsonParse(conn, false); } - - public static HttpURLConnection twoStepWrite(final WebHdfsFileSystem webhdfs, - final HttpOpParam.Op op, HttpURLConnection conn) throws IOException { - return webhdfs.new ConnRunner(op, conn).twoStepWrite(); - } public static FSDataOutputStream write(final WebHdfsFileSystem webhdfs, final HttpOpParam.Op op, final HttpURLConnection conn,
