Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1561802&r1=1561801&r2=1561802&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Mon Jan 27 19:31:47 2014 @@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.junit.Assume; import org.junit.Before; @@ -764,4 +765,37 @@ public class TestDFSUtil { assertEquals(4*24*60*60*1000l, DFSUtil.parseRelativeTime("4d")); assertEquals(999*24*60*60*1000l, DFSUtil.parseRelativeTime("999d")); } + + @Test + public void testAssertAllResultsEqual() { + checkAllResults(new Long[]{}, true); + checkAllResults(new Long[]{1l}, true); + checkAllResults(new Long[]{1l, 1l}, true); + checkAllResults(new Long[]{1l, 1l, 1l}, true); + checkAllResults(new Long[]{new Long(1), new Long(1)}, true); + checkAllResults(new Long[]{null, null, null}, true); + + checkAllResults(new Long[]{1l, 2l}, false); + checkAllResults(new Long[]{2l, 1l}, false); + checkAllResults(new Long[]{1l, 2l, 1l}, false); + checkAllResults(new Long[]{2l, 1l, 1l}, false); + checkAllResults(new Long[]{1l, 1l, 2l}, false); + checkAllResults(new Long[]{1l, null}, false); + checkAllResults(new Long[]{null, 1l}, false); + checkAllResults(new Long[]{1l, null, 1l}, false); + } + + private static void checkAllResults(Long[] toCheck, boolean shouldSucceed) { + if (shouldSucceed) { + DFSUtil.assertAllResultsEqual(Arrays.asList(toCheck)); + } else { + try { + DFSUtil.assertAllResultsEqual(Arrays.asList(toCheck)); + fail("Should not have succeeded with input: " + + Arrays.toString(toCheck)); + } catch (AssertionError ae) { + GenericTestUtils.assertExceptionContains("Not all elements match", ae); + } + } + } }
Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java?rev=1561802&r1=1561801&r2=1561802&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java Mon Jan 27 19:31:47 2014 @@ -19,20 +19,28 @@ package org.apache.hadoop.hdfs; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.TestInterDatanodeProtocol; import org.apache.hadoop.hdfs.server.namenode.LeaseManager; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.Test; public class TestLeaseRecovery { @@ -148,4 +156,55 @@ public class TestLeaseRecovery { if (cluster != null) {cluster.shutdown();} } } + + /** + * Block Recovery when the meta file not having crcs for all chunks in block + * file + */ + @Test + public void testBlockRecoveryWithLessMetafile() throws Exception { + Configuration conf = new Configuration(); + conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, + UserGroupInformation.getCurrentUser().getShortUserName()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .build(); + Path file = new Path("/testRecoveryFile"); + DistributedFileSystem dfs = cluster.getFileSystem(); + FSDataOutputStream out = dfs.create(file); + int count = 0; + while (count < 2 * 1024 * 1024) { + out.writeBytes("Data"); + count += 4; + } + out.hsync(); + // abort the original stream + ((DFSOutputStream) out.getWrappedStream()).abort(); + + LocatedBlocks locations = cluster.getNameNodeRpc().getBlockLocations( + file.toString(), 0, count); + ExtendedBlock block = locations.get(0).getBlock(); + DataNode dn = cluster.getDataNodes().get(0); + BlockLocalPathInfo localPathInfo = dn.getBlockLocalPathInfo(block, null); + File metafile = new File(localPathInfo.getMetaPath()); + assertTrue(metafile.exists()); + + // reduce the block meta file size + RandomAccessFile raf = new RandomAccessFile(metafile, "rw"); + raf.setLength(metafile.length() - 20); + raf.close(); + + // restart DN to make replica to RWR + DataNodeProperties dnProp = cluster.stopDataNode(0); + cluster.restartDataNode(dnProp, true); + + // try to recover the lease + DistributedFileSystem newdfs = (DistributedFileSystem) FileSystem + .newInstance(cluster.getConfiguration(0)); + count = 0; + while (++count < 10 && !newdfs.recoverLease(file)) { + Thread.sleep(1000); + } + assertTrue("File should be closed", newdfs.recoverLease(file)); + + } } Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java?rev=1561802&r1=1561801&r2=1561802&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java Mon Jan 27 19:31:47 2014 @@ -167,8 +167,16 @@ public class MiniJournalCluster { return new File(baseDir, "journalnode-" + idx).getAbsoluteFile(); } + public File getJournalDir(int idx, String jid) { + return new File(getStorageDir(idx), jid); + } + public File getCurrentDir(int idx, String jid) { - return new File(new File(getStorageDir(idx), jid), "current"); + return new File(getJournalDir(idx, jid), "current"); + } + + public File getPreviousDir(int idx, String jid) { + return new File(getJournalDir(idx, jid), "previous"); } public JournalNode getJournalNode(int i) { Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java?rev=1561802&r1=1561801&r2=1561802&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java Mon Jan 27 19:31:47 2014 @@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; @@ -47,6 +48,7 @@ public class MiniQJMHACluster { public static class Builder { private final Configuration conf; + private StartupOption startOpt = null; private final MiniDFSCluster.Builder dfsBuilder; public Builder(Configuration conf) { @@ -61,6 +63,10 @@ public class MiniQJMHACluster { public MiniQJMHACluster build() throws IOException { return new MiniQJMHACluster(this); } + + public void startupOption(StartupOption startOpt) { + this.startOpt = startOpt; + } } public static MiniDFSNNTopology createDefaultTopology() { @@ -95,6 +101,9 @@ public class MiniQJMHACluster { Configuration confNN0 = cluster.getConfiguration(0); NameNode.initializeSharedEdits(confNN0, true); + cluster.getNameNodeInfos()[0].setStartOpt(builder.startOpt); + cluster.getNameNodeInfos()[1].setStartOpt(builder.startOpt); + // restart the cluster cluster.restartNameNodes(); } Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java?rev=1561802&r1=1561801&r2=1561802&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java Mon Jan 27 19:31:47 2014 @@ -27,6 +27,8 @@ import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.common.Storage; +import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.junit.Test; @@ -191,6 +193,29 @@ public class TestGenericJournalConf { shouldPromptCalled = true; return false; } + + @Override + public void doPreUpgrade() throws IOException {} + + @Override + public void doUpgrade(Storage storage) throws IOException {} + + @Override + public void doFinalize() throws IOException {} + + @Override + public boolean canRollBack(StorageInfo storage, StorageInfo prevStorage, int targetLayoutVersion) + throws IOException { + return false; + } + + @Override + public void doRollback() throws IOException {} + + @Override + public long getJournalCTime() throws IOException { + return -1; + } } public static class BadConstructorJournalManager extends DummyJournalManager { Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java?rev=1561802&r1=1561801&r2=1561802&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java Mon Jan 27 19:31:47 2014 @@ -91,7 +91,7 @@ public class TestBootstrapStandby { fail("Did not throw"); } catch (IOException ioe) { GenericTestUtils.assertExceptionContains( - "Cannot start an HA namenode with name dirs that need recovery", + "storage directory does not exist or is not accessible", ioe); } Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSUpgradeWithHA.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSUpgradeWithHA.java?rev=1561802&r1=1561801&r2=1561802&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSUpgradeWithHA.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSUpgradeWithHA.java Mon Jan 27 19:31:47 2014 @@ -1,89 +1,506 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hdfs.server.namenode.ha; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collection; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster; +import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster.Builder; +import org.apache.hadoop.hdfs.qjournal.server.Journal; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.Storage; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.tools.DFSAdmin; +import org.apache.hadoop.hdfs.util.PersistentLongFile; import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Before; import org.junit.Test; -import com.google.common.collect.Lists; +import com.google.common.base.Joiner; /** * Tests for upgrading with HA enabled. */ public class TestDFSUpgradeWithHA { - + private static final Log LOG = LogFactory.getLog(TestDFSUpgradeWithHA.class); + + private Configuration conf; + + @Before + public void createConfiguration() { + conf = new HdfsConfiguration(); + // Turn off persistent IPC, so that the DFSClient can survive NN restart + conf.setInt( + CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, + 0); + } + + private static void assertCTimesEqual(MiniDFSCluster cluster) { + long nn1CTime = cluster.getNamesystem(0).getFSImage().getStorage().getCTime(); + long nn2CTime = cluster.getNamesystem(1).getFSImage().getStorage().getCTime(); + assertEquals(nn1CTime, nn2CTime); + } + + private static void checkClusterPreviousDirExistence(MiniDFSCluster cluster, + boolean shouldExist) { + for (int i = 0; i < 2; i++) { + checkNnPreviousDirExistence(cluster, i, shouldExist); + } + } + + private static void checkNnPreviousDirExistence(MiniDFSCluster cluster, + int index, boolean shouldExist) { + Collection<URI> nameDirs = cluster.getNameDirs(index); + for (URI nnDir : nameDirs) { + checkPreviousDirExistence(new File(nnDir), shouldExist); + } + } + private static void checkJnPreviousDirExistence(MiniQJMHACluster jnCluster, + boolean shouldExist) throws IOException { + for (int i = 0; i < 3; i++) { + checkPreviousDirExistence( + jnCluster.getJournalCluster().getJournalDir(i, "ns1"), shouldExist); + } + if (shouldExist) { + assertEpochFilesCopied(jnCluster); + } + } + + private static void assertEpochFilesCopied(MiniQJMHACluster jnCluster) + throws IOException { + for (int i = 0; i < 3; i++) { + File journalDir = jnCluster.getJournalCluster().getJournalDir(i, "ns1"); + File currDir = new File(journalDir, "current"); + File prevDir = new File(journalDir, "previous"); + for (String fileName : new String[]{ Journal.LAST_PROMISED_FILENAME, + Journal.LAST_WRITER_EPOCH }) { + File prevFile = new File(prevDir, fileName); + // Possible the prev file doesn't exist, e.g. if there has never been a + // writer before the upgrade. + if (prevFile.exists()) { + PersistentLongFile prevLongFile = new PersistentLongFile(prevFile, -10); + PersistentLongFile currLongFile = new PersistentLongFile(new File(currDir, + fileName), -11); + assertTrue("Value in " + fileName + " has decreased on upgrade in " + + journalDir, prevLongFile.get() <= currLongFile.get()); + } + } + } + } + + private static void checkPreviousDirExistence(File rootDir, + boolean shouldExist) { + File previousDir = new File(rootDir, "previous"); + if (shouldExist) { + assertTrue(previousDir + " does not exist", previousDir.exists()); + } else { + assertFalse(previousDir + " does exist", previousDir.exists()); + } + } + + private void runFinalizeCommand(MiniDFSCluster cluster) + throws IOException { + HATestUtil.setFailoverConfigurations(cluster, conf); + new DFSAdmin(conf).finalizeUpgrade(); + } + /** - * Make sure that an HA NN refuses to start if given an upgrade-related - * startup option. + * Ensure that an admin cannot finalize an HA upgrade without at least one NN + * being active. */ @Test - public void testStartingWithUpgradeOptionsFails() throws IOException { - for (StartupOption startOpt : Lists.newArrayList(new StartupOption[] { - StartupOption.UPGRADE, StartupOption.FINALIZE, - StartupOption.ROLLBACK })) { - MiniDFSCluster cluster = null; + public void testCannotFinalizeIfNoActive() throws IOException, + URISyntaxException { + MiniDFSCluster cluster = null; + FileSystem fs = null; + try { + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(MiniDFSNNTopology.simpleHATopology()) + .numDataNodes(0) + .build(); + + File sharedDir = new File(cluster.getSharedEditsDir(0, 1)); + + // No upgrade is in progress at the moment. + checkClusterPreviousDirExistence(cluster, false); + assertCTimesEqual(cluster); + checkPreviousDirExistence(sharedDir, false); + + // Transition NN0 to active and do some FS ops. + cluster.transitionToActive(0); + fs = HATestUtil.configureFailoverFs(cluster, conf); + assertTrue(fs.mkdirs(new Path("/foo1"))); + + // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade + // flag. + cluster.shutdownNameNode(1); + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); + cluster.restartNameNode(0, false); + + checkNnPreviousDirExistence(cluster, 0, true); + checkNnPreviousDirExistence(cluster, 1, false); + checkPreviousDirExistence(sharedDir, true); + + // NN0 should come up in the active state when given the -upgrade option, + // so no need to transition it to active. + assertTrue(fs.mkdirs(new Path("/foo2"))); + + // Restart NN0 without the -upgrade flag, to make sure that works. + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR); + cluster.restartNameNode(0, false); + + // Make sure we can still do FS ops after upgrading. + cluster.transitionToActive(0); + assertTrue(fs.mkdirs(new Path("/foo3"))); + + // Now bootstrap the standby with the upgraded info. + int rc = BootstrapStandby.run( + new String[]{"-force"}, + cluster.getConfiguration(1)); + assertEquals(0, rc); + + // Now restart NN1 and make sure that we can do ops against that as well. + cluster.restartNameNode(1); + cluster.transitionToStandby(0); + cluster.transitionToActive(1); + assertTrue(fs.mkdirs(new Path("/foo4"))); + + assertCTimesEqual(cluster); + + // Now there's no active NN. + cluster.transitionToStandby(1); + try { - cluster = new MiniDFSCluster.Builder(new Configuration()) - .nnTopology(MiniDFSNNTopology.simpleHATopology()) - .startupOption(startOpt) - .numDataNodes(0) - .build(); - fail("Should not have been able to start an HA NN in upgrade mode"); - } catch (IllegalArgumentException iae) { + runFinalizeCommand(cluster); + fail("Should not have been able to finalize upgrade with no NN active"); + } catch (IOException ioe) { GenericTestUtils.assertExceptionContains( - "Cannot perform DFS upgrade with HA enabled.", iae); - LOG.info("Got expected exception", iae); - } finally { - if (cluster != null) { - cluster.shutdown(); - } + "Cannot finalize with no NameNode active", ioe); + } + } finally { + if (fs != null) { + fs.close(); + } + if (cluster != null) { + cluster.shutdown(); } } } - + /** - * Make sure that an HA NN won't start if a previous upgrade was in progress. + * Make sure that an HA NN with NFS-based HA can successfully start and + * upgrade. */ @Test - public void testStartingWithUpgradeInProgressFails() throws Exception { + public void testNfsUpgrade() throws IOException, URISyntaxException { MiniDFSCluster cluster = null; + FileSystem fs = null; try { - cluster = new MiniDFSCluster.Builder(new Configuration()) + cluster = new MiniDFSCluster.Builder(conf) .nnTopology(MiniDFSNNTopology.simpleHATopology()) .numDataNodes(0) .build(); + File sharedDir = new File(cluster.getSharedEditsDir(0, 1)); + + // No upgrade is in progress at the moment. + checkClusterPreviousDirExistence(cluster, false); + assertCTimesEqual(cluster); + checkPreviousDirExistence(sharedDir, false); + + // Transition NN0 to active and do some FS ops. + cluster.transitionToActive(0); + fs = HATestUtil.configureFailoverFs(cluster, conf); + assertTrue(fs.mkdirs(new Path("/foo1"))); + + // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade + // flag. + cluster.shutdownNameNode(1); + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); + cluster.restartNameNode(0, false); + + checkNnPreviousDirExistence(cluster, 0, true); + checkNnPreviousDirExistence(cluster, 1, false); + checkPreviousDirExistence(sharedDir, true); + + // NN0 should come up in the active state when given the -upgrade option, + // so no need to transition it to active. + assertTrue(fs.mkdirs(new Path("/foo2"))); + + // Restart NN0 without the -upgrade flag, to make sure that works. + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR); + cluster.restartNameNode(0, false); + + // Make sure we can still do FS ops after upgrading. + cluster.transitionToActive(0); + assertTrue(fs.mkdirs(new Path("/foo3"))); + + // Now bootstrap the standby with the upgraded info. + int rc = BootstrapStandby.run( + new String[]{"-force"}, + cluster.getConfiguration(1)); + assertEquals(0, rc); + + // Now restart NN1 and make sure that we can do ops against that as well. + cluster.restartNameNode(1); + cluster.transitionToStandby(0); + cluster.transitionToActive(1); + assertTrue(fs.mkdirs(new Path("/foo4"))); + + assertCTimesEqual(cluster); + } finally { + if (fs != null) { + fs.close(); + } + if (cluster != null) { + cluster.shutdown(); + } + } + } + + /** + * Make sure that an HA NN can successfully upgrade when configured using + * JournalNodes. + */ + @Test + public void testUpgradeWithJournalNodes() throws IOException, + URISyntaxException { + MiniQJMHACluster qjCluster = null; + FileSystem fs = null; + try { + Builder builder = new MiniQJMHACluster.Builder(conf); + builder.getDfsBuilder() + .numDataNodes(0); + qjCluster = builder.build(); + + MiniDFSCluster cluster = qjCluster.getDfsCluster(); + + // No upgrade is in progress at the moment. + checkJnPreviousDirExistence(qjCluster, false); + checkClusterPreviousDirExistence(cluster, false); + assertCTimesEqual(cluster); + + // Transition NN0 to active and do some FS ops. + cluster.transitionToActive(0); + fs = HATestUtil.configureFailoverFs(cluster, conf); + assertTrue(fs.mkdirs(new Path("/foo1"))); + + // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade + // flag. + cluster.shutdownNameNode(1); + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); + cluster.restartNameNode(0, false); + + checkNnPreviousDirExistence(cluster, 0, true); + checkNnPreviousDirExistence(cluster, 1, false); + checkJnPreviousDirExistence(qjCluster, true); + + // NN0 should come up in the active state when given the -upgrade option, + // so no need to transition it to active. + assertTrue(fs.mkdirs(new Path("/foo2"))); + + // Restart NN0 without the -upgrade flag, to make sure that works. + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR); + cluster.restartNameNode(0, false); + + // Make sure we can still do FS ops after upgrading. + cluster.transitionToActive(0); + assertTrue(fs.mkdirs(new Path("/foo3"))); + + // Now bootstrap the standby with the upgraded info. + int rc = BootstrapStandby.run( + new String[]{"-force"}, + cluster.getConfiguration(1)); + assertEquals(0, rc); + + // Now restart NN1 and make sure that we can do ops against that as well. + cluster.restartNameNode(1); + cluster.transitionToStandby(0); + cluster.transitionToActive(1); + assertTrue(fs.mkdirs(new Path("/foo4"))); + + assertCTimesEqual(cluster); + } finally { + if (fs != null) { + fs.close(); + } + if (qjCluster != null) { + qjCluster.shutdown(); + } + } + } + + @Test + public void testFinalizeWithJournalNodes() throws IOException, + URISyntaxException { + MiniQJMHACluster qjCluster = null; + FileSystem fs = null; + try { + Builder builder = new MiniQJMHACluster.Builder(conf); + builder.getDfsBuilder() + .numDataNodes(0); + qjCluster = builder.build(); + + MiniDFSCluster cluster = qjCluster.getDfsCluster(); + + // No upgrade is in progress at the moment. + checkJnPreviousDirExistence(qjCluster, false); + checkClusterPreviousDirExistence(cluster, false); + assertCTimesEqual(cluster); + + // Transition NN0 to active and do some FS ops. + cluster.transitionToActive(0); + fs = HATestUtil.configureFailoverFs(cluster, conf); + assertTrue(fs.mkdirs(new Path("/foo1"))); + + // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade + // flag. + cluster.shutdownNameNode(1); + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); + cluster.restartNameNode(0, false); + + assertTrue(fs.mkdirs(new Path("/foo2"))); + + checkNnPreviousDirExistence(cluster, 0, true); + checkNnPreviousDirExistence(cluster, 1, false); + checkJnPreviousDirExistence(qjCluster, true); + + // Now bootstrap the standby with the upgraded info. + int rc = BootstrapStandby.run( + new String[]{"-force"}, + cluster.getConfiguration(1)); + assertEquals(0, rc); + + cluster.restartNameNode(1); + + runFinalizeCommand(cluster); + + checkClusterPreviousDirExistence(cluster, false); + checkJnPreviousDirExistence(qjCluster, false); + assertCTimesEqual(cluster); + } finally { + if (fs != null) { + fs.close(); + } + if (qjCluster != null) { + qjCluster.shutdown(); + } + } + } + + /** + * Make sure that even if the NN which initiated the upgrade is in the standby + * state that we're allowed to finalize. + */ + @Test + public void testFinalizeFromSecondNameNodeWithJournalNodes() + throws IOException, URISyntaxException { + MiniQJMHACluster qjCluster = null; + FileSystem fs = null; + try { + Builder builder = new MiniQJMHACluster.Builder(conf); + builder.getDfsBuilder() + .numDataNodes(0); + qjCluster = builder.build(); + + MiniDFSCluster cluster = qjCluster.getDfsCluster(); + + // No upgrade is in progress at the moment. + checkJnPreviousDirExistence(qjCluster, false); + checkClusterPreviousDirExistence(cluster, false); + assertCTimesEqual(cluster); + + // Transition NN0 to active and do some FS ops. + cluster.transitionToActive(0); + fs = HATestUtil.configureFailoverFs(cluster, conf); + assertTrue(fs.mkdirs(new Path("/foo1"))); + + // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade + // flag. + cluster.shutdownNameNode(1); + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); + cluster.restartNameNode(0, false); + + checkNnPreviousDirExistence(cluster, 0, true); + checkNnPreviousDirExistence(cluster, 1, false); + checkJnPreviousDirExistence(qjCluster, true); + + // Now bootstrap the standby with the upgraded info. + int rc = BootstrapStandby.run( + new String[]{"-force"}, + cluster.getConfiguration(1)); + assertEquals(0, rc); + + cluster.restartNameNode(1); + + // Make the second NN (not the one that initiated the upgrade) active when + // the finalize command is run. + cluster.transitionToStandby(0); + cluster.transitionToActive(1); + + runFinalizeCommand(cluster); + + checkClusterPreviousDirExistence(cluster, false); + checkJnPreviousDirExistence(qjCluster, false); + assertCTimesEqual(cluster); + } finally { + if (fs != null) { + fs.close(); + } + if (qjCluster != null) { + qjCluster.shutdown(); + } + } + } + + /** + * Make sure that an HA NN will start if a previous upgrade was in progress. + */ + @Test + public void testStartingWithUpgradeInProgressSucceeds() throws Exception { + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(MiniDFSNNTopology.simpleHATopology()) + .numDataNodes(0) + .build(); + // Simulate an upgrade having started. for (int i = 0; i < 2; i++) { for (URI uri : cluster.getNameDirs(i)) { @@ -92,18 +509,226 @@ public class TestDFSUpgradeWithHA { assertTrue(prevTmp.mkdirs()); } } - + cluster.restartNameNodes(); - fail("Should not have been able to start an HA NN with an in-progress upgrade"); - } catch (IOException ioe) { - GenericTestUtils.assertExceptionContains( - "Cannot start an HA namenode with name dirs that need recovery.", - ioe); - LOG.info("Got expected exception", ioe); } finally { if (cluster != null) { cluster.shutdown(); } } } + + /** + * Test rollback with NFS shared dir. + */ + @Test + public void testRollbackWithNfs() throws Exception { + MiniDFSCluster cluster = null; + FileSystem fs = null; + try { + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(MiniDFSNNTopology.simpleHATopology()) + .numDataNodes(0) + .build(); + + File sharedDir = new File(cluster.getSharedEditsDir(0, 1)); + + // No upgrade is in progress at the moment. + checkClusterPreviousDirExistence(cluster, false); + assertCTimesEqual(cluster); + checkPreviousDirExistence(sharedDir, false); + + // Transition NN0 to active and do some FS ops. + cluster.transitionToActive(0); + fs = HATestUtil.configureFailoverFs(cluster, conf); + assertTrue(fs.mkdirs(new Path("/foo1"))); + + // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade + // flag. + cluster.shutdownNameNode(1); + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); + cluster.restartNameNode(0, false); + + checkNnPreviousDirExistence(cluster, 0, true); + checkNnPreviousDirExistence(cluster, 1, false); + checkPreviousDirExistence(sharedDir, true); + + // NN0 should come up in the active state when given the -upgrade option, + // so no need to transition it to active. + assertTrue(fs.mkdirs(new Path("/foo2"))); + + // Now bootstrap the standby with the upgraded info. + int rc = BootstrapStandby.run( + new String[]{"-force"}, + cluster.getConfiguration(1)); + assertEquals(0, rc); + + cluster.restartNameNode(1); + + checkNnPreviousDirExistence(cluster, 0, true); + checkNnPreviousDirExistence(cluster, 1, false); + checkPreviousDirExistence(sharedDir, true); + assertCTimesEqual(cluster); + + // Now shut down the cluster and do the rollback. + Collection<URI> nn1NameDirs = cluster.getNameDirs(0); + cluster.shutdown(); + + conf.setStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, Joiner.on(",").join(nn1NameDirs)); + NameNode.doRollback(conf, false); + + // The rollback operation should have rolled back the first NN's local + // dirs, and the shared dir, but not the other NN's dirs. Those have to be + // done by bootstrapping the standby. + checkNnPreviousDirExistence(cluster, 0, false); + checkPreviousDirExistence(sharedDir, false); + } finally { + if (fs != null) { + fs.close(); + } + if (cluster != null) { + cluster.shutdown(); + } + } + } + + @Test + public void testRollbackWithJournalNodes() throws IOException, + URISyntaxException { + MiniQJMHACluster qjCluster = null; + FileSystem fs = null; + try { + Builder builder = new MiniQJMHACluster.Builder(conf); + builder.getDfsBuilder() + .numDataNodes(0); + qjCluster = builder.build(); + + MiniDFSCluster cluster = qjCluster.getDfsCluster(); + + // No upgrade is in progress at the moment. + checkClusterPreviousDirExistence(cluster, false); + assertCTimesEqual(cluster); + checkJnPreviousDirExistence(qjCluster, false); + + // Transition NN0 to active and do some FS ops. + cluster.transitionToActive(0); + fs = HATestUtil.configureFailoverFs(cluster, conf); + assertTrue(fs.mkdirs(new Path("/foo1"))); + + // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade + // flag. + cluster.shutdownNameNode(1); + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); + cluster.restartNameNode(0, false); + + checkNnPreviousDirExistence(cluster, 0, true); + checkNnPreviousDirExistence(cluster, 1, false); + checkJnPreviousDirExistence(qjCluster, true); + + // NN0 should come up in the active state when given the -upgrade option, + // so no need to transition it to active. + assertTrue(fs.mkdirs(new Path("/foo2"))); + + // Now bootstrap the standby with the upgraded info. + int rc = BootstrapStandby.run( + new String[]{"-force"}, + cluster.getConfiguration(1)); + assertEquals(0, rc); + + cluster.restartNameNode(1); + + checkNnPreviousDirExistence(cluster, 0, true); + checkNnPreviousDirExistence(cluster, 1, false); + checkJnPreviousDirExistence(qjCluster, true); + assertCTimesEqual(cluster); + + // Shut down the NNs, but deliberately leave the JNs up and running. + Collection<URI> nn1NameDirs = cluster.getNameDirs(0); + cluster.shutdown(); + + conf.setStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, Joiner.on(",").join(nn1NameDirs)); + NameNode.doRollback(conf, false); + + // The rollback operation should have rolled back the first NN's local + // dirs, and the shared dir, but not the other NN's dirs. Those have to be + // done by bootstrapping the standby. + checkNnPreviousDirExistence(cluster, 0, false); + checkJnPreviousDirExistence(qjCluster, false); + } finally { + if (fs != null) { + fs.close(); + } + if (qjCluster != null) { + qjCluster.shutdown(); + } + } + } + + /** + * Make sure that starting a second NN with the -upgrade flag fails if the + * other NN has already done that. + */ + @Test + public void testCannotUpgradeSecondNameNode() throws IOException, + URISyntaxException { + MiniDFSCluster cluster = null; + FileSystem fs = null; + try { + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(MiniDFSNNTopology.simpleHATopology()) + .numDataNodes(0) + .build(); + + File sharedDir = new File(cluster.getSharedEditsDir(0, 1)); + + // No upgrade is in progress at the moment. + checkClusterPreviousDirExistence(cluster, false); + assertCTimesEqual(cluster); + checkPreviousDirExistence(sharedDir, false); + + // Transition NN0 to active and do some FS ops. + cluster.transitionToActive(0); + fs = HATestUtil.configureFailoverFs(cluster, conf); + assertTrue(fs.mkdirs(new Path("/foo1"))); + + // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade + // flag. + cluster.shutdownNameNode(1); + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); + cluster.restartNameNode(0, false); + + checkNnPreviousDirExistence(cluster, 0, true); + checkNnPreviousDirExistence(cluster, 1, false); + checkPreviousDirExistence(sharedDir, true); + + // NN0 should come up in the active state when given the -upgrade option, + // so no need to transition it to active. + assertTrue(fs.mkdirs(new Path("/foo2"))); + + // Restart NN0 without the -upgrade flag, to make sure that works. + cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR); + cluster.restartNameNode(0, false); + + // Make sure we can still do FS ops after upgrading. + cluster.transitionToActive(0); + assertTrue(fs.mkdirs(new Path("/foo3"))); + + // Make sure that starting the second NN with the -upgrade flag fails. + cluster.getNameNodeInfos()[1].setStartOpt(StartupOption.UPGRADE); + try { + cluster.restartNameNode(1, false); + fail("Should not have been able to start second NN with -upgrade"); + } catch (IOException ioe) { + GenericTestUtils.assertExceptionContains( + "It looks like the shared log is already being upgraded", ioe); + } + } finally { + if (fs != null) { + fs.close(); + } + if (cluster != null) { + cluster.shutdown(); + } + } + } } Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java?rev=1561802&r1=1561801&r2=1561802&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java Mon Jan 27 19:31:47 2014 @@ -96,7 +96,7 @@ public class TestInitializeSharedEdits { } catch (IOException ioe) { LOG.info("Got expected exception", ioe); GenericTestUtils.assertExceptionContains( - "Cannot start an HA namenode with name dirs that need recovery", ioe); + "storage directory does not exist or is not accessible", ioe); } try { cluster.restartNameNode(1, false); @@ -104,7 +104,7 @@ public class TestInitializeSharedEdits { } catch (IOException ioe) { LOG.info("Got expected exception", ioe); GenericTestUtils.assertExceptionContains( - "Cannot start an HA namenode with name dirs that need recovery", ioe); + "storage directory does not exist or is not accessible", ioe); } } Modified: hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java?rev=1561802&r1=1561801&r2=1561802&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java Mon Jan 27 19:31:47 2014 @@ -17,22 +17,26 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; -import java.util.Random; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + import org.apache.hadoop.fs.FileStatus; import org.junit.After; import org.junit.Before; import org.junit.Test; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.util.ToolRunner; public class TestSnapshotFileLength { @@ -112,4 +116,61 @@ public class TestSnapshotFileLength { assertThat(bytesRead, is(BLOCKSIZE)); fis.close(); } + + /** + * Adding as part of jira HDFS-5343 + * Test for checking the cat command on snapshot path it + * cannot read a file beyond snapshot file length + * @throws Exception + */ + @Test (timeout = 600000) + public void testSnapshotFileLengthWithCatCommand() throws Exception { + + FSDataInputStream fis = null; + FileStatus fileStatus = null; + + int bytesRead; + byte[] buffer = new byte[BLOCKSIZE * 8]; + + hdfs.mkdirs(sub); + Path file1 = new Path(sub, file1Name); + DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, SEED); + + hdfs.allowSnapshot(sub); + hdfs.createSnapshot(sub, snapshot1); + + DFSTestUtil.appendFile(hdfs, file1, BLOCKSIZE); + + // Make sure we can read the entire file via its non-snapshot path. + fileStatus = hdfs.getFileStatus(file1); + assertEquals(fileStatus.getLen(), BLOCKSIZE * 2); + fis = hdfs.open(file1); + bytesRead = fis.read(buffer, 0, buffer.length); + assertEquals(bytesRead, BLOCKSIZE * 2); + fis.close(); + + Path file1snap1 = + SnapshotTestHelper.getSnapshotPath(sub, snapshot1, file1Name); + fis = hdfs.open(file1snap1); + fileStatus = hdfs.getFileStatus(file1snap1); + assertEquals(fileStatus.getLen(), BLOCKSIZE); + // Make sure we can only read up to the snapshot length. + bytesRead = fis.read(buffer, 0, buffer.length); + assertEquals(bytesRead, BLOCKSIZE); + fis.close(); + + PrintStream psBackup = System.out; + ByteArrayOutputStream bao = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bao)); + System.setErr(new PrintStream(bao)); + // Make sure we can cat the file upto to snapshot length + FsShell shell = new FsShell(); + try{ + ToolRunner.run(conf, shell, new String[] { "-cat", + "/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" }); + assertEquals(bao.size(), BLOCKSIZE); + }finally{ + System.setOut(psBackup); + } + } }
