Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java?rev=1354832&r1=1354831&r2=1354832&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java Thu Jun 28 06:59:38 2012 @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.na import static org.junit.Assert.*; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import java.util.Collections; @@ -35,6 +36,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNT import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; @@ -118,8 +120,8 @@ public class TestEditLogsDuringFailover } } - @Test - public void testFailoverFinalizesAndReadsInProgress() throws Exception { + private void testFailoverFinalizesAndReadsInProgress( + boolean partialTxAtEnd) throws Exception { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .nnTopology(MiniDFSNNTopology.simpleHATopology()) @@ -130,8 +132,21 @@ public class TestEditLogsDuringFailover URI sharedUri = cluster.getSharedEditsDir(0, 1); File sharedDir = new File(sharedUri.getPath(), "current"); FSImageTestUtil.createAbortedLogWithMkdirs(sharedDir, NUM_DIRS_IN_LOG, 1); + assertEditFiles(Collections.singletonList(sharedUri), NNStorage.getInProgressEditsFileName(1)); + if (partialTxAtEnd) { + FileOutputStream outs = null; + try { + File editLogFile = + new File(sharedDir, NNStorage.getInProgressEditsFileName(1)); + outs = new FileOutputStream(editLogFile, true); + outs.write(new byte[] { 0x18, 0x00, 0x00, 0x00 } ); + LOG.error("editLogFile = " + editLogFile); + } finally { + IOUtils.cleanup(LOG, outs); + } + } // Transition one of the NNs to active cluster.transitionToActive(0); @@ -149,7 +164,18 @@ public class TestEditLogsDuringFailover } finally { cluster.shutdown(); } + } + + @Test + public void testFailoverFinalizesAndReadsInProgressSimple() + throws Exception { + testFailoverFinalizesAndReadsInProgress(false); + } + @Test + public void testFailoverFinalizesAndReadsInProgressWithPartialTxAtEnd() + throws Exception { + testFailoverFinalizesAndReadsInProgress(true); } /**
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java?rev=1354832&r1=1354831&r2=1354832&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java Thu Jun 28 06:59:38 2012 @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.namenode.ha; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY; import static org.junit.Assert.*; import java.io.File; @@ -127,6 +129,7 @@ public class TestFailureOfSharedDir { @Test public void testFailureOfSharedDir() throws Exception { Configuration conf = new Configuration(); + conf.setLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, 2000); // The shared edits dir will automatically be marked required. MiniDFSCluster cluster = null; @@ -151,6 +154,15 @@ public class TestFailureOfSharedDir { assertEquals(0, FileUtil.chmod(sharedEditsDir.getAbsolutePath(), "-w", true)); + Thread.sleep(conf.getLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, + DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT) * 2); + + NameNode nn1 = cluster.getNameNode(1); + assertTrue(nn1.isStandbyState()); + assertFalse( + "StandBy NameNode should not go to SafeMode on resource unavailability", + nn1.isInSafeMode()); + NameNode nn0 = cluster.getNameNode(0); nn0.getNamesystem().getFSImage().getEditLog().getJournalSet() .setRuntimeForTesting(mockRuntime); Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java?rev=1354832&r1=1354831&r2=1354832&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java Thu Jun 28 06:59:38 2012 @@ -22,11 +22,14 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.File; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.util.Map; import java.util.HashMap; import org.junit.Test; import org.junit.Before; + +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import org.apache.commons.logging.Log; @@ -34,12 +37,12 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes; import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer; +import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer.Flags; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.server.namenode.OfflineEditsViewerHelper; public class TestOfflineEditsViewer { - private static final Log LOG = LogFactory.getLog(TestOfflineEditsViewer.class); private static final Map<FSEditLogOpCodes, Boolean> obsoleteOpCodes = @@ -97,8 +100,8 @@ public class TestOfflineEditsViewer { String editsReparsed = cacheDir + "/editsReparsed"; // parse to XML then back to binary - runOev(edits, editsParsedXml, "xml"); - runOev(editsParsedXml, editsReparsed, "binary"); + assertEquals(0, runOev(edits, editsParsedXml, "xml", false)); + assertEquals(0, runOev(editsParsedXml, editsReparsed, "binary", false)); // judgment time assertTrue( @@ -115,6 +118,42 @@ public class TestOfflineEditsViewer { } @Test + public void testRecoveryMode() throws IOException { + LOG.info("START - testing with generated edits"); + + nnHelper.startCluster(buildDir + "/dfs/"); + + // edits generated by nnHelper (MiniDFSCluster), should have all op codes + // binary, XML, reparsed binary + String edits = nnHelper.generateEdits(); + + // Corrupt the file by truncating the end + FileChannel editsFile = new FileOutputStream(edits, true).getChannel(); + editsFile.truncate(editsFile.size() - 5); + + String editsParsedXml = cacheDir + "/editsRecoveredParsed.xml"; + String editsReparsed = cacheDir + "/editsRecoveredReparsed"; + String editsParsedXml2 = cacheDir + "/editsRecoveredParsed2.xml"; + + // Can't read the corrupted file without recovery mode + assertEquals(-1, runOev(edits, editsParsedXml, "xml", false)); + + // parse to XML then back to binary + assertEquals(0, runOev(edits, editsParsedXml, "xml", true)); + assertEquals(0, runOev(editsParsedXml, editsReparsed, "binary", false)); + assertEquals(0, runOev(editsReparsed, editsParsedXml2, "xml", false)); + + // judgment time + assertTrue("Test round trip", + filesEqualIgnoreTrailingZeros(editsParsedXml, editsParsedXml2)); + + // removes edits so do this at the end + nnHelper.shutdownCluster(); + + LOG.info("END"); + } + + @Test public void testStored() throws IOException { LOG.info("START - testing with stored reference edits"); @@ -128,8 +167,9 @@ public class TestOfflineEditsViewer { String editsStoredXml = cacheDir + "/editsStored.xml"; // parse to XML then back to binary - runOev(editsStored, editsStoredParsedXml, "xml"); - runOev(editsStoredParsedXml, editsStoredReparsed, "binary"); + assertEquals(0, runOev(editsStored, editsStoredParsedXml, "xml", false)); + assertEquals(0, runOev(editsStoredParsedXml, editsStoredReparsed, + "binary", false)); // judgement time assertTrue( @@ -151,14 +191,18 @@ public class TestOfflineEditsViewer { * @param inFilename input edits filename * @param outFilename oputput edits filename */ - private void runOev(String inFilename, String outFilename, String processor) - throws IOException { + private int runOev(String inFilename, String outFilename, String processor, + boolean recovery) throws IOException { LOG.info("Running oev [" + inFilename + "] [" + outFilename + "]"); OfflineEditsViewer oev = new OfflineEditsViewer(); - if (oev.go(inFilename, outFilename, processor, true, false, null) != 0) - throw new RuntimeException("oev failed"); + Flags flags = new Flags(); + flags.setPrintToScreen(); + if (recovery) { + flags.setRecoveryMode(); + } + return oev.go(inFilename, outFilename, processor, flags, null); } /** @@ -172,7 +216,7 @@ public class TestOfflineEditsViewer { FileOutputStream fout = new FileOutputStream(outFilename); StatisticsEditsVisitor visitor = new StatisticsEditsVisitor(fout); OfflineEditsViewer oev = new OfflineEditsViewer(); - if (oev.go(inFilename, outFilename, "stats", false, false, visitor) != 0) + if (oev.go(inFilename, outFilename, "stats", new Flags(), visitor) != 0) return false; LOG.info("Statistics for " + inFilename + "\n" + visitor.getStatisticsString()); Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java?rev=1354832&r1=1354831&r2=1354832&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java Thu Jun 28 06:59:38 2012 @@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.hdfs.web.resources.DoAsParam; import org.apache.hadoop.hdfs.web.resources.GetOpParam; import org.apache.hadoop.hdfs.web.resources.HttpOpParam; +import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam; import org.apache.hadoop.hdfs.web.resources.PutOpParam; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; @@ -286,6 +287,10 @@ public class TestWebHdfsFileSystemContra final Path root = new Path("/"); final Path dir = new Path("/test/testUrl"); assertTrue(webhdfs.mkdirs(dir)); + final Path file = new Path("/test/file"); + final FSDataOutputStream out = webhdfs.create(file); + out.write(1); + out.close(); {//test GETHOMEDIRECTORY final URL url = webhdfs.toUrl(GetOpParam.Op.GETHOMEDIRECTORY, root); @@ -351,5 +356,47 @@ public class TestWebHdfsFileSystemContra {//test append. AppendTestUtil.testAppend(fs, new Path(dir, "append")); } + + {//test NamenodeRpcAddressParam not set. + final HttpOpParam.Op op = PutOpParam.Op.CREATE; + final URL url = webhdfs.toUrl(op, dir); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod(op.getType().toString()); + conn.setDoOutput(false); + conn.setInstanceFollowRedirects(false); + conn.connect(); + final String redirect = conn.getHeaderField("Location"); + conn.disconnect(); + + //remove NamenodeRpcAddressParam + WebHdfsFileSystem.LOG.info("redirect = " + redirect); + final int i = redirect.indexOf(NamenodeRpcAddressParam.NAME); + final int j = redirect.indexOf("&", i); + String modified = redirect.substring(0, i - 1) + redirect.substring(j); + WebHdfsFileSystem.LOG.info("modified = " + modified); + + //connect to datanode + conn = (HttpURLConnection)new URL(modified).openConnection(); + conn.setRequestMethod(op.getType().toString()); + conn.setDoOutput(op.getDoOutput()); + conn.connect(); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + } + + {//test jsonParse with non-json type. + final HttpOpParam.Op op = GetOpParam.Op.OPEN; + final URL url = webhdfs.toUrl(op, file); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod(op.getType().toString()); + conn.connect(); + + try { + WebHdfsFileSystem.jsonParse(conn, false); + fail(); + } catch(IOException ioe) { + WebHdfsFileSystem.LOG.info("GOOD", ioe); + } + conn.disconnect(); + } } } Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java?rev=1354832&r1=1354831&r2=1354832&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java Thu Jun 28 06:59:38 2012 @@ -40,6 +40,12 @@ import org.junit.Assert; public class WebHdfsTestUtil { public static final Log LOG = LogFactory.getLog(WebHdfsTestUtil.class); + public static Configuration createConf() { + final Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); + return conf; + } + public static WebHdfsFileSystem getWebHdfsFileSystem(final Configuration conf ) throws IOException, URISyntaxException { final String uri = WebHdfsFileSystem.SCHEME + "://" @@ -49,7 +55,7 @@ public class WebHdfsTestUtil { public static WebHdfsFileSystem getWebHdfsFileSystemAs( final UserGroupInformation ugi, final Configuration conf - ) throws IOException, URISyntaxException, InterruptedException { + ) throws IOException, InterruptedException { return ugi.doAs(new PrivilegedExceptionAction<WebHdfsFileSystem>() { @Override public WebHdfsFileSystem run() throws Exception { @@ -70,7 +76,7 @@ public class WebHdfsTestUtil { final int expectedResponseCode) throws IOException { conn.connect(); Assert.assertEquals(expectedResponseCode, conn.getResponseCode()); - return WebHdfsFileSystem.jsonParse(conn.getInputStream()); + return WebHdfsFileSystem.jsonParse(conn, false); } public static HttpURLConnection twoStepWrite(HttpURLConnection conn, Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml?rev=1354832&r1=1354831&r2=1354832&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml Thu Jun 28 06:59:38 2012 @@ -15470,7 +15470,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>For input string: "a5"</expected-output> + <expected-output>setSpaceQuota: "a5" is not a valid value for a quota.</expected-output> </comparator> </comparators> </test> Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml?rev=1354832&r1=1354831&r2=1354832&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml (original) +++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/pom.xml Thu Jun 28 06:59:38 2012 @@ -34,7 +34,7 @@ http://maven.apache.org/xsd/maven-4.0.0. <module>hadoop-hdfs</module> <module>hadoop-hdfs-httpfs</module> <module>hadoop-hdfs/src/contrib/bkjournal</module> - <module>hadoop-hdfs/src/contrib/fuse-dfs</module> + <module>hadoop-hdfs-raid</module> </modules> <build>
