Author: shv
Date: Thu Sep 15 02:05:45 2011
New Revision: 1170925
URL: http://svn.apache.org/viewvc?rev=1170925&view=rev
Log:
HDFS-2271. startJournalSpool should invoke processIOError. Contributed by Uma
Maheswara Rao G.
Modified:
hadoop/common/branches/branch-0.22/hdfs/CHANGES.txt
hadoop/common/branches/branch-0.22/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
hadoop/common/branches/branch-0.22/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
Modified: hadoop/common/branches/branch-0.22/hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/hdfs/CHANGES.txt?rev=1170925&r1=1170924&r2=1170925&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.22/hdfs/CHANGES.txt Thu Sep 15 02:05:45 2011
@@ -623,6 +623,9 @@ Release 0.22.0 - Unreleased
HDFS-2285. BackupNode should reject requests to modify namespace. (shv)
+ HDFS-2271. startJournalSpool should invoke processIOError.
+ (Uma Maheswara Rao G via shv)
+
Release 0.21.1 - Unreleased
IMPROVEMENTS
Modified:
hadoop/common/branches/branch-0.22/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java?rev=1170925&r1=1170924&r2=1170925&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.22/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
(original)
+++
hadoop/common/branches/branch-0.22/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
Thu Sep 15 02:05:45 2011
@@ -21,8 +21,10 @@ import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.net.URI;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
+import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.server.common.HdfsConstants;
@@ -263,7 +265,8 @@ public class BackupStorage extends FSIma
case WAIT:
waitSpoolEnd();
}
-
+
+ List<StorageDirectory> errorSDs = new ArrayList<StorageDirectory>();
// create journal spool directories
for(Iterator<StorageDirectory> it =
dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
@@ -276,10 +279,18 @@ public class BackupStorage extends FSIma
// create edit file if missing
File eFile = getEditFile(sd);
if(!eFile.exists()) {
- editLog.createEditLogFile(eFile);
+ try {
+ editLog.createEditLogFile(eFile);
+ } catch (IOException ie) {
+ LOG.error("Unable to save edits for " + sd.getRoot(), ie);
+ errorSDs.add(sd);
+ }
}
}
-
+
+ if (errorSDs.size() > 0)
+ processIOError(errorSDs, true);
+
if(!editLog.isOpen())
editLog.open();
Modified:
hadoop/common/branches/branch-0.22/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1170925&r1=1170924&r2=1170925&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.22/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
(original)
+++
hadoop/common/branches/branch-0.22/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
Thu Sep 15 02:05:45 2011
@@ -17,9 +17,14 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -31,20 +36,24 @@ import org.apache.hadoop.hdfs.HdfsConfig
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.TestHDFSServerPorts;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSImage.CheckpointStates;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
-import junit.framework.TestCase;
-public class TestBackupNode extends TestCase {
+public class TestBackupNode {
public static final Log LOG = LogFactory.getLog(TestBackupNode.class);
// reset default 0.0.0.0 addresses in order to avoid IPv6 problem
static final String THIS_HOST = TestHDFSServerPorts.getFullHostName();
static final String BASE_DIR = MiniDFSCluster.getBaseDirectory();
- protected void setUp() throws Exception {
- super.setUp();
+ @Before
+ public void setUp() throws Exception {
File baseDir = new File(BASE_DIR);
if(baseDir.exists())
if(!(FileUtil.fullyDelete(baseDir)))
@@ -56,9 +65,8 @@ public class TestBackupNode extends Test
dirB = new File(getBackupNodeDir(StartupOption.BACKUP, 2));
dirB.mkdirs();
}
-
- protected void tearDown() throws Exception {
- super.tearDown();
+ @After
+ public void tearDown() throws Exception {
File baseDir = new File(BASE_DIR);
if(!(FileUtil.fullyDelete(baseDir)))
throw new IOException("Cannot remove directory: " + baseDir);
@@ -101,7 +109,8 @@ public class TestBackupNode extends Test
} catch (Exception e) {}
} while(backup.getCheckpointState() != CheckpointStates.START);
}
-
+
+ @Test
public void testCheckpoint() throws IOException {
testCheckpoint(StartupOption.CHECKPOINT);
testCheckpoint(StartupOption.BACKUP);
@@ -238,6 +247,7 @@ public class TestBackupNode extends Test
* Test that only one backup node can register.
* @throws IOException
*/
+ @Test
public void testBackupRegistration() throws IOException {
Configuration conf1 = new HdfsConfiguration();
Configuration conf2 = null;
@@ -285,4 +295,62 @@ public class TestBackupNode extends Test
if(cluster != null) cluster.shutdown();
}
}
+
+ @Test
+ public void testStartJournalSpoolHandlesIOExceptions() throws Exception {
+ // Check IOException handled correctly in startJournalSpool;
+ BackupStorage image = new BackupStorage();
+ ArrayList<URI> fsImageDirs = new ArrayList<URI>();
+ ArrayList<URI> editsDirs = new ArrayList<URI>();
+ File filePath1 = new File(BASE_DIR, "storageDirToCheck1/current");
+ File filePath2 = new File(BASE_DIR, "storageDirToCheck2/current");
+ FSEditLog editLog = new FSEditLog(image) {
+ @Override
+ synchronized void divertFileStreams(String dest) throws IOException {
+ }
+ };
+ image.editLog = editLog;
+ assertTrue("Couldn't create directory storageDirToCheck1", filePath1
+ .exists()
+ || filePath1.mkdirs());
+ assertTrue("Couldn't create directory storageDirToCheck2", filePath2
+ .exists()
+ || filePath2.mkdirs());
+ File storageDir1 = filePath1.getParentFile();
+ File storageDir2 = filePath2.getParentFile();
+ try {
+ URI uri1 = storageDir1.toURI();
+ URI uri2 = storageDir2.toURI();
+ fsImageDirs.add(uri1);
+ fsImageDirs.add(uri2);
+ editsDirs.add(uri1);
+ editsDirs.add(uri2);
+ image.setStorageDirectories(fsImageDirs, editsDirs);
+ assertTrue("List of removed storage directories wasn't empty", image
+ .getRemovedStorageDirs().isEmpty());
+ editLog = image.getEditLog();
+ editLog.open();
+ } finally {
+ ArrayList<EditLogOutputStream> editStreams = image.editLog
+ .getEditStreams();
+ // Closing the opened streams
+ for (EditLogOutputStream outStream : editStreams) {
+ outStream.close();
+ }
+ // Delete storage directory to cause IOException in
+ // createEditLogFile
+ FileUtil.fullyDelete(storageDir1);
+ }
+ image.editLog = editLog;
+ image.startJournalSpool(new NamenodeRegistration());
+ List<StorageDirectory> listRsd = image.getRemovedStorageDirs();
+ assertTrue("Removed directory wasn't what was expected", listRsd.size() > 0
+ && listRsd.get(listRsd.size() - 1).getRoot().toString().indexOf(
+ "storageDirToCheck") != -1);
+ // Delete storage directory to cause IOException in
+ // startJournalSpool
+ FileUtil.fullyDelete(storageDir2);
+ // Successfully checked IOException is handled correctly by
+ // startJournalSpool
+ }
}
