Author: tgraves
Date: Tue Feb 5 15:16:39 2013
New Revision: 1442620
URL: http://svn.apache.org/viewvc?rev=1442620&view=rev
Log:
HDFS-4462. 2NN will fail to checkpoint after an HDFS upgrade from a
pre-federation version of HDFS. (atm)
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1442620&r1=1442619&r2=1442620&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Feb 5 15:16:39 2013
@@ -34,6 +34,9 @@ Release 0.23.7 - UNRELEASED
factor is reduced after sync follwed by closing that file. (Ashish Singhi
via umamahesh)
+ HDFS-4462. 2NN will fail to checkpoint after an HDFS upgrade from a
+ pre-federation version of HDFS. (atm)
+
Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1442620&r1=1442619&r2=1442620&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
Tue Feb 5 15:16:39 2013
@@ -783,7 +783,7 @@ public abstract class Storage extends St
props.setProperty("storageType", storageType.toString());
props.setProperty("namespaceID", String.valueOf(namespaceID));
// Set clusterID in version with federation support
- if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) {
+ if (versionSupportsFederation()) {
props.setProperty("clusterID", clusterID);
}
props.setProperty("cTime", String.valueOf(cTime));
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java?rev=1442620&r1=1442619&r2=1442620&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
Tue Feb 5 15:16:39 2013
@@ -22,6 +22,8 @@ import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.LayoutVersion;
+import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
@@ -100,6 +102,10 @@ public class StorageInfo implements Writ
clusterID = WritableUtils.readString(in);
cTime = in.readLong();
}
+
+ public boolean versionSupportsFederation() {
+ return LayoutVersion.supports(Feature.FEDERATION, layoutVersion);
+ }
public String toString() {
StringBuilder sb = new StringBuilder();
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java?rev=1442620&r1=1442619&r2=1442620&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
Tue Feb 5 15:16:39 2013
@@ -112,6 +112,10 @@ public class CheckpointSignature extends
blockpoolID.equals(si.getBlockPoolID());
}
+ boolean namespaceIdMatches(FSImage si) {
+ return namespaceID == si.getStorage().namespaceID;
+ }
+
void validateStorageInfo(FSImage si) throws IOException {
if (!isSameCluster(si)
|| !storageVersionMatches(si.getStorage())) {
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1442620&r1=1442619&r2=1442620&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
Tue Feb 5 15:16:39 2013
@@ -556,7 +556,7 @@ public class NNStorage extends Storage i
}
// Set Block pool ID in version with federation support
- if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) {
+ if (versionSupportsFederation()) {
String sbpid = props.getProperty("blockpoolID");
setBlockPoolID(sd.getRoot(), sbpid);
}
@@ -610,7 +610,7 @@ public class NNStorage extends Storage i
) throws IOException {
super.setPropertiesFromFields(props, sd);
// Set blockpoolID in version with federation support
- if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) {
+ if (versionSupportsFederation()) {
props.setProperty("blockpoolID", blockpoolID);
}
boolean uState = getDistributedUpgradeState();
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1442620&r1=1442619&r2=1442620&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Tue Feb 5 15:16:39 2013
@@ -518,14 +518,20 @@ public class SecondaryNameNode implement
// Returns a token that would be used to upload the merged image.
CheckpointSignature sig = namenode.rollEditLog();
- if ((checkpointImage.getNamespaceID() == 0) ||
- (sig.isSameCluster(checkpointImage) &&
+ boolean loadImage = false;
+ boolean isFreshCheckpointer = (checkpointImage.getNamespaceID() == 0);
+ boolean isSameCluster =
+ (dstStorage.versionSupportsFederation() &&
sig.isSameCluster(checkpointImage)) ||
+ (!dstStorage.versionSupportsFederation() &&
sig.namespaceIdMatches(checkpointImage));
+ if (isFreshCheckpointer ||
+ (isSameCluster &&
!sig.storageVersionMatches(checkpointImage.getStorage()))) {
// if we're a fresh 2NN, or if we're on the same cluster and our storage
// needs an upgrade, just take the storage info from the server.
dstStorage.setStorageInfo(sig);
dstStorage.setClusterID(sig.getClusterID());
dstStorage.setBlockPoolID(sig.getBlockpoolID());
+ loadImage = true;
}
sig.validateStorageInfo(checkpointImage);
@@ -538,7 +544,7 @@ public class SecondaryNameNode implement
RemoteEditLogManifest manifest =
namenode.getEditLogManifest(sig.mostRecentCheckpointTxId + 1);
- boolean loadImage = downloadCheckpointFiles(
+ loadImage |= downloadCheckpointFiles(
fsName, checkpointImage, sig, manifest); // Fetch fsimage and edits
doMerge(sig, manifest, loadImage, checkpointImage);
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java?rev=1442620&r1=1442619&r2=1442620&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
Tue Feb 5 15:16:39 2013
@@ -458,7 +458,11 @@ public abstract class FSImageTestUtil {
props.load(fis);
IOUtils.closeStream(fis);
- props.setProperty(key, value);
+ if (value == null || value.isEmpty()) {
+ props.remove(key);
+ } else {
+ props.setProperty(key, value);
+ }
out = new FileOutputStream(versionFile);
props.store(out, null);
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java?rev=1442620&r1=1442619&r2=1442620&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java
Tue Feb 5 15:16:39 2013
@@ -17,9 +17,12 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import com.google.common.collect.ImmutableMap;
+
import java.io.File;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
import org.junit.Test;
import org.junit.Before;
@@ -57,7 +60,7 @@ public class TestSecondaryNameNodeUpgrad
}
}
- private void doIt(String param, String val) throws IOException {
+ private void doIt(Map<String, String> paramsToCorrupt) throws IOException {
MiniDFSCluster cluster = null;
FileSystem fs = null;
SecondaryNameNode snn = null;
@@ -82,8 +85,12 @@ public class TestSecondaryNameNodeUpgrad
snn.shutdown();
for (File versionFile : versionFiles) {
- System.out.println("Changing '" + param + "' to '" + val + "' in " +
versionFile);
- FSImageTestUtil.corruptVersionFile(versionFile, param, val);
+ for (Map.Entry<String, String> paramToCorrupt :
paramsToCorrupt.entrySet()) {
+ String param = paramToCorrupt.getKey();
+ String val = paramToCorrupt.getValue();
+ System.out.println("Changing '" + param + "' to '" + val + "' in " +
versionFile);
+ FSImageTestUtil.corruptVersionFile(versionFile, param, val);
+ }
}
snn = new SecondaryNameNode(conf);
@@ -100,13 +107,19 @@ public class TestSecondaryNameNodeUpgrad
@Test
public void testUpgradeLayoutVersionSucceeds() throws IOException {
- doIt("layoutVersion", "-39");
+ doIt(ImmutableMap.of("layoutVersion", "-39"));
+ }
+
+ @Test
+ public void testUpgradePreFedSucceeds() throws IOException {
+ doIt(ImmutableMap.of("layoutVersion", "-19", "clusterID", "",
+ "blockpoolID", ""));
}
@Test
public void testChangeNsIDFails() throws IOException {
try {
- doIt("namespaceID", "2");
+ doIt(ImmutableMap.of("namespaceID", "2"));
Assert.fail("Should throw InconsistentFSStateException");
} catch(IOException e) {
GenericTestUtils.assertExceptionContains("Inconsistent checkpoint
fields", e);
