Author: szetszwo
Date: Tue Feb 18 20:12:17 2014
New Revision: 1569515
URL: http://svn.apache.org/r1569515
Log:
HDFS-5945. Add rolling upgrade information to fsimage; and disallow upgrade and
rolling upgrade to be started simultaneously. Contributed by szetszwo & jing9
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-5535.txt
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-5535.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-5535.txt?rev=1569515&r1=1569514&r2=1569515&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-5535.txt
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-5535.txt
Tue Feb 18 20:12:17 2014
@@ -43,3 +43,6 @@ HDFS-5535 subtasks:
HDFS-5920. Support rollback of rolling upgrade in NameNode and
JournalNodes.
(jing9)
+
+ HDFS-5945. Add rolling upgrade information to fsimage; and disallow upgrade
+ and rolling upgrade to be started simultaneously. (szetszwo & jing9)
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1569515&r1=1569514&r2=1569515&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Tue Feb 18 20:12:17 2014
@@ -314,8 +314,9 @@ public class FSImage implements Closeabl
return isFormatted;
}
- void doUpgrade(FSNamesystem target) throws IOException {
- // Upgrade is allowed only if there are
+ /** Check if upgrade is in progress. */
+ void checkUpgrade(FSNamesystem target) throws IOException {
+ // Upgrade or rolling upgrade is allowed only if there are
// no previous fs states in any of the local directories
for (Iterator<StorageDirectory> it = storage.dirIterator(false);
it.hasNext();) {
StorageDirectory sd = it.next();
@@ -324,11 +325,16 @@ public class FSImage implements Closeabl
"previous fs state should not exist during upgrade. "
+ "Finalize or rollback first.");
}
+ }
+
+ void doUpgrade(FSNamesystem target) throws IOException {
+ checkUpgrade(target);
// load the latest image
// Do upgrade for each directory
this.loadFSImage(target, StartupOption.UPGRADE, null);
+ target.checkRollingUpgrade("upgrade namenode");
long oldCTime = storage.getCTime();
storage.cTime = now(); // generate new cTime for the state
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java?rev=1569515&r1=1569514&r2=1569515&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
Tue Feb 18 20:12:17 2014
@@ -42,7 +42,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
import
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
@@ -279,6 +278,9 @@ public final class FSImageFormatProtobuf
fsn.setGenerationStampV1Limit(s.getGenstampV1Limit());
fsn.setLastAllocatedBlockId(s.getLastAllocatedBlockId());
imgTxId = s.getTransactionId();
+ if (s.hasRollingUpgradeStartTime()) {
+ fsn.setRollingUpgradeInfo(s.getRollingUpgradeStartTime());
+ }
}
private void loadStringTableSection(InputStream in) throws IOException {
@@ -519,6 +521,9 @@ public final class FSImageFormatProtobuf
// from the actual saver thread, there's a potential of a
// fairness-related deadlock. See the comments on HDFS-2223.
b.setNamespaceId(fsn.unprotectedGetNamespaceInfo().getNamespaceID());
+ if (fsn.isRollingUpgrade()) {
+
b.setRollingUpgradeStartTime(fsn.getRollingUpgradeInfo().getStartTime());
+ }
NameSystemSection s = b.build();
s.writeDelimitedTo(out);
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java?rev=1569515&r1=1569514&r2=1569515&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java
Tue Feb 18 20:12:17 2014
@@ -25,7 +25,6 @@ import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.Loader;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1569515&r1=1569514&r2=1569515&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Tue Feb 18 20:12:17 2014
@@ -200,7 +200,6 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.common.Util;
import
org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection;
-import
org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection.PersistToken;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
@@ -4512,7 +4511,6 @@ public class FSNamesystem implements Nam
readLock();
try {
checkOperation(OperationCategory.UNCHECKED);
- checkRollingUpgrade("save namespace");
if (!isInSafeMode()) {
throw new IOException("Safe mode should be turned ON "
@@ -7146,6 +7144,7 @@ public class FSNamesystem implements Nam
final String action = "start rolling upgrade";
checkNameNodeSafeMode("Failed to " + action);
checkRollingUpgrade(action);
+ getFSImage().checkUpgrade(this);
getFSImage().saveNamespace(this, NameNodeFile.IMAGE_ROLLBACK, null);
LOG.info("Successfully saved namespace for preparing rolling upgrade.");
@@ -7167,12 +7166,16 @@ public class FSNamesystem implements Nam
rollingUpgradeInfo = new RollingUpgradeInfo(blockPoolId, startTime);
}
+ RollingUpgradeInfo getRollingUpgradeInfo() {
+ return rollingUpgradeInfo;
+ }
+
/** Is rolling upgrade in progress? */
public boolean isRollingUpgrade() {
return rollingUpgradeInfo != null;
}
- private void checkRollingUpgrade(String action) throws
RollingUpgradeException {
+ void checkRollingUpgrade(String action) throws RollingUpgradeException {
if (isRollingUpgrade()) {
throw new RollingUpgradeException("Failed to " + action
+ " since a rolling upgrade is already in progress."
@@ -7197,13 +7200,13 @@ public class FSNamesystem implements Nam
returnInfo = new RollingUpgradeInfo(blockPoolId,
rollingUpgradeInfo.getStartTime(), now());
- getFSImage().purgeCheckpoints(NameNodeFile.IMAGE_ROLLBACK);
rollingUpgradeInfo = null;
getEditLog().logFinalizeRollingUpgrade(returnInfo.getFinalizeTime());
+ getFSImage().saveNamespace(this);
+ getFSImage().purgeCheckpoints(NameNodeFile.IMAGE_ROLLBACK);
} finally {
writeUnlock();
}
- getEditLog().logSync();
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(true, "finalizeRollingUpgrade", null, null, null);
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java?rev=1569515&r1=1569514&r2=1569515&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
Tue Feb 18 20:12:17 2014
@@ -63,7 +63,7 @@ public class NameNodeLayoutVersion {
* </ul>
*/
public static enum Feature implements LayoutFeature {
- ROLLING_UPGRADE_MARKER(-52, "Upgrade marker for rolling upgrade");
+ ROLLING_UPGRADE(-52, "Support rolling upgrade");
private final FeatureInfo info;
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto?rev=1569515&r1=1569514&r2=1569515&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
Tue Feb 18 20:12:17 2014
@@ -70,6 +70,7 @@ message NameSystemSection {
optional uint64 genstampV1Limit = 4;
optional uint64 lastAllocatedBlockId = 5;
optional uint64 transactionId = 6;
+ optional uint64 rollingUpgradeStartTime = 7;
}
/**
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java?rev=1569515&r1=1569514&r2=1569515&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
Tue Feb 18 20:12:17 2014
@@ -35,11 +35,15 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
+import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.TestParallelImageWrite;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.StringUtils;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -214,7 +218,7 @@ public class TestDFSUpgrade {
* This test attempts to upgrade the NameNode and DataNode under
* a number of valid and invalid conditions.
*/
- @Test
+ @Test(timeout = 60000)
public void testUpgrade() throws Exception {
File[] baseDirs;
StorageInfo storageInfo = null;
@@ -227,6 +231,19 @@ public class TestDFSUpgrade {
log("Normal NameNode upgrade", numDirs);
UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current");
cluster = createCluster();
+
+ // make sure that rolling upgrade cannot be started
+ try {
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ dfs.rollingUpgrade(RollingUpgradeAction.START);
+ fail();
+ } catch(RemoteException re) {
+ assertEquals(InconsistentFSStateException.class.getName(),
+ re.getClassName());
+ LOG.info("The exception is expected.", re);
+ }
+
checkNameNode(nameNodeDirs, EXPECTED_TXID);
if (numDirs > 1)
TestParallelImageWrite.checkImages(cluster.getNamesystem(), numDirs);
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java?rev=1569515&r1=1569514&r2=1569515&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
Tue Feb 18 20:12:17 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
import java.io.File;
+import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -27,15 +28,13 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
-import org.apache.hadoop.hdfs.protocol.RollingUpgradeException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster;
import
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
-import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ExitUtil.ExitException;
import org.junit.Assert;
@@ -100,9 +99,13 @@ public class TestRollingUpgrade {
Assert.assertTrue(dfs.exists(foo));
Assert.assertTrue(dfs.exists(bar));
Assert.assertTrue(dfs.exists(baz));
+
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ dfs.saveNamespace();
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
}
- cluster.restartNameNode();
+ cluster.restartNameNode("-rollingupgrade", "started");
{
final DistributedFileSystem dfs = cluster.getFileSystem();
Assert.assertTrue(dfs.exists(foo));
@@ -182,24 +185,6 @@ public class TestRollingUpgrade {
Assert.assertEquals(info1,
dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
dfs.mkdirs(bar);
-
- //save namespace should fail
- try {
- dfs.saveNamespace();
- Assert.fail();
- } catch(RemoteException re) {
- Assert.assertEquals(RollingUpgradeException.class.getName(),
- re.getClassName());
- LOG.info("The exception is expected.", re);
- }
-
- //start checkpoint should fail
- try {
- NameNodeAdapter.startCheckpoint(cluster.getNameNode(), null, null);
- Assert.fail();
- } catch(RollingUpgradeException re) {
- LOG.info("The exception is expected.", re);
- }
}
// cluster2 takes over QJM
@@ -231,8 +216,15 @@ public class TestRollingUpgrade {
Assert.assertTrue(dfs2.exists(bar));
Assert.assertTrue(dfs2.exists(baz));
+ //restart cluster with -upgrade should fail.
+ try {
+ cluster2.restartNameNode("-upgrade");
+ } catch(IOException e) {
+ LOG.info("The exception is expected.", e);
+ }
+
LOG.info("RESTART cluster 2 with -rollingupgrade started again");
- cluster2.restartNameNode();
+ cluster2.restartNameNode("-rollingupgrade", "started");
Assert.assertEquals(info1,
dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
Assert.assertTrue(dfs2.exists(foo));
Assert.assertTrue(dfs2.exists(bar));
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java?rev=1569515&r1=1569514&r2=1569515&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
(original)
+++
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
Tue Feb 18 20:12:17 2014
@@ -78,7 +78,7 @@ public class TestLayoutVersion {
@Test
public void testNameNodeFeature() {
assertTrue(NameNodeLayoutVersion.supports(LayoutVersion.Feature.CACHING,
-
NameNodeLayoutVersion.Feature.ROLLING_UPGRADE_MARKER.getInfo().getLayoutVersion()));
+
NameNodeLayoutVersion.Feature.ROLLING_UPGRADE.getInfo().getLayoutVersion()));
}
/**
