HDFS-11209. SNN can't checkpoint when rolling upgrade is not finalized. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1fce2b8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1fce2b8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1fce2b8 Branch: refs/heads/YARN-2915 Commit: b1fce2b8b14c4ce43f7098a269ac2b95304db0ce Parents: 37468eb Author: Xiaoyu Yao <[email protected]> Authored: Tue Jan 17 14:33:26 2017 -0800 Committer: Xiaoyu Yao <[email protected]> Committed: Tue Jan 17 14:33:26 2017 -0800 ---------------------------------------------------------------------- .../NamenodeProtocolServerSideTranslatorPB.java | 16 ++++++++++++++++ .../protocolPB/NamenodeProtocolTranslatorPB.java | 15 +++++++++++++++ .../hadoop/hdfs/server/namenode/Checkpointer.java | 4 ++-- .../apache/hadoop/hdfs/server/namenode/FSImage.java | 4 ++-- .../hdfs/server/namenode/NameNodeRpcServer.java | 7 +++++++ .../hdfs/server/namenode/SecondaryNameNode.java | 6 +++--- .../hdfs/server/protocol/NamenodeProtocol.java | 8 ++++++++ .../src/main/proto/NamenodeProtocol.proto | 16 ++++++++++++++++ 8 files changed, 69 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fce2b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java index c7b3e74..6a10fe4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java @@ -37,6 +37,8 @@ import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecen import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecentCheckpointTxIdResponseProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdRequestProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeResponseProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedRequestProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedResponseProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterRequestProto; @@ -240,4 +242,18 @@ public class NamenodeProtocolServerSideTranslatorPB implements return IsUpgradeFinalizedResponseProto.newBuilder() .setIsUpgradeFinalized(isUpgradeFinalized).build(); } + + @Override + public IsRollingUpgradeResponseProto isRollingUpgrade( + RpcController controller, IsRollingUpgradeRequestProto request) + throws ServiceException { + boolean isRollingUpgrade; + try { + isRollingUpgrade = impl.isRollingUpgrade(); + } catch (IOException e) { + throw new ServiceException(e); + } + return IsRollingUpgradeResponseProto.newBuilder() + .setIsRollingUpgrade(isRollingUpgrade).build(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fce2b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java index 6fc5fc7..02074f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java @@ -34,6 +34,8 @@ import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetBlocksReq import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetEditLogManifestRequestProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecentCheckpointTxIdRequestProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeResponseProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedRequestProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedResponseProto; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterRequestProto; @@ -247,4 +249,17 @@ public class NamenodeProtocolTranslatorPB implements NamenodeProtocol, throw ProtobufHelper.getRemoteException(e); } } + + @Override + public boolean isRollingUpgrade() throws IOException { + IsRollingUpgradeRequestProto req = IsRollingUpgradeRequestProto + .newBuilder().build(); + try { + IsRollingUpgradeResponseProto response = rpcProxy.isRollingUpgrade( + NULL_CONTROLLER, req); + return response.getIsRollingUpgrade(); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fce2b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java index 9ff40cc..318acfb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java @@ -260,8 +260,8 @@ class Checkpointer extends Daemon { completeBlocksTotal); } bnImage.saveFSImageInAllDirs(backupNode.getNamesystem(), txid); - if (!backupNode.namesystem.isRollingUpgrade()) { - bnStorage.writeAll(); + if (!backupNode.namenode.isRollingUpgrade()) { + bnImage.updateStorageVersion(); } } finally { backupNode.namesystem.writeUnlock("doCheckpoint"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fce2b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index f315d1e..76500a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -539,7 +539,7 @@ public class FSImage implements Closeable { // and save it but keep the same checkpointTime saveNamespace(target); - getStorage().writeAll(); + updateStorageVersion(); } void finalizeUpgrade(boolean finalizeEditLog) throws IOException { @@ -1052,7 +1052,7 @@ public class FSImage implements Closeable { try { saveFSImageInAllDirs(source, nnf, imageTxId, canceler); if (!source.isRollingUpgrade()) { - storage.writeAll(); + updateStorageVersion(); } } finally { if (editLogWasOpen) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fce2b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 6a3f966..4a1e8dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -1250,6 +1250,13 @@ public class NameNodeRpcServer implements NamenodeProtocols { namesystem.checkSuperuserPrivilege(); return namesystem.isUpgradeFinalized(); } + + @Override // NamenodeProtocol + public boolean isRollingUpgrade() throws IOException { + checkNNStartup(); + namesystem.checkSuperuserPrivilege(); + return namesystem.isRollingUpgrade(); + } @Override // ClientProtocol public void finalizeUpgrade() throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fce2b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index ec73468..6dd085a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -1064,7 +1064,7 @@ public class SecondaryNameNode implements Runnable, } - static void doMerge( + void doMerge( CheckpointSignature sig, RemoteEditLogManifest manifest, boolean loadImage, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { @@ -1093,8 +1093,8 @@ public class SecondaryNameNode implements Runnable, Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem); // The following has the side effect of purging old fsimages/edit logs. dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId()); - if (!dstNamesystem.isRollingUpgrade()) { - dstStorage.writeAll(); + if (!namenode.isRollingUpgrade()) { + dstImage.updateStorageVersion(); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fce2b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java index 4048372..ccdf516c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java @@ -193,5 +193,13 @@ public interface NamenodeProtocol { @Idempotent public boolean isUpgradeFinalized() throws IOException; + /** + * return whether the Namenode is rolling upgrade in progress (true) or + * not (false). + * @return + * @throws IOException + */ + @Idempotent + boolean isRollingUpgrade() throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fce2b8/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto index d8b1e44..8aa09d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto @@ -193,6 +193,16 @@ message IsUpgradeFinalizedResponseProto { } /** + * void request + */ +message IsRollingUpgradeRequestProto { +} + +message IsRollingUpgradeResponseProto { + required bool isRollingUpgrade = 1; +} + +/** * Protocol used by the sub-ordinate namenode to send requests * the active/primary namenode. * @@ -267,4 +277,10 @@ service NamenodeProtocolService { */ rpc isUpgradeFinalized(IsUpgradeFinalizedRequestProto) returns (IsUpgradeFinalizedResponseProto); + + /** + * Return whether the NameNode is in rolling upgrade (true) or not (false). + */ + rpc isRollingUpgrade(IsRollingUpgradeRequestProto) + returns (IsRollingUpgradeResponseProto); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
