HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6942cbe Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6942cbe Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6942cbe Branch: refs/heads/HDFS-10467 Commit: b6942cbe9b8c9469e8c2b64c3268d671f5a43e75 Parents: d0046bb Author: Arpit Agarwal <a...@apache.org> Authored: Thu Oct 5 15:26:52 2017 -0700 Committer: Arpit Agarwal <a...@apache.org> Committed: Thu Oct 5 15:26:52 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 ++ .../hadoop/hdfs/server/namenode/NameNode.java | 16 +++++++++ .../namenode/NameNodeFormatException.java | 37 ++++++++++++++++++++ .../src/main/resources/hdfs-default.xml | 10 ++++++ .../src/site/markdown/HDFSCommands.md | 2 +- .../hdfs/server/namenode/TestClusterId.java | 34 ++++++++++++++++++ 6 files changed, 101 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index b4842f9..f2f8730 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -270,6 +270,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.posix.acl.inheritance.enabled"; public static final boolean DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT = true; + public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled"; + public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false; + public static final String DFS_NAMENODE_XATTRS_ENABLED_KEY = "dfs.namenode.xattrs.enabled"; public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true; public static final String DFS_ADMIN = "dfs.cluster.administrators"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 79bbbc5..32b873b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState; import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby; import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; @@ -1146,6 +1147,21 @@ public class NameNode extends ReconfigurableBase implements FSNamesystem fsn = new FSNamesystem(conf, fsImage); fsImage.getEditLog().initJournalsForWrite(); + // Abort NameNode format if reformat is disabled and if + // meta-dir already exists + if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, + DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) { + force = false; + isInteractive = false; + for (StorageDirectory sd : fsImage.storage.dirIterable(null)) { + if (sd.hasSomeData()) { + throw new NameNodeFormatException( + "NameNode format aborted as reformat is disabled for " + + "this cluster."); + } + } + } + if (!fsImage.confirmFormat(force, isInteractive)) { return true; // aborted } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java new file mode 100644 index 0000000..858d6e1 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.IOException; +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Thrown when NameNode format fails. + */ +@InterfaceAudience.Private +public class NameNodeFormatException extends IOException { + + private static final long serialVersionUID = 1L; + + public NameNodeFormatException(String message, Throwable cause) { + super(message, cause); + } + + public NameNodeFormatException(String message) { + super(message); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 4e4ee33..8a60b80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4640,4 +4640,14 @@ </description> </property> + <property> + <name>dfs.reformat.disabled</name> + <value>false</value> + <description> + Disable reformat of NameNode. If it's value is set to "true" + and metadata directories already exist then attempt to format NameNode + will throw NameNodeFormatException. + </description> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index 322609a..c5f80d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -532,7 +532,7 @@ Usage: |:---- |:---- | | `-backup` | Start backup node. | | `-checkpoint` | Start checkpoint node. | -| `-format` `[-clusterid cid]` `[-force]` `[-nonInteractive]` | Formats the specified NameNode. It starts the NameNode, formats it and then shut it down. -force option formats if the name directory exists. -nonInteractive option aborts if the name directory exists, unless -force option is specified. | +| `-format` `[-clusterid cid]` | Formats the specified NameNode. It starts the NameNode, formats it and then shut it down. Will throw NameNodeFormatException if name dir already exist and if reformat is disabled for cluster. | | `-upgrade` `[-clusterid cid]` [`-renameReserved` \<k-v pairs\>] | Namenode should be started with upgrade option after the distribution of new Hadoop version. | | `-upgradeOnly` `[-clusterid cid]` [`-renameReserved` \<k-v pairs\>] | Upgrade the specified NameNode and then shutdown it. | | `-rollback` | Rollback the NameNode to the previous version. This should be used after stopping the cluster and distributing the old Hadoop version. | http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java index fa3399b..d5a3948 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java @@ -39,9 +39,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.ExitUtil.ExitException; @@ -452,4 +456,34 @@ public class TestClusterId { File version = new File(hdfsDir, "current/VERSION"); assertFalse("Check version should not exist", version.exists()); } + + /** + * Test NameNode format failure when reformat is disabled and metadata + * directories exist. + */ + @Test + public void testNNFormatFailure() throws Exception { + NameNode.initMetrics(config, NamenodeRole.NAMENODE); + DFSTestUtil.formatNameNode(config); + config.setBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, true); + // Call to NameNode format will fail as name dir is not empty + try { + NameNode.format(config); + fail("NN format should fail."); + } catch (NameNodeFormatException e) { + GenericTestUtils.assertExceptionContains("NameNode format aborted as " + + "reformat is disabled for this cluster", e); + } + } + + /** + * Test NameNode format when reformat is disabled and metadata directories do + * not exist. + */ + @Test + public void testNNFormatSuccess() throws Exception { + NameNode.initMetrics(config, NamenodeRole.NAMENODE); + config.setBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, true); + DFSTestUtil.formatNameNode(config); + } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
