This is an automated email from the ASF dual-hosted git repository.
erose pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 50dbf08211 HDDS-6695. Enable SCM Ratis by default for new clusters
only (#3499)
50dbf08211 is described below
commit 50dbf08211d846f8d35d1ba130e0b17c4339264e
Author: swamirishi <[email protected]>
AuthorDate: Mon Jun 20 17:09:38 2022 -0700
HDDS-6695. Enable SCM Ratis by default for new clusters only (#3499)
---
.../hadoop/hdds/conf/DefaultConfigManager.java | 54 +++++++++++++++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 4 +-
.../common/src/main/resources/ozone-default.xml | 8 ---
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 59 +++++++++++++++--
.../hdds/scm/server/StorageContainerManager.java | 19 +++---
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 2 +
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 70 ++++++++++++++++++--
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 1 +
.../TestSCMHAUnfinalizedStateValidationAction.java | 14 +++-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 11 +++-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 1 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 8 ++-
.../hadoop/ozone/TestStorageContainerManager.java | 20 ++++--
.../ozone/scm/TestStorageContainerManagerHA.java | 76 ++++++++++++++++++----
15 files changed, 301 insertions(+), 51 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/DefaultConfigManager.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/DefaultConfigManager.java
new file mode 100644
index 0000000000..c33b1a3308
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/DefaultConfigManager.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.conf;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Class to store Default value of configuration keys.
+ */
+public final class DefaultConfigManager {
+
+ /** Never constructed. **/
+ private DefaultConfigManager() {
+ }
+
+ private static final Map<String, Object> CONFIG_DEFAULT_MAP = new
HashMap<>();
+
+ public static <T> void setConfigValue(String config, T value) {
+ T prevValue = getValue(config, value);
+ if (!value.equals(prevValue)) {
+ throw new ConfigurationException(String.format("Setting conflicting " +
+ "Default Configs old default Value: %s New Default Value:%s",
+ prevValue.toString(), value.toString()));
+ }
+ CONFIG_DEFAULT_MAP.putIfAbsent(config, value);
+ }
+ public static <T> T getValue(String config, T defaultValue) {
+ return (T) CONFIG_DEFAULT_MAP.getOrDefault(config, defaultValue);
+ }
+
+ @VisibleForTesting
+ public static void clearDefaultConfigs() {
+ CONFIG_DEFAULT_MAP.clear();
+ }
+}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index b10f05cd1a..3259b5b137 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -453,8 +453,11 @@ public final class ScmConfigKeys {
// SCM Ratis related
public static final String OZONE_SCM_HA_ENABLE_KEY
= "ozone.scm.ratis.enable";
+ /** Default Value would be Overriden based on the current state of Ratis.
+ {@link org.apache.hadoop.hdds.conf.DefaultConfigManager}
+ */
public static final boolean OZONE_SCM_HA_ENABLE_DEFAULT
- = false;
+ = true;
public static final String OZONE_SCM_RATIS_PORT_KEY
= "ozone.scm.ratis.port";
public static final int OZONE_SCM_RATIS_PORT_DEFAULT
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
index 9baa61d434..f07dc56f1f 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.ConfigurationException;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.DefaultConfigManager;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.ratis.ServerNotLeaderException;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -89,7 +90,8 @@ public final class SCMHAUtils {
// Check if SCM HA is enabled.
public static boolean isSCMHAEnabled(ConfigurationSource conf) {
return conf.getBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY,
- ScmConfigKeys.OZONE_SCM_HA_ENABLE_DEFAULT);
+ DefaultConfigManager.getValue(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY,
+ ScmConfigKeys.OZONE_SCM_HA_ENABLE_DEFAULT));
}
public static String getPrimordialSCM(ConfigurationSource conf) {
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index bb0e677a8c..5eab4bd2b2 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2142,14 +2142,6 @@
validation from leader scm during bootstrap
</description>
</property>
- <property>
- <name>ozone.scm.ratis.enable</name>
- <value>false</value>
- <tag>OZONE, SCM, HA, RATIS</tag>
- <description>Property to enable or disable Ratis server on SCM.
- Please note - this is a temporary property to disable SCM Ratis server.
- </description>
- </property>
<property>
<name>ozone.scm.ratis.port</name>
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
index a2a83a2e58..020291e9d2 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
@@ -18,13 +18,18 @@
package org.apache.hadoop.hdds.scm.ha;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.conf.ConfigurationException;
+import org.apache.hadoop.hdds.conf.DefaultConfigManager;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmUtils;
+import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.ozone.ha.ConfUtils;
import org.apache.hadoop.ozone.util.OzoneNetUtils;
+import org.apache.logging.log4j.util.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,15 +52,15 @@ import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRES
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_DIRS;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTPS_ADDRESS_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTPS_BIND_HOST_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTP_BIND_HOST_KEY;
-import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PORT_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PORT_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY;
-import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY;
@@ -144,7 +149,51 @@ public class SCMHANodeDetails {
return new SCMHANodeDetails(scmNodeDetails, Collections.emptyList());
}
- public static SCMHANodeDetails loadSCMHAConfig(OzoneConfiguration conf)
+ /** Validates SCM HA Config.
+ For Non Initialized SCM the value is taken directly based on the config
+ {@link org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY}
+ which defaults to
+ {@link org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_ENABLE_DEFAULT}
+ For Previously Initialized SCM the values are taken from the version file
+ Ratis SCM -> Non Ratis SCM & vice versa is not supported
+ This values is validated with the config provided.
+ **/
+ private static void validateSCMHAConfig(SCMStorageConfig scmStorageConfig,
+ OzoneConfiguration conf) {
+ Storage.StorageState state = scmStorageConfig.getState();
+ boolean scmHAEnableDefault = state == Storage.StorageState.INITIALIZED
+ ? scmStorageConfig.isSCMHAEnabled()
+ : SCMHAUtils.isSCMHAEnabled(conf);
+ boolean scmHAEnabled = SCMHAUtils.isSCMHAEnabled(conf);
+
+ if (Storage.StorageState.INITIALIZED.equals(state) &&
+ scmHAEnabled != scmHAEnableDefault) {
+ String errorMessage = String.format("Current State of SCM: %s",
+ scmHAEnableDefault ? "Ratis SCM is enabled "
+ : "SCM is running in Non-HA without Ratis")
+ + " Ratis SCM -> Non Ratis SCM or " +
+ "Non HA SCM -> HA SCM is not supported";
+ if (Strings.isNotEmpty(conf.get(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY)))
{
+ throw new ConfigurationException(String.format("Invalid Config %s " +
+ "Provided ConfigValue: %s, Expected Config Value: %s. %s",
+ ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, scmHAEnabled,
+ scmHAEnableDefault, errorMessage));
+ } else {
+ LOG.warn("Invalid config {}. The config was not specified, " +
+ "but the default value {} conflicts with " +
+ "the expected config value {}. " +
+ "Falling back to the expected value. {}",
+ ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY,
+ ScmConfigKeys.OZONE_SCM_HA_ENABLE_DEFAULT,
+ scmHAEnableDefault, errorMessage);
+ }
+ }
+ DefaultConfigManager.setConfigValue(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY,
+ scmHAEnableDefault);
+ }
+
+ public static SCMHANodeDetails loadSCMHAConfig(OzoneConfiguration conf,
+ SCMStorageConfig
storageConfig)
throws IOException {
InetSocketAddress localRpcAddress = null;
String localScmServiceId = null;
@@ -158,7 +207,7 @@ public class SCMHANodeDetails {
ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID);
LOG.info("ServiceID for StorageContainerManager is {}", localScmServiceId);
-
+ validateSCMHAConfig(storageConfig, conf);
if (localScmServiceId == null) {
// There is no internal scm service id is being set, fall back to ozone
// .scm.service.ids.
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 3e5deb8cbe..9d9af56035 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -317,15 +317,16 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
Objects.requireNonNull(configurator, "configurator cannot not be null");
Objects.requireNonNull(conf, "configuration cannot not be null");
-
- scmHANodeDetails = SCMHANodeDetails.loadSCMHAConfig(conf);
- configuration = conf;
- initMetrics();
- containerReportCache = buildContainerReportCache();
/**
* It is assumed the scm --init command creates the SCM Storage Config.
*/
scmStorageConfig = new SCMStorageConfig(conf);
+
+ scmHANodeDetails = SCMHANodeDetails.loadSCMHAConfig(conf,
scmStorageConfig);
+ configuration = conf;
+ initMetrics();
+ containerReportCache = buildContainerReportCache();
+
if (scmStorageConfig.getState() != StorageState.INITIALIZED) {
String errMsg = "Please make sure you have run \'ozone scm --init\' " +
"command to generate all the required metadata to " +
@@ -1024,7 +1025,9 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
return false;
}
String primordialSCM = SCMHAUtils.getPrimordialSCM(conf);
- SCMHANodeDetails scmhaNodeDetails = SCMHANodeDetails.loadSCMHAConfig(conf);
+ SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
+ SCMHANodeDetails scmhaNodeDetails = SCMHANodeDetails.loadSCMHAConfig(conf,
+ scmStorageConfig);
String selfNodeId = scmhaNodeDetails.getLocalNodeDetails().getNodeId();
final String selfHostName =
scmhaNodeDetails.getLocalNodeDetails().getHostName();
@@ -1035,7 +1038,6 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
+ "{}, self id {} " + "Ignoring it.", primordialSCM, selfNodeId);
return true;
}
- SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
final String persistedClusterId = scmStorageConfig.getClusterID();
StorageState state = scmStorageConfig.getState();
if (state == StorageState.INITIALIZED && conf
@@ -1136,7 +1138,8 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
String clusterId) throws IOException {
SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
StorageState state = scmStorageConfig.getState();
- final SCMHANodeDetails haDetails = SCMHANodeDetails.loadSCMHAConfig(conf);
+ final SCMHANodeDetails haDetails = SCMHANodeDetails.loadSCMHAConfig(conf,
+ scmStorageConfig);
String primordialSCM = SCMHAUtils.getPrimordialSCM(conf);
final String selfNodeId = haDetails.getLocalNodeDetails().getNodeId();
final String selfHostName = haDetails.getLocalNodeDetails().getHostName();
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java
index 7f71971174..3d0dc963e8 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import org.apache.hadoop.hdds.scm.ha.SCMHAManagerStub;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
@@ -636,6 +637,7 @@ public final class HddsTestUtils {
String scmId = UUID.randomUUID().toString();
scmStore.setClusterId(clusterId);
scmStore.setScmId(scmId);
+ scmStore.setSCMHAFlag(SCMHAUtils.isSCMHAEnabled(conf));
// writes the version file properties
scmStore.initialize();
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
index c9d542777a..4c5522275f 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
@@ -17,15 +17,24 @@
*/
package org.apache.hadoop.hdds.scm.ha;
+import org.apache.hadoop.hdds.conf.ConfigurationException;
+import org.apache.hadoop.hdds.conf.DefaultConfigManager;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.junit.Assert;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.Mockito;
+import java.io.IOException;
import java.net.InetSocketAddress;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY;
@@ -115,8 +124,10 @@ public class TestSCMHAConfiguration {
scmServiceId, nodeId), port++);
}
-
- SCMHANodeDetails.loadSCMHAConfig(conf);
+ SCMStorageConfig scmStorageConfig = Mockito.mock(SCMStorageConfig.class);
+ Mockito.when(scmStorageConfig.getState())
+ .thenReturn(Storage.StorageState.NOT_INITIALIZED);
+ SCMHANodeDetails.loadSCMHAConfig(conf, scmStorageConfig);
port = 9880;
@@ -209,8 +220,11 @@ public class TestSCMHAConfiguration {
conf.set(OZONE_SCM_DATANODE_PORT_KEY, "9898");
conf.set(OZONE_SCM_SECURITY_SERVICE_PORT_KEY, "9899");
-
- SCMHANodeDetails scmhaNodeDetails = SCMHANodeDetails.loadSCMHAConfig(conf);
+ SCMStorageConfig scmStorageConfig = Mockito.mock(SCMStorageConfig.class);
+ Mockito.when(scmStorageConfig.getState())
+ .thenReturn(Storage.StorageState.NOT_INITIALIZED);
+ SCMHANodeDetails scmhaNodeDetails =
+ SCMHANodeDetails.loadSCMHAConfig(conf, scmStorageConfig);
Assertions.assertEquals("10000", conf.get(OZONE_SCM_RATIS_PORT_KEY));
Assertions.assertEquals("10001", conf.get(OZONE_SCM_GRPC_PORT_KEY));
@@ -259,4 +273,52 @@ public class TestSCMHAConfiguration {
}
+
+ @Test
+ public void testRatisEnabledDefaultConfigWithoutInitializedSCM()
+ throws IOException, NoSuchFieldException, IllegalAccessException {
+ SCMStorageConfig scmStorageConfig = Mockito.mock(SCMStorageConfig.class);
+ Mockito.when(scmStorageConfig.getState())
+ .thenReturn(Storage.StorageState.NOT_INITIALIZED);
+ SCMHANodeDetails.loadSCMHAConfig(conf, scmStorageConfig);
+ Assert.assertEquals(SCMHAUtils.isSCMHAEnabled(conf),
+ ScmConfigKeys.OZONE_SCM_HA_ENABLE_DEFAULT);
+ DefaultConfigManager.clearDefaultConfigs();
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, false);
+ SCMHANodeDetails.loadSCMHAConfig(conf, scmStorageConfig);
+ Assert.assertEquals(SCMHAUtils.isSCMHAEnabled(conf), false);
+ DefaultConfigManager.clearDefaultConfigs();
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+ SCMHANodeDetails.loadSCMHAConfig(conf, scmStorageConfig);
+ Assert.assertEquals(SCMHAUtils.isSCMHAEnabled(conf), true);
+ }
+
+ @Test
+ public void testRatisEnabledDefaultConfigWithInitializedSCM()
+ throws IOException, NoSuchFieldException, IllegalAccessException {
+ SCMStorageConfig scmStorageConfig = Mockito.mock(SCMStorageConfig.class);
+ Mockito.when(scmStorageConfig.getState())
+ .thenReturn(Storage.StorageState.INITIALIZED);
+ Mockito.when(scmStorageConfig.isSCMHAEnabled()).thenReturn(false);
+ DefaultConfigManager.clearDefaultConfigs();
+ SCMHANodeDetails.loadSCMHAConfig(conf, scmStorageConfig);
+ Assert.assertEquals(SCMHAUtils.isSCMHAEnabled(conf),
+ scmStorageConfig.isSCMHAEnabled());
+ Mockito.when(scmStorageConfig.isSCMHAEnabled()).thenReturn(false);
+ DefaultConfigManager.clearDefaultConfigs();
+ Assert.assertEquals(SCMHAUtils.isSCMHAEnabled(conf), true);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRatisEnabledDefaultConflictConfigWithInitializedSCM(
+ boolean isRatisEnabled) {
+ SCMStorageConfig scmStorageConfig = Mockito.mock(SCMStorageConfig.class);
+ Mockito.when(scmStorageConfig.getState())
+ .thenReturn(Storage.StorageState.INITIALIZED);
+ Mockito.when(scmStorageConfig.isSCMHAEnabled()).thenReturn(isRatisEnabled);
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, !isRatisEnabled);
+ Assertions.assertThrows(ConfigurationException.class,
+ () -> SCMHANodeDetails.loadSCMHAConfig(conf, scmStorageConfig));
+ }
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
index f781c3f13c..e4b2977fa1 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
@@ -174,6 +174,7 @@ public class TestSCMNodeManager {
TimeUnit.MILLISECONDS);
conf.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
conf.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10);
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, false);
return conf;
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestSCMHAUnfinalizedStateValidationAction.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestSCMHAUnfinalizedStateValidationAction.java
index 17ccc35445..f658805a7b 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestSCMHAUnfinalizedStateValidationAction.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestSCMHAUnfinalizedStateValidationAction.java
@@ -17,10 +17,12 @@
*/
package org.apache.hadoop.hdds.scm.upgrade;
+import org.apache.hadoop.hdds.conf.ConfigurationException;
+import org.apache.hadoop.hdds.conf.DefaultConfigManager;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -74,18 +76,28 @@ public class TestSCMHAUnfinalizedStateValidationAction {
conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, dataPath.toString());
conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, dataPath.toString());
// This init should always succeed, since SCM is not pre-finalized yet.
+ DefaultConfigManager.clearDefaultConfigs();
boolean initResult1 = StorageContainerManager.scmInit(conf, CLUSTER_ID);
Assertions.assertTrue(initResult1);
// Set up new pre-finalized SCM.
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY,
haEnabledPreFinalized);
+ /* Clusters from Ratis SCM -> Non Ratis SCM
+ Ratis SCM -> Non Ratis SCM not supported
+ */
+ if (haEnabledPreFinalized != haEnabledBefore) {
+ Assertions.assertThrows(ConfigurationException.class,
+ () -> StorageContainerManager.scmInit(conf, CLUSTER_ID));
+ return;
+ }
StorageContainerManager scm = HddsTestUtils.getScm(conf);
Assertions.assertEquals(UpgradeFinalizer.Status.FINALIZATION_REQUIRED,
scm.getFinalizationManager().getUpgradeFinalizer().getStatus());
final boolean shouldFail = !haEnabledBefore && haEnabledPreFinalized;
+ DefaultConfigManager.clearDefaultConfigs();
if (shouldFail) {
// Start on its own should fail.
Assertions.assertThrows(UpgradeException.class, scm::start);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 59c2c21672..bf574e3a90 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.recon.ConfigurationProvider;
import org.apache.hadoop.ozone.recon.ReconServer;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.logging.log4j.util.Strings;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.commons.io.FileUtils;
@@ -733,7 +734,6 @@ public class MiniOzoneClusterImpl implements
MiniOzoneCluster {
}
return scm;
}
-
protected void initializeScmStorage(SCMStorageConfig scmStore)
throws IOException {
if (scmStore.getState() == StorageState.INITIALIZED) {
@@ -745,11 +745,16 @@ public class MiniOzoneClusterImpl implements
MiniOzoneCluster {
}
scmStore.setScmId(scmId.get());
scmStore.initialize();
- if (SCMHAUtils.isSCMHAEnabled(conf)) {
+ //TODO: HDDS-6897
+ //Disabling Ratis for only of MiniOzoneClusterImpl.
+ //MiniOzoneClusterImpl doesn't work with Ratis enabled SCM
+ if (Strings.isNotEmpty(conf.get(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY))
+ && SCMHAUtils.isSCMHAEnabled(conf)) {
scmStore.setSCMHAFlag(true);
scmStore.persistCurrentState();
SCMRatisServerImpl.initialize(clusterId, scmId.get(),
- SCMHANodeDetails.loadSCMHAConfig(conf).getLocalNodeDetails(),
conf);
+ SCMHANodeDetails.loadSCMHAConfig(conf, scmStore)
+ .getLocalNodeDetails(), conf);
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index f9245268f1..5fd2eb0ea7 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -97,6 +97,7 @@ public class TestOzoneConfigurationFields extends
TestConfigurationFieldsBase {
OMConfigKeys.OZONE_OM_DECOMMISSIONED_NODES_KEY,
ScmConfigKeys.OZONE_SCM_NODES_KEY,
ScmConfigKeys.OZONE_SCM_ADDRESS_KEY,
+ ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY,
OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY,
OMConfigKeys.OZONE_FS_TRASH_CHECKPOINT_INTERVAL_KEY,
OMConfigKeys.OZONE_OM_S3_GPRC_SERVER_ENABLED,
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 34b0742fbd..76776f86a1 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -34,9 +34,11 @@ import java.util.UUID;
import java.util.concurrent.Callable;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.DefaultConfigManager;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.scm.ScmConfig;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.ha.HASecurityUtils;
@@ -184,7 +186,7 @@ public final class TestSecureOzoneCluster {
conf.setInt(OZONE_SCM_RATIS_PORT_KEY, getPort(1200, 100));
conf.setInt(OZONE_SCM_GRPC_PORT_KEY, getPort(1201, 100));
conf.set(OZONE_OM_ADDRESS_KEY, "localhost:1202");
-
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, false);
DefaultMetricsSystem.setMiniClusterMode(true);
ExitUtils.disableSystemExit();
@@ -218,6 +220,7 @@ public final class TestSecureOzoneCluster {
} catch (Exception e) {
LOG.error("Failed to stop TestSecureOzoneCluster", e);
}
+ DefaultConfigManager.clearDefaultConfigs();
}
private void createCredentialsInKDC() throws Exception {
@@ -388,7 +391,8 @@ public final class TestSecureOzoneCluster {
scmStore.initialize();
if (SCMHAUtils.isSCMHAEnabled(conf)) {
SCMRatisServerImpl.initialize(clusterId, scmId,
- SCMHANodeDetails.loadSCMHAConfig(conf).getLocalNodeDetails(), conf);
+ SCMHANodeDetails.loadSCMHAConfig(conf, scmStore)
+ .getLocalNodeDetails(), conf);
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 7fc05e92b1..d352233a68 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -24,6 +24,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.conf.DefaultConfigManager;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -75,6 +76,7 @@ import org.apache.ozone.test.GenericTestUtils;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.server.RaftServerConfigKeys;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -150,6 +152,11 @@ public class TestStorageContainerManager {
}
}
+ @After
+ public void cleanupDefaults() {
+ DefaultConfigManager.clearDefaultConfigs();
+ }
+
@Test
public void testRpcPermission() throws Exception {
// Test with default configuration
@@ -491,7 +498,7 @@ public class TestStorageContainerManager {
StorageContainerManager.scmInit(conf, testClusterId);
Assert.assertEquals(NodeType.SCM, scmStore.getNodeType());
Assert.assertEquals(testClusterId, scmStore.getClusterID());
- Assert.assertFalse(scmStore.isSCMHAEnabled());
+ Assert.assertTrue(scmStore.isSCMHAEnabled());
}
@Test
@@ -535,8 +542,8 @@ public class TestStorageContainerManager {
}
}
-
- @Test
+ // Unsupported Test case. Non Ratis SCM -> Ratis SCM not supported
+ //@Test
public void testSCMReinitializationWithHAUpgrade() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
final String path = GenericTestUtils.getTempPath(
@@ -599,7 +606,10 @@ public class TestStorageContainerManager {
"Could not find any ratis group with id " + raftGroupId);
}
}
- @Test
+
+ // Non Ratis SCM -> Ratis SCM is not supported {@see HDDS-6695}
+ // Invalid Testcase
+ // @Test
public void testSCMReinitializationWithHAEnabled() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, false);
@@ -621,9 +631,11 @@ public class TestStorageContainerManager {
} catch (IOException ioe) {
// Exception is expected here
}
+
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
// This will re-initialize SCM
StorageContainerManager.scmInit(conf, clusterId);
+ cluster.getStorageContainerManager().start();
// Ratis group with cluster id exists now
validateRatisGroupExists(conf, clusterId);
SCMStorageConfig scmStore = new SCMStorageConfig(conf);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
index fb5b437334..30386fe633 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
@@ -19,18 +19,22 @@
package org.apache.hadoop.ozone.scm;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationException;
+import org.apache.hadoop.hdds.conf.DefaultConfigManager;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
-import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -38,24 +42,24 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.ozone.test.GenericTestUtils;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.Rule;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.time.Instant;
import java.util.HashMap;
import java.util.List;
import java.util.UUID;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.junit.Rule;
-
-import org.junit.rules.Timeout;
-
-import java.io.IOException;
-
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.client.ReplicationType.RATIS;
@@ -74,6 +78,8 @@ public class TestStorageContainerManagerHA {
private String scmServiceId;
private static int numOfSCMs = 3;
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestStorageContainerManagerHA.class);
@Rule
public Timeout timeout = new Timeout(300_000);
@@ -85,7 +91,7 @@ public class TestStorageContainerManagerHA {
*
* @throws IOException
*/
- @Before
+ @BeforeEach
public void init() throws Exception {
conf = new OzoneConfiguration();
conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s");
@@ -107,11 +113,12 @@ public class TestStorageContainerManagerHA {
/**
* Shutdown MiniDFSCluster.
*/
- @After
+ @AfterEach
public void shutdown() {
if (cluster != null) {
cluster.shutdown();
}
+ DefaultConfigManager.clearDefaultConfigs();
}
@Test
@@ -234,6 +241,46 @@ public class TestStorageContainerManagerHA {
StorageContainerManager.scmInit(conf2, scm2.getClusterId()));
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testHAConfig(boolean isRatisEnabled) throws Exception {
+ StorageContainerManager scm0 = cluster.getStorageContainerManager(0);
+ scm0.stop();
+ boolean isDeleted = scm0.getScmStorageConfig().getVersionFile().delete();
+ Assert.assertTrue(isDeleted);
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, isRatisEnabled);
+ final SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
+ scmStorageConfig.setClusterId(UUID.randomUUID().toString());
+ scmStorageConfig.getCurrentDir().delete();
+ scmStorageConfig.setSCMHAFlag(isRatisEnabled);
+ DefaultConfigManager.clearDefaultConfigs();
+ scmStorageConfig.initialize();
+ scm0.scmInit(conf, clusterId);
+ Assert.assertEquals(DefaultConfigManager.getValue(
+ ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, !isRatisEnabled),
+ isRatisEnabled);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testInvalidHAConfig(boolean isRatisEnabled) throws Exception {
+ StorageContainerManager scm0 = cluster.getStorageContainerManager(0);
+ scm0.stop();
+ boolean isDeleted = scm0.getScmStorageConfig().getVersionFile().delete();
+ Assert.assertTrue(isDeleted);
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, isRatisEnabled);
+ final SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
+ scmStorageConfig.setClusterId(UUID.randomUUID().toString());
+ scmStorageConfig.getCurrentDir().delete();
+ scmStorageConfig.setSCMHAFlag(!isRatisEnabled);
+ DefaultConfigManager.clearDefaultConfigs();
+ scmStorageConfig.initialize();
+ Assertions.assertThrows(ConfigurationException.class,
+ () -> StorageContainerManager.scmInit(conf, clusterId));
+ }
+
+
+
@Test
public void testBootStrapSCM() throws Exception {
StorageContainerManager scm2 =
cluster.getStorageContainerManagers().get(1);
@@ -243,6 +290,7 @@ public class TestStorageContainerManagerHA {
final SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf2);
scmStorageConfig.setClusterId(UUID.randomUUID().toString());
scmStorageConfig.getCurrentDir().delete();
+ scmStorageConfig.setSCMHAFlag(true);
scmStorageConfig.initialize();
conf2.setBoolean(ScmConfigKeys.OZONE_SCM_SKIP_BOOTSTRAP_VALIDATION_KEY,
false);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
