This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch HDDS-2939
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-2939 by this push:
new 3c82503 HDDS-5094. [FSO] Fail OM startup when turn on prefix layout
with old buckets (#2151)
3c82503 is described below
commit 3c82503da3e549bb85af36063ff43d7a3889aede
Author: Rakesh Radhakrishnan <[email protected]>
AuthorDate: Fri Apr 30 12:55:01 2021 +0530
HDDS-5094. [FSO] Fail OM startup when turn on prefix layout with old
buckets (#2151)
---
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 4 -
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 5 -
.../hadoop/ozone/om/TestOMStartupWithLayout.java | 197 +++++++++++++++++++++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 109 ++++++++++--
4 files changed, 288 insertions(+), 27 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index acc335d..d322a7a 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -211,10 +211,6 @@ public class TestOzoneFileSystem {
return fs;
}
- public static boolean isEnabledFileSystemPaths() {
- return enabledFileSystemPaths;
- }
-
public static void setIsBucketFSOptimized(boolean isBucketFSO) {
isBucketFSOptimized = isBucketFSO;
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java
index 36f0cb6..c2f67be 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java
@@ -328,11 +328,6 @@ public class TestOzoneFileSystemWithFSO extends
TestOzoneFileSystem {
*/
@Test
public void testRenameDestinationParentDoesntExist() throws Exception {
- // Skip as this will run only in new layout
- if (!isEnabledFileSystemPaths()) {
- return;
- }
-
final String root = "/root_dir";
final String dir1 = root + "/dir1";
final String dir2 = dir1 + "/dir2";
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithLayout.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithLayout.java
new file mode 100644
index 0000000..4a26615
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithLayout.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
+import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT_DEFAULT;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX;
+
+/**
+ * Verifies OM startup with different layout.
+ */
+public class TestOMStartupWithLayout {
+
+ /**
+ * Set a timeout for each test.
+ */
+ @Rule
+ public Timeout timeout = Timeout.seconds(300);
+
+ private static MiniOzoneCluster cluster;
+
+ @BeforeClass
+ public static void startClusterWithSimpleLayout() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ String clusterId = UUID.randomUUID().toString();
+ String scmId = UUID.randomUUID().toString();
+ String omId = UUID.randomUUID().toString();
+ TestOMRequestUtils.configureFSOptimizedPaths(conf, true,
+ OZONE_OM_METADATA_LAYOUT_DEFAULT);
+ cluster = MiniOzoneCluster.newBuilder(conf).setClusterId(clusterId)
+ .setScmId(scmId).setOmId(omId).build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ @AfterClass
+ public static void teardown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testWithDifferentClusterLayout() throws Exception {
+ OzoneConfiguration conf = cluster.getOzoneManager().getConfiguration();
+
+ // create a volume and a bucket with default(SIMPLE) metadata format.
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ verifyBucketLayout(bucket, OZONE_OM_METADATA_LAYOUT_DEFAULT, false);
+
+ cluster.getOzoneManager().stop();
+
+ // case-1) Configured cluster layout as PREFIX. Bucket exists with SIMPLE
+ // layout format. OM startup should fail.
+ conf.set(OZONE_OM_METADATA_LAYOUT, OZONE_OM_METADATA_LAYOUT_PREFIX);
+ verifyOMStartupFailure(OZONE_OM_METADATA_LAYOUT_PREFIX);
+ verifyOMRestartFailure(OZONE_OM_METADATA_LAYOUT_PREFIX);
+
+ // case-2) Configured cluster layout as SIMPLE. Bucket exists with SIMPLE
+ // layout format. OM startup should be successful.
+ conf.set(OZONE_OM_METADATA_LAYOUT, OZONE_OM_METADATA_LAYOUT_DEFAULT);
+ // ensure everything works again with SIMPLE layout format
+ cluster.getOzoneManager().restart();
+ OzoneBucket bucket2 = TestDataUtil.createVolumeAndBucket(cluster);
+ verifyBucketLayout(bucket2, OZONE_OM_METADATA_LAYOUT_DEFAULT, false);
+
+ // Cleanup buckets so that the cluster can be started with PREFIX
+ OzoneClient client = cluster.getClient();
+ OzoneVolume volume =
+ client.getObjectStore().getVolume(bucket.getVolumeName());
+ OzoneVolume volume2 =
+ client.getObjectStore().getVolume(bucket2.getVolumeName());
+ volume.deleteBucket(bucket.getName());
+ volume2.deleteBucket(bucket2.getName());
+
+ // case-3) Configured cluster layout as PREFIX and ENABLE_FSPATH=false.
+ // OM startup should fail as this is INVALID config.
+ cluster.getOzoneManager().stop();
+ conf.set(OZONE_OM_METADATA_LAYOUT, OZONE_OM_METADATA_LAYOUT_PREFIX);
+ conf.setBoolean(OZONE_OM_ENABLE_FILESYSTEM_PATHS, false);
+ verifyOmStartWithInvalidConfig(OZONE_OM_METADATA_LAYOUT_PREFIX);
+ verifyOmRestartWithInvalidConfig(OZONE_OM_METADATA_LAYOUT_PREFIX);
+
+ // case-4) Configured cluster layout as INVALID.
+ // OM startup should fail as this is INVALID config.
+ conf.set(OZONE_OM_METADATA_LAYOUT, "INVALID");
+ verifyOmStartWithInvalidConfig("INVALID");
+ verifyOmRestartWithInvalidConfig("INVALID");
+
+ // case-5) Configured cluster layout as PREFIX and ENABLE_FSPATH=true.
+ // No buckets. OM startup should be successful.
+ conf.set(OZONE_OM_METADATA_LAYOUT, OZONE_OM_METADATA_LAYOUT_PREFIX);
+ conf.setBoolean(OZONE_OM_ENABLE_FILESYSTEM_PATHS, true);
+ cluster.getOzoneManager().restart();
+ OzoneBucket bucket3 = TestDataUtil.createVolumeAndBucket(cluster);
+ verifyBucketLayout(bucket3, OZONE_OM_METADATA_LAYOUT_PREFIX, true);
+
+ // case-6) Configured cluster layout as SIMPLE. Bucket exists with PREFIX
+ // layout format. OM startup should fail.
+ conf.set(OZONE_OM_METADATA_LAYOUT, OZONE_OM_METADATA_LAYOUT_DEFAULT);
+ cluster.getOzoneManager().stop();
+ verifyOMStartupFailure(OZONE_OM_METADATA_LAYOUT_DEFAULT);
+ verifyOMRestartFailure(OZONE_OM_METADATA_LAYOUT_DEFAULT);
+ }
+
+ private void verifyBucketLayout(OzoneBucket bucket, String metadataLayout,
+ boolean isFSOBucket) {
+ Assert.assertNotNull(bucket);
+ Assert.assertEquals(2, bucket.getMetadata().size());
+ Assert.assertEquals(isFSOBucket,
+ OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata()));
+ Assert.assertEquals(metadataLayout,
+ bucket.getMetadata().get(OZONE_OM_METADATA_LAYOUT));
+ }
+
+ private void verifyOMStartupFailure(String clusterLayout) {
+ try {
+ cluster.getOzoneManager().start();
+ Assert.fail("Should fail OM startup in " + clusterLayout + " layout");
+ } catch (IOException ioe) {
+ GenericTestUtils.assertExceptionContains(
+ "Failed to start OM in " + clusterLayout + " layout format",
+ ioe);
+ }
+ cluster.getOzoneManager().stop();
+ }
+
+ private void verifyOMRestartFailure(String clusterLayout) {
+ try {
+ cluster.getOzoneManager().restart();
+ Assert.fail("Should fail OM startup in " + clusterLayout + " layout");
+ } catch (IOException ioe) {
+ GenericTestUtils.assertExceptionContains(
+ "Failed to start OM in " + clusterLayout + " layout format",
+ ioe);
+ }
+ cluster.getOzoneManager().stop();
+ }
+
+ private void verifyOmStartWithInvalidConfig(String clusterLayout)
+ throws IOException {
+ try {
+ cluster.getOzoneManager().start();
+ Assert.fail("Should fail OM startup in " + clusterLayout + " layout");
+ } catch (IllegalArgumentException iae) {
+ GenericTestUtils.assertExceptionContains(
+ "Failed to start OM in " + clusterLayout + " layout format", iae);
+ }
+ cluster.getOzoneManager().stop();
+ }
+
+ private void verifyOmRestartWithInvalidConfig(String clusterLayout)
+ throws IOException {
+ try {
+ cluster.getOzoneManager().restart();
+ Assert.fail("Should fail OM startup in " + clusterLayout + " layout");
+ } catch (IllegalArgumentException iae) {
+ GenericTestUtils.assertExceptionContains(
+ "Failed to start OM in " + clusterLayout + " layout format", iae);
+ }
+ cluster.getOzoneManager().stop();
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 1132fe8..ea83ded 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -84,6 +84,8 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.hdds.utils.db.DBUpdatesWrapper;
import org.apache.hadoop.hdds.utils.db.SequenceNumberNotFoundException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.io.Text;
@@ -1103,23 +1105,23 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
* Start service.
*/
public void start() throws IOException {
+ initFSOLayout();
+
omClientProtocolMetrics.register();
HddsServerUtil.initializeMetrics(configuration, "OzoneManager");
LOG.info(buildRpcServerStartMessage("OzoneManager RPC server",
omRpcAddress));
+ metadataManager.start(configuration);
+
+ validatesBucketLayoutMismatches();
+
// Start Ratis services
if (omRatisServer != null) {
omRatisServer.start();
}
- // TODO: Temporary workaround for OM upgrade path and will be replaced once
- // upgrade HDDS-3698 story reaches consensus. Instead of cluster level
- // configuration, OM needs to check this property on every bucket level.
- getOMMetadataLayout();
-
- metadataManager.start(configuration);
startSecretManagerIfNecessary();
@@ -1173,6 +1175,8 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
* Restarts the service. This method re-initializes the rpc server.
*/
public void restart() throws IOException {
+ initFSOLayout();
+
LOG.info(buildRpcServerStartMessage("OzoneManager RPC server",
omRpcAddress));
@@ -1180,6 +1184,8 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
instantiateServices();
+ validatesBucketLayoutMismatches();
+
startSecretManagerIfNecessary();
// Set metrics and start metrics back ground thread
@@ -3696,18 +3702,8 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
public String getOMMetadataLayout() {
- String version = configuration.getTrimmed(OZONE_OM_METADATA_LAYOUT,
- OZONE_OM_METADATA_LAYOUT_DEFAULT);
- boolean omMetadataLayoutPrefix = StringUtils.equalsIgnoreCase(version,
- OZONE_OM_METADATA_LAYOUT_PREFIX);
- LOG.info("Configured {}={} and enabled:{} optimized OM FS operations",
- OZONE_OM_METADATA_LAYOUT, version, omMetadataLayoutPrefix);
-
- boolean isBucketFSOptimized =
- omMetadataLayoutPrefix && getEnableFileSystemPaths();
- OzoneManagerRatisUtils.setBucketFSOptimized(isBucketFSOptimized);
-
- return version;
+ return configuration
+ .getTrimmed(OZONE_OM_METADATA_LAYOUT,
OZONE_OM_METADATA_LAYOUT_DEFAULT);
}
/**
@@ -3816,4 +3812,81 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
this.minMultipartUploadPartSize = partSizeForTest;
}
+ private void initFSOLayout() {
+ // TODO: Temporary workaround for OM upgrade path and will be replaced once
+ // upgrade HDDS-3698 story reaches consensus. Instead of cluster level
+ // configuration, OM needs to check this property on every bucket level.
+ String metaLayout = getOMMetadataLayout();
+ boolean omMetadataLayoutPrefix = StringUtils.equalsIgnoreCase(metaLayout,
+ OZONE_OM_METADATA_LAYOUT_PREFIX);
+
+ boolean omMetadataLayoutSimple = StringUtils.equalsIgnoreCase(metaLayout,
+ OZONE_OM_METADATA_LAYOUT_DEFAULT);
+
+ if (!(omMetadataLayoutPrefix || omMetadataLayoutSimple)) {
+ StringBuilder msg = new StringBuilder();
+ msg.append("Invalid Configuration. Failed to start OM in ");
+ msg.append(metaLayout);
+ msg.append(" layout format. Supported values are either ");
+ msg.append(OZONE_OM_METADATA_LAYOUT_DEFAULT);
+ msg.append(" or ");
+ msg.append(OZONE_OM_METADATA_LAYOUT_PREFIX);
+
+ LOG.error(msg.toString());
+ throw new IllegalArgumentException(msg.toString());
+ }
+
+ if (omMetadataLayoutPrefix && !getEnableFileSystemPaths()) {
+ StringBuilder msg = new StringBuilder();
+ msg.append("Invalid Configuration. Failed to start OM in ");
+ msg.append(OZONE_OM_METADATA_LAYOUT_PREFIX);
+ msg.append(" layout format as '");
+ msg.append(OZONE_OM_ENABLE_FILESYSTEM_PATHS);
+ msg.append("' is false!");
+
+ LOG.error(msg.toString());
+ throw new IllegalArgumentException(msg.toString());
+ }
+
+ OzoneManagerRatisUtils.setBucketFSOptimized(omMetadataLayoutPrefix);
+ String status = omMetadataLayoutPrefix ? "enabled" : "disabled";
+ LOG.info("Configured {}={} and {} optimized OM FS operations",
+ OZONE_OM_METADATA_LAYOUT, metaLayout, status);
+ }
+
+ private void validatesBucketLayoutMismatches() throws IOException {
+ String clusterLevelMetaLayout = getOMMetadataLayout();
+
+ TableIterator<String, ? extends Table.KeyValue<String, OmBucketInfo>>
+ iterator = metadataManager.getBucketTable().iterator();
+
+ while (iterator.hasNext()) {
+ Map<String, String> bucketMeta =
iterator.next().getValue().getMetadata();
+ verifyBucketMetaLayout(clusterLevelMetaLayout, bucketMeta);
+ }
+ }
+
+ private void verifyBucketMetaLayout(String clusterLevelMetaLayout,
+ Map<String, String> bucketMetadata) throws IOException {
+ String bucketMetaLayout = bucketMetadata.get(OZONE_OM_METADATA_LAYOUT);
+ if (StringUtils.isBlank(bucketMetaLayout)) {
+ // Defaulting to SIMPLE
+ bucketMetaLayout = OZONE_OM_METADATA_LAYOUT_DEFAULT;
+ }
+ boolean supportedMetadataLayout =
+ StringUtils.equalsIgnoreCase(clusterLevelMetaLayout, bucketMetaLayout);
+
+ if (!supportedMetadataLayout) {
+ StringBuilder msg = new StringBuilder();
+ msg.append("Failed to start OM in ");
+ msg.append(clusterLevelMetaLayout);
+ msg.append(" layout format as existing bucket has a different layout ");
+ msg.append(bucketMetaLayout);
+ msg.append(" metadata format");
+
+ LOG.error(msg.toString());
+ throw new IOException(msg.toString());
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
