This is an automated email from the ASF dual-hosted git repository.
arp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ba1bd23211 HDDS-7495. Create OBS buckets by default from S3 API (#3967)
ba1bd23211 is described below
commit ba1bd232110450be914f7971ed04b5de6e4e4780
Author: Ethan Rose <[email protected]>
AuthorDate: Wed May 10 11:36:34 2023 -0700
HDDS-7495. Create OBS buckets by default from S3 API (#3967)
---
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 18 +++++++++---
.../common/src/main/resources/ozone-default.xml | 10 ++++++-
.../apache/hadoop/ozone/client/ObjectStore.java | 33 ++++++++++++++++++++--
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 22 +++++++++++++++
.../src/main/compose/ozonesecure/docker-config | 6 ----
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 2 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 1 +
7 files changed, 77 insertions(+), 15 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 1fd797ef3d..96f9904732 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -575,14 +575,24 @@ public final class OzoneConfigKeys {
OZONE_CLIENT_BUCKET_REPLICATION_CONFIG_REFRESH_PERIOD_DEFAULT_MS =
300 * 1000;
+
+ // Values for bucket layout configurations.
+ public static final String OZONE_BUCKET_LAYOUT_LEGACY =
+ "LEGACY";
+ public static final String OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED =
+ "FILE_SYSTEM_OPTIMIZED";
+ public static final String OZONE_BUCKET_LAYOUT_OBJECT_STORE =
+ "OBJECT_STORE";
+
public static final String OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT =
"ozone.client.fs.default.bucket.layout";
-
public static final String OZONE_CLIENT_FS_BUCKET_LAYOUT_DEFAULT =
- "FILE_SYSTEM_OPTIMIZED";
+ OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED;
- public static final String OZONE_CLIENT_FS_BUCKET_LAYOUT_LEGACY =
- "LEGACY";
+ public static final String OZONE_S3G_DEFAULT_BUCKET_LAYOUT_KEY =
+ "ozone.s3g.default.bucket.layout";
+ public static final String OZONE_S3G_DEFAULT_BUCKET_LAYOUT_DEFAULT =
+ OZONE_BUCKET_LAYOUT_OBJECT_STORE;
public static final String OZONE_AUDIT_LOG_DEBUG_CMD_LIST_OMAUDIT =
"ozone.audit.log.debug.cmd.list.omaudit";
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 56451ab83f..5a3631c33c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1634,7 +1634,15 @@
or be directly or indirectly in a group defined in this property.
</description>
</property>
-
+ <property>
+ <name>ozone.s3g.default.bucket.layout</name>
+ <value>OBJECT_STORE</value>
+ <tag>OZONE, S3GATEWAY</tag>
+ <description>
+ The bucket layout that will be used when buckets are created through
+ the S3 API.
+ </description>
+ </property>
<property>
<name>ozone.readonly.administrators</name>
<value/>
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 4646afb1b3..592fb25279 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -30,9 +30,12 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.DeleteTenantState;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
@@ -73,6 +76,7 @@ public class ObjectStore {
*/
private int listCacheSize;
private final String defaultS3Volume;
+ private BucketLayout s3BucketLayout;
/**
* Creates an instance of ObjectStore.
@@ -84,6 +88,10 @@ public class ObjectStore {
this.proxy = TracingUtil.createProxy(proxy, ClientProtocol.class, conf);
this.listCacheSize = HddsClientUtils.getListCacheSize(conf);
defaultS3Volume = HddsClientUtils.getDefaultS3VolumeName(conf);
+ s3BucketLayout = OmUtils.validateBucketLayout(
+ conf.getTrimmed(
+ OzoneConfigKeys.OZONE_S3G_DEFAULT_BUCKET_LAYOUT_KEY,
+ OzoneConfigKeys.OZONE_S3G_DEFAULT_BUCKET_LAYOUT_DEFAULT));
}
@VisibleForTesting
@@ -125,10 +133,29 @@ public class ObjectStore {
* @param bucketName - S3 bucket Name.
* @throws IOException - On failure, throws an exception like Bucket exists.
*/
- public void createS3Bucket(String bucketName) throws
- IOException {
+ public void createS3Bucket(String bucketName) throws IOException {
OzoneVolume volume = getS3Volume();
- volume.createBucket(bucketName);
+ // Backwards compatibility:
+ // When OM is pre-finalized for the bucket layout feature, it will block
+ // the creation of all bucket types except legacy. If OBS bucket creation
+ // fails for this reason, retry with legacy bucket layout.
+ try {
+ volume.createBucket(bucketName,
+ BucketArgs.newBuilder().setBucketLayout(s3BucketLayout).build());
+ } catch (OMException ex) {
+ if (ex.getResult() ==
+ OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION) {
+ final BucketLayout fallbackLayout = BucketLayout.LEGACY;
+ LOG.info("Failed to create S3 bucket with layout {} since OM is " +
+ "pre-finalized for bucket layouts. Retrying creation with a " +
+ "{} bucket.",
+ s3BucketLayout, fallbackLayout);
+ volume.createBucket(bucketName, BucketArgs.newBuilder()
+ .setBucketLayout(fallbackLayout).build());
+ } else {
+ throw ex;
+ }
+ }
}
public OzoneBucket getS3Bucket(String bucketName) throws IOException {
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 940fffb603..8c40c36279 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -27,6 +27,7 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -41,6 +42,7 @@ import java.util.Comparator;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.ConfigurationException;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -49,6 +51,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.conf.OMClientConfig;
import org.apache.hadoop.ozone.ha.ConfUtils;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OMNodeDetails;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
@@ -510,6 +513,25 @@ public final class OmUtils {
}
}
+ /**
+ * Verify bucket layout is a valid.
+ *
+ * @return The {@link BucketLayout} corresponding to the string.
+ * @throws ConfigurationException If the bucket layout is not valid.
+ */
+ public static BucketLayout validateBucketLayout(String bucketLayoutString) {
+ boolean bucketLayoutValid = Arrays.stream(BucketLayout.values())
+ .anyMatch(layout -> layout.name().equals(bucketLayoutString));
+ if (bucketLayoutValid) {
+ return BucketLayout.fromString(bucketLayoutString);
+ } else {
+ throw new ConfigurationException(bucketLayoutString +
+ " is not a valid default bucket layout. Supported values are " +
+ Arrays.stream(BucketLayout.values())
+ .map(Enum::toString).collect(Collectors.joining(", ")));
+ }
+ }
+
/**
* Verify snapshot name is a valid DNS name.
*/
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index d58b098f85..dc4cae8bb8 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -22,12 +22,6 @@ CORE-SITE.XML_hadoop.proxyuser.httpfs.groups=*
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
-# TODO: HDDS-7495
-# S3 security tests need object store layout from s3 gateway. Other tests on
-# this cluster use `ozone fs`. The config to set an S3 gateway specific
bucket
-# layout will be added in HDDS-7495. Until then, the server side default must
-# be changed for all tests on this cluster to work.
-OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY
OZONE-SITE.XML_ozone.scm.http-address=scm:9876
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index 59972e85c4..753e4d16bc 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -253,7 +253,7 @@ public class TestRootedOzoneFileSystem {
} else {
bucketLayout = BucketLayout.LEGACY;
conf.set(OzoneConfigKeys.OZONE_CLIENT_FS_DEFAULT_BUCKET_LAYOUT,
- OzoneConfigKeys.OZONE_CLIENT_FS_BUCKET_LAYOUT_LEGACY);
+ OzoneConfigKeys.OZONE_BUCKET_LAYOUT_LEGACY);
conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
bucketLayout.name());
conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index e4237f6b9d..2597664774 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -577,6 +577,7 @@ public abstract class TestOzoneRpcClientAbstract {
OzoneBucket bucket = store.getS3Bucket(bucketName);
Assert.assertEquals(bucketName, bucket.getName());
Assert.assertFalse(bucket.getCreationTime().isBefore(testStartTime));
+ Assert.assertEquals(BucketLayout.OBJECT_STORE, bucket.getBucketLayout());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
