This is an automated email from the ASF dual-hosted git repository.
devmadhuu pushed a commit to branch HDDS-11233
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-11233 by this push:
new 8787bc21038 HDDS-15881. Support setting storagePolicy via
SetBucketProperty (#11232)
8787bc21038 is described below
commit 8787bc21038a65e75159a7c66d6a940c7ccedf79
Author: Anurag Parvatikar <[email protected]>
AuthorDate: Mon Oct 5 20:43:30 2026 +0530
HDDS-15881. Support setting storagePolicy via SetBucketProperty (#11232)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../ozone/shell/bucket/CreateBucketHandler.java | 44 ++++
.../ozone/shell/bucket/UpdateBucketHandler.java | 59 ++++++
.../apache/hadoop/ozone/client/OzoneBucket.java | 50 +++++
.../ozone/client/protocol/ClientProtocol.java | 11 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 8 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 30 +++
.../ozone/om/TestOMStoragePolicyUpgrade.java | 223 +++++++++++++++++++++
.../hadoop/ozone/shell/TestOzoneShellHA.java | 156 ++++++++++++++
.../src/main/proto/OmClientProtocol.proto | 1 +
.../om/request/bucket/OMBucketCreateRequest.java | 31 +++
.../request/bucket/OMBucketSetPropertyRequest.java | 35 ++++
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 3 +-
.../request/bucket/TestOMBucketCreateRequest.java | 64 ++++++
.../bucket/TestOMBucketSetPropertyRequest.java | 161 +++++++++++++++
.../hadoop/ozone/client/ClientProtocolStub.java | 6 +
16 files changed, 882 insertions(+), 1 deletion(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 50559ca917a..62f57a3b12c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -300,6 +300,7 @@ public final class OzoneConsts {
public static final String STORAGE_TYPE = "storageType";
public static final String STORAGE_POLICY = "storagePolicy";
public static final String ALLOW_FALLBACK_STORAGE_POLICY =
"allowFallbackStoragePolicy";
+ public static final String UNSET_STORAGE_POLICY = "unsetStoragePolicy";
public static final String RESOURCE_TYPE = "resourceType";
public static final String IS_VERSION_ENABLED = "isVersionEnabled";
public static final String CREATION_TIME = "creationTime";
diff --git
a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
index ab842f184e5..a06db5cb7b2 100644
---
a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
+++
b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
@@ -21,6 +21,8 @@
import java.io.IOException;
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.client.OzoneQuota;
+import org.apache.hadoop.hdds.client.OzoneStoragePolicy;
+import org.apache.hadoop.hdds.client.StoragePolicy;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -33,6 +35,7 @@
import org.apache.hadoop.security.UserGroupInformation;
import picocli.CommandLine;
import picocli.CommandLine.Command;
+import picocli.CommandLine.Help.Visibility;
import picocli.CommandLine.Option;
/**
@@ -67,6 +70,21 @@ public class CreateBucketHandler extends BucketHandler {
@CommandLine.Mixin
private SetSpaceQuotaOptions quotaOptions;
+ @Option(names = {"--storage-policy", "-s"},
+ description = "Bucket StoragePolicy. Allowed values: HOT, WARM, COLD,
null.",
+ defaultValue = "WARM",
+ showDefaultValue = Visibility.ALWAYS)
+ private String storagePolicyStr;
+
+ @Option(names = {"--allow-fallback-storage-policy", "-a"},
+ description = "When true, allocation may fall back to the
StoragePolicy's " +
+ "fallback tier if the creation tier is unavailable.",
+ defaultValue = "true", arity = "1",
+ showDefaultValue = Visibility.ALWAYS)
+ private boolean allowFallBackStoragePolicy;
+
+ private static final String NULL_STORAGE_POLICY = "null";
+
/**
* Executes create bucket.
*/
@@ -78,8 +96,12 @@ public void execute(OzoneClient client, OzoneAddress address)
ownerName = UserGroupInformation.getCurrentUser().getShortUserName();
}
+ StoragePolicy storagePolicy = parseStoragePolicy(storagePolicyStr);
+
BucketArgs.Builder bb =
new BucketArgs.Builder()
+ .setStoragePolicy(storagePolicy)
+ .setAllowFallbackStoragePolicy(allowFallBackStoragePolicy)
.setVersioning(false).setOwner(ownerName);
if (allowedBucketLayout != null) {
bb.setBucketLayout(allowedBucketLayout);
@@ -127,6 +149,28 @@ public void execute(OzoneClient client, OzoneAddress
address)
}
}
+ /**
+ * Parse a user-supplied {@code --storage-policy} value into a
+ * {@link StoragePolicy}. Returns {@code null} when the caller passed
+ * "null" (any case) or an empty value, to leave the bucket without an
+ * explicit StoragePolicy (the server defaults it to WARM on create).
+ *
+ * @throws IllegalArgumentException if the value is not one of HOT, WARM,
+ * COLD, or "null".
+ */
+ private static StoragePolicy parseStoragePolicy(String value) {
+ if (Strings.isNullOrEmpty(value)
+ || NULL_STORAGE_POLICY.equalsIgnoreCase(value)) {
+ return null;
+ }
+ try {
+ return OzoneStoragePolicy.valueOf(value.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Invalid storage policy: " + value
+ + ". Allowed String values are: HOT, WARM, COLD, or null.");
+ }
+ }
+
private static class LayoutConverter implements
CommandLine.ITypeConverter<BucketLayout> {
@Override
public BucketLayout convert(String value) {
diff --git
a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/UpdateBucketHandler.java
b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/UpdateBucketHandler.java
index 58f7190628f..9baad3a284b 100644
---
a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/UpdateBucketHandler.java
+++
b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/bucket/UpdateBucketHandler.java
@@ -17,7 +17,10 @@
package org.apache.hadoop.ozone.shell.bucket;
+import com.google.common.base.Strings;
import java.io.IOException;
+import org.apache.hadoop.hdds.client.OzoneStoragePolicy;
+import org.apache.hadoop.hdds.client.StoragePolicy;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.shell.OzoneAddress;
@@ -35,10 +38,33 @@ public class UpdateBucketHandler extends BucketHandler {
description = "Owner of the bucket to set")
private String ownerName;
+ @Option(names = {"--storage-policy", "-s"},
+ description = "Bucket StoragePolicy. Allowed values: HOT, WARM, COLD,
null "
+ + "(null clears the bucket's StoragePolicy). Leave unset to keep the
"
+ + "current value.")
+ private String storagePolicyStr;
+
+ @Option(names = {"--allow-fallback-storage-policy", "-a"},
+ description = "When true, allocation may fall back to the
StoragePolicy's "
+ + "fallback tier if the creation tier is unavailable. Leave unset to
"
+ + "keep the current value.",
+ arity = "1")
+ private Boolean allowFallBackStoragePolicy;
+
+ private static final String NULL_STORAGE_POLICY = "null";
+
@Override
protected void execute(OzoneClient client, OzoneAddress address)
throws IOException {
+ if (ownerName != null && !ownerName.isEmpty()
+ && (hasStoragePolicy() || allowFallBackStoragePolicy != null)) {
+ throw new IllegalArgumentException(
+ "--user cannot be combined with --storage-policy or "
+ + "--allow-fallback-storage-policy. Run separate 'bucket "
+ + "update' commands for each property.");
+ }
+
String volumeName = address.getVolumeName();
String bucketName = address.getBucketName();
OzoneBucket bucket = client.getObjectStore().getVolume(volumeName)
@@ -52,8 +78,41 @@ protected void execute(OzoneClient client, OzoneAddress
address)
}
}
+ // Update StoragePolicy / allowFallback if requested.
+ if (hasStoragePolicy() || allowFallBackStoragePolicy != null) {
+ StoragePolicy newPolicy = hasStoragePolicy() ? getStoragePolicy() : null;
+ boolean unsetPolicy = hasStoragePolicy() && newPolicy == null;
+ bucket.setStoragePolicyProperty(newPolicy, allowFallBackStoragePolicy,
unsetPolicy);
+ }
+
OzoneBucket updatedBucket = client.getObjectStore().getVolume(volumeName)
.getBucket(bucketName);
printObjectAsJson(updatedBucket);
}
+
+ private boolean hasStoragePolicy() {
+ return !Strings.isNullOrEmpty(storagePolicyStr);
+ }
+
+ /**
+ * Parse the {@code --storagepolicy} value. Returns {@code null} when the
+ * user passed "null" (any case), meaning the policy should be cleared;
+ * otherwise the matching {@link StoragePolicy}.
+ *
+ * @throws IllegalArgumentException if the value is not HOT, WARM, COLD, or
+ * "null".
+ */
+ private StoragePolicy getStoragePolicy() {
+ if (Strings.isNullOrEmpty(storagePolicyStr)
+ || NULL_STORAGE_POLICY.equalsIgnoreCase(storagePolicyStr)) {
+ return null;
+ }
+ try {
+ return OzoneStoragePolicy.valueOf(storagePolicyStr.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Invalid storage policy: "
+ + storagePolicyStr
+ + ". Allowed String values are: HOT, WARM, COLD, or null.");
+ }
+ }
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 1999c8144ca..42d86723c0b 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -55,6 +55,7 @@
import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
+import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
@@ -377,6 +378,55 @@ public void setStoragePolicy(StoragePolicy
newStoragePolicy) throws IOException
storagePolicy = newStoragePolicy;
}
+ /**
+ * Sets the bucket's storage-policy properties (storage policy,
allowFallback,
+ * or unset) on this bucket. Used by the update path, where the policy may be
+ * absent or explicitly cleared.
+ * @param newPolicy new storage policy, or {@code null} to leave unchanged
+ * @param newFallback new allowFallbackStoragePolicy, or {@code null} to
leave unchanged
+ * @param unsetPolicy whether to clear the bucket's storage policy
+ * @throws IOException
+ */
+ public void setStoragePolicyProperty(
+ StoragePolicy newPolicy,
+ Boolean newFallback,
+ boolean unsetPolicy) throws IOException {
+
+ if (newPolicy != null && unsetPolicy) {
+ throw new IllegalArgumentException(
+ "Set storagePolicy and unset storagePolicy cannot "
+ + "be given at the same time");
+ }
+
+ OmBucketArgs.Builder builder = OmBucketArgs.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(name);
+
+ if (newPolicy != null) {
+ builder.setStoragePolicy(newPolicy);
+ }
+
+ if (newFallback != null) {
+ builder.setAllowFallbackStoragePolicy(newFallback);
+ }
+
+ if (unsetPolicy) {
+ builder.setUnsetStoragePolicy(true);
+ }
+
+ proxy.setBucketStoragePolicy(builder.build());
+
+ if (unsetPolicy) {
+ storagePolicy = null;
+ } else if (newPolicy != null) {
+ storagePolicy = newPolicy;
+ }
+
+ if (newFallback != null) {
+ allowFallbackStoragePolicy = newFallback;
+ }
+ }
+
/**
* Enable/Disable versioning of the bucket.
* @param newVersioning
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 49d5b82cf3e..536a69c5a26 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -50,6 +50,7 @@
import org.apache.hadoop.ozone.om.helpers.DeleteTenantState;
import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -293,6 +294,16 @@ void setBucketStoragePolicy(String volumeName, String
bucketName,
StoragePolicy storagePolicy)
throws IOException;
+ /**
+ * Sets the storage-policy related properties of a bucket carried in the
+ * given {@link OmBucketArgs} (storage policy, allowFallback, or unset).
+ * Unlike the three-arg overload, the policy may be absent or explicitly
+ * cleared via {@link OmBucketArgs.Builder#setUnsetStoragePolicy}.
+ * @param args Bucket arguments carrying the properties to update.
+ * @throws IOException
+ */
+ void setBucketStoragePolicy(OmBucketArgs args) throws IOException;
+
/**
* Deletes a bucket if it is empty.
* @param volumeName Name of the Volume
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 0218522b1d6..46375d190ac 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -1236,6 +1236,14 @@ public void setBucketStoragePolicy(
ozoneManagerClient.setBucketProperty(builder.build());
}
+ @Override
+ public void setBucketStoragePolicy(OmBucketArgs args) throws IOException {
+ Objects.requireNonNull(args, "args == null");
+ verifyVolumeName(args.getVolumeName());
+ verifyBucketName(args.getBucketName());
+ ozoneManagerClient.setBucketProperty(args);
+ }
+
@Override
public void setBucketQuota(String volumeName, String bucketName,
long quotaInNamespace, long quotaInBytes) throws IOException {
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java
index c25173e2614..9b582a32715 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java
@@ -55,6 +55,11 @@ public final class OmBucketArgs extends WithMetadata
implements Auditable {
* the flag was not set.
*/
private final Boolean allowFallbackStoragePolicy;
+ /**
+ * Whether to clear (unset) the bucket's storage policy. {@code null} or
+ * {@code false} leaves the existing policy unchanged.
+ */
+ private final Boolean unsetStoragePolicy;
/**
* Bucket encryption key info if encryption is enabled.
@@ -81,6 +86,7 @@ private OmBucketArgs(Builder b) {
this.isVersionEnabled = b.isVersionEnabled;
this.storagePolicy = b.storagePolicy;
this.allowFallbackStoragePolicy = b.allowFallbackStoragePolicy;
+ this.unsetStoragePolicy = b.unsetStoragePolicy;
this.ownerName = b.ownerName;
this.defaultReplicationConfig = b.defaultReplicationConfig;
this.quotaInBytesSet = b.quotaInBytesSet;
@@ -132,6 +138,14 @@ public Boolean getAllowFallbackStoragePolicy() {
return allowFallbackStoragePolicy;
}
+ /**
+ * Returns whether the bucket's storage policy should be cleared (unset).
+ * @return unsetStoragePolicy (may be {@code null} when not set).
+ */
+ public Boolean getUnsetStoragePolicy() {
+ return unsetStoragePolicy;
+ }
+
/**
* Returns Bucket Quota in bytes.
* @return quotaInBytes.
@@ -213,6 +227,10 @@ public Map<String, String> toAuditMap() {
auditMap.put(OzoneConsts.ALLOW_FALLBACK_STORAGE_POLICY,
String.valueOf(this.allowFallbackStoragePolicy));
}
+ if (this.unsetStoragePolicy != null) {
+ auditMap.put(OzoneConsts.UNSET_STORAGE_POLICY,
+ String.valueOf(this.unsetStoragePolicy));
+ }
if (this.ownerName != null) {
auditMap.put(OzoneConsts.OWNER, this.ownerName);
}
@@ -249,6 +267,7 @@ public static class Builder extends WithMetadata.Builder {
private Boolean isVersionEnabled;
private StoragePolicy storagePolicy;
private Boolean allowFallbackStoragePolicy;
+ private Boolean unsetStoragePolicy;
private boolean quotaInBytesSet = false;
private long quotaInBytes;
private boolean quotaInNamespaceSet = false;
@@ -306,6 +325,11 @@ public Builder setAllowFallbackStoragePolicy(Boolean
allowFallback) {
return this;
}
+ public Builder setUnsetStoragePolicy(Boolean unset) {
+ this.unsetStoragePolicy = unset;
+ return this;
+ }
+
public Builder setQuotaInBytes(long quota) {
quotaInBytesSet = true;
quotaInBytes = quota;
@@ -371,6 +395,9 @@ public BucketArgs getProtobuf() {
if (allowFallbackStoragePolicy != null) {
builder.setAllowFallbackStoragePolicy(allowFallbackStoragePolicy);
}
+ if (unsetStoragePolicy != null) {
+ builder.setUnsetStoragePolicy(unsetStoragePolicy);
+ }
if (quotaInBytesSet && (
quotaInBytes > 0 || quotaInBytes == OzoneConsts.QUOTA_RESET)) {
builder.setQuotaInBytes(quotaInBytes);
@@ -416,6 +443,9 @@ public static Builder builderFromProtobuf(BucketArgs
bucketArgs) {
if (bucketArgs.hasAllowFallbackStoragePolicy()) {
builder.setAllowFallbackStoragePolicy(bucketArgs.getAllowFallbackStoragePolicy());
}
+ if (bucketArgs.hasUnsetStoragePolicy()) {
+ builder.setUnsetStoragePolicy(bucketArgs.getUnsetStoragePolicy());
+ }
if (bucketArgs.hasOwnerName()) {
builder.setOwnerName(bucketArgs.getOwnerName());
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStoragePolicyUpgrade.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStoragePolicyUpgrade.java
new file mode 100644
index 00000000000..f76faf9dea7
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStoragePolicyUpgrade.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static
org.apache.hadoop.ozone.om.OMUpgradeTestUtils.waitForFinalization;
+import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.util.UUID;
+import org.apache.hadoop.hdds.client.OzoneStoragePolicy;
+import org.apache.hadoop.hdds.client.StoragePolicy;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.function.Executable;
+
+/**
+ * Upgrade testing for the bucket storage policy feature.
+ * <p>
+ * Expected behavior:
+ * 1. Pre-Finalize:
+ * - CreateBucket requests that carry a storagePolicy and/or
+ * allowFallbackStoragePolicy field succeed, but those fields are
+ * silently stripped from the request before it is applied, so the
+ * created bucket ends up with no storage policy, as if the fields had
+ * never been sent.
+ * - SetBucketProperty requests that carry storagePolicy,
+ * allowFallbackStoragePolicy or unsetStoragePolicy are rejected.
+ * - Requests that carry none of these fields, such as those sent by an
+ * older client, are unaffected either way.
+ * <p>
+ * 2. Post-Finalize: CreateBucket honors the requested storage policy fields
+ * and SetBucketProperty requests carrying them succeed; in both cases the
+ * policy is persisted.
+ * <p>
+ * The cluster starts at the layout version immediately below
+ * {@link OMLayoutFeature#BUCKET_STORAGE_POLICY_SUPPORT} so that only this
+ * feature is un-finalized and the assertions cannot be satisfied by an
+ * unrelated validator.
+ * <p>
+ * Note this covers the pre-finalization gate, not a mixed-version OM ring:
+ * every OM here runs the same binary at the same metadata layout version.
+ * Real mixed-binary coverage lives in the non-rolling-upgrade acceptance suite
+ * under {@code hadoop-ozone/dist/src/main/compose/upgrade}.
+ */
+public class TestOMStoragePolicyUpgrade {
+
+ private static final String VOLUME_NAME = "vol-" + UUID.randomUUID();
+
+ private MiniOzoneCluster cluster;
+ private OzoneClient client;
+ private OzoneManagerProtocol omClient;
+ private final OzoneConfiguration conf = new OzoneConfiguration();
+
+ /** Bucket created before finalization, used by the update assertions. */
+ private String bucketName;
+
+ @BeforeEach
+ public void init() throws Exception {
+ conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY,
+ OMLayoutFeature.MPU_PARTS_TABLE_SPLIT.layoutVersion());
+
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(1)
+ .build();
+ cluster.waitForClusterToBeReady();
+ client = cluster.newClient();
+ omClient =
client.getObjectStore().getClientProxy().getOzoneManagerClient();
+
+ omClient.createVolume(new OmVolumeArgs.Builder()
+ .setVolume(VOLUME_NAME)
+ .setOwnerName("user1")
+ .setAdminName("user1")
+ .build());
+
+ // A bucket without any storage policy field, as an older client would
+ // create it. This must be allowed while pre-finalized.
+ bucketName = createBucket(null, null);
+ }
+
+ @Test
+ public void upgrade() throws Exception {
+ preFinalizationChecks();
+ finalizeOMUpgrade();
+ postFinalizationChecks();
+ }
+
+ private void preFinalizationChecks() throws Exception {
+ assertThat(cluster.getOzoneManager().getVersionManager()
+ .getMetadataLayoutVersion())
+ .isEqualTo(OMLayoutFeature.MPU_PARTS_TABLE_SPLIT.layoutVersion());
+
+ // CreateBucket carrying BucketInfo storage policy field: the request
+ // succeeds, but the storage policy field is silently stripped.
+ String strippedPolicyBucket = createBucket(OzoneStoragePolicy.COLD, null);
+ assertThat(omClient.getBucketInfo(VOLUME_NAME, strippedPolicyBucket)
+ .getStoragePolicy()).isNull();
+
+ // SetBucketProperty carrying any of the three BucketArgs fields.
+ assertRejected(() -> setStoragePolicy(OzoneStoragePolicy.HOT, true, null));
+ assertRejected(() -> setStoragePolicy(null, null, true));
+
+ // Controls: the gate must not reject requests that carry no storage
policy field
+ createBucket(null, null);
+ omClient.setBucketProperty(OmBucketArgs.newBuilder()
+ .setVolumeName(VOLUME_NAME)
+ .setBucketName(bucketName)
+ .setIsVersionEnabled(true)
+ .build());
+ assertThat(omClient.getBucketInfo(VOLUME_NAME, bucketName)
+ .getStoragePolicy()).isNull();
+ }
+
+ /**
+ * Trigger OM upgrade finalization and block until it completes. The client
id
+ * must match the one {@link OMUpgradeTestUtils#waitForFinalization} polls
+ * with.
+ */
+ private void finalizeOMUpgrade() throws Exception {
+ omClient.finalizeUpgrade("finalize-test");
+ waitForFinalization(omClient);
+ }
+
+ private void postFinalizationChecks() throws Exception {
+ String created = createBucket(OzoneStoragePolicy.HOT, true);
+ OmBucketInfo createdInfo = omClient.getBucketInfo(VOLUME_NAME, created);
+
assertThat(createdInfo.getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT);
+ assertThat(createdInfo.getAllowFallbackStoragePolicy()).isTrue();
+
+ setStoragePolicy(OzoneStoragePolicy.HOT, null, null);
+ assertThat(omClient.getBucketInfo(VOLUME_NAME, bucketName)
+ .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT);
+
+ setStoragePolicy(null, null, true);
+ assertThat(omClient.getBucketInfo(VOLUME_NAME, bucketName)
+ .getStoragePolicy()).isNull();
+ }
+
+ private void assertRejected(Executable executable) {
+ OMException ex = assertThrows(OMException.class, executable);
+ assertThat(ex.getResult())
+ .isEqualTo(NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION);
+ assertThat(ex.getMessage())
+ .contains("Cluster does not have the bucket storage policy support"
+ + " feature finalized yet");
+ }
+
+ /**
+ * Creates a bucket through {@link OzoneManagerProtocol} rather than the
+ * {@code OzoneBucket} API, because {@code RpcClient#createBucket}
substitutes
+ * the default storage policy for null and so cannot express a request
without
+ * the field.
+ *
+ * @return the name of the bucket created.
+ */
+ private String createBucket(StoragePolicy storagePolicy,
+ Boolean allowFallbackStoragePolicy) throws Exception {
+ String name = "buck-" + UUID.randomUUID();
+ OmBucketInfo.Builder builder = OmBucketInfo.newBuilder()
+ .setVolumeName(VOLUME_NAME)
+ .setBucketName(name);
+ if (storagePolicy != null) {
+ builder.setStoragePolicy(storagePolicy);
+ }
+ if (allowFallbackStoragePolicy != null) {
+ builder.setAllowFallbackStoragePolicy(allowFallbackStoragePolicy);
+ }
+ omClient.createBucket(builder.build());
+ return name;
+ }
+
+ private void setStoragePolicy(StoragePolicy storagePolicy,
+ Boolean allowFallbackStoragePolicy, Boolean unsetStoragePolicy)
+ throws Exception {
+ OmBucketArgs.Builder builder = OmBucketArgs.newBuilder()
+ .setVolumeName(VOLUME_NAME)
+ .setBucketName(bucketName);
+ if (storagePolicy != null) {
+ builder.setStoragePolicy(storagePolicy);
+ }
+ if (allowFallbackStoragePolicy != null) {
+ builder.setAllowFallbackStoragePolicy(allowFallbackStoragePolicy);
+ }
+ if (unsetStoragePolicy != null) {
+ builder.setUnsetStoragePolicy(unsetStoragePolicy);
+ }
+ omClient.setBucketProperty(builder.build());
+ }
+
+ @AfterEach
+ public void teardown() {
+ IOUtils.closeQuietly(client);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index 00b070e29d2..d7697f4c3bc 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -73,7 +73,9 @@
import org.apache.hadoop.fs.ozone.OzoneTrashPolicy;
import org.apache.hadoop.hdds.JsonTestUtils;
import org.apache.hadoop.hdds.cli.GenericCli;
+import org.apache.hadoop.hdds.client.OzoneStoragePolicy;
import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.client.StoragePolicy;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -1278,6 +1280,160 @@ public void testDeleteTrashNoSkipTrash() throws
Exception {
}
+ @Test
+ public void testShCreateBucketWithStoragePolicy() throws Exception {
+ ObjectStore objectStore = client.getObjectStore();
+ execute(ozoneShell, new String[]{"volume", "create", "spvol1"});
+ out.reset();
+
+ // No -sp: server defaults to WARM and allowFallback true.
+ execute(ozoneShell, new String[]{"bucket", "create", "spvol1/bucket1"});
+ assertThat(objectStore.getVolume("spvol1").getBucket("bucket1")
+ .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.WARM);
+ assertThat(objectStore.getVolume("spvol1").getBucket("bucket1")
+ .getAllowFallbackStoragePolicy()).isTrue();
+
+ // -sp null on create still yields a non-null policy (defaults to WARM).
+ execute(ozoneShell, new String[]{"bucket", "create", "spvol1/bucket2",
"-s", "null"});
+ assertThat(objectStore.getVolume("spvol1").getBucket("bucket2")
+ .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.WARM);
+
+ execute(ozoneShell, new String[]{"bucket", "create", "spvol1/bucket3",
"-s", "HOT"});
+ assertThat(objectStore.getVolume("spvol1").getBucket("bucket3")
+ .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT);
+
+ execute(ozoneShell,
+ new String[]{"bucket", "create", "spvol1/bucket4", "-s", "HOT", "-a",
"false"});
+ assertThat(objectStore.getVolume("spvol1").getBucket("bucket4")
+ .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT);
+ assertThat(objectStore.getVolume("spvol1").getBucket("bucket4")
+ .getAllowFallbackStoragePolicy()).isFalse();
+
+ execute(ozoneShell, new String[]{"bucket", "create", "spvol1/bucket5",
"-s", "COLD"});
+ assertThat(objectStore.getVolume("spvol1").getBucket("bucket5")
+ .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.COLD);
+
+ objectStore.getVolume("spvol1").deleteBucket("bucket1");
+ objectStore.getVolume("spvol1").deleteBucket("bucket2");
+ objectStore.getVolume("spvol1").deleteBucket("bucket3");
+ objectStore.getVolume("spvol1").deleteBucket("bucket4");
+ objectStore.getVolume("spvol1").deleteBucket("bucket5");
+ objectStore.deleteVolume("spvol1");
+ }
+
+ @Test
+ public void testShUpdateBucketStoragePolicy() throws Exception {
+ ObjectStore objectStore = client.getObjectStore();
+ execute(ozoneShell, new String[]{"volume", "create", "spvol2"});
+ out.reset();
+
+ execute(ozoneShell, new String[]{"bucket", "create", "spvol2/bucket1"});
+ assertThat(objectStore.getVolume("spvol2").getBucket("bucket1")
+ .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.WARM);
+ assertThat(objectStore.getVolume("spvol2").getBucket("bucket1")
+ .getAllowFallbackStoragePolicy()).isTrue();
+
+ // Update policy only; allowFallback is left unchanged.
+ execute(ozoneShell, new String[]{"bucket", "update", "spvol2/bucket1",
"-s", "HOT"});
+ assertThat(objectStore.getVolume("spvol2").getBucket("bucket1")
+ .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT);
+ assertThat(objectStore.getVolume("spvol2").getBucket("bucket1")
+ .getAllowFallbackStoragePolicy()).isTrue();
+
+ // Update both policy and allowFallback.
+ execute(ozoneShell,
+ new String[]{"bucket", "update", "spvol2/bucket1", "-s", "COLD", "-a",
"false"});
+ assertThat(objectStore.getVolume("spvol2").getBucket("bucket1")
+ .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.COLD);
+ assertThat(objectStore.getVolume("spvol2").getBucket("bucket1")
+ .getAllowFallbackStoragePolicy()).isFalse();
+
+ // Unset the policy via -sp null (case-insensitive).
+ execute(ozoneShell, new String[]{"bucket", "update", "spvol2/bucket1",
"-s", "NULL"});
+ assertThat(objectStore.getVolume("spvol2").getBucket("bucket1")
+ .getStoragePolicy()).isNull();
+
+ // Set it again, then unset via lowercase null.
+ execute(ozoneShell, new String[]{"bucket", "update", "spvol2/bucket1",
"-s", "HOT"});
+ assertThat(objectStore.getVolume("spvol2").getBucket("bucket1")
+ .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT);
+ execute(ozoneShell, new String[]{"bucket", "update", "spvol2/bucket1",
"-s", "null"});
+ assertThat(objectStore.getVolume("spvol2").getBucket("bucket1")
+ .getStoragePolicy()).isNull();
+
+ objectStore.getVolume("spvol2").deleteBucket("bucket1");
+ objectStore.deleteVolume("spvol2");
+ }
+
+ @Test
+ public void testShUpdateBucketStoragePolicyLocalStateStaysConsistent()
throws Exception {
+ ObjectStore objectStore = client.getObjectStore();
+ objectStore.createVolume("spvol4");
+ objectStore.getVolume("spvol4").createBucket("bucket1");
+
+ OzoneBucket bucket = objectStore.getVolume("spvol4").getBucket("bucket1");
+ bucket.setStoragePolicyProperty(OzoneStoragePolicy.COLD, false, false);
+
+ // The in-memory bucket object reflects the new values immediately,
+ // without needing to be re-fetched.
+ assertThat(bucket.getStoragePolicy()).isEqualTo(OzoneStoragePolicy.COLD);
+ assertThat(bucket.getAllowFallbackStoragePolicy()).isFalse();
+
+ // And the update was actually applied to this bucket (vol/name it was
+ // constructed for), not some other bucket.
+ OzoneBucket refetched =
objectStore.getVolume("spvol4").getBucket("bucket1");
+
assertThat(refetched.getStoragePolicy()).isEqualTo(OzoneStoragePolicy.COLD);
+ assertThat(refetched.getAllowFallbackStoragePolicy()).isFalse();
+
+ bucket.setStoragePolicyProperty(null, null, true);
+ assertThat(bucket.getStoragePolicy()).isNull();
+ assertThat(objectStore.getVolume("spvol4").getBucket("bucket1")
+ .getStoragePolicy()).isNull();
+
+ objectStore.getVolume("spvol4").deleteBucket("bucket1");
+ objectStore.deleteVolume("spvol4");
+ }
+
+ @Test
+ public void testShUpdateBucketRejectsOwnerCombinedWithStoragePolicy() throws
Exception {
+ ObjectStore objectStore = client.getObjectStore();
+ execute(ozoneShell, new String[]{"volume", "create", "spvol3"});
+ out.reset();
+
+ execute(ozoneShell, new String[]{"bucket", "create", "spvol3/bucket1"});
+ String baselineOwner =
objectStore.getVolume("spvol3").getBucket("bucket1").getOwner();
+ StoragePolicy baselinePolicy = objectStore.getVolume("spvol3")
+ .getBucket("bucket1").getStoragePolicy();
+
+ // --user combined with --storage-policy is rejected before any RPC.
+ executeWithError(ozoneShell,
+ new String[]{"bucket", "update", "spvol3/bucket1", "-u", "bob", "-s",
"COLD"},
+ "--user cannot be combined with --storage-policy");
+ assertThat(objectStore.getVolume("spvol3").getBucket("bucket1").getOwner())
+ .isEqualTo(baselineOwner);
+
assertThat(objectStore.getVolume("spvol3").getBucket("bucket1").getStoragePolicy())
+ .isEqualTo(baselinePolicy);
+
+ // --user combined with --allow-fallback-storage-policy is rejected too.
+ executeWithError(ozoneShell,
+ new String[]{"bucket", "update", "spvol3/bucket1", "-u", "bob", "-a",
"false"},
+ "--user cannot be combined with");
+ assertThat(objectStore.getVolume("spvol3").getBucket("bucket1").getOwner())
+ .isEqualTo(baselineOwner);
+ assertThat(objectStore.getVolume("spvol3").getBucket("bucket1")
+ .getAllowFallbackStoragePolicy()).isTrue();
+
+ // --user alone still works as a single RPC.
+ execute(ozoneShell, new String[]{"bucket", "update", "spvol3/bucket1",
"-u", "bob"});
+ assertThat(objectStore.getVolume("spvol3").getBucket("bucket1").getOwner())
+ .isEqualTo("bob");
+
assertThat(objectStore.getVolume("spvol3").getBucket("bucket1").getStoragePolicy())
+ .isEqualTo(baselinePolicy);
+
+ objectStore.getVolume("spvol3").deleteBucket("bucket1");
+ objectStore.deleteVolume("spvol3");
+ }
+
@Test
@SuppressWarnings("methodlength")
public void testShQuota() throws Exception {
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 94b3020f8e3..730c8bfe467 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -901,6 +901,7 @@ message BucketArgs {
repeated hadoop.hdds.KeyValue tags = 13;
optional hadoop.hdds.StoragePolicyProto storagePolicy = 14;
optional bool allowFallbackStoragePolicy = 15;
+ optional bool unsetStoragePolicy = 16;
}
message PrefixInfo {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
index 718f329aaaf..0a196004a95 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
@@ -433,6 +433,24 @@ public static OMRequest
disallowCreateBucketWithECReplicationConfig(
return req;
}
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.CreateBucket
+ )
+ public static OMRequest disallowCreateBucketWithStoragePolicy(
+ OMRequest req, ValidationContext ctx) throws OMException {
+ if (!ctx.versionManager()
+ .isAllowed(OMLayoutFeature.BUCKET_STORAGE_POLICY_SUPPORT)) {
+ BucketInfo bucketInfo = req.getCreateBucketRequest().getBucketInfo();
+ if (bucketInfo.hasStoragePolicy()
+ || bucketInfo.hasAllowFallbackStoragePolicy()) {
+ return stripStoragePolicyFromRequest(req);
+ }
+ }
+ return req;
+ }
+
@RequestFeatureValidator(
conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION,
processingPhase = RequestProcessingPhase.PRE_PROCESS,
@@ -502,4 +520,17 @@ private static OMRequest changeBucketLayout(OMRequest
originalRequest,
return originalRequest.toBuilder()
.setCreateBucketRequest(newCreateRequest).build();
}
+
+ private static OMRequest stripStoragePolicyFromRequest(OMRequest
originalRequest) {
+ CreateBucketRequest createBucketRequest =
+ originalRequest.getCreateBucketRequest();
+ BucketInfo newBucketInfo = createBucketRequest.getBucketInfo().toBuilder()
+ .clearStoragePolicy()
+ .clearAllowFallbackStoragePolicy()
+ .build();
+ CreateBucketRequest newCreateRequest = createBucketRequest.toBuilder()
+ .setBucketInfo(newBucketInfo).build();
+ return originalRequest.toBuilder()
+ .setCreateBucketRequest(newCreateRequest).build();
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
index 2911ba47c53..ac0bc77a404 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
@@ -166,11 +166,22 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
//Check StoragePolicy to update
StoragePolicy storagePolicy = omBucketArgs.getStoragePolicy();
+ Boolean unsetStoragePolicy = omBucketArgs.getUnsetStoragePolicy();
+ if (storagePolicy != null && Boolean.TRUE.equals(unsetStoragePolicy)) {
+ throw new OMException("Set storagePolicy and unset storagePolicy
cannot "
+ + "be given at the same time",
+ OMException.ResultCodes.NOT_SUPPORTED_OPERATION);
+ }
if (storagePolicy != null) {
bucketInfoBuilder.setStoragePolicy(storagePolicy);
LOG.debug("Updating bucket storage policy for bucket: {} in volume:
{}",
bucketName, volumeName);
}
+ if (Boolean.TRUE.equals(unsetStoragePolicy)) {
+ bucketInfoBuilder.setStoragePolicy(null);
+ LOG.debug("Unsetting bucket storage policy for bucket: {} in volume:
{}",
+ bucketName, volumeName);
+ }
//Check allowFallbackStoragePolicy to update
Boolean allowFallbackStoragePolicy =
@@ -386,4 +397,28 @@ public static OMRequest
disallowSetBucketPropertyWithECReplicationConfig(
}
return req;
}
+
+ @RequestFeatureValidator(
+ conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION,
+ processingPhase = RequestProcessingPhase.PRE_PROCESS,
+ requestType = Type.SetBucketProperty
+ )
+ public static OMRequest disallowSetBucketPropertyWithStoragePolicy(
+ OMRequest req, ValidationContext ctx) throws OMException {
+ if (!ctx.versionManager()
+ .isAllowed(OMLayoutFeature.BUCKET_STORAGE_POLICY_SUPPORT)) {
+ SetBucketPropertyRequest propReq = req.getSetBucketPropertyRequest();
+ if (propReq.hasBucketArgs()
+ && (propReq.getBucketArgs().hasStoragePolicy()
+ || propReq.getBucketArgs().hasAllowFallbackStoragePolicy()
+ || propReq.getBucketArgs().hasUnsetStoragePolicy())) {
+ throw new OMException("Cluster does not have the bucket storage policy"
+ + " support feature finalized yet, but the request contains
storage"
+ + " policy arguments. Rejecting the request, please finalize the"
+ + " cluster upgrade and then try again.",
+
OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION);
+ }
+ }
+ return req;
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
index 189bb3f3e1d..71d7fe4211d 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
@@ -46,7 +46,8 @@ public enum OMLayoutFeature implements LayoutFeature {
DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric
key"),
SNAPSHOT_DEFRAG(9, "Supporting defragmentation of snapshot"),
S3_LIFECYCLE_SUPPORT(10, "S3 bucket lifecycle configuration support"),
- MPU_PARTS_TABLE_SPLIT(11, "Split multipart table into separate table for
parts and key");
+ MPU_PARTS_TABLE_SPLIT(11, "Split multipart table into separate table for
parts and key"),
+ BUCKET_STORAGE_POLICY_SUPPORT(12, "Bucket storage policy and fallback
storage policy support");
/////////////////////////////// /////////////////////////////
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
index 64ba9df4e08..a14aba549af 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
@@ -19,6 +19,7 @@
import static
org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newBucketInfoBuilder;
import static
org.apache.hadoop.ozone.om.request.OMRequestTestUtils.newCreateBucketRequest;
+import static
org.apache.hadoop.ozone.om.request.validation.ValidationContext.of;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -27,6 +28,7 @@
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.List;
@@ -44,12 +46,15 @@
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.upgrade.LayoutVersionManager;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import org.junit.jupiter.api.Test;
@@ -413,6 +418,65 @@ public void testIgnoreClientACL(boolean ignoreClientACLs)
throws Exception {
}
}
+ @Test
+ public void testStripStoragePolicyBeforeFinalization() throws Exception {
+ OMRequest request = newCreateBucketRequest(
+ newBucketInfoBuilder(UUID.randomUUID().toString(),
+ UUID.randomUUID().toString())
+ .setStoragePolicy(StoragePolicyProto.HOT)
+ .setAllowFallbackStoragePolicy(true)).build();
+
+ OMRequest result =
OMBucketCreateRequest.disallowCreateBucketWithStoragePolicy(
+ request, preFinalizedContext());
+
+ assertThat(result).isNotEqualTo(request);
+ assertThat(result.getCreateBucketRequest().getBucketInfo()
+ .hasStoragePolicy()).isFalse();
+ assertThat(result.getCreateBucketRequest().getBucketInfo()
+ .hasAllowFallbackStoragePolicy()).isFalse();
+ }
+
+ /**
+ * A request that carries no storage-policy field must pass the gate
+ * untouched, since the validator's return value is chained into the next
+ * validator by the request validation framework.
+ */
+ @Test
+ public void testAllowCreateBucketWithoutStoragePolicyBeforeFinalization()
+ throws Exception {
+ OMRequest request = newCreateBucketRequest(
+ newBucketInfoBuilder(UUID.randomUUID().toString(),
+ UUID.randomUUID().toString())
+ .clearStoragePolicy()).build();
+
+ assertThat(OMBucketCreateRequest.disallowCreateBucketWithStoragePolicy(
+ request, preFinalizedContext())).isEqualTo(request);
+ }
+
+ @Test
+ public void testAllowCreateBucketWithStoragePolicyAfterFinalization()
+ throws Exception {
+ OMRequest request = newCreateBucketRequest(
+ newBucketInfoBuilder(UUID.randomUUID().toString(),
+ UUID.randomUUID().toString())
+ .setStoragePolicy(StoragePolicyProto.COLD)
+ .setAllowFallbackStoragePolicy(true)).build();
+
+ LayoutVersionManager versionManager = mock(LayoutVersionManager.class);
+ when(versionManager.isAllowed(
+ OMLayoutFeature.BUCKET_STORAGE_POLICY_SUPPORT)).thenReturn(true);
+
+ assertThat(OMBucketCreateRequest.disallowCreateBucketWithStoragePolicy(
+ request, of(versionManager, omMetadataManager))).isEqualTo(request);
+ }
+
+ private ValidationContext preFinalizedContext() {
+ LayoutVersionManager versionManager = mock(LayoutVersionManager.class);
+ when(versionManager.isAllowed(
+ OMLayoutFeature.BUCKET_STORAGE_POLICY_SUPPORT)).thenReturn(false);
+ return of(versionManager, omMetadataManager);
+ }
+
private void acceptBucketCreationHelper(String volumeName, String bucketName)
throws Exception {
OMBucketCreateRequest omBucketCreateRequest =
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java
index 2e41d4c8b17..cb1d56950ad 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java
@@ -19,28 +19,38 @@
import static org.apache.hadoop.hdds.client.ReplicationType.EC;
import static org.apache.hadoop.ozone.OzoneConsts.GB;
+import static
org.apache.hadoop.ozone.om.request.validation.ValidationContext.of;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.util.UUID;
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.OzoneStoragePolicy;
+import org.apache.hadoop.hdds.client.StoragePolicy;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketArgs;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetBucketPropertyRequest;
+import org.apache.hadoop.ozone.upgrade.LayoutVersionManager;
import org.apache.ozone.test.GenericTestUtils.LogCapturer;
import org.junit.jupiter.api.Test;
@@ -496,4 +506,155 @@ public void testSettingQuotaRetainsReplication() throws
Exception {
assertEquals(20 * GB, dbBucketInfoAfter.getQuotaInBytes());
assertEquals(1000L, dbBucketInfoAfter.getQuotaInNamespace());
}
+
+ @Test
+ public void testSettingStoragePolicy() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ OmBucketInfo.Builder bucketInfo = new OmBucketInfo.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setStoragePolicy(OzoneStoragePolicy.WARM);
+ OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager);
+ OMRequestTestUtils.addBucketToDB(omMetadataManager, bucketInfo);
+
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+ assertThat(omMetadataManager.getBucketTable().get(bucketKey)
+ .getStoragePolicy()).isEqualTo(OzoneStoragePolicy.WARM);
+
+ // Set the StoragePolicy to HOT and disable fallback.
+ OMClientResponse response = runSetStoragePolicy(volumeName, bucketName,
+ OzoneStoragePolicy.HOT, null, false, 1);
+ assertThat(response.getOMResponse().getSuccess()).isTrue();
+ OmBucketInfo updated = omMetadataManager.getBucketTable().get(bucketKey);
+ assertThat(updated.getStoragePolicy()).isEqualTo(OzoneStoragePolicy.HOT);
+ assertThat(updated.getAllowFallbackStoragePolicy()).isFalse();
+
+ // Set the StoragePolicy to COLD and enable fallback.
+ response = runSetStoragePolicy(volumeName, bucketName,
+ OzoneStoragePolicy.COLD, null, true, 2);
+ assertThat(response.getOMResponse().getSuccess()).isTrue();
+ updated = omMetadataManager.getBucketTable().get(bucketKey);
+ assertThat(updated.getStoragePolicy()).isEqualTo(OzoneStoragePolicy.COLD);
+ assertThat(updated.getAllowFallbackStoragePolicy()).isTrue();
+
+ // Unset the StoragePolicy.
+ response = runSetStoragePolicy(volumeName, bucketName, null, true, null,
3);
+ assertThat(response.getOMResponse().getSuccess()).isTrue();
+ updated = omMetadataManager.getBucketTable().get(bucketKey);
+ assertThat(updated.getStoragePolicy()).isNull();
+
+ // Set and unset the StoragePolicy at the same time is not allowed.
+ response = runSetStoragePolicy(volumeName, bucketName,
+ OzoneStoragePolicy.HOT, true, null, 4);
+ assertThat(response.getOMResponse().getSuccess()).isFalse();
+ assertThat(response.getOMResponse().getStatus())
+ .isEqualTo(OzoneManagerProtocolProtos.Status.NOT_SUPPORTED_OPERATION);
+ }
+
+ private OMClientResponse runSetStoragePolicy(String volumeName,
+ String bucketName, StoragePolicy storagePolicy,
+ Boolean unsetStoragePolicy, Boolean allowFallbackStoragePolicy,
+ long txnId) throws Exception {
+ OmBucketArgs.Builder argsBuilder = OmBucketArgs.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName);
+ if (storagePolicy != null) {
+ argsBuilder.setStoragePolicy(storagePolicy);
+ }
+ if (unsetStoragePolicy != null) {
+ argsBuilder.setUnsetStoragePolicy(unsetStoragePolicy);
+ }
+ if (allowFallbackStoragePolicy != null) {
+ argsBuilder.setAllowFallbackStoragePolicy(allowFallbackStoragePolicy);
+ }
+ OMRequest omRequest = OMRequest.newBuilder().setSetBucketPropertyRequest(
+ SetBucketPropertyRequest.newBuilder()
+ .setBucketArgs(argsBuilder.build().getProtobuf()))
+ .setCmdType(OzoneManagerProtocolProtos.Type.SetBucketProperty)
+ .setClientId(UUID.randomUUID().toString()).build();
+ return new OMBucketSetPropertyRequest(omRequest)
+ .validateAndUpdateCache(ozoneManager, txnId);
+ }
+
+ @Test
+ public void testDisallowSetStoragePolicyBeforeFinalization() {
+ assertStoragePolicyRejectedBeforeFinalization(storagePolicyRequest(
+ OzoneStoragePolicy.HOT, null, true));
+ }
+
+ @Test
+ public void testDisallowUnsetStoragePolicyBeforeFinalization() {
+ assertStoragePolicyRejectedBeforeFinalization(
+ storagePolicyRequest(null, true, null));
+ }
+
+ /**
+ * A SetBucketProperty request that carries no storage-policy field must pass
+ * the gate untouched, since the validator's return value is chained into the
+ * next validator by the request validation framework.
+ */
+ @Test
+ public void testAllowSetOtherBucketPropertyBeforeFinalization()
+ throws Exception {
+ OMRequest request = createSetBucketPropertyRequest(
+ UUID.randomUUID().toString(), UUID.randomUUID().toString(),
+ true, 100 * GB);
+
+ assertThat(OMBucketSetPropertyRequest
+ .disallowSetBucketPropertyWithStoragePolicy(
+ request, preFinalizedContext())).isEqualTo(request);
+ }
+
+ @Test
+ public void testAllowSetStoragePolicyAfterFinalization() throws Exception {
+ OMRequest request = storagePolicyRequest(OzoneStoragePolicy.COLD, null,
true);
+
+ LayoutVersionManager versionManager = mock(LayoutVersionManager.class);
+ when(versionManager.isAllowed(
+ OMLayoutFeature.BUCKET_STORAGE_POLICY_SUPPORT)).thenReturn(true);
+
+ assertThat(OMBucketSetPropertyRequest
+ .disallowSetBucketPropertyWithStoragePolicy(
+ request, of(versionManager,
omMetadataManager))).isEqualTo(request);
+ }
+
+ private void assertStoragePolicyRejectedBeforeFinalization(
+ OMRequest request) {
+ OMException ex = assertThrows(OMException.class, () ->
+ OMBucketSetPropertyRequest.disallowSetBucketPropertyWithStoragePolicy(
+ request, preFinalizedContext()));
+
+ assertThat(ex.getResult()).isEqualTo(
+ OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION);
+ }
+
+ private OMRequest storagePolicyRequest(StoragePolicy storagePolicy,
+ Boolean unsetStoragePolicy, Boolean allowFallbackStoragePolicy) {
+ OmBucketArgs.Builder argsBuilder = OmBucketArgs.newBuilder()
+ .setVolumeName(UUID.randomUUID().toString())
+ .setBucketName(UUID.randomUUID().toString());
+ if (storagePolicy != null) {
+ argsBuilder.setStoragePolicy(storagePolicy);
+ }
+ if (unsetStoragePolicy != null) {
+ argsBuilder.setUnsetStoragePolicy(unsetStoragePolicy);
+ }
+ if (allowFallbackStoragePolicy != null) {
+ argsBuilder.setAllowFallbackStoragePolicy(allowFallbackStoragePolicy);
+ }
+ return OMRequest.newBuilder().setSetBucketPropertyRequest(
+ SetBucketPropertyRequest.newBuilder()
+ .setBucketArgs(argsBuilder.build().getProtobuf()))
+ .setCmdType(OzoneManagerProtocolProtos.Type.SetBucketProperty)
+ .setClientId(UUID.randomUUID().toString()).build();
+ }
+
+ private ValidationContext preFinalizedContext() {
+ LayoutVersionManager versionManager = mock(LayoutVersionManager.class);
+ when(versionManager.isAllowed(
+ OMLayoutFeature.BUCKET_STORAGE_POLICY_SUPPORT)).thenReturn(false);
+ return of(versionManager, omMetadataManager);
+ }
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index 22e85954ae2..a4a516b0cca 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -40,6 +40,7 @@
import org.apache.hadoop.ozone.om.helpers.DeleteTenantState;
import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -198,6 +199,11 @@ public void setBucketStoragePolicy(String volumeName,
String bucketName,
}
+ @Override
+ public void setBucketStoragePolicy(OmBucketArgs args) throws IOException {
+
+ }
+
@Override
public void deleteBucket(String volumeName, String bucketName)
throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
