This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ffe53e7 HDDS-6019. Fix s3g create key error in cluster with
replication one. (#2855)
ffe53e7 is described below
commit ffe53e72461d7f12283974650f9fbc9964e1808e
Author: Gui Hecheng <[email protected]>
AuthorDate: Thu Nov 25 09:31:58 2021 +0800
HDDS-6019. Fix s3g create key error in cluster with replication one. (#2855)
---
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 9 +++++++--
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 22 ++++++++++++++++++++--
.../s3/endpoint/TestAbortMultipartUpload.java | 2 ++
.../s3/endpoint/TestInitiateMultipartUpload.java | 2 ++
.../hadoop/ozone/s3/endpoint/TestListParts.java | 2 ++
.../s3/endpoint/TestMultipartUploadComplete.java | 2 ++
.../s3/endpoint/TestMultipartUploadWithCopy.java | 2 ++
.../hadoop/ozone/s3/endpoint/TestObjectDelete.java | 2 ++
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 2 ++
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 2 ++
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 2 ++
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 2 ++
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 ++++
13 files changed, 51 insertions(+), 4 deletions(-)
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index ad39658..52b9c49 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -171,7 +171,7 @@ public class ObjectEndpoint extends EndpointBase {
S3StorageType s3StorageType;
boolean storageTypeDefault;
if (storageType == null || storageType.equals("")) {
- s3StorageType = S3StorageType.getDefault();
+ s3StorageType = S3StorageType.getDefault(ozoneConfiguration);
storageTypeDefault = true;
} else {
s3StorageType = toS3StorageType(storageType);
@@ -473,7 +473,7 @@ public class ObjectEndpoint extends EndpointBase {
S3StorageType s3StorageType;
if (storageType == null || storageType.equals("")) {
- s3StorageType = S3StorageType.getDefault();
+ s3StorageType = S3StorageType.getDefault(ozoneConfiguration);
} else {
s3StorageType = toS3StorageType(storageType);
}
@@ -900,4 +900,9 @@ public class ObjectEndpoint extends EndpointBase {
return (copySourceIfModifiedSince <= lastModificationTime) &&
(lastModificationTime <= copySourceIfUnmodifiedSince);
}
+
+ @VisibleForTesting
+ public void setOzoneConfiguration(OzoneConfiguration config) {
+ this.ozoneConfiguration = config;
+ }
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java
index 7c0773b..21b3b8d 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.s3.util;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
/**
* Maps S3 storage class values to Ozone replication values.
@@ -48,8 +50,24 @@ public enum S3StorageType {
return type;
}
- public static S3StorageType getDefault() {
- return STANDARD;
+ /**
+ * Get default S3StorageType for a new key to be uploaded.
+ * This should align to the ozone cluster configuration.
+ * @param config OzoneConfiguration
+ * @return S3StorageType which wraps ozone replication type and factor
+ */
+ public static S3StorageType getDefault(ConfigurationSource config) {
+ String replicationString = config.get(OzoneConfigKeys.OZONE_REPLICATION);
+ ReplicationFactor configFactor;
+ try {
+ configFactor = ReplicationFactor.valueOf(
+ Integer.parseInt(replicationString));
+ } catch (NumberFormatException ex) {
+ // conservatively defaults to STANDARD on wrong config value
+ return STANDARD;
+ }
+ return configFactor == ReplicationFactor.ONE
+ ? REDUCED_REDUNDANCY : STANDARD;
}
public static S3StorageType fromReplicationType(
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
index e23fa60..268adca 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
@@ -19,6 +19,7 @@
*/
package org.apache.hadoop.ozone.s3.endpoint;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
@@ -57,6 +58,7 @@ public class TestAbortMultipartUpload {
ObjectEndpoint rest = new ObjectEndpoint();
rest.setHeaders(headers);
rest.setClient(client);
+ rest.setOzoneConfiguration(new OzoneConfiguration());
Response response = rest.initializeMultipartUpload(bucket, key);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
index 02d0f36..2fa396f 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
@@ -20,6 +20,7 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
@@ -55,6 +56,7 @@ public class TestInitiateMultipartUpload {
ObjectEndpoint rest = new ObjectEndpoint();
rest.setHeaders(headers);
rest.setClient(client);
+ rest.setOzoneConfiguration(new OzoneConfiguration());
Response response = rest.initializeMultipartUpload(bucket, key);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
index dcec0df..26e5741 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
@@ -60,6 +61,7 @@ public class TestListParts {
REST.setHeaders(headers);
REST.setClient(client);
+ REST.setOzoneConfiguration(new OzoneConfiguration());
Response response = REST.initializeMultipartUpload(OzoneConsts.S3_BUCKET,
OzoneConsts.KEY);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
index 7626cc1..f6e70ff 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
@@ -20,6 +20,7 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
@@ -68,6 +69,7 @@ public class TestMultipartUploadComplete {
REST.setHeaders(headers);
REST.setClient(CLIENT);
+ REST.setOzoneConfiguration(new OzoneConfiguration());
}
private String initiateMultipartUpload(String key) throws IOException,
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
index f015b63..96e73de 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
@@ -34,6 +34,7 @@ import java.util.Scanner;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
@@ -120,6 +121,7 @@ public class TestMultipartUploadWithCopy {
REST.setHeaders(headers);
REST.setClient(CLIENT);
+ REST.setOzoneConfiguration(new OzoneConfiguration());
}
@Test
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectDelete.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectDelete.java
index e85f598..2eb78f4 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectDelete.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectDelete.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.s3.endpoint;
import java.io.IOException;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
@@ -47,6 +48,7 @@ public class TestObjectDelete {
ObjectEndpoint rest = new ObjectEndpoint();
rest.setClient(client);
+ rest.setOzoneConfiguration(new OzoneConfiguration());
//WHEN
rest.delete("b1", "key1", null);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
index c807f10..a5e8dd0 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
@@ -26,6 +26,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.time.format.DateTimeFormatter;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
@@ -60,6 +61,7 @@ public class TestObjectGet {
ObjectEndpoint rest = new ObjectEndpoint();
rest.setClient(client);
+ rest.setOzoneConfiguration(new OzoneConfiguration());
HttpHeaders headers = Mockito.mock(HttpHeaders.class);
rest.setHeaders(headers);
ByteArrayInputStream body =
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java
index a1b4fb1..66c7456 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
@@ -60,6 +61,7 @@ public class TestObjectHead {
// Create HeadBucket and setClient to OzoneClientStub
keyEndpoint = new ObjectEndpoint();
keyEndpoint.setClient(clientStub);
+ keyEndpoint.setOzoneConfiguration(new OzoneConfiguration());
}
@Test
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index 9684f9c..95b1244 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -26,6 +26,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
@@ -71,6 +72,7 @@ public class TestObjectPut {
// Create PutObject and setClient to OzoneClientStub
objectEndpoint = new ObjectEndpoint();
objectEndpoint.setClient(clientStub);
+ objectEndpoint.setOzoneConfiguration(new OzoneConfiguration());
}
@Test
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
index 0369688..1b0e808 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
@@ -20,6 +20,7 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
@@ -62,6 +63,7 @@ public class TestPartUpload {
REST.setHeaders(headers);
REST.setClient(client);
+ REST.setOzoneConfiguration(new OzoneConfiguration());
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
index 81b0718..237cc71 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
@@ -251,6 +251,7 @@ public class TestPermissionCheck {
ObjectEndpoint objectEndpoint = new ObjectEndpoint();
objectEndpoint.setClient(client);
objectEndpoint.setHeaders(headers);
+ objectEndpoint.setOzoneConfiguration(conf);
try {
objectEndpoint.get("bucketName", "keyPath", null, 1000, "marker",
@@ -271,6 +272,7 @@ public class TestPermissionCheck {
ObjectEndpoint objectEndpoint = new ObjectEndpoint();
objectEndpoint.setClient(client);
objectEndpoint.setHeaders(headers);
+ objectEndpoint.setOzoneConfiguration(conf);
try {
objectEndpoint.put("bucketName", "keyPath", 1024, 0, null, null);
@@ -288,6 +290,7 @@ public class TestPermissionCheck {
ObjectEndpoint objectEndpoint = new ObjectEndpoint();
objectEndpoint.setClient(client);
objectEndpoint.setHeaders(headers);
+ objectEndpoint.setOzoneConfiguration(conf);
try {
objectEndpoint.delete("bucketName", "keyPath", null);
@@ -306,6 +309,7 @@ public class TestPermissionCheck {
ObjectEndpoint objectEndpoint = new ObjectEndpoint();
objectEndpoint.setClient(client);
objectEndpoint.setHeaders(headers);
+ objectEndpoint.setOzoneConfiguration(conf);
try {
objectEndpoint.initializeMultipartUpload("bucketName", "keyPath");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
