This is an automated email from the ASF dual-hosted git repository.
peterxcli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new f0323d7f4fc HDDS-13921. Conditional Copy (CopyObject) (#10207)
f0323d7f4fc is described below
commit f0323d7f4fc0e80f48bdad722c6e476e7f6dd39f
Author: Bolin Lin <[email protected]>
AuthorDate: Sat May 9 14:29:46 2026 -0400
HDDS-13921. Conditional Copy (CopyObject) (#10207)
---
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 101 ++++++++++
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 223 +++++++++++++++++++++
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 39 +++-
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 8 +-
.../ozone/s3/endpoint/S3ConditionalRequest.java | 74 ++++++-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 4 +
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 118 +++++++++++
7 files changed, 543 insertions(+), 24 deletions(-)
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index 42c4de1d503..8238ade3921 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -41,6 +41,8 @@
import com.amazonaws.services.s3.model.CanonicalGrantee;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
+import com.amazonaws.services.s3.model.CopyObjectRequest;
+import com.amazonaws.services.s3.model.CopyObjectResult;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
@@ -494,6 +496,105 @@ public void testPutObjectIfMatchMissingKeyFail() {
assertEquals("NoSuchKey", missingKey.getErrorCode());
}
+ @Test
+ public void testCopyObject() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(sourceBucketName);
+ s3Client.createBucket(destBucketName);
+
+ InputStream is = new
ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ PutObjectResult putResult = s3Client.putObject(sourceBucketName,
sourceKey, is, new ObjectMetadata());
+ assertEquals("37b51d194a7513e45b56f6524f2d51f2", putResult.getETag());
+
+ CopyObjectResult copyResult = s3Client.copyObject(sourceBucketName,
sourceKey, destBucketName, destKey);
+ assertEquals("37b51d194a7513e45b56f6524f2d51f2", copyResult.getETag());
+ }
+
+ @Test
+ public void testCopyObjectWithSourceIfMatch() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(sourceBucketName);
+ s3Client.createBucket(destBucketName);
+
+ InputStream is = new
ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ PutObjectResult putResult = s3Client.putObject(sourceBucketName,
sourceKey, is, new ObjectMetadata());
+ String sourceETag = putResult.getETag();
+
+ CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName,
sourceKey, destBucketName, destKey)
+ .withMatchingETagConstraint(sourceETag);
+ CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
+ assertEquals(sourceETag, copyResult.getETag());
+ }
+
+ @Test
+ public void testCopyObjectWithSourceIfMatchFail() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(sourceBucketName);
+ s3Client.createBucket(destBucketName);
+
+ InputStream is = new
ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata());
+
+ CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName,
sourceKey, destBucketName, destKey)
+ .withMatchingETagConstraint("wrong-etag");
+
+ CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
+ assertNull(copyResult);
+ }
+
+ @Test
+ public void testCopyObjectWithSourceIfNoneMatch() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(sourceBucketName);
+ s3Client.createBucket(destBucketName);
+
+ InputStream is = new
ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ PutObjectResult putResult = s3Client.putObject(sourceBucketName,
sourceKey, is, new ObjectMetadata());
+ String sourceETag = putResult.getETag();
+
+ CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName,
sourceKey, destBucketName, destKey)
+ .withNonmatchingETagConstraint("different-etag");
+ CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
+ assertEquals(sourceETag, copyResult.getETag());
+ }
+
+ @Test
+ public void testCopyObjectWithSourceIfNoneMatchFail() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(sourceBucketName);
+ s3Client.createBucket(destBucketName);
+
+ InputStream is = new
ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ PutObjectResult putResult = s3Client.putObject(sourceBucketName,
sourceKey, is, new ObjectMetadata());
+ String sourceETag = putResult.getETag();
+
+ CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName,
sourceKey, destBucketName, destKey)
+ .withNonmatchingETagConstraint(sourceETag);
+
+ CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
+ assertNull(copyResult);
+ }
+
@Test
public void testPutObjectWithMD5Header() throws Exception {
final String bucketName = getBucketName();
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index 3204e3fe5ff..28d8cbf1f61 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -992,6 +992,229 @@ public void testCopyObject() {
assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"",
copyObjectResponse.copyObjectResult().eTag());
}
+ @Test
+ public void testCopyObjectWithSourceIfMatch() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(sourceBucketName));
+ s3Client.createBucket(b -> b.bucket(destBucketName));
+
+ PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
+ .bucket(sourceBucketName)
+ .key(sourceKey),
+ RequestBody.fromString(content));
+
+ String sourceETag = putObjectResponse.eTag();
+
+ CopyObjectRequest copyReq = CopyObjectRequest.builder()
+ .sourceBucket(sourceBucketName)
+ .sourceKey(sourceKey)
+ .destinationBucket(destBucketName)
+ .destinationKey(destKey)
+ .copySourceIfMatch(sourceETag)
+ .build();
+
+ CopyObjectResponse copyObjectResponse = s3Client.copyObject(copyReq);
+ assertEquals(sourceETag, copyObjectResponse.copyObjectResult().eTag());
+ }
+
+ @Test
+ public void testCopyObjectWithSourceIfMatchFail() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(sourceBucketName));
+ s3Client.createBucket(b -> b.bucket(destBucketName));
+
+ s3Client.putObject(b -> b.bucket(sourceBucketName).key(sourceKey),
RequestBody.fromString(content));
+
+ CopyObjectRequest copyReq = CopyObjectRequest.builder()
+ .sourceBucket(sourceBucketName)
+ .sourceKey(sourceKey)
+ .destinationBucket(destBucketName)
+ .destinationKey(destKey)
+ .copySourceIfMatch("wrong-etag")
+ .build();
+
+ S3Exception exception = assertThrows(S3Exception.class, () ->
s3Client.copyObject(copyReq));
+ assertEquals(412, exception.statusCode());
+ assertEquals("PreconditionFailed",
exception.awsErrorDetails().errorCode());
+ }
+
+ @Test
+ public void testCopyObjectWithSourceIfNoneMatch() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(sourceBucketName));
+ s3Client.createBucket(b -> b.bucket(destBucketName));
+
+ PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
+ .bucket(sourceBucketName)
+ .key(sourceKey),
+ RequestBody.fromString(content));
+
+ String sourceETag = putObjectResponse.eTag();
+
+ CopyObjectRequest copyReq = CopyObjectRequest.builder()
+ .sourceBucket(sourceBucketName)
+ .sourceKey(sourceKey)
+ .destinationBucket(destBucketName)
+ .destinationKey(destKey)
+ .copySourceIfNoneMatch("different-etag")
+ .build();
+
+ CopyObjectResponse copyObjectResponse = s3Client.copyObject(copyReq);
+ assertEquals(sourceETag, copyObjectResponse.copyObjectResult().eTag());
+ }
+
+ @Test
+ public void testCopyObjectWithSourceIfNoneMatchFail() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(sourceBucketName));
+ s3Client.createBucket(b -> b.bucket(destBucketName));
+
+ PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
+ .bucket(sourceBucketName)
+ .key(sourceKey),
+ RequestBody.fromString(content));
+
+ String sourceETag = putObjectResponse.eTag();
+
+ CopyObjectRequest copyReq = CopyObjectRequest.builder()
+ .sourceBucket(sourceBucketName)
+ .sourceKey(sourceKey)
+ .destinationBucket(destBucketName)
+ .destinationKey(destKey)
+ .copySourceIfNoneMatch(sourceETag)
+ .build();
+
+ S3Exception exception = assertThrows(S3Exception.class, () ->
s3Client.copyObject(copyReq));
+ assertEquals(412, exception.statusCode());
+ assertEquals("PreconditionFailed",
exception.awsErrorDetails().errorCode());
+ }
+
+ @Test
+ public void testCopyObjectWithDestinationIfNoneMatch() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(sourceBucketName));
+ s3Client.createBucket(b -> b.bucket(destBucketName));
+
+ PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
+ .bucket(sourceBucketName)
+ .key(sourceKey),
+ RequestBody.fromString(content));
+
+ String sourceETag = putObjectResponse.eTag();
+
+ CopyObjectRequest copyReq = CopyObjectRequest.builder()
+ .sourceBucket(sourceBucketName)
+ .sourceKey(sourceKey)
+ .destinationBucket(destBucketName)
+ .destinationKey(destKey)
+ .ifNoneMatch("*")
+ .build();
+
+ CopyObjectResponse copyObjectResponse = s3Client.copyObject(copyReq);
+ assertEquals(sourceETag, copyObjectResponse.copyObjectResult().eTag());
+ }
+
+ @Test
+ public void testCopyObjectWithDestinationIfNoneMatchFail() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(sourceBucketName));
+ s3Client.createBucket(b -> b.bucket(destBucketName));
+
+ s3Client.putObject(b -> b.bucket(sourceBucketName).key(sourceKey),
RequestBody.fromString(content));
+ s3Client.putObject(b -> b.bucket(destBucketName).key(destKey),
RequestBody.fromString("existing"));
+
+ CopyObjectRequest copyReq = CopyObjectRequest.builder()
+ .sourceBucket(sourceBucketName)
+ .sourceKey(sourceKey)
+ .destinationBucket(destBucketName)
+ .destinationKey(destKey)
+ .ifNoneMatch("*")
+ .build();
+
+ S3Exception exception = assertThrows(S3Exception.class, () ->
s3Client.copyObject(copyReq));
+ assertEquals(412, exception.statusCode());
+ assertEquals("PreconditionFailed",
exception.awsErrorDetails().errorCode());
+ }
+
+ @Test
+ public void testCopyObjectWithDestinationIfMatch() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(sourceBucketName));
+ s3Client.createBucket(b -> b.bucket(destBucketName));
+
+ s3Client.putObject(b -> b.bucket(sourceBucketName).key(sourceKey),
RequestBody.fromString(content));
+ PutObjectResponse destPutResponse = s3Client.putObject(b -> b
+ .bucket(destBucketName)
+ .key(destKey),
+ RequestBody.fromString("existing"));
+ String destETag = destPutResponse.eTag();
+
+ CopyObjectRequest copyReq = CopyObjectRequest.builder()
+ .sourceBucket(sourceBucketName)
+ .sourceKey(sourceKey)
+ .destinationBucket(destBucketName)
+ .destinationKey(destKey)
+ .ifMatch(destETag)
+ .build();
+
+ CopyObjectResponse copyObjectResponse = s3Client.copyObject(copyReq);
+ assertNotNull(copyObjectResponse.copyObjectResult().eTag());
+ }
+
+ @Test
+ public void testCopyObjectWithDestinationIfMatchFail() {
+ final String sourceBucketName = getBucketName("source");
+ final String destBucketName = getBucketName("dest");
+ final String sourceKey = getKeyName("source");
+ final String destKey = getKeyName("dest");
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(sourceBucketName));
+ s3Client.createBucket(b -> b.bucket(destBucketName));
+
+ s3Client.putObject(b -> b.bucket(sourceBucketName).key(sourceKey),
RequestBody.fromString(content));
+ s3Client.putObject(b -> b.bucket(destBucketName).key(destKey),
RequestBody.fromString("existing"));
+
+ CopyObjectRequest copyReq = CopyObjectRequest.builder()
+ .sourceBucket(sourceBucketName)
+ .sourceKey(sourceKey)
+ .destinationBucket(destBucketName)
+ .destinationKey(destKey)
+ .ifMatch("wrong-etag")
+ .build();
+
+ S3Exception exception = assertThrows(S3Exception.class, () ->
s3Client.copyObject(copyReq));
+ assertEquals(412, exception.statusCode());
+ assertEquals("PreconditionFailed",
exception.awsErrorDetails().errorCode());
+ }
+
@Test
public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws
Exception {
final String bucketName = getBucketName();
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 5e301950465..2021790b332 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -388,8 +388,8 @@ Response handleGetRequest(ObjectRequestContext context,
String keyPath)
isFile(keyPath, keyDetails);
- Response conditionalResponse = S3ConditionalRequest
- .evaluateReadPreconditions(getHeaders(), keyPath, keyDetails);
+ Response conditionalResponse =
S3ConditionalRequest.evaluatePreconditions(
+ getHeaders(), keyPath, keyDetails,
S3ConditionalRequest.PreconditionContext.READ);
if (conditionalResponse != null) {
long metadataLatencyNs = getMetrics().updateGetKeyMetadataStats(
startNanos);
@@ -547,8 +547,8 @@ public Response head(
key = getClientProtocol().headS3Object(bucketName, keyPath);
isFile(keyPath, key);
- Response conditionalResponse = S3ConditionalRequest
- .evaluateReadPreconditions(getHeaders(), keyPath, key);
+ Response conditionalResponse =
S3ConditionalRequest.evaluatePreconditions(
+ getHeaders(), keyPath, key,
S3ConditionalRequest.PreconditionContext.READ);
if (conditionalResponse != null) {
getMetrics().updateHeadKeySuccessStats(startNanos);
auditReadSuccess(s3GAction);
@@ -976,20 +976,23 @@ void copy(OzoneVolume volume, DigestInputStream src, long
srcKeyLen,
ReplicationConfig replication,
Map<String, String> metadata,
PerformanceStringBuilder perf, long startNanos,
- Map<String, String> tags)
+ Map<String, String> tags,
+ S3ConditionalRequest.WriteConditions writeConditions)
throws IOException {
long copyLength;
+
if (isDatastreamEnabled() && !(replication != null &&
replication.getReplicationType() == EC) &&
srcKeyLen > getDatastreamMinLength()) {
perf.appendStreamMode();
copyLength = ObjectEndpointStreaming
.copyKeyWithStream(volume.getBucket(destBucket), destKey, srcKeyLen,
- getChunkSize(), replication, metadata, src, perf, startNanos,
tags);
+ getChunkSize(), replication, metadata, src, perf, startNanos,
tags,
+ writeConditions);
} else {
- try (OzoneOutputStream dest = getClientProtocol()
- .createKey(volume.getName(), destBucket, destKey, srcKeyLen,
- replication, metadata, tags)) {
+ try (OzoneOutputStream dest = openKeyForPut(
+ volume.getName(), destBucket, destKey, srcKeyLen,
+ replication, metadata, tags, writeConditions)) {
long metadataLatencyNs =
getMetrics().updateCopyKeyMetadataStats(startNanos);
perf.appendMetaLatencyNanos(metadataLatencyNs);
@@ -1050,6 +1053,14 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
return copyObjectResponse;
}
}
+
+ String sourceKeyPath = sourceBucket + "/" + sourceKey;
+ S3ConditionalRequest.evaluatePreconditions(getHeaders(), sourceKeyPath,
+ sourceKeyDetails,
S3ConditionalRequest.PreconditionContext.COPY_SOURCE);
+
+ S3ConditionalRequest.WriteConditions writeConditions =
+ S3ConditionalRequest.parseWriteConditions(getHeaders(), destkey);
+
long sourceKeyLen = sourceKeyDetails.getDataSize();
// Object tagging in copyObject with tagging directive
@@ -1091,7 +1102,7 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
getMetrics().updateCopyKeyMetadataStats(startNanos);
sourceDigestInputStream = new DigestInputStream(src,
getMD5DigestInstance());
copy(volume, sourceDigestInputStream, sourceKeyLen, destkey,
destBucket, replicationConfig,
- customMetadata, perf, startNanos, tags);
+ customMetadata, perf, startNanos, tags, writeConditions);
}
final OzoneKeyDetails destKeyDetails = getClientProtocol().getKeyDetails(
@@ -1104,9 +1115,17 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
return copyObjectResponse;
} catch (OMException ex) {
if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) {
+ if (getHeaders().getHeaderString(S3Consts.IF_MATCH_HEADER) != null) {
+ throw newError(PRECOND_FAILED, destkey, ex);
+ }
throw newError(S3ErrorTable.NO_SUCH_KEY, sourceKey, ex);
} else if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
throw newError(S3ErrorTable.NO_SUCH_BUCKET, sourceBucket, ex);
+ } else if (ex.getResult() == ResultCodes.ATOMIC_WRITE_CONFLICT
+ || ex.getResult() == ResultCodes.KEY_ALREADY_EXISTS
+ || ex.getResult() == ResultCodes.ETAG_MISMATCH
+ || ex.getResult() == ResultCodes.ETAG_NOT_AVAILABLE) {
+ throw newError(PRECOND_FAILED, destkey, ex);
}
throw newError(destBucket + "/" + destkey, ex);
} finally {
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
index 63f8281177f..92b5d40b963 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
@@ -186,11 +186,13 @@ public static long copyKeyWithStream(
ReplicationConfig replicationConfig,
Map<String, String> keyMetadata,
DigestInputStream body, PerformanceStringBuilder perf, long startNanos,
- Map<String, String> tags)
+ Map<String, String> tags,
+ S3ConditionalRequest.WriteConditions writeConditions)
throws IOException {
long writeLen;
- try (OzoneDataStreamOutput streamOutput = bucket.createStreamKey(keyPath,
- length, replicationConfig, keyMetadata, tags)) {
+ try (OzoneDataStreamOutput streamOutput = openStreamKeyForPut(bucket,
+ keyPath, length, replicationConfig, keyMetadata, tags,
+ writeConditions)) {
long metadataLatencyNs =
METRICS.updateCopyKeyMetadataStats(startNanos);
writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length);
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java
index 4bb6ef79e6e..259abd7d4ed 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java
@@ -45,34 +45,86 @@ final class S3ConditionalRequest {
private S3ConditionalRequest() {
}
- static Response evaluateReadPreconditions(HttpHeaders headers,
- String keyPath, OzoneKey key) throws OS3Exception {
+ /**
+ * Context for evaluating preconditions, defining which headers to check
+ * and how to handle cache validation scenarios.
+ */
+ enum PreconditionContext {
+ /**
+ * For GET/HEAD requests. Returns 304 Not Modified when If-None-Match
+ * matches or If-Modified-Since is not satisfied (cache validation).
+ */
+ READ(
+ S3Consts.IF_MATCH_HEADER,
+ S3Consts.IF_NONE_MATCH_HEADER,
+ S3Consts.IF_MODIFIED_SINCE_HEADER,
+ S3Consts.IF_UNMODIFIED_SINCE_HEADER),
+
+ /**
+ * For CopyObject source validation. Always throws 412 Precondition Failed
+ * when any condition is not met (no 304 responses).
+ */
+ COPY_SOURCE(
+ S3Consts.COPY_SOURCE_IF_MATCH,
+ S3Consts.COPY_SOURCE_IF_NONE_MATCH,
+ S3Consts.COPY_SOURCE_IF_MODIFIED_SINCE,
+ S3Consts.COPY_SOURCE_IF_UNMODIFIED_SINCE);
+
+ private final String ifMatchHeader;
+ private final String ifNoneMatchHeader;
+ private final String ifModifiedSinceHeader;
+ private final String ifUnmodifiedSinceHeader;
+
+ PreconditionContext(String ifMatchHeader, String ifNoneMatchHeader,
+ String ifModifiedSinceHeader, String ifUnmodifiedSinceHeader) {
+ this.ifMatchHeader = ifMatchHeader;
+ this.ifNoneMatchHeader = ifNoneMatchHeader;
+ this.ifModifiedSinceHeader = ifModifiedSinceHeader;
+ this.ifUnmodifiedSinceHeader = ifUnmodifiedSinceHeader;
+ }
+ }
+
+ /**
+ * Evaluates preconditions based on the given context.
+ *
+ * @param headers HTTP headers containing the conditional headers
+ * @param keyPath path of the key for error messages
+ * @param key the key metadata
+ * @param context determines which headers to check and response behavior
+ * @return Response with 304 status for READ context cache hits, null
otherwise
+ * @throws OS3Exception with 412 Precondition Failed if conditions are not
met
+ */
+ static Response evaluatePreconditions(HttpHeaders headers,
+ String keyPath, OzoneKey key, PreconditionContext context) throws
OS3Exception {
String currentETag = key.getMetadata().get(OzoneConsts.ETAG);
- String ifMatch = headers.getHeaderString(S3Consts.IF_MATCH_HEADER);
+
+ String ifMatch = headers.getHeaderString(context.ifMatchHeader);
if (ifMatch != null && !eTagMatches(ifMatch, currentETag)) {
throw newError(PRECOND_FAILED, keyPath);
}
- String ifUnmodifiedSince = headers.getHeaderString(
- S3Consts.IF_UNMODIFIED_SINCE_HEADER);
+ String ifUnmodifiedSince =
headers.getHeaderString(context.ifUnmodifiedSinceHeader);
if (ifMatch == null && ifUnmodifiedSince != null
&& !matchesIfUnmodifiedSince(key, ifUnmodifiedSince)) {
throw newError(PRECOND_FAILED, keyPath);
}
- String ifNoneMatch = headers.getHeaderString(
- S3Consts.IF_NONE_MATCH_HEADER);
+ String ifNoneMatch = headers.getHeaderString(context.ifNoneMatchHeader);
if (ifNoneMatch != null) {
if (eTagMatches(ifNoneMatch, currentETag)) {
+ if (context == PreconditionContext.COPY_SOURCE) {
+ throw newError(PRECOND_FAILED, keyPath);
+ }
return buildNotModifiedResponse(key);
}
return null;
}
- String ifModifiedSince = headers.getHeaderString(
- S3Consts.IF_MODIFIED_SINCE_HEADER);
- if (ifModifiedSince != null && !matchesIfModifiedSince(key,
- ifModifiedSince)) {
+ String ifModifiedSince =
headers.getHeaderString(context.ifModifiedSinceHeader);
+ if (ifModifiedSince != null && !matchesIfModifiedSince(key,
ifModifiedSince)) {
+ if (context == PreconditionContext.COPY_SOURCE) {
+ throw newError(PRECOND_FAILED, keyPath);
+ }
return buildNotModifiedResponse(key);
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index 67f6b4c7d41..76addc4b9e3 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -50,6 +50,10 @@ public final class S3Consts {
// Constants related to Range Header
public static final String COPY_SOURCE_IF_PREFIX = "x-amz-copy-source-if-";
+ public static final String COPY_SOURCE_IF_MATCH =
+ COPY_SOURCE_IF_PREFIX + "match";
+ public static final String COPY_SOURCE_IF_NONE_MATCH =
+ COPY_SOURCE_IF_PREFIX + "none-match";
public static final String COPY_SOURCE_IF_MODIFIED_SINCE =
COPY_SOURCE_IF_PREFIX + "modified-since";
public static final String COPY_SOURCE_IF_UNMODIFIED_SINCE =
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index 15398577b58..185d126ccec 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -84,6 +84,7 @@
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -460,6 +461,123 @@ public void testCopyObjectWithTags() throws Exception {
assertThat(e.getErrorMessage()).contains("The tagging copy directive
specified is invalid");
}
+ @Test
+ void testCopyObjectWithSourceIfMatchSuccess() throws Exception {
+ assertSucceeds(() -> putObject(CONTENT));
+ OzoneKeyDetails sourceKey = bucket.getKey(KEY_NAME);
+ String sourceETag = sourceKey.getMetadata().get(OzoneConsts.ETAG);
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+
when(headers.getHeaderString(S3Consts.COPY_SOURCE_IF_MATCH)).thenReturn("\"" +
sourceETag + "\"");
+
+ assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT));
+ assertKeyContent(destBucket, DEST_KEY, CONTENT);
+ }
+
+ @Test
+ void testCopyObjectWithSourceIfMatchFails() throws Exception {
+ assertSucceeds(() -> putObject(CONTENT));
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+
when(headers.getHeaderString(S3Consts.COPY_SOURCE_IF_MATCH)).thenReturn("\"wrong-etag\"");
+
+ assertErrorResponse(S3ErrorTable.PRECOND_FAILED,
+ () -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT));
+ }
+
+ @Test
+ void testCopyObjectWithSourceIfNoneMatchSuccess() throws Exception {
+ assertSucceeds(() -> putObject(CONTENT));
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+
when(headers.getHeaderString(S3Consts.COPY_SOURCE_IF_NONE_MATCH)).thenReturn("\"different-etag\"");
+
+ assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT));
+ assertKeyContent(destBucket, DEST_KEY, CONTENT);
+ }
+
+ @Test
+ void testCopyObjectWithSourceIfNoneMatchFails() throws Exception {
+ assertSucceeds(() -> putObject(CONTENT));
+ OzoneKeyDetails sourceKey = bucket.getKey(KEY_NAME);
+ String sourceETag = sourceKey.getMetadata().get(OzoneConsts.ETAG);
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+
when(headers.getHeaderString(S3Consts.COPY_SOURCE_IF_NONE_MATCH)).thenReturn("\""
+ sourceETag + "\"");
+
+ assertErrorResponse(S3ErrorTable.PRECOND_FAILED,
+ () -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT));
+ }
+
+ @Test
+ void testCopyObjectWithDestinationIfNoneMatchSuccess() throws Exception {
+ assertSucceeds(() -> putObject(CONTENT));
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+
when(headers.getHeaderString(S3Consts.IF_NONE_MATCH_HEADER)).thenReturn("*");
+
+ assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT));
+ assertKeyContent(destBucket, DEST_KEY, CONTENT);
+ }
+
+ @Test
+ void testCopyObjectWithDestinationIfNoneMatchFails() throws Exception {
+ assertSucceeds(() -> putObject(CONTENT));
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+ assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT));
+
+
when(headers.getHeaderString(S3Consts.IF_NONE_MATCH_HEADER)).thenReturn("*");
+ assertErrorResponse(S3ErrorTable.PRECOND_FAILED,
+ () -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT));
+ }
+
+ @Test
+ void testCopyObjectWithDestinationIfMatchSuccess() throws Exception {
+ assertSucceeds(() -> putObject(CONTENT));
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+ assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT));
+ OzoneKeyDetails destKey = destBucket.getKey(DEST_KEY);
+ String destETag = destKey.getMetadata().get(OzoneConsts.ETAG);
+
+ when(headers.getHeaderString(S3Consts.IF_MATCH_HEADER)).thenReturn("\"" +
destETag + "\"");
+ assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT));
+ }
+
+ @Test
+ void testCopyObjectWithDestinationIfMatchFails() throws Exception {
+ assertSucceeds(() -> putObject(CONTENT));
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+ assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT));
+
+
when(headers.getHeaderString(S3Consts.IF_MATCH_HEADER)).thenReturn("\"wrong-etag\"");
+
+ assertErrorResponse(S3ErrorTable.PRECOND_FAILED,
+ () -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT));
+ }
+
+ @Test
+ void testCopyObjectWithDestinationIfMatchKeyNotFound() throws Exception {
+ assertSucceeds(() -> putObject(CONTENT));
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+
when(headers.getHeaderString(S3Consts.IF_MATCH_HEADER)).thenReturn("\"some-etag\"");
+
+ assertErrorResponse(S3ErrorTable.PRECOND_FAILED,
+ () -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT));
+ }
+
@Test
void testInvalidStorageType() {
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("random");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
