This is an automated email from the ASF dual-hosted git repository.
chungen0126 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 208b031453c HDDS-15140. Piggyback S3 derived key during key creation
(#10652)
208b031453c is described below
commit 208b031453cc001060d745212ff02f88256c8bec
Author: Chung En Lee <[email protected]>
AuthorDate: Wed Aug 19 04:06:08 2026 +0800
HDDS-15140. Piggyback S3 derived key during key creation (#10652)
---
.../apache/hadoop/ozone/client/OzoneBucket.java | 73 ++++++++++++++
.../ozone/client/io/OzoneDataStreamOutput.java | 8 ++
.../hadoop/ozone/client/io/OzoneOutputStream.java | 10 ++
.../ozone/client/protocol/ClientProtocol.java | 76 ++++++++++++++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 112 +++++++++++++++++++--
hadoop-ozone/common/pom.xml | 4 +
.../hadoop/ozone/om}/AWSV4AuthValidator.java | 35 ++-----
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 13 +++
.../hadoop/ozone/om/helpers/OpenKeySession.java | 10 ++
...OzoneManagerProtocolClientSideTranslatorPB.java | 7 +-
.../src/main/proto/OmClientProtocol.proto | 2 +
hadoop-ozone/ozone-manager/pom.xml | 4 -
.../org/apache/hadoop/ozone/om/OzoneManager.java | 5 +
.../ozone/om/request/key/OMKeyCreateRequest.java | 40 +++++++-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 8 +-
.../OzoneDelegationTokenSecretManager.java | 1 +
.../hadoop/ozone/security/S3SecurityUtil.java | 2 +-
.../om/request/key/TestOMKeyCreateRequest.java | 105 +++++++++++++++++++
.../ozone/security/TestAWSV4AuthValidator.java | 1 +
19 files changed, 468 insertions(+), 48 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index fe1cd7b8f2b..9b5f663b7f0 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -499,6 +499,16 @@ public OzoneOutputStream createKey(String key, long size,
.createKey(volumeName, name, key, size, replicationConfig,
keyMetadata, tags);
}
+ public OzoneOutputStream createKey(String key, long size,
+ ReplicationConfig replicationConfig,
+ Map<String, String> keyMetadata,
+ Map<String, String> tags,
+ boolean derivedKeyPiggyBacking)
+ throws IOException {
+ return proxy
+ .createKey(volumeName, name, key, size, replicationConfig,
keyMetadata, tags, derivedKeyPiggyBacking);
+ }
+
/**
* This API allows to atomically update an existing key. The key read before
invoking this API
* should remain unchanged for this key to be written. This is controlled by
the generation
@@ -539,6 +549,13 @@ public OzoneOutputStream createKeyIfNotExists(String
keyName, long size,
replicationConfig, metadata, tags);
}
+ public OzoneOutputStream createKeyIfNotExists(String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags, boolean derivedKeyPiggyBacking) throws
IOException {
+ return proxy.createKeyIfNotExists(volumeName, name, keyName, size,
+ replicationConfig, metadata, tags, derivedKeyPiggyBacking);
+ }
+
/**
* Rewrites a key only if its ETag matches (S3 If-Match semantics).
*
@@ -559,6 +576,15 @@ public OzoneOutputStream rewriteKeyIfMatch(String keyName,
long size,
expectedETag, replicationConfig, metadata, tags);
}
+ public OzoneOutputStream rewriteKeyIfMatch(String keyName, long size,
+ String expectedETag, ReplicationConfig replicationConfig,
+ Map<String, String> metadata, Map<String, String> tags,
+ boolean derivedKeyPiggyBacking)
+ throws IOException {
+ return proxy.rewriteKeyIfMatch(volumeName, name, keyName, size,
+ expectedETag, replicationConfig, metadata, tags,
derivedKeyPiggyBacking);
+ }
+
/**
* Creates a new key in the bucket, with default replication type RATIS and
* with replication factor THREE.
@@ -614,6 +640,17 @@ public OzoneDataStreamOutput createStreamKey(String key,
long size,
replicationConfig, keyMetadata, tags);
}
+ public OzoneDataStreamOutput createStreamKey(String key, long size,
+ ReplicationConfig replicationConfig, Map<String, String> keyMetadata,
+ Map<String, String> tags, boolean derivedKeyPiggyBacking)
+ throws IOException {
+ if (replicationConfig == null) {
+ replicationConfig = defaultReplication;
+ }
+ return proxy.createStreamKey(volumeName, name, key, size,
+ replicationConfig, keyMetadata, tags, derivedKeyPiggyBacking);
+ }
+
/**
* Creates a key with datastream only if it does not exist already
* (S3 If-None-Match: * semantics).
@@ -636,6 +673,16 @@ public OzoneDataStreamOutput
createStreamKeyIfNotExists(String key, long size,
replicationConfig, keyMetadata, tags);
}
+ public OzoneDataStreamOutput createStreamKeyIfNotExists(String key, long
size,
+ ReplicationConfig replicationConfig, Map<String, String> keyMetadata,
+ Map<String, String> tags, boolean derivedKeyPiggyBacking) throws
IOException {
+ if (replicationConfig == null) {
+ replicationConfig = defaultReplication;
+ }
+ return proxy.createStreamKeyIfNotExists(volumeName, name, key, size,
+ replicationConfig, keyMetadata, tags, derivedKeyPiggyBacking);
+ }
+
/**
* Rewrites a key with datastream only if its ETag matches
* (S3 If-Match semantics).
@@ -660,6 +707,18 @@ public OzoneDataStreamOutput
rewriteStreamKeyIfMatch(String key, long size,
expectedETag, replicationConfig, keyMetadata, tags);
}
+ public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String key, long size,
+ String expectedETag, ReplicationConfig replicationConfig,
+ Map<String, String> keyMetadata, Map<String, String> tags,
+ boolean derivedKeyPiggyBacking)
+ throws IOException {
+ if (replicationConfig == null) {
+ replicationConfig = defaultReplication;
+ }
+ return proxy.rewriteStreamKeyIfMatch(volumeName, name, key, size,
+ expectedETag, replicationConfig, keyMetadata, tags,
derivedKeyPiggyBacking);
+ }
+
/**
* Reads an existing key from the bucket.
*
@@ -921,6 +980,14 @@ public OzoneOutputStream createMultipartKey(String key,
long size,
uploadID);
}
+ public OzoneOutputStream createMultipartKey(String key, long size,
+ int partNumber, String uploadID,
+ boolean derivedKeyPiggyBacking)
+ throws IOException {
+ return proxy.createMultipartKey(volumeName, name, key, size, partNumber,
+ uploadID, derivedKeyPiggyBacking);
+ }
+
/**
* Create a part key for a multipart upload key.
* @param key
@@ -936,6 +1003,12 @@ public OzoneDataStreamOutput
createMultipartStreamKey(String key,
key, size, partNumber, uploadID);
}
+ public OzoneDataStreamOutput createMultipartStreamKey(String key,
+ long size, int partNumber, String uploadID, boolean
derivedKeyPiggyBacking) throws IOException {
+ return proxy.createMultipartStreamKey(volumeName, name,
+ key, size, partNumber, uploadID, derivedKeyPiggyBacking);
+ }
+
/**
* Complete Multipart upload. This will combine all the parts and make the
* key visible in ozone.
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java
index 52e79ae5f12..9d87150b2b4 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneDataStreamOutput.java
@@ -39,6 +39,7 @@ public class OzoneDataStreamOutput extends
ByteBufferOutputStream
private final ByteBufferStreamOutput byteBufferStreamOutput;
private boolean enableHsync;
private final Syncable syncable;
+ private ByteBuffer derivedKey;
/**
* Constructs an instance with a {@link Syncable} {@link OutputStream}.
@@ -195,4 +196,11 @@ public Map<String, String> getMetadata() {
return ((KeyMetadataAware)this.byteBufferStreamOutput).getMetadata();
}
+ public ByteBuffer getDerivedKey() {
+ return derivedKey;
+ }
+
+ public void setDerivedKey(ByteBuffer derivedKey) {
+ this.derivedKey = derivedKey;
+ }
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java
index a7eda7da284..38dbff8c5fd 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java
@@ -19,6 +19,7 @@
import java.io.IOException;
import java.io.OutputStream;
+import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -37,6 +38,7 @@ public class OzoneOutputStream extends ByteArrayStreamOutput
private final OutputStream outputStream;
private final Syncable syncable;
private boolean enableHsync;
+ private ByteBuffer derivedKey;
/**
* Constructs an instance with a {@link Syncable} {@link OutputStream}.
@@ -181,4 +183,12 @@ private static OutputStream unwrap(OutputStream out) {
}
return out;
}
+
+ public ByteBuffer getDerivedKey() {
+ return derivedKey;
+ }
+
+ public void setDerivedKey(ByteBuffer derivedKey) {
+ this.derivedKey = derivedKey;
+ }
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 71ca47cef06..a8e75eed746 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -521,6 +521,82 @@ OzoneDataStreamOutput rewriteStreamKeyIfMatch(String
volumeName,
ReplicationConfig replicationConfig, Map<String, String> metadata,
Map<String, String> tags) throws IOException;
+ @SuppressWarnings("checkstyle:parameternumber")
+ default OzoneOutputStream createKey(String volumeName, String bucketName,
+ String keyName, long size, ReplicationConfig replicationConfig,
+ Map<String, String> metadata, Map<String, String> tags,
+ boolean derivedKeyPiggyBacking) throws IOException {
+ return createKey(volumeName, bucketName, keyName, size, replicationConfig,
+ metadata, tags);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ default OzoneOutputStream createKeyIfNotExists(String volumeName, String
bucketName,
+ String keyName, long size, ReplicationConfig replicationConfig,
+ Map<String, String> metadata, Map<String, String> tags,
+ boolean derivedKeyPiggyBacking) throws IOException {
+ return createKeyIfNotExists(volumeName, bucketName, keyName, size,
replicationConfig,
+ metadata, tags);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ default OzoneOutputStream rewriteKeyIfMatch(String volumeName, String
bucketName,
+ String keyName, long size, String expectedETag,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags, boolean derivedKeyPiggyBacking) throws
IOException {
+ return rewriteKeyIfMatch(volumeName, bucketName, keyName, size,
expectedETag,
+ replicationConfig, metadata, tags);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ default OzoneOutputStream createMultipartKey(String volumeName, String
bucketName,
+ String keyName, long size,
+ int partNumber, String uploadID,
+ boolean derivedKeyPiggyBacking)
+ throws IOException {
+ return createMultipartKey(volumeName, bucketName, keyName, size,
partNumber,
+ uploadID);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ default OzoneDataStreamOutput createStreamKey(String volumeName, String
bucketName,
+ String keyName, long size, ReplicationConfig replicationConfig,
+ Map<String, String> metadata, Map<String, String> tags,
+ boolean derivedKeyPiggyBacking) throws IOException {
+ return createStreamKey(volumeName, bucketName, keyName, size,
replicationConfig,
+ metadata, tags);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ default OzoneDataStreamOutput createStreamKeyIfNotExists(String volumeName,
+ String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags, boolean derivedKeyPiggyBacking) throws
IOException {
+ return createStreamKeyIfNotExists(volumeName, bucketName, keyName, size,
+ replicationConfig, metadata, tags);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ default OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName,
+ String bucketName, String keyName, long size, String expectedETag,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags, boolean derivedKeyPiggyBacking) throws
IOException {
+ return rewriteStreamKeyIfMatch(volumeName, bucketName, keyName, size,
expectedETag,
+ replicationConfig, metadata, tags);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ default OzoneDataStreamOutput createMultipartStreamKey(String volumeName,
+ String bucketName,
+ String keyName, long size,
+ int partNumber,
+ String uploadID,
+ boolean
derivedKeyPiggyBacking)
+ throws IOException {
+ return createMultipartStreamKey(volumeName, bucketName, keyName, size,
+ partNumber, uploadID);
+ }
+
/**
* Reads a key from an existing bucket.
* @param volumeName Name of the Volume
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 893fc90138c..01b7b8d9796 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -1404,10 +1404,21 @@ public OzoneOutputStream createKey(
String volumeName, String bucketName, String keyName, long size,
ReplicationConfig replicationConfig,
Map<String, String> metadata, Map<String, String> tags) throws
IOException {
+ return createKey(volumeName, bucketName, keyName, size, replicationConfig,
+ metadata, tags, false);
+ }
+
+ @Override
+ public OzoneOutputStream createKey(
+ String volumeName, String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig,
+ Map<String, String> metadata, Map<String, String> tags,
+ boolean derivedKeyPiggyBacking) throws IOException {
String ownerName = getRealUserInfo().getShortUserName();
OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName,
bucketName, keyName, size, replicationConfig, metadata, tags);
builder.setOwnerName(ownerName);
+ builder.setDerivedKeyPiggyBacking(derivedKeyPiggyBacking);
return openOutputStream(builder.build(), size);
}
@@ -1433,6 +1444,15 @@ public OzoneOutputStream createKeyIfNotExists(String
volumeName,
String bucketName, String keyName, long size,
ReplicationConfig replicationConfig, Map<String, String> metadata,
Map<String, String> tags) throws IOException {
+ return createKeyIfNotExists(volumeName, bucketName, keyName, size,
+ replicationConfig, metadata, tags, false);
+ }
+
+ @Override
+ public OzoneOutputStream createKeyIfNotExists(String volumeName,
+ String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags, boolean derivedKeyPiggyBacking) throws
IOException {
if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) {
throw new IOException(
"OzoneManager does not support atomic key creation.");
@@ -1441,15 +1461,25 @@ public OzoneOutputStream createKeyIfNotExists(String
volumeName,
bucketName, keyName, size, replicationConfig, metadata, tags);
builder.setExpectedDataGeneration(
OzoneConsts.EXPECTED_GEN_CREATE_IF_ABSENT);
+ builder.setDerivedKeyPiggyBacking(derivedKeyPiggyBacking);
return openOutputStream(builder.build(), size);
}
@Override
- @SuppressWarnings("checkstyle:parameternumber")
public OzoneOutputStream rewriteKeyIfMatch(String volumeName,
String bucketName, String keyName, long size, String expectedETag,
ReplicationConfig replicationConfig, Map<String, String> metadata,
Map<String, String> tags) throws IOException {
+ return rewriteKeyIfMatch(volumeName, bucketName, keyName, size,
expectedETag,
+ replicationConfig, metadata, tags, false);
+ }
+
+ @Override
+ @SuppressWarnings("checkstyle:parameternumber")
+ public OzoneOutputStream rewriteKeyIfMatch(String volumeName,
+ String bucketName, String keyName, long size, String expectedETag,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags, boolean derivedKeyPiggyBacking) throws
IOException {
if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) {
throw new IOException(
"OzoneManager does not support conditional key rewrite.");
@@ -1457,6 +1487,7 @@ public OzoneOutputStream rewriteKeyIfMatch(String
volumeName,
OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName,
bucketName, keyName, size, replicationConfig, metadata, tags);
builder.setExpectedETag(expectedETag);
+ builder.setDerivedKeyPiggyBacking(derivedKeyPiggyBacking);
return openOutputStream(builder.build(), size);
}
@@ -1525,9 +1556,20 @@ public OzoneDataStreamOutput createStreamKey(
String volumeName, String bucketName, String keyName, long size,
ReplicationConfig replicationConfig,
Map<String, String> metadata, Map<String, String> tags) throws
IOException {
+ return createStreamKey(volumeName, bucketName, keyName, size,
replicationConfig,
+ metadata, tags, false);
+ }
+
+ @Override
+ public OzoneDataStreamOutput createStreamKey(
+ String volumeName, String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig,
+ Map<String, String> metadata, Map<String, String> tags,
+ boolean derivedKeyPiggyBacking) throws IOException {
OmKeyArgs.Builder builder = createStreamKeyArgsBuilder(
volumeName, bucketName, keyName, size, replicationConfig, metadata,
tags);
+ builder.setDerivedKeyPiggyBacking(derivedKeyPiggyBacking);
return openDataStreamOutput(builder.build());
}
@@ -1536,6 +1578,15 @@ public OzoneDataStreamOutput
createStreamKeyIfNotExists(String volumeName,
String bucketName, String keyName, long size,
ReplicationConfig replicationConfig, Map<String, String> metadata,
Map<String, String> tags) throws IOException {
+ return createStreamKeyIfNotExists(volumeName, bucketName, keyName, size,
+ replicationConfig, metadata, tags, false);
+ }
+
+ @Override
+ public OzoneDataStreamOutput createStreamKeyIfNotExists(String volumeName,
+ String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags, boolean derivedKeyPiggyBacking) throws
IOException {
if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) {
throw new IOException(
"OzoneManager does not support atomic key creation.");
@@ -1545,15 +1596,25 @@ public OzoneDataStreamOutput
createStreamKeyIfNotExists(String volumeName,
tags);
builder.setExpectedDataGeneration(
OzoneConsts.EXPECTED_GEN_CREATE_IF_ABSENT);
+ builder.setDerivedKeyPiggyBacking(derivedKeyPiggyBacking);
return openDataStreamOutput(builder.build());
}
@Override
- @SuppressWarnings("checkstyle:parameternumber")
public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName,
String bucketName, String keyName, long size, String expectedETag,
ReplicationConfig replicationConfig, Map<String, String> metadata,
Map<String, String> tags) throws IOException {
+ return rewriteStreamKeyIfMatch(volumeName, bucketName, keyName, size,
expectedETag,
+ replicationConfig, metadata, tags, false);
+ }
+
+ @Override
+ @SuppressWarnings("checkstyle:parameternumber")
+ public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName,
+ String bucketName, String keyName, long size, String expectedETag,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags, boolean derivedKeyPiggyBacking) throws
IOException {
if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) {
throw new IOException(
"OzoneManager does not support conditional key rewrite.");
@@ -1562,6 +1623,7 @@ public OzoneDataStreamOutput
rewriteStreamKeyIfMatch(String volumeName,
volumeName, bucketName, keyName, size, replicationConfig, metadata,
tags);
builder.setExpectedETag(expectedETag);
+ builder.setDerivedKeyPiggyBacking(derivedKeyPiggyBacking);
return openDataStreamOutput(builder.build());
}
@@ -2066,10 +2128,11 @@ public OmMultipartInfo initiateMultipartUpload(String
volumeName,
return multipartInfo;
}
+ @SuppressWarnings("checkstyle:parameternumber")
private OpenKeySession newMultipartOpenKey(
String volumeName, String bucketName, String keyName,
long size, int partNumber, String uploadID,
- boolean sortDatanodesInPipeline) throws IOException {
+ boolean sortDatanodesInPipeline, boolean derivedKeyPiggyBacking) throws
IOException {
verifyVolumeName(volumeName);
verifyBucketName(bucketName);
if (checkKeyNameEnabled) {
@@ -2094,6 +2157,7 @@ private OpenKeySession newMultipartOpenKey(
.setMultipartUploadPartNumber(partNumber)
.setSortDatanodesInPipeline(sortDatanodesInPipeline)
.setOwnerName(ownerName)
+ .setDerivedKeyPiggyBacking(derivedKeyPiggyBacking)
.build();
return ozoneManagerClient.openKey(keyArgs);
}
@@ -2102,8 +2166,15 @@ private OpenKeySession newMultipartOpenKey(
public OzoneOutputStream createMultipartKey(
String volumeName, String bucketName, String keyName,
long size, int partNumber, String uploadID) throws IOException {
+ return createMultipartKey(volumeName, bucketName, keyName, size,
partNumber, uploadID, false);
+ }
+
+ @Override
+ public OzoneOutputStream createMultipartKey(
+ String volumeName, String bucketName, String keyName,
+ long size, int partNumber, String uploadID, boolean
derivedKeyPiggyBacking) throws IOException {
final OpenKeySession openKey = newMultipartOpenKey(
- volumeName, bucketName, keyName, size, partNumber, uploadID, false);
+ volumeName, bucketName, keyName, size, partNumber, uploadID, false,
derivedKeyPiggyBacking);
return createMultipartOutputStream(openKey, uploadID, partNumber);
}
@@ -2127,8 +2198,21 @@ public OzoneDataStreamOutput createMultipartStreamKey(
int partNumber,
String uploadID)
throws IOException {
+ return createMultipartStreamKey(volumeName, bucketName, keyName, size,
partNumber, uploadID, false);
+ }
+
+ @Override
+ public OzoneDataStreamOutput createMultipartStreamKey(
+ String volumeName,
+ String bucketName,
+ String keyName,
+ long size,
+ int partNumber,
+ String uploadID,
+ boolean derivedKeyPiggyBacking)
+ throws IOException {
final OpenKeySession openKey = newMultipartOpenKey(
- volumeName, bucketName, keyName, size, partNumber, uploadID, true);
+ volumeName, bucketName, keyName, size, partNumber, uploadID, true,
derivedKeyPiggyBacking);
final ByteBufferStreamOutput out;
ReplicationConfig replicationConfig =
openKey.getKeyInfo().getReplicationConfig();
if (replicationConfig.getReplicationType() ==
HddsProtos.ReplicationType.RATIS) {
@@ -2147,7 +2231,11 @@ public OzoneDataStreamOutput createMultipartStreamKey(
} else {
out = createMultipartOutputStream(openKey, uploadID, partNumber);
}
- return new OzoneDataStreamOutput(out, out);
+ OzoneDataStreamOutput ozoneDataStreamOutput = new
OzoneDataStreamOutput(out, out);
+ if (openKey.getDerivedKey() != null) {
+ ozoneDataStreamOutput.setDerivedKey(openKey.getDerivedKey());
+ }
+ return ozoneDataStreamOutput;
}
@Override
@@ -2610,7 +2698,11 @@ private OzoneDataStreamOutput
createDataStreamOutput(OpenKeySession openKey)
} else {
out = createOutputStream(openKey);
}
- return new OzoneDataStreamOutput(out, out);
+ OzoneDataStreamOutput ozoneDataStreamOutput = new
OzoneDataStreamOutput(out, out);
+ if (openKey.getDerivedKey() != null) {
+ ozoneDataStreamOutput.setDerivedKey(openKey.getDerivedKey());
+ }
+ return ozoneDataStreamOutput;
}
private KeyDataStreamOutput.Builder newKeyOutputStreamBuilder() {
@@ -2638,8 +2730,12 @@ private OzoneOutputStream
createOutputStream(OpenKeySession openKey,
openKey.getOpenVersion());
final OzoneOutputStream out = createSecureOutputStream(
openKey, keyOutputStream, keyOutputStream);
- return out != null ? out : new OzoneOutputStream(
+ OzoneOutputStream ozoneOutputStream = out != null ? out : new
OzoneOutputStream(
keyOutputStream, enableHsync);
+ if (openKey.getDerivedKey() != null) {
+ ozoneOutputStream.setDerivedKey(openKey.getDerivedKey());
+ }
+ return ozoneOutputStream;
}
private OzoneOutputStream createSecureOutputStream(OpenKeySession openKey,
diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml
index c5fc6aeb27b..79039d218d7 100644
--- a/hadoop-ozone/common/pom.xml
+++ b/hadoop-ozone/common/pom.xml
@@ -89,6 +89,10 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kerby</groupId>
+ <artifactId>kerby-util</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-client</artifactId>
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/AWSV4AuthValidator.java
similarity index 78%
rename from
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java
rename to
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/AWSV4AuthValidator.java
index 4118894e722..6a2d5628107 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/AWSV4AuthValidator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.security;
+package org.apache.hadoop.ozone.om;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
@@ -33,7 +33,7 @@
* documentation https://docs.aws.amazon.com/general/latest/gr/
* sigv4-create-canonical-request.html.
**/
-final class AWSV4AuthValidator {
+public final class AWSV4AuthValidator {
private static final Logger LOG =
LoggerFactory.getLogger(AWSV4AuthValidator.class);
@@ -75,22 +75,13 @@ private static byte[] sign(byte[] key, String msg) {
}
}
- /**
- * Returns signing key.
- *
- * @param key
- * @param strToSign
- *
- * SignatureKey = HMAC-SHA256(HMAC-SHA256(HMAC-SHA256(HMAC-SHA256("AWS4" +
- * "<YourSecretAccessKey>","20130524"),"us-east-1"),"s3"),"aws4_request")
- *
- * For more details refer to AWS documentation: https://docs.aws.amazon
- * .com/AmazonS3/latest/API/sig-v4-header-based-auth.html
- *
- * */
- private static byte[] getSigningKey(String key, String strToSign) {
- String[] signData = StringUtils.split(StringUtils.split(strToSign,
- '\n')[2], '/');
+ public static byte[] getSigningKey(String key, String strToSign) {
+ String credentialScope = StringUtils.split(strToSign, '\n')[2];
+ return getSigningKeyFromCredentialScope(key, credentialScope);
+ }
+
+ private static byte[] getSigningKeyFromCredentialScope(String key, String
credentialScope) {
+ String[] signData = StringUtils.split(credentialScope, '/');
String dateStamp = signData[0];
String regionName = signData[1];
String serviceName = signData[2];
@@ -105,14 +96,6 @@ private static byte[] getSigningKey(String key, String
strToSign) {
return kSigning;
}
- /**
- * Validate request by comparing Signature from request. Returns true if
- * aws request is legit else returns false.
- * Signature = HEX(HMAC_SHA256(key, String to Sign))
- *
- * For more details refer to AWS documentation: https://docs.aws.amazon.com
- * /AmazonS3/latest/API/sigv4-streaming.html
- */
public static boolean validateRequest(String strToSign, String signature,
String userKey) {
String expectedSignature = Hex.encode(sign(getSigningKey(userKey,
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
index 5d2de09c48e..5c55d7b9b2e 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
@@ -62,6 +62,7 @@ public final class OmKeyArgs extends WithMetadata implements
Auditable {
// been modified.
private Long expectedDataGeneration = null;
private final String expectedETag;
+ private final boolean derivedKeyPiggyBacking;
private OmKeyArgs(Builder b) {
super(b);
@@ -84,6 +85,7 @@ private OmKeyArgs(Builder b) {
this.tags = b.tags.build();
this.expectedDataGeneration = b.expectedDataGeneration;
this.expectedETag = b.expectedETag;
+ this.derivedKeyPiggyBacking = b.derivedKeyPiggyBacking;
}
public boolean getIsMultipartKey() {
@@ -170,6 +172,10 @@ public String getExpectedETag() {
return expectedETag;
}
+ public boolean isDerivedKeyPiggyBacking() {
+ return derivedKeyPiggyBacking;
+ }
+
@Override
public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
@@ -244,6 +250,7 @@ public static class Builder extends WithMetadata.Builder {
private final MapBuilder<String, String> tags;
private Long expectedDataGeneration = null;
private String expectedETag;
+ private boolean derivedKeyPiggyBacking;
public Builder() {
this(AclListBuilder.empty());
@@ -290,6 +297,7 @@ public Builder(OmKeyArgs obj) {
obj.forceUpdateContainerCacheFromSCM;
this.expectedDataGeneration = obj.expectedDataGeneration;
this.expectedETag = obj.expectedETag;
+ this.derivedKeyPiggyBacking = obj.derivedKeyPiggyBacking;
this.tags = MapBuilder.of(obj.tags);
this.acls = AclListBuilder.of(obj.acls);
}
@@ -430,6 +438,11 @@ public Builder setExpectedETag(String eTag) {
return this;
}
+ public Builder setDerivedKeyPiggyBacking(boolean derivedKeyPiggyBacking) {
+ this.derivedKeyPiggyBacking = derivedKeyPiggyBacking;
+ return this;
+ }
+
public OmKeyArgs build() {
return new OmKeyArgs(this);
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OpenKeySession.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OpenKeySession.java
index 125df602b28..e6026cb8483 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OpenKeySession.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OpenKeySession.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.om.helpers;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.nio.ByteBuffer;
/**
* This class represents a open key "session". A session here means a key is
@@ -32,6 +33,7 @@ public class OpenKeySession {
// a block that has a create version equals to open version means it will
// be committed only when this open session is closed.
private long openVersion;
+ private ByteBuffer derivedKey;
public OpenKeySession(long id, OmKeyInfo info, long version) {
this.id = id;
@@ -39,6 +41,14 @@ public OpenKeySession(long id, OmKeyInfo info, long version)
{
this.openVersion = version;
}
+ public void setDerivedKey(ByteBuffer derivedKey) {
+ this.derivedKey = derivedKey;
+ }
+
+ public ByteBuffer getDerivedKey() {
+ return derivedKey;
+ }
+
public long getOpenVersion() {
return this.openVersion;
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index ed1d5c35785..30f32ba2e9d 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -765,15 +765,20 @@ public OpenKeySession openKey(OmKeyArgs args) throws
IOException {
}
req.setKeyArgs(keyArgs.build());
+ req.setDerivedKeyPiggyBacking(args.isDerivedKeyPiggyBacking());
OMRequest omRequest = createOMRequest(Type.CreateKey)
.setCreateKeyRequest(req)
.build();
CreateKeyResponse keyResponse =
handleSubmitRequestAndSCMSafeModeRetry(omRequest).getCreateKeyResponse();
- return new OpenKeySession(keyResponse.getID(),
+ OpenKeySession openKeySession = new OpenKeySession(keyResponse.getID(),
OmKeyInfo.getFromProtobuf(keyResponse.getKeyInfo()),
keyResponse.getOpenVersion());
+ if (keyResponse.hasDerivedKey()) {
+
openKeySession.setDerivedKey(keyResponse.getDerivedKey().asReadOnlyByteBuffer());
+ }
+ return openKeySession;
}
private OMResponse handleError(OMResponse resp) throws OMException {
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 25de443296c..1cf0033cacb 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1361,6 +1361,7 @@ message CreateKeyRequest {
// Set in OM HA during preExecute step. This way all OM's use same ID in
// OM HA.
optional uint64 clientID = 2;
+ optional bool derivedKeyPiggyBacking = 3;
}
message CreateKeyResponse {
@@ -1369,6 +1370,7 @@ message CreateKeyResponse {
// (similar to a cookie).
optional uint64 ID = 3;
optional uint64 openVersion = 4;
+ optional bytes derivedKey = 5;
}
message LookupKeyRequest {
diff --git a/hadoop-ozone/ozone-manager/pom.xml
b/hadoop-ozone/ozone-manager/pom.xml
index 0715e3e9563..9f8a8940351 100644
--- a/hadoop-ozone/ozone-manager/pom.xml
+++ b/hadoop-ozone/ozone-manager/pom.xml
@@ -120,10 +120,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.kerby</groupId>
- <artifactId>kerby-util</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 09381bacb3d..ed0b89c5a01 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -5409,6 +5409,11 @@ public void startQuotaRepair(List<String> buckets)
throws IOException {
new QuotaRepairTask(this).repair(buckets);
}
+ public byte[] getS3DerivedKey(String accessId, String signingKey) throws
IOException {
+ String awsSecretKey = s3SecretManager.getSecretString(accessId);
+ return AWSV4AuthValidator.getSigningKey(awsSecretKey, signingKey);
+ }
+
@Override
public Map<String, String> getObjectTagging(final OmKeyArgs args)
throws IOException {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index 929e46222c0..6059d6c46c7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -22,6 +22,7 @@
import static
org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.FILE_EXISTS_IN_GIVENPATH;
import static org.apache.hadoop.ozone.util.MetricUtil.captureLatencyNs;
+import com.google.protobuf.ByteString;
import java.io.IOException;
import java.nio.file.InvalidPathException;
import java.nio.file.Paths;
@@ -64,9 +65,12 @@
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserInfo;
import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.ozone.security.S3SecurityUtil;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
@@ -195,7 +199,8 @@ public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
@SuppressWarnings("methodlength")
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
ExecutionContext context) {
final long trxnLogIndex = context.getIndex();
- CreateKeyRequest createKeyRequest = getOmRequest().getCreateKeyRequest();
+ OMRequest omRequest = getOmRequest();
+ CreateKeyRequest createKeyRequest = omRequest.getCreateKeyRequest();
KeyArgs keyArgs = createKeyRequest.getKeyArgs();
Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs);
@@ -317,6 +322,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
checkBucketQuotaInBytes(omMetadataManager, bucketInfo,
preAllocatedSpace);
checkBucketQuotaInNamespace(bucketInfo, numMissingParents + 1L);
+ CreateKeyResponse.Builder builder =
+ getResponseBuilderWithDerivedKey(getOmRequest(), ozoneManager,
createKeyRequest);
perfMetrics.addCreateKeyQuotaCheckLatencyNs(Time.monotonicNowNanos() -
quotaCheckStartTime);
bucketInfo.incrUsedNamespace(numMissingParents);
@@ -334,12 +341,12 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry(
dbOpenKeyName, omKeyInfo, trxnLogIndex);
- // Prepare response
- omResponse.setCreateKeyResponse(CreateKeyResponse.newBuilder()
- .setKeyInfo(omKeyInfo.getNetworkProtobuf(getOmRequest().getVersion(),
+
builder.setKeyInfo(omKeyInfo.getNetworkProtobuf(getOmRequest().getVersion(),
keyArgs.getLatestVersionLocation()))
.setID(clientID)
- .setOpenVersion(openVersion).build())
+ .setOpenVersion(openVersion);
+ // Prepare response
+ omResponse.setCreateKeyResponse(builder.build())
.setCmdType(Type.CreateKey);
omClientResponse = new OMKeyCreateResponse(omResponse.build(),
omKeyInfo, missingParentInfos, clientID, bucketInfo.copyObject());
@@ -458,4 +465,27 @@ public static OMRequest
blockCreateKeyWithBucketLayoutFromOldClient(
}
return req;
}
+
+ protected CreateKeyResponse.Builder getResponseBuilderWithDerivedKey(
+ OMRequest omRequest, OzoneManager ozoneManager,
+ CreateKeyRequest createKeyRequest) throws IOException {
+ CreateKeyResponse.Builder builder = CreateKeyResponse.newBuilder();
+ if (omRequest.hasS3Authentication() && ozoneManager.isSecurityEnabled()
+ && createKeyRequest.hasDerivedKeyPiggyBacking()
+ && createKeyRequest.getDerivedKeyPiggyBacking()
+ ) {
+ OzoneTokenIdentifier s3Token =
S3SecurityUtil.constructS3Token(omRequest);
+ if (!s3Token.getTokenType().equals(OMTokenProto.Type.S3AUTHINFO)) {
+ // Piggyback was requested but this token type cannot produce a
derived key.
+ // S3 Gateway should only set this flag for S3AUTHINFO tokens.
+ LOG.warn("Derived key piggyback requested but token type is {}, " +
+ "not S3AUTHINFO. Derived key will not be returned.",
+ s3Token.getTokenType());
+ return builder;
+ }
+ byte[] derivedKey =
ozoneManager.getS3DerivedKey(s3Token.getAwsAccessId(), s3Token.getStrToSign());
+ builder.setDerivedKey(ByteString.copyFrom(derivedKey));
+ }
+ return builder;
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
index 99fabb46de1..3642f96c7fc 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
@@ -184,6 +184,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
preAllocatedSpace);
checkBucketQuotaInNamespace(omBucketInfo, numKeysCreated + 1L);
+ CreateKeyResponse.Builder createKeyResponseBuilder =
+ getResponseBuilderWithDerivedKey(getOmRequest(), ozoneManager,
createKeyRequest);
perfMetrics.addCreateKeyQuotaCheckLatencyNs(Time.monotonicNowNanos() -
quotaCheckStartTime);
omBucketInfo.incrUsedNamespace(numKeysCreated);
@@ -202,11 +204,11 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
// Prepare response. Sets user given full key name in the 'keyName'
// attribute in response object.
int clientVersion = getOmRequest().getVersion();
- omResponse.setCreateKeyResponse(CreateKeyResponse.newBuilder()
- .setKeyInfo(omFileInfo.getNetworkProtobuf(keyName, clientVersion,
+
createKeyResponseBuilder.setKeyInfo(omFileInfo.getNetworkProtobuf(keyName,
clientVersion,
keyArgs.getLatestVersionLocation()))
.setID(clientID)
- .setOpenVersion(openVersion).build())
+ .setOpenVersion(openVersion);
+ omResponse.setCreateKeyResponse(createKeyResponseBuilder.build())
.setCmdType(Type.CreateKey);
omClientResponse = new OMKeyCreateResponseWithFSO(omResponse.build(),
omFileInfo, missingParentInfos, clientID,
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
index 01818aa2d57..acce4015962 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
@@ -44,6 +44,7 @@
import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.security.x509.exception.CertificateException;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.om.AWSV4AuthValidator;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.S3SecretManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java
index cbe0ce414fe..860c69242b2 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java
@@ -79,7 +79,7 @@ public static void validateS3Credential(OMRequest omRequest,
/**
* Construct and return {@link OzoneTokenIdentifier} from {@link OMRequest}.
*/
- private static OzoneTokenIdentifier constructS3Token(OMRequest omRequest) {
+ public static OzoneTokenIdentifier constructS3Token(OMRequest omRequest) {
S3Authentication auth = omRequest.getS3Authentication();
OzoneTokenIdentifier s3Token = new OzoneTokenIdentifier();
s3Token.setTokenType(S3AUTHINFO);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
index 108a10bc240..d9837147594 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
@@ -1518,6 +1518,111 @@ protected OmKeyInfo checkCreatedPaths(
return omKeyInfo;
}
+ @Test
+ public void testCreateKeyWithS3DerivedKey() throws Exception {
+ when(ozoneManager.getOzoneLockProvider()).thenReturn(
+ new OzoneLockProvider(true, true));
+ when(ozoneManager.isSecurityEnabled()).thenReturn(true);
+ byte[] expectedDerivedKey = new byte[] {9, 8, 7, 6};
+ when(ozoneManager.getS3DerivedKey(anyString(),
anyString())).thenReturn(expectedDerivedKey);
+
+ KeyArgs.Builder keyArgs = KeyArgs.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setFactor(((RatisReplicationConfig)
replicationConfig).getReplicationFactor())
+ .setType(replicationConfig.getReplicationType())
+ .setLatestVersionLocation(true)
+ .setDataSize(100L);
+
+ OzoneManagerProtocolProtos.S3Authentication s3Authentication =
+ OzoneManagerProtocolProtos.S3Authentication.newBuilder()
+ .setAccessId("testAccessId")
+ .setSignature("testSignature")
+ .setStringToSign("testStringToSign")
+ .build();
+
+ CreateKeyRequest createKeyRequest = CreateKeyRequest.newBuilder()
+ .setKeyArgs(keyArgs)
+ .setDerivedKeyPiggyBacking(true)
+ .build();
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.CreateKey)
+ .setClientId(UUID.randomUUID().toString())
+ .setCreateKeyRequest(createKeyRequest)
+ .setS3Authentication(s3Authentication)
+ .build();
+
+ OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(omRequest);
+
+ addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager,
getBucketLayout());
+
+ OMRequest modifiedOmRequest = omKeyCreateRequest.preExecute(ozoneManager);
+ omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest);
+
+ OMClientResponse response =
+ omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L);
+
+ assertEquals(OK, response.getOMResponse().getStatus());
+ OzoneManagerProtocolProtos.CreateKeyResponse createKeyResponse =
+ response.getOMResponse().getCreateKeyResponse();
+ assertNotNull(createKeyResponse);
+ assertTrue(createKeyResponse.hasDerivedKey());
+ assertEquals(com.google.protobuf.ByteString.copyFrom(expectedDerivedKey),
createKeyResponse.getDerivedKey());
+ }
+
+ @Test
+ public void testCreateKeyWithoutS3DerivedKey() throws Exception {
+ when(ozoneManager.getOzoneLockProvider()).thenReturn(
+ new OzoneLockProvider(true, true));
+ when(ozoneManager.isSecurityEnabled()).thenReturn(true);
+
+ KeyArgs.Builder keyArgs = KeyArgs.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setFactor(((RatisReplicationConfig)
replicationConfig).getReplicationFactor())
+ .setType(replicationConfig.getReplicationType())
+ .setLatestVersionLocation(true)
+ .setDataSize(100L);
+
+ OzoneManagerProtocolProtos.S3Authentication s3Authentication =
+ OzoneManagerProtocolProtos.S3Authentication.newBuilder()
+ .setAccessId("testAccessId")
+ .setSignature("testSignature")
+ .setStringToSign("testStringToSign")
+ .build();
+
+ CreateKeyRequest createKeyRequest = CreateKeyRequest.newBuilder()
+ .setKeyArgs(keyArgs)
+ .setDerivedKeyPiggyBacking(false)
+ .build();
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.CreateKey)
+ .setClientId(UUID.randomUUID().toString())
+ .setCreateKeyRequest(createKeyRequest)
+ .setS3Authentication(s3Authentication)
+ .build();
+
+ OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(omRequest);
+
+ addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager,
getBucketLayout());
+
+ OMRequest modifiedOmRequest = omKeyCreateRequest.preExecute(ozoneManager);
+ omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest);
+
+ OMClientResponse response =
+ omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L);
+
+ assertEquals(OK, response.getOMResponse().getStatus());
+ OzoneManagerProtocolProtos.CreateKeyResponse createKeyResponse =
+ response.getOMResponse().getCreateKeyResponse();
+ assertNotNull(createKeyResponse);
+ assertFalse(createKeyResponse.hasDerivedKey());
+ }
+
protected long checkIntermediatePaths(Path keyPath) throws Exception {
// Check intermediate paths are created
keyPath = keyPath.getParent();
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestAWSV4AuthValidator.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestAWSV4AuthValidator.java
index 5564947c1bb..361c540dfd3 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestAWSV4AuthValidator.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestAWSV4AuthValidator.java
@@ -21,6 +21,7 @@
import java.util.Arrays;
import java.util.Collection;
+import org.apache.hadoop.ozone.om.AWSV4AuthValidator;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
