This is an automated email from the ASF dual-hosted git repository.
aengineer pushed a commit to branch HDDS-4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/HDDS-4 by this push:
new 1f78b04 HDDS-938. Add Client APIs for using S3 Auth interface.
Contributed by Dinesh Chitlangia.
1f78b04 is described below
commit 1f78b04ff3aaec175b1379474ceaaf2788199b16
Author: Anu Engineer <[email protected]>
AuthorDate: Fri Jan 4 10:04:17 2019 -0800
HDDS-938. Add Client APIs for using S3 Auth interface.
Contributed by Dinesh Chitlangia.
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../apache/hadoop/ozone/client/ObjectStore.java | 5 ++
.../ozone/client/protocol/ClientProtocol.java | 9 +++
.../hadoop/ozone/client/rest/RestClient.java | 7 ++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 15 ++++
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 26 +++++++
.../apache/hadoop/ozone/om/OzoneManagerLock.java | 24 ++++++-
.../hadoop/ozone/om/helpers/S3SecretValue.java | 84 ++++++++++++++++++++++
.../ozone/om/protocol/OzoneManagerProtocol.java | 8 +++
...OzoneManagerProtocolClientSideTranslatorPB.java | 23 ++++++
.../src/main/proto/OzoneManagerProtocol.proto | 17 +++++
.../ozone/client/rpc/TestOzoneRpcClient.java | 18 +++++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 11 +++
.../apache/hadoop/ozone/om/S3SecretManager.java | 30 ++++++++
.../hadoop/ozone/om/S3SecretManagerImpl.java | 82 +++++++++++++++++++++
.../hadoop/ozone/om/exceptions/OMException.java | 3 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 19 +++++
17 files changed, 380 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index dad3bfe..a73fce9 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -86,6 +86,7 @@ public final class OzoneConsts {
public static final String CONTAINER_ROOT_PREFIX = "repository";
public static final String FILE_HASH = "SHA-256";
+ public static final String MD5_HASH = "MD5";
public final static String CHUNK_OVERWRITE = "OverWriteRequested";
public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1 MB
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 353996b..4d5f6d9 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -28,6 +28,7 @@ import java.util.Objects;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.security.UserGroupInformation;
import com.google.common.annotations.VisibleForTesting;
@@ -155,6 +156,10 @@ public class ObjectStore {
return volume;
}
+ public S3SecretValue getS3Secret(String kerberosID) throws IOException {
+ return proxy.getS3Secret(kerberosID);
+ }
+
/**
* Returns Iterator to iterate over all buckets for a user.
* The result can be restricted using bucket prefix, will return all
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 2fe302e..ef7fa59 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import java.io.IOException;
import java.util.List;
+
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.token.Token;
@@ -420,4 +422,11 @@ public interface ClientProtocol {
void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
throws IOException;
+ /**
+ * returns S3 Secret given kerberos user.
+ * @param kerberosID
+ * @return S3SecretValue
+ * @throws IOException
+ */
+ S3SecretValue getS3Secret(String kerberosID) throws IOException;
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index 6e2841d..377092b 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -43,6 +43,7 @@ import
org.apache.hadoop.ozone.client.rest.response.BucketInfo;
import org.apache.hadoop.ozone.client.rest.response.KeyInfoDetails;
import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
@@ -709,6 +710,12 @@ public class RestClient implements ClientProtocol {
}
@Override
+ public S3SecretValue getS3Secret(String kerberosID) throws IOException {
+ throw new UnsupportedOperationException("Ozone REST protocol does not " +
+ "support this operation.");
+ }
+
+ @Override
public OzoneInputStream getKey(
String volumeName, String bucketName, String keyName)
throws IOException {
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index f6b68c4..ee7c571 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import
org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
@@ -446,6 +447,20 @@ public class RpcClient implements ClientProtocol {
ozoneManagerClient.cancelDelegationToken(token);
}
+ /**
+ * Returns s3 secret given a kerberos user.
+ * @param kerberosID
+ * @return S3SecretValue
+ * @throws IOException
+ */
+ @Override
+ public S3SecretValue getS3Secret(String kerberosID) throws IOException {
+ Preconditions.checkArgument(Strings.isNotBlank(kerberosID),
+ "kerberosID cannot be null or empty.");
+
+ return ozoneManagerClient.getS3Secret(kerberosID);
+ }
+
@Override
public void setBucketVersioning(
String volumeName, String bucketName, Boolean versioning)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index e7ed84f..2aae700 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -18,10 +18,15 @@
package org.apache.hadoop.ozone;
import java.io.File;
+import java.io.IOException;
import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.util.Collection;
import java.util.Optional;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.server.ServerUtils;
@@ -140,4 +145,25 @@ public final class OmUtils {
OMConfigKeys.OZONE_OM_DB_DIRS, HddsConfigKeys.OZONE_METADATA_DIRS);
return ServerUtils.getOzoneMetaDirPath(conf);
}
+
+ public static byte[] getMD5Digest(String input) throws IOException {
+ try {
+ MessageDigest md = MessageDigest.getInstance(OzoneConsts.MD5_HASH);
+ return md.digest(input.getBytes(StandardCharsets.UTF_8));
+ } catch (NoSuchAlgorithmException ex) {
+ throw new IOException("Error creating an instance of MD5 digest.\n" +
+ "This could possibly indicate a faulty JRE");
+ }
+ }
+
+ public static byte[] getSHADigest() throws IOException {
+ try {
+ MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
+ return sha.digest(RandomStringUtils.random(32)
+ .getBytes(StandardCharsets.UTF_8));
+ } catch (NoSuchAlgorithmException ex) {
+ throw new IOException("Error creating an instance of SHA-256 digest.\n" +
+ "This could possibly indicate a faulty JRE");
+ }
+ }
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
index c5ce9e2..0e36898 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
@@ -67,6 +67,7 @@ public final class OzoneManagerLock {
private static final String VOLUME_LOCK = "volumeLock";
private static final String BUCKET_LOCK = "bucketLock";
private static final String S3_BUCKET_LOCK = "s3BucketLock";
+ private static final String S3_SECRET_LOCK = "s3SecretetLock";
private final LockManager<String> manager;
@@ -76,7 +77,8 @@ public final class OzoneManagerLock {
() -> ImmutableMap.of(
VOLUME_LOCK, new AtomicInteger(0),
BUCKET_LOCK, new AtomicInteger(0),
- S3_BUCKET_LOCK, new AtomicInteger(0)
+ S3_BUCKET_LOCK, new AtomicInteger(0),
+ S3_SECRET_LOCK, new AtomicInteger(0)
)
);
@@ -219,4 +221,24 @@ public final class OzoneManagerLock {
private boolean hasAnyS3Lock() {
return myLocks.get().get(S3_BUCKET_LOCK).get() != 0;
}
+
+ public void acquireS3SecretLock(String awsAccessId) {
+ if (hasAnyS3SecretLock()) {
+ throw new RuntimeException(
+ "Thread '" + Thread.currentThread().getName() +
+ "' cannot acquire S3 Secret lock while holding S3 " +
+ "awsAccessKey lock(s).");
+ }
+ manager.lock(awsAccessId);
+ myLocks.get().get(S3_SECRET_LOCK).incrementAndGet();
+ }
+
+ private boolean hasAnyS3SecretLock() {
+ return myLocks.get().get(S3_SECRET_LOCK).get() != 0;
+ }
+
+ public void releaseS3SecretLock(String awsAccessId) {
+ manager.unlock(awsAccessId);
+ myLocks.get().get(S3_SECRET_LOCK).decrementAndGet();
+ }
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java
new file mode 100644
index 0000000..8b050a7
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.helpers;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+
+import java.io.IOException;
+
+/**
+ * S3Secret to be saved in database.
+ */
+public class S3SecretValue {
+ private String kerberosID;
+ private String awsSecret;
+ private String awsAccessKey;
+
+ public S3SecretValue(String kerberosID, String awsSecret) throws IOException
{
+ this.kerberosID = kerberosID;
+ this.awsSecret = awsSecret;
+ this.awsAccessKey =
+ DigestUtils.md5Hex(OmUtils.getMD5Digest(kerberosID));
+ }
+
+ public String getKerberosID() {
+ return kerberosID;
+ }
+
+ public void setKerberosID(String kerberosID) {
+ this.kerberosID = kerberosID;
+ }
+
+ public String getAwsSecret() {
+ return awsSecret;
+ }
+
+ public void setAwsSecret(String awsSecret) {
+ this.awsSecret = awsSecret;
+ }
+
+ public String getAwsAccessKey() {
+ return awsAccessKey;
+ }
+
+ public void setAwsAccessKey(String awsAccessKey) {
+ this.awsAccessKey = awsAccessKey;
+ }
+
+ public static S3SecretValue fromProtobuf(
+ OzoneManagerProtocolProtos.S3Secret s3Secret) throws IOException {
+ return new S3SecretValue(s3Secret.getKerberosID(),
s3Secret.getAwsSecret());
+ }
+
+ public OzoneManagerProtocolProtos.S3Secret getProtobuf() {
+ return OzoneManagerProtocolProtos.S3Secret.newBuilder()
+ .setAwsSecret(this.awsSecret)
+ .setKerberosID(this.kerberosID)
+ .build();
+ }
+
+ @Override
+ public String toString() {
+ return "S3SecretValue{" +
+ "kerberosID='" + kerberosID + '\'' +
+ ", awsSecret='" + awsSecret + '\'' +
+ '}';
+ }
+}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 06933d2..fc48d0e 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
@@ -308,5 +309,12 @@ public interface OzoneManagerProtocol extends
OzoneManagerSecurityProtocol {
String bucketPrefix, int maxNumOfBuckets)
throws IOException;
+ /**
+ * Gets s3Secret for given kerberos user.
+ * @param kerberosID
+ * @return S3SecretValue
+ * @throws IOException
+ */
+ S3SecretValue getS3Secret(String kerberosID) throws IOException;
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 85ae4a8..ca2e2fa 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -33,8 +33,10 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.AllocateBlockRequest;
import org.apache.hadoop.ozone.protocol.proto
@@ -883,6 +885,27 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
}
}
+ @Override
+ public S3SecretValue getS3Secret(String kerberosID) throws IOException {
+ OzoneManagerProtocolProtos.S3SecretRequest request =
+ OzoneManagerProtocolProtos.S3SecretRequest.newBuilder()
+ .setKerberosID(kerberosID)
+ .build();
+ final OzoneManagerProtocolProtos.S3SecretResponse resp;
+ try {
+ resp = rpcProxy.getS3Secret(NULL_RPC_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+
+ if(resp.getStatus() != Status.OK) {
+ throw new IOException("Fetch S3 Secret failed, error: " +
+ resp.getStatus());
+ } else {
+ return S3SecretValue.fromProtobuf(resp.getS3Secret());
+ }
+ }
+
/**
* Return the proxy object underlying this protocol translator.
*
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 39d709d..5ce6117 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -61,6 +61,7 @@ enum Status {
SCM_VERSION_MISMATCH_ERROR = 21;
S3_BUCKET_NOT_FOUND = 22;
S3_BUCKET_ALREADY_EXISTS = 23;
+ S3_SECRET_NOT_FOUND = 24;
}
@@ -426,6 +427,19 @@ message S3ListBucketsResponse {
repeated BucketInfo bucketInfo = 2;
}
+message S3Secret {
+ required string kerberosID = 1;
+ required string awsSecret = 2;
+}
+
+message S3SecretRequest {
+ required string kerberosID = 1;
+}
+
+message S3SecretResponse {
+ required Status status = 1;
+ required S3Secret s3Secret = 2;
+}
/**
The OM service that takes care of Ozone namespace.
@@ -580,4 +594,7 @@ service OzoneManagerService {
rpc listS3Buckets(S3ListBucketsRequest)
returns(S3ListBucketsResponse);
+
+ rpc getS3Secret(S3SecretRequest)
+ returns(S3SecretResponse);
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
index 485845c..b2b97f5 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.protocolPB.
StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.junit.AfterClass;
@@ -1258,6 +1259,23 @@ public class TestOzoneRpcClient {
}
}
+ @Test
+ public void testGetS3Secret() throws IOException {
+ //Creates a secret since it does not exist
+ S3SecretValue firstAttempt = store.getS3Secret("HADOOP/JOHNDOE");
+
+ //Fetches the secret from db since it was created in previous step
+ S3SecretValue secondAttempt = store.getS3Secret("HADOOP/JOHNDOE");
+
+ //secret fetched on both attempts must be same
+ Assert.assertTrue(firstAttempt.getAwsSecret()
+ .equals(secondAttempt.getAwsSecret()));
+
+ //access key fetched on both attempts must be same
+ Assert.assertTrue(firstAttempt.getAwsAccessKey()
+ .equals(secondAttempt.getAwsAccessKey()));
+ }
+
/**
* Close OzoneClient and shutdown MiniOzoneCluster.
*/
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index cc458d7..b954762 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.security.OzoneSecurityException;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.AccessControlException;
@@ -187,6 +188,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
private final Runnable shutdownHook;
private final File omMetaDir;
private final SecurityConfig secConfig;
+ private final S3SecretManager s3SecretManager;
private OzoneManager(OzoneConfiguration conf) throws IOException {
Preconditions.checkNotNull(conf);
@@ -245,6 +247,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
volumeManager, bucketManager);
keyManager = new KeyManagerImpl(scmBlockClient, metadataManager,
configuration, omStorage.getOmId(), blockTokenMgr);
+ s3SecretManager = new S3SecretManagerImpl(configuration, metadataManager);
shutdownHook = () -> {
saveOmMetrics();
@@ -1655,6 +1658,14 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
/**
* {@inheritDoc}
*/
+ public S3SecretValue getS3Secret(String kerberosID) throws IOException{
+ return s3SecretManager.getS3Secret(kerberosID);
+ }
+
+ @Override
+ /**
+ * {@inheritDoc}
+ */
public String getOzoneBucketMapping(String s3BucketName)
throws IOException {
return s3BucketManager.getOzoneBucketMapping(s3BucketName);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3SecretManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3SecretManager.java
new file mode 100644
index 0000000..4c87274
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3SecretManager.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.ozone.om;
+
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
+
+import java.io.IOException;
+/**
+ * Interface to manager s3 secret.
+ */
+public interface S3SecretManager {
+
+ S3SecretValue getS3Secret(String kerberosID) throws IOException;
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java
new file mode 100644
index 0000000..902130d
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.logging.log4j.util.Strings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * S3 Secret manager.
+ */
+public class S3SecretManagerImpl implements S3SecretManager {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(S3SecretManagerImpl.class);
+
+ /**
+ * OMMetadataManager is used for accessing OM MetadataDB and ReadWriteLock.
+ */
+ private final OMMetadataManager omMetadataManager;
+ private final OzoneConfiguration configuration;
+
+ /**
+ * Constructs S3SecretManager.
+ *
+ * @param omMetadataManager
+ */
+ public S3SecretManagerImpl(OzoneConfiguration configuration,
+ OMMetadataManager omMetadataManager) {
+ this.configuration = configuration;
+ this.omMetadataManager = omMetadataManager;
+ }
+
+ @Override
+ public S3SecretValue getS3Secret(String kerberosID) throws IOException {
+ Preconditions.checkArgument(Strings.isNotBlank(kerberosID),
+ "kerberosID cannot be null or empty.");
+ byte[] awsAccessKey = OmUtils.getMD5Digest(kerberosID);
+ S3SecretValue result = null;
+ omMetadataManager.getLock().acquireS3SecretLock(kerberosID);
+ try {
+ byte[] s3Secret =
+ omMetadataManager.getS3SecretTable().get(awsAccessKey);
+ if(s3Secret == null) {
+ byte[] secret = OmUtils.getSHADigest();
+ result = new S3SecretValue(kerberosID, DigestUtils.sha256Hex(secret));
+ omMetadataManager.getS3SecretTable()
+ .put(awsAccessKey, result.getProtobuf().toByteArray());
+ } else {
+ result = S3SecretValue.fromProtobuf(
+ OzoneManagerProtocolProtos.S3Secret.parseFrom(s3Secret));
+ }
+ result.setAwsAccessKey(DigestUtils.md5Hex(awsAccessKey));
+ } finally {
+ omMetadataManager.getLock().releaseS3SecretLock(kerberosID);
+ }
+ return result;
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index 3292adf..a1a3fbd 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@ -115,6 +115,7 @@ public class OMException extends IOException {
SCM_VERSION_MISMATCH_ERROR,
SCM_IN_CHILL_MODE,
S3_BUCKET_ALREADY_EXISTS,
- S3_BUCKET_NOT_FOUND
+ S3_BUCKET_NOT_FOUND,
+ S3_SECRET_NOT_FOUND
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
index e2536d3..112cecb 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.AllocateBlockRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@@ -199,6 +200,8 @@ public class OzoneManagerProtocolServerSideTranslatorPB
implements
return Status.S3_BUCKET_ALREADY_EXISTS;
case S3_BUCKET_NOT_FOUND:
return Status.S3_BUCKET_NOT_FOUND;
+ case S3_SECRET_NOT_FOUND:
+ return Status.S3_SECRET_NOT_FOUND;
default:
return Status.INTERNAL_ERROR;
}
@@ -662,6 +665,22 @@ public class OzoneManagerProtocolServerSideTranslatorPB
implements
}
@Override
+ public OzoneManagerProtocolProtos.S3SecretResponse getS3Secret(
+ RpcController controller,
+ OzoneManagerProtocolProtos.S3SecretRequest request)
+ throws ServiceException {
+ OzoneManagerProtocolProtos.S3SecretResponse.Builder resp =
+ OzoneManagerProtocolProtos.S3SecretResponse.newBuilder();
+ try {
+
resp.setS3Secret(impl.getS3Secret(request.getKerberosID()).getProtobuf());
+ resp.setStatus(Status.OK);
+ } catch (IOException e) {
+ resp.setStatus(exceptionToResponseStatus(e));
+ }
+ return resp.build();
+ }
+
+ @Override
public GetDelegationTokenResponseProto getDelegationToken(
RpcController controller, GetDelegationTokenRequestProto request)
throws ServiceException {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
