This is an automated email from the ASF dual-hosted git repository.
mehul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new e5afd84 RANGER-2616 : Add reencryptEncryptedKey, batch reencryption
interface to KMS and improve logs
e5afd84 is described below
commit e5afd844a3020c6c9872481c6dd76a3489dd5c19
Author: fatimaawez <[email protected]>
AuthorDate: Tue Nov 5 16:23:07 2019 +0530
RANGER-2616 : Add reencryptEncryptedKey, batch reencryption interface to
KMS and improve logs
Signed-off-by: Mehul Parikh <[email protected]>
---
...agerKeyGeneratorKeyProviderCryptoExtension.java | 22 +-
.../apache/hadoop/crypto/key/kms/server/KMS.java | 712 ++++++++++++---------
.../hadoop/crypto/key/kms/server/KMSAudit.java | 7 +-
.../key/kms/server/KMSExceptionsProvider.java | 7 +-
.../crypto/key/kms/server/KMSJSONReader.java | 7 +-
.../crypto/key/kms/server/KMSServerJSONUtils.java | 34 +-
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 17 +
.../kms/server/KeyAuthorizationKeyProvider.java | 32 +
.../hadoop/crypto/key/kms/server/TestKMSAudit.java | 62 +-
9 files changed, 566 insertions(+), 334 deletions(-)
diff --git
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
index 854c831..675a357 100644
---
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
+++
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
@@ -139,13 +139,12 @@ public class EagerKeyGeneratorKeyProviderCryptoExtension
@Override
public EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion
arg0)
throws IOException, GeneralSecurityException {
- // TODO Auto-generated method stub
- return null;
+ return keyProviderCryptoExtension.reencryptEncryptedKey(arg0);
}
@Override
public void reencryptEncryptedKeys(List<EncryptedKeyVersion> arg0)
throws IOException, GeneralSecurityException {
- // TODO Auto-generated method stub
+ keyProviderCryptoExtension.reencryptEncryptedKeys(arg0);
}
}
@@ -165,6 +164,16 @@ public class EagerKeyGeneratorKeyProviderCryptoExtension
new CryptoExtension(conf, keyProviderCryptoExtension));
}
+ /**
+ * Roll a new version of the given key generating the material for it.
+ * <p>
+ * Due to the caching on the ValueQueue, even after a rollNewVersion call,
+ * {@link #generateEncryptedKey(String)} may still return an old key - even
+ * when we drain the queue here, the async thread may later fill in old
keys.
+ * This is acceptable since old version keys are still able to decrypt, and
+ * client shall make no assumptions that it will get a new versioned key
+ * after rollNewVersion.
+ */
@Override
public KeyVersion rollNewVersion(String name)
throws NoSuchAlgorithmException, IOException {
@@ -180,4 +189,11 @@ public class EagerKeyGeneratorKeyProviderCryptoExtension
getExtension().drain(name);
return keyVersion;
}
+
+ @Override
+ public void invalidateCache(String name) throws IOException {
+ super.invalidateCache(name);
+ getExtension().drain(name);
+ }
+
}
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
index 04cc984..f67f68b 100644
--- a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+
+import org.apache.hadoop.util.KMSUtil;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.crypto.key.KeyProvider;
@@ -29,7 +33,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.server.KMSACLsType.Type;
import
org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation;
-import org.apache.hadoop.util.KMSUtil;
+import javax.ws.rs.core.UriBuilder;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Consumes;
@@ -47,7 +51,6 @@ import javax.ws.rs.core.Response;
import java.io.IOException;
import java.net.URI;
-import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.LinkedList;
@@ -56,6 +59,10 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.util.KMSUtil.checkNotEmpty;
+import static org.apache.hadoop.util.KMSUtil.checkNotNull;
/**
* Class providing the REST bindings, via Jersey, for the KMS.
*/
@@ -64,16 +71,18 @@ import java.util.regex.Pattern;
public class KMS {
public static enum KMSOp {
- CREATE_KEY, DELETE_KEY, ROLL_NEW_VERSION,
+ CREATE_KEY, DELETE_KEY, ROLL_NEW_VERSION, INVALIDATE_CACHE,
GET_KEYS, GET_KEYS_METADATA,
GET_KEY_VERSIONS, GET_METADATA, GET_KEY_VERSION, GET_CURRENT_KEY,
- GENERATE_EEK, DECRYPT_EEK
+ GENERATE_EEK, DECRYPT_EEK, REENCRYPT_EEK, REENCRYPT_EEK_BATCH
}
private static final String KEY_NAME_VALIDATION =
"[a-z,A-Z,0-9](?!.*--)(?!.*__)(?!.*-_)(?!.*_-)[\\w\\-\\_]*";
private KeyProviderCryptoExtension provider;
private KMSAudit kmsAudit;
+ static final Logger LOG = LoggerFactory.getLogger(KMS.class);
+ private static final int MAX_NUM_PER_BATCH = 10000;
public KMS() throws Exception {
provider = KMSWebApp.getKeyProvider();
@@ -81,7 +90,7 @@ public class KMS {
}
private void assertAccess(Type aclType, UserGroupInformation ugi,
- KMSOp operation, String clientIp) throws AccessControlException {
+ KMSOp operation, String clientIp) throws AccessControlException {
KMSWebApp.getACLs().assertAccess(aclType, ugi, operation, null, clientIp);
}
@@ -96,9 +105,9 @@ public class KMS {
keyVersion.getVersionName(), null);
}
- private static URI getKeyURI(String name) throws URISyntaxException {
- return new URI(KMSRESTConstants.SERVICE_VERSION + "/" +
- KMSRESTConstants.KEY_RESOURCE + "/" + name);
+ private static URI getKeyURI(String domain, String keyName) {
+ return UriBuilder.fromPath("{a}/{b}/{c}")
+ .build(domain, KMSRESTConstants.KEY_RESOURCE, keyName);
}
@POST
@@ -107,133 +116,166 @@ public class KMS {
@Produces(MediaType.APPLICATION_JSON)
@SuppressWarnings("unchecked")
public Response createKey(Map jsonKey, @Context HttpServletRequest request)
throws Exception {
- KMSWebApp.getAdminCallsMeter().mark();
- UserGroupInformation user = HttpUserGroupInformation.get();
- final String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD);
- KMSUtil.checkNotEmpty(name, KMSRESTConstants.NAME_FIELD);
- validateKeyName(name);
- assertAccess(Type.CREATE, user, KMSOp.CREATE_KEY, name,
request.getRemoteAddr());
- String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD);
- final String material = (String)
jsonKey.get(KMSRESTConstants.MATERIAL_FIELD);
- int length = (jsonKey.containsKey(KMSRESTConstants.LENGTH_FIELD))
- ? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0;
- String description = (String)
- jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD);
- Map<String, String> attributes = (Map<String, String>)
- jsonKey.get(KMSRESTConstants.ATTRIBUTES_FIELD);
- if (material != null) {
- assertAccess(Type.SET_KEY_MATERIAL, user,
- KMSOp.CREATE_KEY, name, request.getRemoteAddr());
- }
- final KeyProvider.Options options = new KeyProvider.Options(
+ try{
+ LOG.info("Entering createKey Method.");
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ final String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD);
+ checkNotEmpty(name, KMSRESTConstants.NAME_FIELD);
+ validateKeyName(name);
+ assertAccess(Type.CREATE, user, KMSOp.CREATE_KEY, name,
request.getRemoteAddr());
+ String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD);
+ final String material;
+ material = (String) jsonKey.get(KMSRESTConstants.MATERIAL_FIELD);
+ int length = (jsonKey.containsKey(KMSRESTConstants.LENGTH_FIELD)) ?
(Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0;
+ String description = (String)
jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD);
+ LOG.debug("Creating key with name {}, cipher being used{}, "
+ +"length of key {}, description of key {}", name, cipher,length,
description);
+ Map<String, String> attributes = (Map<String, String>)
+ jsonKey.get(KMSRESTConstants.ATTRIBUTES_FIELD);
+ if (material != null) {
+ assertAccess(Type.SET_KEY_MATERIAL, user,
+ KMSOp.CREATE_KEY, name, request.getRemoteAddr());
+ }
+ final KeyProvider.Options options = new KeyProvider.Options(
KMSWebApp.getConfiguration());
- if (cipher != null) {
- options.setCipher(cipher);
- }
- if (length != 0) {
- options.setBitLength(length);
- }
- options.setDescription(description);
- options.setAttributes(attributes);
-
- KeyProvider.KeyVersion keyVersion = user.doAs(
+ if (cipher != null) {
+ options.setCipher(cipher);
+ }
+ if (length != 0) {
+ options.setBitLength(length);
+ }
+ options.setDescription(description);
+ options.setAttributes(attributes);
+ KeyProvider.KeyVersion keyVersion = user.doAs(
new PrivilegedExceptionAction<KeyVersion>() {
@Override
public KeyVersion run() throws Exception {
- KeyProvider.KeyVersion keyVersion = (material != null)
- ? provider.createKey(name, Base64.decodeBase64(material),
options)
- : provider.createKey(name, options);
+ KeyProvider.KeyVersion keyVersion = (material != null) ?
provider.createKey(name, Base64.decodeBase64(material),options) :
provider.createKey(name, options);
provider.flush();
return keyVersion;
}
}
- );
-
- kmsAudit.ok(user, KMSOp.CREATE_KEY, name, "UserProvidedMaterial:" +
- (material != null) + " Description:" + description);
-
- if (!KMSWebApp.getACLs().hasAccess(Type.GET, user,
request.getRemoteAddr())) {
- keyVersion = removeKeyMaterial(keyVersion);
+ );
+ kmsAudit.ok(user, KMSOp.CREATE_KEY, name, "UserProvidedMaterial:" +
(material != null) + " Description:" + description);
+ if (!KMSWebApp.getACLs().hasAccess(Type.GET, user,
request.getRemoteAddr())) {
+ keyVersion = removeKeyMaterial(keyVersion);
+ }
+ Map json = KMSUtil.toJSON(keyVersion);
+ String requestURL = KMSMDCFilter.getURL();
+ int idx = requestURL.lastIndexOf(KMSRESTConstants.KEYS_RESOURCE);
+ requestURL = requestURL.substring(0, idx);
+ LOG.info("Exiting createKey Method.");
+ return Response.created(getKeyURI(KMSRESTConstants.SERVICE_VERSION,
name))
+ .type(MediaType.APPLICATION_JSON)
+ .header("Location", getKeyURI(requestURL, name)).entity(json).build();
+ }
+ catch (Exception e) {
+ LOG.debug("Exception in createKey.", e);
+ throw e;
}
- Map json = KMSServerJSONUtils.toJSON(keyVersion);
- String requestURL = KMSMDCFilter.getURL();
- int idx = requestURL.lastIndexOf(KMSRESTConstants.KEYS_RESOURCE);
- requestURL = requestURL.substring(0, idx);
- String keyURL = requestURL + KMSRESTConstants.KEY_RESOURCE + "/" + name;
- return Response.created(getKeyURI(name)).type(MediaType.APPLICATION_JSON).
- header("Location", keyURL).entity(json).build();
}
private void validateKeyName(String name) {
- Pattern pattern = Pattern.compile(KEY_NAME_VALIDATION);
- Matcher matcher = pattern.matcher(name);
+ Pattern pattern = Pattern.compile(KEY_NAME_VALIDATION);
+ Matcher matcher = pattern.matcher(name);
if(!matcher.matches()){
throw new IllegalArgumentException("Key Name : " + name +
", should start with alpha/numeric letters and can
have special characters - (hypen) or _ (underscore)");
}
}
- @DELETE
- @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
- public Response deleteKey(@PathParam("name") final String name, @Context
HttpServletRequest request)
- throws Exception {
- KMSWebApp.getAdminCallsMeter().mark();
- UserGroupInformation user = HttpUserGroupInformation.get();
- assertAccess(Type.DELETE, user, KMSOp.DELETE_KEY, name,
request.getRemoteAddr());
- KMSUtil.checkNotEmpty(name, "name");
-
- user.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- provider.deleteKey(name);
- provider.flush();
- return null;
- }
- });
-
- kmsAudit.ok(user, KMSOp.DELETE_KEY, name, "");
-
- return Response.ok().build();
- }
+ @DELETE
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
+ public Response deleteKey(@PathParam("name") final String name, @Context
HttpServletRequest request)
+ throws Exception {
+ try {
+ LOG.info("Entering deleteKey method.");
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ assertAccess(Type.DELETE, user, KMSOp.DELETE_KEY, name,
request.getRemoteAddr());
+ checkNotEmpty(name, "name");
+ LOG.debug("Deleting key with name {}.", name);
+ user.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ provider.deleteKey(name);
+ provider.flush();
+ return null;
+ }
+ });
+ kmsAudit.ok(user, KMSOp.DELETE_KEY, name, "");
+ LOG.info("Exiting deleteKey method.");
+ return Response.ok().build();
+ } catch (Exception e) {
+ LOG.debug("Exception in deleteKey.", e);
+ throw e;
+ }
+ }
@POST
@Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
- public Response rolloverKey(@PathParam("name") final String name,
- Map jsonMaterial, @Context HttpServletRequest request) throws Exception {
- KMSWebApp.getAdminCallsMeter().mark();
- UserGroupInformation user = HttpUserGroupInformation.get();
- assertAccess(Type.ROLLOVER, user, KMSOp.ROLL_NEW_VERSION, name,
request.getRemoteAddr());
- KMSUtil.checkNotEmpty(name, "name");
- final String material = (String)
- jsonMaterial.get(KMSRESTConstants.MATERIAL_FIELD);
- if (material != null) {
- assertAccess(Type.SET_KEY_MATERIAL, user,
- KMSOp.ROLL_NEW_VERSION, name, request.getRemoteAddr());
+ public Response rolloverKey(@PathParam("name") final String name, Map
jsonMaterial, @Context HttpServletRequest request) throws Exception {
+ try {
+ LOG.info("Entering rolloverKey Method.");
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ assertAccess(Type.ROLLOVER, user, KMSOp.ROLL_NEW_VERSION, name,
request.getRemoteAddr());
+ checkNotEmpty(name, "name");
+ LOG.debug("Rolling key with name {}.", name);
+ final String material = (String)
jsonMaterial.get(KMSRESTConstants.MATERIAL_FIELD);
+ if (material != null) {
+ assertAccess(Type.SET_KEY_MATERIAL, user, KMSOp.ROLL_NEW_VERSION,
name, request.getRemoteAddr());
+ }
+ KeyProvider.KeyVersion keyVersion = user.doAs(new
PrivilegedExceptionAction<KeyVersion>(){
+ @Override
+ public KeyVersion run() throws Exception {KeyVersion keyVersion =
(material != null)
+ ? provider.rollNewVersion(name, Base64.decodeBase64(material)):
provider.rollNewVersion(name);
+ provider.flush();
+ return keyVersion;
+ }
+ });
+ kmsAudit.ok(user, KMSOp.ROLL_NEW_VERSION, name, "UserProvidedMaterial:" +
+ (material != null) + " NewVersion:" + keyVersion.getVersionName());
+ if (!KMSWebApp.getACLs().hasAccess(Type.GET, user,
request.getRemoteAddr())) {
+ keyVersion = removeKeyMaterial(keyVersion);
+ }
+ Map json = KMSUtil.toJSON(keyVersion);
+ LOG.info("Exiting rolloverKey Method.");
+ return
Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ } catch (Exception e) {
+ LOG.debug("Exception in rolloverKey.", e);
+ throw e;
}
+ }
- KeyProvider.KeyVersion keyVersion = user.doAs(
- new PrivilegedExceptionAction<KeyVersion>() {
- @Override
- public KeyVersion run() throws Exception {
- KeyVersion keyVersion = (material != null)
- ? provider.rollNewVersion(name, Base64.decodeBase64(material))
- : provider.rollNewVersion(name);
- provider.flush();
- return keyVersion;
- }
+ @POST
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
KMSRESTConstants.INVALIDATECACHE_RESOURCE)
+ public Response invalidateCache(@PathParam("name") final String name) throws
Exception {
+ try {
+ LOG.info("Entering invalidateCache Method.");
+ KMSWebApp.getAdminCallsMeter().mark();
+ checkNotEmpty(name, "name");
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ assertAccess(Type.ROLLOVER, user, KMSOp.INVALIDATE_CACHE, name);
+ LOG.debug("Invalidating cache with key name {}.", name);
+ user.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ provider.invalidateCache(name);
+ provider.flush();
+ return null;
}
- );
-
- kmsAudit.ok(user, KMSOp.ROLL_NEW_VERSION, name, "UserProvidedMaterial:" +
- (material != null) + " NewVersion:" + keyVersion.getVersionName());
-
- if (!KMSWebApp.getACLs().hasAccess(Type.GET, user,
request.getRemoteAddr())) {
- keyVersion = removeKeyMaterial(keyVersion);
+ });
+ kmsAudit.ok(user, KMSOp.INVALIDATE_CACHE, name, "");
+ LOG.info("Exiting invalidateCache for key name {}.", name);
+ return Response.ok().build();
+ } catch (Exception e) {
+ LOG.debug("Exception in invalidateCache for key name {}.", name, e);
+ throw e;
}
- Map json = KMSServerJSONUtils.toJSON(keyVersion);
- return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
}
@GET
@@ -241,52 +283,65 @@ public class KMS {
@Produces(MediaType.APPLICATION_JSON)
public Response getKeysMetadata(@QueryParam(KMSRESTConstants.KEY)
List<String> keyNamesList, @Context HttpServletRequest request) throws
Exception {
- KMSWebApp.getAdminCallsMeter().mark();
- UserGroupInformation user = HttpUserGroupInformation.get();
- final String[] keyNames = keyNamesList.toArray(
- new String[keyNamesList.size()]);
- assertAccess(Type.GET_METADATA, user, KMSOp.GET_KEYS_METADATA,
request.getRemoteAddr());
-
- KeyProvider.Metadata[] keysMeta = user.doAs(
- new PrivilegedExceptionAction<KeyProvider.Metadata[]>() {
- @Override
- public KeyProvider.Metadata[] run() throws Exception {
- return provider.getKeysMetadata(keyNames);
- }
+ try {
+ LOG.info("Entering getKeysMetadata method.");
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ final String[] keyNames = keyNamesList.toArray( new
String[keyNamesList.size()]);
+ assertAccess(Type.GET_METADATA, user, KMSOp.GET_KEYS_METADATA,
request.getRemoteAddr());
+ KeyProvider.Metadata[] keysMeta = user.doAs(new
PrivilegedExceptionAction<KeyProvider.Metadata[]>() {
+ @Override
+ public KeyProvider.Metadata[] run() throws Exception {
+ return provider.getKeysMetadata(keyNames);
}
- );
-
- Object json = KMSServerJSONUtils.toJSON(keyNames, keysMeta);
- kmsAudit.ok(user, KMSOp.GET_KEYS_METADATA, "");
- return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ });
+ Object json = KMSServerJSONUtils.toJSON(keyNames, keysMeta);
+ kmsAudit.ok(user, KMSOp.GET_KEYS_METADATA, "");
+ LOG.info("Exiting getKeysMetadata method.");
+ return
Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ } catch (Exception e) {
+ LOG.debug("Exception in getKeysmetadata.", e);
+ throw e;
+ }
}
@GET
@Path(KMSRESTConstants.KEYS_NAMES_RESOURCE)
@Produces(MediaType.APPLICATION_JSON)
public Response getKeyNames(@Context HttpServletRequest request) throws
Exception {
- KMSWebApp.getAdminCallsMeter().mark();
- UserGroupInformation user = HttpUserGroupInformation.get();
- assertAccess(Type.GET_KEYS, user, KMSOp.GET_KEYS, request.getRemoteAddr());
-
- List<String> json = user.doAs(
- new PrivilegedExceptionAction<List<String>>() {
- @Override
- public List<String> run() throws Exception {
- return provider.getKeys();
- }
+ try {
+ LOG.info("Entering getKeyNames method.");
+ KMSWebApp.getAdminCallsMeter().mark();
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ assertAccess(Type.GET_KEYS, user, KMSOp.GET_KEYS,
request.getRemoteAddr());
+ List<String> json = user.doAs(new
PrivilegedExceptionAction<List<String>>() {
+ @Override
+ public List<String> run() throws Exception {
+ return provider.getKeys();
}
- );
-
- kmsAudit.ok(user, KMSOp.GET_KEYS, "");
- return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ });
+ kmsAudit.ok(user, KMSOp.GET_KEYS, "");
+ LOG.info("Exiting getKeyNames method.");
+ return
Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ } catch (Exception e) {
+ LOG.debug("Exception in getkeyNames.", e);
+ throw e;
+ }
}
@GET
@Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}")
public Response getKey(@PathParam("name") String name, @Context
HttpServletRequest request)
throws Exception {
- return getMetadata(name, request);
+ try {
+ LOG.info("Entering getKey method.");
+ LOG.debug("Getting key information for key with name {}.", name);
+ LOG.info("Exiting getKey method.");
+ return getMetadata(name, request);
+ } catch (Exception e) {
+ LOG.debug("Exception in getKey.", e);
+ throw e;
+ }
}
@GET
@@ -295,23 +350,28 @@ public class KMS {
@Produces(MediaType.APPLICATION_JSON)
public Response getMetadata(@PathParam("name") final String name, @Context
HttpServletRequest request)
throws Exception {
- UserGroupInformation user = HttpUserGroupInformation.get();
- KMSUtil.checkNotEmpty(name, "name");
- KMSWebApp.getAdminCallsMeter().mark();
- assertAccess(Type.GET_METADATA, user, KMSOp.GET_METADATA, name,
request.getRemoteAddr());
-
- KeyProvider.Metadata metadata = user.doAs(
+ try {
+ LOG.info("Entering getMetadata method.");
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ checkNotEmpty(name, "name");
+ KMSWebApp.getAdminCallsMeter().mark();
+ assertAccess(Type.GET_METADATA, user, KMSOp.GET_METADATA, name,
request.getRemoteAddr());
+ LOG.debug("Getting metadata for key with name {}.", name);
+ KeyProvider.Metadata metadata = user.doAs(
new PrivilegedExceptionAction<KeyProvider.Metadata>() {
- @Override
- public KeyProvider.Metadata run() throws Exception {
- return provider.getMetadata(name);
- }
+ @Override
+ public KeyProvider.Metadata run() throws Exception {
+ return provider.getMetadata(name);
}
- );
-
- Object json = KMSServerJSONUtils.toJSON(name, metadata);
- kmsAudit.ok(user, KMSOp.GET_METADATA, name, "");
- return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ });
+ Object json = KMSServerJSONUtils.toJSON(name, metadata);
+ kmsAudit.ok(user, KMSOp.GET_METADATA, name, "");
+ LOG.info("Exiting getMetadata method.");
+ return
Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ } catch (Exception e) {
+ LOG.debug("Exception in getMetadata.", e);
+ throw e;
+ }
}
@GET
@@ -320,25 +380,26 @@ public class KMS {
@Produces(MediaType.APPLICATION_JSON)
public Response getCurrentVersion(@PathParam("name") final String name,
@Context HttpServletRequest request)
throws Exception {
- UserGroupInformation user = HttpUserGroupInformation.get();
- KMSUtil.checkNotEmpty(name, "name");
- KMSWebApp.getKeyCallsMeter().mark();
- assertAccess(Type.GET, user, KMSOp.GET_CURRENT_KEY, name,
request.getRemoteAddr());
-
- KeyVersion keyVersion = user.doAs(
- new PrivilegedExceptionAction<KeyVersion>() {
- @Override
- public KeyVersion run() throws Exception {
- return provider.getCurrentKey(name);
- }
+ try {
+ LOG.info("Entering getCurrentVersion method.");
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ checkNotEmpty(name, "name");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(Type.GET, user, KMSOp.GET_CURRENT_KEY, name,
request.getRemoteAddr());
+ LOG.debug("Getting key version for key with name {}.", name);
+ KeyVersion keyVersion = user.doAs(new
PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ return provider.getCurrentKey(name);
}
- );
- Object json = KMSServerJSONUtils.toJSON(keyVersion);
- if (keyVersion != null) {
- kmsAudit.ok(user, KMSOp.GET_CURRENT_KEY, name, "");
- return
Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
- }else{
- return
Response.status(404).type(MediaType.APPLICATION_JSON).entity(json).build();
+ });
+ Object json = KMSUtil.toJSON(keyVersion);
+ kmsAudit.ok(user, KMSOp.GET_CURRENT_KEY, name, "");
+ LOG.info("Exiting getCurrentVersion method.");
+ return
Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ } catch (Exception e) {
+ LOG.debug("Exception in getCurrentVersion.", e);
+ throw e;
}
}
@@ -347,25 +408,29 @@ public class KMS {
@Produces(MediaType.APPLICATION_JSON)
public Response getKeyVersion(
@PathParam("versionName") final String versionName, @Context
HttpServletRequest request) throws Exception {
- UserGroupInformation user = HttpUserGroupInformation.get();
- KMSUtil.checkNotEmpty(versionName, "versionName");
- KMSWebApp.getKeyCallsMeter().mark();
- assertAccess(Type.GET, user, KMSOp.GET_KEY_VERSION,
request.getRemoteAddr());
-
- KeyVersion keyVersion = user.doAs(
- new PrivilegedExceptionAction<KeyVersion>() {
- @Override
- public KeyVersion run() throws Exception {
- return provider.getKeyVersion(versionName);
- }
+ try {
+ LOG.info("Entering getKeyVersion method.");
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ checkNotEmpty(versionName, "versionName");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(Type.GET, user, KMSOp.GET_KEY_VERSION,
request.getRemoteAddr());
+ LOG.debug("Getting key with version name {}.", versionName);
+ KeyVersion keyVersion = user.doAs(new
PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ return provider.getKeyVersion(versionName);
}
- );
-
- if (keyVersion != null) {
- kmsAudit.ok(user, KMSOp.GET_KEY_VERSION, keyVersion.getName(), "");
+ });
+ if (keyVersion != null) {
+ kmsAudit.ok(user, KMSOp.GET_KEY_VERSION, keyVersion.getName(), "");
+ }
+ Object json = KMSUtil.toJSON(keyVersion);
+ LOG.info("Exiting getKeyVersion method.");
+ return
Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ } catch (Exception e) {
+ LOG.debug("Exception in getKeyVersion.", e);
+ throw e;
}
- Object json = KMSServerJSONUtils.toJSON(keyVersion);
- return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
}
@SuppressWarnings({ "rawtypes", "unchecked" })
@@ -373,52 +438,107 @@ public class KMS {
@Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
KMSRESTConstants.EEK_SUB_RESOURCE)
@Produces(MediaType.APPLICATION_JSON)
- public Response generateEncryptedKeys(
- @PathParam("name") final String name,
- @QueryParam(KMSRESTConstants.EEK_OP) String edekOp,
- @DefaultValue("1")
- @QueryParam(KMSRESTConstants.EEK_NUM_KEYS) final int numKeys,
@Context HttpServletRequest request)
- throws Exception {
- UserGroupInformation user = HttpUserGroupInformation.get();
- KMSUtil.checkNotEmpty(name, "name");
- KMSUtil.checkNotNull(edekOp, "eekOp");
-
- Object retJSON;
- if (edekOp.equals(KMSRESTConstants.EEK_GENERATE)) {
- assertAccess(Type.GENERATE_EEK, user, KMSOp.GENERATE_EEK, name,
request.getRemoteAddr());
-
- final List<EncryptedKeyVersion> retEdeks =
- new LinkedList<EncryptedKeyVersion>();
- try {
-
- user.doAs(
- new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- for (int i = 0; i < numKeys; i++) {
- retEdeks.add(provider.generateEncryptedKey(name));
- }
- return null;
+ public Response generateEncryptedKeys( @PathParam("name") final String name,
+ @QueryParam(KMSRESTConstants.EEK_OP) String edekOp,
+ @DefaultValue("1") @QueryParam(KMSRESTConstants.EEK_NUM_KEYS) final int
numKeys,
+ @Context HttpServletRequest request) throws Exception {
+ try {
+ LOG.info("Entering generateEncryptedKeys method.");
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ checkNotEmpty(name, "name");
+ checkNotNull(edekOp, "eekOp");
+ LOG.debug("Generating encrypted key with name {}, the edek Operation is
{}.", name, edekOp);
+ Object retJSON;
+ if (edekOp.equals(KMSRESTConstants.EEK_GENERATE)) {
+ LOG.debug("edek Operation is Generate.");
+ assertAccess(Type.GENERATE_EEK, user, KMSOp.GENERATE_EEK,
name,request.getRemoteAddr());
+ final List<EncryptedKeyVersion> retEdeks = new
LinkedList<EncryptedKeyVersion>();
+ try {
+ user.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ LOG.debug("Generated Encrypted key for {} number of keys.",
numKeys);
+ for (int i = 0; i < numKeys; i++) {
+ retEdeks.add(provider.generateEncryptedKey(name));
}
+ return null;
}
- );
-
- } catch (Exception e) {
- throw new IOException(e);
+ });
+ } catch (Exception e) {
+ LOG.error("Exception in generateEncryptedKeys:", e);
+ throw new IOException(e);
+ }
+ kmsAudit.ok(user, KMSOp.GENERATE_EEK, name, "");
+ retJSON = new ArrayList();
+ for (EncryptedKeyVersion edek : retEdeks) {
+ ((ArrayList) retJSON).add(KMSUtil.toJSON(edek));
+ }
+ } else {
+ StringBuilder error;
+ error = new StringBuilder("IllegalArgumentException Wrong ");
+ error.append(KMSRESTConstants.EEK_OP);
+ error.append(" value, it must be ");
+ error.append(KMSRESTConstants.EEK_GENERATE);
+ error.append(" or ");
+ error.append(KMSRESTConstants.EEK_DECRYPT);
+ LOG.error(error.toString());
+ throw new IllegalArgumentException(error.toString());
}
- kmsAudit.ok(user, KMSOp.GENERATE_EEK, name, "");
- retJSON = new ArrayList();
- for (EncryptedKeyVersion edek : retEdeks) {
- ((ArrayList)retJSON).add(KMSServerJSONUtils.toJSON(edek));
+ KMSWebApp.getGenerateEEKCallsMeter().mark();
+ LOG.info("Exiting generateEncryptedKeys method.");
+ return
Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON).build();
+ } catch (Exception e) {
+ LOG.debug("Exception in generateEncryptedKeys.", e);
+ throw e;
+ }
+ }
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @POST
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.REENCRYPT_BATCH_SUB_RESOURCE)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON )
+ public Response reencryptEncryptedKeys(
+ @PathParam("name") final String name,
+ final List<Map> jsonPayload)
+ throws Exception {
+ try {
+ LOG.info("Entering reencryptEncryptedKeys method.");
+ final Stopwatch sw = Stopwatch.createStarted();
+ checkNotEmpty(name, "name");
+ checkNotNull(jsonPayload, "jsonPayload");
+ final UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSWebApp.getReencryptEEKBatchCallsMeter().mark();
+ if (jsonPayload.size() > MAX_NUM_PER_BATCH) {
+ LOG.warn("Payload size {} too big for reencryptEncryptedKeys from"
+ + " user {}.", jsonPayload.size(), user);
+ }
+ assertAccess(Type.GENERATE_EEK, user, KMSOp.REENCRYPT_EEK_BATCH,name);
+ LOG.debug("Batch reencrypting {} Encrypted Keys for key name {}",
jsonPayload.size(), name);
+ final List<EncryptedKeyVersion> ekvs =
KMSUtil.parseJSONEncKeyVersions(name, jsonPayload);
+ Preconditions.checkArgument(ekvs.size() ==
jsonPayload.size(),"EncryptedKey size mismatch after parsing from json");
+ for (EncryptedKeyVersion ekv : ekvs) {
+
Preconditions.checkArgument(name.equals(ekv.getEncryptionKeyName()),"All
EncryptedKeys must be under the given key name " + name);
+ }
+ user.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ provider.reencryptEncryptedKeys(ekvs);
+ return null;
+ }
+ });
+ List retJSON = new ArrayList<>(ekvs.size());
+ for (EncryptedKeyVersion ekv: ekvs) {
+ retJSON.add(KMSUtil.toJSON(ekv));
}
- } else {
- throw new IllegalArgumentException("Wrong " + KMSRESTConstants.EEK_OP +
- " value, it must be " + KMSRESTConstants.EEK_GENERATE + " or " +
- KMSRESTConstants.EEK_DECRYPT);
+ kmsAudit.ok(user, KMSOp.REENCRYPT_EEK_BATCH, name,"reencrypted " +
ekvs.size() + " keys");
+ LOG.info("reencryptEncryptedKeys {} keys for key {} took {}",
jsonPayload.size(), name, sw.stop());
+ LOG.info("Exiting reencryptEncryptedKeys method.");
+ return
Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON).build();
+ } catch (Exception e) {
+ LOG.debug("Exception in reencryptEncryptedKeys.", e);
+ throw e;
}
- KMSWebApp.getGenerateEEKCallsMeter().mark();
- return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON)
- .build();
}
@SuppressWarnings("rawtypes")
@@ -426,52 +546,66 @@ public class KMS {
@Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}/" +
KMSRESTConstants.EEK_SUB_RESOURCE)
@Produces(MediaType.APPLICATION_JSON)
- public Response decryptEncryptedKey(
+ public Response handleEncryptedKeyOp(
@PathParam("versionName") final String versionName,
@QueryParam(KMSRESTConstants.EEK_OP) String eekOp,
Map jsonPayload, @Context HttpServletRequest request)
throws Exception {
- UserGroupInformation user = HttpUserGroupInformation.get();
- KMSUtil.checkNotEmpty(versionName, "versionName");
- KMSUtil.checkNotNull(eekOp, "eekOp");
-
- final String keyName = (String) jsonPayload.get(
- KMSRESTConstants.NAME_FIELD);
- String ivStr = (String) jsonPayload.get(KMSRESTConstants.IV_FIELD);
- String encMaterialStr =
- (String) jsonPayload.get(KMSRESTConstants.MATERIAL_FIELD);
- Object retJSON;
- if (eekOp.equals(KMSRESTConstants.EEK_DECRYPT)) {
- assertAccess(Type.DECRYPT_EEK, user, KMSOp.DECRYPT_EEK, keyName,
request.getRemoteAddr());
- KMSUtil.checkNotNull(ivStr, KMSRESTConstants.IV_FIELD);
+ try {
+ LOG.info("Entering decryptEncryptedKey method.");
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ checkNotEmpty(versionName, "versionName");
+ checkNotNull(eekOp, "eekOp");
+ LOG.debug("Decrypting key for {}, the edek Operation is {}.",
versionName, eekOp);
+ final String keyName = (String)
jsonPayload.get(KMSRESTConstants.NAME_FIELD);
+ String ivStr = (String) jsonPayload.get(KMSRESTConstants.IV_FIELD);
+ String encMaterialStr = (String)
jsonPayload.get(KMSRESTConstants.MATERIAL_FIELD);
+ checkNotNull(ivStr, KMSRESTConstants.IV_FIELD);
final byte[] iv = Base64.decodeBase64(ivStr);
- KMSUtil.checkNotNull(encMaterialStr,
- KMSRESTConstants.MATERIAL_FIELD);
+ checkNotNull(encMaterialStr, KMSRESTConstants.MATERIAL_FIELD);
final byte[] encMaterial = Base64.decodeBase64(encMaterialStr);
-
- KeyProvider.KeyVersion retKeyVersion = user.doAs(
- new PrivilegedExceptionAction<KeyVersion>() {
- @Override
- public KeyVersion run() throws Exception {
- return provider.decryptEncryptedKey(
- new KMSClientProvider.KMSEncryptedKeyVersion(keyName,
- versionName, iv, KeyProviderCryptoExtension.EEK,
- encMaterial)
- );
- }
+ Object retJSON;
+ if (eekOp.equals(KMSRESTConstants.EEK_DECRYPT)) {
+ KMSWebApp.getDecryptEEKCallsMeter().mark();
+ assertAccess(Type.DECRYPT_EEK, user, KMSOp.DECRYPT_EEK, keyName,
request.getRemoteAddr());
+ KeyProvider.KeyVersion retKeyVersion = user.doAs(new
PrivilegedExceptionAction<KeyVersion>() {
+ @Override
+ public KeyVersion run() throws Exception {
+ return provider.decryptEncryptedKey(new
KMSClientProvider.KMSEncryptedKeyVersion(
+ keyName, versionName,
iv,KeyProviderCryptoExtension.EEK,encMaterial));
}
- );
-
- retJSON = KMSServerJSONUtils.toJSON(retKeyVersion);
- kmsAudit.ok(user, KMSOp.DECRYPT_EEK, keyName, "");
- } else {
- throw new IllegalArgumentException("Wrong " + KMSRESTConstants.EEK_OP +
- " value, it must be " + KMSRESTConstants.EEK_GENERATE + " or " +
- KMSRESTConstants.EEK_DECRYPT);
- }
- KMSWebApp.getDecryptEEKCallsMeter().mark();
- return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON)
- .build();
+ });
+ retJSON = KMSUtil.toJSON(retKeyVersion);
+ kmsAudit.ok(user, KMSOp.DECRYPT_EEK, keyName, "");
+ } else if (eekOp.equals(KMSRESTConstants.EEK_REENCRYPT)) {
+ KMSWebApp.getReencryptEEKCallsMeter().mark();
+ assertAccess(Type.GENERATE_EEK, user, KMSOp.REENCRYPT_EEK, keyName);
+ EncryptedKeyVersion retEncryptedKeyVersion = user.doAs(new
PrivilegedExceptionAction<EncryptedKeyVersion>() {
+ @Override
+ public EncryptedKeyVersion run() throws Exception {
+ return provider.reencryptEncryptedKey(new
KMSClientProvider.KMSEncryptedKeyVersion(keyName,versionName, iv,
KeyProviderCryptoExtension.EEK,
+ encMaterial));
+ }
+ });
+ retJSON = KMSUtil.toJSON(retEncryptedKeyVersion);
+ kmsAudit.ok(user, KMSOp.REENCRYPT_EEK, keyName, "");
+ } else {
+ StringBuilder error;
+ error = new StringBuilder("IllegalArgumentException Wrong ");
+ error.append(KMSRESTConstants.EEK_OP);
+ error.append(" value, it must be ");
+ error.append(KMSRESTConstants.EEK_GENERATE);
+ error.append(" or ");
+ error.append(KMSRESTConstants.EEK_DECRYPT);
+ LOG.error(error.toString());
+ throw new IllegalArgumentException(error.toString());
+ }
+ LOG.info("Exiting handleEncryptedKeyOp method.");
+ return
Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON).build();
+ } catch (Exception e) {
+ LOG.debug("Exception in handleEncryptedKeyOp.", e);
+ throw e;
+}
}
@GET
@@ -480,23 +614,27 @@ public class KMS {
@Produces(MediaType.APPLICATION_JSON)
public Response getKeyVersions(@PathParam("name") final String name,
@Context HttpServletRequest request)
throws Exception {
- UserGroupInformation user = HttpUserGroupInformation.get();
- KMSUtil.checkNotEmpty(name, "name");
- KMSWebApp.getKeyCallsMeter().mark();
- assertAccess(Type.GET, user, KMSOp.GET_KEY_VERSIONS, name,
request.getRemoteAddr());
-
- List<KeyVersion> ret = user.doAs(
- new PrivilegedExceptionAction<List<KeyVersion>>() {
- @Override
- public List<KeyVersion> run() throws Exception {
- return provider.getKeyVersions(name);
- }
+ try {
+ LOG.info("Entering getKeyVersions method.");
+ UserGroupInformation user = HttpUserGroupInformation.get();
+ checkNotEmpty(name, "name");
+ KMSWebApp.getKeyCallsMeter().mark();
+ assertAccess(Type.GET, user, KMSOp.GET_KEY_VERSIONS, name,
request.getRemoteAddr());
+ LOG.debug("Getting key versions for key {}", name);
+ List<KeyVersion> ret = user.doAs(new
PrivilegedExceptionAction<List<KeyVersion>>() {
+ @Override
+ public List<KeyVersion> run() throws Exception {
+ return provider.getKeyVersions(name);
}
- );
-
- Object json = KMSServerJSONUtils.toJSON(ret);
- kmsAudit.ok(user, KMSOp.GET_KEY_VERSIONS, name, "");
- return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
- }
+ });
+ Object json = KMSServerJSONUtils.toJSON(ret);
+ kmsAudit.ok(user, KMSOp.GET_KEY_VERSIONS, name, "");
+ LOG.info("Exiting getKeyVersions method.");
+ return
Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build();
+ } catch (Exception e) {
+ LOG.debug("Exception in getKeyVersions.", e);
+ throw e;
+ }
+ }
}
diff --git
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
index 56d25d2..053d7e4 100644
--- a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,7 +93,7 @@ public class KMSAudit {
private static Set<KMS.KMSOp> AGGREGATE_OPS_WHITELIST = Sets.newHashSet(
KMS.KMSOp.GET_KEY_VERSION, KMS.KMSOp.GET_CURRENT_KEY,
- KMS.KMSOp.DECRYPT_EEK, KMS.KMSOp.GENERATE_EEK
+ KMS.KMSOp.DECRYPT_EEK, KMS.KMSOp.GENERATE_EEK, KMS.KMSOp.REENCRYPT_EEK
);
private Cache<String, AuditEvent> cache;
@@ -227,4 +228,8 @@ public class KMSAudit {
public void shutdown() {
executor.shutdownNow();
}
+ @VisibleForTesting
+ void evictCacheForTesting() {
+ cache.invalidateAll();
+ }
}
diff --git
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
index cdca8e1..b5be765 100644
---
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
+++
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
@@ -41,9 +41,9 @@ import java.io.IOException;
@Provider
@InterfaceAudience.Private
public class KMSExceptionsProvider implements ExceptionMapper<Exception> {
- private static final Logger LOG =
+ private static Logger LOG =
LoggerFactory.getLogger(KMSExceptionsProvider.class);
-
+ private final static Logger EXCEPTION_LOG = KMS.LOG;
private static final String ENTER = System.getProperty("line.separator");
protected Response createResponse(Response.Status status, Throwable ex) {
@@ -100,6 +100,9 @@ public class KMSExceptionsProvider implements
ExceptionMapper<Exception> {
KMSMDCFilter.getMethod(),
KMSMDCFilter.getURL(), getOneLineMessage(exception));
}
+ EXCEPTION_LOG.warn("User {} request {} {} caused exception.",
+ KMSMDCFilter.getUgi(), KMSMDCFilter.getMethod(),
+ KMSMDCFilter.getURL(), exception);
return createResponse(status, throwable);
}
diff --git
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
index 2b85276..59cc218 100644
---
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
+++
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
@@ -31,20 +31,21 @@ import java.io.InputStream;
import java.lang.annotation.Annotation;
import java.lang.reflect.Type;
import java.util.Map;
+import java.util.List;
@Provider
@Consumes(MediaType.APPLICATION_JSON)
@InterfaceAudience.Private
-public class KMSJSONReader implements MessageBodyReader<Map> {
+public class KMSJSONReader implements MessageBodyReader<Object> {
@Override
public boolean isReadable(Class<?> type, Type genericType,
Annotation[] annotations, MediaType mediaType) {
- return type.isAssignableFrom(Map.class);
+ return type.isAssignableFrom(Map.class) ||
type.isAssignableFrom(List.class);
}
@Override
- public Map readFrom(Class<Map> type, Type genericType,
+ public Object readFrom(Class<Object> type, Type genericType,
Annotation[] annotations, MediaType mediaType,
MultivaluedMap<String, String> httpHeaders, InputStream entityStream)
throws IOException, WebApplicationException {
diff --git
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
index 24af81b..f481486 100644
---
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
+++
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
@@ -17,64 +17,34 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
-import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.crypto.key.KeyProvider;
-import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.util.KMSUtil;
/**
* JSON utility methods for the KMS.
*/
@InterfaceAudience.Private
public class KMSServerJSONUtils {
- @SuppressWarnings("unchecked")
- public static Map toJSON(KeyProvider.KeyVersion keyVersion) {
- Map json = new LinkedHashMap();
- if (keyVersion != null) {
- json.put(KMSRESTConstants.NAME_FIELD,
- keyVersion.getName());
- json.put(KMSRESTConstants.VERSION_NAME_FIELD,
- keyVersion.getVersionName());
- json.put(KMSRESTConstants.MATERIAL_FIELD,
- Base64.encodeBase64URLSafeString(
- keyVersion.getMaterial()));
- }
- return json;
- }
@SuppressWarnings("unchecked")
public static List toJSON(List<KeyProvider.KeyVersion> keyVersions) {
List json = new ArrayList();
if (keyVersions != null) {
for (KeyProvider.KeyVersion version : keyVersions) {
- json.add(toJSON(version));
+ json.add(KMSUtil.toJSON(version));
}
}
return json;
}
@SuppressWarnings("unchecked")
- public static Map toJSON(EncryptedKeyVersion encryptedKeyVersion) {
- Map json = new LinkedHashMap();
- if (encryptedKeyVersion != null) {
- json.put(KMSRESTConstants.VERSION_NAME_FIELD,
- encryptedKeyVersion.getEncryptionKeyVersionName());
- json.put(KMSRESTConstants.IV_FIELD,
- Base64.encodeBase64URLSafeString(
- encryptedKeyVersion.getEncryptedKeyIv()));
- json.put(KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD,
- toJSON(encryptedKeyVersion.getEncryptedKeyVersion()));
- }
- return json;
- }
-
- @SuppressWarnings("unchecked")
public static Map toJSON(String keyName, KeyProvider.Metadata meta) {
Map json = new LinkedHashMap();
if (meta != null) {
diff --git
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
index 501ee30..ae24e5b 100755
--- a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
@@ -65,6 +65,10 @@ public class KMSWebApp implements ServletContextListener {
"generate_eek.calls.meter";
private static final String DECRYPT_EEK_METER = METRICS_PREFIX +
"decrypt_eek.calls.meter";
+ private static final String REENCRYPT_EEK_METER = METRICS_PREFIX +
+ "reencrypt_eek.calls.meter";
+ private static final String REENCRYPT_EEK_BATCH_METER = METRICS_PREFIX +
+ "reencrypt_eek_batch.calls.meter";
private static Logger LOG;
private static MetricRegistry metricRegistry;
@@ -77,6 +81,8 @@ public class KMSWebApp implements ServletContextListener {
private static Meter unauthorizedCallsMeter;
private static Meter unauthenticatedCallsMeter;
private static Meter decryptEEKCallsMeter;
+ private static Meter reencryptEEKCallsMeter;
+ private static Meter reencryptEEKBatchCallsMeter;
private static Meter generateEEKCallsMeter;
private static Meter invalidCallsMeter;
private static KMSAudit kmsAudit;
@@ -140,6 +146,10 @@ public class KMSWebApp implements ServletContextListener {
new Meter());
decryptEEKCallsMeter = metricRegistry.register(DECRYPT_EEK_METER,
new Meter());
+ reencryptEEKCallsMeter = metricRegistry.register(REENCRYPT_EEK_METER,
+ new Meter());
+ reencryptEEKBatchCallsMeter =
metricRegistry.register(REENCRYPT_EEK_BATCH_METER,
+ new Meter());
adminCallsMeter = metricRegistry.register(ADMIN_CALLS_METER, new
Meter());
keyCallsMeter = metricRegistry.register(KEY_CALLS_METER, new Meter());
invalidCallsMeter = metricRegistry.register(INVALID_CALLS_METER,
@@ -283,6 +293,13 @@ public class KMSWebApp implements ServletContextListener {
public static Meter getDecryptEEKCallsMeter() {
return decryptEEKCallsMeter;
}
+ public static Meter getReencryptEEKCallsMeter() {
+ return reencryptEEKCallsMeter;
+ }
+
+ public static Meter getReencryptEEKBatchCallsMeter() {
+ return reencryptEEKBatchCallsMeter;
+ }
public static Meter getUnauthorizedCallsMeter() {
return unauthorizedCallsMeter;
diff --git
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
index bd35a6b..fb9a261 100755
---
a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
+++
b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
@@ -288,6 +288,38 @@ public class KeyAuthorizationKeyProvider extends
KeyProviderCryptoExtension {
}
@Override
+ public EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion ekv)
+ throws IOException, GeneralSecurityException {
+ readLock.lock();
+ try {
+ verifyKeyVersionBelongsToKey(ekv);
+ doAccessCheck(ekv.getEncryptionKeyName(), KeyOpType.GENERATE_EEK);
+ return provider.reencryptEncryptedKey(ekv);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
+ throws IOException, GeneralSecurityException {
+ if (ekvs.isEmpty()) {
+ return;
+ }
+ readLock.lock();
+ try {
+ for (EncryptedKeyVersion ekv : ekvs) {
+ verifyKeyVersionBelongsToKey(ekv);
+ }
+ final String keyName = ekvs.get(0).getEncryptionKeyName();
+ doAccessCheck(keyName, KeyOpType.GENERATE_EEK);
+ provider.reencryptEncryptedKeys(ekvs);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
public List<KeyVersion> getKeyVersions(String name) throws IOException {
readLock.lock();
try {
diff --git
a/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
index 04daeee..ced8610 100644
---
a/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
+++
b/kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
@@ -29,7 +29,9 @@ import org.apache.log4j.PropertyConfigurator;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.Timeout;
import org.mockito.Mockito;
public class TestKMSAudit {
@@ -51,6 +53,9 @@ public class TestKMSAudit {
}
}
+ @Rule
+ public final Timeout testTimeout = new Timeout(180000);
+
@Before
public void setUp() {
originalOut = System.err;
@@ -61,7 +66,7 @@ public class TestKMSAudit {
PropertyConfigurator.configure(Thread.currentThread().
getContextClassLoader()
.getResourceAsStream("log4j-kmsaudit.properties"));
- this.kmsAudit = new KMSAudit(1000);
+ this.kmsAudit = new
KMSAudit(KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT);
}
@After
@@ -80,6 +85,7 @@ public class TestKMSAudit {
}
@Test
+ @SuppressWarnings("checkstyle:linelength")
public void testAggregation() throws Exception {
UserGroupInformation luser = Mockito.mock(UserGroupInformation.class);
Mockito.when(luser.getShortUserName()).thenReturn("luser");
@@ -88,12 +94,20 @@ public class TestKMSAudit {
kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
kmsAudit.ok(luser, KMSOp.DELETE_KEY, "k1", "testmsg");
kmsAudit.ok(luser, KMSOp.ROLL_NEW_VERSION, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.INVALIDATE_CACHE, "k1", "testmsg");
kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
- Thread.sleep(1500);
+ kmsAudit.evictCacheForTesting();
kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
- Thread.sleep(1500);
+ kmsAudit.evictCacheForTesting();
+ kmsAudit.ok(luser, KMSOp.REENCRYPT_EEK, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.REENCRYPT_EEK, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.REENCRYPT_EEK, "k1", "testmsg");
+ kmsAudit.evictCacheForTesting();
+ kmsAudit.ok(luser, KMSOp.REENCRYPT_EEK_BATCH, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.REENCRYPT_EEK_BATCH, "k1", "testmsg");
+ kmsAudit.evictCacheForTesting();
String out = getAndResetLogOutput();
System.out.println(out);
Assert.assertTrue(
@@ -102,34 +116,70 @@ public class TestKMSAudit {
// Not aggregated !!
+ "OK\\[op=DELETE_KEY, key=k1, user=luser\\] testmsg"
+ "OK\\[op=ROLL_NEW_VERSION, key=k1, user=luser\\] testmsg"
+ + "OK\\[op=INVALIDATE_CACHE, key=k1, user=luser\\] testmsg"
// Aggregated
+ "OK\\[op=DECRYPT_EEK, key=k1, user=luser, accessCount=6,
interval=[^m]{1,4}ms\\] testmsg"
- + "OK\\[op=DECRYPT_EEK, key=k1, user=luser, accessCount=1,
interval=[^m]{1,4}ms\\] testmsg"));
+ + "OK\\[op=DECRYPT_EEK, key=k1, user=luser, accessCount=1,
interval=[^m]{1,4}ms\\] testmsg"
+ + "OK\\[op=REENCRYPT_EEK, key=k1, user=luser, accessCount=1,
interval=[^m]{1,4}ms\\] testmsg"
+ + "OK\\[op=REENCRYPT_EEK, key=k1, user=luser, accessCount=3,
interval=[^m]{1,4}ms\\] testmsg"
+ + "OK\\[op=REENCRYPT_EEK_BATCH, key=k1, user=luser\\] testmsg"
+ + "OK\\[op=REENCRYPT_EEK_BATCH, key=k1, user=luser\\] testmsg"));
}
@Test
+ @SuppressWarnings("checkstyle:linelength")
public void testAggregationUnauth() throws Exception {
UserGroupInformation luser = Mockito.mock(UserGroupInformation.class);
Mockito.when(luser.getShortUserName()).thenReturn("luser");
kmsAudit.unauthorized(luser, KMSOp.GENERATE_EEK, "k2");
- Thread.sleep(1000);
+ kmsAudit.evictCacheForTesting();
kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
kmsAudit.unauthorized(luser, KMSOp.GENERATE_EEK, "k3");
+ // wait a bit so the UNAUTHORIZED-triggered cache invalidation happens.
+ Thread.sleep(1000);
kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
- Thread.sleep(2000);
+ kmsAudit.evictCacheForTesting();
String out = getAndResetLogOutput();
System.out.println(out);
+ // The UNAUTHORIZED will trigger cache invalidation, which then triggers
+ // the aggregated OK (accessCount=5). But the order of the UNAUTHORIZED and
+ // the aggregated OK is arbitrary - no correctness concerns, but flaky
here.
Assert.assertTrue(
out.matches(
"UNAUTHORIZED\\[op=GENERATE_EEK, key=k2, user=luser\\] "
+ "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=1,
interval=[^m]{1,4}ms\\] testmsg"
+ "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=5,
interval=[^m]{1,4}ms\\] testmsg"
+ "UNAUTHORIZED\\[op=GENERATE_EEK, key=k3, user=luser\\] "
+ + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=1,
interval=[^m]{1,4}ms\\] testmsg")
+ || out.matches("UNAUTHORIZED\\[op=GENERATE_EEK, key=k2,
user=luser\\] "
+ + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=1,
interval=[^m]{1,4}ms\\] testmsg"
+ + "UNAUTHORIZED\\[op=GENERATE_EEK, key=k3, user=luser\\] "
+ + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=5,
interval=[^m]{1,4}ms\\] testmsg"
+ "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=1,
interval=[^m]{1,4}ms\\] testmsg"));
}
+ @Test
+ public void testAuditLogFormat() throws Exception {
+ UserGroupInformation luser = Mockito.mock(UserGroupInformation.class);
+ Mockito.when(luser.getShortUserName()).thenReturn("luser");
+ kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k4", "testmsg");
+ kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "testmsg");
+ kmsAudit.evictCacheForTesting();
+ kmsAudit.unauthorized(luser, KMSOp.DECRYPT_EEK, "k4");
+ kmsAudit.error(luser, "method", "url", "testmsg");
+ kmsAudit.unauthenticated("remotehost", "method", "url", "testmsg");
+ String out = getAndResetLogOutput();
+ System.out.println(out);
+ Assert.assertTrue(out.matches(
+ "OK\\[op=GENERATE_EEK, key=k4, user=luser, accessCount=1,
interval=[^m]{1,4}ms\\] testmsg"
+ + "OK\\[op=GENERATE_EEK, user=luser\\] testmsg"
+ + "OK\\[op=GENERATE_EEK, key=k4, user=luser, accessCount=1,
interval=[^m]{1,4}ms\\] testmsg"
+ + "UNAUTHORIZED\\[op=DECRYPT_EEK, key=k4, user=luser\\] "
+ + "ERROR\\[user=luser\\] Method:'method' Exception:'testmsg'"
+ + "UNAUTHENTICATED RemoteHost:remotehost Method:method URL:url
ErrorMsg:'testmsg'"));
+ }
}
