http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java new file mode 100644 index 0000000..274b5f8 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java @@ -0,0 +1,341 @@ +package org.apache.hadoop.crypto.key; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.security.Key; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; +import java.util.ArrayList; +import java.util.Date; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.crypto.spec.SecretKeySpec; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProviderFactory; +import org.codehaus.jackson.map.ObjectMapper; +import org.apache.hadoop.fs.Path; +import org.apache.ranger.kms.dao.DaoManager; + +public class RangerKeyStoreProvider extends KeyProvider{ + + public static final String SCHEME_NAME = "dbks"; + public static final String KMS_CONFIG_DIR = "kms.config.dir"; + public static final String DBKS_SITE_XML = "dbks-site.xml"; + public static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + private static final String KEY_METADATA = "KeyMetadata"; + private final RangerKeyStore dbStore; + private char[] masterKey; + private boolean changed = false; + private final Map<String, Metadata> cache = new HashMap<String, Metadata>(); + private DaoManager daoManager; + + public RangerKeyStoreProvider(Configuration conf) throws Throwable { + super(conf); + conf = getDBKSConf(); + RangerKMSDB rangerKMSDB = new RangerKMSDB(conf); + daoManager = rangerKMSDB.getDaoManager(); + RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); + dbStore = new RangerKeyStore(daoManager); + String password = conf.get(ENCRYPTION_KEY); + rangerMasterKey.generateMasterKey(password); + //code to retrieve rangerMasterKey password + masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); + if(masterKey == null){ + // Master Key does not exists + throw new IOException("Ranger MasterKey does not exists"); + } + try { + loadKeys(masterKey); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + throw new IOException("Can't load Keys"); + }catch(CertificateException e){ + e.printStackTrace(); + throw new IOException("Can't load Keys"); + } + } + + public static Configuration getDBKSConf() { + return getConfiguration(true, DBKS_SITE_XML); + } + + static Configuration getConfiguration(boolean loadHadoopDefaults, + String ... resources) { + Configuration conf = new Configuration(loadHadoopDefaults); + String confDir = System.getProperty(KMS_CONFIG_DIR); + if (confDir != null) { + try { + Path confPath = new Path(confDir); + if (!confPath.isUriPathAbsolute()) { + throw new RuntimeException("System property '" + KMS_CONFIG_DIR + + "' must be an absolute path: " + confDir); + } + for (String resource : resources) { + conf.addResource(new URL("file://" + new Path(confDir, resource).toUri())); + } + } catch (MalformedURLException ex) { + ex.printStackTrace(); + throw new RuntimeException(ex); + } + } else { + for (String resource : resources) { + conf.addResource(resource); + } + } + return conf; + } + + private void loadKeys(char[] masterKey) throws NoSuchAlgorithmException, CertificateException, IOException { + dbStore.engineLoad(null, masterKey); + } + + @Override + public KeyVersion createKey(String name, byte[] material, Options options) + throws IOException { + if (dbStore.engineContainsAlias(name) || cache.containsKey(name)) { + throw new IOException("Key " + name + " already exists"); + } + Metadata meta = new Metadata(options.getCipher(), options.getBitLength(), + options.getDescription(), options.getAttributes(), new Date(), 1); + if (options.getBitLength() != 8 * material.length) { + throw new IOException("Wrong key length. Required " + + options.getBitLength() + ", but got " + (8 * material.length)); + } + cache.put(name, meta); + String versionName = buildVersionName(name, 0); + return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes()); + } + + KeyVersion innerSetKeyVersion(String name, String versionName, byte[] material, String cipher, int bitLength, String description, int version, Map<String, String> attributes) throws IOException { + try { + ObjectMapper om = new ObjectMapper(); + String attribute = om.writeValueAsString(attributes); + dbStore.engineSetKeyEntry(versionName, new SecretKeySpec(material, cipher), masterKey, cipher, bitLength, description, version, attribute); + } catch (KeyStoreException e) { + e.printStackTrace(); + throw new IOException("Can't store key " + versionName,e); + } + changed = true; + return new KeyVersion(name, versionName, material); + } + + @Override + public void deleteKey(String name) throws IOException { + Metadata meta = getMetadata(name); + if (meta == null) { + throw new IOException("Key " + name + " does not exist"); + } + for(int v=0; v < meta.getVersions(); ++v) { + String versionName = buildVersionName(name, v); + try { + if (dbStore.engineContainsAlias(versionName)) { + dbStore.engineDeleteEntry(versionName); + } + } catch (KeyStoreException e) { + throw new IOException("Problem removing " + versionName, e); + } + } + try { + if (dbStore.engineContainsAlias(name)) { + dbStore.engineDeleteEntry(name); + } + } catch (KeyStoreException e) { + e.printStackTrace(); + throw new IOException("Problem removing " + name + " from " + this, e); + } + cache.remove(name); + changed = true; + } + + @Override + public void flush() throws IOException { + try { + if (!changed) { + return; + } + // put all of the updates into the db + for(Map.Entry<String, Metadata> entry: cache.entrySet()) { + try { + Metadata metadata = entry.getValue(); + ObjectMapper om = new ObjectMapper(); + String attributes = om.writeValueAsString(metadata.getAttributes()); + dbStore.engineSetKeyEntry(entry.getKey(), new KeyMetadata(metadata), masterKey, metadata.getAlgorithm(), metadata.getBitLength(), metadata.getDescription(), metadata.getVersions(), attributes); + } catch (KeyStoreException e) { + e.printStackTrace(); + throw new IOException("Can't set metadata key " + entry.getKey(),e ); + } + } + try { + dbStore.engineStore(null, masterKey); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + throw new IOException("No such algorithm storing key", e); + } catch (CertificateException e) { + e.printStackTrace(); + throw new IOException("Certificate exception storing key", e); + } + changed = false; + }catch (IOException ioe) { + ioe.printStackTrace(); + throw ioe; + } + } + + @Override + public KeyVersion getKeyVersion(String versionName) throws IOException { + SecretKeySpec key = null; + try { + if (!dbStore.engineContainsAlias(versionName)) { + return null; + } + key = (SecretKeySpec) dbStore.engineGetKey(versionName, masterKey); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + throw new IOException("Can't get algorithm for key " + key, e); + } catch (UnrecoverableKeyException e) { + e.printStackTrace(); + throw new IOException("Can't recover key " + key, e); + } + return new KeyVersion(getBaseName(versionName), versionName, key.getEncoded()); + } + + @Override + public List<KeyVersion> getKeyVersions(String name) throws IOException { + List<KeyVersion> list = new ArrayList<KeyVersion>(); + Metadata km = getMetadata(name); + if (km != null) { + int latestVersion = km.getVersions(); + KeyVersion v = null; + String versionName = null; + for (int i = 0; i < latestVersion; i++) { + versionName = buildVersionName(name, i); + v = getKeyVersion(versionName); + if (v != null) { + list.add(v); + } + } + } + return list; + } + + @Override + public List<String> getKeys() throws IOException { + ArrayList<String> list = new ArrayList<String>(); + String alias = null; + Enumeration<String> e = dbStore.engineAliases(); + while (e.hasMoreElements()) { + alias = e.nextElement(); + // only include the metadata key names in the list of names + if (!alias.contains("@")) { + list.add(alias); + } + } + return list; + } + + @Override + public Metadata getMetadata(String name) throws IOException { + if (cache.containsKey(name)) { + return cache.get(name); + } + try { + if (!dbStore.engineContainsAlias(name)) { + return null; + } + Metadata meta = ((KeyMetadata) dbStore.engineGetKey(name, masterKey)).metadata; + cache.put(name, meta); + return meta; + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + throw new IOException("Can't get algorithm for " + name, e); + } catch (UnrecoverableKeyException e) { + e.printStackTrace(); + throw new IOException("Can't recover key for " + name, e); + } + } + + @Override + public KeyVersion rollNewVersion(String name, byte[] material)throws IOException { + Metadata meta = getMetadata(name); + if (meta == null) { + throw new IOException("Key " + name + " not found"); + } + if (meta.getBitLength() != 8 * material.length) { + throw new IOException("Wrong key length. Required " + meta.getBitLength() + ", but got " + (8 * material.length)); + } + int nextVersion = meta.addVersion(); + String versionName = buildVersionName(name, nextVersion); + return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes()); + } + + /** + * The factory to create JksProviders, which is used by the ServiceLoader. + */ + public static class Factory extends KeyProviderFactory { + @Override + public KeyProvider createProvider(URI providerName, + Configuration conf) throws IOException { + try { + if (SCHEME_NAME.equals(providerName.getScheme())) { + return new RangerKeyStoreProvider(conf); + } + } catch (Throwable e) { + e.printStackTrace(); + } + return null; + } + } + + /** + * An adapter between a KeyStore Key and our Metadata. This is used to store + * the metadata in a KeyStore even though isn't really a key. + */ + public static class KeyMetadata implements Key, Serializable { + private Metadata metadata; + private final static long serialVersionUID = 8405872419967874451L; + + private KeyMetadata(Metadata meta) { + this.metadata = meta; + } + + @Override + public String getAlgorithm() { + return metadata.getCipher(); + } + + @Override + public String getFormat() { + return KEY_METADATA; + } + + @Override + public byte[] getEncoded() { + return new byte[0]; + } + + private void writeObject(ObjectOutputStream out) throws IOException { + byte[] serialized = metadata.serialize(); + out.writeInt(serialized.length); + out.write(serialized); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + byte[] buf = new byte[in.readInt()]; + in.readFully(buf); + metadata = new Metadata(buf); + } + + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java new file mode 100644 index 0000000..6102bfc --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java @@ -0,0 +1,210 @@ +package org.apache.hadoop.crypto.key; + +import java.security.Key; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import javax.crypto.Cipher; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.PBEParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import org.apache.log4j.Logger; +import org.apache.ranger.kms.dao.DaoManager; +import org.apache.ranger.kms.dao.RangerMasterKeyDao; +import org.apache.ranger.entity.XXRangerMasterKey; + +import com.sun.org.apache.xml.internal.security.exceptions.Base64DecodingException; +import com.sun.org.apache.xml.internal.security.utils.Base64; + +public class RangerMasterKey { + + static final Logger logger = Logger.getLogger(RangerMasterKey.class); + + private static final String MK_CIPHER = "AES"; + private static final int MK_KeySize = 256; + private static final int SALT_SIZE = 8; + private static final String PBE_ALGO = "PBEWithMD5AndTripleDES"; + private static final String MD_ALGO = "MD5"; + + public static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + + private DaoManager daoManager; + + public RangerMasterKey() { + } + + public RangerMasterKey(DaoManager daoManager) { + this.daoManager = daoManager; + } + + /** + * To get Master Key + * @param password password to be used for decryption + * @return Decrypted Master Key + * @throws Throwable + */ + public String getMasterKey(String password) throws Throwable{ + logger.info("Getting Master Key"); + byte masterKeyByte[] = getEncryptedMK(); + if(masterKeyByte != null && masterKeyByte.length > 0){ + String masterKey = decryptMasterKey(masterKeyByte, password); + return masterKey; + }else{ + throw new Exception("No Master Key Found"); + } + } + + /** + * Generate the master key encrypt's it and save it in database + * @param password password to be used for encryption + * @return true if successfully created the master key + * false if master key generation was unsuccessful or already master key exists + * @throws Throwable + */ + public boolean generateMasterKey(String password) throws Throwable{ + logger.info("Generating Master Key"); + String encryptedMasterKey = encryptMasterKey(password); + String savedKey = saveEncryptedMK(encryptedMasterKey, daoManager); + if(savedKey != null && !savedKey.trim().equals("")){ + logger.debug("Master Key Created with id = "+savedKey); + return true; + } + return false; + } + + private String decryptMasterKey(byte masterKey[], String password) throws Throwable { + logger.debug("Decrypting Master Key"); + PBEKeySpec pbeKeyspec = getPBEParameterSpec(password) ; + byte[] masterKeyFromDBDecrypted = decryptKey(masterKey, pbeKeyspec) ; + SecretKey masterKeyFromDB = getMasterKeyFromBytes(masterKeyFromDBDecrypted) ; + return Base64.encode(masterKeyFromDB.getEncoded()); + } + + private byte[] getEncryptedMK() throws Base64DecodingException { + logger.debug("Retrieving Encrypted Master Key from database"); + try{ + if(daoManager != null){ + RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager); + List<XXRangerMasterKey> lstRangerMasterKey = rangerKMSDao.getAll(); + if(lstRangerMasterKey.size() < 1){ + throw new Exception("No Master Key exists"); + }else if(lstRangerMasterKey.size() > 1){ + throw new Exception("More than one Master Key exists"); + }else { + XXRangerMasterKey rangerMasterKey = rangerKMSDao.getById(lstRangerMasterKey.get(0).getId()); + String masterKeyStr = rangerMasterKey.getMasterKey(); + byte[] masterKeyFromDBEncrypted = Base64.decode(masterKeyStr) ; + return masterKeyFromDBEncrypted; + } + } + }catch(Exception e){ + e.printStackTrace(); + } + return null; + } + + private String saveEncryptedMK(String encryptedMasterKey, DaoManager daoManager) { + logger.debug("Saving Encrypted Master Key to database"); + XXRangerMasterKey xxRangerMasterKey = new XXRangerMasterKey(); + xxRangerMasterKey.setCipher(MK_CIPHER); + xxRangerMasterKey.setBitLength(MK_KeySize); + xxRangerMasterKey.setMasterKey(encryptedMasterKey); + try{ + if(daoManager != null){ + RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager); + Long l = rangerKMSDao.getAllCount(); + if(l < 1){ + XXRangerMasterKey rangerMasterKey = rangerKMSDao.create(xxRangerMasterKey); + return rangerMasterKey.getId().toString(); + } + } + }catch(Exception e){ + e.printStackTrace(); + } + return null; + } + + private String encryptMasterKey(String password) throws Throwable { + logger.debug("Encrypting Master Key"); + Key secretKey = generateMasterKey(); + PBEKeySpec pbeKeySpec = getPBEParameterSpec(password); + byte[] masterKeyToDB = encryptKey(secretKey.getEncoded(), pbeKeySpec); + String masterKey = Base64.encode(masterKeyToDB) ; + return masterKey; + } + + private Key generateMasterKey() throws NoSuchAlgorithmException{ + KeyGenerator kg = KeyGenerator.getInstance(MK_CIPHER); + kg.init(MK_KeySize); + return kg.generateKey(); + } + + private PBEKeySpec getPBEParameterSpec(String password) throws Throwable { + MessageDigest md = MessageDigest.getInstance(MD_ALGO) ; + byte[] saltGen = md.digest(password.getBytes()) ; + byte[] salt = new byte[SALT_SIZE] ; + System.arraycopy(saltGen, 0, salt, 0, SALT_SIZE); + int iteration = password.toCharArray().length + 1 ; + PBEKeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iteration) ; + return spec ; + } + private byte[] encryptKey(byte[] data, PBEKeySpec keyspec) throws Throwable { + SecretKey key = getPasswordKey(keyspec) ; + PBEParameterSpec paramSpec = new PBEParameterSpec(keyspec.getSalt(), keyspec.getIterationCount()) ; + Cipher c = Cipher.getInstance(key.getAlgorithm()) ; + c.init(Cipher.ENCRYPT_MODE, key,paramSpec); + byte[] encrypted = c.doFinal(data) ; + + return encrypted ; + } + private SecretKey getPasswordKey(PBEKeySpec keyspec) throws Throwable { + SecretKeyFactory factory = SecretKeyFactory.getInstance(PBE_ALGO) ; + SecretKey PbKey = factory.generateSecret(keyspec) ; + return PbKey ; + } + private byte[] decryptKey(byte[] encrypted, PBEKeySpec keyspec) throws Throwable { + SecretKey key = getPasswordKey(keyspec) ; + PBEParameterSpec paramSpec = new PBEParameterSpec(keyspec.getSalt(), keyspec.getIterationCount()) ; + Cipher c = Cipher.getInstance(key.getAlgorithm()) ; + c.init(Cipher.DECRYPT_MODE, key, paramSpec); + byte[] data = c.doFinal(encrypted) ; + return data ; + } + private SecretKey getMasterKeyFromBytes(byte[] keyData) throws Throwable { + SecretKeySpec sks = new SecretKeySpec(keyData, MK_CIPHER) ; + return sks ; + } + + public Map<String, String> getPropertiesWithPrefix(Properties props, String prefix) { + Map<String, String> prefixedProperties = new HashMap<String, String>(); + + if(props != null && prefix != null) { + for(String key : props.stringPropertyNames()) { + if(key == null) { + continue; + } + + String val = props.getProperty(key); + + if(key.startsWith(prefix)) { + key = key.substring(prefix.length()); + + if(key == null) { + continue; + } + + prefixedProperties.put(key, val); + } + } + } + + return prefixedProperties; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java new file mode 100644 index 0000000..64af2b6 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key.kms.server; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.security.NoSuchAlgorithmException; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ExecutionException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; +import org.apache.hadoop.crypto.key.kms.ValueQueue; +import org.apache.hadoop.crypto.key.kms.ValueQueue.SyncGenerationPolicy; + +/** + * A {@link KeyProviderCryptoExtension} that pre-generates and caches encrypted + * keys. + */ [email protected] +public class EagerKeyGeneratorKeyProviderCryptoExtension + extends KeyProviderCryptoExtension { + + private static final String KEY_CACHE_PREFIX = + "hadoop.security.kms.encrypted.key.cache."; + + public static final String KMS_KEY_CACHE_SIZE = + KEY_CACHE_PREFIX + "size"; + public static final int KMS_KEY_CACHE_SIZE_DEFAULT = 100; + + public static final String KMS_KEY_CACHE_LOW_WATERMARK = + KEY_CACHE_PREFIX + "low.watermark"; + public static final float KMS_KEY_CACHE_LOW_WATERMARK_DEFAULT = 0.30f; + + public static final String KMS_KEY_CACHE_EXPIRY_MS = + KEY_CACHE_PREFIX + "expiry"; + public static final int KMS_KEY_CACHE_EXPIRY_DEFAULT = 43200000; + + public static final String KMS_KEY_CACHE_NUM_REFILL_THREADS = + KEY_CACHE_PREFIX + "num.fill.threads"; + public static final int KMS_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT = 2; + + + private static class CryptoExtension + implements KeyProviderCryptoExtension.CryptoExtension { + + private class EncryptedQueueRefiller implements + ValueQueue.QueueRefiller<EncryptedKeyVersion> { + + @Override + public void fillQueueForKey(String keyName, + Queue<EncryptedKeyVersion> keyQueue, int numKeys) throws IOException { + List<EncryptedKeyVersion> retEdeks = + new LinkedList<EncryptedKeyVersion>(); + for (int i = 0; i < numKeys; i++) { + try { + retEdeks.add(keyProviderCryptoExtension.generateEncryptedKey( + keyName)); + } catch (GeneralSecurityException e) { + throw new IOException(e); + } + } + keyQueue.addAll(retEdeks); + } + } + + private KeyProviderCryptoExtension keyProviderCryptoExtension; + private final ValueQueue<EncryptedKeyVersion> encKeyVersionQueue; + + public CryptoExtension(Configuration conf, + KeyProviderCryptoExtension keyProviderCryptoExtension) { + this.keyProviderCryptoExtension = keyProviderCryptoExtension; + encKeyVersionQueue = + new ValueQueue<KeyProviderCryptoExtension.EncryptedKeyVersion>( + conf.getInt(KMS_KEY_CACHE_SIZE, + KMS_KEY_CACHE_SIZE_DEFAULT), + conf.getFloat(KMS_KEY_CACHE_LOW_WATERMARK, + KMS_KEY_CACHE_LOW_WATERMARK_DEFAULT), + conf.getInt(KMS_KEY_CACHE_EXPIRY_MS, + KMS_KEY_CACHE_EXPIRY_DEFAULT), + conf.getInt(KMS_KEY_CACHE_NUM_REFILL_THREADS, + KMS_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT), + SyncGenerationPolicy.LOW_WATERMARK, new EncryptedQueueRefiller() + ); + } + + @Override + public void warmUpEncryptedKeys(String... keyNames) throws + IOException { + try { + encKeyVersionQueue.initializeQueuesForKeys(keyNames); + } catch (ExecutionException e) { + throw new IOException(e); + } + } + + @Override + public void drain(String keyName) { + encKeyVersionQueue.drain(keyName); + } + + @Override + public EncryptedKeyVersion generateEncryptedKey(String encryptionKeyName) + throws IOException, GeneralSecurityException { + try { + return encKeyVersionQueue.getNext(encryptionKeyName); + } catch (ExecutionException e) { + throw new IOException(e); + } + } + + @Override + public KeyVersion + decryptEncryptedKey(EncryptedKeyVersion encryptedKeyVersion) + throws IOException, GeneralSecurityException { + return keyProviderCryptoExtension.decryptEncryptedKey( + encryptedKeyVersion); + } + } + + /** + * This class is a proxy for a <code>KeyProviderCryptoExtension</code> that + * decorates the underlying <code>CryptoExtension</code> with one that eagerly + * caches pre-generated Encrypted Keys using a <code>ValueQueue</code> + * + * @param conf Configuration object to load parameters from + * @param keyProviderCryptoExtension <code>KeyProviderCryptoExtension</code> + * to delegate calls to. + */ + public EagerKeyGeneratorKeyProviderCryptoExtension(Configuration conf, + KeyProviderCryptoExtension keyProviderCryptoExtension) { + super(keyProviderCryptoExtension, + new CryptoExtension(conf, keyProviderCryptoExtension)); + } + + @Override + public KeyVersion rollNewVersion(String name) + throws NoSuchAlgorithmException, IOException { + KeyVersion keyVersion = super.rollNewVersion(name); + getExtension().drain(name); + return keyVersion; + } + + @Override + public KeyVersion rollNewVersion(String name, byte[] material) + throws IOException { + KeyVersion keyVersion = super.rollNewVersion(name, material); + getExtension().drain(name); + return keyVersion; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java new file mode 100644 index 0000000..9b5b5a1 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java @@ -0,0 +1,482 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key.kms.server; + +import org.apache.commons.codec.binary.Base64; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; +import org.apache.hadoop.crypto.key.kms.KMSRESTConstants; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.crypto.key.kms.KMSClientProvider; +import org.apache.hadoop.crypto.key.kms.server.KMSACLsType.Type; +import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Class providing the REST bindings, via Jersey, for the KMS. + */ +@Path(KMSRESTConstants.SERVICE_VERSION) [email protected] +public class KMS { + + public static enum KMSOp { + CREATE_KEY, DELETE_KEY, ROLL_NEW_VERSION, + GET_KEYS, GET_KEYS_METADATA, + GET_KEY_VERSIONS, GET_METADATA, GET_KEY_VERSION, GET_CURRENT_KEY, + GENERATE_EEK, DECRYPT_EEK + } + + private KeyProviderCryptoExtension provider; + private KMSAudit kmsAudit; + + public KMS() throws Exception { + provider = KMSWebApp.getKeyProvider(); + kmsAudit= KMSWebApp.getKMSAudit(); + } + + private void assertAccess(Type aclType, UserGroupInformation ugi, + KMSOp operation) throws AccessControlException { + KMSWebApp.getACLs().assertAccess(aclType, ugi, operation, null); + } + + private void assertAccess(Type aclType, UserGroupInformation ugi, + KMSOp operation, String key) throws AccessControlException { + KMSWebApp.getACLs().assertAccess(aclType, ugi, operation, key); + } + + private static KeyProvider.KeyVersion removeKeyMaterial( + KeyProvider.KeyVersion keyVersion) { + return new KMSClientProvider.KMSKeyVersion(keyVersion.getName(), + keyVersion.getVersionName(), null); + } + + private static URI getKeyURI(String name) throws URISyntaxException { + return new URI(KMSRESTConstants.SERVICE_VERSION + "/" + + KMSRESTConstants.KEY_RESOURCE + "/" + name); + } + + @POST + @Path(KMSRESTConstants.KEYS_RESOURCE) + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @SuppressWarnings("unchecked") + public Response createKey(Map jsonKey) throws Exception { + KMSWebApp.getAdminCallsMeter().mark(); + UserGroupInformation user = HttpUserGroupInformation.get(); + final String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD); + KMSClientProvider.checkNotEmpty(name, KMSRESTConstants.NAME_FIELD); + assertAccess(Type.CREATE, user, KMSOp.CREATE_KEY, name); + String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD); + final String material = (String) jsonKey.get(KMSRESTConstants.MATERIAL_FIELD); + int length = (jsonKey.containsKey(KMSRESTConstants.LENGTH_FIELD)) + ? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0; + String description = (String) + jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD); + Map<String, String> attributes = (Map<String, String>) + jsonKey.get(KMSRESTConstants.ATTRIBUTES_FIELD); + if (material != null) { + assertAccess(Type.SET_KEY_MATERIAL, user, + KMSOp.CREATE_KEY, name); + } + final KeyProvider.Options options = new KeyProvider.Options( + KMSWebApp.getConfiguration()); + if (cipher != null) { + options.setCipher(cipher); + } + if (length != 0) { + options.setBitLength(length); + } + options.setDescription(description); + options.setAttributes(attributes); + + KeyProvider.KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction<KeyVersion>() { + @Override + public KeyVersion run() throws Exception { + KeyProvider.KeyVersion keyVersion = (material != null) + ? provider.createKey(name, Base64.decodeBase64(material), options) + : provider.createKey(name, options); + provider.flush(); + return keyVersion; + } + } + ); + + kmsAudit.ok(user, KMSOp.CREATE_KEY, name, "UserProvidedMaterial:" + + (material != null) + " Description:" + description); + + if (!KMSWebApp.getACLs().hasAccess(Type.GET, user)) { + keyVersion = removeKeyMaterial(keyVersion); + } + Map json = KMSServerJSONUtils.toJSON(keyVersion); + String requestURL = KMSMDCFilter.getURL(); + int idx = requestURL.lastIndexOf(KMSRESTConstants.KEYS_RESOURCE); + requestURL = requestURL.substring(0, idx); + String keyURL = requestURL + KMSRESTConstants.KEY_RESOURCE + "/" + name; + return Response.created(getKeyURI(name)).type(MediaType.APPLICATION_JSON). + header("Location", keyURL).entity(json).build(); + } + + @DELETE + @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") + public Response deleteKey(@PathParam("name") final String name) + throws Exception { + KMSWebApp.getAdminCallsMeter().mark(); + UserGroupInformation user = HttpUserGroupInformation.get(); + assertAccess(Type.DELETE, user, KMSOp.DELETE_KEY, name); + KMSClientProvider.checkNotEmpty(name, "name"); + + user.doAs(new PrivilegedExceptionAction<Void>() { + @Override + public Void run() throws Exception { + provider.deleteKey(name); + provider.flush(); + return null; + } + }); + + kmsAudit.ok(user, KMSOp.DELETE_KEY, name, ""); + + return Response.ok().build(); + } + + @POST + @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public Response rolloverKey(@PathParam("name") final String name, + Map jsonMaterial) throws Exception { + KMSWebApp.getAdminCallsMeter().mark(); + UserGroupInformation user = HttpUserGroupInformation.get(); + assertAccess(Type.ROLLOVER, user, KMSOp.ROLL_NEW_VERSION, name); + KMSClientProvider.checkNotEmpty(name, "name"); + final String material = (String) + jsonMaterial.get(KMSRESTConstants.MATERIAL_FIELD); + if (material != null) { + assertAccess(Type.SET_KEY_MATERIAL, user, + KMSOp.ROLL_NEW_VERSION, name); + } + + KeyProvider.KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction<KeyVersion>() { + @Override + public KeyVersion run() throws Exception { + KeyVersion keyVersion = (material != null) + ? provider.rollNewVersion(name, Base64.decodeBase64(material)) + : provider.rollNewVersion(name); + provider.flush(); + return keyVersion; + } + } + ); + + kmsAudit.ok(user, KMSOp.ROLL_NEW_VERSION, name, "UserProvidedMaterial:" + + (material != null) + " NewVersion:" + keyVersion.getVersionName()); + + if (!KMSWebApp.getACLs().hasAccess(Type.GET, user)) { + keyVersion = removeKeyMaterial(keyVersion); + } + Map json = KMSServerJSONUtils.toJSON(keyVersion); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + } + + @GET + @Path(KMSRESTConstants.KEYS_METADATA_RESOURCE) + @Produces(MediaType.APPLICATION_JSON) + public Response getKeysMetadata(@QueryParam(KMSRESTConstants.KEY) + List<String> keyNamesList) throws Exception { + KMSWebApp.getAdminCallsMeter().mark(); + UserGroupInformation user = HttpUserGroupInformation.get(); + final String[] keyNames = keyNamesList.toArray( + new String[keyNamesList.size()]); + assertAccess(Type.GET_METADATA, user, KMSOp.GET_KEYS_METADATA); + + KeyProvider.Metadata[] keysMeta = user.doAs( + new PrivilegedExceptionAction<KeyProvider.Metadata[]>() { + @Override + public KeyProvider.Metadata[] run() throws Exception { + return provider.getKeysMetadata(keyNames); + } + } + ); + + Object json = KMSServerJSONUtils.toJSON(keyNames, keysMeta); + kmsAudit.ok(user, KMSOp.GET_KEYS_METADATA, ""); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + } + + @GET + @Path(KMSRESTConstants.KEYS_NAMES_RESOURCE) + @Produces(MediaType.APPLICATION_JSON) + public Response getKeyNames() throws Exception { + KMSWebApp.getAdminCallsMeter().mark(); + UserGroupInformation user = HttpUserGroupInformation.get(); + assertAccess(Type.GET_KEYS, user, KMSOp.GET_KEYS); + + List<String> json = user.doAs( + new PrivilegedExceptionAction<List<String>>() { + @Override + public List<String> run() throws Exception { + return provider.getKeys(); + } + } + ); + + kmsAudit.ok(user, KMSOp.GET_KEYS, ""); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + } + + @GET + @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") + public Response getKey(@PathParam("name") String name) + throws Exception { + return getMetadata(name); + } + + @GET + @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + + KMSRESTConstants.METADATA_SUB_RESOURCE) + @Produces(MediaType.APPLICATION_JSON) + public Response getMetadata(@PathParam("name") final String name) + throws Exception { + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(name, "name"); + KMSWebApp.getAdminCallsMeter().mark(); + assertAccess(Type.GET_METADATA, user, KMSOp.GET_METADATA, name); + + KeyProvider.Metadata metadata = user.doAs( + new PrivilegedExceptionAction<KeyProvider.Metadata>() { + @Override + public KeyProvider.Metadata run() throws Exception { + return provider.getMetadata(name); + } + } + ); + + Object json = KMSServerJSONUtils.toJSON(name, metadata); + kmsAudit.ok(user, KMSOp.GET_METADATA, name, ""); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + } + + @GET + @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + + KMSRESTConstants.CURRENT_VERSION_SUB_RESOURCE) + @Produces(MediaType.APPLICATION_JSON) + public Response getCurrentVersion(@PathParam("name") final String name) + throws Exception { + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(name, "name"); + KMSWebApp.getKeyCallsMeter().mark(); + assertAccess(Type.GET, user, KMSOp.GET_CURRENT_KEY, name); + + KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction<KeyVersion>() { + @Override + public KeyVersion run() throws Exception { + return provider.getCurrentKey(name); + } + } + ); + + Object json = KMSServerJSONUtils.toJSON(keyVersion); + kmsAudit.ok(user, KMSOp.GET_CURRENT_KEY, name, ""); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + } + + @GET + @Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}") + @Produces(MediaType.APPLICATION_JSON) + public Response getKeyVersion( + @PathParam("versionName") final String versionName) throws Exception { + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(versionName, "versionName"); + KMSWebApp.getKeyCallsMeter().mark(); + assertAccess(Type.GET, user, KMSOp.GET_KEY_VERSION); + + KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction<KeyVersion>() { + @Override + public KeyVersion run() throws Exception { + return provider.getKeyVersion(versionName); + } + } + ); + + if (keyVersion != null) { + kmsAudit.ok(user, KMSOp.GET_KEY_VERSION, keyVersion.getName(), ""); + } + Object json = KMSServerJSONUtils.toJSON(keyVersion); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @GET + @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + + KMSRESTConstants.EEK_SUB_RESOURCE) + @Produces(MediaType.APPLICATION_JSON) + public Response generateEncryptedKeys( + @PathParam("name") final String name, + @QueryParam(KMSRESTConstants.EEK_OP) String edekOp, + @DefaultValue("1") + @QueryParam(KMSRESTConstants.EEK_NUM_KEYS) final int numKeys) + throws Exception { + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(name, "name"); + KMSClientProvider.checkNotNull(edekOp, "eekOp"); + + Object retJSON; + if (edekOp.equals(KMSRESTConstants.EEK_GENERATE)) { + assertAccess(Type.GENERATE_EEK, user, KMSOp.GENERATE_EEK, name); + + final List<EncryptedKeyVersion> retEdeks = + new LinkedList<EncryptedKeyVersion>(); + try { + + user.doAs( + new PrivilegedExceptionAction<Void>() { + @Override + public Void run() throws Exception { + for (int i = 0; i < numKeys; i++) { + retEdeks.add(provider.generateEncryptedKey(name)); + } + return null; + } + } + ); + + } catch (Exception e) { + throw new IOException(e); + } + kmsAudit.ok(user, KMSOp.GENERATE_EEK, name, ""); + retJSON = new ArrayList(); + for (EncryptedKeyVersion edek : retEdeks) { + ((ArrayList)retJSON).add(KMSServerJSONUtils.toJSON(edek)); + } + } else { + throw new IllegalArgumentException("Wrong " + KMSRESTConstants.EEK_OP + + " value, it must be " + KMSRESTConstants.EEK_GENERATE + " or " + + KMSRESTConstants.EEK_DECRYPT); + } + KMSWebApp.getGenerateEEKCallsMeter().mark(); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON) + .build(); + } + + @SuppressWarnings("rawtypes") + @POST + @Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}/" + + KMSRESTConstants.EEK_SUB_RESOURCE) + @Produces(MediaType.APPLICATION_JSON) + public Response decryptEncryptedKey( + @PathParam("versionName") final String versionName, + @QueryParam(KMSRESTConstants.EEK_OP) String eekOp, + Map jsonPayload) + throws Exception { + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(versionName, "versionName"); + KMSClientProvider.checkNotNull(eekOp, "eekOp"); + + final String keyName = (String) jsonPayload.get( + KMSRESTConstants.NAME_FIELD); + String ivStr = (String) jsonPayload.get(KMSRESTConstants.IV_FIELD); + String encMaterialStr = + (String) jsonPayload.get(KMSRESTConstants.MATERIAL_FIELD); + Object retJSON; + if (eekOp.equals(KMSRESTConstants.EEK_DECRYPT)) { + assertAccess(Type.DECRYPT_EEK, user, KMSOp.DECRYPT_EEK, keyName); + KMSClientProvider.checkNotNull(ivStr, KMSRESTConstants.IV_FIELD); + final byte[] iv = Base64.decodeBase64(ivStr); + KMSClientProvider.checkNotNull(encMaterialStr, + KMSRESTConstants.MATERIAL_FIELD); + final byte[] encMaterial = Base64.decodeBase64(encMaterialStr); + + KeyProvider.KeyVersion retKeyVersion = user.doAs( + new PrivilegedExceptionAction<KeyVersion>() { + @Override + public KeyVersion run() throws Exception { + return provider.decryptEncryptedKey( + new KMSClientProvider.KMSEncryptedKeyVersion(keyName, + versionName, iv, KeyProviderCryptoExtension.EEK, + encMaterial) + ); + } + } + ); + + retJSON = KMSServerJSONUtils.toJSON(retKeyVersion); + kmsAudit.ok(user, KMSOp.DECRYPT_EEK, keyName, ""); + } else { + throw new IllegalArgumentException("Wrong " + KMSRESTConstants.EEK_OP + + " value, it must be " + KMSRESTConstants.EEK_GENERATE + " or " + + KMSRESTConstants.EEK_DECRYPT); + } + KMSWebApp.getDecryptEEKCallsMeter().mark(); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON) + .build(); + } + + @GET + @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + + KMSRESTConstants.VERSIONS_SUB_RESOURCE) + @Produces(MediaType.APPLICATION_JSON) + public Response getKeyVersions(@PathParam("name") final String name) + throws Exception { + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(name, "name"); + KMSWebApp.getKeyCallsMeter().mark(); + assertAccess(Type.GET, user, KMSOp.GET_KEY_VERSIONS, name); + + List<KeyVersion> ret = user.doAs( + new PrivilegedExceptionAction<List<KeyVersion>>() { + @Override + public List<KeyVersion> run() throws Exception { + return provider.getKeyVersions(name); + } + } + ); + + Object json = KMSServerJSONUtils.toJSON(ret); + kmsAudit.ok(user, KMSOp.GET_KEY_VERSIONS, name, ""); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java new file mode 100644 index 0000000..f2298c0 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key.kms.server; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp; +import org.apache.hadoop.crypto.key.kms.server.KMSACLsType.Type; +import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyACLs; +import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Provides access to the <code>AccessControlList</code>s used by KMS, + * hot-reloading them if the <code>kms-acls.xml</code> file where the ACLs + * are defined has been updated. + */ [email protected] +public class KMSACLs implements Runnable, KeyACLs { + private static final Logger LOG = LoggerFactory.getLogger(KMSACLs.class); + + private static final String UNAUTHORIZED_MSG_WITH_KEY = + "User:%s not allowed to do '%s' on '%s'"; + + private static final String UNAUTHORIZED_MSG_WITHOUT_KEY = + "User:%s not allowed to do '%s'"; + + public static final String ACL_DEFAULT = AccessControlList.WILDCARD_ACL_VALUE; + + public static final int RELOADER_SLEEP_MILLIS = 1000; + + private volatile Map<Type, AccessControlList> acls; + private volatile Map<Type, AccessControlList> blacklistedAcls; + private volatile Map<String, HashMap<KeyOpType, AccessControlList>> keyAcls; + private final Map<KeyOpType, AccessControlList> defaultKeyAcls = + new HashMap<KeyOpType, AccessControlList>(); + private ScheduledExecutorService executorService; + private long lastReload; + + KMSACLs(Configuration conf) { + if (conf == null) { + conf = loadACLs(); + } + setKMSACLs(conf); + setKeyACLs(conf); + } + + public KMSACLs() { + this(null); + } + + private void setKMSACLs(Configuration conf) { + Map<Type, AccessControlList> tempAcls = new HashMap<Type, AccessControlList>(); + Map<Type, AccessControlList> tempBlacklist = new HashMap<Type, AccessControlList>(); + for (Type aclType : Type.values()) { + String aclStr = conf.get(aclType.getAclConfigKey(), ACL_DEFAULT); + tempAcls.put(aclType, new AccessControlList(aclStr)); + String blacklistStr = conf.get(aclType.getBlacklistConfigKey()); + if (blacklistStr != null) { + // Only add if blacklist is present + tempBlacklist.put(aclType, new AccessControlList(blacklistStr)); + LOG.info("'{}' Blacklist '{}'", aclType, blacklistStr); + } + LOG.info("'{}' ACL '{}'", aclType, aclStr); + } + acls = tempAcls; + blacklistedAcls = tempBlacklist; + } + + private void setKeyACLs(Configuration conf) { + Map<String, HashMap<KeyOpType, AccessControlList>> tempKeyAcls = + new HashMap<String, HashMap<KeyOpType,AccessControlList>>(); + Map<String, String> allKeyACLS = + conf.getValByRegex(Pattern.quote(KMSConfiguration.KEY_ACL_PREFIX)); + for (Map.Entry<String, String> keyAcl : allKeyACLS.entrySet()) { + String k = keyAcl.getKey(); + // this should be of type "key.acl.<KEY_NAME>.<OP_TYPE>" + int keyNameStarts = KMSConfiguration.KEY_ACL_PREFIX.length(); + int keyNameEnds = k.lastIndexOf("."); + if (keyNameStarts >= keyNameEnds) { + LOG.warn("Invalid key name '{}'", k); + } else { + String aclStr = keyAcl.getValue(); + String keyName = k.substring(keyNameStarts, keyNameEnds); + String keyOp = k.substring(keyNameEnds + 1); + KeyOpType aclType = null; + try { + aclType = KeyOpType.valueOf(keyOp); + } catch (IllegalArgumentException e) { + LOG.warn("Invalid key Operation '{}'", keyOp); + } + if (aclType != null) { + // On the assumption this will be single threaded.. else we need to + // ConcurrentHashMap + HashMap<KeyOpType,AccessControlList> aclMap = + tempKeyAcls.get(keyName); + if (aclMap == null) { + aclMap = new HashMap<KeyOpType, AccessControlList>(); + tempKeyAcls.put(keyName, aclMap); + } + aclMap.put(aclType, new AccessControlList(aclStr)); + LOG.info("KEY_NAME '{}' KEY_OP '{}' ACL '{}'", + keyName, aclType, aclStr); + } + } + } + + keyAcls = tempKeyAcls; + for (KeyOpType keyOp : KeyOpType.values()) { + if (!defaultKeyAcls.containsKey(keyOp)) { + String confKey = KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + keyOp; + String aclStr = conf.get(confKey); + if (aclStr != null) { + if (aclStr.equals("*")) { + LOG.info("Default Key ACL for KEY_OP '{}' is set to '*'", keyOp); + } + defaultKeyAcls.put(keyOp, new AccessControlList(aclStr)); + } + } + } + } + + @Override + public void run() { + try { + if (KMSConfiguration.isACLsFileNewer(lastReload)) { + setKMSACLs(loadACLs()); + setKeyACLs(loadACLs()); + } + } catch (Exception ex) { + LOG.warn( + String.format("Could not reload ACLs file: '%s'", ex.toString()), ex); + } + } + + public synchronized void startReloader() { + if (executorService == null) { + executorService = Executors.newScheduledThreadPool(1); + executorService.scheduleAtFixedRate(this, RELOADER_SLEEP_MILLIS, + RELOADER_SLEEP_MILLIS, TimeUnit.MILLISECONDS); + } + } + + public synchronized void stopReloader() { + if (executorService != null) { + executorService.shutdownNow(); + executorService = null; + } + } + + private Configuration loadACLs() { + LOG.debug("Loading ACLs file"); + lastReload = System.currentTimeMillis(); + Configuration conf = KMSConfiguration.getACLsConf(); + // triggering the resource loading. + conf.get(Type.CREATE.getAclConfigKey()); + return conf; + } + + /** + * First Check if user is in ACL for the KMS operation, if yes, then + * return true if user is not present in any configured blacklist for + * the operation + * @param type KMS Operation + * @param ugi UserGroupInformation of user + * @return true is user has access + */ + @Override + public boolean hasAccess(Type type, UserGroupInformation ugi) { + boolean access = acls.get(type).isUserAllowed(ugi); + if (access) { + AccessControlList blacklist = blacklistedAcls.get(type); + access = (blacklist == null) || !blacklist.isUserInList(ugi); + } + return access; + } + + @Override + public void assertAccess(Type aclType, + UserGroupInformation ugi, KMSOp operation, String key) + throws AccessControlException { + if (!KMSWebApp.getACLs().hasAccess(aclType, ugi)) { + KMSWebApp.getUnauthorizedCallsMeter().mark(); + KMSWebApp.getKMSAudit().unauthorized(ugi, operation, key); + throw new AuthorizationException(String.format( + (key != null) ? UNAUTHORIZED_MSG_WITH_KEY + : UNAUTHORIZED_MSG_WITHOUT_KEY, + ugi.getShortUserName(), operation, key)); + } + } + + @Override + public boolean hasAccessToKey(String keyName, UserGroupInformation ugi, + KeyOpType opType) { + Map<KeyOpType, AccessControlList> keyAcl = keyAcls.get(keyName); + if (keyAcl == null) { + // Get KeyAcl map of DEFAULT KEY. + keyAcl = defaultKeyAcls; + } + // If No key acl defined for this key, check to see if + // there are key defaults configured for this operation + AccessControlList acl = keyAcl.get(opType); + if (acl == null) { + // If no acl is specified for this operation, + // deny access + return false; + } else { + return acl.isUserAllowed(ugi); + } + } + + @Override + public boolean isACLPresent(String keyName, KeyOpType opType) { + return (keyAcls.containsKey(keyName) || defaultKeyAcls.containsKey(opType)); + } + + @Override + public void startACLReloader() { + this.startReloader(); + } + + @Override + public void stopACLReloader() { + this.stopReloader(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java new file mode 100644 index 0000000..0fd3091 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLsType.java @@ -0,0 +1,17 @@ +package org.apache.hadoop.crypto.key.kms.server; + +public class KMSACLsType { + + public enum Type { + CREATE, DELETE, ROLLOVER, GET, GET_KEYS, GET_METADATA, + SET_KEY_MATERIAL, GENERATE_EEK, DECRYPT_EEK; + + public String getAclConfigKey() { + return KMSConfiguration.CONFIG_PREFIX + "acl." + this.toString(); + } + + public String getBlacklistConfigKey() { + return KMSConfiguration.CONFIG_PREFIX + "blacklist." + this.toString(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java new file mode 100644 index 0000000..7ff76e5 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java @@ -0,0 +1,230 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key.kms.server; + +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Joiner; +import com.google.common.base.Strings; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Provides convenience methods for audit logging consistently the different + * types of events. + */ +public class KMSAudit { + + private static class AuditEvent { + private final AtomicLong accessCount = new AtomicLong(-1); + private final String keyName; + private final String user; + private final KMS.KMSOp op; + private final String extraMsg; + private final long startTime = System.currentTimeMillis(); + + private AuditEvent(String keyName, String user, KMS.KMSOp op, String msg) { + this.keyName = keyName; + this.user = user; + this.op = op; + this.extraMsg = msg; + } + + public String getExtraMsg() { + return extraMsg; + } + + public AtomicLong getAccessCount() { + return accessCount; + } + + public String getKeyName() { + return keyName; + } + + public String getUser() { + return user; + } + + public KMS.KMSOp getOp() { + return op; + } + + public long getStartTime() { + return startTime; + } + } + + public static enum OpStatus { + OK, UNAUTHORIZED, UNAUTHENTICATED, ERROR; + } + + private static Set<KMS.KMSOp> AGGREGATE_OPS_WHITELIST = Sets.newHashSet( + KMS.KMSOp.GET_KEY_VERSION, KMS.KMSOp.GET_CURRENT_KEY, + KMS.KMSOp.DECRYPT_EEK, KMS.KMSOp.GENERATE_EEK + ); + + private Cache<String, AuditEvent> cache; + + private ScheduledExecutorService executor; + + public static final String KMS_LOGGER_NAME = "kms-audit"; + + private static Logger AUDIT_LOG = LoggerFactory.getLogger(KMS_LOGGER_NAME); + + /** + * Create a new KMSAudit. + * + * @param windowMs Duplicate events within the aggregation window are quashed + * to reduce log traffic. A single message for aggregated + * events is printed at the end of the window, along with a + * count of the number of aggregated events. + */ + KMSAudit(long windowMs) { + cache = CacheBuilder.newBuilder() + .expireAfterWrite(windowMs, TimeUnit.MILLISECONDS) + .removalListener( + new RemovalListener<String, AuditEvent>() { + @Override + public void onRemoval( + RemovalNotification<String, AuditEvent> entry) { + AuditEvent event = entry.getValue(); + if (event.getAccessCount().get() > 0) { + KMSAudit.this.logEvent(event); + event.getAccessCount().set(0); + KMSAudit.this.cache.put(entry.getKey(), event); + } + } + }).build(); + executor = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder() + .setDaemon(true).setNameFormat(KMS_LOGGER_NAME + "_thread").build()); + executor.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + cache.cleanUp(); + } + }, windowMs / 10, windowMs / 10, TimeUnit.MILLISECONDS); + } + + private void logEvent(AuditEvent event) { + AUDIT_LOG.info( + "OK[op={}, key={}, user={}, accessCount={}, interval={}ms] {}", + event.getOp(), event.getKeyName(), event.getUser(), + event.getAccessCount().get(), + (System.currentTimeMillis() - event.getStartTime()), + event.getExtraMsg()); + } + + private void op(OpStatus opStatus, final KMS.KMSOp op, final String user, + final String key, final String extraMsg) { + if (!Strings.isNullOrEmpty(user) && !Strings.isNullOrEmpty(key) + && (op != null) + && AGGREGATE_OPS_WHITELIST.contains(op)) { + String cacheKey = createCacheKey(user, key, op); + if (opStatus == OpStatus.UNAUTHORIZED) { + cache.invalidate(cacheKey); + AUDIT_LOG.info("UNAUTHORIZED[op={}, key={}, user={}] {}", op, key, user, + extraMsg); + } else { + try { + AuditEvent event = cache.get(cacheKey, new Callable<AuditEvent>() { + @Override + public AuditEvent call() throws Exception { + return new AuditEvent(key, user, op, extraMsg); + } + }); + // Log first access (initialized as -1 so + // incrementAndGet() == 0 implies first access) + if (event.getAccessCount().incrementAndGet() == 0) { + event.getAccessCount().incrementAndGet(); + logEvent(event); + } + } catch (ExecutionException ex) { + throw new RuntimeException(ex); + } + } + } else { + List<String> kvs = new LinkedList<String>(); + if (op != null) { + kvs.add("op=" + op); + } + if (!Strings.isNullOrEmpty(key)) { + kvs.add("key=" + key); + } + if (!Strings.isNullOrEmpty(user)) { + kvs.add("user=" + user); + } + if (kvs.size() == 0) { + AUDIT_LOG.info("{} {}", opStatus.toString(), extraMsg); + } else { + String join = Joiner.on(", ").join(kvs); + AUDIT_LOG.info("{}[{}] {}", opStatus.toString(), join, extraMsg); + } + } + } + + public void ok(UserGroupInformation user, KMS.KMSOp op, String key, + String extraMsg) { + op(OpStatus.OK, op, user.getShortUserName(), key, extraMsg); + } + + public void ok(UserGroupInformation user, KMS.KMSOp op, String extraMsg) { + op(OpStatus.OK, op, user.getShortUserName(), null, extraMsg); + } + + public void unauthorized(UserGroupInformation user, KMS.KMSOp op, String key) { + op(OpStatus.UNAUTHORIZED, op, user.getShortUserName(), key, ""); + } + + public void error(UserGroupInformation user, String method, String url, + String extraMsg) { + op(OpStatus.ERROR, null, user.getShortUserName(), null, "Method:'" + method + + "' Exception:'" + extraMsg + "'"); + } + + public void unauthenticated(String remoteHost, String method, + String url, String extraMsg) { + op(OpStatus.UNAUTHENTICATED, null, null, null, "RemoteHost:" + + remoteHost + " Method:" + method + + " URL:" + url + " ErrorMsg:'" + extraMsg + "'"); + } + + private static String createCacheKey(String user, String key, KMS.KMSOp op) { + return user + "#" + key + "#" + op; + } + + public void shutdown() { + executor.shutdownNow(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java new file mode 100644 index 0000000..79652f3 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key.kms.server; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.kms.KMSClientProvider; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler; + +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletResponseWrapper; +import java.io.IOException; +import java.util.Map; +import java.util.Properties; + +/** + * Authentication filter that takes the configuration from the KMS configuration + * file. + */ [email protected] +public class KMSAuthenticationFilter + extends DelegationTokenAuthenticationFilter { + + public static final String CONFIG_PREFIX = KMSConfiguration.CONFIG_PREFIX + + "authentication."; + + @Override + protected Properties getConfiguration(String configPrefix, + FilterConfig filterConfig) { + Properties props = new Properties(); + Configuration conf = KMSWebApp.getConfiguration(); + for (Map.Entry<String, String> entry : conf) { + String name = entry.getKey(); + if (name.startsWith(CONFIG_PREFIX)) { + String value = conf.get(name); + name = name.substring(CONFIG_PREFIX.length()); + props.setProperty(name, value); + } + } + String authType = props.getProperty(AUTH_TYPE); + if (authType.equals(PseudoAuthenticationHandler.TYPE)) { + props.setProperty(AUTH_TYPE, + PseudoDelegationTokenAuthenticationHandler.class.getName()); + } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) { + props.setProperty(AUTH_TYPE, + KerberosDelegationTokenAuthenticationHandler.class.getName()); + } + props.setProperty(DelegationTokenAuthenticationHandler.TOKEN_KIND, + KMSClientProvider.TOKEN_KIND); + return props; + } + + protected Configuration getProxyuserConfiguration(FilterConfig filterConfig) { + Map<String, String> proxyuserConf = KMSWebApp.getConfiguration(). + getValByRegex("hadoop\\.kms\\.proxyuser\\."); + Configuration conf = new Configuration(false); + for (Map.Entry<String, String> entry : proxyuserConf.entrySet()) { + conf.set(entry.getKey().substring("hadoop.kms.".length()), + entry.getValue()); + } + return conf; + } + + private static class KMSResponse extends HttpServletResponseWrapper { + public int statusCode; + public String msg; + + public KMSResponse(ServletResponse response) { + super((HttpServletResponse)response); + } + + @Override + public void setStatus(int sc) { + statusCode = sc; + super.setStatus(sc); + } + + @Override + public void sendError(int sc, String msg) throws IOException { + statusCode = sc; + this.msg = msg; + super.sendError(sc, msg); + } + + @Override + public void sendError(int sc) throws IOException { + statusCode = sc; + super.sendError(sc); + } + + @Override + public void setStatus(int sc, String sm) { + statusCode = sc; + msg = sm; + super.setStatus(sc, sm); + } + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain filterChain) throws IOException, ServletException { + KMSResponse kmsResponse = new KMSResponse(response); + super.doFilter(request, kmsResponse, filterChain); + + if (kmsResponse.statusCode != HttpServletResponse.SC_OK && + kmsResponse.statusCode != HttpServletResponse.SC_CREATED && + kmsResponse.statusCode != HttpServletResponse.SC_UNAUTHORIZED) { + KMSWebApp.getInvalidCallsMeter().mark(); + } + + // HttpServletResponse.SC_UNAUTHORIZED is because the request does not + // belong to an authenticated user. + if (kmsResponse.statusCode == HttpServletResponse.SC_UNAUTHORIZED) { + KMSWebApp.getUnauthenticatedCallsMeter().mark(); + String method = ((HttpServletRequest) request).getMethod(); + StringBuffer requestURL = ((HttpServletRequest) request).getRequestURL(); + String queryString = ((HttpServletRequest) request).getQueryString(); + if (queryString != null) { + requestURL.append("?").append(queryString); + } + + KMSWebApp.getKMSAudit().unauthenticated( + request.getRemoteHost(), method, requestURL.toString(), + kmsResponse.msg); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java new file mode 100644 index 0000000..d6c77ea --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key.kms.server; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; + +/** + * Utility class to load KMS configuration files. + */ [email protected] +public class KMSConfiguration { + + public static final String KMS_CONFIG_DIR = "kms.config.dir"; + public static final String KMS_SITE_XML = "kms-site.xml"; + public static final String KMS_ACLS_XML = "kms-acls.xml"; + + public static final String CONFIG_PREFIX = "hadoop.kms."; + + public static final String KEY_ACL_PREFIX = "key.acl."; + public static final String DEFAULT_KEY_ACL_PREFIX = "default.key.acl."; + + // Property to set the backing KeyProvider + public static final String KEY_PROVIDER_URI = CONFIG_PREFIX + + "key.provider.uri"; + + // Property to Enable/Disable Caching + public static final String KEY_CACHE_ENABLE = CONFIG_PREFIX + + "cache.enable"; + // Timeout for the Key and Metadata Cache + public static final String KEY_CACHE_TIMEOUT_KEY = CONFIG_PREFIX + + "cache.timeout.ms"; + // TImeout for the Current Key cache + public static final String CURR_KEY_CACHE_TIMEOUT_KEY = CONFIG_PREFIX + + "current.key.cache.timeout.ms"; + // Delay for Audit logs that need aggregation + public static final String KMS_AUDIT_AGGREGATION_WINDOW = CONFIG_PREFIX + + "audit.aggregation.window.ms"; + + //for authorizer + public static final String KMS_SECURITY_AUTHORIZER = CONFIG_PREFIX + "security.authorization.manager"; + + public static final boolean KEY_CACHE_ENABLE_DEFAULT = true; + // 10 mins + public static final long KEY_CACHE_TIMEOUT_DEFAULT = 10 * 60 * 1000; + // 30 secs + public static final long CURR_KEY_CACHE_TIMEOUT_DEFAULT = 30 * 1000; + // 10 secs + public static final long KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT = 10000; + + // Property to Enable/Disable per Key authorization + public static final String KEY_AUTHORIZATION_ENABLE = CONFIG_PREFIX + + "key.authorization.enable"; + + public static final boolean KEY_AUTHORIZATION_ENABLE_DEFAULT = true; + + static Configuration getConfiguration(boolean loadHadoopDefaults, + String ... resources) { + Configuration conf = new Configuration(loadHadoopDefaults); + String confDir = System.getProperty(KMS_CONFIG_DIR); + if (confDir != null) { + try { + Path confPath = new Path(confDir); + if (!confPath.isUriPathAbsolute()) { + throw new RuntimeException("System property '" + KMS_CONFIG_DIR + + "' must be an absolute path: " + confDir); + } + for (String resource : resources) { + conf.addResource(new URL("file://" + new Path(confDir, resource).toUri())); + } + } catch (MalformedURLException ex) { + throw new RuntimeException(ex); + } + } else { + for (String resource : resources) { + conf.addResource(resource); + } + } + return conf; + } + + public static Configuration getKMSConf() { + return getConfiguration(true, "core-site.xml", KMS_SITE_XML); + } + + public static Configuration getACLsConf() { + return getConfiguration(false, KMS_ACLS_XML); + } + + public static boolean isACLsFileNewer(long time) { + boolean newer = false; + String confDir = System.getProperty(KMS_CONFIG_DIR); + if (confDir != null) { + Path confPath = new Path(confDir); + if (!confPath.isUriPathAbsolute()) { + throw new RuntimeException("System property '" + KMS_CONFIG_DIR + + "' must be an absolute path: " + confDir); + } + File f = new File(confDir, KMS_ACLS_XML); + // at least 100ms newer than time, we do this to ensure the file + // has been properly closed/flushed + newer = f.lastModified() - time > 100; + } + return newer; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java new file mode 100644 index 0000000..5cb0885 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key.kms.server; + +import org.apache.hadoop.classification.InterfaceAudience; + +import com.sun.jersey.api.container.ContainerException; + +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.util.HttpExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; + +import java.io.IOException; + +/** + * Jersey provider that converts KMS exceptions into detailed HTTP errors. + */ +@Provider [email protected] +public class KMSExceptionsProvider implements ExceptionMapper<Exception> { + private static Logger LOG = + LoggerFactory.getLogger(KMSExceptionsProvider.class); + + private static final String ENTER = System.getProperty("line.separator"); + + protected Response createResponse(Response.Status status, Throwable ex) { + return HttpExceptionUtils.createJerseyExceptionResponse(status, ex); + } + + protected String getOneLineMessage(Throwable exception) { + String message = exception.getMessage(); + if (message != null) { + int i = message.indexOf(ENTER); + if (i > -1) { + message = message.substring(0, i); + } + } + return message; + } + + /** + * Maps different exceptions thrown by KMS to HTTP status codes. + */ + @Override + public Response toResponse(Exception exception) { + Response.Status status; + boolean doAudit = true; + Throwable throwable = exception; + if (exception instanceof ContainerException) { + throwable = exception.getCause(); + } + if (throwable instanceof SecurityException) { + status = Response.Status.FORBIDDEN; + } else if (throwable instanceof AuthenticationException) { + status = Response.Status.FORBIDDEN; + // we don't audit here because we did it already when checking access + doAudit = false; + } else if (throwable instanceof AuthorizationException) { + status = Response.Status.FORBIDDEN; + // we don't audit here because we did it already when checking access + doAudit = false; + } else if (throwable instanceof AccessControlException) { + status = Response.Status.FORBIDDEN; + } else if (exception instanceof IOException) { + status = Response.Status.INTERNAL_SERVER_ERROR; + } else if (exception instanceof UnsupportedOperationException) { + status = Response.Status.BAD_REQUEST; + } else if (exception instanceof IllegalArgumentException) { + status = Response.Status.BAD_REQUEST; + } else { + status = Response.Status.INTERNAL_SERVER_ERROR; + } + if (doAudit) { + KMSWebApp.getKMSAudit().error(KMSMDCFilter.getUgi(), + KMSMDCFilter.getMethod(), + KMSMDCFilter.getURL(), getOneLineMessage(exception)); + } + return createResponse(status, throwable); + } + + protected void log(Response.Status status, Throwable ex) { + UserGroupInformation ugi = KMSMDCFilter.getUgi(); + String method = KMSMDCFilter.getMethod(); + String url = KMSMDCFilter.getURL(); + String msg = getOneLineMessage(ex); + LOG.warn("User:'{}' Method:{} URL:{} Response:{}-{}", ugi, method, url, + status, msg, ex); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java new file mode 100644 index 0000000..6918015 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key.kms.server; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.jmx.JMXJsonServlet; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import java.io.IOException; + [email protected] +public class KMSJMXServlet extends JMXJsonServlet { + + @Override + protected boolean isInstrumentationAccessAllowed(HttpServletRequest request, + HttpServletResponse response) throws IOException { + return true; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/bb0bdced/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java new file mode 100644 index 0000000..d3e0064 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key.kms.server; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.codehaus.jackson.map.ObjectMapper; + +import javax.ws.rs.Consumes; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.ext.MessageBodyReader; +import javax.ws.rs.ext.Provider; +import java.io.IOException; +import java.io.InputStream; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; +import java.util.Map; + +@Provider +@Consumes(MediaType.APPLICATION_JSON) [email protected] +public class KMSJSONReader implements MessageBodyReader<Map> { + + @Override + public boolean isReadable(Class<?> type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + return type.isAssignableFrom(Map.class); + } + + @Override + public Map readFrom(Class<Map> type, Type genericType, + Annotation[] annotations, MediaType mediaType, + MultivaluedMap<String, String> httpHeaders, InputStream entityStream) + throws IOException, WebApplicationException { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(entityStream, type); + } +}
