Repository: ranger Updated Branches: refs/heads/master 5259c1e9d -> 20db6e67b
RANGER-2017 : Ranger KMS encryption good practices Signed-off-by: Mehul Parikh <me...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/20db6e67 Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/20db6e67 Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/20db6e67 Branch: refs/heads/master Commit: 20db6e67bdaf65e1f6012ac798fe6c9d3b1cd110 Parents: 5259c1e Author: Bhavik Patel <bhavikpatel...@gmail.com> Authored: Wed Apr 4 11:30:24 2018 +0530 Committer: Mehul Parikh <me...@apache.org> Committed: Fri Apr 6 16:34:37 2018 +0530 ---------------------------------------------------------------------- kms/config/kms-webapp/dbks-site.xml | 79 +++++- kms/scripts/DBMK2HSM.sh | 2 +- kms/scripts/HSMMK2DB.sh | 2 +- kms/scripts/importJCEKSKeys.sh | 2 +- .../hadoop/crypto/key/RangerMasterKey.java | 239 ++++++++++++++----- 5 files changed, 255 insertions(+), 69 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/20db6e67/kms/config/kms-webapp/dbks-site.xml ---------------------------------------------------------------------- diff --git a/kms/config/kms-webapp/dbks-site.xml b/kms/config/kms-webapp/dbks-site.xml index 2fc5177..ec649a7 100755 --- a/kms/config/kms-webapp/dbks-site.xml +++ b/kms/config/kms-webapp/dbks-site.xml @@ -29,17 +29,74 @@ </description> </property> - <!-- Encryption key Password --> - - <property> - <name>ranger.db.encrypt.key.password</name> - <value>Str0ngPassw0rd</value> - <description> - Password used for encrypting Master Key - </description> - </property> - - <!-- db Details --> + <!-- Encryption key Password --> + + <property> + <name>ranger.db.encrypt.key.password</name> + <value>Str0ngPassw0rd</value> + <description> + Password used for encrypting Master Key + </description> + </property> + + <property> + <name>ranger.kms.service.masterkey.password.cipher</name> + <value>AES</value> + <description> + Cipher used for encrypting Master Key + </description> + </property> + + <property> + <name>ranger.kms.service.masterkey.password.size</name> + <value>256</value> + <description> + Size of masterkey + </description> + </property> + + <property> + <name>ranger.kms.service.masterkey.password.salt.size</name> + <value>8</value> + <description> + Salt size to encrypt Master Key + </description> + </property> + + <property> + <name>ranger.kms.service.masterkey.password.salt</name> + <value>abcdefghijklmnopqrstuvwxyz01234567890</value> + <description> + Salt to encrypt Master Key + </description> + </property> + + + <property> + <name>ranger.kms.service.masterkey.password.iteration.count</name> + <value>1000</value> + <description> + Iteration count to encrypt Master Key + </description> + </property> + + <property> + <name>ranger.kms.service.masterkey.password.encryption.algorithm</name> + <value>PBEWithMD5AndDES</value> + <description> + Algorithm to encrypt Master Key + </description> + </property> + + <property> + <name>ranger.kms.service.masterkey.password.md.algorithm</name> + <value>SHA</value> + <description> + Message Digest algorithn to encrypt Master Key + </description> + </property> + + <!-- db Details --> <property> <name>ranger.ks.jpa.jdbc.url</name> http://git-wip-us.apache.org/repos/asf/ranger/blob/20db6e67/kms/scripts/DBMK2HSM.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/DBMK2HSM.sh b/kms/scripts/DBMK2HSM.sh index 89c8c2d..001199d 100644 --- a/kms/scripts/DBMK2HSM.sh +++ b/kms/scripts/DBMK2HSM.sh @@ -20,5 +20,5 @@ else exit ; fi RANGER_KMS_HOME=`dirname $0` -cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/WEB-INF/classes/lib/*:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" java -cp "${cp}" org.apache.hadoop.crypto.key.DB2HSMMKUtil ${1} ${2} http://git-wip-us.apache.org/repos/asf/ranger/blob/20db6e67/kms/scripts/HSMMK2DB.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/HSMMK2DB.sh b/kms/scripts/HSMMK2DB.sh index 2637cf6..6c77f73 100644 --- a/kms/scripts/HSMMK2DB.sh +++ b/kms/scripts/HSMMK2DB.sh @@ -20,5 +20,5 @@ else exit ; fi RANGER_KMS_HOME=`dirname $0` -cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/WEB-INF/classes/lib/*:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" java -cp "${cp}" org.apache.hadoop.crypto.key.HSM2DBMKUtil ${1} ${2} http://git-wip-us.apache.org/repos/asf/ranger/blob/20db6e67/kms/scripts/importJCEKSKeys.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/importJCEKSKeys.sh b/kms/scripts/importJCEKSKeys.sh index d72c93e..5d4fe97 100755 --- a/kms/scripts/importJCEKSKeys.sh +++ b/kms/scripts/importJCEKSKeys.sh @@ -20,5 +20,5 @@ else exit ; fi RANGER_KMS_HOME=`dirname $0` -cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/WEB-INF/classes/lib/*:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" java -cp "${cp}" org.apache.hadoop.crypto.key.JKS2RangerUtil ${1} ${2} http://git-wip-us.apache.org/repos/asf/ranger/blob/20db6e67/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java index 5614c16..6cfd37e 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java @@ -20,6 +20,7 @@ package org.apache.hadoop.crypto.key; import java.security.Key; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,33 +33,79 @@ import javax.crypto.SecretKeyFactory; import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.PBEParameterSpec; import javax.crypto.spec.SecretKeySpec; + +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.ranger.kms.dao.DaoManager; import org.apache.ranger.kms.dao.RangerMasterKeyDao; +import org.apache.ranger.plugin.util.XMLUtils; import org.apache.ranger.entity.XXRangerMasterKey; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; import com.sun.org.apache.xml.internal.security.exceptions.Base64DecodingException; import com.sun.org.apache.xml.internal.security.utils.Base64; public class RangerMasterKey implements RangerKMSMKI{ private static final Logger logger = Logger.getLogger(RangerMasterKey.class); - - private static final String MK_CIPHER = "AES"; - private static final int MK_KeySize = 256; - private static final int SALT_SIZE = 8; - private static final String PBE_ALGO = "PBEWithMD5AndTripleDES"; - private static final String MD_ALGO = "MD5"; - + + private static final String DEFAULT_MK_CIPHER = "AES"; + private static final int DEFAULT_MK_KeySize = 256; + private static final int DEFAULT_SALT_SIZE = 8; + private static final String DEFAULT_SALT = "abcdefghijklmnopqrstuvwxyz01234567890"; + private static final String DEFAULT_CRYPT_ALGO = "PBEWithMD5AndTripleDES"; + private static final String DEFAULT_MD_ALGO = "MD5"; + private static final int DEFAULT_ITERATION_COUNT = 1000; + private static String password = null; + + public static final String DBKS_SITE_XML = "dbks-site.xml"; + private static Properties serverConfigProperties = new Properties(); + + public static String MK_CIPHER; + public static Integer MK_KeySize = 0; + public static Integer SALT_SIZE = 0; + public static String SALT; + public static String PBE_ALGO; + public static String MD_ALGO; + public static Integer ITERATION_COUNT = 0; + public static String paddingString; + private DaoManager daoManager; - - public RangerMasterKey() { + + public RangerMasterKey() { } public RangerMasterKey(DaoManager daoManager) { this.daoManager = daoManager; } + protected static String getConfig(String key, String defaultValue) { + String value = serverConfigProperties.getProperty(key); + if (value == null || value.trim().isEmpty()) { + // Value not found in properties file, let's try to get from + // System's property + value = System.getProperty(key); + } + if (value == null || value.trim().isEmpty()) { + value = defaultValue; + } + return value; + } + + protected static int getIntConfig(String key, int defaultValue) { + int ret = defaultValue; + String retStr = serverConfigProperties.getProperty(key); + try { + if (retStr != null) { + ret = Integer.parseInt(retStr); + } + } catch (Exception err) { + logger.warn(retStr + " can't be parsed to int. Reason: " + err.toString()); + } + return ret; + } /** * To get Master Key * @param password password to be used for decryption @@ -66,24 +113,40 @@ public class RangerMasterKey implements RangerKMSMKI{ * @throws Throwable */ @Override - public String getMasterKey(String password) throws Throwable{ + public String getMasterKey(String password) throws Throwable { logger.info("Getting Master Key"); - byte masterKeyByte[] = getEncryptedMK(); - if(masterKeyByte != null && masterKeyByte.length > 0){ - return decryptMasterKey(masterKeyByte, password); - }else{ + List result = getEncryptedMK(); + String encryptedPassString = null; + byte masterKeyByte[] = null; + if (CollectionUtils.isNotEmpty(result) && result.size() == 2) { + masterKeyByte = (byte[]) result.get(0); + encryptedPassString = (String) result.get(1); + } else if (CollectionUtils.isNotEmpty(result)) { + masterKeyByte = (byte[]) result.get(0); + } + if (masterKeyByte != null && masterKeyByte.length > 0) { + return decryptMasterKey(masterKeyByte, password, encryptedPassString); + } else { throw new Exception("No Master Key Found"); - } + } } - public SecretKey getMasterSecretKey(String password) throws Throwable{ + public SecretKey getMasterSecretKey(String password) throws Throwable { logger.info("Getting Master Key"); - byte masterKeyByte[] = getEncryptedMK(); - if(masterKeyByte != null && masterKeyByte.length > 0){ - return decryptMasterKeySK(masterKeyByte, password); - }else{ + List result = getEncryptedMK(); + String encryptedPassString = null; + byte masterKeyByte[] = null; + if (CollectionUtils.isNotEmpty(result) && result.size() == 2) { + masterKeyByte = (byte[]) result.get(0); + encryptedPassString = (String) result.get(1); + } else if (CollectionUtils.isNotEmpty(result)) { + masterKeyByte = (byte[]) result.get(0); + } + if (masterKeyByte != null && masterKeyByte.length > 0) { + return decryptMasterKeySK(masterKeyByte, password, encryptedPassString); + } else { throw new Exception("No Master Key Found"); - } + } } /** @@ -93,13 +156,29 @@ public class RangerMasterKey implements RangerKMSMKI{ * false if master key generation was unsuccessful or the master key already exists * @throws Throwable */ + + public void init() { + XMLUtils.loadConfig(DBKS_SITE_XML, serverConfigProperties); + MK_CIPHER = getConfig("ranger.kms.service.masterkey.password.cipher", DEFAULT_MK_CIPHER); + MK_KeySize = getIntConfig("ranger.kms.service.masterkey.password.size", DEFAULT_MK_KeySize); + SALT_SIZE = getIntConfig("ranger.kms.service.masterkey.password.salt.size", DEFAULT_SALT_SIZE); + SALT = getConfig("ranger.kms.service.masterkey.password.salt", DEFAULT_SALT); + PBE_ALGO = getConfig("ranger.kms.service.masterkey.password.encryption.algorithm", DEFAULT_CRYPT_ALGO); + MD_ALGO = getConfig("ranger.kms.service.masterkey.password.md.algorithm", DEFAULT_MD_ALGO); + ITERATION_COUNT = getIntConfig("ranger.kms.service.masterkey.password.iteration.count", + DEFAULT_ITERATION_COUNT); + paddingString = Joiner.on(",").skipNulls().join(MK_CIPHER, MK_KeySize, SALT_SIZE, PBE_ALGO, MD_ALGO, + ITERATION_COUNT, SALT); + } + @Override - public boolean generateMasterKey(String password) throws Throwable{ + public boolean generateMasterKey(String password) throws Throwable { logger.info("Generating Master Key"); - String encryptedMasterKey = encryptMasterKey(password); - String savedKey = saveEncryptedMK(encryptedMasterKey, daoManager); - if(savedKey != null && !savedKey.trim().equals("")){ - logger.debug("Master Key Created with id = "+savedKey); + init(); + String encryptedMasterKey = encryptMasterKey(password); + String savedKey = saveEncryptedMK(paddingString + "," + encryptedMasterKey, daoManager); + if (savedKey != null && !savedKey.trim().equals("")) { + logger.debug("Master Key Created with id = " + savedKey); return true; } return false; @@ -107,8 +186,9 @@ public class RangerMasterKey implements RangerKMSMKI{ public boolean generateMKFromHSMMK(String password, byte[] key) throws Throwable{ logger.info("Generating Master Key"); + init(); String encryptedMasterKey = encryptMasterKey(password, key); - String savedKey = saveEncryptedMK(encryptedMasterKey, daoManager); + String savedKey = saveEncryptedMK(paddingString + "," + encryptedMasterKey, daoManager); if(savedKey != null && !savedKey.trim().equals("")){ logger.debug("Master Key Created with id = "+savedKey); return true; @@ -116,41 +196,90 @@ public class RangerMasterKey implements RangerKMSMKI{ return false; } - private String decryptMasterKey(byte masterKey[], String password) throws Throwable { + private String decryptMasterKey(byte masterKey[], String password, String encryptedPassString) throws Throwable { logger.debug("Decrypting Master Key"); + if (encryptedPassString == null) { + getPasswordParam(password); + } PBEKeySpec pbeKeyspec = getPBEParameterSpec(password); byte[] masterKeyFromDBDecrypted = decryptKey(masterKey, pbeKeyspec); SecretKey masterKeyFromDB = getMasterKeyFromBytes(masterKeyFromDBDecrypted); return Base64.encode(masterKeyFromDB.getEncoded()); } - - private SecretKey decryptMasterKeySK(byte masterKey[], String password) throws Throwable { + + public static void getPasswordParam(String paddedEncryptedPwd) { + String[] encryptedPwd = null; + if (paddedEncryptedPwd != null && paddedEncryptedPwd.contains(",")) { + encryptedPwd = Lists.newArrayList(Splitter.on(",").split(paddedEncryptedPwd)).toArray(new String[0]); + } + if (encryptedPwd != null && encryptedPwd.length >= 7) { + int index = 0; + MK_CIPHER = encryptedPwd[index]; + MK_KeySize = Integer.parseInt(encryptedPwd[++index]); + SALT_SIZE = Integer.parseInt(encryptedPwd[++index]); + PBE_ALGO = encryptedPwd[++index]; + MD_ALGO = encryptedPwd[++index]; + ITERATION_COUNT = Integer.parseInt(encryptedPwd[++index]); + SALT = encryptedPwd[++index]; + password = encryptedPwd[++index]; + + if (index > 7) { + for (int i = 8; i <= encryptedPwd.length; i++) { + password = password + "," + encryptedPwd[i]; + } + } + } else { + MK_CIPHER = DEFAULT_MK_CIPHER; + MK_KeySize = DEFAULT_MK_KeySize; + SALT_SIZE = DEFAULT_SALT_SIZE; + PBE_ALGO = DEFAULT_CRYPT_ALGO; + MD_ALGO = DEFAULT_MD_ALGO; + password = paddedEncryptedPwd; + SALT = password; + ITERATION_COUNT = password.toCharArray().length + 1; + } + } + + private SecretKey decryptMasterKeySK(byte masterKey[], String password, String encryptedPassString) + throws Throwable { logger.debug("Decrypting Master Key"); + if (encryptedPassString == null) { + getPasswordParam(password); + } PBEKeySpec pbeKeyspec = getPBEParameterSpec(password); byte[] masterKeyFromDBDecrypted = decryptKey(masterKey, pbeKeyspec); - return getMasterKeyFromBytes(masterKeyFromDBDecrypted); + return getMasterKeyFromBytes(masterKeyFromDBDecrypted); } - private byte[] getEncryptedMK() throws Base64DecodingException { + private List getEncryptedMK() throws Base64DecodingException { logger.debug("Retrieving Encrypted Master Key from database"); - try{ - if(daoManager != null){ - RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager); - List<XXRangerMasterKey> lstRangerMasterKey = rangerKMSDao.getAll(); - if(lstRangerMasterKey.size() < 1){ - throw new Exception("No Master Key exists"); - }else if(lstRangerMasterKey.size() > 1){ - throw new Exception("More than one Master Key exists"); - }else { - XXRangerMasterKey rangerMasterKey = rangerKMSDao.getById(lstRangerMasterKey.get(0).getId()); - String masterKeyStr = rangerMasterKey.getMasterKey(); - return Base64.decode(masterKeyStr); - } - } - }catch(Exception e){ - e.printStackTrace(); - } - return null; + try { + if (daoManager != null) { + ArrayList ret = new ArrayList<>(); + RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager); + List<XXRangerMasterKey> lstRangerMasterKey = rangerKMSDao.getAll(); + if (lstRangerMasterKey.size() < 1) { + throw new Exception("No Master Key exists"); + } else if (lstRangerMasterKey.size() > 1) { + throw new Exception("More than one Master Key exists"); + } else { + XXRangerMasterKey rangerMasterKey = rangerKMSDao.getById(lstRangerMasterKey.get(0).getId()); + String masterKeyStr = rangerMasterKey.getMasterKey(); + if (masterKeyStr.contains(",")) { + getPasswordParam(masterKeyStr); + ret.add(Base64.decode(password)); + ret.add(masterKeyStr); + return ret; + } else { + ret.add(Base64.decode(masterKeyStr)); + return ret; + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return null; } private String saveEncryptedMK(String encryptedMasterKey, DaoManager daoManager) { @@ -197,12 +326,12 @@ public class RangerMasterKey implements RangerKMSMKI{ private PBEKeySpec getPBEParameterSpec(String password) throws Throwable { MessageDigest md = MessageDigest.getInstance(MD_ALGO); - byte[] saltGen = md.digest(password.getBytes()); - byte[] salt = new byte[SALT_SIZE]; - System.arraycopy(saltGen, 0, salt, 0, SALT_SIZE); - int iteration = password.toCharArray().length + 1; - return new PBEKeySpec(password.toCharArray(), salt, iteration); + byte[] saltGen = md.digest(SALT.getBytes()); + byte[] salt = new byte[SALT_SIZE]; + System.arraycopy(saltGen, 0, salt, 0, SALT_SIZE); + return new PBEKeySpec(password.toCharArray(), salt, ITERATION_COUNT); } + private byte[] encryptKey(byte[] data, PBEKeySpec keyspec) throws Throwable { SecretKey key = getPasswordKey(keyspec); if(keyspec.getSalt() != null) {
