Repository: incubator-ranger Updated Branches: refs/heads/master e85eb01d5 -> 73dc5031e
RANGER-868: Add Support for HSM to Ranger Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/73dc5031 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/73dc5031 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/73dc5031 Branch: refs/heads/master Commit: 73dc5031e3cfd8b9671b41e0f619b854e7aafd23 Parents: e85eb01 Author: Ankita Sinha <[email protected]> Authored: Wed Apr 13 19:38:45 2016 +0530 Committer: Gautam Borad <[email protected]> Committed: Thu Apr 14 10:27:52 2016 +0530 ---------------------------------------------------------------------- kms/config/kms-webapp/dbks-site.xml | 31 +++++ kms/scripts/DBMK2HSM.sh | 19 +++ kms/scripts/HSMMK2DB.sh | 19 +++ kms/scripts/install.properties | 6 + kms/scripts/setup.sh | 64 ++++++++- .../apache/hadoop/crypto/key/DB2HSMMKUtil.java | 129 ++++++++++++++++++ .../apache/hadoop/crypto/key/HSM2DBMKUtil.java | 127 ++++++++++++++++++ .../org/apache/hadoop/crypto/key/RangerHSM.java | 131 +++++++++++++++++++ .../apache/hadoop/crypto/key/RangerKMSMKI.java | 25 ++++ .../crypto/key/RangerKeyStoreProvider.java | 18 ++- .../hadoop/crypto/key/RangerMasterKey.java | 40 +++++- src/main/assembly/kms.xml | 2 + 12 files changed, 605 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/config/kms-webapp/dbks-site.xml ---------------------------------------------------------------------- diff --git a/kms/config/kms-webapp/dbks-site.xml b/kms/config/kms-webapp/dbks-site.xml index edaff93..f649264 100755 --- a/kms/config/kms-webapp/dbks-site.xml +++ b/kms/config/kms-webapp/dbks-site.xml @@ -113,4 +113,35 @@ </description> </property> + <!-- HSM Config --> + <property> + <name>ranger.ks.hsm.type</name> + <value>LunaProvider</value> + <description></description> + </property> + + <property> + <name>ranger.ks.hsm.enabled</name> + <value>false</value> + <description></description> + </property> + + <property> + <name>ranger.ks.hsm.partition.name</name> + <value>par19</value> + <description></description> + </property> + + <property> + <name>ranger.ks.hsm.partition.password</name> + <value>S@fenet123</value> + <description></description> + </property> + + <property> + <name>ranger.ks.hsm.partition.password.alias</name> + <value>ranger.kms.hsm.partition.password</value> + <description></description> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/scripts/DBMK2HSM.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/DBMK2HSM.sh b/kms/scripts/DBMK2HSM.sh new file mode 100644 index 0000000..e5beec3 --- /dev/null +++ b/kms/scripts/DBMK2HSM.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +RANGER_KMS_HOME=`dirname $0` +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" +java -cp "${cp}" org.apache.hadoop.crypto.key.DB2HSMMKUtil ${1} ${2} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/scripts/HSMMK2DB.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/HSMMK2DB.sh b/kms/scripts/HSMMK2DB.sh new file mode 100644 index 0000000..03e9888 --- /dev/null +++ b/kms/scripts/HSMMK2DB.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +RANGER_KMS_HOME=`dirname $0` +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" +java -cp "${cp}" org.apache.hadoop.crypto.key.HSM2DBMKUtil ${1} ${2} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/scripts/install.properties ---------------------------------------------------------------------- diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties index d30b28c..7762948 100755 --- a/kms/scripts/install.properties +++ b/kms/scripts/install.properties @@ -65,6 +65,12 @@ db_password= #------------------------- RANGER KMS Master Key Crypt Key ------------------ KMS_MASTER_KEY_PASSWD=Str0ngPassw0rd +#------------------------- Ranger KMS HSM CONFIG ------------------------------ +HSM_TYPE=LunaProvider +HSM_ENABLED=false +HSM_PARTITION_NAME=par19 +HSM_PARTITION_PASSWORD=S@fenet123 + # # ------- UNIX User CONFIG ---------------- # http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/scripts/setup.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh index 64abcc7..6019526 100755 --- a/kms/scripts/setup.sh +++ b/kms/scripts/setup.sh @@ -84,6 +84,10 @@ sqlanywhere_core_file=$(get_prop 'sqlanywhere_core_file' $PROPFILE) cred_keystore_filename=$(eval echo "$(get_prop 'cred_keystore_filename' $PROPFILE)") KMS_BLACKLIST_DECRYPT_EEK=$(get_prop 'KMS_BLACKLIST_DECRYPT_EEK' $PROPFILE) RANGER_KMS_LOG_DIR=$(eval echo "$(get_prop 'RANGER_KMS_LOG_DIR' $PROPFILE)") +HSM_TYPE=$(get_prop 'HSM_TYPE' $PROPFILE) +HSM_ENABLED=$(get_prop 'HSM_ENABLED' $PROPFILE) +HSM_PARTITION_NAME=$(get_prop 'HSM_PARTITION_NAME' $PROPFILE) +HSM_PARTITION_PASSWORD=$(get_prop 'HSM_PARTITION_PASSWORD' $PROPFILE) DB_HOST="${db_host}" @@ -159,7 +163,7 @@ updatePropertyToFile(){ #$1 -> propertyName $2 -> newPropertyValue $3 -> fileName updatePropertyToFilePy(){ python update_property.py $1 $2 $3 - check_ret_status $? "Update property failed for: " $1 + check_ret_status $? "Update property failed for: {'$1'}" } @@ -207,7 +211,12 @@ init_variables(){ DB_FLAVOR="MYSQL" fi log "[I] DB_FLAVOR=${DB_FLAVOR}" - password_validation "$KMS_MASTER_KEY_PASSWD" "KMS Master key" + ########## HSM Config ########## + + propertyName=ranger.ks.hsm.enabled + HSM_ENABLED=`echo $HSM_ENABLED | tr '[:lower:]' '[:upper:]'` + password_validation "$KMS_MASTER_KEY_PASSWD" "KMS Master key" + #getPropertyFromFile 'db_root_user' $PROPFILE db_root_user #getPropertyFromFile 'db_root_password' $PROPFILE db_user #getPropertyFromFile 'db_user' $PROPFILE db_user @@ -494,6 +503,11 @@ update_properties() { MK_CREDENTIAL_ALIAS="ranger.ks.masterkey.password" DB_CREDENTIAL_ALIAS="ranger.ks.jpa.jdbc.credential.alias" + HSM_PARTITION_PASSWD="ranger.ks.hsm.partition.password" + HSM_PARTITION_PASSWORD_ALIAS="ranger.kms.hsm.partition.password" + + HSM_ENABLED=`echo $HSM_ENABLED | tr '[:lower:]' '[:upper:]'` + if [ "${keystore}" != "" ] then mkdir -p `dirname "${keystore}"` @@ -503,6 +517,21 @@ update_properties() { #$JAVA_HOME/bin/java -cp "cred/lib/*" org.apache.ranger.credentialapi.buildks create "${DB_CREDENTIAL_ALIAS}" -value "$db_password" -provider jceks://file$keystore #$JAVA_HOME/bin/java -cp "cred/lib/*" org.apache.ranger.credentialapi.buildks create "${MK_CREDENTIAL_ALIAS}" -value "${KMS_MASTER_KEY_PASSWD}" -provider jceks://file$keystore + if [ "${HSM_ENABLED}" == "TRUE" ] + then + password_validation "$HSM_PARTITION_PASSWORD" "HSM Partition Password" + + $PYTHON_COMMAND_INVOKER ranger_credential_helper.py -l "cred/lib/*" -f "$keystore" -k "${HSM_PARTITION_PASSWORD_ALIAS}" -v "${HSM_PARTITION_PASSWORD}" -c 1 + + propertyName=ranger.ks.hsm.partition.password.alias + newPropertyValue="${HSM_PARTITION_PASSWORD_ALIAS}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.hsm.partition.password + newPropertyValue="_" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + fi + propertyName=ranger.ks.jpa.jdbc.credential.alias newPropertyValue="${DB_CREDENTIAL_ALIAS}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file @@ -530,6 +559,10 @@ update_properties() { propertyName="${MK_CREDENTIAL_ATTR}" newPropertyValue="${KMS_MASTER_KEY_PASSWD}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName="${HSM_PARTITION_PASSWD}" + newPropertyValue="${HSM_PARTITION_PASSWORD}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file fi if test -f $keystore; then @@ -546,13 +579,38 @@ update_properties() { propertyName="${MK_CREDENTIAL_ATTR}" newPropertyValue="${KMS_MASTER_KEY_PASSWD}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName="${HSM_PARTITION_PASSWD}" + newPropertyValue="${HSM_PARTITION_PASSWORD}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file fi propertyName=hadoop.kms.blacklist.DECRYPT_EEK newPropertyValue="${KMS_BLACKLIST_DECRYPT_EEK}" updatePropertyToFilePy $propertyName $newPropertyValue $to_file - ########### + ########### HSM CONFIG ################# + + + if [ "${HSM_ENABLED}" != "TRUE" ] + then + propertyName=ranger.ks.hsm.enabled + newPropertyValue="false" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + else + propertyName=ranger.ks.hsm.enabled + newPropertyValue="true" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.hsm.type + newPropertyValue="${HSM_TYPE}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + + propertyName=ranger.ks.hsm.partition.name + newPropertyValue="${HSM_PARTITION_NAME}" + updatePropertyToFilePy $propertyName $newPropertyValue $to_file + fi + } #===================================================================== http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/DB2HSMMKUtil.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/DB2HSMMKUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/DB2HSMMKUtil.java new file mode 100644 index 0000000..ca69dc0 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/DB2HSMMKUtil.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key; + +import java.io.Console; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; + +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.kms.dao.DaoManager; + +import com.sun.org.apache.xml.internal.security.utils.Base64; + +public class DB2HSMMKUtil { + + private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password" ; + private static final String PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; + private static final String PARTITION_NAME = "ranger.ks.hsm.partition.name"; + private static final String HSM_TYPE = "ranger.ks.hsm.type"; + + public static void showUsage() { + System.err.println("USAGE: java " + DB2HSMMKUtil.class.getName() + " <HSMType> <partitionName>") ; + } + + public static void main(String[] args) { + if (args.length < 2) { + System.err.println("Invalid number of parameters found.") ; + showUsage() ; + System.exit(1) ; + } + else { + String hsmType = args[0]; + if (hsmType == null || hsmType.trim().isEmpty()) { + System.err.println("HSM Type does not exists.") ; + showUsage() ; + System.exit(1); + } + + String partitionName = args[1]; + if (partitionName == null || partitionName.trim().isEmpty()) { + System.err.println("Partition name does not exists.") ; + showUsage() ; + System.exit(1) ; + } + + boolean result = new DB2HSMMKUtil().doExportMKToHSM(hsmType, partitionName); + if(result){ + System.out.println("Master Key from Ranger KMS DB has been successfully imported into HSM.") ; + }else{ + System.out.println("Import of Master Key from DB has been unsuccessful.") ; + } + System.exit(0) ; + + } + } + + private boolean doExportMKToHSM(String hsmType, String partitionName) { + try { + String partitionPassword = getPasswordFromConsole("Enter Password for the Partition "+partitionName+" : ") ; + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + conf.set(HSM_TYPE, hsmType); + conf.set(PARTITION_NAME, partitionName); + conf.set(PARTITION_PASSWORD, partitionPassword); + + RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); + DaoManager daoManager = rangerkmsDb.getDaoManager(); + RangerKeyStore dbStore = new RangerKeyStore(daoManager); + String password = conf.get(ENCRYPTION_KEY); + + // Get Master Key from Ranger DB + RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); + String mkey = rangerMasterKey.getMasterKey(password); + byte[] key = Base64.decode(mkey); + + // Put Master Key in HSM + RangerHSM rangerHSM = new RangerHSM(conf); + + return rangerHSM.setMasterKey(password, key); + } + catch(Throwable t) { + throw new RuntimeException("Unable to import Master key from Ranger DB to HSM ", t) ; + } + } + + private String getPasswordFromConsole(String prompt) throws IOException { + String ret = null ; + Console c=System.console(); + if (c == null) { + System.out.print(prompt + " "); + InputStream in=System.in; + int max=50; + byte[] b=new byte[max]; + int l= in.read(b); + l--; //last character is \n + if (l>0) { + byte[] e=new byte[l]; + System.arraycopy(b,0, e, 0, l); + ret = new String(e, Charset.defaultCharset()); + } + } else { + char[] pwd = c.readPassword(prompt + " ") ; + if (pwd == null) { + ret = null ; + } + else { + ret = new String(pwd); + } + } + if (ret == null) { + ret = "" ; + } + return ret; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/HSM2DBMKUtil.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/HSM2DBMKUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/HSM2DBMKUtil.java new file mode 100644 index 0000000..73a5830 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/HSM2DBMKUtil.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key; + +import java.io.Console; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; + +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.kms.dao.DaoManager; + +import com.sun.org.apache.xml.internal.security.utils.Base64; + +public class HSM2DBMKUtil { + + private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password" ; + private static final String PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; + private static final String PARTITION_NAME = "ranger.ks.hsm.partition.name"; + private static final String HSM_TYPE = "ranger.ks.hsm.type"; + + public static void showUsage() { + System.err.println("USAGE: java " + HSM2DBMKUtil.class.getName() + " <HSMType> <partitionName>") ; + } + + + public static void main(String[] args) { + if (args.length < 2) { + System.err.println("Invalid number of parameters found.") ; + showUsage() ; + System.exit(1) ; + } + else { + String hsmType = args[0]; + if (hsmType == null || hsmType.trim().isEmpty()) { + System.err.println("HSM Type does not exists.") ; + showUsage() ; + System.exit(1); + } + + String partitionName = args[1]; + if (partitionName == null || partitionName.trim().isEmpty()) { + System.err.println("Partition name does not exists.") ; + showUsage() ; + System.exit(1) ; + } + + new HSM2DBMKUtil().doImportMKFromHSM(hsmType, partitionName); + + System.out.println("Master Key from HSM has been successfully imported into Ranger KMS DB.") ; + + System.exit(0) ; + + } + } + + private void doImportMKFromHSM(String hsmType, String partitionName) { + try { + String partitionPassword = getPasswordFromConsole("Enter Password for the Partition "+partitionName+" : ") ; + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + conf.set(HSM_TYPE, hsmType); + conf.set(PARTITION_NAME, partitionName); + conf.set(PARTITION_PASSWORD, partitionPassword); + + RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); + DaoManager daoManager = rangerkmsDb.getDaoManager(); + RangerKeyStore dbStore = new RangerKeyStore(daoManager); + String password = conf.get(ENCRYPTION_KEY); + + // Get Master Key from HSM + RangerHSM rangerHSM = new RangerHSM(conf); + String mKey = rangerHSM.getMasterKey(password); + byte[] key = Base64.decode(mKey); + + // Put Master Key in Ranger DB + RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); + rangerMasterKey.generateMKFromHSMMK(password, key); + } + catch(Throwable t) { + throw new RuntimeException("Unable to import Master key from HSM to Ranger DB", t) ; + } + } + + private String getPasswordFromConsole(String prompt) throws IOException { + String ret = null ; + Console c=System.console(); + if (c == null) { + System.out.print(prompt + " "); + InputStream in=System.in; + int max=50; + byte[] b=new byte[max]; + int l= in.read(b); + l--; //last character is \n + if (l>0) { + byte[] e=new byte[l]; + System.arraycopy(b,0, e, 0, l); + ret = new String(e, Charset.defaultCharset()); + } + } else { + char[] pwd = c.readPassword(prompt + " ") ; + if (pwd == null) { + ret = null ; + } + else { + ret = new String(pwd); + } + } + if (ret == null) { + ret = "" ; + } + return ret; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java new file mode 100644 index 0000000..6ab91d9 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Logger; + +import com.sun.org.apache.xml.internal.security.utils.Base64; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.security.Key; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; + +/** + * This Class is for HSM Keystore + */ +public class RangerHSM implements RangerKMSMKI { + + static final Logger logger = Logger.getLogger(RangerHSM.class); + + // Configure these as required. + private String passwd = null; + private String alias = "RangerKMSKey"; + private String partitionName = null; + private KeyStore myStore = null; + private String hsm_keystore=null; + private static final String MK_CIPHER = "AES"; + private static final int MK_KeySize = 128; + private static final String PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; + private static final String PARTITION_NAME = "ranger.ks.hsm.partition.name"; + private static final String HSM_TYPE = "ranger.ks.hsm.type"; + + public RangerHSM(){ + } + + public RangerHSM(Configuration conf) { + logger.info("RangerHSM provider"); + /* + * We will log in to the HSM + */ + passwd = conf.get(PARTITION_PASSWORD); + partitionName = conf.get(PARTITION_NAME); + hsm_keystore = conf.get(HSM_TYPE); + try { + ByteArrayInputStream is1 = new ByteArrayInputStream(("tokenlabel:" + partitionName).getBytes()); + logger.debug("Loading HSM tokenlabel : "+partitionName); + myStore = KeyStore.getInstance("Luna"); + myStore.load(is1, passwd.toCharArray()); + } catch (KeyStoreException kse) { + logger.error("Unable to create keystore object : "+kse.getMessage()); + } catch (NoSuchAlgorithmException nsae) { + logger.error("Unexpected NoSuchAlgorithmException while loading keystore : " + nsae.getMessage()); + } catch (CertificateException e) { + logger.error("Unexpected CertificateException while loading keystore : "+e.getMessage()); + } catch (IOException e) { + logger.error("Unexpected IOException while loading keystore : "+e.getMessage()); + } + } + + @Override + public boolean generateMasterKey(String password) throws Throwable { + if(myStore.size() < 1){ + KeyGenerator keyGen = null; + SecretKey aesKey = null; + try { + logger.info("Generating AES Master Key for HSM Provider"); + keyGen = KeyGenerator.getInstance(MK_CIPHER, hsm_keystore); + keyGen.init(MK_KeySize); + aesKey = keyGen.generateKey(); + String masterKey = Base64.encode(aesKey.getEncoded()) ; + myStore.setKeyEntry(alias, aesKey, password.toCharArray(), (java.security.cert.Certificate[]) null); + return true; + } catch (Exception e) { + logger.error("generateMasterKey : Exception during Ranger Master Key Generation - " + e.getMessage()); + return false; + } + } + return false; + } + + @Override + public String getMasterKey(String password) throws Throwable { + try { + logger.debug("Searching for Ranger Master Key in Luna Keystore"); + boolean result = myStore.containsAlias(alias); + if (result == true) { + logger.debug("Ranger Master Key is present in Keystore"); + SecretKey key = (SecretKey)myStore.getKey(alias, password.toCharArray()); + String masterKey = Base64.encode(key.getEncoded()) ; + return masterKey; + } + } catch (Exception e) { + logger.error("getMasterKey : Exception searching for Ranger Master Key - " + e.getMessage()); + } + return null; + } + + public boolean setMasterKey(String password, byte[] key){ + try { + Key aesKey = new SecretKeySpec(key, MK_CIPHER); + myStore.setKeyEntry(alias, aesKey, password.toCharArray(), (java.security.cert.Certificate[]) null); + return true; + } catch (KeyStoreException e) { + logger.error("setMasterKey : Exception while setting Master Key - " + e.getMessage()); + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java new file mode 100644 index 0000000..75e70ff --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key; + +public interface RangerKMSMKI { + + boolean generateMasterKey(String password) throws Throwable; + + String getMasterKey(String password) throws Throwable; +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java index a9e43fc..12c9ed4 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java @@ -64,6 +64,9 @@ public class RangerKeyStoreProvider extends KeyProvider{ private static final String MK_CREDENTIAL_ALIAS = "ranger.ks.masterkey.credential.alias"; private static final String DB_CREDENTIAL_ALIAS = "ranger.ks.jpa.jdbc.credential.alias"; private static final String DB_PASSWORD = "ranger.ks.jpa.jdbc.password"; + private static final String HSM_ENABLED = "ranger.ks.hsm.enabled"; + private static final String HSM_PARTITION_PASSWORD_ALIAS = "ranger.ks.hsm.partition.password.alias"; + private static final String HSM_PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; private final RangerKeyStore dbStore; private char[] masterKey; @@ -78,14 +81,25 @@ public class RangerKeyStoreProvider extends KeyProvider{ conf = getDBKSConf(); getFromJceks(conf,CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY); getFromJceks(conf,CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD); + getFromJceks(conf,CREDENTIAL_PATH, HSM_PARTITION_PASSWORD_ALIAS, HSM_PARTITION_PASSWORD); RangerKMSDB rangerKMSDB = new RangerKMSDB(conf); daoManager = rangerKMSDB.getDaoManager(); - RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); - dbStore = new RangerKeyStore(daoManager); + + RangerKMSMKI rangerMasterKey = null; String password = conf.get(ENCRYPTION_KEY); if(password == null || password.trim().equals("") || password.trim().equals("_") || password.trim().equals("crypted")){ throw new IOException("Master Key Jceks does not exists"); } + if(conf.get(HSM_ENABLED).equalsIgnoreCase("false")){ + rangerMasterKey = new RangerMasterKey(daoManager); + }else{ + rangerMasterKey = new RangerHSM(conf); + String partitionPasswd = conf.get(HSM_PARTITION_PASSWORD); + if(partitionPasswd == null || partitionPasswd.trim().equals("") || partitionPasswd.trim().equals("_") || partitionPasswd.trim().equals("crypted")){ + throw new IOException("Partition Password doesn't exists"); + } + } + dbStore = new RangerKeyStore(daoManager); rangerMasterKey.generateMasterKey(password); //code to retrieve rangerMasterKey password masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java index 75a34b2..d70ec4e 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java @@ -40,7 +40,7 @@ import org.apache.ranger.entity.XXRangerMasterKey; import com.sun.org.apache.xml.internal.security.exceptions.Base64DecodingException; import com.sun.org.apache.xml.internal.security.utils.Base64; -public class RangerMasterKey { +public class RangerMasterKey implements RangerKMSMKI{ static final Logger logger = Logger.getLogger(RangerMasterKey.class); @@ -65,6 +65,7 @@ public class RangerMasterKey { * @return Decrypted Master Key * @throws Throwable */ + @Override public String getMasterKey(String password) throws Throwable{ logger.info("Getting Master Key"); byte masterKeyByte[] = getEncryptedMK(); @@ -76,6 +77,16 @@ public class RangerMasterKey { } } + public SecretKey getMasterSecretKey(String password) throws Throwable{ + logger.info("Getting Master Key"); + byte masterKeyByte[] = getEncryptedMK(); + if(masterKeyByte != null && masterKeyByte.length > 0){ + return decryptMasterKeySK(masterKeyByte, password); + }else{ + throw new Exception("No Master Key Found"); + } + } + /** * Generate the master key encrypt's it and save it in database * @param password password to be used for encryption @@ -83,6 +94,7 @@ public class RangerMasterKey { * false if master key generation was unsuccessful or already master key exists * @throws Throwable */ + @Override public boolean generateMasterKey(String password) throws Throwable{ logger.info("Generating Master Key"); String encryptedMasterKey = encryptMasterKey(password); @@ -93,6 +105,17 @@ public class RangerMasterKey { } return false; } + + public boolean generateMKFromHSMMK(String password, byte[] key) throws Throwable{ + logger.info("Generating Master Key"); + String encryptedMasterKey = encryptMasterKey(password, key); + String savedKey = saveEncryptedMK(encryptedMasterKey, daoManager); + if(savedKey != null && !savedKey.trim().equals("")){ + logger.debug("Master Key Created with id = "+savedKey); + return true; + } + return false; + } private String decryptMasterKey(byte masterKey[], String password) throws Throwable { logger.debug("Decrypting Master Key"); @@ -101,6 +124,13 @@ public class RangerMasterKey { SecretKey masterKeyFromDB = getMasterKeyFromBytes(masterKeyFromDBDecrypted) ; return Base64.encode(masterKeyFromDB.getEncoded()); } + + private SecretKey decryptMasterKeySK(byte masterKey[], String password) throws Throwable { + logger.debug("Decrypting Master Key"); + PBEKeySpec pbeKeyspec = getPBEParameterSpec(password) ; + byte[] masterKeyFromDBDecrypted = decryptKey(masterKey, pbeKeyspec) ; + return getMasterKeyFromBytes(masterKeyFromDBDecrypted) ; + } private byte[] getEncryptedMK() throws Base64DecodingException { logger.debug("Retrieving Encrypted Master Key from database"); @@ -155,6 +185,14 @@ public class RangerMasterKey { return masterKey; } + private String encryptMasterKey(String password, byte[] secretKey) throws Throwable { + logger.debug("Encrypting Master Key"); + PBEKeySpec pbeKeySpec = getPBEParameterSpec(password); + byte[] masterKeyToDB = encryptKey(secretKey, pbeKeySpec); + String masterKey = Base64.encode(masterKeyToDB) ; + return masterKey; + } + private Key generateMasterKey() throws NoSuchAlgorithmException{ KeyGenerator kg = KeyGenerator.getInstance(MK_CIPHER); kg.init(MK_KeySize); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/src/main/assembly/kms.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/kms.xml b/src/main/assembly/kms.xml index e267687..01f14b9 100755 --- a/src/main/assembly/kms.xml +++ b/src/main/assembly/kms.xml @@ -304,6 +304,8 @@ <include>install.properties</include> <include>importJCEKSKeys.sh</include> <include>exportKeysToJCEKS.sh</include> + <include>HSMMK2DB.sh</include> + <include>DBMK2HSM.sh</include> </includes> <fileMode>544</fileMode> </fileSet>
