Repository: incubator-ranger Updated Branches: refs/heads/master ec2ea9213 -> 44d365434
RANGER-749 : Ranger KMS to support multiple KMS instances with keys across multiple clusters Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/44d36543 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/44d36543 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/44d36543 Branch: refs/heads/master Commit: 44d365434a37ccd5a4fb8fc2988d17c8d9098046 Parents: ec2ea92 Author: Gautam Borad <[email protected]> Authored: Wed Nov 25 13:48:14 2015 +0530 Committer: Gautam Borad <[email protected]> Committed: Wed Dec 2 09:04:53 2015 +0530 ---------------------------------------------------------------------- kms/scripts/exportKeysToJCEKS.sh | 19 +++ kms/scripts/importJCEKSKeys.sh | 2 +- .../hadoop/crypto/key/Ranger2JKSUtil.java | 134 +++++++++++++++++++ .../hadoop/crypto/key/RangerKeyStore.java | 40 +++++- src/main/assembly/kms.xml | 1 + 5 files changed, 188 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44d36543/kms/scripts/exportKeysToJCEKS.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/exportKeysToJCEKS.sh b/kms/scripts/exportKeysToJCEKS.sh new file mode 100644 index 0000000..03dbe48 --- /dev/null +++ b/kms/scripts/exportKeysToJCEKS.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +RANGER_KMS_HOME=`dirname $0` +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" +java -cp "${cp}" org.apache.hadoop.crypto.key.Ranger2JKSUtil ${1} ${2} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44d36543/kms/scripts/importJCEKSKeys.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/importJCEKSKeys.sh b/kms/scripts/importJCEKSKeys.sh index 9c2f9fb..57a2053 100755 --- a/kms/scripts/importJCEKSKeys.sh +++ b/kms/scripts/importJCEKSKeys.sh @@ -15,5 +15,5 @@ # limitations under the License. # ------------------------------------------------------------------------------------- RANGER_KMS_HOME=`dirname $0` -cp="${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF:" +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" java -cp "${cp}" org.apache.hadoop.crypto.key.JKS2RangerUtil ${1} ${2} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44d36543/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java new file mode 100644 index 0000000..0def5e5 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key; + +import java.io.Console; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.security.KeyStore; +import java.security.KeyStoreException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.kms.dao.DaoManager; + +public class Ranger2JKSUtil { + + private static final String DEFAULT_KEYSTORE_TYPE = "jceks" ; + private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password" ; + + public static void showUsage() { + System.err.println("USAGE: java " + Ranger2JKSUtil.class.getName() + " <KMS_FileName> [KeyStoreType]") ; + System.err.println(" If KeyStoreType is not provided, it will be considered as " + DEFAULT_KEYSTORE_TYPE) ; + System.err.println(" When execution of this utility, it will prompt for both keystore password and key password.") ; + } + + + public static void main(String[] args) throws IOException { + if (args.length == 0) { + System.err.println("Invalid number of parameters found.") ; + showUsage() ; + System.exit(1) ; + } + else { + String keyStoreFileName = args[0] ; + File f = new File(keyStoreFileName) ; + if (! f.exists()) { + f.createNewFile(); + } + String keyStoreType = (args.length == 2 ? args[1] : DEFAULT_KEYSTORE_TYPE) ; + try { + KeyStore.getInstance(keyStoreType) ; + } catch (KeyStoreException e) { + System.err.println("ERROR: Unable to get valid keystore for the type [" + keyStoreType + "]") ; + showUsage() ; + System.exit(1) ; + } + + new Ranger2JKSUtil().doExportKeysFromJKS(keyStoreFileName, keyStoreType); + + System.out.println("Keys from Ranger KMS Database has been successfully exported into " + keyStoreFileName); + + System.exit(0) ; + + } + } + + private void doExportKeysFromJKS(String keyStoreFileName, String keyStoreType) { + try { + char[] keyStorePassword = getPasswordFromConsole("Enter Password for the keystore FILE :") ; + char[] keyPassword = getPasswordFromConsole("Enter Password for the KEY(s) stored in the keystore:") ; + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); + DaoManager daoManager = rangerkmsDb.getDaoManager(); + RangerKeyStore dbStore = new RangerKeyStore(daoManager); + String password = conf.get(ENCRYPTION_KEY); + RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); + char[] masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); + OutputStream out = null; + try { + out = new FileOutputStream(new File(keyStoreFileName)); + dbStore.engineLoadToKeyStoreFile(out, keyStorePassword, keyPassword, masterKey, keyStoreType); + } + finally { + if (out != null) { + try { + out.close(); + } catch (Exception e) { + throw new RuntimeException("ERROR: Unable to close file stream for [" + keyStoreFileName + "]", e) ; + } + } + } + } + catch(Throwable t) { + throw new RuntimeException("Unable to export keys to [" + keyStoreFileName + "] due to exception.", t) ; + } + } + + private char[] getPasswordFromConsole(String prompt) throws IOException { + String ret = null ; + Console c=System.console(); + if (c == null) { + System.out.print(prompt + " "); + InputStream in=System.in; + int max=50; + byte[] b=new byte[max]; + int l= in.read(b); + l--; //last character is \n + if (l>0) { + byte[] e=new byte[l]; + System.arraycopy(b,0, e, 0, l); + ret = new String(e, Charset.defaultCharset()); + } + } else { + char[] pwd = c.readPassword(prompt + " ") ; + if (pwd == null) { + ret = null ; + } + else { + ret = new String(pwd); + } + } + if (ret == null) { + ret = "" ; + } + return ret.toCharArray() ; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44d36543/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java index ff82f53..850104f 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java @@ -42,6 +42,7 @@ import java.security.NoSuchAlgorithmException; import java.security.UnrecoverableKeyException; import java.security.cert.Certificate; import java.security.cert.CertificateException; +import java.util.ArrayList; import java.util.Date; import java.util.Enumeration; import java.util.Hashtable; @@ -223,10 +224,10 @@ public class RangerKeyStore extends KeyStoreSpi { throw new IllegalArgumentException("Ranger Master Key can't be null"); } - MessageDigest md = getKeyedMessageDigest(password); + MessageDigest md = getKeyedMessageDigest(password); byte digest[] = md.digest(); - for (Enumeration<String> e = deltaEntries.keys(); e.hasMoreElements();) { + for (Enumeration<String> e = deltaEntries.keys(); e.hasMoreElements();) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(new DigestOutputStream(baos, md)); @@ -282,7 +283,6 @@ public class RangerKeyStore extends KeyStoreSpi { xxRangerKeyStore = new XXRangerKeyStore(); keyStoreExists = false; } - xxRangerKeyStore = mapToEntityBean(rangerKeyStore, xxRangerKeyStore, 0); if (keyStoreExists) { xxRangerKeyStore = rangerKMSDao.update(xxRangerKeyStore); @@ -483,13 +483,13 @@ public class RangerKeyStore extends KeyStoreSpi { public void engineLoadKeyStoreFile(InputStream stream, char[] storePass, char[] keyPass, char[] masterKey, String fileFormat) throws IOException, NoSuchAlgorithmException, CertificateException { - synchronized(keyEntries) { + synchronized(deltaEntries) { KeyStore ks; try { ks = KeyStore.getInstance(fileFormat); ks.load(stream, storePass); - keyEntries.clear(); + deltaEntries.clear(); for (Enumeration<String> name = ks.aliases(); name.hasMoreElements();){ SecretKeyEntry entry = new SecretKeyEntry(); String alias = (String) name.nextElement(); @@ -532,8 +532,34 @@ public class RangerKeyStore extends KeyStoreSpi { entry.date = ks.getCreationDate(alias); entry.version = (alias.split("@").length == 2)?(Integer.parseInt(alias.split("@")[1])):0; entry.description = k.getFormat()+" - "+ks.getType(); - keyEntries.put(alias, entry); - } + deltaEntries.put(alias, entry); + } + } catch (Throwable t) { + logger.error("Unable to load keystore file ", t); + throw new IOException(t) ; + } + } + } + + public void engineLoadToKeyStoreFile(OutputStream stream, char[] storePass, char[] keyPass, char[] masterKey, String fileFormat) + throws IOException, NoSuchAlgorithmException, CertificateException + { + synchronized(keyEntries) { + KeyStore ks; + try { + ks = KeyStore.getInstance(fileFormat); + ks.load(null, storePass); + String alias = null; + engineLoad(null, masterKey); + Enumeration<String> e = engineAliases(); + Key key; + while (e.hasMoreElements()) { + alias = e.nextElement(); + key = engineGetKey(alias, masterKey); + ks.setKeyEntry(alias, key, keyPass, null); + } + + ks.store(stream, storePass); } catch (Throwable t) { logger.error("Unable to load keystore file ", t); throw new IOException(t) ; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/44d36543/src/main/assembly/kms.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/kms.xml b/src/main/assembly/kms.xml index 52ab5a0..5c9e11b 100755 --- a/src/main/assembly/kms.xml +++ b/src/main/assembly/kms.xml @@ -304,6 +304,7 @@ <include>setup.sh</include> <include>install.properties</include> <include>importJCEKSKeys.sh</include> + <include>exportKeysToJCEKS.sh</include> </includes> <fileMode>544</fileMode> </fileSet>
