Repository: incubator-ranger Updated Branches: refs/heads/master 168c13531 -> b802bd3ee
RANGER-259: added an import utility for JCEKS keys Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/b802bd3e Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/b802bd3e Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/b802bd3e Branch: refs/heads/master Commit: b802bd3eeb19fb885c8086dd303c043b05990ee8 Parents: 168c135 Author: sneethiraj <[email protected]> Authored: Thu Apr 23 04:40:39 2015 -0400 Committer: sneethiraj <[email protected]> Committed: Thu Apr 23 04:40:39 2015 -0400 ---------------------------------------------------------------------- kms/scripts/importJCEKSKeys.sh | 19 +++ .../hadoop/crypto/key/JKS2RangerUtil.java | 139 +++++++++++++++++++ .../hadoop/crypto/key/RangerKeyStore.java | 88 +++++++++++- .../crypto/key/RangerKeyStoreProvider.java | 8 +- src/main/assembly/kms.xml | 4 +- 5 files changed, 251 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b802bd3e/kms/scripts/importJCEKSKeys.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/importJCEKSKeys.sh b/kms/scripts/importJCEKSKeys.sh new file mode 100755 index 0000000..9c2f9fb --- /dev/null +++ b/kms/scripts/importJCEKSKeys.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +RANGER_KMS_HOME=`dirname $0` +cp="${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF:" +java -cp "${cp}" org.apache.hadoop.crypto.key.JKS2RangerUtil ${1} ${2} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b802bd3e/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java new file mode 100644 index 0000000..4e9ea59 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/JKS2RangerUtil.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key; + +import java.io.Console; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.security.KeyStore; +import java.security.KeyStoreException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.ProviderUtils; +import org.apache.ranger.kms.dao.DaoManager; + +public class JKS2RangerUtil { + + private static final String DEFAULT_KEYSTORE_TYPE = "jceks" ; + private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password" ; + + public static void showUsage() { + System.err.println("USAGE: java " + JKS2RangerUtil.class.getName() + " <KMS_FileName> [KeyStoreType]") ; + System.err.println(" If KeyStoreType is not provided, it will be considered as " + DEFAULT_KEYSTORE_TYPE) ; + System.err.println(" When execution of this utility, it will prompt for both keystore password and key password.") ; + } + + + public static void main(String[] args) { + if (args.length == 0) { + System.err.println("Invalid number of parameters found.") ; + showUsage() ; + System.exit(1) ; + } + else { + String keyStoreFileName = args[0] ; + File f = new File(keyStoreFileName) ; + if (! f.exists()) { + System.err.println("File: [" + f.getAbsolutePath() + "] does not exists.") ; + showUsage() ; + System.exit(1); + } + String keyStoreType = (args.length == 2 ? args[1] : DEFAULT_KEYSTORE_TYPE) ; + try { + KeyStore.getInstance(keyStoreType) ; + } catch (KeyStoreException e) { + System.err.println("ERROR: Unable to get valid keystore for the type [" + keyStoreType + "]") ; + showUsage() ; + System.exit(1) ; + } + + new JKS2RangerUtil().doImportKeysFromJKS(keyStoreFileName, keyStoreType); + + System.out.println("Keys from " + keyStoreFileName + " has been successfully imported into RangerDB.") ; + + System.exit(0) ; + + } + } + + private void doImportKeysFromJKS(String keyStoreFileName, String keyStoreType) { + try { + char[] keyStorePassword = getPasswordFromConsole("Enter Password for the keystore FILE :") ; + char[] keyPassword = getPasswordFromConsole("Enter Password for the KEY(s) stored in the keystore:") ; + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); + DaoManager daoManager = rangerkmsDb.getDaoManager(); + RangerKeyStore dbStore = new RangerKeyStore(daoManager); + String password = conf.get(ENCRYPTION_KEY); + RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager); + rangerMasterKey.generateMasterKey(password); + char[] masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); + InputStream in = null ; + try { + in = new FileInputStream(new File(keyStoreFileName)) ; + dbStore.engineLoadKeyStoreFile(in, keyStorePassword, keyPassword, masterKey, keyStoreType); + dbStore.engineStore(null,masterKey); + } + finally { + if (in != null) { + try { + in.close(); + } catch (Exception e) { + System.err.println("ERROR: Unable to close file stream for [" + keyStoreFileName + "]") ; + System.exit(1); + } + } + } + } + catch(Throwable t) { + System.err.println("Unable to import keys from [" + keyStoreFileName + "] due to exception :" + t ) ; + t.printStackTrace(); + System.exit(1); + } + } + + + private char[] getPasswordFromConsole(String prompt) throws IOException { + String ret = null ; + Console c=System.console(); + if (c == null) { + System.out.print(prompt + " "); + InputStream in=System.in; + int max=50; + byte[] b=new byte[max]; + int l= in.read(b); + l--; //last character is \n + if (l>0) { + byte[] e=new byte[l]; + System.arraycopy(b,0, e, 0, l); + ret = new String(e); + } + } else { + ret = new String(c.readPassword(prompt + " ")); + } + if (ret == null) { + ret = "" ; + } + return ret.toCharArray() ; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b802bd3e/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java index acff11c..93109e2 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java @@ -17,15 +17,41 @@ package org.apache.hadoop.crypto.key; -import java.io.*; -import java.security.*; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.security.DigestInputStream; +import java.security.DigestOutputStream; +import java.security.Key; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.KeyStoreSpi; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; import java.security.cert.Certificate; import java.security.cert.CertificateException; -import java.util.*; +import java.util.Date; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Properties; import javax.crypto.SealedObject; import javax.xml.bind.DatatypeConverter; +import org.apache.hadoop.crypto.key.KeyProvider.Metadata; import org.apache.log4j.Logger; import org.apache.ranger.entity.XXRangerKeyStore; import org.apache.ranger.kms.dao.DaoManager; @@ -550,4 +576,60 @@ public class RangerKeyStore extends KeyStoreSpi { return prefixedProperties; } + + // + // The method is created to support JKS migration (from hadoop-common KMS keystore to RangerKMS keystore) + // + + private static final String METADATA_FIELDNAME = "metadata" ; + private static final int NUMBER_OF_BITS_PER_BYTE = 8 ; + + public void engineLoadKeyStoreFile(InputStream stream, char[] storePass, char[] keyPass, char[] masterKey, String fileFormat) + throws IOException, NoSuchAlgorithmException, CertificateException + { + synchronized(entries) { + KeyStore ks; + + try { + ks = KeyStore.getInstance(fileFormat); + ks.load(stream, storePass); + entries.clear(); + for (Enumeration<String> name = ks.aliases(); name.hasMoreElements();){ + SecretKeyEntry entry = new SecretKeyEntry(); + String alias = (String) name.nextElement(); + Key k = ks.getKey(alias, keyPass); + + if (k instanceof JavaKeyStoreProvider.KeyMetadata) { + JavaKeyStoreProvider.KeyMetadata keyMetadata = (JavaKeyStoreProvider.KeyMetadata)k ; + Field f = JavaKeyStoreProvider.KeyMetadata.class.getDeclaredField(METADATA_FIELDNAME) ; + f.setAccessible(true); + Metadata metadata = (Metadata)f.get(keyMetadata) ; + entry.bit_length = metadata.getBitLength() ; + entry.cipher_field = metadata.getAlgorithm() ; + Constructor<RangerKeyStoreProvider.KeyMetadata> constructor = RangerKeyStoreProvider.KeyMetadata.class.getDeclaredConstructor(Metadata.class); + constructor.setAccessible(true); + RangerKeyStoreProvider.KeyMetadata nk = constructor.newInstance(metadata); + k = nk ; + } + else { + entry.bit_length = (k.getEncoded().length * NUMBER_OF_BITS_PER_BYTE) ; + entry.cipher_field = k.getAlgorithm(); + } + String keyName = alias.split("@")[0] ; + entry.attributes = "{\"key.acl.name\":\"" + keyName + "\"}" ; + KeyProtector keyProtector = new KeyProtector(masterKey); + entry.sealedKey = keyProtector.seal(k); + entry.date = ks.getCreationDate(alias); + entry.version = (alias.split("@").length == 2)?(Integer.parseInt(alias.split("@")[1])):0; + entry.description = k.getFormat()+" - "+ks.getType(); + entries.put(alias, entry); + System.out.println("+ adding key alias [" + alias + "]") ; + } + } catch (Throwable t) { + logger.error("Unable to load keystore file ", t); + throw new IOException(t) ; + } + } + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b802bd3e/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java index db0dc38..7b8e977 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java @@ -99,7 +99,11 @@ public class RangerKeyStoreProvider extends KeyProvider{ } public static Configuration getDBKSConf() { - return getConfiguration(true, DBKS_SITE_XML); + Configuration newConfig = getConfiguration(true, DBKS_SITE_XML); + getFromJceks(newConfig,CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY); + getFromJceks(newConfig,CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD); + return newConfig ; + } static Configuration getConfiguration(boolean loadHadoopDefaults, @@ -315,7 +319,7 @@ public class RangerKeyStoreProvider extends KeyProvider{ return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes()); } - private void getFromJceks(Configuration conf, String path, String alias, String key){ + private static void getFromJceks(Configuration conf, String path, String alias, String key){ //update credential from keystore if(conf!=null){ String pathValue=conf.get(path); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b802bd3e/src/main/assembly/kms.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/kms.xml b/src/main/assembly/kms.xml index 6882d1b..44eda67 100644 --- a/src/main/assembly/kms.xml +++ b/src/main/assembly/kms.xml @@ -295,8 +295,8 @@ <include>ranger-kms-initd</include> <include>ranger-kms</include> <include>setup.sh</include> - <include>install.properties</include> - <include>migrate-keystore-dbks.sh</include> + <include>install.properties</include> + <include>importJCEKSKeys.sh</include> </includes> <fileMode>544</fileMode> </fileSet>
