RANGER-548: Fixes for Key Rollover command failure Signed-off-by: sneethiraj <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/b70ec703 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/b70ec703 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/b70ec703 Branch: refs/heads/ranger-0.5 Commit: b70ec703ad34e4c4996b7ba5a41a233d338297da Parents: 12ab54a Author: Velmurugan Periasamy <[email protected]> Authored: Thu Jun 11 21:59:42 2015 -0700 Committer: sneethiraj <[email protected]> Committed: Fri Jun 12 09:16:24 2015 -0700 ---------------------------------------------------------------------- .../hadoop/crypto/key/RangerKeyStore.java | 13 ++--- .../crypto/key/RangerKeyStoreProvider.java | 57 +++++++++++++------- .../hadoop/crypto/key/kms/server/KMS.java | 16 +++++- .../apache/ranger/entity/XXRangerKeyStore.java | 2 + .../java/org/apache/ranger/kms/dao/BaseDao.java | 16 ++++-- .../org/apache/ranger/kms/dao/RangerKMSDao.java | 7 +++ .../META-INF/kms_jpa_named_queries.xml | 5 ++ kms/src/main/resources/META-INF/persistence.xml | 7 ++- 8 files changed, 90 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b70ec703/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java index f38f8b0..dc8efde 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java @@ -82,14 +82,12 @@ public class RangerKeyStore extends KeyStoreSpi { int version; } - private final Hashtable<String, Object> keyEntries ; + private Hashtable<String, Object> keyEntries = new Hashtable<String, Object>(); RangerKeyStore() { - keyEntries = new Hashtable<String, Object>(); } RangerKeyStore(DaoManager daoManager) { - keyEntries = new Hashtable<String, Object>(); this.daoManager = daoManager; } @@ -117,7 +115,7 @@ public class RangerKeyStore extends KeyStoreSpi { o = constructor.newInstance(password); Method m = c.getDeclaredMethod("unseal", SealedObject.class); m.setAccessible(true); - key = (Key) m.invoke(o, ((SecretKeyEntry)entry).sealedKey); + key = (Key) m.invoke(o, ((SecretKeyEntry)entry).sealedKey); } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { logger.error(e.getMessage()); } @@ -313,6 +311,7 @@ public class RangerKeyStore extends KeyStoreSpi { { synchronized(keyEntries) { List<XXRangerKeyStore> rangerKeyDetails = dbOperationLoad(); + DataInputStream dis; MessageDigest md = null; @@ -372,7 +371,6 @@ public class RangerKeyStore extends KeyStoreSpi { entry.description = rangerKey.getDescription(); entry.version = rangerKey.getVersion(); entry.attributes = rangerKey.getAttributes(); - //read the sealed key try { ois = new ObjectInputStream(dis); @@ -380,7 +378,7 @@ public class RangerKeyStore extends KeyStoreSpi { } catch (ClassNotFoundException cnfe) { throw new IOException(cnfe.getMessage()); } - + //Add the entry to the list keyEntries.put(alias, entry); }finally { @@ -398,7 +396,7 @@ public class RangerKeyStore extends KeyStoreSpi { try{ if(daoManager != null){ RangerKMSDao rangerKMSDao = new RangerKMSDao(daoManager); - return rangerKMSDao.getAll(); + return rangerKMSDao.getAllKeys(); } }catch(Exception e){ e.printStackTrace(); @@ -531,7 +529,6 @@ public class RangerKeyStore extends KeyStoreSpi { entry.version = (alias.split("@").length == 2)?(Integer.parseInt(alias.split("@")[1])):0; entry.description = k.getFormat()+" - "+ks.getType(); keyEntries.put(alias, entry); - System.out.println("+ adding key alias [" + alias + "]") ; } } catch (Throwable t) { logger.error("Unable to load keystore file ", t); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b70ec703/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java index ee48c7c..23547a7 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Map; import javax.crypto.spec.SecretKeySpec; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; @@ -46,6 +47,7 @@ import org.apache.hadoop.fs.Path; import org.apache.ranger.credentialapi.CredentialReader; import org.apache.ranger.kms.dao.DaoManager; import org.apache.log4j.Logger; + import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -93,7 +95,7 @@ public class RangerKeyStoreProvider extends KeyProvider{ // Master Key does not exists throw new IOException("Ranger MasterKey does not exists"); } - reloadKeys() ; + reloadKeys(); ReadWriteLock lock = new ReentrantReadWriteLock(true); readLock = lock.readLock(); } @@ -133,13 +135,13 @@ public class RangerKeyStoreProvider extends KeyProvider{ } private void loadKeys(char[] masterKey) throws NoSuchAlgorithmException, CertificateException, IOException { - dbStore.engineLoad(null, masterKey); + dbStore.engineLoad(null, masterKey); } @Override public KeyVersion createKey(String name, byte[] material, Options options) throws IOException { - reloadKeys() ; + reloadKeys() ; if (dbStore.engineContainsAlias(name) || cache.containsKey(name)) { throw new IOException("Key " + name + " already exists"); } @@ -158,7 +160,7 @@ public class RangerKeyStoreProvider extends KeyProvider{ try { ObjectMapper om = new ObjectMapper(); String attribute = om.writeValueAsString(attributes); - dbStore.addKeyEntry(versionName, new SecretKeySpec(material, cipher), masterKey, cipher, bitLength, description, version, attribute); + dbStore.addKeyEntry(versionName, new SecretKeySpec(material, cipher), masterKey, cipher, bitLength, description, version, attribute); } catch (KeyStoreException e) { throw new IOException("Can't store key " + versionName,e); } @@ -168,7 +170,8 @@ public class RangerKeyStoreProvider extends KeyProvider{ @Override public void deleteKey(String name) throws IOException { - Metadata meta = getMetadata(name); + reloadKeys(); + Metadata meta = getMetadata(name); if (meta == null) { throw new IOException("Key " + name + " does not exist"); } @@ -190,7 +193,7 @@ public class RangerKeyStoreProvider extends KeyProvider{ throw new IOException("Problem removing " + name + " from " + this, e); } cache.remove(name); - changed = true; + changed = true; } @Override @@ -212,15 +215,18 @@ public class RangerKeyStoreProvider extends KeyProvider{ } try { dbStore.engineStore(null, masterKey); + reloadKeys(); } catch (NoSuchAlgorithmException e) { throw new IOException("No such algorithm storing key", e); } catch (CertificateException e) { throw new IOException("Certificate exception storing key", e); - } + } changed = false; }catch (IOException ioe) { + cache.clear(); + reloadKeys(); throw ioe; - } + } } @Override @@ -230,14 +236,20 @@ public class RangerKeyStoreProvider extends KeyProvider{ SecretKeySpec key = null; try { if (!dbStore.engineContainsAlias(versionName)) { - return null; - } + dbStore.engineLoad(null, masterKey); + if (!dbStore.engineContainsAlias(versionName)) { + return null; + } + } key = (SecretKeySpec) dbStore.engineGetKey(versionName, masterKey); } catch (NoSuchAlgorithmException e) { throw new IOException("Can't get algorithm for key " + key, e); } catch (UnrecoverableKeyException e) { throw new IOException("Can't recover key " + key, e); } + catch (CertificateException e) { + throw new IOException("Certificate exception storing key", e); + } if (key == null) { return null; } else { @@ -285,15 +297,18 @@ public class RangerKeyStoreProvider extends KeyProvider{ @Override public Metadata getMetadata(String name) throws IOException { - try { + try { readLock.lock(); - reloadKeys() ; - if (cache.containsKey(name)) { - return cache.get(name); + if (cache.containsKey(name)) { + Metadata meta = cache.get(name); + return meta; } try { if (!dbStore.engineContainsAlias(name)) { - return null; + dbStore.engineLoad(null, masterKey); + if (!dbStore.engineContainsAlias(name)) { + return null; + } } Key key = dbStore.engineGetKey(name, masterKey); if(key != null){ @@ -307,13 +322,18 @@ public class RangerKeyStoreProvider extends KeyProvider{ throw new IOException("Can't recover key for " + name, e); } return null; - } finally { + } + catch(Exception e){ + throw new IOException("Please try again ", e); + } + finally { readLock.unlock(); } } @Override public KeyVersion rollNewVersion(String name, byte[] material)throws IOException { + reloadKeys(); Metadata meta = getMetadata(name); if (meta == null) { throw new IOException("Key " + name + " not found"); @@ -345,12 +365,13 @@ public class RangerKeyStoreProvider extends KeyProvider{ private void reloadKeys() throws IOException { try { - loadKeys(masterKey); + cache.clear(); + loadKeys(masterKey); } catch (NoSuchAlgorithmException e) { throw new IOException("Can't load Keys"); }catch(CertificateException e){ throw new IOException("Can't load Keys"); - } + } } /** http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b70ec703/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java index 404b710..ae6d8f8 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java @@ -52,6 +52,8 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Class providing the REST bindings, via Jersey, for the KMS. @@ -67,6 +69,8 @@ public class KMS { GENERATE_EEK, DECRYPT_EEK } + private static final String KEY_NAME_VALIDATION = "[a-z,A-Z,0-9](?!.*--)(?!.*__)(?!.*-_)(?!.*_-)[\\w\\-\\_]*"; + private KeyProviderCryptoExtension provider; private KMSAudit kmsAudit; @@ -105,7 +109,8 @@ public class KMS { KMSWebApp.getAdminCallsMeter().mark(); UserGroupInformation user = HttpUserGroupInformation.get(); final String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD); - KMSClientProvider.checkNotEmpty(name, KMSRESTConstants.NAME_FIELD); + KMSClientProvider.checkNotEmpty(name, KMSRESTConstants.NAME_FIELD); + validateKeyName(name); assertAccess(Type.CREATE, user, KMSOp.CREATE_KEY, name, request.getRemoteAddr()); String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD); final String material = (String) jsonKey.get(KMSRESTConstants.MATERIAL_FIELD); @@ -158,6 +163,15 @@ public class KMS { header("Location", keyURL).entity(json).build(); } + private void validateKeyName(String name) { + Pattern pattern = Pattern.compile(KEY_NAME_VALIDATION); + Matcher matcher = pattern.matcher(name); + if(!matcher.matches()){ + throw new IllegalArgumentException("Key Name : " + name + + ", should start with alpha/numeric letters and can have special characters - (hypen) or _ (underscore)"); + } + } + @DELETE @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") public Response deleteKey(@PathParam("name") final String name, @Context HttpServletRequest request) http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b70ec703/kms/src/main/java/org/apache/ranger/entity/XXRangerKeyStore.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/ranger/entity/XXRangerKeyStore.java b/kms/src/main/java/org/apache/ranger/entity/XXRangerKeyStore.java index 9bc53c2..8defdf6 100755 --- a/kms/src/main/java/org/apache/ranger/entity/XXRangerKeyStore.java +++ b/kms/src/main/java/org/apache/ranger/entity/XXRangerKeyStore.java @@ -17,6 +17,7 @@ package org.apache.ranger.entity; +import javax.persistence.Cacheable; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.GeneratedValue; @@ -28,6 +29,7 @@ import javax.persistence.Table; import javax.xml.bind.annotation.XmlRootElement; @Entity +@Cacheable(false) @Table(name="ranger_keystore") @XmlRootElement public class XXRangerKeyStore extends XXDBBase implements java.io.Serializable { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b70ec703/kms/src/main/java/org/apache/ranger/kms/dao/BaseDao.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/ranger/kms/dao/BaseDao.java b/kms/src/main/java/org/apache/ranger/kms/dao/BaseDao.java index f835bcc..dbaedd0 100644 --- a/kms/src/main/java/org/apache/ranger/kms/dao/BaseDao.java +++ b/kms/src/main/java/org/apache/ranger/kms/dao/BaseDao.java @@ -177,9 +177,8 @@ public abstract class BaseDao<T> { TypedQuery<T> qry = getEntityManager().createQuery( "SELECT t FROM " + tClass.getSimpleName() + " t", tClass); - + qry.setHint("eclipselink.refresh", "true"); ret = qry.getResultList(); - return ret; } @@ -189,9 +188,8 @@ public abstract class BaseDao<T> { TypedQuery<Long> qry = getEntityManager().createQuery( "SELECT count(t) FROM " + tClass.getSimpleName() + " t", Long.class); - + qry.setHint("eclipselink.refresh", "true"); ret = qry.getSingleResult(); - return ret; } @@ -258,4 +256,14 @@ public abstract class BaseDao<T> { } return 0; } + + public List<T> getAllKeys(String namedQuery) { + try { + return getEntityManager() + .createNamedQuery(namedQuery, tClass).setHint("eclipselink.refresh", "true").getResultList(); + } catch (NoResultException e) { + e.printStackTrace(); + } + return null; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b70ec703/kms/src/main/java/org/apache/ranger/kms/dao/RangerKMSDao.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/ranger/kms/dao/RangerKMSDao.java b/kms/src/main/java/org/apache/ranger/kms/dao/RangerKMSDao.java index 2f750aa..cb64310 100644 --- a/kms/src/main/java/org/apache/ranger/kms/dao/RangerKMSDao.java +++ b/kms/src/main/java/org/apache/ranger/kms/dao/RangerKMSDao.java @@ -17,6 +17,8 @@ package org.apache.ranger.kms.dao; +import java.util.List; + import org.apache.ranger.entity.XXRangerKeyStore; public class RangerKMSDao extends BaseDao<XXRangerKeyStore> { @@ -32,4 +34,9 @@ public class RangerKMSDao extends BaseDao<XXRangerKeyStore> { public int deleteByAlias(String alias){ return super.deleteByAlias("XXRangerKeyStore.deleteByAlias", alias); } + + public List<XXRangerKeyStore> getAllKeys(){ + List<XXRangerKeyStore> xxr = super.getAllKeys("XXRangerKeyStore.getAllKeys"); + return xxr; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b70ec703/kms/src/main/resources/META-INF/kms_jpa_named_queries.xml ---------------------------------------------------------------------- diff --git a/kms/src/main/resources/META-INF/kms_jpa_named_queries.xml b/kms/src/main/resources/META-INF/kms_jpa_named_queries.xml index 8fd3128..94d5fa6 100644 --- a/kms/src/main/resources/META-INF/kms_jpa_named_queries.xml +++ b/kms/src/main/resources/META-INF/kms_jpa_named_queries.xml @@ -23,6 +23,11 @@ WHERE obj.alias=:alias </query> </named-query> + + <named-query name="XXRangerKeyStore.getAllKeys"> + <query>SELECT Obj FROM XXRangerKeyStore obj + </query> + </named-query> <named-query name="XXRangerKeyStore.deleteByAlias"> <query>DELETE FROM XXRangerKeyStore obj http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/b70ec703/kms/src/main/resources/META-INF/persistence.xml ---------------------------------------------------------------------- diff --git a/kms/src/main/resources/META-INF/persistence.xml b/kms/src/main/resources/META-INF/persistence.xml index 31c0bc4..57445b5 100644 --- a/kms/src/main/resources/META-INF/persistence.xml +++ b/kms/src/main/resources/META-INF/persistence.xml @@ -20,9 +20,12 @@ <mapping-file>META-INF/kms_jpa_named_queries.xml</mapping-file> <class>org.apache.ranger.entity.XXRangerMasterKey</class> <class>org.apache.ranger.entity.XXRangerKeyStore</class> - + <shared-cache-mode>NONE</shared-cache-mode> + <properties> - <property name="eclipselink.logging.level" value="SEVERE"/> + <property name="eclipselink.logging.level" value="WARNING"/> + <property name="eclipselink.cache.shared.default" value="false"/> + <property name="eclipselink.query-results-cache" value="false"/> </properties> </persistence-unit> </persistence>
