SQOOP-2890: Provide tooling to encrypt non-encrypted repository and rotate keys
(Abraham Fine via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c6fc7f95 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c6fc7f95 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c6fc7f95 Branch: refs/heads/sqoop2 Commit: c6fc7f95ac0e70aa2cb8902da3e74111d7c142bf Parents: 5b897a4 Author: Jarek Jarcec Cecho <[email protected]> Authored: Mon Apr 11 13:20:57 2016 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Mon Apr 11 13:20:57 2016 -0700 ---------------------------------------------------------------------- .../sqoop/error/code/CommonRepositoryError.java | 7 + .../java/org/apache/sqoop/driver/Driver.java | 13 +- .../apache/sqoop/repository/JdbcRepository.java | 36 +- .../sqoop/repository/JdbcRepositoryHandler.java | 21 +- .../sqoop/repository/MasterKeyManager.java | 251 ++++++----- .../org/apache/sqoop/repository/Repository.java | 28 +- .../apache/sqoop/security/SecurityError.java | 6 + .../sqoop/repository/TestMasterKeyManager.java | 13 +- dist/src/main/conf/sqoop.properties | 4 +- docs/src/site/sphinx/security/Encryption.rst | 218 ++++++++- .../common/CommonRepositoryHandler.java | 136 +++++- ...RepositoryInsertUpdateDeleteSelectQuery.java | 91 +++- .../derby/DerbySchemaCreateQuery.java | 2 +- test/pom.xml | 5 + .../tools/RepositoryDumpLoadToolTest.java | 47 +- .../tools/RepositoryEncryptionToolTest.java | 444 +++++++++++++++++++ .../apache/sqoop/tools/tool/BuiltinTools.java | 1 + .../tools/tool/RepositoryEncryptionTool.java | 145 ++++++ 18 files changed, 1298 insertions(+), 170 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java b/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java index dd52c82..4d523d2 100644 --- a/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java +++ b/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java @@ -218,6 +218,13 @@ public enum CommonRepositoryError implements ErrorCode { COMMON_0058("Resource doesn't exist"), COMMON_0059("Unable to retrieve master key"), + + COMMON_0060("Failed to change the master key for inputs"), + + COMMON_0061("Unable to delete Master Key"), + + COMMON_0062("Invalid primary key after registering entity"), + ; private final String message; http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/driver/Driver.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/Driver.java b/core/src/main/java/org/apache/sqoop/driver/Driver.java index 2d1d73e..b6be4f1 100644 --- a/core/src/main/java/org/apache/sqoop/driver/Driver.java +++ b/core/src/main/java/org/apache/sqoop/driver/Driver.java @@ -114,10 +114,6 @@ public class Driver implements Reconfigurable { } private Driver() { - List<MConfig> driverConfig = ConfigUtils.toConfigs(getDriverJobConfigurationClass()); - List<MValidator> mValidators = ConfigUtils.getMValidatorsFromConfigurationClass(getDriverJobConfigurationClass()); - mDriver = new MDriver(new MDriverConfig(driverConfig, mValidators), DriverBean.CURRENT_DRIVER_VERSION); - // Build upgrader driverUpgrader = new DriverUpgrader(); } @@ -130,15 +126,20 @@ public class Driver implements Reconfigurable { public synchronized void initialize(boolean autoUpgrade) { LOG.trace("Begin Driver initialization"); + List<MConfig> driverConfig = ConfigUtils.toConfigs(getDriverJobConfigurationClass()); + List<MValidator> mValidators = ConfigUtils.getMValidatorsFromConfigurationClass(getDriverJobConfigurationClass()); + // Register driver in repository - mDriver = RepositoryManager.getInstance().getRepository().registerDriver(mDriver, autoUpgrade); + mDriver = RepositoryManager.getInstance().getRepository().registerDriver( + new MDriver(new MDriverConfig(driverConfig, mValidators), + DriverBean.CURRENT_DRIVER_VERSION), autoUpgrade); SqoopConfiguration.getInstance().getProvider().registerListener(new CoreConfigurationListener(this)); LOG.info("Driver initialized: OK"); } - public synchronized void destroy() { + public synchronized void destroy() { LOG.trace("Begin Driver destroy"); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java index 44ab41b..b665d79 100644 --- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java +++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java @@ -662,20 +662,20 @@ public class JdbcRepository extends Repository { * {@inheritDoc} */ @Override - public MMasterKey getMasterKey() { + public MMasterKey getMasterKey(RepositoryTransaction txn) { return (MMasterKey) doWithConnection(new DoWithConnection() { @Override public Object doIt(Connection conn) throws Exception { return handler.getMasterKey(conn); } - }); + }, (JdbcRepositoryTransaction) txn); } /** * {@inheritDoc} */ @Override - public void createMasterKey(final MMasterKey mMasterKey) { + public void createMasterKey(final MMasterKey mMasterKey, RepositoryTransaction txn) { doWithConnection(new DoWithConnection() { @Override public Object doIt(Connection conn) { @@ -685,7 +685,35 @@ public class JdbcRepository extends Repository { handler.createMasterKey(mMasterKey, conn); return null; } - }); + }, (JdbcRepositoryTransaction) txn); + } + + /** + * {@inheritDoc} + */ + @Override + public void deleteMasterKey(final long masterKeyId, RepositoryTransaction txn) { + doWithConnection(new DoWithConnection() { + @Override + public Object doIt(Connection conn) { + handler.deleteMasterKey(masterKeyId, conn); + return null; + } + }, (JdbcRepositoryTransaction) txn); + } + + /** + * {@inheritDoc} + */ + @Override + public void changeMasterKeyManager(final MasterKeyManager fromMasterKeyManager, final MasterKeyManager toMasterKeyManager, RepositoryTransaction txn) { + doWithConnection(new DoWithConnection() { + @Override + public Object doIt(Connection conn) { + handler.changeMasterKeyManager(fromMasterKeyManager, toMasterKeyManager, conn); + return null; + } + }, (JdbcRepositoryTransaction) txn); } @Override http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java index 94f5e6f..efd8806 100644 --- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java +++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java @@ -477,9 +477,28 @@ public abstract class JdbcRepositoryHandler { /** * Create the master key in the database * - * @param mMasterKey MMasterKey representing the Master Key + * @param mMasterKey MMasterKey representing the Master Key, this method populates + * its persistence id * @param conn Connection to the repository */ public abstract void createMasterKey(MMasterKey mMasterKey, Connection conn); + /** + * Delete the master key from the database + * + * @param conn Connection to the repository + */ + public abstract void deleteMasterKey(long masterKeyId, Connection conn); + + /** + * Change MasterKeyManager for inputs + * + * @param fromMasterKeyManager The master key manager that currently represents + * the encryption strategy for the inputs + * @param toMasterKeyManager The master key manager that will represent the + * encryption strategy for the inputs + * @param conn Connection to the repository + */ + public abstract void changeMasterKeyManager(MasterKeyManager fromMasterKeyManager, MasterKeyManager toMasterKeyManager, Connection conn); + } http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java b/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java index df84d54..bf251e0 100644 --- a/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java +++ b/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java @@ -53,6 +53,9 @@ public class MasterKeyManager { private int pbkdf2Rounds; private int ivLength; + private RepositoryTransaction repositoryTransaction; + private MMasterKey mMasterKey; + private SecretKey masterEncryptionKey; private SecretKey masterHmacKey; @@ -64,7 +67,7 @@ public class MasterKeyManager { instance = new MasterKeyManager(); } - private MasterKeyManager() { + public MasterKeyManager() { } public static MasterKeyManager getInstance() { @@ -79,128 +82,168 @@ public class MasterKeyManager { initialize(true); } - public synchronized void initialize(boolean createMasterKey) throws SqoopException { - // This is used for the generation of random initialization vectors and salts - random = new SecureRandom(); + public void initialize(boolean createMasterKey) throws SqoopException { + initialize(createMasterKey, false, null); + } + public void initialize(boolean createMasterKey, boolean createKeyEvenIfKeyExists, RepositoryTransaction repositoryTransactionArg) throws SqoopException { MapContext configurationContext = SqoopConfiguration.getInstance().getContext(); if (configurationContext.getBoolean(SecurityConstants.REPO_ENCRYPTION_ENABLED, false)) { // Grab configuration from the sqoop properties file. All of this configuration is required // and an exception will be thrown if any of it is missing - hmacAlgorithm = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM); - cipherAlgorithm = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM); - cipherSpec = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC); - cipherKeySize = populateIntConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE); - ivLength = populateIntConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE); - pbkdf2Algorithm = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM); - pbkdf2Rounds = populateIntConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS); - - // The size of the hmac key can be derived from the provided HMAC algorithm - try { - hmacKeySizeBytes = Mac.getInstance(hmacAlgorithm).getMacLength(); - } catch (NoSuchAlgorithmException e) { - throw new SqoopException(SecurityError.ENCRYPTION_0011, e); - } - - Repository repository = RepositoryManager.getInstance().getRepository(); - String password = PasswordUtils.readPassword(configurationContext, SecurityConstants.REPO_ENCRYPTION_PASSWORD, + String hmacAlgorithm = populateStringConfiguration(configurationContext, + SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM); + String cipherAlgorithm = populateStringConfiguration(configurationContext, + SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM); + String cipherSpec = populateStringConfiguration(configurationContext, + SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC); + int cipherKeySize = populateIntConfiguration(configurationContext, + SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE); + int ivLength = populateIntConfiguration(configurationContext, + SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE); + String pbkdf2Algorithm = populateStringConfiguration(configurationContext, + SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM); + int pbkdf2Rounds = populateIntConfiguration(configurationContext, + SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS); + + String password = PasswordUtils.readPassword(configurationContext, + SecurityConstants.REPO_ENCRYPTION_PASSWORD, SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR); - if (StringUtils.isEmpty(password)) { - throw new SqoopException(SecurityError.ENCRYPTION_0008); - } - MMasterKey existingEncryptedMasterKey = repository.getMasterKey(); - String salt; + initialize(createMasterKey, hmacAlgorithm, cipherAlgorithm, cipherSpec, cipherKeySize, + ivLength, pbkdf2Algorithm, pbkdf2Rounds, password, createKeyEvenIfKeyExists, repositoryTransactionArg); + } + } - if (existingEncryptedMasterKey == null) { - // Since the master key does not exist, we can generate a random salt that we will use - // for encryption of the Master Key - // We will use a salt that is the same size as the encryption key - salt = Base64.encodeBase64String(generateRandomByteArray(hmacKeySizeBytes)); - } else { - // Since the master key already exists, we will read the salt from the repository - salt = existingEncryptedMasterKey.getSalt(); - } - // Derive two keys (that we will be used to encrypt and verify the master key) - // from the configuration provided password and the salt we just read/created. - byte[] keyBytes = getKeysFromPassword(password, salt); - SecretKey passwordEncryptionKey = new SecretKeySpec(keyBytes, 0, - cipherKeySize, cipherAlgorithm); - SecretKey passwordHmacKey = new SecretKeySpec(keyBytes, - cipherKeySize, hmacKeySizeBytes, hmacAlgorithm); - - byte[] masterEncryptionKeyBytes; - byte[] masterHmacKeyBytes; - if (existingEncryptedMasterKey == null) { - if (createMasterKey) { - // A master key does not exist so we must create one. We will simply - // use two random byte arrays for the encryption and hmac components. - // The sizes of these keys is determined by the values provided to - // configuration. - masterEncryptionKeyBytes = generateRandomByteArray(cipherKeySize); - masterHmacKeyBytes = generateRandomByteArray(hmacKeySizeBytes); - - // The initialization vector for the encryption of the master key is - // randomly generated. - String iv = Base64.encodeBase64String(generateRandomByteArray(ivLength)); - - // We append our two keys together and encrypt the resulting byte array. - // This is the secret that all of the encryption in the repository depends upon - byte[] secret = ArrayUtils.addAll(masterEncryptionKeyBytes, masterHmacKeyBytes); - String encryptedSecret = encryptToString(passwordEncryptionKey, secret, iv); - - // We store our new master key in the repository in its encrypted form - // along with an HMAC to verify the key when we read it, the salt needed to - // generate keys to decrypt it, and the initialization vector used - repository.createMasterKey(new MMasterKey(encryptedSecret, generateHmac(passwordHmacKey, - encryptedSecret), salt, iv)); - } else { - // If a master key does not exist and we are trying to initialize the - // manager without allowing it to create a master key, we should fail - throw new SqoopException(SecurityError.ENCRYPTION_0002); - } + public synchronized void initialize(boolean createMasterKey, String hmacAlgorithmArg, + String cipherAlgorithmArg, String cipherSpecArg, + int cipherKeySizeArg, int ivLengthArg, + String pbkdf2AlgorithmArg, int pbkdf2RoundsArg, + String password, boolean createKeyEvenIfKeyExists, RepositoryTransaction repositoryTransactionArg) throws SqoopException { + hmacAlgorithm = hmacAlgorithmArg; + cipherAlgorithm = cipherAlgorithmArg; + cipherSpec = cipherSpecArg; + cipherKeySize = cipherKeySizeArg; + ivLength = ivLengthArg; + pbkdf2Algorithm = pbkdf2AlgorithmArg; + pbkdf2Rounds = pbkdf2RoundsArg; + + repositoryTransaction = repositoryTransactionArg; + + // This is used for the generation of random initialization vectors and salts + random = new SecureRandom(); + // The size of the hmac key can be derived from the provided HMAC algorithm + try { + hmacKeySizeBytes = Mac.getInstance(hmacAlgorithm).getMacLength(); + } catch (NoSuchAlgorithmException e) { + throw new SqoopException(SecurityError.ENCRYPTION_0011, e); + } + + Repository repository = RepositoryManager.getInstance().getRepository(); + if (StringUtils.isEmpty(password)) { + throw new SqoopException(SecurityError.ENCRYPTION_0008); + } + + MMasterKey existingEncryptedMasterKey = repository.getMasterKey(repositoryTransaction); + String salt; + + if (existingEncryptedMasterKey == null || createKeyEvenIfKeyExists) { + // Since the master key does not exist, we can generate a random salt that we will use + // for encryption of the Master Key + // We will use a salt that is the same size as the encryption key + salt = Base64.encodeBase64String(generateRandomByteArray(hmacKeySizeBytes)); + } else { + // Since the master key already exists, we will read the salt from the repository + salt = existingEncryptedMasterKey.getSalt(); + } + + // Derive two keys (that we will be used to encrypt and verify the master key) + // from the configuration provided password and the salt we just read/created. + byte[] keyBytes = getKeysFromPassword(password, salt); + SecretKey passwordEncryptionKey = new SecretKeySpec(keyBytes, 0, + cipherKeySize, cipherAlgorithm); + SecretKey passwordHmacKey = new SecretKeySpec(keyBytes, + cipherKeySize, hmacKeySizeBytes, hmacAlgorithm); + + byte[] masterEncryptionKeyBytes; + byte[] masterHmacKeyBytes; + if (existingEncryptedMasterKey == null || createKeyEvenIfKeyExists) { + if (createMasterKey) { + // A master key does not exist so we must create one. We will simply + // use two random byte arrays for the encryption and hmac components. + // The sizes of these keys is determined by the values provided to + // configuration. + masterEncryptionKeyBytes = generateRandomByteArray(cipherKeySize); + masterHmacKeyBytes = generateRandomByteArray(hmacKeySizeBytes); + + // The initialization vector for the encryption of the master key is + // randomly generated. + String iv = Base64.encodeBase64String(generateRandomByteArray(ivLength)); + + // We append our two keys together and encrypt the resulting byte array. + // This is the secret that all of the encryption in the repository depends upon + byte[] secret = ArrayUtils.addAll(masterEncryptionKeyBytes, masterHmacKeyBytes); + String encryptedSecret = encryptToString(passwordEncryptionKey, secret, iv); + + // We store our new master key in the repository in its encrypted form + // along with an HMAC to verify the key when we read it, the salt needed to + // generate keys to decrypt it, and the initialization vector used + mMasterKey = new MMasterKey(encryptedSecret, generateHmac(passwordHmacKey, encryptedSecret), salt, iv); + repository.createMasterKey(mMasterKey, repositoryTransaction); } else { - // A master key exists so we need to read it from the repository and - // decrypt it. - String iv = existingEncryptedMasterKey.getIv(); - String encryptedSecret = existingEncryptedMasterKey.getEncryptedSecret(); - - // Before we go about decrypting the master key we should verify the hmac - // to ensure that it has not been tampered with - String hmac = existingEncryptedMasterKey.getHmac(); - if (!validHmac(passwordHmacKey, encryptedSecret, hmac)) { - throw new SqoopException(SecurityError.ENCRYPTION_0001); - } - - // The master key has not been tampered with, lets decrypt it using the key - // derived from the password and the initialization vector from the repository - byte[] decryptedKey = decryptToBytes(passwordEncryptionKey, encryptedSecret, iv); - - // Since the master key is stored as the concatenation of an encryption - // key and an hmac key, we need to split it according to the sizes derived - // from the configuration - masterEncryptionKeyBytes = new byte[cipherKeySize]; - masterHmacKeyBytes = new byte[hmacKeySizeBytes]; - System.arraycopy(decryptedKey, 0, masterEncryptionKeyBytes, 0, - cipherKeySize); - System.arraycopy(decryptedKey, cipherKeySize, - masterHmacKeyBytes, 0, hmacKeySizeBytes); + // If a master key does not exist and we are trying to initialize the + // manager without allowing it to create a master key, we should fail + throw new SqoopException(SecurityError.ENCRYPTION_0002); + } + } else { + // A master key exists so we need to read it from the repository and + // decrypt it. + mMasterKey = existingEncryptedMasterKey; + String iv = existingEncryptedMasterKey.getIv(); + String encryptedSecret = existingEncryptedMasterKey.getEncryptedSecret(); + + // Before we go about decrypting the master key we should verify the hmac + // to ensure that it has not been tampered with + String hmac = existingEncryptedMasterKey.getHmac(); + if (!validHmac(passwordHmacKey, encryptedSecret, hmac)) { + throw new SqoopException(SecurityError.ENCRYPTION_0001); } - // Place the master encryption and master hmac key in SecretKey objects - // so we can use them to encrypt and decrypt data - masterEncryptionKey = new SecretKeySpec(masterEncryptionKeyBytes, 0, cipherKeySize, cipherAlgorithm); - masterHmacKey = new SecretKeySpec(masterHmacKeyBytes, 0, hmacKeySizeBytes, hmacAlgorithm); + // The master key has not been tampered with, lets decrypt it using the key + // derived from the password and the initialization vector from the repository + byte[] decryptedKey = decryptToBytes(passwordEncryptionKey, encryptedSecret, iv); + + // Since the master key is stored as the concatenation of an encryption + // key and an hmac key, we need to split it according to the sizes derived + // from the configuration + masterEncryptionKeyBytes = new byte[cipherKeySize]; + masterHmacKeyBytes = new byte[hmacKeySizeBytes]; + System.arraycopy(decryptedKey, 0, masterEncryptionKeyBytes, 0, + cipherKeySize); + System.arraycopy(decryptedKey, cipherKeySize, + masterHmacKeyBytes, 0, hmacKeySizeBytes); } + + // Place the master encryption and master hmac key in SecretKey objects + // so we can use them to encrypt and decrypt data + masterEncryptionKey = new SecretKeySpec(masterEncryptionKeyBytes, 0, cipherKeySize, cipherAlgorithm); + masterHmacKey = new SecretKeySpec(masterHmacKeyBytes, 0, hmacKeySizeBytes, hmacAlgorithm); } public synchronized void destroy() { hmacAlgorithm = null; + hmacKeySizeBytes = 0; cipherAlgorithm = null; + cipherKeySize = 0; cipherSpec = null; pbkdf2Algorithm = null; + pbkdf2Rounds = 0; + ivLength = 0; + + repositoryTransaction = null; + mMasterKey = null; masterEncryptionKey = null; masterHmacKey = null; @@ -208,6 +251,10 @@ public class MasterKeyManager { random = null; } + public void deleteMasterKeyFromRepository() { + RepositoryManager.getInstance().getRepository().deleteMasterKey(mMasterKey.getPersistenceId(), repositoryTransaction); + } + /** * Returns a Base64 representation of the encrypted cleartext, using the Master Key * http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/repository/Repository.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java index c2e3c74..ff02f3a 100644 --- a/core/src/main/java/org/apache/sqoop/repository/Repository.java +++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java @@ -33,7 +33,6 @@ import org.apache.sqoop.driver.Driver; import org.apache.sqoop.driver.DriverUpgrader; import org.apache.sqoop.json.DriverBean; import org.apache.sqoop.model.ConfigUtils; -import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MConfigList; import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MDriver; @@ -332,19 +331,40 @@ public abstract class Repository { /** * Get the encrypted master key from the repository + * There should only be one unless the repository encryption tool is in use * * @return The encrypted master key, null if no master key exists */ - public abstract MMasterKey getMasterKey(); + public abstract MMasterKey getMasterKey(RepositoryTransaction tx); /** * Create the encrypted master key in the repository * - * @param mMasterKey The encrypted master key + * @param mMasterKey The encrypted master key. Its persistenceId will be populated */ - public abstract void createMasterKey(MMasterKey mMasterKey); + public abstract void createMasterKey(MMasterKey mMasterKey, RepositoryTransaction tx); + /** + * Delete the master key record for the given id + * + * @param masterKeyId id of the master key to delete + * @param tx Transaction to perform the change within + */ + public abstract void deleteMasterKey(long masterKeyId, RepositoryTransaction tx); + + /** + * Change MasterKeyManager for inputs + * + * @param fromMasterKeyManager The master key manager that currently represents + * the encryption strategy for the inputs. + * null if the inputs are not currently encrypted + * @param toMasterKeyManager The master key manager that will represent the + * encryption strategy for the inputs. + * null if we would like to decrypt the inputs + * @param tx Transaction to perform the change within + */ + public abstract void changeMasterKeyManager(MasterKeyManager fromMasterKeyManager, MasterKeyManager toMasterKeyManager, RepositoryTransaction tx); /*********************Configurable Upgrade APIs ******************************/ http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/security/SecurityError.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/security/SecurityError.java b/core/src/main/java/org/apache/sqoop/security/SecurityError.java index 2ba849c..fc7b6b2 100644 --- a/core/src/main/java/org/apache/sqoop/security/SecurityError.java +++ b/core/src/main/java/org/apache/sqoop/security/SecurityError.java @@ -102,6 +102,12 @@ public enum SecurityError implements ErrorCode { /** The HMAC algorithm specified in configuration could not be found */ ENCRYPTION_0011("HMAC algorithm not found"), + /** An error occured when running the repository encryption tool */ + ENCRYPTION_0012("Failed to run repository encryption tool"), + + /** An error occured when running the repository encryption tool */ + ENCRYPTION_0013("Repository is encrypted, need configuration to decrypt"), + ; http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java b/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java index f9579bf..9e368a7 100644 --- a/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java +++ b/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java @@ -81,7 +81,7 @@ public class TestMasterKeyManager { jdbcRepoMock = mock(JdbcRepository.class); - when(jdbcRepoMock.getMasterKey()).thenReturn(null); + when(jdbcRepoMock.getMasterKey(null)).thenReturn(null); when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock); masterKeyManager = MasterKeyManager.getInstance(); @@ -106,8 +106,7 @@ public class TestMasterKeyManager { ArgumentCaptor<MMasterKey> mMasterKeyArgumentCaptor = ArgumentCaptor .forClass(MMasterKey.class); - verify(jdbcRepoMock, times(1)).createMasterKey(mMasterKeyArgumentCaptor - .capture()); + verify(jdbcRepoMock, times(1)).createMasterKey(mMasterKeyArgumentCaptor.capture(), any(RepositoryTransaction.class)); // Encrypt something with that master key String secret = "imasecret"; @@ -120,12 +119,12 @@ public class TestMasterKeyManager { // Create a new MasterKeyManager instance with existing master key // coming from the "db" jdbcRepoMock = mock(JdbcRepository.class); - when(jdbcRepoMock.getMasterKey()).thenReturn(mMasterKeyArgumentCaptor + when(jdbcRepoMock.getMasterKey(null)).thenReturn(mMasterKeyArgumentCaptor .getValue()); when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock); masterKeyManager.initialize(); - verify(jdbcRepoMock, times(1)).getMasterKey(); + verify(jdbcRepoMock, times(1)).getMasterKey(null); // Try to decrypt assertEquals(masterKeyManager.decryptWithMasterKey(encrypted, iv, masterKeyManager.generateHmacWithMasterHmacKey(encrypted)), secret); @@ -135,7 +134,7 @@ public class TestMasterKeyManager { public void testInitializeWithKeyCreationWithoutExistingKey() { masterKeyManager.initialize(); - verify(jdbcRepoMock, times(1)).createMasterKey(any(MMasterKey.class)); + verify(jdbcRepoMock, times(1)).createMasterKey(any(MMasterKey.class), any(RepositoryTransaction.class)); } @Test( @@ -144,7 +143,7 @@ public class TestMasterKeyManager { ) public void testMasterKeyWithInvalidHmac() { jdbcRepoMock = mock(JdbcRepository.class); - when(jdbcRepoMock.getMasterKey()).thenReturn(new MMasterKey( + when(jdbcRepoMock.getMasterKey(null)).thenReturn(new MMasterKey( Base64.encodeBase64String(generateRandomByteArray(CIPHER_KEY_SIZE_BYTES)), Base64.encodeBase64String(generateRandomByteArray(HMAC_KEY_SIZE_BYTES)), Base64.encodeBase64String(generateRandomByteArray(CIPHER_KEY_SIZE_BYTES)), http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/dist/src/main/conf/sqoop.properties ---------------------------------------------------------------------- diff --git a/dist/src/main/conf/sqoop.properties b/dist/src/main/conf/sqoop.properties index 58b60fd..d5aa0c3 100755 --- a/dist/src/main/conf/sqoop.properties +++ b/dist/src/main/conf/sqoop.properties @@ -197,9 +197,9 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce #org.apache.sqoop.security.repo_encryption.password_generator= #org.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 #org.apache.sqoop.security.repo_encryption.cipher_algorithm=AES -#org.apache.sqoop.security.repo_encryption.cipher_key_size=128 +#org.apache.sqoop.security.repo_encryption.cipher_key_size=16 #org.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding -#org.apache.sqoop.security.repo_encryption.initialization_vector_size=128 +#org.apache.sqoop.security.repo_encryption.initialization_vector_size=16 #org.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 #org.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/docs/src/site/sphinx/security/Encryption.rst ---------------------------------------------------------------------- diff --git a/docs/src/site/sphinx/security/Encryption.rst b/docs/src/site/sphinx/security/Encryption.rst index 6471a07..45ffc17 100644 --- a/docs/src/site/sphinx/security/Encryption.rst +++ b/docs/src/site/sphinx/security/Encryption.rst @@ -14,9 +14,9 @@ limitations under the License. -========== -Encryption -========== +=========== +API TLS/SSL +=========== Sqoop 2 offers an HTTP REST-like API as the mechanism by which clients can communicate with the Sqoop 2 server. The Sqoop 2 server and the Sqoop 2 shell @@ -73,7 +73,7 @@ is configured. org.apache.sqoop.security.tls.keymanager_password=echo keymanagerpassword Client/Shell Configuration --------------------------- +========================== When using TLS on the Sqoop 2 server, especially with a self-signed certificate, it may be useful to specify a truststore for the client/shell to use. @@ -99,4 +99,212 @@ You may also use a password generator. :: sqoop:000> set truststore --truststore /Users/abefine/keystore/node2.truststore --truststore-password-generator "echo changeme" - Truststore set successfully \ No newline at end of file + Truststore set successfully + +===================== +Repository Encryption +===================== + +Sqoop 2 uses a database to store metadata about the various data sources it talks to, we call this database the repository. + +The repository can store passwords and other pieces of information that are security sensitive, within the context of Sqoop +2, this information is referred to as sensitive inputs. Which inputs are considered sensitive is determined by the connector. + +We support encrypting sensitive inputs in the repository using a provided password or password generator. Sqoop 2 uses the +provided password and the provided key generation algorithm (such as PBKDF2) to generate a key to encrypt sensitive inputs +and another hmac key to verify their integrity. + +Only the sensitive inputs are encrypted. If an input is not defined as sensitive by the connector, it is NOT encrypted. + +Server Configuration +===================== + +Note: This configuration will allow a new Sqoop instance to encrypt information or read from an already encrypted repository. +It will not encrypt sensitive inputs in an existing repository. Please see below for instructions on how to encrypt an existing repository. + +First, repository encryption must be enabled. +:: + + org.apache.sqoop.security.repo_encryption.enabled=true + +Then we configure the password: + +:: + + org.apache.sqoop.security.repo_encryption.password=supersecret + +Or the password generator: + +:: + + org.apache.sqoop.security.repo_encryption.password_generator=echo supersecret + +The plaintext password is always given preference to the password generator if both are present. + +Then we can configure the HMAC algorithm. Please find the list of possibilities here: +`Standard Algorithm Name Documentation - Mac <http://docs.oracle.com/javase/7/docs/technotes/guides/security/StandardNames.html#Mac>`_ +We can store digests with up to 1024 bits. + +:: + + org.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 + +Then we configure the cipher algorithm. Possibilities can be found here: +`Standard Algorithm Name Documentation - Cipher <http://docs.oracle.com/javase/7/docs/technotes/guides/security/StandardNames.html#Cipher>`_ + +:: + + org.apache.sqoop.security.repo_encryption.cipher_algorithm=AES + +Then we configure the key size for the cipher in bytes. We can store up to 1024 bit keys. + +:: + + org.apache.sqoop.security.repo_encryption.cipher_key_size=16 + +Next we need to specify the cipher transformation. The options for this field are listed here: +`Cipher (Java Platform SE 7) <http://docs.oracle.com/javase/7/docs/api/javax/crypto/Cipher.html>`_ + +:: + + org.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding + +The size of the initialization vector to use in bytes. We support up to 1024 bit initialization vectors. + +:: + + org.apache.sqoop.security.repo_encryption.initialization_vector_size=16 + +Next we need to specfy the algorithm for secret key generation. Please refer to: +`Standard Algorithm Name Documentation - SecretKeyFactory <http://docs.oracle.com/javase/7/docs/technotes/guides/security/StandardNames.html#SecretKeyFactory>`_ + +:: + + org.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 + +Finally specify the number of rounds/iterations for the generation of a key from a password. + +:: + + org.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 + +Repository Encryption Tool +========================== + +Sometimes we may want to change the password that is used to encrypt our data, generate a new key for our existing password, +encrypt an existing unencrypted repository, or decrypt an existing encrypting repository. Sqoop 2 provides the +Repository Encryption Tool to allow us to do this. + +Before using the tool it is important to shut down the Sqoop 2 server. + +All changes that the tool makes occur in a single transaction with the repository, which should prevent leaving the +repository in a bad state. + +The Repository Encryption Tool is very simple, it uses the exact same configuration specified above (with the exception +of ``useConf``). Configuration prefixed with a "-F" represents the existing repository state, configuration prefixed with +a "-T" represents the desired repository state. If one of these configuration sets is left out that means unencrypted. + +Changing the Password +--------------------- + +In order to change the password, we need to specify the current configuration with the existing password and the desired +configuration with the new password. It looks like this: + +:: + + sqoop.sh tool repositoryencryption \ + -Forg.apache.sqoop.security.repo_encryption.password=old_password \ + -Forg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \ + -Forg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \ + -Forg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \ + -Forg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \ + -Forg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \ + -Forg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \ + -Forg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 \ + -Torg.apache.sqoop.security.repo_encryption.password=new_password \ + -Torg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \ + -Torg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \ + -Torg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \ + -Torg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \ + -Torg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \ + -Torg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \ + -Torg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 + +Generate a New Key for the Existing Password +-------------------------------------------- + +Just like with the previous scenario you could copy the same configuration twice like this: + +:: + + sqoop.sh tool repositoryencryption \ + -Forg.apache.sqoop.security.repo_encryption.password=password \ + -Forg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \ + -Forg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \ + -Forg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \ + -Forg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \ + -Forg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \ + -Forg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \ + -Forg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 \ + -Torg.apache.sqoop.security.repo_encryption.password=password \ + -Torg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \ + -Torg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \ + -Torg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \ + -Torg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \ + -Torg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \ + -Torg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \ + -Torg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 + +But we do have a shortcut to make this easier: + +:: + + sqoop.sh tool repositoryencryption -FuseConf -TuseConf + +The ``useConf`` option will read whatever configuration is already in the configured sqoop properties file and apply it +for the specified direction. + +Encrypting an Existing Unencrypted Repository +--------------------------------------------- + +:: + + sqoop.sh tool repositoryencryption \ + -Torg.apache.sqoop.security.repo_encryption.password=password \ + -Torg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \ + -Torg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \ + -Torg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \ + -Torg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \ + -Torg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \ + -Torg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \ + -Torg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 + +If the configuration for the encrypted repository has already been written to the sqoop properties file, one can simply +execute: + +:: + + sqoop.sh tool repositoryencryption -TuseConf + + +Decrypting an Existing Encrypted Repository +------------------------------------------- + +:: + + sqoop.sh tool repositoryencryption \ + -Forg.apache.sqoop.security.repo_encryption.password=password \ + -Forg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \ + -Forg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \ + -Forg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \ + -Forg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \ + -Forg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \ + -Forg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \ + -Forg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 + +If the configuration for the encrypted repository has not yet been removed from the sqoop properties file, one can simply +execute: + +:: + + sqoop.sh tool repositoryencryption -FuseConf http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java index a9168ae..4cae811 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java @@ -1164,12 +1164,15 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { return null; } - String aesKey = rs.getString(1); - String hmac = rs.getString(2); - String salt = rs.getString(3); - String iv = rs.getString(4); + String aesKey = rs.getString(2); + String hmac = rs.getString(3); + String salt = rs.getString(4); + String iv = rs.getString(5); - return new MMasterKey(aesKey, hmac, salt, iv); + MMasterKey mMasterKey = new MMasterKey(aesKey, hmac, salt, iv); + mMasterKey.setPersistenceId(rs.getLong(1)); + + return mMasterKey; } } catch (SQLException ex) { logException(ex); @@ -1183,8 +1186,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { @Override public void createMasterKey(MMasterKey mMasterKey, Connection conn) { int result; - try (PreparedStatement preparedStatement = conn.prepareStatement(crudQueries.getStmtInsertSqMasterKey(), - Statement.RETURN_GENERATED_KEYS)) { + try (PreparedStatement preparedStatement = conn.prepareStatement(crudQueries.getStmtInsertSqMasterKey(), Statement.RETURN_GENERATED_KEYS)) { preparedStatement.setString(1, mMasterKey.getEncryptedSecret()); preparedStatement.setString(2, mMasterKey.getHmac()); preparedStatement.setString(3, mMasterKey.getSalt()); @@ -1195,12 +1197,60 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { throw new SqoopException(CommonRepositoryError.COMMON_0009, Integer.toString(result)); } + + long masterKeyId = -1; + try(ResultSet primaryKeyResultSet = preparedStatement.getGeneratedKeys()) { + if (primaryKeyResultSet.next()) { + masterKeyId = primaryKeyResultSet.getLong(1); + } + } + + if (masterKeyId < 0) { + throw new SqoopException(CommonRepositoryError.COMMON_0062); + } + + mMasterKey.setPersistenceId(masterKeyId); + } catch (SQLException ex) { logException(ex, mMasterKey); throw new SqoopException(CommonRepositoryError.COMMON_0031, ex); } } + + /** + * {@inheritDoc} + */ + @Override + public void deleteMasterKey(long masterKeyId, Connection conn) { + try (PreparedStatement deleteMasterKeyPreparedStatement = conn.prepareStatement(crudQueries.getDeleteSqMasterKey())) { + deleteMasterKeyPreparedStatement.setLong(1, masterKeyId); + deleteMasterKeyPreparedStatement.executeUpdate(); + } catch (SQLException ex) { + logException(ex); + throw new SqoopException(CommonRepositoryError.COMMON_0061, ex); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void changeMasterKeyManager(MasterKeyManager fromMasterKeyManager, MasterKeyManager toMasterKeyManager, Connection conn) { + try ( + PreparedStatement fetchLinkInputs = conn.prepareStatement(crudQueries.getStmtFetchLinkInput()); + PreparedStatement updateLinkInput = conn.prepareStatement(crudQueries.getUpdateLinkInput()); + PreparedStatement fetchJobInputs = conn.prepareStatement(crudQueries.getStmtFetchJobInput()); + PreparedStatement updateJobInput = conn.prepareStatement(crudQueries.getUpdateJobInput()) + ) { + transitionInputs(fromMasterKeyManager, toMasterKeyManager, fetchLinkInputs, updateLinkInput); + transitionInputs(fromMasterKeyManager, toMasterKeyManager, fetchJobInputs, updateJobInput); + } catch (SQLException exception) { + logException(exception); + throw new SqoopException(CommonRepositoryError.COMMON_0060, exception); + } + } + private void insertConnectorDirection(Long connectorId, Direction direction, Connection conn) throws SQLException { try (PreparedStatement stmt = conn.prepareStatement(crudQueries.getStmtInsertSqConnectorDirections())) { @@ -1646,7 +1696,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { try (ResultSet rsConnection = stmt.executeQuery(); PreparedStatement connectorConfigFetchStatement = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable()); - PreparedStatement connectorConfigInputStatement = conn.prepareStatement(crudQueries.getStmtFetchLinkInput());) { + PreparedStatement connectorConfigInputStatement = conn.prepareStatement(crudQueries.getStmtFetchLinkInputByJob());) { while(rsConnection.next()) { long id = rsConnection.getLong(1); String name = rsConnection.getString(2); @@ -1691,7 +1741,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { try (ResultSet rsConnection = stmt.executeQuery(); PreparedStatement configStmt = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfiguration()); - PreparedStatement inputStmt = conn.prepareStatement(crudQueries.getStmtFetchLinkInput()); + PreparedStatement inputStmt = conn.prepareStatement(crudQueries.getStmtFetchLinkInputByJob()); ) { while(rsConnection.next()) { long id = rsConnection.getLong(1); @@ -1734,7 +1784,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { PreparedStatement fromConfigFetchStmt = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable()); PreparedStatement toConfigFetchStmt = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable()); PreparedStatement driverConfigfetchStmt = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable()); - PreparedStatement jobInputFetchStmt = conn.prepareStatement(crudQueries.getStmtFetchJobInput());) { + PreparedStatement jobInputFetchStmt = conn.prepareStatement(crudQueries.getStmtFetchJobInputByJob());) { // Note: Job does not hold a explicit reference to the driver since every // job has the same driver @@ -1813,9 +1863,9 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { (crudQueries.getStmtSelectConfigForConfigurable()); PreparedStatement configStmt = conn.prepareStatement(crudQueries .getStmtSelectConfigForConfiguration()); - PreparedStatement jobInputFetchStmt = conn.prepareStatement(crudQueries.getStmtFetchJobInput()); + PreparedStatement jobInputFetchStmt = conn.prepareStatement(crudQueries.getStmtFetchJobInputByJob()); PreparedStatement inputStmt = conn.prepareStatement(crudQueries - .getStmtFetchLinkInput()) + .getStmtFetchLinkInputByJob()) ) { // Note: Job does not hold a explicit reference to the driver since every @@ -2125,7 +2175,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { // get the overrides value from the SQ_INPUT_RELATION table String overrides = getOverrides(inputId, conn); String inputEnumValues = rsetInput.getString(9); - String value = readInputValue(rsetInput.getString(10), rsetInput.getBoolean(11), rsetInput.getString(12), rsetInput.getString(13)); + String value = readInputValue(MasterKeyManager.getInstance(), rsetInput.getString(10), rsetInput.getBoolean(11), rsetInput.getString(12), rsetInput.getString(13)); MInputType mit = MInputType.valueOf(inputType); MInput input = null; switch (mit) { @@ -2222,6 +2272,58 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { } } + private void transitionInputs(MasterKeyManager fromMasterKeyManager, MasterKeyManager toMasterKeyManager, + PreparedStatement selectInputsStatement, PreparedStatement updateInputsStatement) throws SQLException { + try (ResultSet inputs = selectInputsStatement.executeQuery()) { + while (inputs.next()) { + long inputId = inputs.getLong(1); + boolean encrypted = inputs.getBoolean(11); + boolean sensitive = inputs.getBoolean(6); + + if (encrypted) { + assert(fromMasterKeyManager != null); + // We need to decrypt the input first + String encryptedValue = inputs.getString(10); + String iv = inputs.getString(12); + String hmac = inputs.getString(13); + + String plainTextValue = readInputValue(fromMasterKeyManager, encryptedValue, encrypted, iv, hmac); + + if (toMasterKeyManager != null && sensitive) { + // We need to encrypt the input + String newIv = toMasterKeyManager.generateRandomIv(); + String encryptedInput = toMasterKeyManager.encryptWithMasterKey(plainTextValue, newIv); + + updateInputsStatement.setString(1, encryptedInput); + updateInputsStatement.setBoolean(2, true); + updateInputsStatement.setString(3, newIv); + updateInputsStatement.setString(4, toMasterKeyManager.generateHmacWithMasterHmacKey(encryptedInput)); + updateInputsStatement.setLong(5, inputId); + } else { + // Store the plaintext + updateInputsStatement.setString(1, plainTextValue); + updateInputsStatement.setBoolean(2, false); + updateInputsStatement.setNull(3, Types.VARCHAR); + updateInputsStatement.setNull(4, Types.VARCHAR); + updateInputsStatement.setLong(5, inputId); + } + updateInputsStatement.executeUpdate(); + } else if (toMasterKeyManager != null && sensitive) { + // We need to encrypt the input + String plainTextValue = inputs.getString(10); + String newIv = toMasterKeyManager.generateRandomIv(); + String encryptedInput = toMasterKeyManager.encryptWithMasterKey(plainTextValue, newIv); + + updateInputsStatement.setString(1, encryptedInput); + updateInputsStatement.setBoolean(2, true); + updateInputsStatement.setString(3, newIv); + updateInputsStatement.setString(4, toMasterKeyManager.generateHmacWithMasterHmacKey(encryptedInput)); + updateInputsStatement.setLong(5, inputId); + updateInputsStatement.executeUpdate(); + } + } + } + } private void loadInputsForConfigs(MConfigList mConfigList, PreparedStatement configStmt, PreparedStatement inputStmt) throws SQLException { for (MConfig mConfig : mConfigList.getConfigs()) { @@ -2237,7 +2339,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { while (inputResults.next()) { long inputId = inputResults.getLong(1); String inputName = inputResults.getString(2); - String value = readInputValue(inputResults.getString(10), inputResults.getBoolean(11), inputResults.getString(12), inputResults.getString(13)); + String value = readInputValue(MasterKeyManager.getInstance(), inputResults.getString(10), inputResults.getBoolean(11), inputResults.getString(12), inputResults.getString(13)); if (mConfig.getName().equals(configName) && mConfig.getInputNames().contains(inputName)) { MInput mInput = mConfig.getInput(inputName); mInput.setPersistenceId(inputId); @@ -2262,9 +2364,9 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { * @param hmac HMAC for tamper resistance * @return The input value */ - private String readInputValue(String possiblyEncryptedValue, boolean encrypted, String iv, String hmac) throws SqoopException { + private String readInputValue(MasterKeyManager masterKeyManager, String possiblyEncryptedValue, boolean encrypted, String iv, String hmac) throws SqoopException { if (encrypted) { - return MasterKeyManager.getInstance().decryptWithMasterKey(possiblyEncryptedValue, iv, hmac); + return masterKeyManager.decryptWithMasterKey(possiblyEncryptedValue, iv, hmac); } else { return possiblyEncryptedValue; } @@ -2318,7 +2420,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { // get the overrides value from the SQ_INPUT_RELATION table String overrides = getOverrides(inputId, conn); String inputEnumValues = rsetInput.getString(9); - String value = readInputValue(rsetInput.getString(10), rsetInput.getBoolean(11), rsetInput.getString(12), rsetInput.getString(13)); + String value = readInputValue(MasterKeyManager.getInstance(), rsetInput.getString(10), rsetInput.getBoolean(11), rsetInput.getString(12), rsetInput.getString(13)); MInputType mit = MInputType.valueOf(inputType); http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java index d527290..9118722 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java @@ -215,7 +215,7 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { * *******LINK INPUT TABLE ************* */ //DML: Get inputs and values for a given link - private static final String STMT_FETCH_LINK_INPUT = + private static final String STMT_FETCH_LINK_INPUT_BY_JOB = "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", " @@ -237,11 +237,38 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + " = ?" + " ORDER BY " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_INDEX); + private static final String STMT_FETCH_LINK_INPUT = + "SELECT " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_INDEX) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_TYPE) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRMASK) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + "," + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC) + + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + + " RIGHT OUTER JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + + " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + " = " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT); + + private static final String UPDATE_LINK_INPUT = + "UPDATE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + " SET " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + " = ?, " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + " = ?, " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + " = ?, " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC) + " = ? " + + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + " = ?"; + /** * *******JOB INPUT TABLE ************* */ //DML: Fetch inputs and values for a given job - private static final String STMT_FETCH_JOB_INPUT = + private static final String STMT_FETCH_JOB_INPUT_BY_JOB = "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", " @@ -263,6 +290,35 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + " = ?" + " ORDER BY " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_INDEX); + + private static final String STMT_FETCH_JOB_INPUT = + "SELECT " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_INDEX) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_TYPE) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRMASK) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC) + + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + + " RIGHT OUTER JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + + " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + " = " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + + " ORDER BY " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_INDEX); + + private static final String UPDATE_JOB_INPUT = + "UPDATE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + " SET " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + " = ?, " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + " = ?, " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + " = ?, " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC) + " = ? " + + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + " = ?"; + /** * *******LINK TABLE ************* */ @@ -758,7 +814,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_CONFIGURABLE) + " = ?)"; private static final String STMT_SELECT_SQ_MASTER_KEY = - "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + ", " + "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_ID) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV) @@ -772,6 +829,10 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV) + ") VALUES (?, ?, ?, ?)"; + private static final String DELETE_SQ_MASTER_KEY = + "DELETE FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME) + + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_ID) + " = ?"; + public String getStmtSelectSqdIdBySqdName() { return STMT_SELECT_SQD_ID_BY_SQD_NAME; @@ -821,14 +882,30 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { return STMT_INSERT_INTO_INPUT; } - public String getStmtFetchLinkInput() { - return STMT_FETCH_LINK_INPUT; + public String getStmtFetchLinkInputByJob() { + return STMT_FETCH_LINK_INPUT_BY_JOB; + } + + public String getUpdateLinkInput() { + return UPDATE_LINK_INPUT; + } + + public String getUpdateJobInput() { + return UPDATE_JOB_INPUT; } public String getStmtFetchJobInput() { return STMT_FETCH_JOB_INPUT; } + public String getStmtFetchJobInputByJob() { + return STMT_FETCH_JOB_INPUT_BY_JOB; + } + + public String getStmtFetchLinkInput() { + return STMT_FETCH_LINK_INPUT; + } + public String getStmtInsertLink() { return STMT_INSERT_LINK; } @@ -1082,4 +1159,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { public String getStmtInsertSqMasterKey() { return STMT_INSERT_SQ_MASTER_KEY; } + + public String getDeleteSqMasterKey() { + return DELETE_SQ_MASTER_KEY; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java index b4f66bb..81de886 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java @@ -193,7 +193,7 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*; * | SQ_LNKI_INPUT: BIGINT PK | FK SQ_INPUT(SQI_ID) * | SQ_LNKI_VALUE: LONG VARCHAR| * | SQ_LNKI_ENCRYPTED: BOOLEAN | - * | SQ_LNKI_IV: VARCHAR(171) | + * | SQ_LNKI_IV: VARCHAR(171) | * | SQ_LNKI_HMAC: VARCHAR(171) | * +----------------------------+ * </pre> http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/test/pom.xml ---------------------------------------------------------------------- diff --git a/test/pom.xml b/test/pom.xml index 4bac683..81e599d 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -185,6 +185,11 @@ limitations under the License. <artifactId>parquet-avro</artifactId> </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + </dependency> + </dependencies> <!-- Add classifier name to the JAR name --> http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryDumpLoadToolTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryDumpLoadToolTest.java b/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryDumpLoadToolTest.java index c1a10bc..c17c483 100644 --- a/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryDumpLoadToolTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryDumpLoadToolTest.java @@ -19,6 +19,7 @@ package org.apache.sqoop.integration.tools; import org.apache.commons.io.Charsets; import org.apache.commons.io.IOUtils; +import org.apache.sqoop.client.SqoopClient; import org.apache.sqoop.common.VersionInfo; import org.apache.sqoop.json.JSONUtils; import org.apache.sqoop.json.JobBean; @@ -28,16 +29,18 @@ import org.apache.sqoop.model.*; import org.apache.sqoop.submission.SubmissionStatus; import org.apache.sqoop.test.infrastructure.Infrastructure; import org.apache.sqoop.test.infrastructure.SqoopTestCase; +import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider; +import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider; import org.apache.sqoop.test.infrastructure.providers.SqoopInfrastructureProvider; +import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; import org.apache.sqoop.test.utils.HdfsUtils; import org.apache.sqoop.tools.tool.JSONConstants; import org.apache.sqoop.tools.tool.RepositoryDumpTool; import org.apache.sqoop.tools.tool.RepositoryLoadTool; import org.apache.sqoop.utils.UrlSafeUtils; import org.json.simple.JSONObject; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; +import org.testng.annotations.*; import java.io.*; import java.util.List; @@ -46,14 +49,24 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @Test(groups = "no-real-cluster") -@Infrastructure(dependencies = {KdcInfrastructureProvider.class, SqoopInfrastructureProvider.class}) +@Infrastructure(dependencies = {HadoopInfrastructureProvider.class}) public class RepositoryDumpLoadToolTest extends SqoopTestCase { + private JettySqoopMiniCluster jettySqoopMiniCluster; + private SqoopClient client; private String jsonFilePath; // do the load test and insert data to repo first, then do the dump test. - @Test(dependsOnMethods = { "testLoad" }) + @Test public void testDump() throws Exception { + // load data into repository + RepositoryLoadTool rlt = new RepositoryLoadTool(); + rlt.setInTest(true); + rlt.runToolWithConfiguration(new String[]{"-i", jsonFilePath}); + verifyLinks(client.getLinks()); + verifyJobs(client.getJobs()); + verifySubmissions(client.getSubmissions()); + // dump the repository RepositoryDumpTool rdt = new RepositoryDumpTool(); rdt.setInTest(true); @@ -87,16 +100,6 @@ public class RepositoryDumpLoadToolTest extends SqoopTestCase { } } - @Test - public void testLoad() throws Exception { - RepositoryLoadTool rlt = new RepositoryLoadTool(); - rlt.setInTest(true); - rlt.runToolWithConfiguration(new String[]{"-i", jsonFilePath}); - verifyLinks(getClient().getLinks()); - verifyJobs(getClient().getJobs()); - verifySubmissions(getClient().getSubmissions()); - } - private void verifyLinks(List<MLink> links) { for (MLink link : links) { String linkName = link.getName(); @@ -126,8 +129,15 @@ public class RepositoryDumpLoadToolTest extends SqoopTestCase { assertEquals(submission.getStatus(), SubmissionStatus.SUCCEEDED); } - // generate the json file without the license @BeforeMethod + private void startCluster() throws Exception { + jettySqoopMiniCluster = new JettySqoopMiniCluster(HdfsUtils.joinPathFragments(super.getTemporaryPath(), this.getClass().getName()), getHadoopConf()); + jettySqoopMiniCluster.start(); + client = new SqoopClient(jettySqoopMiniCluster.getServerUrl()); + } + + // generate the json file without the license + @BeforeMethod(dependsOnMethods = { "startCluster" }) public void prepareJsonFile() throws Exception { String testFilePath = getClass().getResource("/repoLoadToolTest.json").getPath(); jsonFilePath = HdfsUtils.joinPathFragments(getTemporaryPath(), "repoLoadTest.json"); @@ -140,7 +150,7 @@ public class RepositoryDumpLoadToolTest extends SqoopTestCase { // for hdfs connector, DirectoryExistsValidator is responsible for validation // replace the link config dir by the local path. if (line.indexOf("linkConfReplacement") > 0) { - line = line.replaceAll("linkConfReplacement", UrlSafeUtils.urlEncode(getSqoopMiniClusterTemporaryPath() + "/config/")); + line = line.replaceAll("linkConfReplacement", UrlSafeUtils.urlEncode(jettySqoopMiniCluster.getTemporaryPath() + "/config/")); } writer.write(line); } @@ -148,4 +158,9 @@ public class RepositoryDumpLoadToolTest extends SqoopTestCase { writer.flush(); } } + + @AfterMethod + private void stopCluster() throws Exception { + jettySqoopMiniCluster.stop(); + } }
