Updated Branches: refs/heads/trunk a555a1f31 -> ed73f885a
SQOOP-1223. Enhance the password file capability to enable plugging-in custom loaders. (Jarek Jarcec Cecho via Hari Shreedharan) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ed73f885 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ed73f885 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ed73f885 Branch: refs/heads/trunk Commit: ed73f885a1e6b526945072090e1a83e74fd3a23c Parents: a555a1f Author: Hari Shreedharan <[email protected]> Authored: Tue Nov 5 11:30:08 2013 -0800 Committer: Hari Shreedharan <[email protected]> Committed: Tue Nov 5 11:30:08 2013 -0800 ---------------------------------------------------------------------- src/java/org/apache/sqoop/SqoopOptions.java | 3 +- .../org/apache/sqoop/tool/BaseSqoopTool.java | 5 +- .../org/apache/sqoop/util/CredentialsUtil.java | 89 ++++++---- .../sqoop/util/password/CryptoFileLoader.java | 164 +++++++++++++++++++ .../sqoop/util/password/FilePasswordLoader.java | 89 ++++++++++ .../sqoop/util/password/PasswordLoader.java | 49 ++++++ .../credentials/TestPassingSecurePassword.java | 88 +++++++++- 7 files changed, 449 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/ed73f885/src/java/org/apache/sqoop/SqoopOptions.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index 13637b5..5c7a56a 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -613,8 +613,7 @@ public class SqoopOptions implements Cloneable { passwordFilePath = props.getProperty("db.password.file"); if (passwordFilePath != null) { try { - password = CredentialsUtil.fetchPasswordFromFile( - getConf(), passwordFilePath); + password = CredentialsUtil.fetchPasswordFromLoader(passwordFilePath, getConf()); return; // short-circuit } catch (IOException e) { throw new RuntimeException("Unable to fetch password from file.", e); http://git-wip-us.apache.org/repos/asf/sqoop/blob/ed73f885/src/java/org/apache/sqoop/tool/BaseSqoopTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 9230f82..018d11f 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -26,7 +26,6 @@ import java.sql.SQLException; import java.util.Arrays; import java.util.Properties; -import com.cloudera.sqoop.util.ImportException; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; @@ -878,7 +877,9 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { try { out.setPasswordFilePath(in.getOptionValue(PASSWORD_PATH_ARG)); // apply password from file into password in options - out.setPassword(CredentialsUtil.fetchPasswordFromFile(out)); + out.setPassword(CredentialsUtil.fetchPassword(out)); + // And allow the PasswordLoader to clean up any sensitive properties + CredentialsUtil.cleanUpSensitiveProperties(out.getConf()); } catch (IOException ex) { LOG.warn("Failed to load connection parameter file", ex); throw new InvalidOptionsException( http://git-wip-us.apache.org/repos/asf/sqoop/blob/ed73f885/src/java/org/apache/sqoop/util/CredentialsUtil.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/util/CredentialsUtil.java b/src/java/org/apache/sqoop/util/CredentialsUtil.java index 6412469..5b5375e 100644 --- a/src/java/org/apache/sqoop/util/CredentialsUtil.java +++ b/src/java/org/apache/sqoop/util/CredentialsUtil.java @@ -21,64 +21,91 @@ package org.apache.sqoop.util; import com.cloudera.sqoop.SqoopOptions; -import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; +import org.apache.sqoop.util.password.FilePasswordLoader; +import org.apache.sqoop.util.password.PasswordLoader; import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; /** * A utility class for fetching passwords from a file. */ public final class CredentialsUtil { - public static final Log LOG = LogFactory.getLog( - CredentialsUtil.class.getName()); + /** + * Property for specifying which loader should be used to fetch the password. + */ + private static String PROPERTY_LOADER_CLASSS = "org.apache.sqoop.credentials.loader.class"; + + /** + * The default loader is a FilePasswordLoader that will fetch the password from a file. + */ + private static String DEFAULT_PASSWORD_LOADER = FilePasswordLoader.class.getCanonicalName(); + + public static final Log LOG = LogFactory.getLog(CredentialsUtil.class.getName()); private CredentialsUtil() { } - public static String fetchPasswordFromFile(SqoopOptions options) + /** + * Return password that was specified (either on command line or via other facilities). + * + * @param options Sqoop Options + * @return Password + * @throws IOException + */ + public static String fetchPassword(SqoopOptions options) throws IOException { String passwordFilePath = options.getPasswordFilePath(); if (passwordFilePath == null) { return options.getPassword(); } - return fetchPasswordFromFile(options.getConf(), passwordFilePath); + return fetchPasswordFromLoader(options.getPasswordFilePath(), options.getConf()); } - public static String fetchPasswordFromFile(Configuration conf, - String passwordFilePath) - throws IOException { - LOG.debug("Fetching password from specified path: " + passwordFilePath); - Path path = new Path(passwordFilePath); - FileSystem fs = path.getFileSystem(conf); + /** + * Return password via --password-file argument. + * + * Given loader can be overridden using PROPERTY_LOADER_CLASSS. + * + * @param path Path with the password file. + * @param conf Configuration + * @return Password + * @throws IOException + */ + public static String fetchPasswordFromLoader(String path, Configuration conf) throws IOException { + PasswordLoader loader = getLoader(conf); + return loader.loadPassword(path, conf); + } - if (!fs.exists(path)) { - throw new IOException("The password file does not exist! " - + passwordFilePath); - } + /** + * Remove any potentially sensitive information from the configuration object. + * + * @param configuration Associated configuration object. + * @throws IOException + */ + public static void cleanUpSensitiveProperties(Configuration configuration) throws IOException { + PasswordLoader loader = getLoader(configuration); + loader.cleanUpConfiguration(configuration); + } - if (!fs.isFile(path)) { - throw new IOException("The password file cannot be a directory! " - + passwordFilePath); - } + /** + * Instantiate configured PasswordLoader class. + * + * @param configuration Associated configuration object. + * @return + * @throws IOException + */ + public static PasswordLoader getLoader(Configuration configuration) throws IOException { + String loaderClass = configuration.get(PROPERTY_LOADER_CLASSS, DEFAULT_PASSWORD_LOADER); - InputStream is = fs.open(path); - StringWriter writer = new StringWriter(); try { - IOUtils.copy(is, writer); - return writer.toString(); - } finally { - IOUtils.closeQuietly(is); - IOUtils.closeQuietly(writer); - fs.close(); + return (PasswordLoader) Class.forName(loaderClass).newInstance(); + } catch (Exception e) { + throw new IOException(e); } } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/ed73f885/src/java/org/apache/sqoop/util/password/CryptoFileLoader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/util/password/CryptoFileLoader.java b/src/java/org/apache/sqoop/util/password/CryptoFileLoader.java new file mode 100644 index 0000000..54d2cf5 --- /dev/null +++ b/src/java/org/apache/sqoop/util/password/CryptoFileLoader.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.util.password; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import javax.crypto.Cipher; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; + + +/** + * Example implementation of "advanced" file loader that will read password from + * encrypted file. Please note that this method is merely obfuscating the password, + * as malicious user will be able to retrieve the password if he intercepts the + * Sqoop commands. He won't be able to get it if he will just get the password + * file though. Current implementation is limited to ECB and is not supporting other + * methods. + * + * Example usage: + * sqoop import \ + * -Dorg.apache.sqoop.credentials.loader.class=org.apache.sqoop.util.password.CryptoFileLoader \ + * -Dorg.apache.sqoop.credentials.loader.crypto.passphrase=sqooppass \ + * --connect ... + */ +public class CryptoFileLoader extends FilePasswordLoader { + + /** + * Crypto algorithm that should be used. + * + * List of available ciphers is for example available here: + * http://docs.oracle.com/javase/7/docs/api/javax/crypto/Cipher.html + */ + private static String PROPERTY_CRYPTO_ALG = "org.apache.sqoop.credentials.loader.crypto.alg"; + + /** + * Salt that should be used. + * + * Some algorithms are requiring salt to be present. + */ + private static String PROPERTY_CRYPTO_SALT = "org.apache.sqoop.credentials.loader.crypto.salt"; + + /** + * Iterations argument for creating key. + */ + private static String PROPERTY_CRYPTO_ITERATIONS = "org.apache.sqoop.credentials.loader.crypto.iterations"; + + /** + * Length of the key (driven by the used algorithm). + */ + private static String PROPERTY_CRYPTO_KEY_LEN = "org.apache.sqoop.credentials.loader.crypto.salt.key.len"; + + /** + * Passphrase (encryption password). + */ + private static String PROPERTY_CRYPTO_PASSPHRASE = "org.apache.sqoop.credentials.loader.crypto.passphrase"; + + /** + * Default is AES in electronic code book with padding. + */ + private static String DEFAULT_ALG = "AES/ECB/PKCS5Padding"; + + /** + * Default salt is not much secure, use your own! + */ + private static String DEFAULT_SALT = "SALT"; + + /** + * Iterate 10000 times by default. + */ + private static int DEFAULT_ITERATIONS = 10000; + + /** + * One of valid key sizes for default algorithm (AES). + */ + private static int DEFAULT_KEY_LEN = 128; + + @Override + public String loadPassword(String p, Configuration configuration) throws IOException { + LOG.debug("Fetching password from specified path: " + p); + Path path = new Path(p); + FileSystem fs = path.getFileSystem(configuration); + + byte [] encrypted; + try { + verifyPath(fs, path); + encrypted = readBytes(fs, path); + } finally { + fs.close(); + } + + String passPhrase = configuration.get(PROPERTY_CRYPTO_PASSPHRASE); + if(passPhrase == null) { + throw new IOException("Passphrase is missing in property " + PROPERTY_CRYPTO_PASSPHRASE); + } + + String alg = configuration.get(PROPERTY_CRYPTO_ALG, DEFAULT_ALG); + String algOnly = alg.split("/")[0]; + String salt = configuration.get(PROPERTY_CRYPTO_SALT, DEFAULT_SALT); + int iterations = configuration.getInt(PROPERTY_CRYPTO_ITERATIONS, DEFAULT_ITERATIONS); + int keyLen = configuration.getInt(PROPERTY_CRYPTO_KEY_LEN, DEFAULT_KEY_LEN); + + SecretKeyFactory factory = null; + try { + factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1"); + } catch (NoSuchAlgorithmException e) { + throw new IOException("Can't load SecretKeyFactory", e); + } + + SecretKeySpec key = null; + try { + key = new SecretKeySpec(factory.generateSecret(new PBEKeySpec(passPhrase.toCharArray(), salt.getBytes(), iterations, keyLen)).getEncoded(), algOnly); + } catch (Exception e) { + throw new IOException("Can't generate secret key", e); + } + + Cipher crypto = null; + + try { + crypto = Cipher.getInstance(alg); + } catch (Exception e) { + throw new IOException("Can't initialize the decryptor", e); + } + + byte[] decryptedBytes; + + try { + crypto.init(Cipher.DECRYPT_MODE, key); + decryptedBytes = crypto.doFinal(encrypted); + } catch (Exception e) { + throw new IOException("Can't decrypt the password", e); + } + + return new String(decryptedBytes); + } + + @Override + public void cleanUpConfiguration(Configuration configuration) { + configuration.unset(PROPERTY_CRYPTO_PASSPHRASE); + configuration.unset(PROPERTY_CRYPTO_SALT); + configuration.unset(PROPERTY_CRYPTO_KEY_LEN); + configuration.unset(PROPERTY_CRYPTO_ITERATIONS); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/ed73f885/src/java/org/apache/sqoop/util/password/FilePasswordLoader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/util/password/FilePasswordLoader.java b/src/java/org/apache/sqoop/util/password/FilePasswordLoader.java new file mode 100644 index 0000000..86b71ec --- /dev/null +++ b/src/java/org/apache/sqoop/util/password/FilePasswordLoader.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.util.password; + +import org.apache.hadoop.conf.Configuration; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import java.io.IOException; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import java.io.InputStream; + +/** + * Simple password loader that will load file from file system. + * + * The file system can be either local or any remote implementation supported + * by your Hadoop installation. As the password needs to be stored in the file + * in a clear text, the password file should have very restrictive permissions + * (400). + */ +public class FilePasswordLoader extends PasswordLoader { + + public static final Log LOG = LogFactory.getLog(FilePasswordLoader.class.getName()); + + /** + * Verify that given path leads to a file that we can read. + * + * @param fs Associated FileSystem + * @param path Path + * @throws IOException + */ + protected void verifyPath(FileSystem fs, Path path) throws IOException { + if (!fs.exists(path)) { + throw new IOException("The password file does not exist! " + path); + } + + if (!fs.isFile(path)) { + throw new IOException("The password file cannot be a directory! " + path); + } + } + + /** + * Read bytes from given file. + * + * @param fs Associated FileSystem + * @param path Path + * @return + * @throws IOException + */ + protected byte [] readBytes(FileSystem fs, Path path) throws IOException { + InputStream is = fs.open(path); + try { + return IOUtils.toByteArray(is); + } finally { + IOUtils.closeQuietly(is); + } + } + + @Override + public String loadPassword(String p, Configuration configuration) throws IOException { + LOG.debug("Fetching password from specified path: " + p); + Path path = new Path(p); + FileSystem fs = path.getFileSystem(configuration); + + try { + verifyPath(fs, path); + return new String(readBytes(fs, path)); + } finally { + fs.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/ed73f885/src/java/org/apache/sqoop/util/password/PasswordLoader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/util/password/PasswordLoader.java b/src/java/org/apache/sqoop/util/password/PasswordLoader.java new file mode 100644 index 0000000..6515a1d --- /dev/null +++ b/src/java/org/apache/sqoop/util/password/PasswordLoader.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.util.password; + +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; + +/** + * Abstract class for describing password loader. + */ +abstract public class PasswordLoader { + + /** + * Load password from given path. + * + * @param path Path to load password from. + * @param configuration Configuration object. + * @return Password + * @throws IOException + */ + public abstract String loadPassword(String path, Configuration configuration) throws IOException; + + /** + * Callback that allows to clean up configuration properties out of the + * configuration object that will be passed down to the mapreduce framework. + * + * It's defined by the Sqoop framework that after calling cleanUpConfiguration(), + * method loadPassword() will never be called again. + */ + public void cleanUpConfiguration(Configuration configuration) { + // Default implementation is empty + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/ed73f885/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java index 41a432a..bbf82f4 100644 --- a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java +++ b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java @@ -29,10 +29,17 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.sqoop.mapreduce.db.DBConfiguration; import org.apache.sqoop.tool.BaseSqoopTool; import org.apache.sqoop.tool.ImportTool; +import org.apache.sqoop.util.password.CryptoFileLoader; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.security.NoSuchAlgorithmException; import java.sql.Connection; import java.util.ArrayList; import java.util.Collections; @@ -314,18 +321,93 @@ public class TestPassingSecurePassword extends BaseSqoopTestCase { return args.toArray(new String[0]); } + public void testCryptoFileLoader() throws Exception { + // Current implementation is limited to ECB mode + Object[][] ciphers = { +// {"AES/CBC/NoPadding", 128}, +// {"AES/CBC/PKCS5Padding", 128}, + {"AES/ECB/NoPadding", 128}, + {"AES/ECB/PKCS5Padding", 128}, +// {"DES/CBC/NoPadding", 56}, +// {"DES/CBC/PKCS5Padding", 56}, + {"DES/ECB/NoPadding", 64}, + {"DES/ECB/PKCS5Padding", 64}, +// {"DESede/CBC/NoPadding", 168}, +// {"DESede/CBC/PKCS5Padding", 168}, + {"DESede/ECB/NoPadding", 192}, + {"DESede/ECB/PKCS5Padding", 192} + }; + + String [] passphrases = { + "Simple password", + "!@#$%^&*()_+<>?:" + }; + + // Execute all ciphers with all pass phrases + for(Object [] cipher : ciphers) { + for(String pass : passphrases) { + executeCipherTest(pass, pass, (String)cipher[0], (Integer)cipher[1]); + } + } + } + + public void executeCipherTest(String password, String passphrase, String cipher, int keySize) throws Exception { + LOG.info("Using cipher: " + cipher + " with keySize " + keySize + " and passphrase " + passphrase ); + String passwordFilePath = TEMP_BASE_DIR + ".pwd"; + createTempFile(passwordFilePath); + writeToFile(passwordFilePath, encryptPassword(password, passphrase, cipher, 10000, keySize)); + LOG.info("Generated encrypted password file in: " + passwordFilePath); + + ArrayList<String> extraArgs = new ArrayList<String>(); + extraArgs.add("--username"); + extraArgs.add("username"); + extraArgs.add("--password-file"); + extraArgs.add(passwordFilePath); + String[] commonArgs = getCommonArgs(false, extraArgs); + + Configuration conf = getConf(); + conf.set("org.apache.sqoop.credentials.loader.class", CryptoFileLoader.class.getCanonicalName()); + conf.set("org.apache.sqoop.credentials.loader.crypto.alg", cipher); + conf.set("org.apache.sqoop.credentials.loader.crypto.passphrase", passphrase); + conf.setInt("org.apache.sqoop.credentials.loader.crypto.salt.key.len", keySize); + + SqoopOptions in = getSqoopOptions(conf); + ImportTool importTool = new ImportTool(); + + SqoopOptions out = importTool.parseArguments(commonArgs, conf, in, true); + assertNotNull(out.getPasswordFilePath()); + assertNotNull(out.getPassword()); + assertEquals(passphrase, out.getPassword()); + } + + private byte[] encryptPassword(String password, String passPhrase, String alg, int iterations, int keySize) throws Exception { + String algOnly = alg.split("/")[0]; + + SecretKeyFactory factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1"); + SecretKey secKey = factory.generateSecret(new PBEKeySpec(passPhrase.toCharArray(), "SALT".getBytes(), iterations, keySize)); + SecretKeySpec key = new SecretKeySpec(secKey.getEncoded(), algOnly); + + Cipher crypto = Cipher.getInstance(alg); + crypto.init(Cipher.ENCRYPT_MODE, key); + + return crypto.doFinal(password.getBytes()); + } + private void createTempFile(String filePath) throws IOException { File pwdFile = new File(filePath); pwdFile.createNewFile(); } - private void writeToFile(String filePath, String contents) - throws IOException { + private void writeToFile(String filePath, String contents) throws IOException { + writeToFile(filePath, contents.getBytes()); + } + + private void writeToFile(String filePath, byte [] contents) throws IOException { File pwdFile = new File(filePath); FileOutputStream fos = null; try { fos = new FileOutputStream(pwdFile); - fos.write(contents.getBytes()); + fos.write(contents); } finally { if (fos != null) { fos.close();
