SQOOP-2889: Read and write from encrypted repository (Abraham Fine via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5b897a46 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5b897a46 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5b897a46 Branch: refs/heads/sqoop2 Commit: 5b897a46fcbbe8a83fdb7ce1ebe8d862c34e9736 Parents: bfcfedf Author: Jarek Jarcec Cecho <[email protected]> Authored: Tue Mar 29 14:13:47 2016 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Tue Mar 29 14:13:47 2016 -0700 ---------------------------------------------------------------------- .../sqoop/common/test/db/DerbyProvider.java | 10 +- .../sqoop/error/code/CommonRepositoryError.java | 2 + .../apache/sqoop/error/code/PasswordError.java | 44 ++ .../java/org/apache/sqoop/model/MMasterKey.java | 54 +++ .../org/apache/sqoop/utils/PasswordUtils.java | 99 ++++ .../org/apache/sqoop/utils/ProcessUtils.java | 40 -- .../apache/sqoop/utils/TestPasswordUtils.java | 86 ++++ .../oracle-jdbc-connector-config.properties | 170 +++---- .../java/org/apache/sqoop/core/SqoopServer.java | 3 + .../apache/sqoop/repository/JdbcRepository.java | 31 ++ .../sqoop/repository/JdbcRepositoryHandler.java | 17 + .../sqoop/repository/MasterKeyManager.java | 479 +++++++++++++++++++ .../org/apache/sqoop/repository/Repository.java | 16 + .../sqoop/security/SecurityConstants.java | 79 +++ .../apache/sqoop/security/SecurityError.java | 38 +- .../sqoop/repository/TestMasterKeyManager.java | 208 ++++++++ dist/src/main/conf/sqoop.properties | 15 + .../common/CommonRepositoryHandler.java | 107 ++++- ...RepositoryInsertUpdateDeleteSelectQuery.java | 64 ++- .../common/CommonRepositorySchemaConstants.java | 26 + .../derby/DerbyRepositoryHandler.java | 8 + .../derby/DerbySchemaCreateQuery.java | 42 ++ .../derby/DerbySchemaUpgradeQuery.java | 35 ++ .../sqoop/repository/derby/DerbyTestCase.java | 8 + .../mysql/MySqlRepositoryHandler.java | 1 + .../mysql/MySqlSchemaCreateQuery.java | 18 +- ...RepositoryInsertUpdateDeleteSelectQuery.java | 4 + .../postgresql/PostgresqlRepositoryHandler.java | 7 + .../postgresql/PostgresqlSchemaCreateQuery.java | 44 ++ .../PostgresqlSchemaUpgradeQuery.java | 48 ++ .../apache/sqoop/server/SqoopJettyServer.java | 38 +- .../sqoop/shell/SetTruststoreFunction.java | 4 +- .../test/minicluster/SqoopMiniCluster.java | 11 + 33 files changed, 1657 insertions(+), 199 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/common-test/src/main/java/org/apache/sqoop/common/test/db/DerbyProvider.java ---------------------------------------------------------------------- diff --git a/common-test/src/main/java/org/apache/sqoop/common/test/db/DerbyProvider.java b/common-test/src/main/java/org/apache/sqoop/common/test/db/DerbyProvider.java index 839e561..b879320 100644 --- a/common-test/src/main/java/org/apache/sqoop/common/test/db/DerbyProvider.java +++ b/common-test/src/main/java/org/apache/sqoop/common/test/db/DerbyProvider.java @@ -43,6 +43,10 @@ public class DerbyProvider extends DatabaseProvider { public static final String DRIVER = "org.apache.derby.jdbc.ClientDriver"; + private static String DERBY_USERNAME = "sqoop2"; + + private static String DERBY_PASSWORD = "encryptme"; + // Used port for this instance int port; @@ -62,7 +66,7 @@ public class DerbyProvider extends DatabaseProvider { port = NetworkUtils.findAvailablePort(); LOG.info("Will bind to port " + port); - server = new NetworkServerControl(InetAddress.getByName("localhost"), port); + server = new NetworkServerControl(InetAddress.getByName("localhost"), port, DERBY_USERNAME, DERBY_PASSWORD); server.start(new LoggerWriter(LOG, Level.INFO)); // Start won't thrown an exception in case that it fails to start, one @@ -160,12 +164,12 @@ public class DerbyProvider extends DatabaseProvider { @Override public String getConnectionUsername() { - return null; + return DERBY_USERNAME; } @Override public String getConnectionPassword() { - return null; + return DERBY_PASSWORD; } @Override http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java b/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java index 37eb04a..dd52c82 100644 --- a/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java +++ b/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java @@ -216,6 +216,8 @@ public enum CommonRepositoryError implements ErrorCode { COMMON_0057("Unable to load specific connector"), COMMON_0058("Resource doesn't exist"), + + COMMON_0059("Unable to retrieve master key"), ; private final String message; http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/common/src/main/java/org/apache/sqoop/error/code/PasswordError.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/error/code/PasswordError.java b/common/src/main/java/org/apache/sqoop/error/code/PasswordError.java new file mode 100644 index 0000000..c4f8e63 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/error/code/PasswordError.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.error.code; + +import org.apache.sqoop.common.ErrorCode; + +public enum PasswordError implements ErrorCode { + + PASSWORD_0000("Failed to execute password generator"), + + PASSWORD_0001("No password returned from generator"), + + ; + + private final String message; + + private PasswordError(String message) { + this.message = message; + } + + public String getCode() { + return name(); + } + + public String getMessage() { + return message; + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/common/src/main/java/org/apache/sqoop/model/MMasterKey.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/MMasterKey.java b/common/src/main/java/org/apache/sqoop/model/MMasterKey.java new file mode 100644 index 0000000..e87a8f9 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/model/MMasterKey.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.model; + +public class MMasterKey extends MPersistableEntity { + + private String encryptedSecret; + private String hmac; + private String salt; + private String iv; + + public MMasterKey(String encryptedSecret, String hmac, String salt, String iv) { + this.encryptedSecret = encryptedSecret; + this.hmac = hmac; + this.salt = salt; + this.iv = iv; + } + + @Override + public String toString() { + return "hmac: " + getHmac() + " salt: " + getSalt() + " IV: " + getIv(); + } + + public String getEncryptedSecret() { + return encryptedSecret; + } + + public String getHmac() { + return hmac; + } + + public String getSalt() { + return salt; + } + + public String getIv() { + return iv; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/common/src/main/java/org/apache/sqoop/utils/PasswordUtils.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/utils/PasswordUtils.java b/common/src/main/java/org/apache/sqoop/utils/PasswordUtils.java new file mode 100644 index 0000000..f30ed6c --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/utils/PasswordUtils.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.utils; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.apache.sqoop.common.MapContext; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.error.code.PasswordError; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; + +public class PasswordUtils { + private static final Logger LOG = Logger.getLogger(PasswordUtils.class); + + /** + * Attempts to read a password from the configuration via plaintext or a generator. + * + * First, we attempt to read the plaintext password, if that value does not exist + * we will look for and try to run a configured password generator. + * + * If a password cannot be found and a password generator cannot be found, an + * exception is thrown. + * + * If both a password and a password generator are set, we prefer the password. + * + * @param configurationContext MapContext holding the sqoop configuration + * @param passwordProperty String containing the property that maps to the + * plaintext version of the password + * @param passwordGeneratorProperty String containing the property that maps to + * the generator that prints the password to + * standard out + * @return A String password value, null if neither password property is set + * @throws SqoopException + */ + public static String readPassword(MapContext configurationContext, String passwordProperty, String passwordGeneratorProperty) { + String plaintextPassword = configurationContext.getString(passwordProperty); + String passwordGenerator = configurationContext.getString(passwordGeneratorProperty); + + if (StringUtils.isNotBlank(plaintextPassword)) { + if (StringUtils.isNotBlank(passwordGenerator)) { + LOG.warn(passwordProperty + " and " + passwordGeneratorProperty + " are both set, using " + passwordProperty); + } + return plaintextPassword; + } else if (StringUtils.isNotBlank(passwordGenerator)) { + try { + String password = PasswordUtils.readOutputFromGenerator(passwordGenerator); + + if (StringUtils.isNotBlank(password)) { + return password; + } else { + throw new SqoopException(PasswordError.PASSWORD_0001); + } + } catch (IOException exception) { + throw new SqoopException(PasswordError.PASSWORD_0000, exception); + } + } else { + return null; + } + } + + /** + * Executes the given command under /bin/sh and reads one line of standard output + * + * @param generatorCommand String containing the command to execute + * @return The first line of standard output from the generator command + * @throws IOException + */ + public static String readOutputFromGenerator(String generatorCommand) throws IOException { + ProcessBuilder processBuilder = new ProcessBuilder("/bin/sh", "-c", generatorCommand); + Process process = processBuilder.start(); + String output; + try ( + InputStreamReader inputStreamReader = new InputStreamReader(process.getInputStream(), Charset.forName("UTF-8")); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader); + ) { + output = bufferedReader.readLine(); + } + return output; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/common/src/main/java/org/apache/sqoop/utils/ProcessUtils.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/utils/ProcessUtils.java b/common/src/main/java/org/apache/sqoop/utils/ProcessUtils.java deleted file mode 100644 index 0a32d68..0000000 --- a/common/src/main/java/org/apache/sqoop/utils/ProcessUtils.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.utils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.Charset; - -public class ProcessUtils { - public static String readOutputFromGenerator(String generatorCommand) throws IOException { - ProcessBuilder processBuilder = new ProcessBuilder("/bin/sh", "-c", generatorCommand); - Process process = processBuilder.start(); - String output; - try ( - InputStreamReader inputStreamReader = new InputStreamReader(process.getInputStream(), Charset.forName("UTF-8")); - BufferedReader bufferedReader = new BufferedReader(inputStreamReader); - ) { - output = bufferedReader.readLine(); - } catch(IOException exception) { - throw exception; - } - return output; - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/common/src/test/java/org/apache/sqoop/utils/TestPasswordUtils.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/utils/TestPasswordUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestPasswordUtils.java new file mode 100644 index 0000000..18a096b --- /dev/null +++ b/common/src/test/java/org/apache/sqoop/utils/TestPasswordUtils.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.utils; + +import org.apache.sqoop.common.MapContext; +import org.apache.sqoop.common.SqoopException; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; + +public class TestPasswordUtils { + + private static String PASSWORD_KEY = "PASSWORD_KEY"; + private static String PASSWORD_GENERATOR_KEY = "PASSWORD_GENERATOR_KEY"; + + private static String PASSWORD = "password"; + private static String PASSWORD_FROM_GENERATOR = "password_from_generator"; + private static String PASSWORD_GENERATOR = "echo " + PASSWORD_FROM_GENERATOR; + + @Test + public void passwordExistsAndPasswordGeneratorExists() { + Map<String, String> passwordMap = new HashMap<>(); + passwordMap.put(PASSWORD_KEY, PASSWORD); + passwordMap.put(PASSWORD_GENERATOR_KEY, PASSWORD_GENERATOR); + + assertEquals( + PasswordUtils.readPassword(new MapContext(passwordMap), PASSWORD_KEY, PASSWORD_GENERATOR_KEY), + PASSWORD); + } + + @Test + public void passwordExistsAndPasswordGeneratorDoesNotExist() { + Map<String, String> passwordMap = new HashMap<>(); + passwordMap.put(PASSWORD_KEY, PASSWORD); + + assertEquals( + PasswordUtils.readPassword(new MapContext(passwordMap), PASSWORD_KEY, PASSWORD_GENERATOR_KEY), + PASSWORD); + } + + @Test + public void passwordDoesNotExistAndPasswordGeneratorExists() { + Map<String, String> passwordMap = new HashMap<>(); + passwordMap.put(PASSWORD_GENERATOR_KEY, PASSWORD_GENERATOR); + + assertEquals( + PasswordUtils.readPassword(new MapContext(passwordMap), PASSWORD_KEY, PASSWORD_GENERATOR_KEY), + PASSWORD_FROM_GENERATOR); + } + + @Test + public void passwordDoesNotExistAndPasswordGeneratorDoesNotExist() { + Map<String, String> passwordMap = new HashMap<>(); + + assertNull(PasswordUtils.readPassword(new MapContext(passwordMap), PASSWORD_KEY, PASSWORD_GENERATOR_KEY)); + } + + @Test( + expectedExceptions = {SqoopException.class}, + expectedExceptionsMessageRegExp = ".*No password returned from generator") + public void passwordGeneratorFailsToExecute() { + Map<String, String> passwordMap = new HashMap<>(); + passwordMap.put(PASSWORD_GENERATOR_KEY, "ISUREDOHOPEYOUDONTHAVESOMETHINGWITHTHISNAMEINYOURPATH"); + + PasswordUtils.readPassword(new MapContext(passwordMap), PASSWORD_KEY, PASSWORD_GENERATOR_KEY); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/connector/connector-oracle-jdbc/src/main/resources/oracle-jdbc-connector-config.properties ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/resources/oracle-jdbc-connector-config.properties b/connector/connector-oracle-jdbc/src/main/resources/oracle-jdbc-connector-config.properties index 1e36ae7..d99ab9b 100644 --- a/connector/connector-oracle-jdbc/src/main/resources/oracle-jdbc-connector-config.properties +++ b/connector/connector-oracle-jdbc/src/main/resources/oracle-jdbc-connector-config.properties @@ -13,160 +13,130 @@ # See the License for the specific language governing permissions and # limitations under the License. -connector.name = Oracle connector - +# Oracle JDBC Connector Resources +############################ +# Connection Config +# connectionConfig.label = Oracle connection configuration -connectionConfig.help = Information required to connect to an Oracle server. +connectionConfig.help = You must supply the information requested in order to \ + create an Oracle connection object. -connectionConfig.connectionString.label = Connection string -connectionConfig.connectionString.example = jdbc:oracle:thin:@oracle.sqoop.org:1521:sqoop_db -connectionConfig.connectionString.help = JDBC connection string associated with your Oracle database. +# connect string +connectionConfig.connectionString.label = JDBC connection string +connectionConfig.connectionString.help = Enter the value of JDBC connection string to be \ + used by this connector for creating Oracle connections. +# username string connectionConfig.username.label = Username -connectionConfig.username.example = sqoop-user -connectionConfig.username.help = Username to be used for connection to the Oracle server. +connectionConfig.username.help = Enter the username to be used for connecting to the \ + database. +# password string connectionConfig.password.label = Password -connectionConfig.password.label = Sup3rS3rcr3t! -connectionConfig.password.help = Password to be used for connection to the Oracle server. +connectionConfig.password.help = Enter the password to be used for connecting to the \ + database. -connectionConfig.jdbcProperties.label = Connection properties -connectionConfig.jdbcProperties.example = defaultRowPrefetch=1000 -connectionConfig.jdbcProperties.help = Key-value pairs that should be passed down to JDBC driver when establishing connection. +# jdbc properties +connectionConfig.jdbcProperties.label = JDBC connection properties +connectionConfig.jdbcProperties.help = Enter any JDBC properties that should be \ + supplied during the creation of connection. -connectionConfig.timeZone.label = Time zone -connectionConfig.timeZone.example = GMT -connectionConfig.timeZone.help = Time zone that will be used for all created sessions. +connectionConfig.timeZone.label = Session time zone +connectionConfig.timeZone.help = timeZone connectionConfig.actionName.label = Session action name -connectionConfig.actionName.example = sqoop-import -connectionConfig.actionName.help = The connector will call DMS_APPLICATION_INFO.SET_MODULE procedure with action_name set to this value. +connectionConfig.actionName.help = actionName -connectionConfig.fetchSize.label = fetch size -connectionConfig.fetchSize.example = 1000 -connectionConfig.fetchSize.help = Optional hint specifying requested JDBC fetch size. +connectionConfig.fetchSize.label = JDBC fetch size +connectionConfig.fetchSize.help = fetchSize -connectionConfig.initializationStatements.label = Initialization statements -connectionConfig.initializationStatements.example = alter session disable parallel query -connectionConfig.initializationStatements.help = List of statements that will be executed on each connection immediately after opening and before any metadata/data retrieving queries. +connectionConfig.initializationStatements.label = Session initialization statements +connectionConfig.initializationStatements.help = initializationStatements -connectionConfig.jdbcUrlVerbatim.label = Strict connection string -connectionConfig.jdbcUrlVerbatim.example = false -connectionConfig.jdbcUrlVerbatim.help = By default, OraOop will use specified connection string only to retrieve all instances of your RAC and \ - then for actual data transfer jobs will generate connection strings to distribute the load across all nodes. This distribution can be turned \ - off by setting this property to false. +connectionConfig.jdbcUrlVerbatim.label = Use JDBC connection string verbatim +connectionConfig.jdbcUrlVerbatim.help = jdbcUrlVerbatim connectionConfig.racServiceName.label = RAC service name -connectionConfig.racServiceName.example = sales.sqoop.org -connectionConfig.racServiceName.help = Service name that will be used for automatically generated connection strings when connecting to RAC. - +connectionConfig.racServiceName.help = racServiceName -toJobConfig.label = Database target -toJobConfig.help = Describes target destination and way how data should be persisted on the RDBMS system. +# ToJob Config +# +toJobConfig.label = To database configuration +toJobConfig.help = You must supply the information requested in order to create \ + the TO part of the job object. +# To table name toJobConfig.tableName.label = Table name -toJobConfig.tableName.example = target_table -toJobConfig.tableName.help = Destination table name to store transfer results. +toJobConfig.tableName.help = Table name to write data into toJobConfig.columns.label = Columns -toJobConfig.columns.example = id,text,city -toJobConfig.columns.help = Subset of columns that will will be written to. Omitted columns have to either allow \ - NULL values or have defined default value. +toJobConfig.columns.help = Columns toJobConfig.templateTable.label = Template table name -toJobConfig.templateTable.example = existing_table -toJobConfig.templateTable.help = If this field is not empty, then target table will get created with the same structure as the \ - template table. +toJobConfig.templateTable.help = templateTable toJobConfig.partitioned.label = Partitioned -toJobConfig.partitioned.example = true -toJobConfig.partitioned.help = If creating a new target table with structure from template table, this option controls whether \ - the new table should be partitioned or not. +toJobConfig.partitioned.help = partitioned toJobConfig.nologging.label = Nologging -toJobConfig.nologging.example = Nologging -toJobConfig.nologging.help = If creating a new target table with structure from template table, when set to true, this option will \ - add NOLOGGING clause to the CREATE TABLE statement. +toJobConfig.nologging.help = nologging -toJobConfig.updateKey.label = Update columns -toJobConfig.updateKey.example = id,date -toJobConfig.updateKey.help = Specifying this option will switch OraOop to update mode. Instead of generating INSERT statements to \ - insert data to Oracle, it will generate UPDATE statements. Configured options will then be used in WHERE clause. +toJobConfig.updateKey.label = Update key columns +toJobConfig.updateKey.help = updateKey toJobConfig.updateMerge.label = Merge updates -toJobConfig.updateMerge.example = true -toJobConfig.updateMerge.help = Instead of generating only UPDATE statements, OraOop will generate MERGE statement that will do so \ - called upsert - will either update existing rows or insert new rows. +toJobConfig.updateMerge.help = updateMerge -toJobConfig.dropTableIfExists.label = Drop table -toJobConfig.dropTableIfExists.example = true -toJobConfig.dropTableIfExists.help = If set to true then, then OraOop will drop existing target table and re-create it using the \ - template table structure. +toJobConfig.dropTableIfExists.label = Drop table if exists +toJobConfig.dropTableIfExists.help = dropTableIfExists -toJobConfig.storageClause.label = Template storage clause -toJobConfig.storageClause.example = STORAGE (INITIAL 100K NEXT 50K MINEXTENTS 1 MAXEXTENTS 50 PCTINCREASE 5) -toJobConfig.storageClause.help = If target table will get created, insert specified storage clause to the CREATE TABLE statement. +toJobConfig.storageClause.label = Template table storage clause +toJobConfig.storageClause.help = storageClause -toJobConfig.temporaryStorageClause.label = Temporary storage clause -toJobConfig.temporaryStorageClause.example = STORAGE (INITIAL 100K NEXT 50K MINEXTENTS 1 MAXEXTENTS 50 PCTINCREASE 5) -toJobConfig.temporaryStorageClause.help = Storage clause that will be used for all temporary tables that OraOop might need to create. +toJobConfig.temporaryStorageClause.label = Temporary table storage clause +toJobConfig.temporaryStorageClause.help = temporaryStorageClause -toJobConfig.appendValuesHint.label = Append values hint -toJobConfig.appendValuesHint.example = AUTO -toJobConfig.appendValuesHint.help = Specifies whether generated INSERT queries should take advantage of Oracle's APPEND hint. +toJobConfig.appendValuesHint.label = Append values hint usage +toJobConfig.appendValuesHint.help = appendValuesHint toJobConfig.parallel.label = Parallel -toJobConfig.parallel.example = true -toJobConfig.parallel.help = When moving data from temporary tables, this option controls whether generated queries shoud include \ - PARALLEL keyword. - +toJobConfig.parallel.help = parallel -fromJobConfig.label = Database source -fromJobConfig.help = Specifies source and way how the data should be fetched from source database. +# FromJob Config +# +fromJobConfig.label = From Oracle configuration +fromJobConfig.help = You must supply the information requested in order to create \ + the FROM part of the job object. fromJobConfig.tableName.label = Table name -fromJobConfig.tableName.example = input_table -fromJobConfig.tableName.help = Input table name from from which data will be retrieved. +fromJobConfig.tableName.help = tableName fromJobConfig.columns.label = Columns -fromJobConfig.columns.example = id,text,city -fromJobConfig.columns.help = Subset of columns that should be retrieved from source table. +fromJobConfig.columns.help = Columns fromJobConfig.consistentRead.label = Consistent read -fromJobConfig.consistentRead.example = true -fromJobConfig.consistentRead.help = If true, connector will use Oracle Flashback technology to achieve consistency across \ - independent connections. +fromJobConfig.consistentRead.help = consistentRead fromJobConfig.consistentReadScn.label = Consistent read SCN -fromJobConfig.consistentReadScn.example = 10093466 -fromJobConfig.consistentReadScn.help = Optional SCN value that should be used to read the consistent point in time. Connector \ - will use latest value during initialization if not specified. +fromJobConfig.consistentReadScn.help = consistentReadScn fromJobConfig.partitionList.label = Partitions -fromJobConfig.partitionList.example = PART1,PART2 -fromJobConfig.partitionList.help = Subset of partitions that should be retrieved from source table. +fromJobConfig.partitionList.help = partitionList fromJobConfig.dataChunkMethod.label = Data chunk method -fromJobConfig.dataChunkMethod.example = ROWID -fromJobConfig.dataChunkMethod.help = Method that will be used to slice the data in source table to transfer them in parallel. +fromJobConfig.dataChunkMethod.help = dataChunkMethod fromJobConfig.dataChunkAllocationMethod.label = Data chunk allocation method -fromJobConfig.dataChunkAllocationMethod.example = ROUNDROBIN -fromJobConfig.dataChunkAllocationMethod.help = Specific way in which various data chunks gets distributed to partitions. +fromJobConfig.dataChunkAllocationMethod.help = dataChunkAllocationMethod -fromJobConfig.whereClauseLocation.label = Where clause -fromJobConfig.whereClauseLocation.example = SUBSPLIT -fromJobConfig.whereClauseLocation.help = Determines whether the where clause should be used inside each data chunk separately \ - or for union of all chunks that are selected in single partition. +fromJobConfig.whereClauseLocation.label = Where clause location +fromJobConfig.whereClauseLocation.help = whereClauseLocation fromJobConfig.omitLobColumns.label = Omit LOB columns -fromJobConfig.omitLobColumns.example = true -fromJobConfig.omitLobColumns.help = If set to true, then OraOop will ignore importing all LOB based columns (CLOB, BLOB). +fromJobConfig.omitLobColumns.help = omitLobColumns fromJobConfig.queryHint.label = Query hint -fromJobConfig.queryHint.example = ALL_ROWS -fromJobConfig.queryHint.help = Hints that should be used for queries that are transferring data. +fromJobConfig.queryHint.help = queryHint fromJobConfig.conditions.label = Conditions -fromJobConfig.conditions.example = id > 100 -fromJobConfig.conditions.help = Additional conditions that should be used inside WHERE clause to limit imported data. \ No newline at end of file +fromJobConfig.conditions.help = conditions \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/core/src/main/java/org/apache/sqoop/core/SqoopServer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/core/SqoopServer.java b/core/src/main/java/org/apache/sqoop/core/SqoopServer.java index 80a7b88..0c983a0 100644 --- a/core/src/main/java/org/apache/sqoop/core/SqoopServer.java +++ b/core/src/main/java/org/apache/sqoop/core/SqoopServer.java @@ -22,6 +22,7 @@ import org.apache.sqoop.audit.AuditLoggerManager; import org.apache.sqoop.connector.ConnectorManager; import org.apache.sqoop.driver.Driver; import org.apache.sqoop.driver.JobManager; +import org.apache.sqoop.repository.MasterKeyManager; import org.apache.sqoop.repository.RepositoryManager; import org.apache.sqoop.security.AuthenticationManager; import org.apache.sqoop.security.AuthorizationManager; @@ -39,6 +40,7 @@ public class SqoopServer { JobManager.getInstance().destroy(); Driver.getInstance().destroy(); ConnectorManager.getInstance().destroy(); + MasterKeyManager.getInstance().destroy(); RepositoryManager.getInstance().destroy(); AuditLoggerManager.getInstance().destroy(); AuthorizationManager.getInstance().destroy(); @@ -56,6 +58,7 @@ public class SqoopServer { AuthorizationManager.getInstance().initialize(); AuditLoggerManager.getInstance().initialize(); RepositoryManager.getInstance().initialize(); + MasterKeyManager.getInstance().initialize(); ConnectorManager.getInstance().initialize(); Driver.getInstance().initialize(); JobManager.getInstance().initialize(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java index 5b70f95..44ab41b 100644 --- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java +++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java @@ -27,6 +27,7 @@ import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MDriver; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; +import org.apache.sqoop.model.MMasterKey; import org.apache.sqoop.model.MSubmission; public class JdbcRepository extends Repository { @@ -657,6 +658,36 @@ public class JdbcRepository extends Repository { }); } + /** + * {@inheritDoc} + */ + @Override + public MMasterKey getMasterKey() { + return (MMasterKey) doWithConnection(new DoWithConnection() { + @Override + public Object doIt(Connection conn) throws Exception { + return handler.getMasterKey(conn); + } + }); + } + + /** + * {@inheritDoc} + */ + @Override + public void createMasterKey(final MMasterKey mMasterKey) { + doWithConnection(new DoWithConnection() { + @Override + public Object doIt(Connection conn) { + if(mMasterKey.hasPersistenceId()) { + throw new SqoopException(RepositoryError.JDBCREPO_0023); + } + handler.createMasterKey(mMasterKey, conn); + return null; + } + }); + } + @Override protected void deleteJobInputs(final String jobName, RepositoryTransaction tx) { doWithConnection(new DoWithConnection() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java index feab2ad..94f5e6f 100644 --- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java +++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java @@ -25,6 +25,7 @@ import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MDriver; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; +import org.apache.sqoop.model.MMasterKey; import org.apache.sqoop.model.MSubmission; /** @@ -465,4 +466,20 @@ public abstract class JdbcRepositoryHandler { */ public abstract MSubmission findLastSubmissionForJob(String jobName, Connection conn); + /** + * Read the master key from the database + * + * @param conn Connection to the repository + * @return MMasterKey representing the Master Key + */ + public abstract MMasterKey getMasterKey(Connection conn); + + /** + * Create the master key in the database + * + * @param mMasterKey MMasterKey representing the Master Key + * @param conn Connection to the repository + */ + public abstract void createMasterKey(MMasterKey mMasterKey, Connection conn); + } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java b/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java new file mode 100644 index 0000000..df84d54 --- /dev/null +++ b/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java @@ -0,0 +1,479 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.repository; + +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.sqoop.common.MapContext; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.core.SqoopConfiguration; +import org.apache.sqoop.model.MMasterKey; +import org.apache.sqoop.security.SecurityConstants; +import org.apache.sqoop.security.SecurityError; +import org.apache.sqoop.utils.PasswordUtils; +import org.apache.commons.codec.binary.Base64; + +import javax.crypto.Cipher; +import javax.crypto.Mac; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.Charset; +import java.security.GeneralSecurityException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Random; + +public class MasterKeyManager { + + private String hmacAlgorithm; + private int hmacKeySizeBytes; + private String cipherAlgorithm; + private int cipherKeySize; + private String cipherSpec; + private String pbkdf2Algorithm; + private int pbkdf2Rounds; + private int ivLength; + + private SecretKey masterEncryptionKey; + private SecretKey masterHmacKey; + + private Random random; + + private static MasterKeyManager instance; + + static { + instance = new MasterKeyManager(); + } + + private MasterKeyManager() { + } + + public static MasterKeyManager getInstance() { + return instance; + } + + public static void setInstance(MasterKeyManager newInstance) { + instance = newInstance; + } + + public void initialize() throws SqoopException { + initialize(true); + } + + public synchronized void initialize(boolean createMasterKey) throws SqoopException { + // This is used for the generation of random initialization vectors and salts + random = new SecureRandom(); + + MapContext configurationContext = SqoopConfiguration.getInstance().getContext(); + if (configurationContext.getBoolean(SecurityConstants.REPO_ENCRYPTION_ENABLED, false)) { + + // Grab configuration from the sqoop properties file. All of this configuration is required + // and an exception will be thrown if any of it is missing + hmacAlgorithm = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM); + cipherAlgorithm = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM); + cipherSpec = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC); + cipherKeySize = populateIntConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE); + ivLength = populateIntConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE); + pbkdf2Algorithm = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM); + pbkdf2Rounds = populateIntConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS); + + // The size of the hmac key can be derived from the provided HMAC algorithm + try { + hmacKeySizeBytes = Mac.getInstance(hmacAlgorithm).getMacLength(); + } catch (NoSuchAlgorithmException e) { + throw new SqoopException(SecurityError.ENCRYPTION_0011, e); + } + + Repository repository = RepositoryManager.getInstance().getRepository(); + String password = PasswordUtils.readPassword(configurationContext, SecurityConstants.REPO_ENCRYPTION_PASSWORD, + SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR); + if (StringUtils.isEmpty(password)) { + throw new SqoopException(SecurityError.ENCRYPTION_0008); + } + + MMasterKey existingEncryptedMasterKey = repository.getMasterKey(); + String salt; + + if (existingEncryptedMasterKey == null) { + // Since the master key does not exist, we can generate a random salt that we will use + // for encryption of the Master Key + // We will use a salt that is the same size as the encryption key + salt = Base64.encodeBase64String(generateRandomByteArray(hmacKeySizeBytes)); + } else { + // Since the master key already exists, we will read the salt from the repository + salt = existingEncryptedMasterKey.getSalt(); + } + + // Derive two keys (that we will be used to encrypt and verify the master key) + // from the configuration provided password and the salt we just read/created. + byte[] keyBytes = getKeysFromPassword(password, salt); + SecretKey passwordEncryptionKey = new SecretKeySpec(keyBytes, 0, + cipherKeySize, cipherAlgorithm); + SecretKey passwordHmacKey = new SecretKeySpec(keyBytes, + cipherKeySize, hmacKeySizeBytes, hmacAlgorithm); + + byte[] masterEncryptionKeyBytes; + byte[] masterHmacKeyBytes; + if (existingEncryptedMasterKey == null) { + if (createMasterKey) { + // A master key does not exist so we must create one. We will simply + // use two random byte arrays for the encryption and hmac components. + // The sizes of these keys is determined by the values provided to + // configuration. + masterEncryptionKeyBytes = generateRandomByteArray(cipherKeySize); + masterHmacKeyBytes = generateRandomByteArray(hmacKeySizeBytes); + + // The initialization vector for the encryption of the master key is + // randomly generated. + String iv = Base64.encodeBase64String(generateRandomByteArray(ivLength)); + + // We append our two keys together and encrypt the resulting byte array. + // This is the secret that all of the encryption in the repository depends upon + byte[] secret = ArrayUtils.addAll(masterEncryptionKeyBytes, masterHmacKeyBytes); + String encryptedSecret = encryptToString(passwordEncryptionKey, secret, iv); + + // We store our new master key in the repository in its encrypted form + // along with an HMAC to verify the key when we read it, the salt needed to + // generate keys to decrypt it, and the initialization vector used + repository.createMasterKey(new MMasterKey(encryptedSecret, generateHmac(passwordHmacKey, + encryptedSecret), salt, iv)); + } else { + // If a master key does not exist and we are trying to initialize the + // manager without allowing it to create a master key, we should fail + throw new SqoopException(SecurityError.ENCRYPTION_0002); + } + } else { + // A master key exists so we need to read it from the repository and + // decrypt it. + String iv = existingEncryptedMasterKey.getIv(); + String encryptedSecret = existingEncryptedMasterKey.getEncryptedSecret(); + + // Before we go about decrypting the master key we should verify the hmac + // to ensure that it has not been tampered with + String hmac = existingEncryptedMasterKey.getHmac(); + if (!validHmac(passwordHmacKey, encryptedSecret, hmac)) { + throw new SqoopException(SecurityError.ENCRYPTION_0001); + } + + // The master key has not been tampered with, lets decrypt it using the key + // derived from the password and the initialization vector from the repository + byte[] decryptedKey = decryptToBytes(passwordEncryptionKey, encryptedSecret, iv); + + // Since the master key is stored as the concatenation of an encryption + // key and an hmac key, we need to split it according to the sizes derived + // from the configuration + masterEncryptionKeyBytes = new byte[cipherKeySize]; + masterHmacKeyBytes = new byte[hmacKeySizeBytes]; + System.arraycopy(decryptedKey, 0, masterEncryptionKeyBytes, 0, + cipherKeySize); + System.arraycopy(decryptedKey, cipherKeySize, + masterHmacKeyBytes, 0, hmacKeySizeBytes); + } + + // Place the master encryption and master hmac key in SecretKey objects + // so we can use them to encrypt and decrypt data + masterEncryptionKey = new SecretKeySpec(masterEncryptionKeyBytes, 0, cipherKeySize, cipherAlgorithm); + masterHmacKey = new SecretKeySpec(masterHmacKeyBytes, 0, hmacKeySizeBytes, hmacAlgorithm); + } + } + + public synchronized void destroy() { + hmacAlgorithm = null; + cipherAlgorithm = null; + cipherSpec = null; + pbkdf2Algorithm = null; + + masterEncryptionKey = null; + masterHmacKey = null; + + random = null; + } + + /** + * Returns a Base64 representation of the encrypted cleartext, using the Master Key + * + * @param clearText Text to encrypt + * @param iv Initialization vector for the cipher + * @return Base64 representation of the encrypted cleartext + * @throws SqoopException + */ + public String encryptWithMasterKey(String clearText, String iv) throws SqoopException { + return encryptToString(masterEncryptionKey, clearText, iv); + } + + /** + * Validates the HMAC against the cipher text and then returns a UTF-8 string + * of the data decrypted using the Master Key + * + * Throws an exception of the HMAC is incorrect + * + * @param cipherText Base64 representation of the encrypted text + * @param iv Initialization vector for the cipher + * @param hmac HMAC for tamper resistance + * @return UTF-8 string from the decrypted data + * @throws SqoopException + */ + public String decryptWithMasterKey(String cipherText, String iv, String hmac) throws SqoopException { + if (!validWithMasterHmacKey(cipherText, hmac)) { + throw new SqoopException(SecurityError.ENCRYPTION_0010); + } + return decryptWithMasterKey(cipherText, iv); + } + + /** + * Generates the hmac for the provided cipher text using the Master Key + * + * @param cipherText Base64 representation of the encrypted text + * @return Base64 representation of the HMAC + * @throws SqoopException + */ + public String generateHmacWithMasterHmacKey(String cipherText) throws SqoopException { + return generateHmac(masterHmacKey, cipherText); + } + + /** + * Generates a random initialization vector of the expected size + * @return Base64 representation of the initialization vector + */ + public String generateRandomIv() { + return Base64.encodeBase64String(generateRandomByteArray(ivLength)); + } + + /** + * Determines if the provided HMAC matches the HMAC generated for the cipher text + * using the Master Key + * + * @param cipherText Base64 representation of the encrypted data + * @param expectedHmac Provided HMAC to compare against + * @return True if expectedHmac matches what is generated by the Master Key, false otherwise + * @throws SqoopException + */ + private boolean validWithMasterHmacKey(String cipherText, String expectedHmac) throws SqoopException { + return validHmac(masterHmacKey, cipherText, expectedHmac); + } + + /** + * Decrypts the provided ciphertext with the IV provided and the Master Key + * + * @param cipherText Base64 representation of the encrypted data + * @param iv Initialization vector for use by the cipher + * @return UTF-8 representation of the decrypted data + * @throws SqoopException + */ + private String decryptWithMasterKey(String cipherText, String iv) throws SqoopException { + return decryptToString(masterEncryptionKey, cipherText, iv); + } + + /** + * Encrypts the provided cleartext with the provided IV and SecretKey + * + * @param secretKey Key to use for the encryption + * @param clearText String that is to be encrypted + * @param iv Initialization vector for use by the cipher, + * @return byte array representing the encrypted data + * @throws SqoopException + */ + private byte[] encryptToBytes(SecretKey secretKey, String clearText, String iv) throws SqoopException { + return encryptToBytes(secretKey, clearText.getBytes(Charset.forName("UTF-8")), iv); + } + + /** + * Encrypts the provided cleartext with the provided IV and SecretKey + * + * @param secretKey Key to use for the encryption + * @param clearText Byte array that is to be encrypted + * @param iv Initialization vector for use by the cipher, + * @return byte array representing the encrypted data + * @throws SqoopException + */ + private byte[] encryptToBytes(SecretKey secretKey, byte[] clearText, String iv) throws SqoopException { + try { + Cipher cipher = Cipher.getInstance(cipherSpec); + cipher.init(Cipher.ENCRYPT_MODE, secretKey, new IvParameterSpec(Base64.decodeBase64(iv))); + return cipher.doFinal(clearText); + } catch (GeneralSecurityException exception) { + throw new SqoopException(SecurityError.ENCRYPTION_0004, exception); + } + } + + /** + * Encrypts the provided cleartext with the provided IV and SecretKey + * + * @param secretKey Key to use for the encryption + * @param clearText Byte array that is to be encrypted + * @param iv Initialization vector for use by the cipher, + * @return Base64 representation of the encrypted data + * @throws SqoopException + */ + private String encryptToString(SecretKey secretKey, byte[] clearText, String iv) throws SqoopException { + return Base64.encodeBase64String(encryptToBytes(secretKey, clearText, iv)); + } + + /** + * Encrypts the provided cleartext with the provided IV and SecretKey + * + * @param secretKey Key to use for the encryption + * @param clearText String that is to be encrypted + * @param iv Initialization vector for use by the cipher, + * @return Base64 representation of the encrypted data + * @throws SqoopException + */ + private String encryptToString(SecretKey secretKey, String clearText, String iv) throws SqoopException { + return Base64.encodeBase64String(encryptToBytes(secretKey, clearText, iv)); + } + + /** + * Validates an HMAC against some cipherText using the provided hmacKey + * + * @param hmacKey SecretKey which defines the HMAC key and the HMAC algorithm + * @param cipherText Encrypted text from which the HMAC is generated + * @param expectedHmac The expected value for the HMAC that we will be comparing against + * @return True if the generated HMAC matches the expectedHmac, false otherwise + * @throws SqoopException + */ + private boolean validHmac(SecretKey hmacKey, String cipherText, String expectedHmac) throws SqoopException { + try { + Mac hmac = Mac.getInstance(hmacAlgorithm); + hmac.init(hmacKey); + byte[] calculatedHmac = hmac.doFinal(Base64.decodeBase64(cipherText)); + return Arrays.equals(calculatedHmac, Base64.decodeBase64(expectedHmac)); + } catch (GeneralSecurityException exception) { + throw new SqoopException(SecurityError.ENCRYPTION_0005, exception); + } + } + + /** + * Generates an HMAC based on a SecretKey and some cipherText + * + * @param hmacKey SecretKey which defines the HMAC key and the HMAC algorithm + * @param cipherText Encrypted text from which the HMAC is generated + * @return Base64 representation of the HMAC value + * @throws SqoopException + */ + private String generateHmac(SecretKey hmacKey, String cipherText) throws SqoopException { + try { + Mac hmac = Mac.getInstance(hmacAlgorithm); + hmac.init(hmacKey); + return Base64.encodeBase64String(hmac.doFinal(Base64.decodeBase64(cipherText))); + } catch (GeneralSecurityException exception) { + throw new SqoopException(SecurityError.ENCRYPTION_0005, exception); + } + } + + /** + * Decrypts the provided cipherText using the provided encryptionKey and iv + * + * @param encryptionKey SecretKey which defines the encryption key and algorithm + * @param cipherText Base64 representation of the data we want to decrypt + * @param iv Base64 representation of the initialization vector used when the data was encrypted + * @return Byte array representing the decrypted data + * @throws SqoopException + */ + private byte[] decryptToBytes(SecretKey encryptionKey, String cipherText, String iv) throws SqoopException { + try { + Cipher cipher = Cipher.getInstance(cipherSpec); + cipher.init(Cipher.DECRYPT_MODE, encryptionKey, new IvParameterSpec(Base64.decodeBase64(iv))); + return cipher.doFinal(Base64.decodeBase64(cipherText)); + } catch (GeneralSecurityException exception) { + throw new SqoopException(SecurityError.ENCRYPTION_0006, exception); + } + } + + /** + * Decrypts the provided cipherText using the provided encryptionKey and iv + * + * @param encryptionKey SecretKey which defines the encryption key and algorithm + * @param cipherText Base64 representation of the data we want to decrypt + * @param iv Base64 representation of the initialization vector used when the data was encrypted + * @return String representing the decrypted data using the UTF-8 character set + * @throws SqoopException + */ + private String decryptToString(SecretKey encryptionKey, String cipherText, String iv) throws SqoopException { + return new String(decryptToBytes(encryptionKey, cipherText, iv), Charset.forName("UTF-8")); + } + + /** + * Reads the specified String configuration value from the provided configurationContext, + * throws an exception if the value cannot be found + * + * @param configurationContext MapContext containing the sqoop configuration + * @param configuration Configuration value that we would like from the configurationContext + * @return String value from the configuration + * @throws SqoopException + */ + private String populateStringConfiguration(MapContext configurationContext, String configuration) throws SqoopException { + String value = configurationContext.getString(configuration); + if (StringUtils.isEmpty(value)){ + throw new SqoopException(SecurityError.ENCRYPTION_0009, configuration); + } + return value; + } + + /** + * Reads the specified integer configuration value from the provided configurationContext, + * throws an exception if the value cannot be found + * + * @param configurationContext MapContext containing the sqoop configuration + * @param configuration Configuration value that we would like from the configurationContext + * @return int value from the configuration + * @throws SqoopException + */ + private int populateIntConfiguration(MapContext configurationContext, String configuration) throws SqoopException { + int value = configurationContext.getInt(configuration, 0); + if (value < 1){ + throw new SqoopException(SecurityError.ENCRYPTION_0009, configuration); + } + return value; + } + + /** + * Generates a random byte array of the specified length + * + * @param size number of random bytes to return + * @return byte array containing random bytes of the specified size + */ + @edu.umd.cs.findbugs.annotations.SuppressWarnings("IS2_INCONSISTENT_SYNC") + private byte[] generateRandomByteArray(int size) { + byte[] randomBytes = new byte[size]; + random.nextBytes(randomBytes); + return randomBytes; + } + + /** + * Using the PBKDF2 algorithm, we will generate a Master Key. + * + * @param password The password that will be used for the generation of encryption keys + * @param salt Salt to be used for the generation of encryption keys + * @return byte[] containing the key generated by PDKDF2 + * @throws SqoopException + */ + private byte[] getKeysFromPassword(String password, String salt) throws SqoopException { + try { + PBEKeySpec spec = new PBEKeySpec(password.toCharArray(), Base64.decodeBase64(salt), + pbkdf2Rounds, (cipherKeySize + hmacKeySizeBytes) * 8); + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance(pbkdf2Algorithm); + return secretKeyFactory.generateSecret(spec).getEncoded(); + } catch (GeneralSecurityException exception) { + throw new SqoopException(SecurityError.ENCRYPTION_0003, exception); + } + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/core/src/main/java/org/apache/sqoop/repository/Repository.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java index 03989a3..c2e3c74 100644 --- a/core/src/main/java/org/apache/sqoop/repository/Repository.java +++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java @@ -42,6 +42,7 @@ import org.apache.sqoop.model.MFromConfig; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; import org.apache.sqoop.model.MLinkConfig; +import org.apache.sqoop.model.MMasterKey; import org.apache.sqoop.model.MPersistableEntity; import org.apache.sqoop.model.MSubmission; import org.apache.sqoop.model.MToConfig; @@ -329,6 +330,21 @@ public abstract class Repository { */ public abstract MSubmission findLastSubmissionForJob(String jobName); + /** + * Get the encrypted master key from the repository + * + * @return The encrypted master key, null if no master key exists + */ + public abstract MMasterKey getMasterKey(); + + + /** + * Create the encrypted master key in the repository + * + * @param mMasterKey The encrypted master key + */ + public abstract void createMasterKey(MMasterKey mMasterKey); + /*********************Configurable Upgrade APIs ******************************/ http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java b/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java index 0241c86..9d0974c 100644 --- a/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java +++ b/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java @@ -44,6 +44,13 @@ public final class SecurityConstants { PREFIX_SECURITY_CONFIG + "tls."; /** + * All repository encryption related configuration is prefixed with this: + * <tt>org.apache.sqoop.security.repo_encryption.</tt> + */ + public static final String PREFIX_REPO_ENCRYPTION_CONFIG = + PREFIX_SECURITY_CONFIG + "repo_encryption."; + + /** * The config specifies the sqoop authentication type (SIMPLE, KERBEROS). * The default type is SIMPLE * <tt>org.apache.sqoop.security.authentication.type</tt>. @@ -217,6 +224,78 @@ public final class SecurityConstants { PREFIX_TLS_CONFIG + "keymanager_password_generator"; /** + * The config specifies if repository encryption is enabled + * <tt>org.apache.sqoop.security.repo_encryption.enabled</tt>. + */ + public static final String REPO_ENCRYPTION_ENABLED = + PREFIX_REPO_ENCRYPTION_CONFIG + "enabled"; + + /** + * The config specifies the password used to encrypt the repository + * <tt>org.apache.sqoop.security.repo_encryption.password</tt>. + */ + public static final String REPO_ENCRYPTION_PASSWORD = + PREFIX_REPO_ENCRYPTION_CONFIG + "password"; + + /** + * The config specifies a command that prints the password used to encrypt + * the repository to standard out + * <tt>org.apache.sqoop.security.repo_encryption.password_generator</tt>. + */ + public static final String REPO_ENCRYPTION_PASSWORD_GENERATOR= + PREFIX_REPO_ENCRYPTION_CONFIG + "password_generator"; + + /** + * The config specifies the algorithm to be used for hmac generation + * <tt>org.apache.sqoop.security.repo_encryption.hmac_algorithm</tt>. + */ + public static final String REPO_ENCRYPTION_HMAC_ALGORITHM= + PREFIX_REPO_ENCRYPTION_CONFIG + "hmac_algorithm"; + + /** + * The config specifies the algorithm to be used for repository encryption + * <tt>org.apache.sqoop.security.repo_encryption.cipher_algorithm</tt>. + */ + public static final String REPO_ENCRYPTION_CIPHER_ALGORITHM= + PREFIX_REPO_ENCRYPTION_CONFIG + "cipher_algorithm"; + + /** + * The config specifies the spec to be used for repository encryption + * <tt>org.apache.sqoop.security.repo_encryption.cipher_spec</tt>. + */ + public static final String REPO_ENCRYPTION_CIPHER_SPEC= + PREFIX_REPO_ENCRYPTION_CONFIG + "cipher_spec"; + + /** + * The config specifies the size of the key used for repository encryption + * <tt>org.apache.sqoop.security.repo_encryption.cipher_key_size</tt>. + */ + public static final String REPO_ENCRYPTION_CIPHER_KEY_SIZE= + PREFIX_REPO_ENCRYPTION_CONFIG + "cipher_key_size"; + + /** + * The config specifies the size of the initialization vector used for repository encryption + * <tt>org.apache.sqoop.security.repo_encryption.initialization_vector_size</tt>. + */ + public static final String REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE= + PREFIX_REPO_ENCRYPTION_CONFIG + "initialization_vector_size"; + + /** + * The config specifies the pbkdf2 algorithm to be used for master key generation + * <tt>org.apache.sqoop.security.repo_encryption.pbkdf2_algorithm</tt>. + */ + public static final String REPO_ENCRYPTION_PBKDF2_ALGORITHM= + PREFIX_REPO_ENCRYPTION_CONFIG + "pbkdf2_algorithm"; + + /** + * The config specifies the number of rounds of the pbkdf2 algorithm + * to be used for master key generation + * <tt>org.apache.sqoop.security.repo_encryption.pbkdf2_algorithm</tt>. + */ + public static final String REPO_ENCRYPTION_PBKDF2_ROUNDS= + PREFIX_REPO_ENCRYPTION_CONFIG + "pbkdf2_rounds"; + + /** * The config specifies the token kind in delegation token. */ public static final String TOKEN_KIND = "sqoop_token_kind"; http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/core/src/main/java/org/apache/sqoop/security/SecurityError.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/security/SecurityError.java b/core/src/main/java/org/apache/sqoop/security/SecurityError.java index 988e425..2ba849c 100644 --- a/core/src/main/java/org/apache/sqoop/security/SecurityError.java +++ b/core/src/main/java/org/apache/sqoop/security/SecurityError.java @@ -67,7 +67,43 @@ public enum SecurityError implements ErrorCode { AUTH_0014("Authorization exception"), /** Don't support to grant/remoke privileges for default user. */ - AUTH_0015("Cannot grant/revoke privileges for default user"); + AUTH_0015("Cannot grant/revoke privileges for default user"), + + /** The HMAC calculation yielded a result different than what was stored in the database */ + ENCRYPTION_0001("HMAC validation failed for Master Key"), + + /** The Master Key is found in the database and the creation of one is disabled */ + ENCRYPTION_0002("No Master Key found"), + + /** Calculation of the Master Key from the provided password failed */ + ENCRYPTION_0003("Failed to generate Master Key from password"), + + /** Could not encrypt the provided plaintext */ + ENCRYPTION_0004("Failed to perform encryption"), + + /** HMAC calculation failed */ + ENCRYPTION_0005("Failed to calculate HMAC"), + + /** Could not decrypt the provided ciphertext */ + ENCRYPTION_0006("Failed to perform decryption"), + + /** Could not execute the generator script to create a password */ + ENCRYPTION_0007("Failed to execute password generator"), + + /** No password or password generator found in the configuration file */ + ENCRYPTION_0008("No password or password generator set"), + + /** One of the necessary configuration entries for encryption is missing */ + ENCRYPTION_0009("Invalid configuration"), + + /** HMAC validation failed for input, it may have been tampered with */ + ENCRYPTION_0010("HMAC validation failed for input"), + + /** The HMAC algorithm specified in configuration could not be found */ + ENCRYPTION_0011("HMAC algorithm not found"), + + ; + private final String message; http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java b/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java new file mode 100644 index 0000000..f9579bf --- /dev/null +++ b/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java @@ -0,0 +1,208 @@ + /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.repository; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang.StringUtils; +import org.apache.sqoop.common.MapContext; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.core.SqoopConfiguration; +import org.apache.sqoop.model.MMasterKey; +import org.apache.sqoop.security.SecurityConstants; +import org.mockito.ArgumentCaptor; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +public class TestMasterKeyManager { + private MasterKeyManager masterKeyManager; + private RepositoryManager repositoryManagerMock; + private Repository jdbcRepoMock; + private Map<String, String> configurationMap; + + private static final int HMAC_KEY_SIZE_BYTES = 32; + private static final int CIPHER_KEY_SIZE_BYTES = 16; + + @BeforeMethod(alwaysRun = true) + public void setUp() throws Exception { + SqoopConfiguration configurationMock = mock(SqoopConfiguration.class); + configurationMap = new HashMap<>(); + configurationMap.put(SecurityConstants.REPO_ENCRYPTION_ENABLED, String + .valueOf(true)); + configurationMap.put(SecurityConstants + .REPO_ENCRYPTION_PASSWORD_GENERATOR, "echo youwillnevergetthis"); + configurationMap.put(SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM, + "HmacSHA256"); + configurationMap.put(SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM, + "AES"); + configurationMap.put(SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE, + String.valueOf(CIPHER_KEY_SIZE_BYTES)); + configurationMap.put(SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE, + String.valueOf(CIPHER_KEY_SIZE_BYTES)); + configurationMap.put(SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC, + "AES/CBC/PKCS5Padding"); + configurationMap.put(SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM, + "PBKDF2WithHmacSHA1"); + configurationMap.put(SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS, + "4000"); + doReturn(new MapContext(configurationMap)).when(configurationMock) + .getContext(); + SqoopConfiguration.setInstance(configurationMock); + + repositoryManagerMock = mock(RepositoryManager.class); + RepositoryManager.setInstance(repositoryManagerMock); + + + jdbcRepoMock = mock(JdbcRepository.class); + when(jdbcRepoMock.getMasterKey()).thenReturn(null); + when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock); + + masterKeyManager = MasterKeyManager.getInstance(); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + masterKeyManager.destroy(); + + } + + @Test( + expectedExceptions = {SqoopException.class}, + expectedExceptionsMessageRegExp = ".*No Master Key found") + public void testInitializeWithoutKeyCreationWithoutExistingKey() { + masterKeyManager.initialize(false); + } + + @Test + public void testInitializeWithoutKeyCreationWithExistingKey() { + masterKeyManager.initialize(); + + ArgumentCaptor<MMasterKey> mMasterKeyArgumentCaptor = ArgumentCaptor + .forClass(MMasterKey.class); + verify(jdbcRepoMock, times(1)).createMasterKey(mMasterKeyArgumentCaptor + .capture()); + + // Encrypt something with that master key + String secret = "imasecret"; + String iv = masterKeyManager.generateRandomIv(); + String encrypted = masterKeyManager + .encryptWithMasterKey(secret, iv); + + masterKeyManager.destroy(); + + // Create a new MasterKeyManager instance with existing master key + // coming from the "db" + jdbcRepoMock = mock(JdbcRepository.class); + when(jdbcRepoMock.getMasterKey()).thenReturn(mMasterKeyArgumentCaptor + .getValue()); + when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock); + + masterKeyManager.initialize(); + verify(jdbcRepoMock, times(1)).getMasterKey(); + + // Try to decrypt + assertEquals(masterKeyManager.decryptWithMasterKey(encrypted, iv, masterKeyManager.generateHmacWithMasterHmacKey(encrypted)), secret); + } + + @Test + public void testInitializeWithKeyCreationWithoutExistingKey() { + masterKeyManager.initialize(); + + verify(jdbcRepoMock, times(1)).createMasterKey(any(MMasterKey.class)); + } + + @Test( + expectedExceptions = {SqoopException.class}, + expectedExceptionsMessageRegExp = ".*HMAC validation failed for Master Key" + ) + public void testMasterKeyWithInvalidHmac() { + jdbcRepoMock = mock(JdbcRepository.class); + when(jdbcRepoMock.getMasterKey()).thenReturn(new MMasterKey( + Base64.encodeBase64String(generateRandomByteArray(CIPHER_KEY_SIZE_BYTES)), + Base64.encodeBase64String(generateRandomByteArray(HMAC_KEY_SIZE_BYTES)), + Base64.encodeBase64String(generateRandomByteArray(CIPHER_KEY_SIZE_BYTES)), + Base64.encodeBase64String(generateRandomByteArray(CIPHER_KEY_SIZE_BYTES)) + )); + when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock); + + masterKeyManager.initialize(); + } + + @Test( + expectedExceptions = {SqoopException.class}, + expectedExceptionsMessageRegExp = ".*No password or password generator set") + public void testNoPasswordOrGenerator() { + configurationMap.put(SecurityConstants + .REPO_ENCRYPTION_PASSWORD_GENERATOR, StringUtils.EMPTY); + + masterKeyManager.initialize(); + } + + @Test + public void testEncryptAndDecryptWithMasterKey() { + masterKeyManager.initialize(); + + String secret = "imasecret"; + String iv = masterKeyManager.generateRandomIv(); + String encrypted = masterKeyManager + .encryptWithMasterKey(secret, iv); + + assertEquals(masterKeyManager.decryptWithMasterKey(encrypted, iv, masterKeyManager.generateHmacWithMasterHmacKey(encrypted)), secret); + } + + @Test( + expectedExceptions = {SqoopException.class}, + expectedExceptionsMessageRegExp = ".*HMAC validation failed for input") + public void testEncryptAndDecryptWithMasterKeyWithInvalidHmac() { + masterKeyManager.initialize(); + + String secret = "imasecret"; + String iv = masterKeyManager.generateRandomIv(); + String encrypted = masterKeyManager.encryptWithMasterKey(secret, iv); + + String invalidHmac = Base64.encodeBase64String(generateRandomByteArray(HMAC_KEY_SIZE_BYTES)); + masterKeyManager.decryptWithMasterKey(encrypted, iv, invalidHmac); + } + + @Test( + expectedExceptions = {SqoopException.class}, + expectedExceptionsMessageRegExp = ".*Invalid configuration.*" + + SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM) + public void testMissingConfiguration() { + configurationMap.put(SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM, ""); + masterKeyManager.initialize(); + } + + private static byte[] generateRandomByteArray(int size) { + byte[] randomBytes = new byte[size]; + new Random().nextBytes(randomBytes); + return randomBytes; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/dist/src/main/conf/sqoop.properties ---------------------------------------------------------------------- diff --git a/dist/src/main/conf/sqoop.properties b/dist/src/main/conf/sqoop.properties index 767d3f2..58b60fd 100755 --- a/dist/src/main/conf/sqoop.properties +++ b/dist/src/main/conf/sqoop.properties @@ -188,6 +188,21 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce #org.apache.sqoop.security.tls.keystore= #org.apache.sqoop.security.tls.keystore_password= +# +# Repository Encryption +# + +#org.apache.sqoop.security.repo_encryption.enabled=true +#org.apache.sqoop.security.repo_encryption.password= +#org.apache.sqoop.security.repo_encryption.password_generator= +#org.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 +#org.apache.sqoop.security.repo_encryption.cipher_algorithm=AES +#org.apache.sqoop.security.repo_encryption.cipher_key_size=128 +#org.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding +#org.apache.sqoop.security.repo_encryption.initialization_vector_size=128 +#org.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 +#org.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 + # External connectors load path # "/path/to/external/connectors/": Add all the connector JARs in the specified folder http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java index 15cc41b..a9168ae 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java @@ -42,6 +42,7 @@ import org.apache.sqoop.common.MutableMapContext; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.common.SupportedDirections; import org.apache.sqoop.connector.ConnectorManager; +import org.apache.sqoop.core.SqoopConfiguration; import org.apache.sqoop.driver.Driver; import org.apache.sqoop.error.code.CommonRepositoryError; import org.apache.sqoop.model.InputEditable; @@ -65,11 +66,14 @@ import org.apache.sqoop.model.MLinkConfig; import org.apache.sqoop.model.MListInput; import org.apache.sqoop.model.MLongInput; import org.apache.sqoop.model.MMapInput; +import org.apache.sqoop.model.MMasterKey; import org.apache.sqoop.model.MStringInput; import org.apache.sqoop.model.MSubmission; import org.apache.sqoop.model.MToConfig; import org.apache.sqoop.model.SubmissionError; import org.apache.sqoop.repository.JdbcRepositoryHandler; +import org.apache.sqoop.repository.MasterKeyManager; +import org.apache.sqoop.security.SecurityConstants; import org.apache.sqoop.submission.SubmissionStatus; import org.apache.sqoop.submission.counter.Counter; import org.apache.sqoop.submission.counter.CounterGroup; @@ -1147,6 +1151,56 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { } } + /** + * {@inheritDoc} + */ + @Override + public MMasterKey getMasterKey(Connection conn) { + try (PreparedStatement stmt = conn.prepareStatement(crudQueries.getStmtSelectSqMasterKey())) { + stmt.setMaxRows(1); + + try (ResultSet rs = stmt.executeQuery()) { + if (!rs.next()) { + return null; + } + + String aesKey = rs.getString(1); + String hmac = rs.getString(2); + String salt = rs.getString(3); + String iv = rs.getString(4); + + return new MMasterKey(aesKey, hmac, salt, iv); + } + } catch (SQLException ex) { + logException(ex); + throw new SqoopException(CommonRepositoryError.COMMON_0059, ex); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void createMasterKey(MMasterKey mMasterKey, Connection conn) { + int result; + try (PreparedStatement preparedStatement = conn.prepareStatement(crudQueries.getStmtInsertSqMasterKey(), + Statement.RETURN_GENERATED_KEYS)) { + preparedStatement.setString(1, mMasterKey.getEncryptedSecret()); + preparedStatement.setString(2, mMasterKey.getHmac()); + preparedStatement.setString(3, mMasterKey.getSalt()); + preparedStatement.setString(4, mMasterKey.getIv()); + + result = preparedStatement.executeUpdate(); + if (result != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0009, + Integer.toString(result)); + } + } catch (SQLException ex) { + logException(ex, mMasterKey); + throw new SqoopException(CommonRepositoryError.COMMON_0031, ex); + } + } + private void insertConnectorDirection(Long connectorId, Direction direction, Connection conn) throws SQLException { try (PreparedStatement stmt = conn.prepareStatement(crudQueries.getStmtInsertSqConnectorDirections())) { @@ -2071,8 +2125,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { // get the overrides value from the SQ_INPUT_RELATION table String overrides = getOverrides(inputId, conn); String inputEnumValues = rsetInput.getString(9); - String value = rsetInput.getString(10); - + String value = readInputValue(rsetInput.getString(10), rsetInput.getBoolean(11), rsetInput.getString(12), rsetInput.getString(13)); MInputType mit = MInputType.valueOf(inputType); MInput input = null; switch (mit) { @@ -2184,7 +2237,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { while (inputResults.next()) { long inputId = inputResults.getLong(1); String inputName = inputResults.getString(2); - String value = inputResults.getString(10); + String value = readInputValue(inputResults.getString(10), inputResults.getBoolean(11), inputResults.getString(12), inputResults.getString(13)); if (mConfig.getName().equals(configName) && mConfig.getInputNames().contains(inputName)) { MInput mInput = mConfig.getInput(inputName); mInput.setPersistenceId(inputId); @@ -2199,6 +2252,24 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { } } + + /** + * Reads the value of an input, while handling the possibility that the input is encrypted + * + * @param possiblyEncryptedValue Cleartext or base64 encoded ciphertext representing the input + * @param encrypted Is the input encrypted + * @param iv Encryption initialization vector + * @param hmac HMAC for tamper resistance + * @return The input value + */ + private String readInputValue(String possiblyEncryptedValue, boolean encrypted, String iv, String hmac) throws SqoopException { + if (encrypted) { + return MasterKeyManager.getInstance().decryptWithMasterKey(possiblyEncryptedValue, iv, hmac); + } else { + return possiblyEncryptedValue; + } + } + /** * Load configs and corresponding inputs related to a connector. * @@ -2247,7 +2318,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { // get the overrides value from the SQ_INPUT_RELATION table String overrides = getOverrides(inputId, conn); String inputEnumValues = rsetInput.getString(9); - String value = rsetInput.getString(10); + String value = readInputValue(rsetInput.getString(10), rsetInput.getBoolean(11), rsetInput.getString(12), rsetInput.getString(13)); MInputType mit = MInputType.valueOf(inputType); @@ -2401,24 +2472,40 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { throws SQLException { int result; - try (PreparedStatement stmt = conn.prepareStatement(query)) { + boolean encryptionEnabled = SqoopConfiguration.getInstance().getContext().getBoolean(SecurityConstants.REPO_ENCRYPTION_ENABLED, false); + MasterKeyManager masterKeyManager = MasterKeyManager.getInstance(); for (MConfig config : configs) { for (MInput<?> input : config.getInputs()) { // Skip empty values as we're not interested in storing those in db if (input.isEmpty()) { continue; } - stmt.setLong(1, id); - stmt.setLong(2, input.getPersistenceId()); - stmt.setString(3, input.getUrlSafeValueString()); - result = stmt.executeUpdate(); + try (PreparedStatement stmt = conn.prepareStatement(query)) { + stmt.setLong(1, id); + stmt.setLong(2, input.getPersistenceId()); + if (input.isSensitive() && encryptionEnabled) { + String iv = MasterKeyManager.getInstance().generateRandomIv(); + String hmac = null; + String encryptedInput = masterKeyManager.encryptWithMasterKey(input.getUrlSafeValueString(), iv); + stmt.setString(3, encryptedInput); + hmac = masterKeyManager.generateHmacWithMasterHmacKey(encryptedInput); + stmt.setBoolean(4, true); + stmt.setString(5, iv); + stmt.setString(6, hmac); + } else { + stmt.setString(3, input.getUrlSafeValueString()); + stmt.setBoolean(4, false); + stmt.setNull(5, Types.VARCHAR); + stmt.setNull(6, Types.VARCHAR); + } + result = stmt.executeUpdate(); + } if (result != 1) { throw new SqoopException(CommonRepositoryError.COMMON_0017, Integer.toString(result)); } } } - } } /**
