Repository: sqoop Updated Branches: refs/heads/sqoop2 bfcfedfa7 -> 5b897a46f
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java index ae16b85..d527290 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java @@ -161,6 +161,9 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", " + + "cast(null as varchar(100))," + + "false," + + "cast(null as varchar(100))," + "cast(null as varchar(100))" + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + " = ?" @@ -223,7 +226,10 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + "," + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", " - + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC) + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " LEFT OUTER JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + " = " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) @@ -246,7 +252,10 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", " - + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC) + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " LEFT OUTER JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + " = " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) @@ -271,11 +280,14 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { // DML: Insert new link inputs private static final String STMT_INSERT_LINK_INPUT = - "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + " (" - + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_LINK) + ", " - + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + ", " - + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) - + ") VALUES (?, ?, ?)"; + "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + " (" + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_LINK) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC) + + ") VALUES (?, ?, ?, ?, ?, ?)"; // DML: Update link private static final String STMT_UPDATE_LINK = @@ -385,13 +397,17 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + CommonRepoUtils.escapeColumnName(COLUMN_SQB_UPDATE_DATE) + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; + // DML: Insert new job inputs private static final String STMT_INSERT_JOB_INPUT = - "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + " (" - + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_JOB) + ", " - + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + ", " - + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) - + ") VALUES (?, ?, ?)"; + "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + " (" + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_JOB) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC) + + ") VALUES (?, ?, ?, ?, ?, ?)"; private static final String STMT_UPDATE_JOB = "UPDATE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_NAME) + " SET " @@ -741,6 +757,22 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + " IN (SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_ID) + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIG_NAME) + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_CONFIGURABLE) + " = ?)"; + private static final String STMT_SELECT_SQ_MASTER_KEY = + "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV) + + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME); + + private static final String STMT_INSERT_SQ_MASTER_KEY = + "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME) + " (" + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV) + + ") VALUES (?, ?, ?, ?)"; + + public String getStmtSelectSqdIdBySqdName() { return STMT_SELECT_SQD_ID_BY_SQD_NAME; } @@ -1042,4 +1074,12 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { public String getStmtInsertIntoInputRelation() { return STMT_INSERT_INTO_INPUT_RELATION; } + + public String getStmtSelectSqMasterKey() { + return STMT_SELECT_SQ_MASTER_KEY; + } + + public String getStmtInsertSqMasterKey() { + return STMT_INSERT_SQ_MASTER_KEY; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java ---------------------------------------------------------------------- diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java index d1940e8..6fd9d6c 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java @@ -175,10 +175,16 @@ public final class CommonRepositorySchemaConstants { public static final String COLUMN_SQ_LNKI_LINK = "SQ_LNKI_LINK"; + public static final String COLUMN_SQ_LNKI_ENCRYPTED = "SQ_LNKI_ENCRYPTED"; + public static final String COLUMN_SQ_LNKI_INPUT = "SQ_LNKI_INPUT"; public static final String COLUMN_SQ_LNKI_VALUE = "SQ_LNKI_VALUE"; + public static final String COLUMN_SQ_LNKI_IV = "SQ_LNKI_IV"; + + public static final String COLUMN_SQ_LNKI_HMAC = "SQ_LNKI_HMAC"; + // SQ_JOB_INPUT public static final String TABLE_SQ_JOB_INPUT_NAME = "SQ_JOB_INPUT"; @@ -187,8 +193,14 @@ public final class CommonRepositorySchemaConstants { public static final String COLUMN_SQBI_INPUT = "SQBI_INPUT"; + public static final String COLUMN_SQBI_ENCRYPTED = "SQBI_ENCRYPTED"; + public static final String COLUMN_SQBI_VALUE = "SQBI_VALUE"; + public static final String COLUMN_SQBI_IV = "SQBI_IV"; + + public static final String COLUMN_SQBI_HMAC = "SQBI_HMAC"; + // SQ_SUBMISSION public static final String TABLE_SQ_SUBMISSION_NAME = "SQ_SUBMISSION"; @@ -273,6 +285,20 @@ public final class CommonRepositorySchemaConstants { public static final String COLUMN_SQCO_VALUE = "SQCO_VALUE"; + // SQ_MASTER_KEY + + public static final String TABLE_SQ_MASTER_KEY_NAME = "SQ_MASTER_KEY"; + + public static final String COLUMN_SQMK_ID = "SQMK_ID"; + + public static final String COLUMN_SQMK_SECRET = "SQMK_SECRET"; + + public static final String COLUMN_SQMK_HMAC = "SQMK_HMAC"; + + public static final String COLUMN_SQMK_SALT = "SQMK_SALT"; + + public static final String COLUMN_SQMK_IV = "SQMK_IV"; + // Constraints public static final String CONSTRAINT_SQCD_SQC_NAME http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java index ee5e8d1..75636d4 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java @@ -319,6 +319,14 @@ public class DerbyRepositoryHandler extends CommonRepositoryHandler { runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_COLUMN_SQB_NAME, conn); runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ALTER_COLUMN_SQB_NAME_NOT_NULL, conn); runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIGURABLE_ALTER_COLUMN_SQB_NAME_NOT_NULL, conn); + + runQuery(QUERY_CREATE_TABLE_SQ_MASTER_KEY, conn); + runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED, conn); + runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV, conn); + runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC, conn); + runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED, conn); + runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV, conn); + runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC, conn); } // last step upgrade the repository version to the latest value in the code http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java index 1770036..b4f66bb 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java @@ -181,6 +181,10 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*; * <p> * <strong>SQ_LINK_INPUT</strong>: N:M relationship link and input * + * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum length + * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where + * 1024 is the maximum possible key size in bits for the IV and the digest. + * * <pre> * +----------------------------+ * | SQ_LINK_INPUT | @@ -188,6 +192,9 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*; * | SQ_LNKI_LINK: BIGINT PK | FK SQ_LINK(SQ_LNK_ID) * | SQ_LNKI_INPUT: BIGINT PK | FK SQ_INPUT(SQI_ID) * | SQ_LNKI_VALUE: LONG VARCHAR| + * | SQ_LNKI_ENCRYPTED: BOOLEAN | + * | SQ_LNKI_IV: VARCHAR(171) | + * | SQ_LNKI_HMAC: VARCHAR(171) | * +----------------------------+ * </pre> * @@ -195,6 +202,10 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*; * <p> * <strong>SQ_JOB_INPUT</strong>: N:M relationship job and input * + * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum length + * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where + * 1024 is the maximum possible key size in bits for the IV and the digest. + * * <pre> * +----------------------------+ * | SQ_JOB_INPUT | @@ -202,6 +213,9 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*; * | SQBI_JOB: BIGINT PK | FK SQ_JOB(SQB_ID) * | SQBI_INPUT: BIGINT PK | FK SQ_INPUT(SQI_ID) * | SQBI_VALUE: LONG VARCHAR | + * | SQBI_ENCRYPTED: BOOLEAN | + * | SQBI_IV: VARCHAR(24) | + * | SQBI_HMAC: VARCHAR(171) | * +----------------------------+ * </pre> * @@ -310,6 +324,34 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*; * +----------------------------+ * </pre> * </p> + * <p> + * <strong>SQ_MASTER_KEY</strong>: Master Key for sensitive value encryption + * + * SQMK_SECRET length determined by a generous assumption of the maximum length + * of the base64 encoded secret. Calculated by ceil((4/3)(1024/8+1024/8)). Where + * 1024 is the maximum possible key size in bits for encryption and HMAC. + * + * SQMK_HMAC length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8)) + * where 1024 is the maximum possible size of the digest in bits. + * + * SQMK_SALT length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8)) + * where 1024 is the maximum possible size of the salt in bits. + * + * SQMK_IV length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(128)) + * where 128 is the maximum possible size of the salt in bytes. + * + * <pre> + * +---------------------------+ + * | SQ_MASTER_KEY | + * +---------------------------+ + * | SQMK_ID: BIGINT PK | + * | SQMK_SECRET: VARCHAR(342) | + * | SQMK_HMAC: VARCHAR(171) | + * | SQMK_SALT: VARCHAR(171) | + * | SQMK_IV: VARCHAR(171) | + * +---------------------------+ + * </pre> + * </p> */ // NOTE: If you have signed yourself to modify the schema for the repository http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java index 5081b82..a427421 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java @@ -662,6 +662,41 @@ public final class DerbySchemaUpgradeQuery { + " ALTER COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQC_NAME) + " NOT NULL"; + // DDL: Create table SQ_MASTER_KEY + public static final String QUERY_CREATE_TABLE_SQ_MASTER_KEY = + "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME) + " (" + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_ID) + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1), " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + " VARCHAR(342), " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + " VARCHAR(171), " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + " VARCHAR(171), " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV) + " VARCHAR(171)" + + ")"; + + public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + " BOOLEAN DEFAULT FALSE"; + + public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + " VARCHAR(171)"; + + public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC) + " VARCHAR(171)"; + + public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + " BOOLEAN DEFAULT FALSE"; + + public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + " VARCHAR(171)"; + + public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC) + " VARCHAR(171)"; + + private DerbySchemaUpgradeQuery() { // Disable explicit object creation } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java index e4cca07..9bcfe01 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java @@ -247,6 +247,14 @@ abstract public class DerbyTestCase { runQuery(QUERY_CREATE_TABLE_SQ_CONTEXT_TYPE); runQuery(QUERY_CREATE_TABLE_SQ_CONTEXT_PROPERTY); runQuery(QUERY_CREATE_TABLE_SQ_CONTEXT); + + runQuery(QUERY_CREATE_TABLE_SQ_MASTER_KEY); + runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED); + runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV); + runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC); + runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED); + runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV); + runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC); } // deprecated repository version http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java index 2c74c32..87bd6d8 100644 --- a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java +++ b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java @@ -146,6 +146,7 @@ public class MySqlRepositoryHandler extends CommonRepositoryHandler { runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_CONTEXT_TYPE, conn); runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_CONTEXT_PROPERTY, conn); runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_CONTEXT, conn); + runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_MASTER_KEY, conn); // Insert FROM and TO directions. insertDirections(conn); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java index 47f12fe..29355d5 100644 --- a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java +++ b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java @@ -24,7 +24,7 @@ import org.apache.sqoop.repository.common.CommonRepositorySchemaConstants; public class MySqlSchemaCreateQuery { - public static final String QUERY_CREATE_DATABASE_SQOOP = "CREATE DATABASE " + CommonRepoUtils.escapeDatabaseName(SCHEMA_SQOOP); + public static final String QUERY_CREATE_DATABASE_SQOOP = "CREATE DATABASE IF NOT EXISTS" + CommonRepoUtils.escapeDatabaseName(SCHEMA_SQOOP); public static final String QUERY_CREATE_TABLE_SQ_SYSTEM = "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, CommonRepositorySchemaConstants.TABLE_SQ_SYSTEM_NAME) + " (" @@ -177,6 +177,9 @@ public class MySqlSchemaCreateQuery { + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_LINK) + " BIGINT, " + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_INPUT) + " BIGINT, " + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_VALUE) + " VARCHAR(1000), " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_ENCRYPTED) + " BOOLEAN DEFAULT FALSE, " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_IV) + " VARCHAR(171), " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_HMAC) + " VARCHAR(171)," + "PRIMARY KEY (" + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_LINK) + ", " + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_INPUT) + "), " + "CONSTRAINT " + CommonRepoUtils.escapeConstraintName(CommonRepositorySchemaConstants.CONSTRAINT_SQ_LNKI_SQ_LNK_NAME) @@ -194,6 +197,9 @@ public class MySqlSchemaCreateQuery { + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_JOB) + " BIGINT, " + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_INPUT) + " BIGINT, " + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_VALUE) + " VARCHAR(1000), " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_ENCRYPTED) + " BOOLEAN DEFAULT FALSE, " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_IV) + " VARCHAR(171), " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_HMAC) + " VARCHAR(171)," + "PRIMARY KEY (" + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_JOB) + ", " + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_INPUT) + "), " + "CONSTRAINT " + CommonRepoUtils.escapeConstraintName(CommonRepositorySchemaConstants.CONSTRAINT_SQBI_SQB_NAME) @@ -291,6 +297,16 @@ public class MySqlSchemaCreateQuery { + "(" + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQCP_ID) + ") " + ")"; + // DDL: Create table SQ_MASTER_KEY + public static final String QUERY_CREATE_TABLE_SQ_MASTER_KEY = + "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, CommonRepositorySchemaConstants.TABLE_SQ_MASTER_KEY_NAME) + " (" + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_ID) + " BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_SECRET) + " VARCHAR(342), " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_HMAC) + " VARCHAR(171), " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_SALT) + " VARCHAR(171), " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_IV) + " VARCHAR(171)" + + ")"; + private MySqlSchemaCreateQuery() { // Disable explicit object creation } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java index 4c295c0..08d3374 100644 --- a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java +++ b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java @@ -36,6 +36,7 @@ public class MysqlRepositoryInsertUpdateDeleteSelectQuery extends CommonRepositoryInsertUpdateDeleteSelectQuery { // DML: Get inputs for a given config + // MySQL requires that we cast to null to char instead of varchar private static final String STMT_SELECT_INPUT = "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", " @@ -46,6 +47,9 @@ public class MysqlRepositoryInsertUpdateDeleteSelectQuery extends + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", " + + "cast(null as char(100))," + + "false," + + "cast(null as char(100))," + "cast(null as char(100))" + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java index 400d706..b18a7d5 100644 --- a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java +++ b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java @@ -145,6 +145,13 @@ public class PostgresqlRepositoryHandler extends CommonRepositoryHandler { runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_COLUMN_SQB_NAME, conn); runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ALTER_COLUMN_SQB_NAME_NOT_NULL, conn); runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_CONFIGURABLE_ALTER_COLUMN_SQB_NAME_NOT_NULL, conn); + runQuery(PostgresqlSchemaUpgradeQuery.QUERY_CREATE_TABLE_SQ_MASTER_KEY, conn); + runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED, conn); + runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV, conn); + runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC, conn); + runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED, conn); + runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV, conn); + runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC, conn); } try (PreparedStatement stmtDel = conn.prepareStatement(PostgresqlSchemaQuery.STMT_DELETE_SYSTEM); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java index 8358df0..e31886e 100644 --- a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java +++ b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java @@ -169,6 +169,11 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants * </p> * <p> * <strong>SQ_LINK_INPUT</strong>: N:M relationship link and input + * + * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum length + * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where + * 1024 is the maximum possible key size in bits for the IV and the digest. + * * <pre> * +----------------------------+ * | SQ_LINK_INPUT | @@ -176,11 +181,19 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants * | SQ_LNK_LINK: BIGSERIAL | FK SQ_LINK(SQ_LNK_ID) * | SQ_LNK_INPUT: BIGINT | FK SQ_INPUT(SQI_ID) * | SQ_LNK_VALUE: VARCHAR | + * | SQ_LNK_ENCRYPTED: BOOLEAN | + * | SQ_LNKI_IV: VARCHAR(171) | + * | SQ_LNKI_HMAC: VARCHAR(171) | * +----------------------------+ * </pre> * </p> * <p> * <strong>SQ_JOB_INPUT</strong>: N:M relationship job and input + * + * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum length + * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where + * 1024 is the maximum possible key size in bits for the IV and the digest. + * * <pre> * +----------------------------+ * | SQ_JOB_INPUT | @@ -188,6 +201,9 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants * | SQBI_JOB: BIGINT | FK SQ_JOB(SQB_ID) * | SQBI_INPUT: BIGINT | FK SQ_INPUT(SQI_ID) * | SQBI_VALUE: VARCHAR(1000) | + * | SQBI_ENCRYPTED: BOOLEAN | + * | SQBI_IV: VARCHAR(171) | + * | SQBI_HMAC: VARCHAR(171) | * +----------------------------+ * </pre> * </p> @@ -287,6 +303,34 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants * +----------------------------+ * </pre> * </p> + * <p> + * <strong>SQ_MASTER_KEY</strong>: Master Key for sensitive value encryption + * + * SQMK_SECRET length determined by a generous assumption of the maximum length + * of the base64 encoded secret. Calculated by ceil((4/3)(1024/8+1024/8)). Where + * 1024 is the maximum possible key size in bits for encryption and HMAC. + * + * SQMK_HMAC length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8)) + * where 1024 is the maximum possible size of the digest in bits. + * + * SQMK_SALT length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8)) + * where 1024 is the maximum possible size of the salt in bits. + * + * SQMK_IV length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(128)) + * where 128 is the maximum possible size of the salt in bytes. + * + * <pre> + * +---------------------------+ + * | SQ_MASTER_KEY | + * +---------------------------+ + * | SQMK_ID: BIGINT PK | + * | SQMK_SECRET: VARCHAR(342) | + * | SQMK_HMAC: VARCHAR(171) | + * | SQMK_SALT: VARCHAR(171) | + * | SQMK_IV: VARCHAR(171) | + * +---------------------------+ + * </pre> + * </p> */ public class PostgresqlSchemaCreateQuery { http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java index 52954e6..3021039 100644 --- a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java +++ b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java @@ -17,15 +17,30 @@ */ package org.apache.sqoop.repository.postgresql; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQBI_ENCRYPTED; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQBI_HMAC; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQBI_IV; import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQB_ID; import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQC_NAME; + +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_HMAC; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_ID; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_IV; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_SALT; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_SECRET; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_ENCRYPTED; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_HMAC; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_IV; import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNK_ID; import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNK_NAME; import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQB_NAME; import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.SCHEMA_SQOOP; import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_CONFIGURABLE_NAME; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_JOB_INPUT_NAME; import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_JOB_NAME; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_LINK_INPUT_NAME; import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_LINK_NAME; +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_MASTER_KEY_NAME; import org.apache.sqoop.repository.common.CommonRepoUtils; @@ -58,4 +73,37 @@ public class PostgresqlSchemaUpgradeQuery { "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIGURABLE_NAME) + " ALTER COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQC_NAME) + " SET NOT NULL"; + + public static final String QUERY_CREATE_TABLE_SQ_MASTER_KEY = + "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME) + " (" + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_ID) + " BIGSERIAL PRIMARY KEY NOT NULL, " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + " VARCHAR(342), " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + " VARCHAR(171), " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + " VARCHAR(171), " + + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV) + " VARCHAR(171)" + + ")"; + + public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + " BOOLEAN DEFAULT FALSE"; + + public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + " VARCHAR(171)"; + + public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC) + " VARCHAR(171)"; + + public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + " BOOLEAN DEFAULT FALSE"; + + public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + " VARCHAR(171)"; + + public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC = + "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC) + " VARCHAR(171)"; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java index 00aa214..c2f5b84 100644 --- a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java +++ b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java @@ -34,7 +34,7 @@ import org.apache.sqoop.server.v1.DriverServlet; import org.apache.sqoop.server.v1.JobServlet; import org.apache.sqoop.server.v1.LinkServlet; import org.apache.sqoop.server.v1.SubmissionsServlet; -import org.apache.sqoop.utils.ProcessUtils; +import org.apache.sqoop.utils.PasswordUtils; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.SecureRequestCustomizer; @@ -46,7 +46,6 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.ExecutorThreadPool; import javax.servlet.DispatcherType; -import java.io.IOException; import java.util.EnumSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; @@ -94,40 +93,19 @@ public class SqoopJettyServer { sslContextFactory.setProtocol(protocol.trim()); } - String keyStorePassword = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD); - String keyStorePasswordGenerator = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD_GENERATOR); - if (StringUtils.isNotBlank(keyStorePassword)) { - if (StringUtils.isNotBlank(keyStorePasswordGenerator)) { - LOG.warn(SecurityConstants.KEYSTORE_PASSWORD + " and " + SecurityConstants.KEYSTORE_PASSWORD_GENERATOR - + "are both set, using " + SecurityConstants.KEYSTORE_PASSWORD); - } + String keyStorePassword = PasswordUtils.readPassword(configurationContext, SecurityConstants.KEYSTORE_PASSWORD, + SecurityConstants.KEYSTORE_PASSWORD_GENERATOR); + if (StringUtils.isNotEmpty(keyStorePassword)) { sslContextFactory.setKeyStorePassword(keyStorePassword); - } else if (StringUtils.isNotBlank(keyStorePasswordGenerator)) { - try { - String passwordFromGenerator = ProcessUtils.readOutputFromGenerator(keyStorePasswordGenerator); - sslContextFactory.setKeyStorePassword(passwordFromGenerator); - } catch (IOException exception) { - throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator: " + SecurityConstants.KEYSTORE_PASSWORD_GENERATOR, exception); - } } - String keyManagerPassword = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD); - String keyManagerPasswordGenerator = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR); - if (StringUtils.isNotBlank(keyManagerPassword)) { + String keyManagerPassword = PasswordUtils.readPassword(configurationContext, SecurityConstants.KEYMANAGER_PASSWORD, + SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR); + if (StringUtils.isNotEmpty(keyManagerPassword)) { sslContextFactory.setKeyManagerPassword(keyManagerPassword); - if (StringUtils.isNotBlank(keyManagerPasswordGenerator)) { - LOG.warn(SecurityConstants.KEYMANAGER_PASSWORD + " and " + SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR - + "are both set, using " + SecurityConstants.KEYMANAGER_PASSWORD); - } - } else if (StringUtils.isNotBlank(keyManagerPasswordGenerator)) { - try { - String passwordFromGenerator = ProcessUtils.readOutputFromGenerator(keyManagerPasswordGenerator); - sslContextFactory.setKeyManagerPassword(passwordFromGenerator); - } catch (IOException exception) { - throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator: " + SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR, exception); - } } + HttpConfiguration https = new HttpConfiguration(); https.addCustomizer(new SecureRequestCustomizer()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java b/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java index 0b603a1..acd055d 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java @@ -20,7 +20,7 @@ package org.apache.sqoop.shell; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.OptionBuilder; import org.apache.sqoop.shell.core.Constants; -import org.apache.sqoop.utils.ProcessUtils; +import org.apache.sqoop.utils.PasswordUtils; import org.apache.sqoop.validation.Status; import javax.net.ssl.HttpsURLConnection; @@ -70,7 +70,7 @@ public class SetTruststoreFunction extends SqoopFunction { truststorePassword = line.getOptionValue(Constants.OPT_TRUSTSTORE_PASSWORD).toCharArray(); } else if (line.hasOption(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR)) { String generator = line.getOptionValue(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR); - truststorePassword = ProcessUtils.readOutputFromGenerator(generator).toCharArray(); + truststorePassword = PasswordUtils.readOutputFromGenerator(generator).toCharArray(); } KeyStore keyStore = KeyStore.getInstance("JKS"); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java index c7a4db8..9fa9467 100644 --- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java @@ -22,6 +22,7 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.sqoop.core.ConfigurationConstants; import org.apache.sqoop.common.test.repository.RepositoryProviderFactory; +import org.apache.sqoop.security.SecurityConstants; import org.apache.sqoop.test.kdc.KdcRunner; import java.io.File; @@ -231,6 +232,16 @@ public abstract class SqoopMiniCluster { properties.put("org.apache.sqoop.authentication.proxyuser." + "hadoop" + ".groups", "*"); properties.put("org.apache.sqoop.authentication.proxyuser." + "hadoop" + ".hosts", "*"); + properties.put("org.apache.sqoop.security.repo_encryption.enabled", String.valueOf(true)); + properties.put("org.apache.sqoop.security.repo_encryption.password_generator", "echo youwillnevergetthis"); + properties.put("org.apache.sqoop.security.repo_encryption.hmac_algorithm", "HmacSHA256"); + properties.put("org.apache.sqoop.security.repo_encryption.cipher_algorithm", "AES"); + properties.put("org.apache.sqoop.security.repo_encryption.cipher_key_size", "16"); + properties.put("org.apache.sqoop.security.repo_encryption.initialization_vector_size", "16"); + properties.put("org.apache.sqoop.security.repo_encryption.cipher_spec", "AES/CBC/PKCS5Padding"); + properties.put("org.apache.sqoop.security.repo_encryption.pbkdf2_algorithm", "PBKDF2WithHmacSHA1"); + properties.put("org.apache.sqoop.security.repo_encryption.pbkdf2_rounds", "4000"); + return properties; }
