Repository: sqoop Updated Branches: refs/heads/sqoop2 6fc50b08b -> d3062f340
SQOOP-2072: Sqoop2: Add editable and override attributes for postgres (Veena Basavaraj via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/d3062f34 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/d3062f34 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/d3062f34 Branch: refs/heads/sqoop2 Commit: d3062f340f511fdc59d234ca14b349b7c5a7446a Parents: 6fc50b0 Author: Abraham Elmahrek <[email protected]> Authored: Thu Feb 5 13:16:46 2015 -0800 Committer: Abraham Elmahrek <[email protected]> Committed: Thu Feb 5 13:16:46 2015 -0800 ---------------------------------------------------------------------- .../repository/derby/TestConnectorHandling.java | 5 ++++ .../repository/derby/TestDriverHandling.java | 1 + .../postgresql/PostgresqlRepositoryHandler.java | 1 + .../postgresql/PostgresqlSchemaCreateQuery.java | 30 ++++++++++++++++++-- .../postgresql/PostgresqlTestCase.java | 8 +++--- .../postgresql/PostgresqlTestUtils.java | 1 - .../postgresql/TestConnectorHandling.java | 4 +++ .../postgresql/TestDriverHandling.java | 1 + .../repository/postgresql/TestStructure.java | 3 ++ 9 files changed, 46 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/d3062f34/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java index 1056c69..cacba4c 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java @@ -85,6 +85,8 @@ public class TestConnectorHandling extends DerbyTestCase { assertCountForTable("SQOOP.SQ_CONFIGURABLE", 1); assertCountForTable("SQOOP.SQ_CONFIG", 6); assertCountForTable("SQOOP.SQ_INPUT", 12); + assertCountForTable("SQOOP.SQ_INPUT_RELATION", 9); + // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", getDerbyDatabaseConnection()); @@ -122,6 +124,7 @@ public class TestConnectorHandling extends DerbyTestCase { assertCountForTable("SQOOP.SQ_CONFIGURABLE", 1); assertCountForTable("SQOOP.SQ_CONFIG", 4); assertCountForTable("SQOOP.SQ_INPUT", 8); + assertCountForTable("SQOOP.SQ_INPUT_RELATION", 6); // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", getDerbyDatabaseConnection()); @@ -142,6 +145,7 @@ public class TestConnectorHandling extends DerbyTestCase { assertCountForTable("SQOOP.SQ_CONFIGURABLE", 1); assertCountForTable("SQOOP.SQ_CONFIG", 4); assertCountForTable("SQOOP.SQ_INPUT", 8); + assertCountForTable("SQOOP.SQ_INPUT_RELATION", 6); // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", getDerbyDatabaseConnection()); @@ -162,6 +166,7 @@ public class TestConnectorHandling extends DerbyTestCase { assertCountForTable("SQOOP.SQ_CONFIGURABLE", 1); assertCountForTable("SQOOP.SQ_CONFIG", 2); assertCountForTable("SQOOP.SQ_INPUT", 4); + assertCountForTable("SQOOP.SQ_INPUT_RELATION", 3); // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", getDerbyDatabaseConnection()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/d3062f34/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java index a2c50f8..4a81827 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java @@ -75,6 +75,7 @@ public class TestDriverHandling extends DerbyTestCase { assertCountForTable("SQOOP.SQ_CONNECTOR", 0); assertCountForTable("SQOOP.SQ_CONFIG", 2); assertCountForTable("SQOOP.SQ_INPUT", 4); + assertCountForTable("SQOOP.SQ_INPUT_RELATION", 4); // Registered driver and config should be easily recovered back MDriver retrieved = handler.findDriver(MDriver.DRIVER_NAME, getDerbyDatabaseConnection()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/d3062f34/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java index e72c8c1..3740f8d 100644 --- a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java +++ b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java @@ -119,6 +119,7 @@ public class PostgresqlRepositoryHandler extends CommonRepositoryHandler { runQuery(PostgresqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_CONFIGURABLE, conn); runQuery(PostgresqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_CONFIG, conn); runQuery(PostgresqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_INPUT, conn); + runQuery(PostgresqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_INPUT_RELATION, conn); runQuery(PostgresqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_LINK, conn); runQuery(PostgresqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_JOB, conn); runQuery(PostgresqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_LINK_INPUT, conn); http://git-wip-us.apache.org/repos/asf/sqoop/blob/d3062f34/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java index be612ce..51d0bd6 100644 --- a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java +++ b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java @@ -18,6 +18,7 @@ package org.apache.sqoop.repository.postgresql; import org.apache.sqoop.repository.common.CommonRepoUtils; +import org.apache.sqoop.repository.common.CommonRepositorySchemaConstants; import static org.apache.sqoop.repository.postgresql.PostgresqlSchemaConstants.*; @@ -101,20 +102,35 @@ import static org.apache.sqoop.repository.postgresql.PostgresqlSchemaConstants.* * </p> * <p> * <strong>SQ_INPUT</strong>: Input details + * * <pre> * +----------------------------+ * | SQ_INPUT | * +----------------------------+ - * | SQI_ID: BIGSERIAL PK | + * | SQI_ID: BIGSERIAL PK | * | SQI_NAME: VARCHAR(64) | - * | SQI_CONFIG: BIGINT | FK SQI_CONFIG(SQ_CFG_ID) + * | SQI_CONFIG: BIGINT |FK SQ_CONFIG(SQ_CFG_ID) * | SQI_INDEX: SMALLINT | - * | SQI_TYPE: VARCHAR(32) | "STRING"|"MAP" + * | SQI_TYPE: VARCHAR(32) |"STRING"|"MAP" * | SQI_STRMASK: BOOLEAN | * | SQI_STRLENGTH: SMALLINT | * | SQI_ENUMVALS: VARCHAR(100) | + * | SQI_EDITABLE: VARCHAR(32) | + * +----------------------------+ + * </pre> +* <p> + * <strong>SQ_INPUT_RELATION</strong>: Input to Input relationship + * + * <pre> + * +----------------------------+ + * | SQ_INPUT_RELATION | + * +----------------------------+ + * | SQIR_ID: BIGSERIAL PK | + * | SQIR_PARENT_ID: BIGINT |FK SQ_INPUT(SQI_ID) + * | SQIR_CHILD_ID: BIGINT |FK SQ_INPUT(SQI_ID) * +----------------------------+ * </pre> + * * </p> * <p> * <strong>SQ_LINK</strong>: Stored connections @@ -291,10 +307,18 @@ public class PostgresqlSchemaCreateQuery { + CommonRepoUtils.escapeColumnName(COLUMN_SQI_TYPE) + " VARCHAR(32), " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRMASK) + " BOOLEAN, " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + " SMALLINT, " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQI_EDITABLE) + " VARCHAR(32), " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + " VARCHAR(100), " + " UNIQUE (" + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_TYPE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + ") " + ")"; + public static final String QUERY_CREATE_TABLE_SQ_INPUT_RELATION = + "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, CommonRepositorySchemaConstants.TABLE_SQ_INPUT_RELATION_NAME) + " (" + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQIR_ID) + " BIGSERIAL PRIMARY KEY NOT NULL, " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQIR_PARENT) + " BIGINT REFERENCES " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + "(" + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ")" + ", " + + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQIR_CHILD) + " BIGINT REFERENCES " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + "(" + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ")" + + ")"; + public static final String QUERY_CREATE_TABLE_SQ_LINK = "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNK_ID) + " BIGSERIAL PRIMARY KEY NOT NULL, " http://git-wip-us.apache.org/repos/asf/sqoop/blob/d3062f34/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java index 79381d8..34dbd8c 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java @@ -149,16 +149,16 @@ abstract public class PostgresqlTestCase { List<MConfig> configs = new LinkedList<MConfig>(); List<MInput<?>> inputs = new LinkedList<MInput<?>>(); - MInput input = new MStringInput("I1", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30); + MInput<?> input = new MStringInput("I1", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30); inputs.add(input); - input = new MMapInput("I2", false, InputEditable.ANY, StringUtils.EMPTY); + input = new MMapInput("I2", false, InputEditable.ANY, "I1"); inputs.add(input); configs.add(new MConfig(configName1, inputs)); inputs = new LinkedList<MInput<?>>(); - input = new MStringInput("I3", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30); + input = new MStringInput("I3", false, InputEditable.ANY, "I4", (short) 30); inputs.add(input); - input = new MMapInput("I4", false, InputEditable.ANY, StringUtils.EMPTY); + input = new MMapInput("I4", false, InputEditable.ANY, "I3"); inputs.add(input); configs.add(new MConfig(configName2, inputs)); http://git-wip-us.apache.org/repos/asf/sqoop/blob/d3062f34/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestUtils.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestUtils.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestUtils.java index 0742e2d..19066b8 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestUtils.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestUtils.java @@ -22,7 +22,6 @@ import org.apache.sqoop.repository.common.CommonRepoUtils; import java.sql.DatabaseMetaData; import java.sql.ResultSet; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Set; http://git-wip-us.apache.org/repos/asf/sqoop/blob/d3062f34/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestConnectorHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestConnectorHandling.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestConnectorHandling.java index c1f5a20..d62f585 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestConnectorHandling.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestConnectorHandling.java @@ -83,6 +83,7 @@ public class TestConnectorHandling extends PostgresqlTestCase { assertEquals(provider.rowCount("SQOOP", "SQ_CONFIGURABLE"), 1); assertEquals(provider.rowCount("SQOOP", "SQ_CONFIG"), 6); assertEquals(provider.rowCount("SQOOP", "SQ_INPUT"), 12); + assertEquals(provider.rowCount("SQOOP", "SQ_INPUT_RELATION"), 9); // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", provider.getConnection()); @@ -103,6 +104,7 @@ public class TestConnectorHandling extends PostgresqlTestCase { assertEquals(provider.rowCount("SQOOP", "SQ_CONFIGURABLE"), 1); assertEquals(provider.rowCount("SQOOP", "SQ_CONFIG"), 4); assertEquals(provider.rowCount("SQOOP", "SQ_INPUT"), 8); + assertEquals(provider.rowCount("SQOOP", "SQ_INPUT_RELATION"), 6); // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", provider.getConnection()); @@ -123,6 +125,7 @@ public class TestConnectorHandling extends PostgresqlTestCase { assertEquals(provider.rowCount("SQOOP", "SQ_CONFIGURABLE"), 1); assertEquals(provider.rowCount("SQOOP", "SQ_CONFIG"), 4); assertEquals(provider.rowCount("SQOOP", "SQ_INPUT"), 8); + assertEquals(provider.rowCount("SQOOP", "SQ_INPUT_RELATION"), 6); // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", provider.getConnection()); @@ -143,6 +146,7 @@ public class TestConnectorHandling extends PostgresqlTestCase { assertEquals(provider.rowCount("SQOOP", "SQ_CONFIGURABLE"), 1); assertEquals(provider.rowCount("SQOOP", "SQ_CONFIG"), 2); assertEquals(provider.rowCount("SQOOP", "SQ_INPUT"), 4); + assertEquals(provider.rowCount("SQOOP", "SQ_INPUT_RELATION"), 3); // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", provider.getConnection()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/d3062f34/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestDriverHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestDriverHandling.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestDriverHandling.java index 2139720..be95926 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestDriverHandling.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestDriverHandling.java @@ -63,6 +63,7 @@ public class TestDriverHandling extends PostgresqlTestCase { assertEquals(provider.rowCount("SQOOP", "SQ_CONFIGURABLE"), 1); assertEquals(provider.rowCount("SQOOP", "SQ_CONFIG"), 2); assertEquals(provider.rowCount("SQOOP", "SQ_INPUT"), 4); + assertEquals(provider.rowCount("SQOOP", "SQ_INPUT_RELATION"), 3); // Registered driver and config should be easily recovered back MDriver retrieved = handler.findDriver(MDriver.DRIVER_NAME, provider.getConnection()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/d3062f34/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestStructure.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestStructure.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestStructure.java index 42cf3ed..2ff4d35 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestStructure.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestStructure.java @@ -34,6 +34,7 @@ public class TestStructure extends PostgresqlTestCase { utils.assertTableExists("SQOOP", "SQ_CONFIG"); utils.assertTableExists("SQOOP", "SQ_CONNECTOR_DIRECTIONS"); utils.assertTableExists("SQOOP", "SQ_INPUT"); + utils.assertTableExists("SQOOP", "SQ_INPUT_RELATION"); utils.assertTableExists("SQOOP", "SQ_LINK"); utils.assertTableExists("SQOOP", "SQ_JOB"); utils.assertTableExists("SQOOP", "SQ_LINK_INPUT"); @@ -57,6 +58,8 @@ public class TestStructure extends PostgresqlTestCase { utils.assertForeignKey("SQOOP", "SQ_LINK", "SQ_LNK_ID", "SQ_JOB", "SQB_TO_LINK"); utils.assertForeignKey("SQOOP", "SQ_LINK", "SQ_LNK_ID", "SQ_LINK_INPUT", "SQ_LNKI_LINK"); utils.assertForeignKey("SQOOP", "SQ_INPUT", "SQI_ID", "SQ_LINK_INPUT", "SQ_LNKI_INPUT"); + utils.assertForeignKey("SQOOP", "SQ_INPUT", "SQI_ID", "SQ_INPUT_RELATION", "SQIR_PARENT_ID"); + utils.assertForeignKey("SQOOP", "SQ_INPUT", "SQI_ID", "SQ_INPUT_RELATION", "SQIR_CHILD_ID"); utils.assertForeignKey("SQOOP", "SQ_JOB", "SQB_ID", "SQ_JOB_INPUT", "SQBI_JOB"); utils.assertForeignKey("SQOOP", "SQ_INPUT", "SQI_ID", "SQ_JOB_INPUT", "SQBI_INPUT"); utils.assertForeignKey("SQOOP", "SQ_JOB", "SQB_ID", "SQ_SUBMISSION", "SQS_JOB");
