Repository: sqoop Updated Branches: refs/heads/sqoop2 b055d4eba -> 756a3c17e
SQOOP-2242: Sqoop2: Add support for full table names into our DatabaseProvider infrastructure (Jarek Jarcec Cecho via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/756a3c17 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/756a3c17 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/756a3c17 Branch: refs/heads/sqoop2 Commit: 756a3c17e20c109e355f386e330985f1d4c695b5 Parents: b055d4e Author: Abraham Elmahrek <[email protected]> Authored: Thu Mar 19 19:10:40 2015 -0700 Committer: Abraham Elmahrek <[email protected]> Committed: Thu Mar 19 19:10:40 2015 -0700 ---------------------------------------------------------------------- .../common/test/asserts/ProviderAsserts.java | 5 +- .../sqoop/common/test/db/DatabaseProvider.java | 74 ++++++++------------ .../apache/sqoop/common/test/db/TableName.java | 50 +++++++++++++ .../postgresql/TestConnectorHandling.java | 33 ++++----- .../postgresql/TestDriverHandling.java | 9 +-- .../repository/postgresql/TestJobHandling.java | 21 +++--- .../repository/postgresql/TestLinkHandling.java | 17 ++--- .../postgresql/TestSubmissionHandling.java | 19 ++--- .../java/org/apache/sqoop/test/data/Cities.java | 3 +- .../org/apache/sqoop/test/data/DataSet.java | 7 +- .../apache/sqoop/test/data/ShortStories.java | 3 +- .../apache/sqoop/test/data/UbuntuReleases.java | 3 +- .../sqoop/test/testcases/ConnectorTestCase.java | 16 ++++- .../connector/hdfs/AppendModeTest.java | 6 +- .../connector/hdfs/OutputDirectoryTest.java | 12 +--- .../connector/jdbc/generic/AllTypesTest.java | 8 +-- .../jdbc/generic/FromHDFSToRDBMSTest.java | 5 +- .../jdbc/generic/FromRDBMSToHDFSTest.java | 20 +++--- .../jdbc/generic/IncrementalReadTest.java | 5 +- .../connector/jdbc/generic/PartitionerTest.java | 5 +- .../jdbc/generic/TableStagedRDBMSTest.java | 14 ++-- .../connector/kafka/FromRDBMSToKafkaTest.java | 4 +- .../connector/kite/FromRDBMSToKiteTest.java | 5 +- .../SubmissionWithDisabledModelObjectsTest.java | 8 +-- 24 files changed, 191 insertions(+), 161 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/common-test/src/main/java/org/apache/sqoop/common/test/asserts/ProviderAsserts.java ---------------------------------------------------------------------- diff --git a/common-test/src/main/java/org/apache/sqoop/common/test/asserts/ProviderAsserts.java b/common-test/src/main/java/org/apache/sqoop/common/test/asserts/ProviderAsserts.java index e11290b..505e084 100644 --- a/common-test/src/main/java/org/apache/sqoop/common/test/asserts/ProviderAsserts.java +++ b/common-test/src/main/java/org/apache/sqoop/common/test/asserts/ProviderAsserts.java @@ -19,6 +19,7 @@ package org.apache.sqoop.common.test.asserts; import org.apache.sqoop.common.test.db.DatabaseProvider; import org.apache.log4j.Logger; +import org.apache.sqoop.common.test.db.TableName; import java.sql.ResultSet; import java.sql.SQLException; @@ -41,7 +42,7 @@ public class ProviderAsserts { * @param conditions Conditions for identifying the row * @param values Values that should be present in the table */ - public static void assertRow(DatabaseProvider provider, String tableName, Object []conditions, Object ...values) { + public static void assertRow(DatabaseProvider provider, TableName tableName, Object []conditions, Object ...values) { assertRow(provider, tableName, true, conditions, values); } @@ -54,7 +55,7 @@ public class ProviderAsserts { * @param conditions Conditions for identifying the row * @param values Values that should be present in the table */ - public static void assertRow(DatabaseProvider provider, String tableName, boolean escapeValues, Object []conditions, Object ...values) { + public static void assertRow(DatabaseProvider provider, TableName tableName, boolean escapeValues, Object []conditions, Object ...values) { ResultSet rs = null; try { rs = provider.getRows(tableName, escapeValues, conditions); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/common-test/src/main/java/org/apache/sqoop/common/test/db/DatabaseProvider.java ---------------------------------------------------------------------- diff --git a/common-test/src/main/java/org/apache/sqoop/common/test/db/DatabaseProvider.java b/common-test/src/main/java/org/apache/sqoop/common/test/db/DatabaseProvider.java index 948431b..7e17c09 100644 --- a/common-test/src/main/java/org/apache/sqoop/common/test/db/DatabaseProvider.java +++ b/common-test/src/main/java/org/apache/sqoop/common/test/db/DatabaseProvider.java @@ -157,26 +157,6 @@ abstract public class DatabaseProvider { } /** - * Get full table name with qualifications - * @param schemaName - * @param tableName - * @param escape - * @return String table name - */ - public String getTableName(String schemaName, String tableName, boolean escape) { - StringBuilder sb = new StringBuilder(); - - if (schemaName != null) { - sb.append(escape ? escapeSchemaName(schemaName) : schemaName); - sb.append("."); - } - - sb.append(escape ? escapeTableName(tableName) : tableName); - - return sb.toString(); - } - - /** * Start the handler. */ public void start() { @@ -321,7 +301,7 @@ abstract public class DatabaseProvider { * @param primaryKey Primary key column(0) or null if table should not have any * @param columns List of double values column name and value for example ... "id", "varchar(50)"... */ - public void createTable(String name, String primaryKey, String ...columns) { + public void createTable(TableName name, String primaryKey, String ...columns) { // Columns are in form of two strings - name and type if(columns.length == 0 || columns.length % 2 != 0) { throw new RuntimeException("Incorrect number of parameters."); @@ -331,7 +311,7 @@ abstract public class DatabaseProvider { dropTable(name); StringBuilder sb = new StringBuilder("CREATE TABLE "); - sb.append(escapeTableName(name)).append("("); + sb.append(getTableFragment(name)).append("("); // Column list List<String> columnList = new LinkedList<String>(); @@ -356,7 +336,7 @@ abstract public class DatabaseProvider { * @param tableName Table name * @param values List of objects that should be inserted */ - public void insertRow(String tableName, Object ...values) { + public void insertRow(TableName tableName, Object ...values) { insertRow(tableName, true, values); } @@ -367,9 +347,9 @@ abstract public class DatabaseProvider { * @param escapeValues Should the values be escaped based on their type or not * @param values List of objects that should be inserted */ - public void insertRow(String tableName, boolean escapeValues, Object ...values) { + public void insertRow(TableName tableName, boolean escapeValues, Object ...values) { StringBuilder sb = new StringBuilder("INSERT INTO "); - sb.append(escapeTableName(tableName)); + sb.append(getTableFragment(tableName)); sb.append(" VALUES ("); List<String> valueList = new LinkedList<String>(); @@ -390,7 +370,7 @@ abstract public class DatabaseProvider { * @param conditions Conditions in form of double values - column name and value, for example: "id", 1 or "last_update_date", null * @return ResultSet with given criteria */ - public ResultSet getRows(String tableName, Object []conditions) { + public ResultSet getRows(TableName tableName, Object []conditions) { return getRows(tableName, true, conditions); } @@ -402,14 +382,14 @@ abstract public class DatabaseProvider { * @param conditions Conditions in form of double values - column name and value, for example: "id", 1 or "last_update_date", null * @return ResultSet with given criteria */ - public ResultSet getRows(String tableName, boolean escapeValues, Object []conditions) { + public ResultSet getRows(TableName tableName, boolean escapeValues, Object []conditions) { // Columns are in form of two strings - name and value if(conditions.length % 2 != 0) { throw new RuntimeException("Incorrect number of parameters."); } StringBuilder sb = new StringBuilder("SELECT * FROM "); - sb.append(escapeTableName(tableName)); + sb.append(getTableFragment(tableName)); List<String> conditionList = new LinkedList<String>(); for(int i = 0; i < conditions.length; i += 2) { @@ -458,9 +438,9 @@ abstract public class DatabaseProvider { * * @param tableName */ - public void dropTable(String tableName) { + public void dropTable(TableName tableName) { StringBuilder sb = new StringBuilder("DROP TABLE "); - sb.append(escapeTableName(tableName)); + sb.append(getTableFragment(tableName)); try { executeUpdate(sb.toString()); @@ -491,13 +471,12 @@ abstract public class DatabaseProvider { /** * Return number of rows from given table. * - * @param schemaName Schema name * @param tableName Table name * @return Number of rows */ - public long rowCount(String schemaName, String tableName) { + public long rowCount(TableName tableName) { StringBuilder sb = new StringBuilder("SELECT COUNT(*) FROM "); - sb.append(getTableName(schemaName, tableName, true)); + sb.append(getTableFragment(tableName)); ResultSet rs = null; try { @@ -516,16 +495,6 @@ abstract public class DatabaseProvider { } /** - * Return number of rows from a given table. - * - * @param tableName - * @return Number of rows - */ - public long rowCount(String tableName) { - return rowCount(null, tableName); - } - - /** * Close given result set (if not null) and associated statement. * * @param rs ResultSet to close. @@ -560,8 +529,8 @@ abstract public class DatabaseProvider { * * @param tableName Name of the table */ - public void dumpTable(String tableName) { - String query = "SELECT * FROM " + escapeTableName(tableName); + public void dumpTable(TableName tableName) { + String query = "SELECT * FROM " + getTableFragment(tableName); List<String> list = new LinkedList<String>(); ResultSet rs = null; @@ -591,4 +560,19 @@ abstract public class DatabaseProvider { closeResultSetWithStatement(rs); } } + + /** + * Generated properly escaped table name fragment that can be used + * in SQL query. + * + * @param tableName Full table name + * @return + */ + public String getTableFragment(TableName tableName) { + if(tableName.getSchemaName() == null) { + return escapeTableName(tableName.getTableName()); + } + + return escapeSchemaName(tableName.getSchemaName()) + "." + escapeTableName(tableName.getTableName()); + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/common-test/src/main/java/org/apache/sqoop/common/test/db/TableName.java ---------------------------------------------------------------------- diff --git a/common-test/src/main/java/org/apache/sqoop/common/test/db/TableName.java b/common-test/src/main/java/org/apache/sqoop/common/test/db/TableName.java new file mode 100644 index 0000000..27cfc6e --- /dev/null +++ b/common-test/src/main/java/org/apache/sqoop/common/test/db/TableName.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.common.test.db; + +/** + * Describes table full name. + */ +public class TableName { + + private String schemaName; + + private String tableName; + + public TableName(String tableName) { + this.tableName = tableName; + } + + public TableName(String schemaName, String tableName) { + this(tableName); + this.schemaName = schemaName; + } + + @Override + public String toString() { + return schemaName + "." + tableName; + } + + public String getSchemaName() { + return schemaName; + } + + public String getTableName() { + return tableName; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestConnectorHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestConnectorHandling.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestConnectorHandling.java index d62f585..e9b5bcd 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestConnectorHandling.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestConnectorHandling.java @@ -17,6 +17,7 @@ */ package org.apache.sqoop.integration.repository.postgresql; +import org.apache.sqoop.common.test.db.TableName; import org.apache.sqoop.model.MConnector; import org.testng.annotations.Test; @@ -80,10 +81,10 @@ public class TestConnectorHandling extends PostgresqlTestCase { assertEquals(1, connector.getPersistenceId()); // Now check content in corresponding tables - assertEquals(provider.rowCount("SQOOP", "SQ_CONFIGURABLE"), 1); - assertEquals(provider.rowCount("SQOOP", "SQ_CONFIG"), 6); - assertEquals(provider.rowCount("SQOOP", "SQ_INPUT"), 12); - assertEquals(provider.rowCount("SQOOP", "SQ_INPUT_RELATION"), 9); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_CONFIGURABLE")), 1); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_CONFIG")), 6); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_INPUT")), 12); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_INPUT_RELATION")), 9); // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", provider.getConnection()); @@ -101,10 +102,10 @@ public class TestConnectorHandling extends PostgresqlTestCase { assertEquals(1, connector.getPersistenceId()); // Now check content in corresponding tables - assertEquals(provider.rowCount("SQOOP", "SQ_CONFIGURABLE"), 1); - assertEquals(provider.rowCount("SQOOP", "SQ_CONFIG"), 4); - assertEquals(provider.rowCount("SQOOP", "SQ_INPUT"), 8); - assertEquals(provider.rowCount("SQOOP", "SQ_INPUT_RELATION"), 6); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_CONFIGURABLE")), 1); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_CONFIG")), 4); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_INPUT")), 8); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_INPUT_RELATION")), 6); // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", provider.getConnection()); @@ -122,10 +123,10 @@ public class TestConnectorHandling extends PostgresqlTestCase { assertEquals(1, connector.getPersistenceId()); // Now check content in corresponding tables - assertEquals(provider.rowCount("SQOOP", "SQ_CONFIGURABLE"), 1); - assertEquals(provider.rowCount("SQOOP", "SQ_CONFIG"), 4); - assertEquals(provider.rowCount("SQOOP", "SQ_INPUT"), 8); - assertEquals(provider.rowCount("SQOOP", "SQ_INPUT_RELATION"), 6); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_CONFIGURABLE")), 1); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_CONFIG")), 4); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_INPUT")), 8); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_INPUT_RELATION")), 6); // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", provider.getConnection()); @@ -143,10 +144,10 @@ public class TestConnectorHandling extends PostgresqlTestCase { assertEquals(1, connector.getPersistenceId()); // Now check content in corresponding tables - assertEquals(provider.rowCount("SQOOP", "SQ_CONFIGURABLE"), 1); - assertEquals(provider.rowCount("SQOOP", "SQ_CONFIG"), 2); - assertEquals(provider.rowCount("SQOOP", "SQ_INPUT"), 4); - assertEquals(provider.rowCount("SQOOP", "SQ_INPUT_RELATION"), 3); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_CONFIGURABLE")), 1); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_CONFIG")), 2); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_INPUT")), 4); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_INPUT_RELATION")), 3); // Registered connector should be easily recovered back MConnector retrieved = handler.findConnector("A", provider.getConnection()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestDriverHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestDriverHandling.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestDriverHandling.java index be95926..d48d49e 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestDriverHandling.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestDriverHandling.java @@ -17,6 +17,7 @@ */ package org.apache.sqoop.integration.repository.postgresql; +import org.apache.sqoop.common.test.db.TableName; import org.apache.sqoop.model.MDriver; import org.testng.annotations.Test; @@ -60,10 +61,10 @@ public class TestDriverHandling extends PostgresqlTestCase { assertEquals(1, driver.getPersistenceId()); // Now check content in corresponding tables - assertEquals(provider.rowCount("SQOOP", "SQ_CONFIGURABLE"), 1); - assertEquals(provider.rowCount("SQOOP", "SQ_CONFIG"), 2); - assertEquals(provider.rowCount("SQOOP", "SQ_INPUT"), 4); - assertEquals(provider.rowCount("SQOOP", "SQ_INPUT_RELATION"), 3); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_CONFIGURABLE")), 1); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_CONFIG")), 2); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_INPUT")), 4); + assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_INPUT_RELATION")), 3); // Registered driver and config should be easily recovered back MDriver retrieved = handler.findDriver(MDriver.DRIVER_NAME, provider.getConnection()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestJobHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestJobHandling.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestJobHandling.java index 25827f0..5d3d46f 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestJobHandling.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestJobHandling.java @@ -19,6 +19,7 @@ package org.apache.sqoop.integration.repository.postgresql; import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.common.test.db.TableName; import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MJob; @@ -180,8 +181,8 @@ public class TestJobHandling extends PostgresqlTestCase { @Test public void testCreateJob() throws Exception { - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_JOB"), 2); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_JOB_INPUT"), 12); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_JOB")), 2); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_JOB_INPUT")), 12); MJob retrieved = handler.findJob(1, provider.getConnection()); assertEquals(1, retrieved.getPersistenceId()); @@ -211,8 +212,8 @@ public class TestJobHandling extends PostgresqlTestCase { @Test public void testUpdateJob() throws Exception { - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_JOB"), 2); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_JOB_INPUT"), 12); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_JOB")), 2); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_JOB_INPUT")), 12); MJob job = handler.findJob(1, provider.getConnection()); @@ -237,8 +238,8 @@ public class TestJobHandling extends PostgresqlTestCase { handler.updateJob(job, provider.getConnection()); assertEquals(1, job.getPersistenceId()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_JOB"), 2); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_JOB_INPUT"), 14); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_JOB")), 2); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_JOB_INPUT")), 14); MJob retrieved = handler.findJob(1, provider.getConnection()); assertEquals("name", retrieved.getName()); @@ -279,11 +280,11 @@ public class TestJobHandling extends PostgresqlTestCase { @Test public void testDeleteJob() throws Exception { handler.deleteJob(1, provider.getConnection()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_JOB"), 1); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_JOB_INPUT"), 6); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_JOB")), 1); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_JOB_INPUT")), 6); handler.deleteJob(2, provider.getConnection()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_JOB"), 0); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_JOB_INPUT"), 0); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_JOB")), 0); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_JOB_INPUT")), 0); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestLinkHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestLinkHandling.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestLinkHandling.java index eb35b26..35736d4 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestLinkHandling.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestLinkHandling.java @@ -20,6 +20,7 @@ package org.apache.sqoop.integration.repository.postgresql; import java.util.List; import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.common.test.db.TableName; import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MJob; @@ -195,8 +196,8 @@ public class TestLinkHandling extends PostgresqlTestCase { assertEquals("Value2", configs.get(1).getInputs().get(0).getValue()); assertNull(configs.get(1).getInputs().get(1).getValue()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_LINK"), 2); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_LINK_INPUT"), 4); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_LINK")), 2); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_LINK_INPUT")), 4); } @Test(expectedExceptions = SqoopException.class) @@ -241,8 +242,8 @@ public class TestLinkHandling extends PostgresqlTestCase { handler.updateLink(link, provider.getConnection()); assertEquals(1, link.getPersistenceId()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_LINK"), 2); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_LINK_INPUT"), 4); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_LINK")), 2); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_LINK_INPUT")), 4); MLink retrieved = handler.findLink(1, provider.getConnection()); assertEquals("name", link.getName()); @@ -274,11 +275,11 @@ public class TestLinkHandling extends PostgresqlTestCase { @Test public void testDeleteLink() throws Exception { handler.deleteLink(1, provider.getConnection()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_LINK"), 1); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_LINK_INPUT"), 2); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_LINK")), 1); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_LINK_INPUT")), 2); handler.deleteLink(2, provider.getConnection()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_LINK"), 0); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_LINK_INPUT"), 0); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_LINK")), 0); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_LINK_INPUT")), 0); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestSubmissionHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestSubmissionHandling.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestSubmissionHandling.java index 9842754..3bce806 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestSubmissionHandling.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/TestSubmissionHandling.java @@ -17,6 +17,7 @@ */ package org.apache.sqoop.integration.repository.postgresql; +import org.apache.sqoop.common.test.db.TableName; import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; @@ -153,7 +154,7 @@ public class TestSubmissionHandling extends PostgresqlTestCase { assertEquals(1, submission.getPersistenceId()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_SUBMISSION"), 1); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_SUBMISSION")), 1); List<MSubmission> submissions = handler.findUnfinishedSubmissions(provider.getConnection()); assertNotNull(submissions); @@ -202,7 +203,7 @@ public class TestSubmissionHandling extends PostgresqlTestCase { handler.createSubmission(submission, provider.getConnection()); assertEquals(2, submission.getPersistenceId()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_SUBMISSION"), 2); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_SUBMISSION")), 2); } @Test @@ -303,7 +304,7 @@ public class TestSubmissionHandling extends PostgresqlTestCase { submissions = handler.findUnfinishedSubmissions(provider.getConnection()); assertNotNull(submissions); assertEquals(1, submissions.size()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_SUBMISSION"), 4); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_SUBMISSION")), 4); Calendar calendar = Calendar.getInstance(); // 2012-01-03 05:05:05 @@ -313,21 +314,21 @@ public class TestSubmissionHandling extends PostgresqlTestCase { submissions = handler.findUnfinishedSubmissions(provider.getConnection()); assertNotNull(submissions); assertEquals(1, submissions.size()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_SUBMISSION"), 4); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_SUBMISSION")), 4); handler.purgeSubmissions(new Date(), provider.getConnection()); submissions = handler.findUnfinishedSubmissions(provider.getConnection()); assertNotNull(submissions); assertEquals(0, submissions.size()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_SUBMISSION"), 0); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_SUBMISSION")), 0); handler.purgeSubmissions(new Date(), provider.getConnection()); submissions = handler.findUnfinishedSubmissions(provider.getConnection()); assertNotNull(submissions); assertEquals(0, submissions.size()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_SUBMISSION"), 0); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_SUBMISSION")), 0); } /** @@ -342,12 +343,12 @@ public class TestSubmissionHandling extends PostgresqlTestCase { MJob jobB = handler.findJob(JOB_B_NAME, provider.getConnection()); loadSubmissions(); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_SUBMISSION"), 4); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_SUBMISSION")), 4); handler.deleteJob(jobA.getPersistenceId(), provider.getConnection()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_SUBMISSION"), 2); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_SUBMISSION")), 2); handler.deleteJob(jobB.getPersistenceId(), provider.getConnection()); - Assert.assertEquals(provider.rowCount("SQOOP", "SQ_SUBMISSION"), 0); + Assert.assertEquals(provider.rowCount(new TableName("SQOOP", "SQ_SUBMISSION")), 0); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/main/java/org/apache/sqoop/test/data/Cities.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/data/Cities.java b/test/src/main/java/org/apache/sqoop/test/data/Cities.java index 2589061..fbbd7ef 100644 --- a/test/src/main/java/org/apache/sqoop/test/data/Cities.java +++ b/test/src/main/java/org/apache/sqoop/test/data/Cities.java @@ -18,13 +18,14 @@ package org.apache.sqoop.test.data; import org.apache.sqoop.common.test.db.DatabaseProvider; +import org.apache.sqoop.common.test.db.TableName; /** * Simple listing of few world's cities to do basic sanity tests. */ public class Cities extends DataSet { - public Cities(DatabaseProvider provider, String tableBaseName) { + public Cities(DatabaseProvider provider, TableName tableBaseName) { super(provider, tableBaseName); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/main/java/org/apache/sqoop/test/data/DataSet.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/data/DataSet.java b/test/src/main/java/org/apache/sqoop/test/data/DataSet.java index 06f386e..5e109b1 100644 --- a/test/src/main/java/org/apache/sqoop/test/data/DataSet.java +++ b/test/src/main/java/org/apache/sqoop/test/data/DataSet.java @@ -18,6 +18,7 @@ package org.apache.sqoop.test.data; import org.apache.sqoop.common.test.db.DatabaseProvider; +import org.apache.sqoop.common.test.db.TableName; /** * Abstract class for basic testing data sets. @@ -35,9 +36,9 @@ public abstract class DataSet { /** * Base name for created tables. */ - protected String tableBaseName; + protected TableName tableBaseName; - public DataSet(DatabaseProvider provider, String tableBaseName) { + public DataSet(DatabaseProvider provider, TableName tableBaseName) { setProvider(provider); setTableBaseName(tableBaseName); } @@ -47,7 +48,7 @@ public abstract class DataSet { return this; } - public DataSet setTableBaseName(String tableBaseName) { + public DataSet setTableBaseName(TableName tableBaseName) { this.tableBaseName = tableBaseName; return this; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/main/java/org/apache/sqoop/test/data/ShortStories.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/data/ShortStories.java b/test/src/main/java/org/apache/sqoop/test/data/ShortStories.java index 17148bc..b84339e 100644 --- a/test/src/main/java/org/apache/sqoop/test/data/ShortStories.java +++ b/test/src/main/java/org/apache/sqoop/test/data/ShortStories.java @@ -18,6 +18,7 @@ package org.apache.sqoop.test.data; import org.apache.sqoop.common.test.db.DatabaseProvider; +import org.apache.sqoop.common.test.db.TableName; /** * Releases of Ubuntu Linux. @@ -26,7 +27,7 @@ import org.apache.sqoop.common.test.db.DatabaseProvider; */ public class ShortStories extends DataSet { - public ShortStories(DatabaseProvider provider, String tableBaseName) { + public ShortStories(DatabaseProvider provider, TableName tableBaseName) { super(provider, tableBaseName); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java b/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java index 7979686..f8aeb38 100644 --- a/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java +++ b/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java @@ -18,6 +18,7 @@ package org.apache.sqoop.test.data; import org.apache.sqoop.common.test.db.DatabaseProvider; +import org.apache.sqoop.common.test.db.TableName; /** * Releases of Ubuntu Linux. @@ -26,7 +27,7 @@ import org.apache.sqoop.common.test.db.DatabaseProvider; */ public class UbuntuReleases extends DataSet { - public UbuntuReleases(DatabaseProvider provider, String tableBaseName) { + public UbuntuReleases(DatabaseProvider provider, TableName tableBaseName) { super(provider, tableBaseName); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java index 80f21cc..94e8c7a 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java @@ -28,6 +28,7 @@ import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.test.asserts.ProviderAsserts; import org.apache.sqoop.common.test.db.DatabaseProvider; import org.apache.sqoop.common.test.db.DatabaseProviderFactory; +import org.apache.sqoop.common.test.db.TableName; import org.apache.sqoop.connector.hdfs.configuration.ToFormat; import org.apache.sqoop.model.MConfigList; import org.apache.sqoop.model.MJob; @@ -103,8 +104,8 @@ abstract public class ConnectorTestCase extends TomcatTestCase { provider.stop(); } - public String getTableName() { - return getClass().getSimpleName(); + public TableName getTableName() { + return new TableName(getClass().getSimpleName()); } protected void createTable(String primaryKey, String ...columns) { @@ -144,6 +145,17 @@ abstract public class ConnectorTestCase extends TomcatTestCase { configs.getStringInput("linkConfig.password").setValue(provider.getConnectionPassword()); } + protected void fillRdbmsFromConfig(MJob job, String partitionColumn) { + MConfigList fromConfig = job.getJobConfig(Direction.FROM); + fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName().getTableName())); + fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName(partitionColumn)); + } + + protected void fillRdbmsToConfig(MJob job) { + MConfigList toConfig = job.getJobConfig(Direction.TO); + toConfig.getStringInput("toJobConfig.tableName").setValue(provider.escapeTableName(getTableName().getTableName())); + } + protected void fillHdfsLink(MLink link) { MConfigList configs = link.getConnectorLinkConfig(); configs.getStringInput("linkConfig.confDir").setValue(getCluster().getConfigurationPath()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/AppendModeTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/AppendModeTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/AppendModeTest.java index 1ba3bd4..89405e6 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/AppendModeTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/AppendModeTest.java @@ -47,11 +47,9 @@ public class AppendModeTest extends ConnectorTestCase { MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); // Set rdbms "FROM" config - MConfigList fromConfig = job.getJobConfig(Direction.FROM); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + fillRdbmsFromConfig(job, "id"); - // fill the hdfs "TO" config + // Fill the hdfs "TO" config fillHdfsToConfig(job, ToFormat.TEXT_FILE); MConfigList toConfig = job.getJobConfig(Direction.TO); toConfig.getBooleanInput("toJobConfig.appendMode").setValue(true); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/OutputDirectoryTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/OutputDirectoryTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/OutputDirectoryTest.java index b454263..9b77d98 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/OutputDirectoryTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/OutputDirectoryTest.java @@ -57,9 +57,7 @@ public class OutputDirectoryTest extends ConnectorTestCase { MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); // Set rdbms "FROM" config - MConfigList fromConfig = job.getJobConfig(Direction.FROM); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + fillRdbmsFromConfig(job, "id"); // fill the hdfs "TO" config fillHdfsToConfig(job, ToFormat.TEXT_FILE); @@ -95,9 +93,7 @@ public class OutputDirectoryTest extends ConnectorTestCase { MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); // Set rdbms "FROM" config - MConfigList fromConfig = job.getJobConfig(Direction.FROM); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + fillRdbmsFromConfig(job, "id"); // fill the hdfs "TO" config fillHdfsToConfig(job, ToFormat.TEXT_FILE); @@ -132,9 +128,7 @@ public class OutputDirectoryTest extends ConnectorTestCase { MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); // Set rdbms "FROM" config - MConfigList fromConfig = job.getJobConfig(Direction.FROM); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + fillRdbmsFromConfig(job, "id"); // fill the hdfs "TO" config fillHdfsToConfig(job, ToFormat.TEXT_FILE); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/AllTypesTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/AllTypesTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/AllTypesTest.java index 6823ed2..740732f 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/AllTypesTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/AllTypesTest.java @@ -84,10 +84,9 @@ public class AllTypesTest extends ConnectorTestCase implements ITest { MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); // Fill rdbms "FROM" config + fillRdbmsFromConfig(job, "id"); MConfigList fromConfig = job.getJobConfig(Direction.FROM); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("value")); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); // Fill the hdfs "TO" config fillHdfsToConfig(job, ToFormat.TEXT_FILE); @@ -128,9 +127,8 @@ public class AllTypesTest extends ConnectorTestCase implements ITest { MJob job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId()); fillHdfsFromConfig(job); - // Set the rdms "TO" config here - MConfigList toConfig = job.getJobConfig(Direction.TO); - toConfig.getStringInput("toJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); + // Set the rdbms "TO" config here + fillRdbmsToConfig(job); // Driver config MDriverConfig driverConfig = job.getDriverConfig(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java index 034ae43..9e0e6d9 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java @@ -69,9 +69,8 @@ public class FromHDFSToRDBMSTest extends ConnectorTestCase { // set hdfs "FROM" config for the job, since the connector test case base class only has utilities for hdfs! fillHdfsFromConfig(job); - // set the rdms "TO" config here - MConfigList toConfig = job.getJobConfig(Direction.TO); - toConfig.getStringInput("toJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); + // set the rdbms "TO" config here + fillRdbmsToConfig(job); // driver config MDriverConfig driverConfig = job.getDriverConfig(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java index 6e1e031..b5baf87 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java @@ -51,10 +51,8 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { // Job creation MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); - // srt rdbms "FROM" config - MConfigList fromConfig = job.getJobConfig(Direction.FROM); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + // Set rdbms "FROM" config + fillRdbmsFromConfig(job, "id"); // fill the hdfs "TO" config fillHdfsToConfig(job, ToFormat.TEXT_FILE); @@ -96,9 +94,8 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); // Connector values + fillRdbmsFromConfig(job, "id"); MConfigList configs = job.getJobConfig(Direction.FROM); - configs.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - configs.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); configs.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("name") + "," + provider.escapeColumnName("story")); fillHdfsToConfig(job, ToFormat.TEXT_FILE); saveJob(job); @@ -142,9 +139,8 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); // Connector values + fillRdbmsFromConfig(job, "id"); MConfigList configs = job.getJobConfig(Direction.FROM); - configs.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - configs.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); configs.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("country")); fillHdfsToConfig(job, ToFormat.TEXT_FILE); saveJob(job); @@ -191,7 +187,7 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { // Connector values MConfigList configs = job.getJobConfig(Direction.FROM); configs.getStringInput("fromJobConfig.sql").setValue("SELECT " + provider.escapeColumnName("id") - + " FROM " + provider.escapeTableName(getTableName()) + " WHERE ${CONDITIONS}"); + + " FROM " + provider.escapeTableName(getTableName().getTableName()) + " WHERE ${CONDITIONS}"); configs.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); fillHdfsToConfig(job, ToFormat.TEXT_FILE); saveJob(job); @@ -236,16 +232,16 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); // Connector values - String partitionColumn = provider.escapeTableName(getTableName()) + "." + provider.escapeColumnName("id"); + String partitionColumn = provider.escapeTableName(getTableName().getTableName()) + "." + provider.escapeColumnName("id"); MConfigList configs = job.getJobConfig(Direction.FROM); configs.getStringInput("fromJobConfig.sql").setValue( "SELECT " + provider.escapeColumnName("id") + " as " + provider.escapeColumnName("i") + ", " + provider.escapeColumnName("id") + " as " + provider.escapeColumnName("j") - + " FROM " + provider.escapeTableName(getTableName()) + " WHERE ${CONDITIONS}"); + + " FROM " + provider.escapeTableName(getTableName().getTableName()) + " WHERE ${CONDITIONS}"); configs.getStringInput("fromJobConfig.partitionColumn").setValue(partitionColumn); configs.getStringInput("fromJobConfig.boundaryQuery").setValue( "SELECT MIN(" + partitionColumn + "), MAX(" + partitionColumn + ") FROM " - + provider.escapeTableName(getTableName())); + + provider.escapeTableName(getTableName().getTableName())); fillHdfsToConfig(job, ToFormat.TEXT_FILE); saveJob(job); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java index b37cdb4..e36283e 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java @@ -78,9 +78,8 @@ public class IncrementalReadTest extends ConnectorTestCase implements ITest { MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); // Set the rdbms "FROM" config + fillRdbmsFromConfig(job, "id"); MConfigList fromConfig = job.getJobConfig(Direction.FROM); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); fromConfig.getStringInput("incrementalRead.checkColumn").setValue(provider.escapeColumnName(checkColumn)); fromConfig.getStringInput("incrementalRead.lastValue").setValue(lastValue); @@ -128,7 +127,7 @@ public class IncrementalReadTest extends ConnectorTestCase implements ITest { // Job creation MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); - String query = "SELECT * FROM " + provider.escapeTableName(getTableName()) + " WHERE ${CONDITIONS}"; + String query = "SELECT * FROM " + provider.escapeTableName(getTableName().getTableName()) + " WHERE ${CONDITIONS}"; // Set the rdbms "FROM" config MConfigList fromConfig = job.getJobConfig(Direction.FROM); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java index ef9720a..a1e2a41 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java @@ -95,9 +95,8 @@ public class PartitionerTest extends ConnectorTestCase implements ITest { MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); // set the rdbms "FROM" config - MConfigList fromConfig = job.getJobConfig(Direction.FROM); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName(partitionColumn)); + fillRdbmsFromConfig(job, partitionColumn); + // fill hdfs "TO" config fillHdfsToConfig(job, ToFormat.TEXT_FILE); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java index 5ef7c8f..5c2428d 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java @@ -20,6 +20,7 @@ package org.apache.sqoop.integration.connector.jdbc.generic; import static org.testng.AssertJUnit.assertEquals; import org.apache.sqoop.common.Direction; +import org.apache.sqoop.common.test.db.TableName; import org.apache.sqoop.model.MConfigList; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; @@ -34,7 +35,7 @@ public class TableStagedRDBMSTest extends ConnectorTestCase { @Test public void testStagedTransfer() throws Exception { - final String stageTableName = "STAGE_" + getTableName(); + final TableName stageTableName = new TableName("STAGE_" + getTableName()); createTableCities(); createFromFile("input-0001", "1,'USA','2004-10-23','San Francisco'", @@ -62,11 +63,10 @@ public class TableStagedRDBMSTest extends ConnectorTestCase { fillHdfsFromConfig(job); // fill rdbms "TO" config here + fillRdbmsToConfig(job); MConfigList configs = job.getJobConfig(Direction.TO); - configs.getStringInput("toJobConfig.tableName").setValue( - provider.escapeTableName(getTableName())); - configs.getStringInput("toJobConfig.stageTableName").setValue( - provider.escapeTableName(stageTableName)); + configs.getStringInput("toJobConfig.stageTableName").setValue(provider.escapeTableName(stageTableName.getTableName())); + // driver config MConfigList driverConfig = job.getDriverConfig(); driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3); @@ -75,8 +75,8 @@ public class TableStagedRDBMSTest extends ConnectorTestCase { executeJob(job); - assertEquals(0L, provider.rowCount(null, stageTableName)); - assertEquals(4L, provider.rowCount(null, getTableName())); + assertEquals(0L, provider.rowCount(stageTableName)); + assertEquals(4L, provider.rowCount(getTableName())); assertRowInCities(1, "USA", "2004-10-23", "San Francisco"); assertRowInCities(2, "USA", "2004-10-24", "Sunnyvale"); assertRowInCities(3, "Czech Republic", "2004-10-25", "Brno"); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java index 8a09d7e..92a52b8 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java @@ -53,9 +53,7 @@ public class FromRDBMSToKafkaTest extends KafkaConnectorTestCase { MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kafkaLink.getPersistenceId()); // set rdbms "FROM" job config - MConfigList fromConfig = job.getJobConfig(Direction.FROM); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + fillRdbmsFromConfig(job, "id"); // set Kafka "TO" job config fillKafkaToConfig(job); http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java index 1d4e7e6..8ca1c3a 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java @@ -70,10 +70,9 @@ public class FromRDBMSToKiteTest extends ConnectorTestCase { MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kiteLink.getPersistenceId()); // Set rdbms "FROM" config - MConfigList fromConfig = job.getJobConfig(Direction.FROM); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + fillRdbmsFromConfig(job, "id"); // TODO: Kite have troubles with some data types, so we're limiting the columns to int only + MConfigList fromConfig = job.getJobConfig(Direction.FROM); fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id")); // Fill the Kite "TO" config http://git-wip-us.apache.org/repos/asf/sqoop/blob/756a3c17/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java b/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java index 5f9f41d..40dc7d7 100644 --- a/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java @@ -77,17 +77,11 @@ public class SubmissionWithDisabledModelObjectsTest extends ConnectorTestCase { MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); // rdms "FROM" config - MConfigList fromConfig = job.getJobConfig(Direction.FROM); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + fillRdbmsFromConfig(job, "id"); // hdfs "TO" config fillHdfsToConfig(job, ToFormat.TEXT_FILE); - // driver config - MConfigList driverConfig = job.getDriverConfig(); - //driverConfig.getIntegerInput("throttlingConfig.extractors").setValue(3); - saveJob(job); // Disable model entities as per parameterized run
