Repository: sqoop Updated Branches: refs/heads/sqoop2 7202fe3a5 -> 3362fbb1b
SQOOP-2244: Sqoop2: Generic JDBC: Automatically escape table and column names from configuration objects (Jarek Jarcec Cecho via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3362fbb1 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3362fbb1 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3362fbb1 Branch: refs/heads/sqoop2 Commit: 3362fbb1b1369f598f9d88860989bf859bd5af0d Parents: 7202fe3 Author: Abraham Elmahrek <[email protected]> Authored: Mon Aug 10 13:44:17 2015 -0700 Committer: Abraham Elmahrek <[email protected]> Committed: Mon Aug 10 13:44:17 2015 -0700 ---------------------------------------------------------------------- .../connector/jdbc/GenericJdbcExecutor.java | 145 +++++++++++++------ .../jdbc/GenericJdbcFromInitializer.java | 71 +++------ .../jdbc/GenericJdbcToInitializer.java | 12 +- .../jdbc/configuration/LinkConfiguration.java | 3 + .../jdbc/configuration/SqlDialect.java | 33 +++++ .../generic-jdbc-connector-config.properties | 7 + .../connector/jdbc/GenericJdbcExecutorTest.java | 90 +++++++++--- .../jdbc/GenericJdbcTestConstants.java | 5 + .../sqoop/connector/jdbc/TestExtractor.java | 16 +- .../connector/jdbc/TestFromInitializer.java | 128 ++++++++-------- .../apache/sqoop/connector/jdbc/TestLoader.java | 7 +- .../sqoop/connector/jdbc/TestToInitializer.java | 24 +-- .../sqoop/test/testcases/ConnectorTestCase.java | 6 +- .../jdbc/generic/FromRDBMSToHDFSTest.java | 8 +- .../jdbc/generic/IncrementalReadTest.java | 6 +- .../jdbc/generic/TableStagedRDBMSTest.java | 2 +- 16 files changed, 337 insertions(+), 226 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutor.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutor.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutor.java index cab0917..1aeca7e 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutor.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutor.java @@ -17,9 +17,9 @@ */ package org.apache.sqoop.connector.jdbc; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.sqoop.common.SqoopException; -import org.apache.sqoop.connector.jdbc.configuration.LinkConfig; import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration; import org.apache.sqoop.error.code.GenericJdbcConnectorError; import org.apache.sqoop.schema.Schema; @@ -38,6 +38,8 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; +import java.util.LinkedList; +import java.util.List; import java.util.Properties; /** @@ -112,6 +114,11 @@ public class GenericJdbcExecutor { logSQLException(e); throw new SqoopException(GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0001, e); } + + // Fill in defaults if they were not pre-entered by user + if(link.dialect.identifierEnclose == null) { + link.dialect.identifierEnclose = "\""; + } } public ResultSet executeQuery(String sql) { @@ -144,14 +151,69 @@ public class GenericJdbcExecutor { } } + /** + * Enclose given identifier based on the configuration from user. + * + * @param identifier Identifier to enclose + * @return Enclosed variant + */ + public String encloseIdentifier(String identifier) { + assert identifier != null; + return link.dialect.identifierEnclose + identifier + link.dialect.identifierEnclose; + } + + /** + * Enclose multiple identifiers and join them together to one string. + * + * Used to convert (schema, table) to string "schema"."table" needed for SQL queries. + * + * @param identifiers Identifiers to enclose + * @return Enclose identifiers joined with "." + */ + public String encloseIdentifiers(String ...identifiers) { + assert identifiers != null; + + List<String> enclosedIdentifiers = new LinkedList<String>(); + for(String identifier: identifiers) { + if(identifier != null) { + enclosedIdentifiers.add(encloseIdentifier(identifier)); + } + } + + return StringUtils.join(enclosedIdentifiers, "."); + } + + /** + * Create column list fragment for SELECT SQL. + * + * For example for (id, text, date), will automatically escape the column + * names and return one string: + * "id", "text", "date" + * + * This method won't work correctly if the column name contains an expression + * or anything else beyond just a column name. + * + * @param columnNames Column names to escape and join. + * @return + */ + public String columnList(String ...columnNames) { + assert columnNames != null; + + List<String> escapedColumns = new LinkedList<String>(); + for(String column : columnNames) { + escapedColumns.add(encloseIdentifier(column)); + } + + return StringUtils.join(escapedColumns, ", "); + } + public void deleteTableData(String tableName) { LOG.info("Deleting all the rows from: " + tableName); - executeUpdate("DELETE FROM " + tableName); + executeUpdate("DELETE FROM " + encloseIdentifier(tableName)); } public void migrateData(String fromTable, String toTable) { - String insertQuery = "INSERT INTO " + toTable + - " SELECT * FROM " + fromTable; + String insertQuery = "INSERT INTO " + encloseIdentifier(toTable) + " SELECT * FROM " + encloseIdentifier(fromTable); Statement stmt = null; Boolean oldAutoCommit = null; try { @@ -196,7 +258,7 @@ public class GenericJdbcExecutor { } public long getTableRowCount(String tableName) { - ResultSet resultSet = executeQuery("SELECT COUNT(1) FROM " + tableName); + ResultSet resultSet = executeQuery("SELECT COUNT(1) FROM " + encloseIdentifier(tableName)); try { resultSet.next(); return resultSet.getLong(1); @@ -295,11 +357,26 @@ public class GenericJdbcExecutor { } } - public String getPrimaryKey(String table) { + /** + * Return primary key for given table. + * + * @param identifiers Identifiers that are used to build the table's name. Following + * variants are accepted: + * * (catalog, schema, table) + * * (schema, table) + * * (table) + * Return value of any combination is "undefined". + * @return Primary key's name + */ + public String getPrimaryKey(String ...identifiers) { + int index = 0; + String catalog = identifiers.length >= 3 ? identifiers[index++] : null; + String schema = identifiers.length >= 2 ? identifiers[index++] : null; + String table = identifiers[index]; + try { - String[] splitNames = dequalify(table); DatabaseMetaData dbmd = connection.getMetaData(); - ResultSet rs = dbmd.getPrimaryKeys(null, splitNames[0], splitNames[1]); + ResultSet rs = dbmd.getPrimaryKeys(catalog, schema, table); if (rs != null && rs.next()) { return rs.getString("COLUMN_NAME"); @@ -335,12 +412,26 @@ public class GenericJdbcExecutor { } } - public boolean existTable(String table) { - try { - String[] splitNames = dequalify(table); + /** + * Verifies existence of table in the database. + * + * @param identifiers Identifiers that are used to build the table's name. Following + * variants are accepted: + * * (catalog, schema, table) + * * (schema, table) + * * (table) + * Return value of any combination is "undefined". + * @return True if given table exists + */ + public boolean existTable(String ...identifiers) { + int index = 0; + String catalog = identifiers.length >= 3 ? identifiers[index++] : null; + String schema = identifiers.length >= 2 ? identifiers[index++] : null; + String table = identifiers[index]; + try { DatabaseMetaData dbmd = connection.getMetaData(); - ResultSet rs = dbmd.getTables(null, splitNames[0], splitNames[1], null); + ResultSet rs = dbmd.getTables(catalog, schema, table, null); if (rs.next()) { return true; @@ -354,36 +445,6 @@ public class GenericJdbcExecutor { } } - /* - * If not qualified already, the name will be added with the qualifier. - * If qualified already, old qualifier will be replaced. - */ - public String qualify(String name, String qualifier) { - String[] splits = dequalify(name); - return qualifier + "." + splits[1]; - } - - /* - * Split the name into a qualifier (element 0) and a base (element 1). - */ - public String[] dequalify(String name) { - String qualifier; - String base; - int dot = name.indexOf("."); - if (dot != -1) { - qualifier = name.substring(0, dot); - base = name.substring(dot + 1); - } else { - qualifier = null; - base = name; - } - return new String[] {qualifier, base}; - } - - public String delimitIdentifier(String name) { - return name; - } - public void close() { try { connection.close(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java index 20fabf6..9d8e4e7 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java @@ -70,11 +70,11 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F public Schema getSchema(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) { executor = new GenericJdbcExecutor(linkConfig); - String schemaName = fromJobConfig.fromJobConfig.tableName; - if(schemaName == null) { + String schemaName; + if(fromJobConfig.fromJobConfig.tableName != null) { + schemaName = executor.encloseIdentifiers(fromJobConfig.fromJobConfig.schemaName, fromJobConfig.fromJobConfig.tableName); + } else { schemaName = "Query"; - } else if(fromJobConfig.fromJobConfig.schemaName != null) { - schemaName = fromJobConfig.fromJobConfig.schemaName + "." + schemaName; } Schema schema = new Schema(schemaName); @@ -135,11 +135,11 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F String partitionColumnName = jobConf.fromJobConfig.partitionColumn; // If it's not specified, we can use primary key of given table (if it's table based import) if (StringUtils.isBlank(partitionColumnName) && tableImport) { - partitionColumnName = executor.getPrimaryKey(jobConf.fromJobConfig.tableName); + partitionColumnName = executor.getPrimaryKey(jobConf.fromJobConfig.schemaName, jobConf.fromJobConfig.tableName); } // If we don't have partition column name, we will error out if (partitionColumnName != null) { - context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME, partitionColumnName); + context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME, executor.encloseIdentifier(partitionColumnName)); } else { throw new SqoopException(GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0005); } @@ -148,13 +148,7 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F // From fragment for subsequent queries String fromFragment; if(tableImport) { - String tableName = jobConf.fromJobConfig.tableName; - String schemaName = jobConf.fromJobConfig.schemaName; - - fromFragment = executor.delimitIdentifier(tableName); - if(schemaName != null) { - fromFragment = executor.delimitIdentifier(schemaName) + "." + fromFragment; - } + fromFragment = executor.encloseIdentifiers(jobConf.fromJobConfig.schemaName, jobConf.fromJobConfig.tableName); } else { sb.setLength(0); sb.append("("); @@ -169,7 +163,7 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F if(incrementalImport) { sb.setLength(0); sb.append("SELECT "); - sb.append("MAX(").append(jobConf.incrementalRead.checkColumn).append(") "); + sb.append("MAX(").append(executor.encloseIdentifier(jobConf.incrementalRead.checkColumn)).append(") "); sb.append("FROM "); sb.append(fromFragment); @@ -199,15 +193,15 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F if (minMaxQuery == null) { sb.setLength(0); sb.append("SELECT "); - sb.append("MIN(").append(partitionColumnName).append("), "); - sb.append("MAX(").append(partitionColumnName).append(") "); + sb.append("MIN(").append(executor.encloseIdentifier(partitionColumnName)).append("), "); + sb.append("MAX(").append(executor.encloseIdentifier(partitionColumnName)).append(") "); sb.append("FROM ").append(fromFragment).append(" "); if(incrementalImport) { sb.append("WHERE "); - sb.append(jobConf.incrementalRead.checkColumn).append(" > ?"); + sb.append(executor.encloseIdentifier(jobConf.incrementalRead.checkColumn)).append(" > ?"); sb.append(" AND "); - sb.append(jobConf.incrementalRead.checkColumn).append(" <= ?"); + sb.append(executor.encloseIdentifier(jobConf.incrementalRead.checkColumn)).append(" <= ?"); } minMaxQuery = sb.toString(); @@ -263,16 +257,12 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F String tableSql = fromJobConfig.fromJobConfig.sql; String tableColumns = fromJobConfig.fromJobConfig.columns; - if (tableName != null && tableSql != null) { - // when both fromTable name and fromTable sql are specified: - throw new SqoopException( - GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0007); - - } else if (tableName != null) { - // when fromTable name is specified: + // Assertion that should be true based on our validations + assert (tableName != null && tableSql == null) || (tableName == null && tableSql != null); + if (tableName != null) { // For databases that support schemas (IE: postgresql). - String fullTableName = (schemaName == null) ? executor.delimitIdentifier(tableName) : executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName); + String fullTableName = executor.encloseIdentifiers(schemaName, tableName); if (tableColumns == null) { StringBuilder builder = new StringBuilder(); @@ -282,10 +272,8 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F builder.append(GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN); dataSql = builder.toString(); - String[] queryColumns = executor.getQueryColumns(dataSql.replace( - GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN, "1 = 0")); - fieldNames = StringUtils.join(queryColumns, ','); - + String[] queryColumns = executor.getQueryColumns(dataSql.replace(GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN, "1 = 0")); + fieldNames = executor.columnList(queryColumns); } else { StringBuilder builder = new StringBuilder(); builder.append("SELECT "); @@ -298,29 +286,18 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F fieldNames = tableColumns; } - } else if (tableSql != null) { - // when fromTable sql is specified: - + } else { assert tableSql.contains(GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN); if (tableColumns == null) { dataSql = tableSql; - String[] queryColumns = executor.getQueryColumns(dataSql.replace( - GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN, "1 = 0")); - fieldNames = StringUtils.join(queryColumns, ','); - + String[] queryColumns = executor.getQueryColumns(dataSql.replace(GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN, "1 = 0")); + fieldNames = executor.columnList(queryColumns); } else { - String[] columns = StringUtils.split(tableColumns, ','); StringBuilder builder = new StringBuilder(); builder.append("SELECT "); - builder.append(executor.qualify( - columns[0], GenericJdbcConnectorConstants.SUBQUERY_ALIAS)); - for (int i = 1; i < columns.length; i++) { - builder.append(","); - builder.append(executor.qualify( - columns[i], GenericJdbcConnectorConstants.SUBQUERY_ALIAS)); - } + builder.append(tableColumns); builder.append(" FROM "); builder.append("("); builder.append(tableSql); @@ -330,10 +307,6 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F fieldNames = tableColumns; } - } else { - // when neither are specified: - throw new SqoopException( - GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0008); } LOG.info("Using dataSql: " + dataSql); http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java index 4688de3..f97e731 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java @@ -63,17 +63,12 @@ public class GenericJdbcToInitializer extends Initializer<LinkConfiguration, ToJ public Schema getSchema(InitializerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) { executor = new GenericJdbcExecutor(linkConfig); - String schemaName = toJobConfig.toJobConfig.tableName; - + String schemaName = executor.encloseIdentifiers(toJobConfig.toJobConfig.schemaName, toJobConfig.toJobConfig.tableName); if (schemaName == null) { throw new SqoopException(GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0019, "Table name extraction not supported yet."); } - if(toJobConfig.toJobConfig.schemaName != null) { - schemaName = toJobConfig.toJobConfig.schemaName + "." + schemaName; - } - Schema schema = new Schema(schemaName); ResultSet rs = null; ResultSetMetaData rsmt = null; @@ -147,10 +142,7 @@ public class GenericJdbcToInitializer extends Initializer<LinkConfiguration, ToJ // For databases that support schemas (IE: postgresql). final String tableInUse = stageEnabled ? stageTableName : tableName; - String fullTableName = (schemaName == null) ? - executor.delimitIdentifier(tableInUse) : - executor.delimitIdentifier(schemaName) + - "." + executor.delimitIdentifier(tableInUse); + String fullTableName = executor.encloseIdentifiers(schemaName, tableInUse); if (tableColumns == null) { String[] columns = executor.getQueryColumns("SELECT * FROM " http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java index ed55bff..ceb6e6d 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java @@ -28,7 +28,10 @@ public class LinkConfiguration { @Config public LinkConfig linkConfig; + @Config public SqlDialect dialect; + public LinkConfiguration() { linkConfig = new LinkConfig(); + dialect = new SqlDialect(); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/SqlDialect.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/SqlDialect.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/SqlDialect.java new file mode 100644 index 0000000..12defa3 --- /dev/null +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/SqlDialect.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.connector.jdbc.configuration; + +import org.apache.sqoop.model.ConfigClass; +import org.apache.sqoop.model.Input; + +/** + * Enables user to configure various aspects of the way the JDBC Connector generates + * SQL queries. + */ +@ConfigClass +public class SqlDialect { + /** + * Character(s) that we should use to escape SQL identifiers (tables, column names, ...) + */ + @Input(size = 5) public String identifierEnclose; +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties b/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties index 52bf631..73fa308 100644 --- a/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties +++ b/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties @@ -122,3 +122,10 @@ incrementalRead.checkColumn.help = Column that is checked during incremental rea incrementalRead.lastValue.label = Last value incrementalRead.lastValue.help = Last read value, fetch will resume with higher values + +# Dialect +dialect.label = SQL Dialect +dialect.help = Dialect that should be used for generated queries + +dialect.identifierEnclose.label = Identifier enclose +dialect.identifierEnclose.help = Character(s) that should be used to enclose table name, schema, column names, ... http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java index 22c9e15..a482ac4 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java @@ -18,46 +18,60 @@ package org.apache.sqoop.connector.jdbc; import org.apache.sqoop.common.SqoopException; -import org.apache.sqoop.connector.jdbc.configuration.LinkConfig; import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.sql.DriverManager; +import java.sql.SQLException; + import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; public class GenericJdbcExecutorTest { private final String table; private final String emptyTable; - private final GenericJdbcExecutor executor; + private final String schema; + private GenericJdbcExecutor executor; - private static final int START = -50; - private static final int NUMBER_OF_ROWS = 974; + private static final int START = -10; + private static final int NUMBER_OF_ROWS = 20; public GenericJdbcExecutorTest() { table = getClass().getSimpleName().toUpperCase(); emptyTable = table + "_EMPTY"; - executor = new GenericJdbcExecutor(GenericJdbcTestConstants.LINK_CONFIGURATION); + schema = table + "_SCHEMA"; } @BeforeMethod(alwaysRun = true) public void setUp() { - if(executor.existTable(emptyTable)) { - executor.executeUpdate("DROP TABLE " + emptyTable); - } - executor.executeUpdate("CREATE TABLE " - + emptyTable + "(ICOL INTEGER PRIMARY KEY, VCOL VARCHAR(20))"); - - if(executor.existTable(table)) { - executor.executeUpdate("DROP TABLE " + table); - } - executor.executeUpdate("CREATE TABLE " - + table + "(ICOL INTEGER PRIMARY KEY, VCOL VARCHAR(20))"); + executor = new GenericJdbcExecutor(GenericJdbcTestConstants.LINK_CONFIGURATION); + executor.executeUpdate("CREATE SCHEMA " + executor.encloseIdentifier(schema)); + executor.executeUpdate("CREATE TABLE " + executor.encloseIdentifier(emptyTable )+ "(ICOL INTEGER PRIMARY KEY, VCOL VARCHAR(20))"); + executor.executeUpdate("CREATE TABLE " + executor.encloseIdentifier(table) + "(ICOL INTEGER PRIMARY KEY, VCOL VARCHAR(20))"); + executor.executeUpdate("CREATE TABLE " + executor.encloseIdentifiers(schema, table) + "(ICOL INTEGER PRIMARY KEY, VCOL VARCHAR(20))"); for (int i = 0; i < NUMBER_OF_ROWS; i++) { int value = START + i; - String sql = "INSERT INTO " + table - + " VALUES(" + value + ", '" + value + "')"; - executor.executeUpdate(sql); + executor.executeUpdate("INSERT INTO " + executor.encloseIdentifier(table) + " VALUES(" + value + ", '" + value + "')"); + executor.executeUpdate("INSERT INTO " + executor.encloseIdentifiers(schema, table) + " VALUES(" + value + ", '" + value + "')"); + } + } + + @AfterMethod + public void tearDown() throws SQLException { + executor.close(); + try { + DriverManager.getConnection(GenericJdbcTestConstants.URL_DROP); + } catch(SQLException e) { + // Code 8006 means that the database has been successfully drooped + if(e.getErrorCode() != 45000 && e.getNextException().getErrorCode() == 8006) { + throw e; + } + } } @@ -71,6 +85,44 @@ public class GenericJdbcExecutorTest { } @Test + public void testGetPrimaryKey() { + assertNull(executor.getPrimaryKey("non-existing-table")); + assertNull(executor.getPrimaryKey("non-existing-schema", "non-existing-table")); + assertNull(executor.getPrimaryKey("non-existing-catalog", "non-existing-schema", "non-existing-table")); + + assertEquals(executor.getPrimaryKey(table), "ICOL"); + assertEquals(executor.getPrimaryKey(schema, table), "ICOL"); + } + + @Test + public void testExistsTable() { + assertFalse(executor.existTable("non-existing-table")); + assertFalse(executor.existTable("non-existing-schema", "non-existing-table")); + assertFalse(executor.existTable("non-existing-catalog", "non-existing-schema", "non-existing-table")); + + assertTrue(executor.existTable(table)); + assertTrue(executor.existTable(schema, table)); + } + + @Test + public void testEncloseIdentifier() { + assertEquals(executor.encloseIdentifier("a"), "\"a\""); + } + + @Test + public void testEncloseIdentifiers() { + assertEquals(executor.encloseIdentifiers("a"), "\"a\""); + assertEquals(executor.encloseIdentifiers(null, "a"), "\"a\""); + assertEquals(executor.encloseIdentifiers("a", "b"), "\"a\".\"b\""); + } + + @Test + public void testColumnList() { + assertEquals(executor.columnList("a"), "\"a\""); + assertEquals(executor.columnList("a", "b"), "\"a\", \"b\""); + } + + @Test public void testDeleteTableData() throws Exception { executor.deleteTableData(table); assertEquals(0, executor.getTableRowCount(table), http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcTestConstants.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcTestConstants.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcTestConstants.java index 8a5dba4..e16c631 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcTestConstants.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcTestConstants.java @@ -32,6 +32,11 @@ public class GenericJdbcTestConstants { public static final String URL = "jdbc:derby:memory:TESTDB;create=true"; /** + * URL to drop the in-memory database + */ + public static final String URL_DROP = "jdbc:derby:memory:TESTDB;drop=true"; + + /** * Test link configuration */ public static final LinkConfiguration LINK_CONFIGURATION = new LinkConfiguration(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java index 77ac9c3..264cadf 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java @@ -63,12 +63,12 @@ public class TestExtractor { if (!executor.existTable(tableName)) { executor.executeUpdate("CREATE TABLE " - + executor.delimitIdentifier(tableName) + + executor.encloseIdentifier(tableName) + "(ICOL INTEGER PRIMARY KEY, DCOL DOUBLE, VCOL VARCHAR(20), DATECOL DATE)"); for (int i = 0; i < NUMBER_OF_ROWS; i++) { int value = START + i; - String sql = "INSERT INTO " + executor.delimitIdentifier(tableName) + String sql = "INSERT INTO " + executor.encloseIdentifier(tableName) + " VALUES(" + value + ", " + value + ", '" + value + "', '2004-10-19')"; executor.executeUpdate(sql); } @@ -93,7 +93,7 @@ public class TestExtractor { FromJobConfiguration jobConfig = new FromJobConfiguration(); context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL, - "SELECT * FROM " + executor.delimitIdentifier(tableName) + " WHERE ${CONDITIONS}"); + "SELECT * FROM " + executor.encloseIdentifier(tableName) + " WHERE ${CONDITIONS}"); GenericJdbcPartition partition; @@ -133,7 +133,7 @@ public class TestExtractor { context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL, "SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL,SQOOP_SUBQUERY_ALIAS.DATECOL FROM " + "(SELECT * FROM " - + executor.delimitIdentifier(tableName) + " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS"); + + executor.encloseIdentifier(tableName) + " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS"); GenericJdbcPartition partition; @@ -174,7 +174,7 @@ public class TestExtractor { context.setString( GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL, "SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL FROM " + "(SELECT * FROM " - + executor.delimitIdentifier(tableName) + " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS"); + + executor.encloseIdentifier(tableName) + " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS"); GenericJdbcPartition partition = new GenericJdbcPartition(); @@ -192,12 +192,12 @@ public class TestExtractor { public void testNullValueExtracted() throws Exception { if (!executor.existTable(nullDataTableName)) { - executor.executeUpdate("CREATE TABLE " + executor.delimitIdentifier(nullDataTableName) + executor.executeUpdate("CREATE TABLE " + executor.encloseIdentifier(nullDataTableName) + "(ICOL INTEGER PRIMARY KEY, DCOL DOUBLE, VCOL VARCHAR(20), DATECOL DATE)"); for (int i = 0; i < NUMBER_OF_ROWS; i++) { int value = i; - String sql = "INSERT INTO " + executor.delimitIdentifier(nullDataTableName) + " VALUES(" + value + ",null,null,null)"; + String sql = "INSERT INTO " + executor.encloseIdentifier(nullDataTableName) + " VALUES(" + value + ",null,null,null)"; executor.executeUpdate(sql); } } @@ -210,7 +210,7 @@ public class TestExtractor { FromJobConfiguration jobConfig = new FromJobConfiguration(); context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL, - "SELECT * FROM " + executor.delimitIdentifier(nullDataTableName) + " WHERE ${CONDITIONS}"); + "SELECT * FROM " + executor.encloseIdentifier(nullDataTableName) + " WHERE ${CONDITIONS}"); Extractor extractor = new GenericJdbcExtractor(); DummyNullDataWriter writer = new DummyNullDataWriter(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java index 6ae6f90..31fafb1 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java @@ -63,9 +63,9 @@ public class TestFromInitializer { public void setUp() { executor = new GenericJdbcExecutor(GenericJdbcTestConstants.LINK_CONFIGURATION); - String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName); + String fullTableName = executor.encloseIdentifier(schemaName) + "." + executor.encloseIdentifier(tableName); if (!executor.existTable(tableName)) { - executor.executeUpdate("CREATE SCHEMA " + executor.delimitIdentifier(schemaName)); + executor.executeUpdate("CREATE SCHEMA " + executor.encloseIdentifier(schemaName)); executor.executeUpdate("CREATE TABLE " + fullTableName + "(ICOL INTEGER PRIMARY KEY, DCOL DOUBLE, VCOL VARCHAR(20))"); @@ -78,7 +78,7 @@ public class TestFromInitializer { } } - fullTableName = executor.delimitIdentifier(schemalessTableName); + fullTableName = executor.encloseIdentifier(schemalessTableName); if (!executor.existTable(schemalessTableName)) { executor.executeUpdate("CREATE TABLE " + fullTableName @@ -130,10 +130,9 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT * FROM " + executor.delimitIdentifier(schemalessTableName) - + " WHERE ${CONDITIONS}", - "ICOL,DCOL,VCOL", - "ICOL", + "SELECT * FROM " + executor.encloseIdentifier(schemalessTableName) + " WHERE ${CONDITIONS}", + "\"ICOL\", \"DCOL\", \"VCOL\"", + "\"ICOL\"", String.valueOf(Types.INTEGER), String.valueOf(START), String.valueOf(START+NUMBER_OF_ROWS-1)); @@ -159,9 +158,9 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT * FROM " + executor.delimitIdentifier(schemalessTableName) + " WHERE ${CONDITIONS}", - "ICOL,DCOL,VCOL", - "ICOL", + "SELECT * FROM " + executor.encloseIdentifier(schemalessTableName) + " WHERE ${CONDITIONS}", + "\"ICOL\", \"DCOL\", \"VCOL\"", + "\"ICOL\"", String.valueOf(Types.INTEGER), String.valueOf(START), String.valueOf(START+NUMBER_OF_ROWS-1)); @@ -189,9 +188,9 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT * FROM " + executor.delimitIdentifier(schemalessTableName) + " WHERE ${CONDITIONS}", - "ICOL,DCOL,VCOL", - "ICOL", + "SELECT * FROM " + executor.encloseIdentifier(schemalessTableName) + " WHERE ${CONDITIONS}", + "\"ICOL\", \"DCOL\", \"VCOL\"", + "\"ICOL\"", String.valueOf(Types.INTEGER), String.valueOf(1), String.valueOf(START+NUMBER_OF_ROWS-1)); @@ -218,10 +217,9 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT ICOL,VCOL FROM " + executor.delimitIdentifier(schemalessTableName) - + " WHERE ${CONDITIONS}", + "SELECT ICOL,VCOL FROM " + executor.encloseIdentifier(schemalessTableName) + " WHERE ${CONDITIONS}", tableColumns, - "ICOL", + "\"ICOL\"", String.valueOf(Types.INTEGER), String.valueOf(START), String.valueOf(START+NUMBER_OF_ROWS-1)); @@ -246,10 +244,9 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT * FROM " + executor.delimitIdentifier(schemalessTableName) - + " WHERE ${CONDITIONS}", - "ICOL,DCOL,VCOL", - "DCOL", + "SELECT * FROM " + schemalessTableName + " WHERE ${CONDITIONS}", + "\"ICOL\", \"DCOL\", \"VCOL\"", + "\"DCOL\"", String.valueOf(Types.DOUBLE), String.valueOf((double)START), String.valueOf((double)(START+NUMBER_OF_ROWS-1))); @@ -276,9 +273,9 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT * FROM " + executor.delimitIdentifier(schemalessTableName) + " WHERE ${CONDITIONS}", - "ICOL,DCOL,VCOL", - "ICOL", + "SELECT * FROM " + schemalessTableName + " WHERE ${CONDITIONS}", + "\"ICOL\", \"DCOL\", \"VCOL\"", + "\"ICOL\"", String.valueOf(Types.INTEGER), String.valueOf(START), String.valueOf((START+NUMBER_OF_ROWS-1))); @@ -306,9 +303,9 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT * FROM " + executor.delimitIdentifier(schemalessTableName) + " WHERE ${CONDITIONS}", - "ICOL,DCOL,VCOL", - "ICOL", + "SELECT * FROM " + schemalessTableName + " WHERE ${CONDITIONS}", + "\"ICOL\", \"DCOL\", \"VCOL\"", + "\"ICOL\"", String.valueOf(Types.INTEGER), String.valueOf(1), String.valueOf((START+NUMBER_OF_ROWS-1))); @@ -335,11 +332,9 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL FROM " - + "(SELECT * FROM " + executor.delimitIdentifier(schemalessTableName) - + " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS", + "SELECT ICOL,VCOL FROM (" + schemalessTableSql + ") SQOOP_SUBQUERY_ALIAS", tableColumns, - "DCOL", + "\"DCOL\"", String.valueOf(Types.DOUBLE), String.valueOf((double)START), String.valueOf((double)(START+NUMBER_OF_ROWS-1))); @@ -351,7 +346,7 @@ public class TestFromInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); FromJobConfiguration jobConfig = new FromJobConfiguration(); - String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName); + String fullTableName = executor.encloseIdentifiers(schemaName, tableName); linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; @@ -366,10 +361,9 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT * FROM " + fullTableName - + " WHERE ${CONDITIONS}", - "ICOL,DCOL,VCOL", - "ICOL", + "SELECT * FROM " + fullTableName + " WHERE ${CONDITIONS}", + "\"ICOL\", \"DCOL\", \"VCOL\"", + "\"ICOL\"", String.valueOf(Types.INTEGER), String.valueOf(START), String.valueOf(START+NUMBER_OF_ROWS-1)); @@ -381,7 +375,7 @@ public class TestFromInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); FromJobConfiguration jobConfig = new FromJobConfiguration(); - String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName); + String fullTableName = executor.encloseIdentifiers(schemaName, tableName); linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; @@ -397,10 +391,9 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT ICOL,VCOL FROM " + fullTableName - + " WHERE ${CONDITIONS}", + "SELECT ICOL,VCOL FROM " + fullTableName + " WHERE ${CONDITIONS}", tableColumns, - "ICOL", + "\"ICOL\"", String.valueOf(Types.INTEGER), String.valueOf(START), String.valueOf(START+NUMBER_OF_ROWS-1)); @@ -412,8 +405,6 @@ public class TestFromInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); FromJobConfiguration jobConfig = new FromJobConfiguration(); - String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName); - linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; jobConfig.fromJobConfig.schemaName = schemaName; @@ -428,10 +419,9 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT * FROM " + fullTableName - + " WHERE ${CONDITIONS}", - "ICOL,DCOL,VCOL", - "DCOL", + tableSql, + "\"ICOL\", \"DCOL\", \"VCOL\"", + "\"DCOL\"", String.valueOf(Types.DOUBLE), String.valueOf((double)START), String.valueOf((double)(START+NUMBER_OF_ROWS-1))); @@ -443,6 +433,8 @@ public class TestFromInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); FromJobConfiguration jobConfig = new FromJobConfiguration(); + String fullTableName = executor.encloseIdentifiers(schemaName, tableName); + linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; jobConfig.fromJobConfig.schemaName = schemaName; @@ -456,7 +448,7 @@ public class TestFromInitializer { Initializer initializer = new GenericJdbcFromInitializer(); initializer.initialize(initializerContext, linkConfig, jobConfig); Schema schema = initializer.getSchema(initializerContext, linkConfig, jobConfig); - assertEquals(getSchema(jobConfig.fromJobConfig.schemaName + "." + tableName), schema); + assertEquals(getSchema(fullTableName), schema); } @Test @@ -487,8 +479,6 @@ public class TestFromInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); FromJobConfiguration jobConfig = new FromJobConfiguration(); - String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName); - linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; jobConfig.fromJobConfig.schemaName = schemaName; @@ -504,32 +494,32 @@ public class TestFromInitializer { initializer.initialize(initializerContext, linkConfig, jobConfig); verifyResult(context, - "SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL FROM " - + "(SELECT * FROM " + fullTableName - + " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS", + "SELECT ICOL,VCOL FROM (" + tableSql + ") SQOOP_SUBQUERY_ALIAS", tableColumns, - "DCOL", + "\"DCOL\"", String.valueOf(Types.DOUBLE), String.valueOf((double)START), String.valueOf((double)(START+NUMBER_OF_ROWS-1))); } - private void verifyResult(MutableContext context, - String dataSql, String fieldNames, - String partitionColumnName, String partitionColumnType, - String partitionMinValue, String partitionMaxValue) { - assertEquals(dataSql, context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL)); - assertEquals(fieldNames, context.getString( - Constants.JOB_ETL_FIELD_NAMES)); - - assertEquals(partitionColumnName, context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME)); - assertEquals(partitionColumnType, context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNTYPE)); - assertEquals(partitionMinValue, context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE)); - assertEquals(partitionMaxValue, context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE)); + /** + * Asserts expected content inside the generated context. + * + * @param context Context that we're validating against + * @param dataSql Expected SQL fragment + * @param fieldNames All detected field names, they need to be properly escaped + * @param partitionColumnName Partition column name, it needs to be properly escaped + * @param partitionColumnType Partition column type + * @param partitionMinValue Minimal value for partitioning + * @param partitionMaxValue Maximal value for partitioning + */ + private void verifyResult(MutableContext context, String dataSql, String fieldNames, String partitionColumnName, String partitionColumnType, String partitionMinValue, String partitionMaxValue) { + assertEquals(context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL), dataSql); + assertEquals(context.getString(Constants.JOB_ETL_FIELD_NAMES), fieldNames); + + assertEquals(context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME), partitionColumnName); + assertEquals(context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNTYPE), partitionColumnType); + assertEquals(context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE), partitionMinValue); + assertEquals(context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE), partitionMaxValue); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java index f192c22..c69ec03 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java @@ -72,8 +72,7 @@ public class TestLoader { executor = new GenericJdbcExecutor(GenericJdbcTestConstants.LINK_CONFIGURATION); if (!executor.existTable(tableName)) { - executor.executeUpdate("CREATE TABLE " - + executor.delimitIdentifier(tableName) + executor.executeUpdate("CREATE TABLE " + executor.encloseIdentifier(tableName) + "(ICOL INTEGER PRIMARY KEY, DCOL DOUBLE, VCOL VARCHAR(20), DATECOL DATE, DATETIMECOL TIMESTAMP, TIMECOL TIME)"); } else { executor.deleteTableData(tableName); @@ -98,7 +97,7 @@ public class TestLoader { ToJobConfiguration jobConfig = new ToJobConfiguration(); context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_TO_DATA_SQL, - "INSERT INTO " + executor.delimitIdentifier(tableName) + " VALUES (?,?,?,?,?,?)"); + "INSERT INTO " + executor.encloseIdentifier(tableName) + " VALUES (?,?,?,?,?,?)"); Loader loader = new GenericJdbcLoader(); @@ -112,7 +111,7 @@ public class TestLoader { int index = START; ResultSet rs = executor.executeQuery("SELECT * FROM " - + executor.delimitIdentifier(tableName) + " ORDER BY ICOL"); + + executor.encloseIdentifier(tableName) + " ORDER BY ICOL"); while (rs.next()) { assertEquals(index, rs.getObject(1)); assertEquals((double) index, rs.getObject(2)); http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java index 1c65fc3..7e36666 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java @@ -61,13 +61,13 @@ public class TestToInitializer { public void setUp() { executor = new GenericJdbcExecutor(GenericJdbcTestConstants.LINK_CONFIGURATION); - String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName); + String fullTableName = executor.encloseIdentifier(schemaName) + "." + executor.encloseIdentifier(tableName); if (!executor.existTable(tableName)) { - executor.executeUpdate("CREATE SCHEMA " + executor.delimitIdentifier(schemaName)); + executor.executeUpdate("CREATE SCHEMA " + executor.encloseIdentifier(schemaName)); executor.executeUpdate("CREATE TABLE " + fullTableName + "(ICOL INTEGER PRIMARY KEY, DCOL DOUBLE, VCOL VARCHAR(20))"); } - fullTableName = executor.delimitIdentifier(schemalessTableName); + fullTableName = executor.encloseIdentifier(schemalessTableName); if (!executor.existTable(schemalessTableName)) { executor.executeUpdate("CREATE TABLE " + fullTableName + "(ICOL INTEGER PRIMARY KEY, DCOL DOUBLE, VCOL VARCHAR(20))"); } @@ -84,7 +84,7 @@ public class TestToInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); ToJobConfiguration jobConfig = new ToJobConfiguration(); - String fullTableName = executor.delimitIdentifier(schemalessTableName); + String fullTableName = executor.encloseIdentifier(schemalessTableName); linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; @@ -106,7 +106,7 @@ public class TestToInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); ToJobConfiguration jobConfig = new ToJobConfiguration(); - String fullTableName = executor.delimitIdentifier(schemalessTableName); + String fullTableName = executor.encloseIdentifier(schemalessTableName); linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; @@ -140,7 +140,7 @@ public class TestToInitializer { Initializer initializer = new GenericJdbcToInitializer(); initializer.initialize(initializerContext, linkConfig, jobConfig); - verifyResult(context, "INSERT INTO " + executor.delimitIdentifier(schemalessTableName) + " VALUES (?,?,?)"); + verifyResult(context, schemalessTableSql); } @Test @@ -149,7 +149,7 @@ public class TestToInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); ToJobConfiguration jobConfig = new ToJobConfiguration(); - String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName); + String fullTableName = executor.encloseIdentifiers(schemaName, tableName); linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; @@ -172,7 +172,7 @@ public class TestToInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); ToJobConfiguration jobConfig = new ToJobConfiguration(); - String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName); + String fullTableName = executor.encloseIdentifiers(schemaName, tableName); linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; @@ -208,7 +208,7 @@ public class TestToInitializer { Initializer initializer = new GenericJdbcToInitializer(); initializer.initialize(initializerContext, linkConfig, jobConfig); - verifyResult(context, "INSERT INTO " + executor.delimitIdentifier(tableName) + " VALUES (?,?,?)"); + verifyResult(context, tableSql); } private void verifyResult(MutableContext context, String dataSql) { @@ -255,7 +255,7 @@ public class TestToInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); ToJobConfiguration jobConfig = new ToJobConfiguration(); - String fullStageTableName = executor.delimitIdentifier(stageTableName); + String fullStageTableName = executor.encloseIdentifier(stageTableName); linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; @@ -333,7 +333,7 @@ public class TestToInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); ToJobConfiguration jobConfig = new ToJobConfiguration(); - String fullStageTableName = executor.delimitIdentifier(stageTableName); + String fullStageTableName = executor.encloseIdentifier(stageTableName); linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; @@ -359,7 +359,7 @@ public class TestToInitializer { LinkConfiguration linkConfig = new LinkConfiguration(); ToJobConfiguration jobConfig = new ToJobConfiguration(); - String fullStageTableName = executor.delimitIdentifier(stageTableName); + String fullStageTableName = executor.encloseIdentifier(stageTableName); linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER; linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL; http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java index c84e799..0bf62d5 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java @@ -132,13 +132,13 @@ abstract public class ConnectorTestCase extends TomcatTestCase { protected void fillRdbmsFromConfig(MJob job, String partitionColumn) { MConfigList fromConfig = job.getFromJobConfig(); - fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName().getTableName())); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName(partitionColumn)); + fromConfig.getStringInput("fromJobConfig.tableName").setValue(getTableName().getTableName()); + fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(partitionColumn); } protected void fillRdbmsToConfig(MJob job) { MConfigList toConfig = job.getToJobConfig(); - toConfig.getStringInput("toJobConfig.tableName").setValue(provider.escapeTableName(getTableName().getTableName())); + toConfig.getStringInput("toJobConfig.tableName").setValue(getTableName().getTableName()); } protected void fillHdfsLink(MLink link) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java index dac6db7..aa4fdde 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java @@ -17,15 +17,11 @@ */ package org.apache.sqoop.integration.connector.jdbc.generic; -import static org.testng.Assert.assertTrue; - -import org.apache.sqoop.common.Direction; import org.apache.sqoop.connector.hdfs.configuration.ToFormat; import org.apache.sqoop.model.MConfigList; import org.apache.sqoop.model.MDriverConfig; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; -import org.apache.sqoop.model.MSubmission; import org.apache.sqoop.test.testcases.ConnectorTestCase; import org.testng.annotations.Test; @@ -174,7 +170,7 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { MConfigList configs = job.getFromJobConfig(); configs.getStringInput("fromJobConfig.sql").setValue("SELECT " + provider.escapeColumnName("id") + " FROM " + provider.escapeTableName(getTableName().getTableName()) + " WHERE ${CONDITIONS}"); - configs.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + configs.getStringInput("fromJobConfig.partitionColumn").setValue("id"); fillHdfsToConfig(job, ToFormat.TEXT_FILE); saveJob(job); @@ -217,7 +213,7 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { "SELECT " + provider.escapeColumnName("id") + " as " + provider.escapeColumnName("i") + ", " + provider.escapeColumnName("id") + " as " + provider.escapeColumnName("j") + " FROM " + provider.escapeTableName(getTableName().getTableName()) + " WHERE ${CONDITIONS}"); - configs.getStringInput("fromJobConfig.partitionColumn").setValue(partitionColumn); + configs.getStringInput("fromJobConfig.partitionColumn").setValue("id"); configs.getStringInput("fromJobConfig.boundaryQuery").setValue( "SELECT MIN(" + partitionColumn + "), MAX(" + partitionColumn + ") FROM " + provider.escapeTableName(getTableName().getTableName())); http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java index 66c016d..a7be9c6 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java @@ -95,7 +95,7 @@ public class IncrementalReadTest extends ConnectorTestCase implements ITest { // Set the rdbms "FROM" config fillRdbmsFromConfig(job, "id"); MConfigList fromConfig = job.getFromJobConfig(); - fromConfig.getStringInput("incrementalRead.checkColumn").setValue(provider.escapeColumnName(checkColumn)); + fromConfig.getStringInput("incrementalRead.checkColumn").setValue(checkColumn); fromConfig.getStringInput("incrementalRead.lastValue").setValue(lastValue); // Fill hdfs "TO" config @@ -149,8 +149,8 @@ public class IncrementalReadTest extends ConnectorTestCase implements ITest { // Set the rdbms "FROM" config MConfigList fromConfig = job.getFromJobConfig(); fromConfig.getStringInput("fromJobConfig.sql").setValue(query); - fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); - fromConfig.getStringInput("incrementalRead.checkColumn").setValue(provider.escapeColumnName(checkColumn)); + fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue("id"); + fromConfig.getStringInput("incrementalRead.checkColumn").setValue(checkColumn); fromConfig.getStringInput("incrementalRead.lastValue").setValue(lastValue); // Fill hdfs "TO" config http://git-wip-us.apache.org/repos/asf/sqoop/blob/3362fbb1/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java index e9c4543..0008cac 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java @@ -65,7 +65,7 @@ public class TableStagedRDBMSTest extends ConnectorTestCase { // fill rdbms "TO" config here fillRdbmsToConfig(job); MConfigList configs = job.getToJobConfig(); - configs.getStringInput("toJobConfig.stageTableName").setValue(provider.escapeTableName(stageTableName.getTableName())); + configs.getStringInput("toJobConfig.stageTableName").setValue(stageTableName.getTableName()); // driver config MConfigList driverConfig = job.getDriverConfig();
