Repository: sqoop Updated Branches: refs/heads/sqoop2 63a83f5b0 -> 30fe4da2c
SQOOP-2699: Sqoop2: Oraoop: Improve Oracle parameters (David Robson via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/30fe4da2 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/30fe4da2 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/30fe4da2 Branch: refs/heads/sqoop2 Commit: 30fe4da2cd5a9e311c9f6702ef084a025889e036 Parents: 63a83f5 Author: Jarek Jarcec Cecho <[email protected]> Authored: Fri Nov 20 08:19:01 2015 -0800 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Fri Nov 20 08:19:01 2015 -0800 ---------------------------------------------------------------------- .../oracle/OracleJdbcCommonInitializer.java | 8 +- .../oracle/OracleJdbcConnectorConstants.java | 22 +- .../jdbc/oracle/OracleJdbcExtractor.java | 3 +- .../jdbc/oracle/OracleJdbcFromInitializer.java | 14 + .../connector/jdbc/oracle/OracleJdbcLoader.java | 136 ++++--- .../jdbc/oracle/OracleJdbcPartitioner.java | 7 +- .../jdbc/oracle/OracleJdbcToInitializer.java | 12 +- .../oracle/configuration/ConnectionConfig.java | 3 +- .../oracle/configuration/FromJobConfig.java | 26 +- .../jdbc/oracle/configuration/ToJobConfig.java | 14 +- .../oracle/util/OracleConnectionFactory.java | 63 +-- .../jdbc/oracle/util/OracleUtilities.java | 404 +++++++------------ .../oracle-jdbc-connector-config.properties | 70 ++-- .../OracleConnectionFactoryTest.java | 128 +----- 14 files changed, 357 insertions(+), 553 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcCommonInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcCommonInitializer.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcCommonInitializer.java index 1fd95c0..31768c1 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcCommonInitializer.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcCommonInitializer.java @@ -45,7 +45,7 @@ import org.apache.sqoop.job.etl.InitializerContext; import org.apache.sqoop.schema.Schema; import org.apache.sqoop.schema.type.Column; -public class OracleJdbcCommonInitializer<JobConfiguration> extends Initializer<LinkConfiguration, JobConfiguration> { +public abstract class OracleJdbcCommonInitializer<JobConfiguration> extends Initializer<LinkConfiguration, JobConfiguration> { private static final Logger LOG = Logger.getLogger(OracleJdbcCommonInitializer.class); @@ -107,6 +107,9 @@ public class OracleJdbcCommonInitializer<JobConfiguration> extends Initializer<L showUserTheOracleCommandToKillOraOop(context.getContext()); } + protected abstract List<String> + getColumnNames(JobConfiguration jobConfiguration) throws SQLException; + @Override public Schema getSchema(InitializerContext context, LinkConfiguration linkConfiguration, @@ -123,8 +126,7 @@ public class OracleJdbcCommonInitializer<JobConfiguration> extends Initializer<L Schema schema = new Schema(table.toString()); try { - List<String> colNames = OracleQueries.getToTableColumnNames( - connection, table, true, true); + List<String> colNames = getColumnNames(jobConfiguration); List<Column> columnTypes = OracleQueries.getColDataTypes(connection, table, colNames); http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcConnectorConstants.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcConnectorConstants.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcConnectorConstants.java index 2215cf3..a016399 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcConnectorConstants.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcConnectorConstants.java @@ -17,6 +17,10 @@ */ package org.apache.sqoop.connector.jdbc.oracle; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + public final class OracleJdbcConnectorConstants { // Resource bundle name @@ -39,12 +43,12 @@ public final class OracleJdbcConnectorConstants { public static final String ORACLE_JDBC_DRIVER_CLASS = "oracle.jdbc.OracleDriver"; - public static final String ORACLE_SESSION_INITIALIZATION_STATEMENTS_DEFAULT = - "alter session disable parallel query;" + - "alter session set \"_serial_direct_read\"=true;" + - "alter session set tracefile_identifier=oraoop;" + - "--alter session set events '10046 trace name context forever, level 8';"; - + public static final List<String> + ORACLE_SESSION_INITIALIZATION_STATEMENTS_DEFAULT = + Collections.unmodifiableList( + Arrays.asList("alter session disable parallel query", + "alter session set \"_serial_direct_read\"=true", + "alter session set tracefile_identifier=sqoop")); ///////////////////////////////////////////////////////////////////// @@ -287,12 +291,6 @@ public final class OracleJdbcConnectorConstants { // public static final String ORAOOP_ORACLE_APPEND_VALUES_HINT_USAGE = // "oraoop.oracle.append.values.hint.usage"; // - /** - * Whether to use the append values hint for exports. - */ - public enum AppendValuesHintUsage { - AUTO, ON, OFF - } // http://download.oracle.com/docs/cd/E11882_01/server.112/e17118/ // sql_elements001.htm#i45441 http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcExtractor.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcExtractor.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcExtractor.java index df15fc2..4d0da9c 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcExtractor.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcExtractor.java @@ -248,8 +248,7 @@ public class OracleJdbcExtractor extends } OracleUtilities.OracleTableImportWhereClauseLocation whereClauseLocation = - OracleUtilities.getTableImportWhereClauseLocation(jobConfig, - OracleUtilities.OracleTableImportWhereClauseLocation.SUBSPLIT); + OracleUtilities.getTableImportWhereClauseLocation(jobConfig); int numberOfDataChunks = this.dbInputSplit.getNumberOfDataChunks(); for (int idx = 0; idx < numberOfDataChunks; idx++) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcFromInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcFromInitializer.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcFromInitializer.java index 62a0e84..7c54cf1 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcFromInitializer.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcFromInitializer.java @@ -18,6 +18,7 @@ package org.apache.sqoop.connector.jdbc.oracle; import java.sql.SQLException; +import java.util.List; import org.apache.commons.lang.BooleanUtils; import org.apache.log4j.Logger; @@ -87,4 +88,17 @@ public class OracleJdbcFromInitializer extends } } + @Override + protected List<String> getColumnNames(FromJobConfiguration jobConfiguration) + throws SQLException { + List<String> colNames = OracleQueries.getFromTableColumnNames(connection, + table, OracleUtilities.omitLobAndLongColumnsDuringImport( + jobConfiguration.fromJobConfig), + true // <- onlyOraOopSupportedTypes + ); + + return OracleUtilities.getSelectedColumnNamesInOracleTable(table, + colNames, jobConfiguration.fromJobConfig.columns); + } + } http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcLoader.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcLoader.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcLoader.java index b741dc8..708f589 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcLoader.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcLoader.java @@ -96,11 +96,6 @@ public class OracleJdbcLoader extends Loader<LinkConfiguration, ToJobConfigurati linkConfiguration.connectionConfig.username, jobConfiguration.toJobConfig.tableName); } - tableColumns = OracleQueries.getToTableColumns( - connection, table, true, false); - tableHasMapperRowNumberColumn = - tableColumns.findColumnByName( - OracleJdbcConnectorConstants.COLUMN_NAME_EXPORT_MAPPER_ROW) != null; // Should we use the APPEND_VALUES Oracle hint?... useAppendValuesOracleHint = false; @@ -125,9 +120,6 @@ public class OracleJdbcLoader extends Loader<LinkConfiguration, ToJobConfigurati updateColumnNames = OracleUtilities. getExportUpdateKeyColumnNames(jobConfiguration.toJobConfig); - tableColumns = OracleQueries.getToTableColumns( - connection, table, true, false); - if (updateMode == UpdateMode.Merge || updateMode == UpdateMode.Update) { // Should we use the APPEND_VALUES Oracle hint?... useAppendValuesOracleHint = canUseOracleAppendValuesHint(); @@ -184,6 +176,13 @@ public class OracleJdbcLoader extends Loader<LinkConfiguration, ToJobConfigurati setupUpdate(linkConfiguration, jobConfiguration); } + tableColumns = OracleQueries.getToTableColumns( + connection, table, true, false); + + tableHasMapperRowNumberColumn = + tableColumns.findColumnByName( + OracleJdbcConnectorConstants.COLUMN_NAME_EXPORT_MAPPER_ROW) != null; + // Has the user forced the use of APPEND_VALUES either on or off?... useAppendValuesOracleHint = allowUserToOverrideUseOfTheOracleAppendValuesHint( @@ -304,74 +303,85 @@ public class OracleJdbcLoader extends Loader<LinkConfiguration, ToJobConfigurati * table (that OraOop created), there are two pseudo-columns we added to * the table to identify the export job and the mapper. */ + List<String> columnNamesList = new ArrayList<String>(); + for(Column column : context.getSchema().getColumnsList()) { + columnNamesList.add(column.getName()); + } int colCount = 0; for (int idx = 0; idx < this.tableColumns.size(); idx++) { OracleTableColumn oracleTableColumn = this.tableColumns.get(idx); String columnName = oracleTableColumn.getName(); + if(columnNamesList.contains(columnName) || + OracleJdbcConnectorConstants.COLUMN_NAME_EXPORT_PARTITION + .equals(columnName) || + OracleJdbcConnectorConstants.COLUMN_NAME_EXPORT_SUBPARTITION + .equals(columnName) || + OracleJdbcConnectorConstants.COLUMN_NAME_EXPORT_MAPPER_ROW + .equals(columnName)) { + // column names... + if (colCount > 0) { + sqlNames.append("\n,"); + } + sqlNames.append(columnName); - // column names... - if (colCount > 0) { - sqlNames.append("\n,"); - } - sqlNames.append(columnName); - - // column values... - if (colCount > 0) { - sqlValues.append("\n,"); - } + // column values... + if (colCount > 0) { + sqlValues.append("\n,"); + } - String pseudoColumnValue = - generateInsertValueForPseudoColumn(columnName); + String pseudoColumnValue = + generateInsertValueForPseudoColumn(columnName); - String bindVarName = null; + String bindVarName = null; - if (pseudoColumnValue != null) { - bindVarName = pseudoColumnValue; - } else if (oracleTableColumn.getOracleType() == OracleQueries - .getOracleType("STRUCT")) { - if (oracleTableColumn.getDataType().equals( - OracleJdbcConnectorConstants.Oracle.URITYPE)) { - bindVarName = - String.format("urifactory.getUri(%s)", - columnNameToBindVariable(columnName)); + if (pseudoColumnValue != null) { + bindVarName = pseudoColumnValue; + } else if (oracleTableColumn.getOracleType() == OracleQueries + .getOracleType("STRUCT")) { + if (oracleTableColumn.getDataType().equals( + OracleJdbcConnectorConstants.Oracle.URITYPE)) { + bindVarName = + String.format("urifactory.getUri(%s)", + columnNameToBindVariable(columnName)); + } + //TODO: Date as string? + /*} else if (getConf().getBoolean( + OraOopConstants.ORAOOP_MAP_TIMESTAMP_AS_STRING, + OraOopConstants.ORAOOP_MAP_TIMESTAMP_AS_STRING_DEFAULT)) { + if (oracleTableColumn.getOracleType() == OraOopOracleQueries + .getOracleType("DATE")) { + bindVarName = + String.format("to_date(%s, 'yyyy-mm-dd hh24:mi:ss')", + columnNameToBindVariable(columnName)); + } else if (oracleTableColumn.getOracleType() == OraOopOracleQueries + .getOracleType("TIMESTAMP")) { + bindVarName = + String.format("to_timestamp(%s, 'yyyy-mm-dd hh24:mi:ss.ff')", + columnNameToBindVariable(columnName)); + } else if (oracleTableColumn.getOracleType() == OraOopOracleQueries + .getOracleType("TIMESTAMPTZ")) { + bindVarName = + String.format( + "to_timestamp_tz(%s, 'yyyy-mm-dd hh24:mi:ss.ff TZR')", + columnNameToBindVariable(columnName)); + } else if (oracleTableColumn.getOracleType() == OraOopOracleQueries + .getOracleType("TIMESTAMPLTZ")) { + bindVarName = + String.format( + "to_timestamp_tz(%s, 'yyyy-mm-dd hh24:mi:ss.ff TZR')", + columnNameToBindVariable(columnName)); + }*/ } - //TODO: Date as string? - /*} else if (getConf().getBoolean( - OraOopConstants.ORAOOP_MAP_TIMESTAMP_AS_STRING, - OraOopConstants.ORAOOP_MAP_TIMESTAMP_AS_STRING_DEFAULT)) { - if (oracleTableColumn.getOracleType() == OraOopOracleQueries - .getOracleType("DATE")) { - bindVarName = - String.format("to_date(%s, 'yyyy-mm-dd hh24:mi:ss')", - columnNameToBindVariable(columnName)); - } else if (oracleTableColumn.getOracleType() == OraOopOracleQueries - .getOracleType("TIMESTAMP")) { - bindVarName = - String.format("to_timestamp(%s, 'yyyy-mm-dd hh24:mi:ss.ff')", - columnNameToBindVariable(columnName)); - } else if (oracleTableColumn.getOracleType() == OraOopOracleQueries - .getOracleType("TIMESTAMPTZ")) { - bindVarName = - String.format( - "to_timestamp_tz(%s, 'yyyy-mm-dd hh24:mi:ss.ff TZR')", - columnNameToBindVariable(columnName)); - } else if (oracleTableColumn.getOracleType() == OraOopOracleQueries - .getOracleType("TIMESTAMPLTZ")) { - bindVarName = - String.format( - "to_timestamp_tz(%s, 'yyyy-mm-dd hh24:mi:ss.ff TZR')", - columnNameToBindVariable(columnName)); - }*/ - } - if (bindVarName == null) { - bindVarName = columnNameToBindVariable(columnName); - } + if (bindVarName == null) { + bindVarName = columnNameToBindVariable(columnName); + } - sqlValues.append(bindVarName); + sqlValues.append(bindVarName); - colCount++; + colCount++; + } } String sql = http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcPartitioner.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcPartitioner.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcPartitioner.java index 00c7752..24aa08e 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcPartitioner.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcPartitioner.java @@ -104,9 +104,7 @@ public class OracleJdbcPartitioner extends OracleUtilities.OracleBlockToSplitAllocationMethod blockAllocationMethod = OracleUtilities .getOracleBlockToSplitAllocationMethod( - jobConfiguration.fromJobConfig, - OracleUtilities. - OracleBlockToSplitAllocationMethod.ROUNDROBIN); + jobConfiguration.fromJobConfig); // Group the Oracle data-chunks into splits... splits = @@ -145,8 +143,7 @@ public class OracleJdbcPartitioner extends private List<String> getPartitionList(FromJobConfig jobConfig) { LOG.debug("Partition list = " + jobConfig.partitionList); - List<String> result = - OracleUtilities.splitOracleStringList(jobConfig.partitionList); + List<String> result = jobConfig.partitionList; if (result != null && result.size() > 0) { LOG.debug("Partition filter list: " + result.toString()); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcToInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcToInitializer.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcToInitializer.java index f1d92f0..b58e4ad 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcToInitializer.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/OracleJdbcToInitializer.java @@ -121,7 +121,7 @@ public class OracleJdbcToInitializer extends boolean noLoggingOnNewTable = BooleanUtils.isTrue(jobConfig.nologging); - String updateKeyCol = jobConfig.updateKey; + List<String> updateKeyCol = jobConfig.updateKey; /* =========================== */ /* VALIDATION OF INPUTS */ @@ -495,4 +495,14 @@ public class OracleJdbcToInitializer extends return result; } + + @Override + protected List<String> getColumnNames(ToJobConfiguration jobConfiguration) + throws SQLException { + List<String> colNames = OracleQueries.getToTableColumnNames( + connection, table, true, true); + + return OracleUtilities.getSelectedColumnNamesInOracleTable(table, + colNames, jobConfiguration.toJobConfig.columns); + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/ConnectionConfig.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/ConnectionConfig.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/ConnectionConfig.java index c355a77..54e12ed 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/ConnectionConfig.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/ConnectionConfig.java @@ -27,6 +27,7 @@ import org.apache.sqoop.validation.validators.StartsWith; import java.sql.DriverManager; import java.sql.SQLException; +import java.util.List; import java.util.Map; /** @@ -56,7 +57,7 @@ public class ConnectionConfig { public Integer fetchSize; @Input - public String initializationStatements; + public List<String> initializationStatements; @Input public Boolean jdbcUrlVerbatim; http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/FromJobConfig.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/FromJobConfig.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/FromJobConfig.java index 38c808f..fab7348 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/FromJobConfig.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/FromJobConfig.java @@ -17,6 +17,9 @@ */ package org.apache.sqoop.connector.jdbc.oracle.configuration; +import java.util.List; + +import org.apache.sqoop.connector.jdbc.oracle.util.OracleUtilities; import org.apache.sqoop.model.ConfigClass; import org.apache.sqoop.model.Input; import org.apache.sqoop.model.Validator; @@ -32,27 +35,32 @@ public class FromJobConfig { public String tableName; @Input + public List<String> columns; + + @Input public Boolean consistentRead; @Input public Long consistentReadScn; - @Input(size = 2000) - public String partitionList; + @Input + public List<String> partitionList; - @Input(size = 2000) - public String dataChunkMethod; + @Input + public OracleUtilities.OracleDataChunkMethod dataChunkMethod; - @Input(size = 2000) - public String dataChunkAllocationMethod; + @Input + public OracleUtilities.OracleBlockToSplitAllocationMethod + dataChunkAllocationMethod; - @Input(size = 2000) - public String whereClauseLocation; + @Input + public OracleUtilities.OracleTableImportWhereClauseLocation + whereClauseLocation; @Input public Boolean omitLobColumns; - @Input + @Input(size = 2000) public String queryHint; @Input(size = 2000) http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/ToJobConfig.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/ToJobConfig.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/ToJobConfig.java index 939a87a..f3d9b75 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/ToJobConfig.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/configuration/ToJobConfig.java @@ -17,6 +17,9 @@ */ package org.apache.sqoop.connector.jdbc.oracle.configuration; +import java.util.List; + +import org.apache.sqoop.connector.jdbc.oracle.util.OracleUtilities; import org.apache.sqoop.model.ConfigClass; import org.apache.sqoop.model.Input; import org.apache.sqoop.model.Validator; @@ -31,6 +34,9 @@ public class ToJobConfig { @Input(size = 2000, validators = { @Validator(NotEmpty.class)}) public String tableName; + @Input + public List<String> columns; + @Input(size = 2000) public String templateTable; @@ -40,8 +46,8 @@ public class ToJobConfig { @Input public Boolean nologging; - @Input(size = 2000) - public String updateKey; + @Input + public List<String> updateKey; @Input public Boolean updateMerge; @@ -55,8 +61,8 @@ public class ToJobConfig { @Input(size = 2000) public String temporaryStorageClause; - @Input(size = 2000) - public String appendValuesHint; + @Input + public OracleUtilities.AppendValuesHintUsage appendValuesHint; @Input public Boolean parallel; http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/util/OracleConnectionFactory.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/util/OracleConnectionFactory.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/util/OracleConnectionFactory.java index 3ebb0d4..28af8da 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/util/OracleConnectionFactory.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/util/OracleConnectionFactory.java @@ -180,22 +180,24 @@ public class OracleConnectionFactory { } public static void executeOraOopSessionInitializationStatements( - Connection connection, String sessionInitializationStatements) { - String statementsStr = sessionInitializationStatements; - if(StringUtils.isEmpty(statementsStr)) { - statementsStr =OracleJdbcConnectorConstants. + Connection connection, List<String> sessionInitializationStatements) { + List<String> statements = sessionInitializationStatements; + + if(statements == null || statements.isEmpty()) { + statements = OracleJdbcConnectorConstants. ORACLE_SESSION_INITIALIZATION_STATEMENTS_DEFAULT; } - List<String> statements = - parseOraOopSessionInitializationStatements(statementsStr); - - if (statements.size() == 0) { - LOG.warn("No Oracle 'session initialization' statements were found to " - + "execute."); - } else { - for (String statement : statements) { + int numStatements = 0; + for (String statement : statements) { + String initializationStatement = statement.trim(); + if (initializationStatement != null + && !initializationStatement.isEmpty() + && !initializationStatement + .startsWith(OracleJdbcConnectorConstants.Oracle. + ORACLE_SQL_STATEMENT_COMMENT_TOKEN)) { try { + numStatements++; connection.createStatement().execute(statement); LOG.info("Initializing Oracle session with SQL : " + statement); } catch (Exception ex) { @@ -206,41 +208,10 @@ public class OracleConnectionFactory { } } } - } - - public static List<String> parseOraOopSessionInitializationStatements( - String sessionInitializationStatements) { - - ArrayList<String> result = new ArrayList<String>(); - - if (sessionInitializationStatements != null - && !sessionInitializationStatements.isEmpty()) { - String[] initializationStatements = - sessionInitializationStatements.split(";"); - for (String initializationStatement : initializationStatements) { - initializationStatement = initializationStatement.trim(); - if (initializationStatement != null - && !initializationStatement.isEmpty() - && !initializationStatement - .startsWith(OracleJdbcConnectorConstants.Oracle. - ORACLE_SQL_STATEMENT_COMMENT_TOKEN)) { - - LOG.debug(String - .format( - "initializationStatement (quoted & pre-expression " - + "evaluation) = \"%s\"", - initializationStatement)); - - //TODO: Not supported in Sqoop 2? - /*initializationStatement = - OracleUtilities.replaceConfigurationExpression( - initializationStatement, conf);*/ - - result.add(initializationStatement); - } - } + if(numStatements==0) { + LOG.warn("No Oracle 'session initialization' statements were found to " + + "execute."); } - return result; } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/util/OracleUtilities.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/util/OracleUtilities.java b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/util/OracleUtilities.java index 4219af2..11717cb 100644 --- a/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/util/OracleUtilities.java +++ b/connector/connector-oracle-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/oracle/util/OracleUtilities.java @@ -93,6 +93,13 @@ public final class OracleUtilities { SUBSPLIT, SPLIT } + /** + * Whether to use the append values hint for exports. + */ + public enum AppendValuesHintUsage { + AUTO, ON, OFF + } + // /** // * Used for testing purposes - can get OraOop to call a class to run a report // * on various performance metrics. @@ -272,34 +279,34 @@ public final class OracleUtilities { return String.format("%s()", stackTraceElements[1].getMethodName()); } -// public static String[] getDuplicatedStringArrayValues(String[] list, -// boolean ignoreCase) { -// -// if (list == null) { -// throw new IllegalArgumentException("The list argument cannot be null"); -// } -// -// ArrayList<String> duplicates = new ArrayList<String>(); -// -// for (int idx1 = 0; idx1 < list.length - 1; idx1++) { -// for (int idx2 = idx1 + 1; idx2 < list.length; idx2++) { -// if (list[idx1].equals(list[idx2])) { -// // If c is a duplicate of both a & b, don't add c to the list twice... -// if (!duplicates.contains(list[idx2])) { -// duplicates.add(list[idx2]); -// } -// -// } else if (ignoreCase && list[idx1].equalsIgnoreCase((list[idx2]))) { -// // If c is a duplicate of both a & b, don't add c to the list twice... -// if (stringListIndexOf(duplicates, list[idx2], ignoreCase) == -1) { -// duplicates.add(list[idx2]); -// } -// } -// } -// } -// -// return duplicates.toArray(new String[duplicates.size()]); -// } + public static String[] getDuplicatedStringArrayValues(String[] list, + boolean ignoreCase) { + + if (list == null) { + throw new IllegalArgumentException("The list argument cannot be null"); + } + + ArrayList<String> duplicates = new ArrayList<String>(); + + for (int idx1 = 0; idx1 < list.length - 1; idx1++) { + for (int idx2 = idx1 + 1; idx2 < list.length; idx2++) { + if (list[idx1].equals(list[idx2])) { + // If c is a duplicate of both a & b, don't add c to the list twice... + if (!duplicates.contains(list[idx2])) { + duplicates.add(list[idx2]); + } + + } else if (ignoreCase && list[idx1].equalsIgnoreCase((list[idx2]))) { + // If c is a duplicate of both a & b, don't add c to the list twice... + if (stringListIndexOf(duplicates, list[idx2], ignoreCase) == -1) { + duplicates.add(list[idx2]); + } + } + } + } + + return duplicates.toArray(new String[duplicates.size()]); + } public static String getFullExceptionMessage(Exception ex) { @@ -365,140 +372,34 @@ public final class OracleUtilities { public static OracleDataChunkMethod getOraOopOracleDataChunkMethod(FromJobConfig jobConfig) { - - String strMethod = jobConfig.dataChunkMethod; - if (strMethod == null) { - return OracleDataChunkMethod.ROWID; - } - - OracleDataChunkMethod result; - - try { - strMethod = strMethod.toUpperCase().trim(); - result = OracleDataChunkMethod.valueOf(strMethod); - } catch (IllegalArgumentException ex) { + OracleDataChunkMethod result = jobConfig.dataChunkMethod; + if(result == null) { result = OracleDataChunkMethod.ROWID; - LOG.error("An invalid value of \"" + strMethod - + "\" was specified for the data chunk method " - + "configuration property value.\n" + "\tThe default value of " - + OracleDataChunkMethod.ROWID - + " will be used."); } return result; } public static OracleBlockToSplitAllocationMethod getOracleBlockToSplitAllocationMethod( - FromJobConfig jobConfig, - OracleBlockToSplitAllocationMethod defaultMethod) { - - String strMethod = jobConfig.dataChunkAllocationMethod; - if (strMethod == null) { - return defaultMethod; - } - - OracleBlockToSplitAllocationMethod result; - - try { - strMethod = strMethod.toUpperCase().trim(); - result = - OracleBlockToSplitAllocationMethod - .valueOf(strMethod); - } catch (IllegalArgumentException ex) { - result = defaultMethod; - - String errorMsg = - String - .format( - "An invalid value of \"%s\" was specified for the data chunk " - + " allocation method configuration property value.\n" - + "\tValid values are: %s\n" - + "\tThe default value of %s will be used.", - strMethod, - getOracleBlockToSplitAllocationMethods(), defaultMethod.name()); - LOG.error(errorMsg); + FromJobConfig jobConfig) { + OracleBlockToSplitAllocationMethod result = + jobConfig.dataChunkAllocationMethod; + if(result == null) { + result = OracleBlockToSplitAllocationMethod.ROUNDROBIN; } - return result; } - private static String getOracleBlockToSplitAllocationMethods() { - - OracleBlockToSplitAllocationMethod[] values = - OracleBlockToSplitAllocationMethod.values(); - - StringBuilder result = - new StringBuilder((2 * values.length) - 1); // <- Include capacity - // for commas - - for (int idx = 0; idx < values.length; idx++) { - OracleBlockToSplitAllocationMethod value = - values[idx]; - if (idx > 0) { - result.append(" or "); - } - result.append(value.name()); - } - return result.toString(); - } - public static OracleTableImportWhereClauseLocation getTableImportWhereClauseLocation( - FromJobConfig jobConfig, - OracleTableImportWhereClauseLocation defaultLocation) { - - String strLocation = jobConfig.whereClauseLocation; - - if (strLocation == null) { - return defaultLocation; + FromJobConfig jobConfig) { + OracleTableImportWhereClauseLocation result = jobConfig.whereClauseLocation; + if(result == null) { + result = OracleTableImportWhereClauseLocation.SUBSPLIT; } - - OracleTableImportWhereClauseLocation result; - - try { - strLocation = strLocation.toUpperCase().trim(); - result = - OracleTableImportWhereClauseLocation - .valueOf(strLocation); - } catch (IllegalArgumentException ex) { - result = defaultLocation; - - String errorMsg = - String - .format( - "An invalid value of \"%s\"was specified for the " - + "where clause location configuration property value.\n" - + "\tValid values are: %s\n" - + "\tThe default value of %s will be used.", strLocation, - getOracleTableImportWhereClauseLocations(), defaultLocation - .name()); - LOG.error(errorMsg); - } - return result; } - private static String getOracleTableImportWhereClauseLocations() { - - OracleTableImportWhereClauseLocation[] locationValues = - OracleTableImportWhereClauseLocation.values(); - - StringBuilder result = - new StringBuilder((2 * locationValues.length) - 1); // <- Include - // capacity for - // commas - - for (int idx = 0; idx < locationValues.length; idx++) { - OracleTableImportWhereClauseLocation locationValue = - locationValues[idx]; - if (idx > 0) { - result.append(" or "); - } - result.append(locationValue.name()); - } - return result.toString(); - } - // public static String getOutputDirectory( // org.apache.hadoop.conf.Configuration conf) { // @@ -516,52 +417,6 @@ public final class OracleUtilities { // return StringUtils.rightPad(s, n); // } // -// public static String replaceConfigurationExpression(String str, -// org.apache.hadoop.conf.Configuration conf) { -// -// int startPos = str.indexOf('{'); -// int endPos = str.indexOf('}'); -// -// // Example: -// // alter session set timezone = '{oracle.sessionTimeZone|GMT}'; -// -// if (startPos == -1 || endPos == -1) { -// return str; -// } -// -// String configName = null; -// String defaultValue = null; -// -// String expression = str.substring(startPos + 1, endPos); -// int defaultValuePos = expression.indexOf('|'); -// if (defaultValuePos == -1) { -// // return expression; -// configName = expression; -// } else { -// configName = expression.substring(0, defaultValuePos); -// defaultValue = expression.substring(defaultValuePos + 1); -// } -// -// if (defaultValue == null) { -// defaultValue = ""; -// } -// -// String configValue = conf.get(configName); -// if (configValue == null) { -// configValue = defaultValue; -// } -// -// String result = str.replace(String.format("{%s}", expression), configValue); -// -// LOG.debug(String.format("The expression:\n%s\nwas replaced with:\n%s", str, -// result)); -// -// // Recurse to evaluate any other expressions... -// result = replaceConfigurationExpression(result, conf); -// -// return result; -// } -// // public static boolean stackContainsClass(String className) { // // StackTraceElement[] stackTraceElements = (new Throwable()).getStackTrace(); @@ -676,20 +531,20 @@ public final class OracleUtilities { return result.toString(); } -// public static int stringListIndexOf(List<String> list, String value, -// boolean ignoreCase) { -// -// for (int idx = 0; idx < list.size(); idx++) { -// if (list.get(idx).equals(value)) { -// return idx; -// } -// if (ignoreCase && list.get(idx).equalsIgnoreCase(value)) { -// return idx; -// } -// } -// return -1; -// } -// + public static int stringListIndexOf(List<String> list, String value, + boolean ignoreCase) { + + for (int idx = 0; idx < list.size(); idx++) { + if (list.get(idx).equals(value)) { + return idx; + } + if (ignoreCase && list.get(idx).equalsIgnoreCase(value)) { + return idx; + } + } + return -1; + } + // public static void writeOutputFile(org.apache.hadoop.conf.Configuration conf, // String fileName, String fileText) { // @@ -1240,20 +1095,12 @@ public final class OracleUtilities { public static String[] getExportUpdateKeyColumnNames(ToJobConfig jobConfig) { - if (jobConfig.updateKey == null) { + if (jobConfig.updateKey == null || jobConfig.updateKey.isEmpty()) { // This must be an "insert-export" if no --update-key has been specified! return new String[0]; } - String[] columnNames = jobConfig.updateKey.split(","); - for (int idx = 0; idx < columnNames.length; idx++) { - columnNames[idx] = columnNames[idx].trim(); - if (!columnNames[idx].startsWith("\"")) { - columnNames[idx] = columnNames[idx].toUpperCase(); - } - - } - return columnNames; + return jobConfig.updateKey.toArray(new String[jobConfig.updateKey.size()]); } // /** @@ -1303,60 +1150,15 @@ public final class OracleUtilities { return result; } - public static OracleJdbcConnectorConstants.AppendValuesHintUsage - getOracleAppendValuesHintUsage(ToJobConfig jobConfig) { - - String strUsage = jobConfig.appendValuesHint; - if (strUsage == null) { - return OracleJdbcConnectorConstants.AppendValuesHintUsage.AUTO; - } - - OracleJdbcConnectorConstants.AppendValuesHintUsage result; - - try { - strUsage = strUsage.toUpperCase().trim(); - result = OracleJdbcConnectorConstants. - AppendValuesHintUsage.valueOf(strUsage); - } catch (IllegalArgumentException ex) { - result = OracleJdbcConnectorConstants.AppendValuesHintUsage.AUTO; - - String errorMsg = - String - .format( - "An invalid value of \"%s\" was specified for the " - + "append values hint configuration property value.\n" - + "\tValid values are: %s\n" - + "\tThe default value of %s will be used.", strUsage, - getOraOopOracleAppendValuesHintUsageValues(), - OracleJdbcConnectorConstants. - AppendValuesHintUsage.AUTO.name()); - LOG.error(errorMsg); + public static AppendValuesHintUsage getOracleAppendValuesHintUsage( + ToJobConfig jobConfig) { + AppendValuesHintUsage result = jobConfig.appendValuesHint; + if (result == null) { + result = AppendValuesHintUsage.AUTO; } - return result; } - private static String getOraOopOracleAppendValuesHintUsageValues() { - - OracleJdbcConnectorConstants.AppendValuesHintUsage[] values = - OracleJdbcConnectorConstants.AppendValuesHintUsage.values(); - - StringBuilder result = new StringBuilder((2 * values.length) - 1); // <- - // Include - // capacity - // for - // commas - - for (int idx = 0; idx < values.length; idx++) { - OracleJdbcConnectorConstants.AppendValuesHintUsage value = values[idx]; - if (idx > 0) { - result.append(" or "); - } - result.append(value.name()); - } - return result.toString(); - } - public static String getImportHint(FromJobConfig jobConfig) { String result = null; result = jobConfig.queryHint; @@ -1443,4 +1245,76 @@ public final class OracleUtilities { return result; } + + private static boolean isEscaped(String name) { + return name.startsWith("\"") && name.endsWith("\""); + } + + private String escapeOracleColumnName(String columnName) { + if (isEscaped(columnName)) { + return columnName; + } else { + return "\"" + columnName + "\""; + } + } + + private static String unescapeOracleColumnName(String columnName) { + if (isEscaped(columnName)) { + return columnName.substring(1, columnName.length() - 1); + } else { + return columnName; + } + } + + public static List<String> getSelectedColumnNamesInOracleTable( + OracleTable table, List<String> colNamesInTable, + List<String> selectedColumnsInput) { + if (selectedColumnsInput != null && selectedColumnsInput.size() > 0) { + String[] selectedColumns = new String[selectedColumnsInput.size()]; + + for (int idx = 0; idx < selectedColumnsInput.size(); idx++) { + String column = selectedColumnsInput.get(idx); + // If the user did not escape this column name, then we should + // uppercase it... + if (!isEscaped(column)) { + selectedColumns[idx] = column.toUpperCase(); + } else { + // If the user escaped this column name, then we should + // retain its case... + selectedColumns[idx] = unescapeOracleColumnName(column); + } + } + + // Ensure there are no duplicated column names... + String[] duplicates = + OracleUtilities + .getDuplicatedStringArrayValues(selectedColumns, false); + if (duplicates.length > 0) { + StringBuilder msg = new StringBuilder(); + msg.append("The following column names have been duplicated in the "); + msg.append("\"--columns\" clause:\n"); + + for (String duplicate : duplicates) { + msg.append("\t" + duplicate + "\n"); + } + + throw new RuntimeException(msg.toString()); + } + + List<String> result = new ArrayList<String>(selectedColumnsInput.size()); + // Ensure the user selected column names that actually exist... + for (String selectedColumn : selectedColumns) { + if (!colNamesInTable.contains(selectedColumn)) { + throw new RuntimeException(String.format( + "The column named \"%s\" does not exist within the table" + + "%s (or is of an unsupported data-type).", selectedColumn, + table.toString())); + } + result.add(selectedColumn); + } + return result; + } else { + return colNamesInTable; + } + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/main/resources/oracle-jdbc-connector-config.properties ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/main/resources/oracle-jdbc-connector-config.properties b/connector/connector-oracle-jdbc/src/main/resources/oracle-jdbc-connector-config.properties index e0b9951..d99ab9b 100644 --- a/connector/connector-oracle-jdbc/src/main/resources/oracle-jdbc-connector-config.properties +++ b/connector/connector-oracle-jdbc/src/main/resources/oracle-jdbc-connector-config.properties @@ -17,14 +17,14 @@ ############################ # Connection Config # -connectionConfig.label = Link configuration +connectionConfig.label = Oracle connection configuration connectionConfig.help = You must supply the information requested in order to \ - create a link object. + create an Oracle connection object. # connect string -connectionConfig.connectionString.label = JDBC Connection String +connectionConfig.connectionString.label = JDBC connection string connectionConfig.connectionString.help = Enter the value of JDBC connection string to be \ - used by this connector for creating database connections. + used by this connector for creating Oracle connections. # username string connectionConfig.username.label = Username @@ -37,26 +37,26 @@ connectionConfig.password.help = Enter the password to be used for connecting to database. # jdbc properties -connectionConfig.jdbcProperties.label = JDBC Connection Properties +connectionConfig.jdbcProperties.label = JDBC connection properties connectionConfig.jdbcProperties.help = Enter any JDBC properties that should be \ supplied during the creation of connection. -connectionConfig.timeZone.label = timeZone +connectionConfig.timeZone.label = Session time zone connectionConfig.timeZone.help = timeZone -connectionConfig.actionName.label = actionName +connectionConfig.actionName.label = Session action name connectionConfig.actionName.help = actionName -connectionConfig.fetchSize.label = fetchSize +connectionConfig.fetchSize.label = JDBC fetch size connectionConfig.fetchSize.help = fetchSize -connectionConfig.initializationStatements.label = initializationStatements +connectionConfig.initializationStatements.label = Session initialization statements connectionConfig.initializationStatements.help = initializationStatements -connectionConfig.jdbcUrlVerbatim.label = jdbcUrlVerbatim +connectionConfig.jdbcUrlVerbatim.label = Use JDBC connection string verbatim connectionConfig.jdbcUrlVerbatim.help = jdbcUrlVerbatim -connectionConfig.racServiceName.label = racServiceName +connectionConfig.racServiceName.label = RAC service name connectionConfig.racServiceName.help = racServiceName # ToJob Config @@ -69,68 +69,74 @@ toJobConfig.help = You must supply the information requested in order to create toJobConfig.tableName.label = Table name toJobConfig.tableName.help = Table name to write data into -toJobConfig.templateTable.label = templateTable +toJobConfig.columns.label = Columns +toJobConfig.columns.help = Columns + +toJobConfig.templateTable.label = Template table name toJobConfig.templateTable.help = templateTable -toJobConfig.partitioned.label = partitioned +toJobConfig.partitioned.label = Partitioned toJobConfig.partitioned.help = partitioned -toJobConfig.nologging.label = nologging +toJobConfig.nologging.label = Nologging toJobConfig.nologging.help = nologging -toJobConfig.updateKey.label = updateKey +toJobConfig.updateKey.label = Update key columns toJobConfig.updateKey.help = updateKey -toJobConfig.updateMerge.label = updateMerge +toJobConfig.updateMerge.label = Merge updates toJobConfig.updateMerge.help = updateMerge -toJobConfig.dropTableIfExists.label = dropTableIfExists +toJobConfig.dropTableIfExists.label = Drop table if exists toJobConfig.dropTableIfExists.help = dropTableIfExists -toJobConfig.storageClause.label = storageClause +toJobConfig.storageClause.label = Template table storage clause toJobConfig.storageClause.help = storageClause -toJobConfig.temporaryStorageClause.label = temporaryStorageClause +toJobConfig.temporaryStorageClause.label = Temporary table storage clause toJobConfig.temporaryStorageClause.help = temporaryStorageClause -toJobConfig.appendValuesHint.label = appendValuesHint +toJobConfig.appendValuesHint.label = Append values hint usage toJobConfig.appendValuesHint.help = appendValuesHint -toJobConfig.parallel.label = parallel +toJobConfig.parallel.label = Parallel toJobConfig.parallel.help = parallel # FromJob Config # -fromJobConfig.label = From database configuration +fromJobConfig.label = From Oracle configuration fromJobConfig.help = You must supply the information requested in order to create \ the FROM part of the job object. -fromJobConfig.tableName.label = tableName +fromJobConfig.tableName.label = Table name fromJobConfig.tableName.help = tableName -fromJobConfig.consistentRead.label = consistentRead +fromJobConfig.columns.label = Columns +fromJobConfig.columns.help = Columns + +fromJobConfig.consistentRead.label = Consistent read fromJobConfig.consistentRead.help = consistentRead -fromJobConfig.consistentReadScn.label = consistentReadScn +fromJobConfig.consistentReadScn.label = Consistent read SCN fromJobConfig.consistentReadScn.help = consistentReadScn -fromJobConfig.partitionList.label = partitionList +fromJobConfig.partitionList.label = Partitions fromJobConfig.partitionList.help = partitionList -fromJobConfig.dataChunkMethod.label = dataChunkMethod +fromJobConfig.dataChunkMethod.label = Data chunk method fromJobConfig.dataChunkMethod.help = dataChunkMethod -fromJobConfig.dataChunkAllocationMethod.label = dataChunkAllocationMethod +fromJobConfig.dataChunkAllocationMethod.label = Data chunk allocation method fromJobConfig.dataChunkAllocationMethod.help = dataChunkAllocationMethod -fromJobConfig.whereClauseLocation.label = whereClauseLocation +fromJobConfig.whereClauseLocation.label = Where clause location fromJobConfig.whereClauseLocation.help = whereClauseLocation -fromJobConfig.omitLobColumns.label = omitLobColumns +fromJobConfig.omitLobColumns.label = Omit LOB columns fromJobConfig.omitLobColumns.help = omitLobColumns -fromJobConfig.queryHint.label = queryHint +fromJobConfig.queryHint.label = Query hint fromJobConfig.queryHint.help = queryHint -fromJobConfig.conditions.label = conditions +fromJobConfig.conditions.label = Conditions fromJobConfig.conditions.help = conditions \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/30fe4da2/connector/connector-oracle-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/oracle/integration/OracleConnectionFactoryTest.java ---------------------------------------------------------------------- diff --git a/connector/connector-oracle-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/oracle/integration/OracleConnectionFactoryTest.java b/connector/connector-oracle-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/oracle/integration/OracleConnectionFactoryTest.java index a4caa85..f2898bf 100644 --- a/connector/connector-oracle-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/oracle/integration/OracleConnectionFactoryTest.java +++ b/connector/connector-oracle-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/oracle/integration/OracleConnectionFactoryTest.java @@ -24,6 +24,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -223,15 +224,19 @@ public class OracleConnectionFactoryTest extends OracleTestCase { // Check that the absence of session-initialization statements is reflected // in the log... + List<String> statements = new ArrayList<String>(); + statements.add(" "); stringWriter.getBuffer().setLength(0); - checkExecuteOraOopSessionInitializationStatements(";"); + checkExecuteOraOopSessionInitializationStatements(statements); checkLogContainsText(stringWriter.toString(), "No Oracle 'session initialization' statements were found to execute"); // This should throw an exception, as Oracle won't know what to do with // this... + statements.clear(); + statements.add("loremipsum"); stringWriter.getBuffer().setLength(0); - checkExecuteOraOopSessionInitializationStatements("loremipsum"); + checkExecuteOraOopSessionInitializationStatements(statements); checkLogContainsText(stringWriter.toString(), "loremipsum"); checkLogContainsText(stringWriter.toString(), "ORA-00900: invalid SQL statement"); @@ -240,22 +245,24 @@ public class OracleConnectionFactoryTest extends OracleTestCase { try { // Try a session-initialization statement that creates a table... + statements.clear(); + statements.add("create table " + TEST_TABLE_NAME + " (col1 varchar2(1))"); dropTable(conn, TEST_TABLE_NAME); - checkExecuteOraOopSessionInitializationStatements("create table " - + TEST_TABLE_NAME + " (col1 varchar2(1))"); + checkExecuteOraOopSessionInitializationStatements(statements); if (!doesTableExist(conn, TEST_TABLE_NAME)) { Assert.fail("The session-initialization statement to create the table " + TEST_TABLE_NAME + " did not work."); } // Try a sequence of a few statements... + statements.clear(); + statements.add("create table " + TEST_TABLE_NAME + " (col1 number)"); + statements.add("insert into " + TEST_TABLE_NAME + " values (1) "); + statements.add("--update " + TEST_TABLE_NAME + " set col1 = col1 + 1"); + statements.add("update " + TEST_TABLE_NAME + " set col1 = col1 + 1"); + statements.add("commit"); dropTable(conn, TEST_TABLE_NAME); - checkExecuteOraOopSessionInitializationStatements("create table " - + TEST_TABLE_NAME + " (col1 number);insert into " - + TEST_TABLE_NAME + " values (1) ; --update " - + TEST_TABLE_NAME + " set col1 = col1 + 1; update " - + TEST_TABLE_NAME - + " set col1 = col1 + 1; commit ;;"); + checkExecuteOraOopSessionInitializationStatements(statements); ResultSet resultSet = conn.createStatement().executeQuery( @@ -275,105 +282,6 @@ public class OracleConnectionFactoryTest extends OracleTestCase { log.removeAppender(writerAppender); } - @Test - public void testParseOraOopSessionInitializationStatements() { - - List<String> statements = null; - - statements = - OracleConnectionFactory - .parseOraOopSessionInitializationStatements(null); - Assert.assertEquals(0, statements.size()); - - statements = - OracleConnectionFactory - .parseOraOopSessionInitializationStatements(""); - Assert.assertEquals(0, statements.size()); - - statements = - OracleConnectionFactory - .parseOraOopSessionInitializationStatements(";"); - Assert.assertEquals(0, statements.size()); - - statements = - OracleConnectionFactory - .parseOraOopSessionInitializationStatements(";--;\t--"); - Assert.assertEquals(0, statements.size()); - - statements = - OracleConnectionFactory - .parseOraOopSessionInitializationStatements("\ta"); - Assert.assertEquals(1, statements.size()); - if (!statements.get(0).equalsIgnoreCase("a")) { - Assert.fail("Expected a session initialization statement of \"a\""); - } - - statements = - OracleConnectionFactory - .parseOraOopSessionInitializationStatements("a;b;--c;d;"); - Assert.assertEquals(3, statements.size()); - if (!statements.get(0).equalsIgnoreCase("a")) { - Assert.fail("Expected a session initialization statement of \"a\""); - } - if (!statements.get(1).equalsIgnoreCase("b")) { - Assert.fail("Expected a session initialization statement of \"b\""); - } - if (!statements.get(2).equalsIgnoreCase("d")) { - Assert.fail("Expected a session initialization statement of \"d\""); - } - - // Expressions without default values... - /*conf.set(OraOopConstants.ORAOOP_SESSION_INITIALIZATION_STATEMENTS, - "set a={expr1};b={expr2}/{expr3};"); - conf.set("expr1", "1"); - conf.set("expr2", "2"); - conf.set("expr3", "3"); - statements = - OracleConnectionFactory - .parseOraOopSessionInitializationStatements(conf); - Assert.assertEquals(2, statements.size()); - String actual = statements.get(0); - String expected = "set a=1"; - if (!actual.equalsIgnoreCase(expected)) { - Assert.fail(String.format( - "Expected a session initialization statement of \"%s\", but got \"%s\"." - , expected, actual)); - } - actual = statements.get(1); - expected = "b=2/3"; - if (!actual.equalsIgnoreCase(expected)) { - Assert.fail(String.format( - "Expected a session initialization statement of \"%s\", but got \"%s\"." - , expected, actual)); - } - - // Expressions with default values... - conf.set(OraOopConstants.ORAOOP_SESSION_INITIALIZATION_STATEMENTS, - "set c={expr3|66};d={expr4|15}/{expr5|90};"); - conf.set("expr3", "20"); - // conf.set("expr4", "21"); - // conf.set("expr5", "23"); - statements = - OracleConnectionFactory - .parseOraOopSessionInitializationStatements(conf); - Assert.assertEquals(2, statements.size()); - actual = statements.get(0); - expected = "set c=20"; - if (!actual.equalsIgnoreCase(expected)) { - Assert.fail(String.format( - "Expected a session initialization statement of \"%s\", but got \"%s\"." - , expected, actual)); - } - actual = statements.get(1); - expected = "d=15/90"; - if (!actual.equalsIgnoreCase(expected)) { - Assert.fail(String.format( - "Expected a session initialization statement of \"%s\", but got \"%s\"." - , expected, actual)); - }*/ - - } - private void dropTable(Connection conn, String tableName) { try { @@ -417,7 +325,7 @@ public class OracleConnectionFactoryTest extends OracleTestCase { } private void checkExecuteOraOopSessionInitializationStatements( - String statements) { + List<String> statements) { Connection conn = getConnection();
