Repository: sqoop Updated Branches: refs/heads/trunk c3ac21f71 -> 3e5bab49e
SQOOP-3066: Introduce an option + env variable to enable/disable SQOOP-2737 feature (Attila Szabo) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3e5bab49 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3e5bab49 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3e5bab49 Branch: refs/heads/trunk Commit: 3e5bab49ea3b48626abe3b6316820159fe4bed41 Parents: c3ac21f Author: Attila Szabo <[email protected]> Authored: Mon Dec 5 20:25:50 2016 +0100 Committer: Attila Szabo <[email protected]> Committed: Mon Dec 5 20:25:50 2016 +0100 ---------------------------------------------------------------------- src/java/org/apache/sqoop/SqoopOptions.java | 23 +++ .../org/apache/sqoop/manager/OracleManager.java | 24 +-- .../sqoop/manager/oracle/OraOopConnManager.java | 30 +-- .../manager/oracle/OraOopDBRecordReader.java | 5 +- .../manager/oracle/OraOopOracleQueries.java | 36 ++-- .../manager/oracle/OraOopOutputFormatBase.java | 5 +- .../oracle/OraOopOutputFormatInsert.java | 14 +- .../oracle/OraOopOutputFormatUpdate.java | 21 ++- .../sqoop/manager/oracle/OracleUtils.java | 12 +- .../mapreduce/db/OracleDBRecordReader.java | 4 +- .../org/apache/sqoop/tool/BaseSqoopTool.java | 12 ++ .../com/cloudera/sqoop/TestSqoopOptions.java | 181 ++++++++++++++++++- .../sqoop/manager/oracle/util/OracleData.java | 3 +- 13 files changed, 305 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/java/org/apache/sqoop/SqoopOptions.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index 2350c7a..53cc03a 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -59,6 +59,8 @@ import static org.apache.sqoop.Sqoop.SQOOP_RETHROW_PROPERTY; */ public class SqoopOptions implements Cloneable { + public static final String ORACLE_ESCAPING_DISABLED = "sqoop.oracle.escaping.disabled"; + private static final String OLD_SQOOP_TEST_IMPORT_ROOT_DIR = "sqoop.test.import.rootDir"; public static final Log LOG = LogFactory.getLog(SqoopOptions.class.getName()); @@ -381,6 +383,9 @@ public class SqoopOptions implements Cloneable { private Class validationThresholdClass; // ValidationThreshold implementation private Class validationFailureHandlerClass; // FailureHandler implementation + @StoredAsProperty(ORACLE_ESCAPING_DISABLED) + private boolean oracleEscapingDisabled; + public SqoopOptions() { initDefaults(null); } @@ -701,6 +706,10 @@ public class SqoopOptions implements Cloneable { if (this.verbose) { LoggingUtils.setDebugLevel(); } + + // Ensuring that oracleEscapingDisabled property is propagated to + // the level of Hadoop configuration as well + this.setOracleEscapingDisabled(this.isOracleEscapingDisabled()); } private void loadPasswordProperty(Properties props) { @@ -1039,6 +1048,8 @@ public class SqoopOptions implements Cloneable { //to support backward compatibility. Do not exchange it. this.throwOnError = isSqoopRethrowSystemPropertySet(); + setOracleEscapingDisabled(Boolean.parseBoolean(System.getProperty(ORACLE_ESCAPING_DISABLED, "true"))); + this.isValidationEnabled = false; // validation is disabled by default this.validatorClass = RowCountValidator.class; this.validationThresholdClass = AbsoluteValidationThreshold.class; @@ -2708,4 +2719,16 @@ public class SqoopOptions implements Cloneable { public void setMetadataTransactionIsolationLevel(int transactionIsolationLevel) { this.metadataTransactionIsolationLevel = transactionIsolationLevel; } + + public boolean isOracleEscapingDisabled() { + return oracleEscapingDisabled; + } + + public void setOracleEscapingDisabled(boolean escapingDisabled) { + this.oracleEscapingDisabled = escapingDisabled; + // important to have custom setter to ensure option is available through + // Hadoop configuration on those places where SqoopOptions is not reachable + getConf().setBoolean(ORACLE_ESCAPING_DISABLED, escapingDisabled); + } + } http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/java/org/apache/sqoop/manager/OracleManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/OracleManager.java b/src/java/org/apache/sqoop/manager/OracleManager.java index d769972..3273435 100644 --- a/src/java/org/apache/sqoop/manager/OracleManager.java +++ b/src/java/org/apache/sqoop/manager/OracleManager.java @@ -917,20 +917,20 @@ public class OracleManager } } - @Override - public String escapeColName(String colName) { - return OracleUtils.escapeIdentifier(colName); - } + @Override + public String escapeColName(String colName) { + return OracleUtils.escapeIdentifier(colName, options.isOracleEscapingDisabled()); + } - @Override - public String escapeTableName(String tableName) { - return OracleUtils.escapeIdentifier(tableName); - } + @Override + public String escapeTableName(String tableName) { + return OracleUtils.escapeIdentifier(tableName, options.isOracleEscapingDisabled()); + } - @Override - public boolean escapeTableNameOnExport() { - return true; - } + @Override + public boolean escapeTableNameOnExport() { + return true; + } @Override public String[] getColumnNames(String tableName) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java b/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java index 216c771..2026c43 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java +++ b/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java @@ -122,13 +122,17 @@ public class OraOopConnManager extends GenericJdbcManager { Configuration conf = this.options.getConf(); this.columnNamesInOracleTable = - OraOopOracleQueries.getTableColumnNames(getConnection(), - tableContext, OraOopUtilities - .omitLobAndLongColumnsDuringImport(conf), OraOopUtilities - .recallSqoopJobType(conf), true, // <- - // onlyOraOopSupportedTypes - true // <- omitOraOopPseudoColumns - ); + OraOopOracleQueries.getTableColumnNames( + getConnection(), + tableContext, + OraOopUtilities + .omitLobAndLongColumnsDuringImport(conf), + OraOopUtilities + .recallSqoopJobType(conf), + true, // <- onlyOraOopSupportedTypes + true, // <- omitOraOopPseudoColumns + options.isOracleEscapingDisabled() + ); } catch (SQLException ex) { throw new RuntimeException(ex); } @@ -529,14 +533,14 @@ public class OraOopConnManager extends GenericJdbcManager { @Override public String escapeColName(String colName) { - return OracleUtils.escapeIdentifier(colName); // <- See notes at top about escaped - // column names + return OracleUtils.escapeIdentifier(colName, options.isOracleEscapingDisabled()); // <- See notes at top about escaped + // column names } - @Override - public String escapeTableName(String tableName) { - return OracleUtils.escapeIdentifier(tableName); - } + @Override + public String escapeTableName(String tableName) { + return OracleUtils.escapeIdentifier(tableName, options.isOracleEscapingDisabled()); + } @Override public boolean escapeTableNameOnExport() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/java/org/apache/sqoop/manager/oracle/OraOopDBRecordReader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopDBRecordReader.java b/src/java/org/apache/sqoop/manager/oracle/OraOopDBRecordReader.java index f3ee4aa..f7d1889 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OraOopDBRecordReader.java +++ b/src/java/org/apache/sqoop/manager/oracle/OraOopDBRecordReader.java @@ -31,6 +31,8 @@ import com.cloudera.sqoop.mapreduce.db.DBConfiguration; import com.cloudera.sqoop.mapreduce.db.DBInputFormat; import com.cloudera.sqoop.mapreduce.db.DBInputFormat.DBInputSplit; import com.cloudera.sqoop.mapreduce.db.DataDrivenDBRecordReader; + +import org.apache.sqoop.SqoopOptions; import org.apache.sqoop.manager.oracle.OraOopConstants. OraOopTableImportWhereClauseLocation; import org.apache.sqoop.manager.oracle.OraOopUtilities.OraOopStatsReports; @@ -198,7 +200,8 @@ class OraOopDBRecordReader<T extends SqoopRecord> extends OraOopUtilities.recallSqoopJobType(conf) , true // <- onlyOraOopSupportedTypes , true // <- omitOraOopPseudoColumns - ); + , OracleUtils.isOracleEscapingDisabled(conf) + ); } catch (SQLException ex) { LOG.error(String.format( "Unable to obtain the data-types of the columns in table %s.\n" http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/java/org/apache/sqoop/manager/oracle/OraOopOracleQueries.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopOracleQueries.java b/src/java/org/apache/sqoop/manager/oracle/OraOopOracleQueries.java index 2603f83..b26bc71 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OraOopOracleQueries.java +++ b/src/java/org/apache/sqoop/manager/oracle/OraOopOracleQueries.java @@ -604,9 +604,9 @@ public final class OraOopOracleQueries { } public static List<String> getTableColumnNames(Connection connection, - OracleTable table) throws SQLException { + OracleTable table, boolean escapingDisabled) throws SQLException { - OracleTableColumns oracleTableColumns = getTableColumns(connection, table); + OracleTableColumns oracleTableColumns = getTableColumns(connection, table, escapingDisabled); List<String> result = new ArrayList<String>(oracleTableColumns.size()); for (int idx = 0; idx < oracleTableColumns.size(); idx++) { @@ -619,11 +619,11 @@ public final class OraOopOracleQueries { public static List<String> getTableColumnNames(Connection connection, OracleTable table, boolean omitLobAndLongColumnsDuringImport, OraOopConstants.Sqoop.Tool sqoopTool, boolean onlyOraOopSupportedTypes, - boolean omitOraOopPseudoColumns) throws SQLException { + boolean omitOraOopPseudoColumns, boolean escapingDisabled) throws SQLException { OracleTableColumns oracleTableColumns = getTableColumns(connection, table, omitLobAndLongColumnsDuringImport, - sqoopTool, onlyOraOopSupportedTypes, omitOraOopPseudoColumns); + sqoopTool, onlyOraOopSupportedTypes, omitOraOopPseudoColumns, escapingDisabled); List<String> result = new ArrayList<String>(oracleTableColumns.size()); @@ -636,7 +636,7 @@ public final class OraOopOracleQueries { private static OracleTableColumns getTableColumns(Connection connection, OracleTable table, boolean omitLobColumns, String dataTypesClause, - HashSet<String> columnNamesToOmit) throws SQLException { + HashSet<String> columnNamesToOmit, boolean escapingDisabled) throws SQLException { String sql = "SELECT column_name, data_type " + " FROM dba_tab_columns" @@ -680,7 +680,7 @@ public final class OraOopOracleQueries { if (idx > 0) { columnList.append(","); } - columnList.append(OracleUtils.escapeIdentifier(result.get(idx).getName())); + columnList.append(OracleUtils.escapeIdentifier(result.get(idx).getName(), escapingDisabled)); } sql = String.format("SELECT %s FROM %s WHERE 0=1", columnList.toString(), @@ -700,16 +700,16 @@ public final class OraOopOracleQueries { } public static OracleTableColumns getTableColumns(Connection connection, - OracleTable table) throws SQLException { + OracleTable table, boolean escapingDisabled) throws SQLException { return getTableColumns(connection, table, false, null // <- dataTypesClause - , null); // <-columnNamesToOmit + , null, escapingDisabled); // <-columnNamesToOmit } public static OracleTableColumns getTableColumns(Connection connection, OracleTable table, boolean omitLobAndLongColumnsDuringImport, OraOopConstants.Sqoop.Tool sqoopTool, boolean onlyOraOopSupportedTypes, - boolean omitOraOopPseudoColumns) throws SQLException { + boolean omitOraOopPseudoColumns, boolean escapingDisabled) throws SQLException { String dataTypesClause = ""; HashSet<String> columnNamesToOmit = null; @@ -765,7 +765,7 @@ public final class OraOopOracleQueries { } return getTableColumns(connection, table, - omitLobAndLongColumnsDuringImport, dataTypesClause, columnNamesToOmit); + omitLobAndLongColumnsDuringImport, dataTypesClause, columnNamesToOmit, escapingDisabled); } public static List<OracleActiveInstance> getOracleActiveInstances( @@ -1232,7 +1232,7 @@ public final class OraOopOracleQueries { public static void updateTable(Connection connection, OracleTable targetTable, OracleTable sourceTable, String[] mergeColumnNames, OracleTableColumns oracleTableColumns, - Object oraOopSysDate, int oraOopMapperId, boolean parallelizationEnabled) + Object oraOopSysDate, int oraOopMapperId, boolean parallelizationEnabled, boolean escapingDisabled) throws SQLException { StringBuilder targetColumnsClause = new StringBuilder(); @@ -1244,12 +1244,13 @@ public final class OraOopOracleQueries { if (targetColumnsClause.length() > 0) { targetColumnsClause.append(","); } - targetColumnsClause.append(String.format("a.%s", OracleUtils.escapeIdentifier(columnName))); + targetColumnsClause + .append(String.format("a.%s", OracleUtils.escapeIdentifier(columnName, escapingDisabled))); if (sourceColumnsClause.length() > 0) { sourceColumnsClause.append(","); } - sourceColumnsClause.append(String.format("b.%s", OracleUtils.escapeIdentifier(columnName))); + sourceColumnsClause.append(String.format("b.%s", OracleUtils.escapeIdentifier(columnName, escapingDisabled))); } String sourceClause = sourceColumnsClause.toString(); @@ -1298,7 +1299,7 @@ public final class OraOopOracleQueries { OracleTable tableToCreate, String tableToCreateStorageClause, OracleTable tableContainingUpdates, OracleTable tableToBeUpdated, String[] joinColumnNames, CreateExportChangesTableOptions options, - boolean parallelizationEnabled) throws SQLException { + boolean parallelizationEnabled, boolean escapingDisabled) throws SQLException { List<String> columnNames = getTableColumnNames(connection, tableToBeUpdated @@ -1306,6 +1307,7 @@ public final class OraOopOracleQueries { , OraOopConstants.Sqoop.Tool.EXPORT , true // <- onlyOraOopSupportedTypes , false // <- omitOraOopPseudoColumns + , escapingDisabled ); StringBuilder columnClause = new StringBuilder(2 * columnNames.size()); @@ -1313,7 +1315,7 @@ public final class OraOopOracleQueries { if (idx > 0) { columnClause.append(","); } - columnClause.append(String.format("a.%s", OracleUtils.escapeIdentifier(columnNames.get(idx)))); + columnClause.append(String.format("a.%s", OracleUtils.escapeIdentifier(columnNames.get(idx), escapingDisabled))); } StringBuilder rowEqualityClause = new StringBuilder(); @@ -1432,10 +1434,10 @@ public final class OraOopOracleQueries { public static void insertRowsIntoExportTable(Connection connection, OracleTable tableToInsertRowsInto, OracleTable tableContainingRowsToInsert, Object oraOopSysDate, - int oraOopMapperId, boolean parallelizationEnabled) throws SQLException { + int oraOopMapperId, boolean parallelizationEnabled, boolean escapingDisabled) throws SQLException { List<String> columnNames = - getTableColumnNames(connection, tableToInsertRowsInto); + getTableColumnNames(connection, tableToInsertRowsInto, escapingDisabled); StringBuilder columnClause = new StringBuilder(2 + (2 * columnNames.size())); http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatBase.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatBase.java b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatBase.java index 8f94cf8..c6b7098 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatBase.java +++ b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatBase.java @@ -31,6 +31,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.sqoop.SqoopOptions; import com.cloudera.sqoop.lib.SqoopRecord; import com.cloudera.sqoop.mapreduce.AsyncSqlOutputFormat; @@ -279,6 +280,7 @@ abstract class OraOopOutputFormatBase<K extends SqoopRecord, V> extends .omitLobAndLongColumnsDuringImport(conf), OraOopUtilities .recallSqoopJobType(conf), true // <- onlyOraOopSupportedTypes , false // <- omitOraOopPseudoColumns + , OracleUtils.isOracleEscapingDisabled(conf) )); } @@ -354,7 +356,8 @@ abstract class OraOopOutputFormatBase<K extends SqoopRecord, V> extends if (colCount > 0) { sqlNames.append("\n,"); } - sqlNames.append(OracleUtils.escapeIdentifier(columnName)); + boolean escapingDisabled = OracleUtils.isOracleEscapingDisabled(getConf()); + sqlNames.append(OracleUtils.escapeIdentifier(columnName, escapingDisabled)); // column values... if (colCount > 0) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatInsert.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatInsert.java b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatInsert.java index d430ef7..1874b9f 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatInsert.java +++ b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatInsert.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.sqoop.SqoopOptions; import com.cloudera.sqoop.lib.SqoopRecord; @@ -148,11 +149,14 @@ public class OraOopOutputFormatInsert<K extends SqoopRecord, V> extends // a subpartition of the 'real' export table... this.oracleTable = createUniqueMapperTable(context); - setOracleTableColumns(OraOopOracleQueries.getTableColumns(this - .getConnection(), this.oracleTable, OraOopUtilities - .omitLobAndLongColumnsDuringImport(conf), OraOopUtilities - .recallSqoopJobType(conf), true // <- onlyOraOopSupportedTypes - , false) // <- omitOraOopPseudoColumns + setOracleTableColumns(OraOopOracleQueries.getTableColumns( + this.getConnection(), + this.oracleTable, + OraOopUtilities.omitLobAndLongColumnsDuringImport(conf), + OraOopUtilities.recallSqoopJobType(conf), + true, // <- onlyOraOopSupportedTypes + false, // <- omitOraOopPseudoColumns + OracleUtils.isOracleEscapingDisabled(conf)) ); this.subPartitionName = http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatUpdate.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatUpdate.java b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatUpdate.java index a33768f..33bcb84 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatUpdate.java +++ b/src/java/org/apache/sqoop/manager/oracle/OraOopOutputFormatUpdate.java @@ -29,6 +29,8 @@ import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import com.cloudera.sqoop.lib.SqoopRecord; + +import org.apache.sqoop.SqoopOptions; import org.apache.sqoop.manager.oracle.OraOopOracleQueries. CreateExportChangesTableOptions; @@ -161,11 +163,14 @@ public class OraOopOutputFormatUpdate<K extends SqoopRecord, V> extends Configuration conf = context.getConfiguration(); this.oracleTable = createUniqueMapperTable(context); - setOracleTableColumns(OraOopOracleQueries.getTableColumns(this - .getConnection(), this.oracleTable, OraOopUtilities - .omitLobAndLongColumnsDuringImport(conf), OraOopUtilities - .recallSqoopJobType(conf), true // <- onlyOraOopSupportedTypes - , false) // <- omitOraOopPseudoColumns + setOracleTableColumns(OraOopOracleQueries.getTableColumns( + this.getConnection(), + this.oracleTable, + OraOopUtilities.omitLobAndLongColumnsDuringImport(conf), + OraOopUtilities.recallSqoopJobType(conf), + true, // <- onlyOraOopSupportedTypes + false, // <- omitOraOopPseudoColumns + OracleUtils.isOracleEscapingDisabled(conf)) ); } @@ -292,7 +297,7 @@ public class OraOopOutputFormatUpdate<K extends SqoopRecord, V> extends OraOopOracleQueries.createExportChangesTable(this.getConnection(), changesTable, temporaryTableStorageClause, this.oracleTable, targetTable, this.updateColumnNames, changesTableOptions, - parallelizationEnabled); + parallelizationEnabled, OracleUtils.isOracleEscapingDisabled(context.getConfiguration())); if (changeTableRowCount == 0) { LOG.debug(String.format( @@ -312,7 +317,7 @@ public class OraOopOutputFormatUpdate<K extends SqoopRecord, V> extends OraOopOracleQueries.insertRowsIntoExportTable(this.getConnection(), targetTable, changesTable, sysDateTime, this.mapperId, - parallelizationEnabled); + parallelizationEnabled, OracleUtils.isOracleEscapingDisabled(context.getConfiguration())); break; case UpdateSql: @@ -322,7 +327,7 @@ public class OraOopOutputFormatUpdate<K extends SqoopRecord, V> extends OraOopOracleQueries.updateTable(this.getConnection(), targetTable, changesTable, this.updateColumnNames, this .getOracleTableColumns(), sysDateTime, this.mapperId, - parallelizationEnabled); + parallelizationEnabled, context.getConfiguration().getBoolean(SqoopOptions.ORACLE_ESCAPING_DISABLED,false)); double timeInSec = (System.nanoTime() - start) / Math.pow(10, 9); LOG.info(String.format("Time spent performing an update: %f sec.", http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/java/org/apache/sqoop/manager/oracle/OracleUtils.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OracleUtils.java b/src/java/org/apache/sqoop/manager/oracle/OracleUtils.java index b46034b..aa56e70 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OracleUtils.java +++ b/src/java/org/apache/sqoop/manager/oracle/OracleUtils.java @@ -19,6 +19,8 @@ package org.apache.sqoop.manager.oracle; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.SqoopOptions; /** * Utility class for Oracle. @@ -28,12 +30,20 @@ public final class OracleUtils { private static final String PERIOD_REGEX = "\\."; private static final String PERIOD_DELIMITER = "."; + public static boolean isOracleEscapingDisabled(Configuration conf) { + return conf.getBoolean(SqoopOptions.ORACLE_ESCAPING_DISABLED, true); + } + public static boolean isEscaped(final String identifier) { return !StringUtils.isBlank(identifier) && identifier.startsWith("\"") && identifier.endsWith("\""); } public static String escapeIdentifier(final String identifier) { - if (StringUtils.isBlank(identifier)) { + return escapeIdentifier(identifier, false); + } + + public static String escapeIdentifier(final String identifier, boolean escapingDisabled) { + if (escapingDisabled || StringUtils.isBlank(identifier)) { return identifier; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/java/org/apache/sqoop/mapreduce/db/OracleDBRecordReader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/db/OracleDBRecordReader.java b/src/java/org/apache/sqoop/mapreduce/db/OracleDBRecordReader.java index f1647b8..4d7da39 100644 --- a/src/java/org/apache/sqoop/mapreduce/db/OracleDBRecordReader.java +++ b/src/java/org/apache/sqoop/mapreduce/db/OracleDBRecordReader.java @@ -22,6 +22,7 @@ import java.lang.reflect.Method; import java.sql.Connection; import java.sql.SQLException; +import org.apache.sqoop.SqoopOptions; import org.apache.sqoop.manager.oracle.OracleUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -66,8 +67,9 @@ extends DBRecordReader<T> { if (dbConf.getInputQuery() == null) { query.append("SELECT "); + boolean escapingDisabled = OracleUtils.isOracleEscapingDisabled(getConf()); for (int i = 0; i < fieldNames.length; i++) { - query.append(OracleUtils.escapeIdentifier(fieldNames[i])); + query.append(OracleUtils.escapeIdentifier(fieldNames[i], escapingDisabled)); if (i != fieldNames.length -1) { query.append(", "); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/java/org/apache/sqoop/tool/BaseSqoopTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 6082296..3d37859 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -171,6 +171,7 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { public static final String SKIP_DISTCACHE_ARG = "skip-dist-cache"; public static final String RELAXED_ISOLATION = "relaxed-isolation"; public static final String THROW_ON_ERROR_ARG = "throw-on-error"; + public static final String ORACLE_ESCAPING_DISABLED = "oracle-escaping-disabled"; // Arguments for validation. public static final String VALIDATE_ARG = "validate"; @@ -511,6 +512,13 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { .withLongOpt(RELAXED_ISOLATION) .create()); + commonOpts.addOption(OptionBuilder + .withDescription("Disable the escaping mechanism of the Oracle/OraOop connection managers") + .withLongOpt(ORACLE_ESCAPING_DISABLED) + .hasArg() + .withArgName("boolean") + .create()); + return commonOpts; } @@ -1072,6 +1080,10 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { + "java.sql.Connection javadocs for more details", e); } } + + if (in.hasOption(ORACLE_ESCAPING_DISABLED)) { + out.setOracleEscapingDisabled(Boolean.parseBoolean(in.getOptionValue(ORACLE_ESCAPING_DISABLED))); + } } private void applyCredentialsOptions(CommandLine in, SqoopOptions out) http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/test/com/cloudera/sqoop/TestSqoopOptions.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/TestSqoopOptions.java b/src/test/com/cloudera/sqoop/TestSqoopOptions.java index d5d09b6..d95f904 100644 --- a/src/test/com/cloudera/sqoop/TestSqoopOptions.java +++ b/src/test/com/cloudera/sqoop/TestSqoopOptions.java @@ -24,18 +24,26 @@ import com.cloudera.sqoop.tool.BaseSqoopTool; import junit.framework.TestCase; import org.apache.commons.lang.ArrayUtils; +import org.apache.sqoop.manager.oracle.OracleUtils; + import com.cloudera.sqoop.lib.DelimiterSet; import com.cloudera.sqoop.tool.ImportTool; import com.cloudera.sqoop.testutil.HsqldbTestServer; import org.junit.Before; import org.junit.After; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import static org.apache.sqoop.Sqoop.SQOOP_RETHROW_PROPERTY; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; /** * Test aspects of the SqoopOptions class. */ +@RunWith(JUnit4.class) public class TestSqoopOptions extends TestCase { private Properties originalSystemProperties; @@ -51,10 +59,12 @@ public class TestSqoopOptions extends TestCase { } // tests for the toChar() parser + @Test public void testNormalChar() throws Exception { assertEquals('a', SqoopOptions.toChar("a")); } + @Test public void testEmptyString() throws Exception { try { SqoopOptions.toChar(""); @@ -64,6 +74,7 @@ public class TestSqoopOptions extends TestCase { } } + @Test public void testNullString() throws Exception { try { SqoopOptions.toChar(null); @@ -73,45 +84,55 @@ public class TestSqoopOptions extends TestCase { } } + @Test public void testTooLong() throws Exception { // Should just use the first character and log a warning. assertEquals('x', SqoopOptions.toChar("xyz")); } + @Test public void testHexChar1() throws Exception { assertEquals(0xF, SqoopOptions.toChar("\\0xf")); } + @Test public void testHexChar2() throws Exception { assertEquals(0xF, SqoopOptions.toChar("\\0xF")); } + @Test public void testHexChar3() throws Exception { assertEquals(0xF0, SqoopOptions.toChar("\\0xf0")); } + @Test public void testHexChar4() throws Exception { assertEquals(0xF0, SqoopOptions.toChar("\\0Xf0")); } + @Test public void testEscapeChar1() throws Exception { assertEquals('\n', SqoopOptions.toChar("\\n")); } + @Test public void testEscapeChar2() throws Exception { assertEquals('\\', SqoopOptions.toChar("\\\\")); } + @Test public void testEscapeChar3() throws Exception { assertEquals('\\', SqoopOptions.toChar("\\")); } + @Test public void testWhitespaceToChar() throws Exception { assertEquals(' ', SqoopOptions.toChar(" ")); assertEquals(' ', SqoopOptions.toChar(" ")); assertEquals('\t', SqoopOptions.toChar("\t")); } + @Test public void testUnknownEscape1() throws Exception { try { SqoopOptions.toChar("\\Q"); @@ -121,6 +142,7 @@ public class TestSqoopOptions extends TestCase { } } + @Test public void testUnknownEscape2() throws Exception { try { SqoopOptions.toChar("\\nn"); @@ -130,30 +152,37 @@ public class TestSqoopOptions extends TestCase { } } + @Test public void testEscapeNul1() throws Exception { assertEquals(DelimiterSet.NULL_CHAR, SqoopOptions.toChar("\\0")); } + @Test public void testEscapeNul2() throws Exception { assertEquals(DelimiterSet.NULL_CHAR, SqoopOptions.toChar("\\00")); } + @Test public void testEscapeNul3() throws Exception { assertEquals(DelimiterSet.NULL_CHAR, SqoopOptions.toChar("\\0000")); } + @Test public void testEscapeNul4() throws Exception { assertEquals(DelimiterSet.NULL_CHAR, SqoopOptions.toChar("\\0x0")); } + @Test public void testOctalChar1() throws Exception { assertEquals(04, SqoopOptions.toChar("\\04")); } + @Test public void testOctalChar2() throws Exception { assertEquals(045, SqoopOptions.toChar("\\045")); } + @Test public void testErrOctalChar() throws Exception { try { SqoopOptions.toChar("\\095"); @@ -163,6 +192,7 @@ public class TestSqoopOptions extends TestCase { } } + @Test public void testErrHexChar() throws Exception { try { SqoopOptions.toChar("\\0x9K5"); @@ -178,6 +208,7 @@ public class TestSqoopOptions extends TestCase { } // test that setting output delimiters also sets input delimiters + @Test public void testDelimitersInherit() throws Exception { String [] args = { "--fields-terminated-by", @@ -191,6 +222,7 @@ public class TestSqoopOptions extends TestCase { // Test that setting output delimiters and setting input delims // separately works. + @Test public void testDelimOverride1() throws Exception { String [] args = { "--fields-terminated-by", @@ -205,6 +237,7 @@ public class TestSqoopOptions extends TestCase { } // test that the order in which delims are specified doesn't matter + @Test public void testDelimOverride2() throws Exception { String [] args = { "--input-fields-terminated-by", @@ -218,6 +251,7 @@ public class TestSqoopOptions extends TestCase { assertEquals('|', opts.getOutputFieldDelim()); } + @Test public void testBadNumMappers1() throws Exception { String [] args = { "--num-mappers", @@ -232,6 +266,7 @@ public class TestSqoopOptions extends TestCase { } } + @Test public void testBadNumMappers2() throws Exception { String [] args = { "-m", @@ -246,6 +281,7 @@ public class TestSqoopOptions extends TestCase { } } + @Test public void testGoodNumMappers() throws Exception { String [] args = { "-m", @@ -256,6 +292,7 @@ public class TestSqoopOptions extends TestCase { assertEquals(4, opts.getNumMappers()); } + @Test public void testHivePartitionParams() throws Exception { String[] args = { "--hive-partition-key", "ds", @@ -266,6 +303,7 @@ public class TestSqoopOptions extends TestCase { assertEquals("20110413", opts.getHivePartitionValue()); } + @Test public void testBoundaryQueryParams() throws Exception { String[] args = { "--boundary-query", "select 1, 2", @@ -275,6 +313,7 @@ public class TestSqoopOptions extends TestCase { assertEquals("select 1, 2", opts.getBoundaryQuery()); } + @Test public void testMapColumnHiveParams() throws Exception { String[] args = { "--map-column-hive", "id=STRING", @@ -286,6 +325,7 @@ public class TestSqoopOptions extends TestCase { assertEquals("STRING", mapping.get("id")); } + @Test public void testMalformedMapColumnHiveParams() throws Exception { String[] args = { "--map-column-hive", "id", @@ -298,6 +338,7 @@ public class TestSqoopOptions extends TestCase { } } + @Test public void testMapColumnJavaParams() throws Exception { String[] args = { "--map-column-java", "id=String", @@ -309,6 +350,7 @@ public class TestSqoopOptions extends TestCase { assertEquals("String", mapping.get("id")); } + @Test public void testMalfromedMapColumnJavaParams() throws Exception { String[] args = { "--map-column-java", "id", @@ -321,12 +363,14 @@ public class TestSqoopOptions extends TestCase { } } + @Test public void testSkipDistCacheOption() throws Exception { String[] args = {"--skip-dist-cache"}; SqoopOptions opts = parse(args); assertTrue(opts.isSkipDistCache()); } + @Test public void testPropertySerialization1() { // Test that if we write a SqoopOptions out to a Properties, // and then read it back in, we get all the same results. @@ -367,6 +411,7 @@ public class TestSqoopOptions extends TestCase { connParams, in.getConnectionParams()); } + @Test public void testPropertySerialization2() { // Test that if we write a SqoopOptions out to a Properties, // and then read it back in, we get all the same results. @@ -407,12 +452,14 @@ public class TestSqoopOptions extends TestCase { connParams, in.getConnectionParams()); } + @Test public void testDefaultTempRootDir() { SqoopOptions opts = new SqoopOptions(); assertEquals("_sqoop", opts.getTempRootDir()); } + @Test public void testDefaultLoadedTempRootDir() { SqoopOptions out = new SqoopOptions(); Properties props = out.writeProperties(); @@ -422,6 +469,7 @@ public class TestSqoopOptions extends TestCase { assertEquals("_sqoop", opts.getTempRootDir()); } + @Test public void testLoadedTempRootDir() { SqoopOptions out = new SqoopOptions(); final String tempRootDir = "customRoot"; @@ -433,6 +481,7 @@ public class TestSqoopOptions extends TestCase { assertEquals(tempRootDir, opts.getTempRootDir()); } + @Test public void testNulledTempRootDir() { SqoopOptions out = new SqoopOptions(); out.setTempRootDir(null); @@ -501,7 +550,122 @@ public class TestSqoopOptions extends TestCase { assertTrue(opts.isThrowOnError()); } + @Test + public void defaultValueOfOracleEscapingDisabledShouldBeFalse() { + System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED); + SqoopOptions opts = new SqoopOptions(); + + assertThat(opts.isOracleEscapingDisabled(), is(equalTo(true))); + } + + @Test + public void valueOfOracleEscapingDisabledShouldBeFalseIfTheValueOfTheRelatedEnvironmentVariableIsSetToFalse() { + System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "false"); + SqoopOptions opts = new SqoopOptions(); + + assertThat(opts.isOracleEscapingDisabled(), is(equalTo(false))); + } + + @Test + public void valueOfOracleEscapingDisabledShouldBeTrueIfTheValueOfTheRelatedEnvironmentVariableIsSetToTrue() { + System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "true"); + SqoopOptions opts = new SqoopOptions(); + + assertThat(opts.isOracleEscapingDisabled(), is(equalTo(true))); + } + + @Test + public void valueOfOracleEscapingDisabledShouldBeFalseIfTheValueOfTheRelatedEnvironmentVariableIsSetToAnyNonBooleanValue() { + System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "falsetrue"); + SqoopOptions opts = new SqoopOptions(); + + assertThat(opts.isOracleEscapingDisabled(), is(equalTo(false))); + } + + @Test + public void hadoopConfigurationInstanceOfSqoopOptionsShouldContainTheSameValueForOracleEscapingDisabledAsSqoopOptionsProperty() { + SqoopOptions opts = new SqoopOptions(); + + assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), + is(equalTo(opts.isOracleEscapingDisabled()))); + } + + @Test + public void hadoopConfigurationInstanceOfSqoopOptionsShouldContainTrueForOracleEscapingDisabledAsTheValueDirectlyHasBeenSetToSqoopOptions() { + System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED); + SqoopOptions opts = new SqoopOptions(); + opts.setOracleEscapingDisabled(true); + + assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), + is(equalTo(true))); + } + + @Test + public void hadoopConfigurationInstanceOfSqoopOptionsShouldContainFalseForOracleEscapingDisabledAsTheValueDirectlyHasBeenSetToSqoopOptions() { + System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED); + SqoopOptions opts = new SqoopOptions(); + opts.setOracleEscapingDisabled(false); + + assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), + is(equalTo(false))); + } + + @Test + public void valueOfOracleEscapingDisabledInHadoopConfigurationInstanceOfSqoopOptionsShouldBeFalseIfTheValueOfTheRelatedEnvironmentVariableIsSetToFalse() { + System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "false"); + SqoopOptions opts = new SqoopOptions(); + + assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), + is(equalTo(false))); + } + + @Test + public void valueOfOracleEscapingDisabledInHadoopConfigurationInstanceOfSqoopOptionsShouldBeTrueIfTheValueOfTheRelatedEnvironmentVariableIsSetToTrue() { + System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "true"); + SqoopOptions opts = new SqoopOptions(); + + assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), + is(equalTo(true))); + } + + @Test + public void valueOfOracleEscapingDisabledInHadoopConfigurationInstanceOfSqoopOptionsShouldBeFalseIfTheValueOfTheRelatedEnvironmentVariableIsSetToAnyNonBooleanValue() { + System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "falsetrue"); + SqoopOptions opts = new SqoopOptions(); + + assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), + is(equalTo(false))); + } + + @Test + public void valueOfOracleEscapingDisabledShouldBeAbleToSavedAndLoadedBackWithTheSameValue() { + System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED); + SqoopOptions opts = new SqoopOptions(); + opts.setOracleEscapingDisabled(false); + Properties out = opts.writeProperties(); + opts = new SqoopOptions(); + opts.loadProperties(out); + + assertThat(opts.isOracleEscapingDisabled(), is(equalTo(false))); + assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), + is(equalTo(false))); + } + + @Test + public void valueOfOracleEscapingDisabledShouldBeEqualToNullIfASqoopOptionsInstanceWasLoadedWhichDidntContainASavedValueForIt() { + System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED); + SqoopOptions opts = new SqoopOptions(); + Properties out = opts.writeProperties(); + opts = new SqoopOptions(); + opts.loadProperties(out); + + assertThat(opts.isOracleEscapingDisabled(), is(equalTo(true))); + assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), + is(equalTo(true))); + } + // test that hadoop-home is accepted as an option + @Test public void testHadoopHome() throws Exception { String [] args = { "--hadoop-home", @@ -513,12 +677,12 @@ public class TestSqoopOptions extends TestCase { } // test that hadoop-home is accepted as an option + @Test public void testHadoopMapRedOverridesHadoopHome() throws Exception { - String[] args = { "--hadoop-home", "/usr/lib/hadoop-ignored", - "--hadoop-mapred-home", "/usr/lib/hadoop", }; + String[] args = { "--hadoop-home", "/usr/lib/hadoop-ignored", "--hadoop-mapred-home", "/usr/lib/hadoop", }; - SqoopOptions opts = parse(args); - assertEquals("/usr/lib/hadoop", opts.getHadoopMapRedHome()); + SqoopOptions opts = parse(args); + assertEquals("/usr/lib/hadoop", opts.getHadoopMapRedHome()); } @@ -536,6 +700,7 @@ public class TestSqoopOptions extends TestCase { } //test compatability of --detele-target-dir with import + @Test public void testDeteleTargetDir() throws Exception { String [] extraArgs = { "--delete-target-dir", @@ -548,6 +713,7 @@ public class TestSqoopOptions extends TestCase { } //test incompatability of --delete-target-dir & --append with import + @Test public void testDeleteTargetDirWithAppend() throws Exception { String [] extraArgs = { "--append", @@ -562,6 +728,7 @@ public class TestSqoopOptions extends TestCase { } //test incompatability of --delete-target-dir with incremental import + @Test public void testDeleteWithIncrementalImport() throws Exception { String [] extraArgs = { "--incremental", "append", @@ -577,6 +744,7 @@ public class TestSqoopOptions extends TestCase { // test that hbase bulk load import with table name and target dir // passes validation + @Test public void testHBaseBulkLoad() throws Exception { String [] extraArgs = { longArgument(BaseSqoopTool.HBASE_BULK_LOAD_ENABLED_ARG), @@ -588,6 +756,7 @@ public class TestSqoopOptions extends TestCase { } // test that hbase bulk load import with a missing --hbase-table fails + @Test public void testHBaseBulkLoadMissingHbaseTable() throws Exception { String [] extraArgs = { longArgument(BaseSqoopTool.HBASE_BULK_LOAD_ENABLED_ARG), @@ -604,6 +773,7 @@ public class TestSqoopOptions extends TestCase { return String.format("--%s", argument); } + @Test public void testRelaxedIsolation() throws Exception { String extraArgs[] = { "--relaxed-isolation", @@ -611,6 +781,7 @@ public class TestSqoopOptions extends TestCase { validateImportOptions(extraArgs); } + @Test public void testResetToOneMapper() throws Exception { String extraArgs[] = { "--autoreset-to-one-mapper", @@ -618,6 +789,7 @@ public class TestSqoopOptions extends TestCase { validateImportOptions(extraArgs); } + @Test public void testResetToOneMapperAndSplitBy() throws Exception { String extraArgs[] = { "--autoreset-to-one-mapper", @@ -631,5 +803,4 @@ public class TestSqoopOptions extends TestCase { // Expected } } - } http://git-wip-us.apache.org/repos/asf/sqoop/blob/3e5bab49/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java b/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java index 355b1e2..79638eb 100644 --- a/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java +++ b/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java @@ -53,7 +53,8 @@ public final class OracleData { StringBuilder result = new StringBuilder(); String delim = ""; for (OracleDataDefinition column : tableDefinition.getColumnList()) { - result.append(delim).append(OracleUtils.escapeIdentifier(column.getColumnName())).append(" ").append( + result.append(delim).append(OracleUtils.escapeIdentifier(column.getColumnName())).append(" ") + .append( column.getDataType()); delim = ",\n"; }
