Repository: sqoop Updated Branches: refs/heads/branch-1.4.7 559abda92 -> 232897141
SQOOP-3226: ImportTest, ExportTest and TimestampDataTest fail because of column escaping problems (Szabolcs Vasas via Boglarka Egyed) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5b551e14 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5b551e14 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5b551e14 Branch: refs/heads/branch-1.4.7 Commit: 5b551e14a44fd3ab229e33577bce2d9be1882d39 Parents: 559abda Author: Boglarka Egyed <[email protected]> Authored: Tue Aug 29 11:56:24 2017 +0200 Committer: Attila Szabo <[email protected]> Committed: Wed Dec 6 00:19:48 2017 +0100 ---------------------------------------------------------------------- .../apache/sqoop/manager/oracle/ExportTest.java | 8 +++-- .../apache/sqoop/manager/oracle/ImportTest.java | 31 ++++++++++---------- .../sqoop/manager/oracle/OraOopTestCase.java | 22 ++++++++++++-- .../sqoop/manager/oracle/TimestampDataTest.java | 4 ++- 4 files changed, 42 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b551e14/src/test/org/apache/sqoop/manager/oracle/ExportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/ExportTest.java b/src/test/org/apache/sqoop/manager/oracle/ExportTest.java index 23b4c73..a601687 100644 --- a/src/test/org/apache/sqoop/manager/oracle/ExportTest.java +++ b/src/test/org/apache/sqoop/manager/oracle/ExportTest.java @@ -31,6 +31,8 @@ public class ExportTest extends OraOopTestCase { private static final ExportTest TEST_CASE = new ExportTest(); + private static final boolean DISABLE_ORACLE_ESCAPING_FLAG = false; + @BeforeClass public static void setUpHdfsData() throws Exception { // Copy the TST_PRODUCT table into HDFS which can be used for the export @@ -40,14 +42,14 @@ public class ExportTest extends OraOopTestCase { TEST_CASE.createTable("table_tst_product.xml"); int retCode = - TEST_CASE.runImport("TST_PRODUCT", TEST_CASE.getSqoopConf(), false); + TEST_CASE.runImport("TST_PRODUCT", TEST_CASE.getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } @Test public void testProductExport() throws Exception { int retCode = - TEST_CASE.runExportFromTemplateTable("TST_PRODUCT", "TST_PRODUCT_EXP", false); + TEST_CASE.runExportFromTemplateTable("TST_PRODUCT", "TST_PRODUCT_EXP", false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } @@ -55,7 +57,7 @@ public class ExportTest extends OraOopTestCase { public void testProductExportMixedCaseTableName() throws Exception { int retCode = TEST_CASE.runExportFromTemplateTable("TST_PRODUCT", - "\"\"T5+_Pr#duct_Exp\"\"", false); + "\"\"T5+_Pr#duct_Exp\"\"", false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b551e14/src/test/org/apache/sqoop/manager/oracle/ImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/ImportTest.java b/src/test/org/apache/sqoop/manager/oracle/ImportTest.java index 0002128..5db9fe3 100644 --- a/src/test/org/apache/sqoop/manager/oracle/ImportTest.java +++ b/src/test/org/apache/sqoop/manager/oracle/ImportTest.java @@ -19,7 +19,6 @@ package org.apache.sqoop.manager.oracle; import org.apache.hadoop.conf.Configuration; -import org.apache.sqoop.SqoopOptions; import org.apache.sqoop.manager.oracle.OraOopConstants. OraOopOracleDataChunkMethod; import org.junit.Test; @@ -31,13 +30,15 @@ import static org.junit.Assert.assertEquals; */ public class ImportTest extends OraOopTestCase { + private static final boolean DISABLE_ORACLE_ESCAPING_FLAG = false; + @Test public void testProductImport() throws Exception { setSqoopTargetDirectory(getSqoopTargetDirectory() + "tst_product"); createTable("table_tst_product.xml"); try { - int retCode = runImport("TST_PRODUCT", getSqoopConf(), false); + int retCode = runImport("TST_PRODUCT", getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { @@ -49,16 +50,14 @@ public class ImportTest extends OraOopTestCase { @Test public void testProductWithWhiteSpaceImport() throws Exception { - System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "false"); setSqoopTargetDirectory(getSqoopTargetDirectory() + "tst_product"); createTable("table_tst_product_with_white_space.xml"); try { - int retCode = runImport("TST_Pr OdUCT", getSqoopConf(), false); + int retCode = runImport("TST_Pr OdUCT", getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { - System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED); cleanupFolders(); closeTestEnvConnection(); } @@ -70,7 +69,7 @@ public class ImportTest extends OraOopTestCase { createTable("table_tst_product_part.xml"); try { - int retCode = runImport("TST_PRODUCT_PART", getSqoopConf(), false); + int retCode = runImport("TST_PRODUCT_PART", getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { @@ -89,7 +88,7 @@ public class ImportTest extends OraOopTestCase { OraOopConstants.OraOopOracleDataChunkMethod.PARTITION.toString()); try { - int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { @@ -110,7 +109,7 @@ public class ImportTest extends OraOopTestCase { "tst_product_part_1,tst_product_part_2,\"tst_product_pa#rt_6\""); try { - int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { @@ -134,7 +133,7 @@ public class ImportTest extends OraOopTestCase { +"tst_product_part_3,\"tst_product_pa#rt_6\""); try { - int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { @@ -149,7 +148,7 @@ public class ImportTest extends OraOopTestCase { createTable("table_tst_product_subpart.xml"); try { - int retCode = runImport("TST_PRODUCT_SUBPART", getSqoopConf(), false); + int retCode = runImport("TST_PRODUCT_SUBPART", getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { @@ -168,7 +167,7 @@ public class ImportTest extends OraOopTestCase { OraOopConstants.OraOopOracleDataChunkMethod.PARTITION.toString()); try { - int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { @@ -191,7 +190,7 @@ public class ImportTest extends OraOopTestCase { +"TST_PRODUCT_PART_3,TST_PRODUCT_PART_4"); try { - int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { @@ -212,7 +211,7 @@ public class ImportTest extends OraOopTestCase { "TST_PRODUCT_PART_1,TST_PRODUCT_PART_2,TST_PRODUCT_PART_3"); try { - int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { @@ -233,7 +232,7 @@ public class ImportTest extends OraOopTestCase { sqoopConf.setBoolean(OraOopConstants.ORAOOP_IMPORT_CONSISTENT_READ, true); try { - int retCode = runImport("TST_PRODUCT", sqoopConf, false); + int retCode = runImport("TST_PRODUCT", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { @@ -248,7 +247,7 @@ public class ImportTest extends OraOopTestCase { createTable("table_tst_product_special_chars.xml"); try { - int retCode = runImport("\"\"T5+_Pr#duct\"\"", getSqoopConf(), false); + int retCode = runImport("\"\"T5+_Pr#duct\"\"", getSqoopConf(), false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { @@ -267,7 +266,7 @@ public class ImportTest extends OraOopTestCase { OraOopConstants.OraOopOracleDataChunkMethod.PARTITION.toString()); try { - int retCode = runImport("TST_PRODUCT_PART_IOT", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_PART_IOT", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally { http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b551e14/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java b/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java index 631e4f9..56dcac6 100644 --- a/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java +++ b/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java @@ -55,7 +55,7 @@ public abstract class OraOopTestCase { private static final OraOopLog LOG = OraOopLogFactory.getLog( OraOopTestCase.class.getName()); - + private static final boolean ESCAPING_DISABLED_DEFAULT = true; private String sqoopGenLibDirectory = System.getProperty("user.dir") + "/target/tmp/lib"; private String sqoopGenSrcDirectory = System.getProperty("user.dir") @@ -206,6 +206,11 @@ public abstract class OraOopTestCase { protected int runImport(String tableName, Configuration sqoopConf, boolean sequenceFile) { + return runImport(tableName, sqoopConf, sequenceFile, ESCAPING_DISABLED_DEFAULT); + } + + protected int runImport(String tableName, Configuration sqoopConf, + boolean sequenceFile, boolean escapingDisabled) { Logger rootLogger = Logger.getRootLogger(); StringWriter stringWriter = new StringWriter(); Layout layout = new PatternLayout("%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"); @@ -246,6 +251,9 @@ public abstract class OraOopTestCase { sqoopArgs.add("--outdir"); sqoopArgs.add(this.sqoopGenSrcDirectory); + sqoopArgs.add("--oracle-escaping-disabled"); + sqoopArgs.add(Boolean.toString(escapingDisabled)); + if (OracleUtils.NUM_MAPPERS != 0) { sqoopArgs.add("--num-mappers"); sqoopArgs.add(Integer.toString(OracleUtils.NUM_MAPPERS)); @@ -276,16 +284,21 @@ public abstract class OraOopTestCase { protected int runExportFromTemplateTable(String templateTableName, String tableName, boolean isPartitoned) { + return runExportFromTemplateTable(templateTableName, tableName, isPartitoned, ESCAPING_DISABLED_DEFAULT); + } + + protected int runExportFromTemplateTable(String templateTableName, + String tableName, boolean isPartitoned, boolean escapingDisabled) { Map<String, String> stringConfigEntries = new HashMap<String, String>(); stringConfigEntries.put("oraoop.template.table", templateTableName); Map<String, Boolean> booleanConfigEntries = new HashMap<String, Boolean>(); booleanConfigEntries.put("oraoop.partitioned", isPartitoned); - return runExport(tableName, new ArrayList<String>(), stringConfigEntries, booleanConfigEntries); + return runExport(tableName, new ArrayList<String>(), stringConfigEntries, booleanConfigEntries, escapingDisabled); } protected int runExport(String tableName, List<String> additionalArgs, Map<String, String> stringConfigEntries, - Map<String, Boolean> booleanConfigEntries) { + Map<String, Boolean> booleanConfigEntries, boolean escapingDisabled) { List<String> sqoopArgs = new ArrayList<String>(); sqoopArgs.add("export"); @@ -316,6 +329,9 @@ public abstract class OraOopTestCase { sqoopArgs.add("--outdir"); sqoopArgs.add(this.sqoopGenSrcDirectory); + sqoopArgs.add("--oracle-escaping-disabled"); + sqoopArgs.add(Boolean.toString(escapingDisabled)); + sqoopArgs.addAll(additionalArgs); Configuration sqoopConf = getSqoopConf(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b551e14/src/test/org/apache/sqoop/manager/oracle/TimestampDataTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/TimestampDataTest.java b/src/test/org/apache/sqoop/manager/oracle/TimestampDataTest.java index 1babf6c..be086c5 100644 --- a/src/test/org/apache/sqoop/manager/oracle/TimestampDataTest.java +++ b/src/test/org/apache/sqoop/manager/oracle/TimestampDataTest.java @@ -30,6 +30,8 @@ import static org.junit.Assert.assertEquals; */ public class TimestampDataTest extends OraOopTestCase { + private static final boolean DISABLE_ORACLE_ESCAPING_FLAG = false; + @Test public void testProductImportTimezone() throws Exception { setSqoopTargetDirectory(getSqoopTargetDirectory() + "tst_product_timezone"); @@ -39,7 +41,7 @@ public class TimestampDataTest extends OraOopTestCase { sqoopConf.setBoolean(OraOopConstants.ORAOOP_MAP_TIMESTAMP_AS_STRING, false); try { - int retCode = runImport("TST_PRODUCT", sqoopConf, false); + int retCode = runImport("TST_PRODUCT", sqoopConf, false, DISABLE_ORACLE_ESCAPING_FLAG); assertEquals("Return code should be 0", 0, retCode); } finally {
