Repository: sqoop Updated Branches: refs/heads/trunk 729e681f1 -> dc2568431
SQOOP-2909: Oracle related ImportTest fails after SQOOP-2737 (Attila Szabo via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/dc256843 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/dc256843 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/dc256843 Branch: refs/heads/trunk Commit: dc256843181831e3916860f4fe530ea260d6f0c3 Parents: 729e681 Author: Jarek Jarcec Cecho <[email protected]> Authored: Mon Apr 18 08:50:37 2016 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Mon Apr 18 08:50:37 2016 -0700 ---------------------------------------------------------------------- src/test/oraoop/table_tst_product_part.xml | 2 +- src/test/oraoop/table_tst_product_part_iot.xml | 2 +- src/test/oraoop/table_tst_product_subpart.xml | 4 +- .../table_tst_product_with_white_space.xml | 90 ++++++++++++++++++++ .../apache/sqoop/manager/oracle/ImportTest.java | 38 ++++++--- .../sqoop/manager/oracle/OraOopTestCase.java | 2 +- .../sqoop/manager/oracle/util/OracleData.java | 7 +- 7 files changed, 126 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/dc256843/src/test/oraoop/table_tst_product_part.xml ---------------------------------------------------------------------- diff --git a/src/test/oraoop/table_tst_product_part.xml b/src/test/oraoop/table_tst_product_part.xml index 6014476..b5c5352 100644 --- a/src/test/oraoop/table_tst_product_part.xml +++ b/src/test/oraoop/table_tst_product_part.xml @@ -88,7 +88,7 @@ <uniqueKeyColumn>product_code</uniqueKeyColumn> </uniqueKeyColumns> <partitionClause> - PARTITION BY RANGE(sell_from_date) + PARTITION BY RANGE("sell_from_date") ( PARTITION tst_product_part_1 values less than (to_date(''' || to_char(add_months(sysdate,-9),'DD/MM/YYYY') || ''',''DD/MM/YYYY'')), PARTITION tst_product_part_2 values less than (to_date(''' || to_char(add_months(sysdate,-6),'DD/MM/YYYY') || ''',''DD/MM/YYYY'')), http://git-wip-us.apache.org/repos/asf/sqoop/blob/dc256843/src/test/oraoop/table_tst_product_part_iot.xml ---------------------------------------------------------------------- diff --git a/src/test/oraoop/table_tst_product_part_iot.xml b/src/test/oraoop/table_tst_product_part_iot.xml index b09015b..97e487c 100644 --- a/src/test/oraoop/table_tst_product_part_iot.xml +++ b/src/test/oraoop/table_tst_product_part_iot.xml @@ -89,7 +89,7 @@ </uniqueKeyColumns> <indexOrganizedTable>true</indexOrganizedTable> <partitionClause> - PARTITION BY HASH(product_id) + PARTITION BY HASH("product_id") PARTITIONS 4 </partitionClause> </table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/dc256843/src/test/oraoop/table_tst_product_subpart.xml ---------------------------------------------------------------------- diff --git a/src/test/oraoop/table_tst_product_subpart.xml b/src/test/oraoop/table_tst_product_subpart.xml index 4883244..137d99d 100644 --- a/src/test/oraoop/table_tst_product_subpart.xml +++ b/src/test/oraoop/table_tst_product_subpart.xml @@ -88,8 +88,8 @@ <uniqueKeyColumn>product_code</uniqueKeyColumn> </uniqueKeyColumns> <partitionClause> - PARTITION BY RANGE(sell_from_date) - SUBPARTITION BY HASH(supplier_code) + PARTITION BY RANGE("sell_from_date") + SUBPARTITION BY HASH("supplier_code") SUBPARTITIONS 2 ( PARTITION tst_product_part_1 values less than (to_date(''' || to_char(add_months(sysdate,-9),'DD/MM/YYYY') || ''',''DD/MM/YYYY'')), http://git-wip-us.apache.org/repos/asf/sqoop/blob/dc256843/src/test/oraoop/table_tst_product_with_white_space.xml ---------------------------------------------------------------------- diff --git a/src/test/oraoop/table_tst_product_with_white_space.xml b/src/test/oraoop/table_tst_product_with_white_space.xml new file mode 100644 index 0000000..65e97aa --- /dev/null +++ b/src/test/oraoop/table_tst_product_with_white_space.xml @@ -0,0 +1,90 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<table> + <name>TST_Pr OdUCT</name> + <columns> + <column> + <name>product_id</name> + <dataType>INTEGER</dataType> + <dataExpression>id</dataExpression> + </column> + <column> + <name>supplier_code</name> + <dataType>VARCHAR2 (30)</dataType> + <dataExpression>TO_CHAR (id - MOD (id, 5000),'FMXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')</dataExpression> + </column> + <column> + <name>product_code</name> + <dataType>VARCHAR2 (30)</dataType> + <dataExpression>TO_CHAR (MOD (id, 100000), 'FMXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')</dataExpression> + </column> + <column> + <name>product_descr</name> + <dataType>VARCHAR2 (255)</dataType> + <dataExpression>DBMS_RANDOM.string ('x', ROUND (DBMS_RANDOM.VALUE (1, 100)))</dataExpression> + </column> + <column> + <name>product_long_descr</name> + <dataType>VARCHAR2 (4000)</dataType> + <dataExpression>DBMS_RANDOM.string ('x', ROUND (DBMS_RANDOM.VALUE (1, 200)))</dataExpression> + </column> + <column> + <name>product_cost_price</name> + <dataType>NUMBER</dataType> + <dataExpression>ROUND (DBMS_RANDOM.VALUE (0, 100000), 2)</dataExpression> + </column> + <column> + <name>sell_from_date</name> + <dataType>DATE</dataType> + <dataExpression>TRUNC (SYSDATE + DBMS_RANDOM.VALUE (-365, 365))</dataExpression> + </column> + <column> + <name>sell_price</name> + <dataType>NUMBER</dataType> + <dataExpression>ROUND (DBMS_RANDOM.VALUE (0, 200000), 2)</dataExpression> + </column> + <column> + <name>create_user</name> + <dataType>VARCHAR2 (30)</dataType> + <dataExpression>DBMS_RANDOM.string ('U', 30)</dataExpression> + </column> + <column> + <name>crea te_time</name> + <dataType>TIMESTAMP</dataType> + <dataExpression>TO_TIMESTAMP (TO_CHAR (SYSDATE + DBMS_RANDOM.VALUE (-730, 0),'YYYYMMDDHH24MISS') || '.' || TRUNC (TO_CHAR (DBMS_RANDOM.VALUE * 999999999)), 'YYYYMMDDHH24MISSXFF')</dataExpression> + </column> + <column> + <name>last_update_user</name> + <dataType>VARCHAR2 (30)</dataType> + <dataExpression>DBMS_RANDOM.string ('U', 30)</dataExpression> + </column> + <column> + <name>last_update_time</name> + <dataType>TIMESTAMP</dataType> + <dataExpression>TO_TIMESTAMP (TO_CHAR (SYSDATE + DBMS_RANDOM.VALUE (-730, 0),'YYYYMMDDHH24MISS') || '.' || TRUNC (TO_CHAR (DBMS_RANDOM.VALUE * 999999999)), 'YYYYMMDDHH24MISSXFF')</dataExpression> + </column> + </columns> + <primaryKeyColumns> + <primaryKeyColumn>product_id</primaryKeyColumn> + </primaryKeyColumns> + <uniqueKeyColumns> + <uniqueKeyColumn>supplier_code</uniqueKeyColumn> + <uniqueKeyColumn>product_code</uniqueKeyColumn> + </uniqueKeyColumns> +</table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/dc256843/src/test/org/apache/sqoop/manager/oracle/ImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/ImportTest.java b/src/test/org/apache/sqoop/manager/oracle/ImportTest.java index b2134c6..cf03cc0 100644 --- a/src/test/org/apache/sqoop/manager/oracle/ImportTest.java +++ b/src/test/org/apache/sqoop/manager/oracle/ImportTest.java @@ -36,7 +36,23 @@ public class ImportTest extends OraOopTestCase { createTable("table_tst_product.xml"); try { - int retCode = runImport("tst_product", getSqoopConf(), false); + int retCode = runImport("TST_PRODUCT", getSqoopConf(), false); + Assert.assertEquals("Return code should be 0", 0, retCode); + + } finally { + cleanupFolders(); + closeTestEnvConnection(); + } + } + + + @Test + public void testProductWithWhiteSpaceImport() throws Exception { + setSqoopTargetDirectory(getSqoopTargetDirectory() + "tst_product"); + createTable("table_tst_product_with_white_space.xml"); + + try { + int retCode = runImport("TST_Pr OdUCT", getSqoopConf(), false); Assert.assertEquals("Return code should be 0", 0, retCode); } finally { @@ -51,7 +67,7 @@ public class ImportTest extends OraOopTestCase { createTable("table_tst_product_part.xml"); try { - int retCode = runImport("tst_product_part", getSqoopConf(), false); + int retCode = runImport("TST_PRODUCT_PART", getSqoopConf(), false); Assert.assertEquals("Return code should be 0", 0, retCode); } finally { @@ -70,7 +86,7 @@ public class ImportTest extends OraOopTestCase { OraOopConstants.OraOopOracleDataChunkMethod.PARTITION.toString()); try { - int retCode = runImport("tst_product_part", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false); Assert.assertEquals("Return code should be 0", 0, retCode); } finally { @@ -91,7 +107,7 @@ public class ImportTest extends OraOopTestCase { "tst_product_part_1,tst_product_part_2,\"tst_product_pa#rt_6\""); try { - int retCode = runImport("tst_product_part", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false); Assert.assertEquals("Return code should be 0", 0, retCode); } finally { @@ -115,7 +131,7 @@ public class ImportTest extends OraOopTestCase { +"tst_product_part_3,\"tst_product_pa#rt_6\""); try { - int retCode = runImport("tst_product_part", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_PART", sqoopConf, false); Assert.assertEquals("Return code should be 0", 0, retCode); } finally { @@ -130,7 +146,7 @@ public class ImportTest extends OraOopTestCase { createTable("table_tst_product_subpart.xml"); try { - int retCode = runImport("tst_product_subpart", getSqoopConf(), false); + int retCode = runImport("TST_PRODUCT_SUBPART", getSqoopConf(), false); Assert.assertEquals("Return code should be 0", 0, retCode); } finally { @@ -149,7 +165,7 @@ public class ImportTest extends OraOopTestCase { OraOopConstants.OraOopOracleDataChunkMethod.PARTITION.toString()); try { - int retCode = runImport("tst_product_subpart", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false); Assert.assertEquals("Return code should be 0", 0, retCode); } finally { @@ -172,7 +188,7 @@ public class ImportTest extends OraOopTestCase { +"TST_PRODUCT_PART_3,TST_PRODUCT_PART_4"); try { - int retCode = runImport("tst_product_subpart", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false); Assert.assertEquals("Return code should be 0", 0, retCode); } finally { @@ -193,7 +209,7 @@ public class ImportTest extends OraOopTestCase { "TST_PRODUCT_PART_1,TST_PRODUCT_PART_2,TST_PRODUCT_PART_3"); try { - int retCode = runImport("tst_product_subpart", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_SUBPART", sqoopConf, false); Assert.assertEquals("Return code should be 0", 0, retCode); } finally { @@ -214,7 +230,7 @@ public class ImportTest extends OraOopTestCase { sqoopConf.setBoolean(OraOopConstants.ORAOOP_IMPORT_CONSISTENT_READ, true); try { - int retCode = runImport("tst_product", sqoopConf, false); + int retCode = runImport("TST_PRODUCT", sqoopConf, false); Assert.assertEquals("Return code should be 0", 0, retCode); } finally { @@ -248,7 +264,7 @@ public class ImportTest extends OraOopTestCase { OraOopConstants.OraOopOracleDataChunkMethod.PARTITION.toString()); try { - int retCode = runImport("tst_product_part_iot", sqoopConf, false); + int retCode = runImport("TST_PRODUCT_PART_IOT", sqoopConf, false); Assert.assertEquals("Return code should be 0", 0, retCode); } finally { http://git-wip-us.apache.org/repos/asf/sqoop/blob/dc256843/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java b/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java index 6d0cf89..9fe4821 100644 --- a/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java +++ b/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java @@ -141,7 +141,7 @@ public abstract class OraOopTestCase { } protected int countTable(String inputTableName, List<String> partitionList) { - String tableName = inputTableName; + String tableName = org.apache.sqoop.manager.oracle.OracleUtils.escapeIdentifier(inputTableName); if (tableName.startsWith("\"\"") && tableName.endsWith("\"\"")) { // Table names may be double quoted to work around Sqoop issue tableName = tableName.substring(1, tableName.length() - 1); http://git-wip-us.apache.org/repos/asf/sqoop/blob/dc256843/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java b/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java index 3f16e04..8846f65 100644 --- a/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java +++ b/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java @@ -27,6 +27,7 @@ import java.util.List; import org.apache.commons.io.IOUtils; import org.apache.sqoop.manager.oracle.OraOopTestCase; +import org.apache.sqoop.manager.oracle.OracleUtils; /** * Class to load an Oracle table with data based on configuration file. @@ -51,7 +52,7 @@ public final class OracleData { StringBuilder result = new StringBuilder(); String delim = ""; for (OracleDataDefinition column : tableDefinition.getColumnList()) { - result.append(delim).append(column.getColumnName()).append(" ").append( + result.append(delim).append(OracleUtils.escapeIdentifier(column.getColumnName())).append(" ").append( column.getDataType()); delim = ",\n"; } @@ -62,7 +63,7 @@ public final class OracleData { getDataExpression(List<OracleDataDefinition> columnList) { StringBuilder result = new StringBuilder(); for (OracleDataDefinition column : columnList) { - result.append("l_ret_rec.").append(column.getColumnName()).append(" := ") + result.append("l_ret_rec.").append(OracleUtils.escapeIdentifier(column.getColumnName())).append(" := ") .append(column.getDataExpression()).append(";\n"); } return result.toString(); @@ -126,7 +127,7 @@ public final class OracleData { StringBuilder keyColumnList = new StringBuilder(); String delim = ""; for (String column : columns) { - keyColumnList.append(delim).append(column); + keyColumnList.append(delim).append(OracleUtils.escapeIdentifier(column)); delim = ","; } result = keyColumnList.toString();
