Repository: sqoop Updated Branches: refs/heads/trunk 73ef0c133 -> de6fa8279
SQOOP-2470: Incremental Hive import with append not working after validation check for --hive-import and --import (Aaditya Raj via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/de6fa827 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/de6fa827 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/de6fa827 Branch: refs/heads/trunk Commit: de6fa8279d8125afb4fd2d7f1e8c7e731d70451d Parents: 73ef0c1 Author: Jarek Jarcec Cecho <[email protected]> Authored: Thu Aug 13 17:10:42 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Thu Aug 13 17:10:42 2015 -0700 ---------------------------------------------------------------------- .../org/apache/sqoop/tool/BaseSqoopTool.java | 4 +- .../cloudera/sqoop/TestIncrementalImport.java | 63 +++++++++++++++++++- testdata/hive/scripts/incrementalHiveAppend10.q | 17 ++++++ testdata/hive/scripts/incrementalHiveAppend20.q | 17 ++++++ .../hive/scripts/incrementalHiveAppendEmpty.q | 17 ++++++ 5 files changed, 114 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/src/java/org/apache/sqoop/tool/BaseSqoopTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 4e2e66d..a7f6aec 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -40,6 +40,7 @@ import org.apache.sqoop.util.password.CredentialProviderHelper; import com.cloudera.sqoop.ConnFactory; import com.cloudera.sqoop.Sqoop; import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.SqoopOptions.IncrementalMode; import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException; import com.cloudera.sqoop.cli.RelatedOptions; import com.cloudera.sqoop.cli.ToolOptions; @@ -1398,7 +1399,8 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { } if (options.doHiveImport() - && options.isAppendMode()) { + && options.isAppendMode() + && !options.getIncrementalMode().equals(IncrementalMode.AppendRows)) { throw new InvalidOptionsException("Append mode for hive imports is not " + " yet supported. Please remove the parameter --append-mode"); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/src/test/com/cloudera/sqoop/TestIncrementalImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/TestIncrementalImport.java b/src/test/com/cloudera/sqoop/TestIncrementalImport.java index a6680e8..3e3f929 100644 --- a/src/test/com/cloudera/sqoop/TestIncrementalImport.java +++ b/src/test/com/cloudera/sqoop/TestIncrementalImport.java @@ -19,6 +19,7 @@ package com.cloudera.sqoop; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.sql.Connection; @@ -29,9 +30,7 @@ import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; import java.util.List; - import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -39,7 +38,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.StringUtils; - +import org.apache.sqoop.hive.HiveImport; import com.cloudera.sqoop.manager.ConnManager; import com.cloudera.sqoop.manager.HsqldbManager; import com.cloudera.sqoop.manager.ManagerFactory; @@ -1216,6 +1215,64 @@ public class TestIncrementalImport extends TestCase { runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); } + + public void testIncrementalHiveAppendEmptyThenFull() throws Exception { + // This is to test Incremental Hive append feature. SQOOP-2470 + final String TABLE_NAME = "incrementalHiveAppendEmptyThenFull"; + Configuration conf = newConf(); + conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, + InstrumentHsqldbManagerFactory.class.getName()); + clearDir(TABLE_NAME); + createIdTable(TABLE_NAME, 0); + List<String> args = new ArrayList<String>(); + args.add("--connect"); + args.add(SOURCE_DB_URL); + args.add("--table"); + args.add(TABLE_NAME); + args.add("--warehouse-dir"); + args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + args.add("--hive-import"); + args.add("--hive-table"); + args.add(TABLE_NAME + "hive"); + args.add("--incremental"); + args.add("append"); + args.add("--check-column"); + args.add("id"); + args.add("-m"); + args.add("1"); + createJob(TABLE_NAME, args, conf); + HiveImport.setTestMode(true); + String hiveHome = org.apache.sqoop.SqoopOptions.getHiveHomeDefault(); + assertNotNull("hive.home was not set", hiveHome); + String testDataPath = new Path(new Path(hiveHome), "scripts/" + + "incrementalHiveAppendEmpty.q").toString(); + System.clearProperty("expected.script"); + System.setProperty("expected.script", + new File(testDataPath).getAbsolutePath()); + runJob(TABLE_NAME); + assertDirOfNumbers(TABLE_NAME, 0); + // Now add some rows. + insertIdRows(TABLE_NAME, 0, 10); + String testDataPath10 = new Path(new Path(hiveHome), "scripts/" + + "incrementalHiveAppend10.q").toString(); + System.clearProperty("expected.script"); + System.setProperty("expected.script", + new File(testDataPath10).getAbsolutePath()); + System.getProperty("expected.script"); + // Running the job a second time should import 10 rows. + runJob(TABLE_NAME); + assertDirOfNumbers(TABLE_NAME, 10); + // Add some more rows. + insertIdRows(TABLE_NAME, 10, 20); + String testDataPath20 = new Path(new Path(hiveHome), "scripts/" + + "incrementalHiveAppend20.q").toString(); + System.clearProperty("expected.script"); + System.setProperty("expected.script", + new File(testDataPath20).getAbsolutePath()); + // Import only those rows. + runJob(TABLE_NAME); + assertDirOfNumbers(TABLE_NAME, 20); + } // SQOOP-1890 public void testTableNameWithSpecialCharacters() throws Exception { http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/testdata/hive/scripts/incrementalHiveAppend10.q ---------------------------------------------------------------------- diff --git a/testdata/hive/scripts/incrementalHiveAppend10.q b/testdata/hive/scripts/incrementalHiveAppend10.q new file mode 100644 index 0000000..383c7b4 --- /dev/null +++ b/testdata/hive/scripts/incrementalHiveAppend10.q @@ -0,0 +1,17 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/testdata/hive/scripts/incrementalHiveAppend20.q ---------------------------------------------------------------------- diff --git a/testdata/hive/scripts/incrementalHiveAppend20.q b/testdata/hive/scripts/incrementalHiveAppend20.q new file mode 100644 index 0000000..383c7b4 --- /dev/null +++ b/testdata/hive/scripts/incrementalHiveAppend20.q @@ -0,0 +1,17 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/testdata/hive/scripts/incrementalHiveAppendEmpty.q ---------------------------------------------------------------------- diff --git a/testdata/hive/scripts/incrementalHiveAppendEmpty.q b/testdata/hive/scripts/incrementalHiveAppendEmpty.q new file mode 100644 index 0000000..383c7b4 --- /dev/null +++ b/testdata/hive/scripts/incrementalHiveAppendEmpty.q @@ -0,0 +1,17 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`; \ No newline at end of file
