SQOOP-2281: Set overwrite on kite dataset (Abraham Elmahrek via Jarek Jarcec Cecho)
(cherry picked from commit 032bc54c86c79f2f032211ce6505ff9c9fbce31e) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/75c7b5ab Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/75c7b5ab Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/75c7b5ab Branch: refs/heads/branch-1.4.6 Commit: 75c7b5ab472649bf75bab1aac1db0155e691ff64 Parents: 9348d6a Author: Jarek Jarcec Cecho <[email protected]> Authored: Fri Apr 3 09:45:55 2015 -0700 Committer: Gwen Shapira <[email protected]> Committed: Mon Apr 6 15:37:25 2015 -0700 ---------------------------------------------------------------------- .../org/apache/sqoop/mapreduce/DataDrivenImportJob.java | 5 ++--- src/java/org/apache/sqoop/mapreduce/ParquetJob.java | 11 ++++++++--- src/test/com/cloudera/sqoop/hive/TestHiveImport.java | 3 +++ 3 files changed, 13 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/75c7b5ab/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java index bcf6611..e70d23c 100644 --- a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java +++ b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java @@ -103,9 +103,8 @@ public class DataDrivenImportJob extends ImportJobBase { // Parquet data records. The import will fail, if schema is invalid. Schema schema = generateAvroSchema(tableName); String uri = getKiteUri(conf, tableName); - boolean reuseExistingDataset = options.isAppendMode() || - (options.doHiveImport() && options.doOverwriteHiveTable()); - ParquetJob.configureImportJob(conf, schema, uri, reuseExistingDataset); + ParquetJob.configureImportJob(conf, schema, uri, options.isAppendMode(), + options.doHiveImport() && options.doOverwriteHiveTable()); } job.setMapperClass(getMapperClass()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/75c7b5ab/src/java/org/apache/sqoop/mapreduce/ParquetJob.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/ParquetJob.java b/src/java/org/apache/sqoop/mapreduce/ParquetJob.java index 85d88ff..df55dbc 100644 --- a/src/java/org/apache/sqoop/mapreduce/ParquetJob.java +++ b/src/java/org/apache/sqoop/mapreduce/ParquetJob.java @@ -71,9 +71,9 @@ public final class ParquetJob { * {@link org.apache.avro.generic.GenericRecord}. */ public static void configureImportJob(Configuration conf, Schema schema, - String uri, boolean reuseExistingDataset) throws IOException { + String uri, boolean reuseExistingDataset, boolean overwrite) throws IOException { Dataset dataset; - if (reuseExistingDataset) { + if (reuseExistingDataset || overwrite) { try { dataset = Datasets.load(uri); } catch (DatasetNotFoundException ex) { @@ -89,7 +89,12 @@ public final class ParquetJob { dataset = createDataset(schema, getCompressionType(conf), uri); } conf.set(CONF_AVRO_SCHEMA, schema.toString()); - DatasetKeyOutputFormat.configure(conf).writeTo(dataset); + + if (overwrite) { + DatasetKeyOutputFormat.configure(conf).overwrite(dataset); + } else { + DatasetKeyOutputFormat.configure(conf).writeTo(dataset); + } } private static Dataset createDataset(Schema schema, http://git-wip-us.apache.org/repos/asf/sqoop/blob/75c7b5ab/src/test/com/cloudera/sqoop/hive/TestHiveImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java index d6df196..fa717cb 100644 --- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java +++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java @@ -307,6 +307,9 @@ public class TestHiveImport extends ImportJobTestCase { runImportTest(TABLE_NAME, types, vals, "createOverwriteImport.q", getCreateHiveTableArgs(extraArgs), new CreateHiveTableTool()); + runImportTest(TABLE_NAME, types, vals, + "createOverwriteImport.q", getCreateHiveTableArgs(extraArgs), + new CreateHiveTableTool()); } /** Test that dates are coerced properly to strings. */
