Repository: sqoop Updated Branches: refs/heads/trunk 64c5fc375 -> 032bc54c8
SQOOP-2281: Set overwrite on kite dataset (Abraham Elmahrek via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/032bc54c Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/032bc54c Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/032bc54c Branch: refs/heads/trunk Commit: 032bc54c86c79f2f032211ce6505ff9c9fbce31e Parents: 64c5fc3 Author: Jarek Jarcec Cecho <[email protected]> Authored: Fri Apr 3 09:45:55 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Fri Apr 3 09:45:55 2015 -0700 ---------------------------------------------------------------------- .../org/apache/sqoop/mapreduce/DataDrivenImportJob.java | 5 ++--- src/java/org/apache/sqoop/mapreduce/ParquetJob.java | 11 ++++++++--- src/test/com/cloudera/sqoop/hive/TestHiveImport.java | 3 +++ 3 files changed, 13 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/032bc54c/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java index bcf6611..e70d23c 100644 --- a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java +++ b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java @@ -103,9 +103,8 @@ public class DataDrivenImportJob extends ImportJobBase { // Parquet data records. The import will fail, if schema is invalid. Schema schema = generateAvroSchema(tableName); String uri = getKiteUri(conf, tableName); - boolean reuseExistingDataset = options.isAppendMode() || - (options.doHiveImport() && options.doOverwriteHiveTable()); - ParquetJob.configureImportJob(conf, schema, uri, reuseExistingDataset); + ParquetJob.configureImportJob(conf, schema, uri, options.isAppendMode(), + options.doHiveImport() && options.doOverwriteHiveTable()); } job.setMapperClass(getMapperClass()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/032bc54c/src/java/org/apache/sqoop/mapreduce/ParquetJob.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/ParquetJob.java b/src/java/org/apache/sqoop/mapreduce/ParquetJob.java index 85d88ff..df55dbc 100644 --- a/src/java/org/apache/sqoop/mapreduce/ParquetJob.java +++ b/src/java/org/apache/sqoop/mapreduce/ParquetJob.java @@ -71,9 +71,9 @@ public final class ParquetJob { * {@link org.apache.avro.generic.GenericRecord}. */ public static void configureImportJob(Configuration conf, Schema schema, - String uri, boolean reuseExistingDataset) throws IOException { + String uri, boolean reuseExistingDataset, boolean overwrite) throws IOException { Dataset dataset; - if (reuseExistingDataset) { + if (reuseExistingDataset || overwrite) { try { dataset = Datasets.load(uri); } catch (DatasetNotFoundException ex) { @@ -89,7 +89,12 @@ public final class ParquetJob { dataset = createDataset(schema, getCompressionType(conf), uri); } conf.set(CONF_AVRO_SCHEMA, schema.toString()); - DatasetKeyOutputFormat.configure(conf).writeTo(dataset); + + if (overwrite) { + DatasetKeyOutputFormat.configure(conf).overwrite(dataset); + } else { + DatasetKeyOutputFormat.configure(conf).writeTo(dataset); + } } private static Dataset createDataset(Schema schema, http://git-wip-us.apache.org/repos/asf/sqoop/blob/032bc54c/src/test/com/cloudera/sqoop/hive/TestHiveImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java index d6df196..fa717cb 100644 --- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java +++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java @@ -307,6 +307,9 @@ public class TestHiveImport extends ImportJobTestCase { runImportTest(TABLE_NAME, types, vals, "createOverwriteImport.q", getCreateHiveTableArgs(extraArgs), new CreateHiveTableTool()); + runImportTest(TABLE_NAME, types, vals, + "createOverwriteImport.q", getCreateHiveTableArgs(extraArgs), + new CreateHiveTableTool()); } /** Test that dates are coerced properly to strings. */
