Repository: incubator-carbondata Updated Branches: refs/heads/12-dev 974cb1e58 -> 8843aecfe
Added option to specify single pass load in data frame Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4c93fe8d Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4c93fe8d Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4c93fe8d Branch: refs/heads/12-dev Commit: 4c93fe8d5f7b0e0762aa8946aad3f10a3ef2fb87 Parents: 974cb1e Author: Sanoj MG <sanoj.george....@gmail.com> Authored: Wed Apr 12 22:32:51 2017 +0400 Committer: Sanoj MG <sanoj.george....@gmail.com> Committed: Fri Apr 14 04:34:42 2017 +0400 ---------------------------------------------------------------------- .../testsuite/dataload/TestLoadDataFrame.scala | 32 ++++++++++++++++++++ .../spark/CarbonDataFrameWriter.scala | 3 +- 2 files changed, 34 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4c93fe8d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala index 6e79a10..b790131 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala @@ -59,6 +59,8 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll { sql("DROP TABLE IF EXISTS carbon5") sql("DROP TABLE IF EXISTS carbon6") sql("DROP TABLE IF EXISTS carbon7") + sql("DROP TABLE IF EXISTS carbon8") + sql("DROP TABLE IF EXISTS carbon9") } @@ -160,6 +162,36 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll { ) } + test("test load dataframe with single pass enabled") { + // save dataframe to carbon file + df.write + .format("carbondata") + .option("tableName", "carbon8") + .option("tempCSV", "false") + .option("single_pass", "true") + .option("compress", "false") + .mode(SaveMode.Overwrite) + .save() + checkAnswer( + sql("select count(*) from carbon8 where c3 > 500"), Row(500) + ) + } + + test("test load dataframe with single pass disabled") { + // save dataframe to carbon file + df.write + .format("carbondata") + .option("tableName", "carbon9") + .option("tempCSV", "true") + .option("single_pass", "false") + .option("compress", "false") + .mode(SaveMode.Overwrite) + .save() + checkAnswer( + sql("select count(*) from carbon9 where c3 > 500"), Row(500) + ) + } + override def afterAll { dropTable } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4c93fe8d/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala index 9f813a8..9089be1 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala @@ -194,7 +194,8 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) { s""" LOAD DATA INPATH '$csvFolder' INTO TABLE ${options.dbName}.${options.tableName} - OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}') + OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}', + 'SINGLE_PASS' = '${options.singlePass}') """ }
