[28/50] [abbrv] carbondata git commit: [CARBONDATA-2110]deprecate 'tempCSV' option of dataframe load

2018-02-03 Thread ravipesala
[CARBONDATA-2110]deprecate 'tempCSV' option of dataframe load

deprecate 'tempCSV' option of dataframe load, it won't generate temp file on 
hdfs, no matter the value of tempCSV

This closes #1916


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/da129d52
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/da129d52
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/da129d52

Branch: refs/heads/branch-1.3
Commit: da129d5277babe498fa5686fe53d01433d112bab
Parents: 6c097cb
Author: qiuchenjian <807169...@qq.com>
Authored: Sat Feb 3 00:14:07 2018 +0800
Committer: Jacky Li 
Committed: Sat Feb 3 15:29:08 2018 +0800

--
 .../testsuite/dataload/TestLoadDataFrame.scala  | 19 
 .../spark/sql/CarbonDataFrameWriter.scala   | 98 +---
 2 files changed, 20 insertions(+), 97 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/da129d52/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
index 6f03493..693c145 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
@@ -29,6 +29,7 @@ class TestLoadDataFrame extends QueryTest with 
BeforeAndAfterAll {
   var df: DataFrame = _
   var dataFrame: DataFrame = _
   var df2: DataFrame = _
+  var df3: DataFrame = _
   var booldf:DataFrame = _
 
 
@@ -52,6 +53,10 @@ class TestLoadDataFrame extends QueryTest with 
BeforeAndAfterAll {
   .map(x => ("key_" + x, "str_" + x, x, x * 2, x * 3))
   .toDF("c1", "c2", "c3", "c4", "c5")
 
+df3 = sqlContext.sparkContext.parallelize(1 to 3)
+  .map(x => (x.toString + "te,s\nt", x))
+  .toDF("c1", "c2")
+
 val boolrdd = sqlContext.sparkContext.parallelize(
   Row("anubhav",true) ::
 Row("prince",false) :: Nil)
@@ -74,6 +79,7 @@ class TestLoadDataFrame extends QueryTest with 
BeforeAndAfterAll {
 sql("DROP TABLE IF EXISTS carbon9")
 sql("DROP TABLE IF EXISTS carbon10")
 sql("DROP TABLE IF EXISTS carbon11")
+sql("DROP TABLE IF EXISTS carbon12")
 sql("DROP TABLE IF EXISTS df_write_sort_column_not_specified")
 sql("DROP TABLE IF EXISTS df_write_specify_sort_column")
 sql("DROP TABLE IF EXISTS df_write_empty_sort_column")
@@ -261,6 +267,19 @@ test("test the boolean data type"){
 val isStreaming: String = 
descResult.collect().find(row=>row(0).asInstanceOf[String].trim.equalsIgnoreCase("streaming")).get.get(1).asInstanceOf[String]
 assert(isStreaming.contains("true"))
   }
+
+  test("test datasource table with specified char") {
+
+df3.write
+  .format("carbondata")
+  .option("tableName", "carbon12")
+  .option("tempCSV", "true")
+  .mode(SaveMode.Overwrite)
+  .save()
+checkAnswer(
+  sql("select count(*) from carbon12"), Row(3)
+)
+  }
   private def getSortColumnValue(tableName: String): Array[String] = {
 val desc = sql(s"desc formatted $tableName")
 val sortColumnRow = desc.collect.find(r =>

http://git-wip-us.apache.org/repos/asf/carbondata/blob/da129d52/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index 2b06375..2be89b1 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -17,16 +17,12 @@
 
 package org.apache.spark.sql
 
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.util.CarbonException
 
 import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonType}
-import org.apache.carbondata.core.util.CarbonProperties
 import org.apache.carbondata.spark.CarbonOption
 
 class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: 

carbondata git commit: [CARBONDATA-2110]deprecate 'tempCSV' option of dataframe load

2018-02-02 Thread jackylk
Repository: carbondata
Updated Branches:
  refs/heads/master 6c097cbf3 -> da129d527


[CARBONDATA-2110]deprecate 'tempCSV' option of dataframe load

deprecate 'tempCSV' option of dataframe load, it won't generate temp file on 
hdfs, no matter the value of tempCSV

This closes #1916


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/da129d52
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/da129d52
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/da129d52

Branch: refs/heads/master
Commit: da129d5277babe498fa5686fe53d01433d112bab
Parents: 6c097cb
Author: qiuchenjian <807169...@qq.com>
Authored: Sat Feb 3 00:14:07 2018 +0800
Committer: Jacky Li 
Committed: Sat Feb 3 15:29:08 2018 +0800

--
 .../testsuite/dataload/TestLoadDataFrame.scala  | 19 
 .../spark/sql/CarbonDataFrameWriter.scala   | 98 +---
 2 files changed, 20 insertions(+), 97 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/da129d52/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
index 6f03493..693c145 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
@@ -29,6 +29,7 @@ class TestLoadDataFrame extends QueryTest with 
BeforeAndAfterAll {
   var df: DataFrame = _
   var dataFrame: DataFrame = _
   var df2: DataFrame = _
+  var df3: DataFrame = _
   var booldf:DataFrame = _
 
 
@@ -52,6 +53,10 @@ class TestLoadDataFrame extends QueryTest with 
BeforeAndAfterAll {
   .map(x => ("key_" + x, "str_" + x, x, x * 2, x * 3))
   .toDF("c1", "c2", "c3", "c4", "c5")
 
+df3 = sqlContext.sparkContext.parallelize(1 to 3)
+  .map(x => (x.toString + "te,s\nt", x))
+  .toDF("c1", "c2")
+
 val boolrdd = sqlContext.sparkContext.parallelize(
   Row("anubhav",true) ::
 Row("prince",false) :: Nil)
@@ -74,6 +79,7 @@ class TestLoadDataFrame extends QueryTest with 
BeforeAndAfterAll {
 sql("DROP TABLE IF EXISTS carbon9")
 sql("DROP TABLE IF EXISTS carbon10")
 sql("DROP TABLE IF EXISTS carbon11")
+sql("DROP TABLE IF EXISTS carbon12")
 sql("DROP TABLE IF EXISTS df_write_sort_column_not_specified")
 sql("DROP TABLE IF EXISTS df_write_specify_sort_column")
 sql("DROP TABLE IF EXISTS df_write_empty_sort_column")
@@ -261,6 +267,19 @@ test("test the boolean data type"){
 val isStreaming: String = 
descResult.collect().find(row=>row(0).asInstanceOf[String].trim.equalsIgnoreCase("streaming")).get.get(1).asInstanceOf[String]
 assert(isStreaming.contains("true"))
   }
+
+  test("test datasource table with specified char") {
+
+df3.write
+  .format("carbondata")
+  .option("tableName", "carbon12")
+  .option("tempCSV", "true")
+  .mode(SaveMode.Overwrite)
+  .save()
+checkAnswer(
+  sql("select count(*) from carbon12"), Row(3)
+)
+  }
   private def getSortColumnValue(tableName: String): Array[String] = {
 val desc = sql(s"desc formatted $tableName")
 val sortColumnRow = desc.collect.find(r =>

http://git-wip-us.apache.org/repos/asf/carbondata/blob/da129d52/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index 2b06375..2be89b1 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -17,16 +17,12 @@
 
 package org.apache.spark.sql
 
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.util.CarbonException
 
 import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonType}
-import org.apache.carbondata.core.util.CarbonProperties
 import