Github user anubhav100 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1774#discussion_r161367337
--- Diff:
integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
---
@@ -167,13 +167,20 @@ class CarbonDataFrameWriter(sqlContext: SQLContext,
val dataFrame: DataFrame) {
val carbonSchema = schema.map { field =>
s"${ field.name } ${ convertToCarbonType(field.dataType) }"
}
+ val isStreaming = if (options.isStreaming) Some("true") else None
+
val property = Map(
"SORT_COLUMNS" -> options.sortColumns,
"DICTIONARY_INCLUDE" -> options.dictionaryInclude,
"DICTIONARY_EXCLUDE" -> options.dictionaryExclude,
- "TABLE_BLOCKSIZE" -> options.tableBlockSize
- ).filter(_._2.isDefined).map(p => s"'${p._1}' =
'${p._2.get}'").mkString(",")
+ "TABLE_BLOCKSIZE" -> options.tableBlockSize,
+ "STREAMING" -> isStreaming
+ )
+ .filter(_._2.isDefined).
--- End diff --
using it directly will cause this test case to fail
test("test datasource table with specified table path") {
val path = "./source"
df2.write
.format("carbondata")
.option("tableName", "carbon10")
.option("tablePath", path)
.mode(SaveMode.Overwrite)
.save()
assert(new File(path).exists())
checkAnswer(
sql("select count(*) from carbon10 where c3 > 500"), Row(500)
)
sql("drop table carbon10")
assert(!new File(path).exists())
assert(intercept[AnalysisException](
sql("select count(*) from carbon10 where c3 > 500"))
.message
.contains("not found"))
}
this is because of some problem at parsing level giving tableproperties
followed by table path is not parsing correctlly a seperate jira is already
created for it here is the link
https://issues.apache.org/jira/browse/CARBONDATA-2005
are we going to support it?
thats why what i do is that if is streaming is false then do not include it
in property map make it none
---
