Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2433#discussion_r201266791
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
---
@@ -2291,6 +2295,132 @@ class TestNonTransactionalCarbonTable extends
QueryTest with BeforeAndAfterAll {
|'$writerPath' """.stripMargin)
checkAnswer(sql("select * from sdkOutputTable"),
Seq(Row(Timestamp.valueOf("1970-01-02 16:00:00"),
Row(Timestamp.valueOf("1970-01-02 16:00:00")))))
}
+
+ test("test LocalDictionary with True") {
+ FileUtils.deleteDirectory(new File(writerPath))
+ val builder = CarbonWriter.builder.isTransactionalTable(false)
+
.sortBy(Array[String]("name")).withBlockSize(12).enableLocalDictionary(true)
+
.uniqueIdentifier(System.currentTimeMillis).taskNo(System.nanoTime).outputPath(writerPath)
+ generateCarbonData(builder)
+ assert(FileFactory.getCarbonFile(writerPath).exists())
+
assert(avroUtil.checkForLocalDictionary(avroUtil.getDimRawChunk(0,writerPath)))
+ sql("DROP TABLE IF EXISTS sdkTable")
+ sql(
+ s"""CREATE EXTERNAL TABLE sdkTable STORED BY 'carbondata' LOCATION
+ |'$writerPath' """.stripMargin)
+ val descLoc = sql("describe formatted sdkTable").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) =>
assert(row.get(1).toString.contains("name,surname"))
+ }
+ FileUtils.deleteDirectory(new File(writerPath))
+ }
+
+ test("test LocalDictionary with custom Threshold") {
+ FileUtils.deleteDirectory(new File(writerPath))
+ val builder = CarbonWriter.builder.isTransactionalTable(false)
+
.sortBy(Array[String]("name")).withBlockSize(12).enableLocalDictionary(true)
+ .localDictionaryThreshold(200)
+
.uniqueIdentifier(System.currentTimeMillis).taskNo(System.nanoTime).outputPath(writerPath)
+ generateCarbonData(builder)
+ assert(FileFactory.getCarbonFile(writerPath).exists())
+
assert(avroUtil.checkForLocalDictionary(avroUtil.getDimRawChunk(0,writerPath)))
+ sql("DROP TABLE IF EXISTS sdkTable")
+ sql(
+ s"""CREATE EXTERNAL TABLE sdkTable STORED BY 'carbondata' LOCATION
+ |'$writerPath' """.stripMargin)
+ val descLoc = sql("describe formatted sdkTable").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
--- End diff --
no need of this check, as this value is not inferred value from carbondata
file.
---
