Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2422#discussion_r199126782
--- Diff:
integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
---
@@ -2585,67 +2585,445 @@ class LocalDictionarySupportCreateTableTest
extends QueryTest with BeforeAndAfte
}
}
- test("test preaggregate table local dictionary enabled table")
+ test("test local dictionary foer varchar datatype columns") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_include'='city',
+ | 'LONG_STRING_COLUMNS'='city')
+ """.stripMargin)
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ }
+
+ test("test local dictionary describe formatted only with default
configs")
{
sql("drop table if exists local1")
- sql("CREATE TABLE local1 (id Int, date date, country string, phonetype
string, " +
- "serialname String,salary int ) STORED BY
'org.apache.carbondata.format' " +
-
"tblproperties('dictionary_include'='country','local_dictionary_enable'='true','local_dictionary_include'
= 'phonetype','local_dictionary_exclude' ='serialname')")
- sql("create datamap PreAggCount on table local1 using 'preaggregate'
as " +
- "select country,count(salary) as count from local1 group by
country")
- val descLoc = sql("describe formatted local1_PreAggCount").collect
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
case Some(row) => assert(row.get(1).toString.contains("10000"))
}
descLoc.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
- case Some(row) => assert(row.get(1).toString.contains("phonetype"))
+ case Some(row) => assert(row.get(1).toString.contains("name,city"))
}
- descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude"))
match {
- case Some(row) => assert(row.get(1).toString.contains("serialname"))
+ }
+
+ test("test local dictionary for invalid threshold")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_threshold'='300000')
+ """.stripMargin)
+
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
}
- descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ }
+
+ test("test alter set for local dictionary enable to disable")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_threshold'='300000')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
case Some(row) => assert(row.get(1).toString.contains("true"))
}
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary
Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_enable'='false')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ checkExistence(sql("DESC FORMATTED local1"), false,
+ "Local Dictionary Threshold")
}
- test("test local dictionary foer varchar datatype columns") {
+ test("test alter set for local dictionary disable to enable")
+ {
sql("drop table if exists local1")
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
tblproperties('local_dictionary_include'='city',
- | 'LONG_STRING_COLUMNS'='city')
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='300000')
""".stripMargin)
- val descLoc = sql("describe formatted local1").collect
- descLoc.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
- case Some(row) => assert(row.get(1).toString.contains("city"))
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
}
- descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold"))
match {
+ checkExistence(sql("DESC FORMATTED local1"), false,
+ "Local Dictionary Threshold")
+ sql("alter table local1 set
tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='30000')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary
Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("30000"))
+ }
+ }
+
+ test("test alter set for local dictionary threshold valid and invalid")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_threshold'='20000')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary
Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("20000"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_threshold'='30000')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary
Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("30000"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_threshold'='500000')")
+ val descLoc3 = sql("describe formatted local1").collect
+ descLoc3.find(_.get(0).toString.contains("Local Dictionary
Threshold")) match {
case Some(row) => assert(row.get(1).toString.contains("10000"))
}
}
- test("test local dictionary describe formatted only with default
configs")
+ test("test alter set for local dictionary include valid and invalid")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age
int,add string)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='city','dictionary_include'='add')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='name')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='id')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='city,city')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='add')")
+ }
+ }
+
+ test("test alter set for local dictionary exclude valid and invalid")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age
int,add string)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_exclude'='city','dictionary_include'='add')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Exclude"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='name')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name"))
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='id')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='city,city')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='add')")
+ }
+ }
+
+ test("test alter set same column for local dictionary exclude and
include")
{
sql("drop table if exists local1")
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata'
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_exclude'='city')
""".stripMargin)
+ intercept[Exception] {
+ sql(
+ "alter table local1 set
tblproperties('local_dictionary_include'='name'," +
+ "'local_dictionary_exclude'='name')")
+ }
+ }
+ test("test alter set for valid and invalid complex type as
include/exclude") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int,st
struct<s_id:int,
+ | s_name:string,s_city:array<string>>, dcity array<string>)
+ | STORED BY 'org.apache.carbondata.format'
+ |
tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city,dcity',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ val descFormatted1 = sql("describe formatted local1").collect
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary
Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descFormatted1.find(_.get(0).toString.contains("Local Dictionary
Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city,dcity"))
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='dcity')")
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='st')")
+ val descFormatted2 = sql("describe formatted local1").collect
+ descFormatted2.find(_.get(0).toString.contains("Local Dictionary
Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("st"))
+ }
+ descFormatted2.find(_.get(0).toString.contains("Local Dictionary
Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city,dcity"))
+ }
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='st'," +
+ "'local_dictionary_include'='dcity')")
+ val descFormatted3 = sql("describe formatted local1").collect
+ descFormatted3.find(_.get(0).toString.contains("Local Dictionary
Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("st"))
+ }
+ descFormatted3.find(_.get(0).toString.contains("Local Dictionary
Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("dcity"))
+ }
+ }
+
+ test("test alter set for invalid complex type as include/exclude") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int,st
struct<s_id:int,
+ | s_name:int,s_city:array<int>>, dcity array<int>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('local_dictionary_exclude'='name',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_exclude'='dcity')")
+ }
+ intercept[Exception] {
+ sql("alter table local1 set
tblproperties('local_dictionary_include'='st')")
+ }
+ }
+
+ test("test alter unset for local dictionary disable")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age
int,add string)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_enable'='false')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Enable"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("false"))
+ }
+ sql("alter table local1 unset
tblproperties('local_dictionary_enable')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Enable"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ }
+
+ test("test alter unset for local dictionary enable local dict include")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age
int,add string)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='city')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ sql("alter table local1 unset
tblproperties('local_dictionary_include')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("name,city"))
+ }
+ }
+
+ test("test alter unset for local dictionary enable local dict exclude")
+ {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age
int,add string)
+ | STORED BY 'carbondata'
tblproperties('local_dictionary_include'='city',
+ | 'local_dictionary_exclude'='name')
+ """.stripMargin)
+
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ sql("alter table local1 unset
tblproperties('local_dictionary_include')")
+ val descLoc2 = sql("describe formatted local1").collect
+ descLoc2.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test("test alter unset for valid/invalid complex type as
include/exclude") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string, city string, age int,st
struct<s_id:string,
+ | s_name:int,s_city:array<int>>, dcity array<string>)
+ | STORED BY 'org.apache.carbondata.format'
+ |
tblproperties('local_dictionary_exclude'='st','local_dictionary_include'='city',
+ | 'local_dictionary_enable'='true')
+ """.
+ stripMargin)
+ val descLoc1 = sql("describe formatted local1").collect
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Include"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ descLoc1.find(_.get(0).toString.contains("Local Dictionary Exclude"))
match {
+ case Some(row) => assert(row.get(1).toString.contains("st"))
--- End diff --
add check for child column also
---
