Repository: carbondata Updated Branches: refs/heads/master 334e64778 -> b0d24eac5
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d24eac/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala index e139f8c..692b95b 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala @@ -26,7 +26,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -138,7 +138,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='10000') + | tblproperties('local_dictionary_threshold'='20000') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -146,7 +146,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) + case Some(row) => assert(row.get(1).toString.contains("20000")) } } @@ -162,7 +162,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -178,7 +178,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -194,7 +194,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -210,7 +210,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -222,7 +222,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='name') + | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -230,7 +230,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) + case Some(row) => assert(row.get(1).toString.contains("20000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { case Some(row) => assert(row.get(1).toString.contains("name")) @@ -253,7 +253,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { case Some(row) => assert(row.get(1).toString.contains("name")) @@ -276,7 +276,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { case Some(row) => assert(row.get(1).toString.contains("name")) @@ -299,7 +299,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { case Some(row) => assert(row.get(1).toString.contains("name")) @@ -314,7 +314,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='name, + | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name, | name') """.stripMargin) } @@ -329,7 +329,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'=' ') + | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'=' ') """.stripMargin) } } @@ -343,7 +343,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='hello') + | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='hello') """.stripMargin) } } @@ -357,7 +357,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='10000','local_dictionary_include'='name', + | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name', | 'dictionary_include'='name') """.stripMargin) } @@ -421,7 +421,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -812,7 +812,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -820,7 +820,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) + case Some(row) => assert(row.get(1).toString.contains("20000")) } } @@ -836,7 +836,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -852,7 +852,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -868,7 +868,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -884,7 +884,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -897,7 +897,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000', + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000', | 'local_dictionary_include'='name') """.stripMargin) @@ -906,7 +906,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) + case Some(row) => assert(row.get(1).toString.contains("20000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { case Some(row) => assert(row.get(1).toString.contains("name")) @@ -931,7 +931,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { case Some(row) => assert(row.get(1).toString.contains("name")) @@ -956,7 +956,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { case Some(row) => assert(row.get(1).toString.contains("name")) @@ -981,7 +981,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { case Some(row) => assert(row.get(1).toString.contains("name")) @@ -998,7 +998,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000', + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000', | 'local_dictionary_include'='name,name') """.stripMargin) } @@ -1014,7 +1014,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000', + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000', | 'local_dictionary_include'=' ') """.stripMargin) } @@ -1030,7 +1030,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000', + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000', | 'local_dictionary_include'='hello') """.stripMargin) } @@ -1046,7 +1046,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000', + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000', | 'local_dictionary_include'='name','dictionary_include'='name') """.stripMargin) } @@ -1214,7 +1214,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000') + | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='20000') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -1296,7 +1296,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000', + | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='20000', | 'local_dictionary_include'='name') """.stripMargin) @@ -1373,7 +1373,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000', + | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='20000', | 'local_dictionary_include'='name,name') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -1391,7 +1391,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000', + | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='20000', | 'local_dictionary_include'=' ') """.stripMargin) @@ -1410,7 +1410,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000', + | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='20000', | 'local_dictionary_include'='hello') """.stripMargin) @@ -1429,7 +1429,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='10000', + | tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='20000', | 'local_dictionary_include'='name','dictionary_include'='name') """.stripMargin) @@ -1598,7 +1598,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -2054,7 +2054,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte sql( """ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000') | as select * from local """.stripMargin) @@ -2063,7 +2063,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte case Some(row) => assert(row.get(1).toString.contains("true")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) + case Some(row) => assert(row.get(1).toString.contains("20000")) } } @@ -2087,7 +2087,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -2112,7 +2112,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } } @@ -2130,14 +2130,14 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte sql( """ | CREATE TABLE local1 STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='10000', + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000', | 'local_dictionary_include'='city') | as select * from local """.stripMargin) val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("20000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) @@ -2154,13 +2154,13 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', - | 'local_dictionary_threshold'='10000','local_dictionary_include'='city', + | 'local_dictionary_threshold'='20000','local_dictionary_include'='city', | 'no_inverted_index'='name') """.stripMargin) val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("20000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) @@ -2220,7 +2220,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """.stripMargin) val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) @@ -2243,7 +2243,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """.stripMargin) val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) @@ -2266,7 +2266,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """.stripMargin) val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) @@ -2289,7 +2289,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """.stripMargin) val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) @@ -2312,7 +2312,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """.stripMargin) val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("1000")) + case Some(row) => assert(row.get(1).toString.contains("10000")) } descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) @@ -2341,6 +2341,311 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte "local_dictionary_include are not of string dataType.")) } + test("test alter table add column") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name') + """.stripMargin) + sql("alter table local1 add columns (alt string) tblproperties('local_dictionary_include'='alt')") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("20000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city,alt")) + } + } + + test("test alter table add column default configs for local dictionary") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','no_inverted_index'='name') + """.stripMargin) + sql("alter table local1 add columns (alt string)") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("20000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name,city,alt")) + } + } + + test("test alter table add column where same column is in dictionary include and local dictionary include") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name') + """.stripMargin) + val exception = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string) tblproperties('local_dictionary_include'='alt','dictionary_include'='alt')") + } + assert(exception.getMessage + .contains( + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: alt specified in Dictionary " + + "include. Local Dictionary will not be generated for Dictionary include columns. Please " + + "check create table statement.")) + } + + test("test alter table add column where duplicate columns present in local dictionary include") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name') + """.stripMargin) + val exception = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string) tblproperties('local_dictionary_include'='alt,alt')") + } + assert(exception.getMessage + .contains( + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE contains Duplicate Columns: alt. " + + "Please check create table statement.")) + } + + test("test alter table add column where duplicate columns present in local dictionary include/exclude") + { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','local_dictionary_include'='city', + | 'no_inverted_index'='name') + """.stripMargin) + val exception1 = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string) tblproperties" + + "('local_dictionary_include'='abc')") + } + assert(exception1.getMessage + .contains( + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + + "Please check create table statement.")) + val exception2 = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string) tblproperties" + + "('local_dictionary_exclude'='abc')") + } + assert(exception2.getMessage + .contains( + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + + "Please check create table statement.")) + } + + test("test alter table add column for datatype validation") + { + sql("drop table if exists local1") + sql( + """ | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_include'='city', 'no_inverted_index'='name') + """.stripMargin) + val exception = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string,abc int) tblproperties" + + "('local_dictionary_include'='abc')") + } + assert(exception.getMessage + .contains( + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc is not a String/complex " + + "datatype column. LOCAL_DICTIONARY_COLUMN should be no dictionary string/complex datatype" + + " column.Please check create table statement.")) + } + + test("test alter table add column where duplicate columns are present in local dictionary include and exclude") + { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_include'='city', 'no_inverted_index'='name') + """.stripMargin) + val exception = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string,abc string) tblproperties" + + "('local_dictionary_include'='abc','local_dictionary_exclude'='alt,abc')") + } + assert(exception.getMessage + .contains( + "Column ambiguity as duplicate column(s):abc is present in LOCAL_DICTIONARY_INCLUDE " + + "and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed.")) + } + + test("test alter table add column unsupported table property") + { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_include'='city', 'no_inverted_index'='name') + """.stripMargin) + val exception = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string,abc string) tblproperties" + + "('local_dictionary_enable'='abc')") + } + assert(exception.getMessage + .contains( + "Unsupported Table property in add column: local_dictionary_enable")) + val exception1 = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string,abc string) tblproperties" + + "('local_dictionary_threshold'='10000')") + } + assert(exception1.getMessage + .contains( + "Unsupported Table property in add column: local_dictionary_threshold")) + } + + test("test alter table add column when main table is disabled for local dictionary") + { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false', + | 'local_dictionary_include'='city', 'no_inverted_index'='name') + """.stripMargin) + sql( + "alter table local1 add columns (alt string,abc string) tblproperties" + + "('local_dictionary_include'='abc')") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("false")) + } + + checkExistence(sql("DESC FORMATTED local1"), false, + "Local Dictionary Include") + } + + test("test local dictionary threshold for boundary values") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_threshold'='300000') + """.stripMargin) + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_threshold'='500') + """.stripMargin) + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + } + + test("test alter table add column for local dictionary include and exclude configs") + { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_include'='city', 'no_inverted_index'='name') + """.stripMargin) + sql( + "alter table local1 add columns (alt string,abc string) tblproperties" + + "('local_dictionary_include'='abc','local_dictionary_exclude'='alt')") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city,abc")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("alt")) + } + } + + test("test preaggregate table local dictionary enabled table") + { + sql("drop table if exists local1") + sql("CREATE TABLE local1 (id Int, date date, country string, phonetype string, " + + "serialname String,salary int ) STORED BY 'org.apache.carbondata.format' " + + "tblproperties('dictionary_include'='country','local_dictionary_enable'='true','local_dictionary_include' = 'phonetype','local_dictionary_exclude' ='serialname')") + sql("create datamap PreAggCount on table local1 using 'preaggregate' as " + + "select country,count(salary) as count from local1 group by country") + val descLoc = sql("describe formatted local1_PreAggCount").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("phonetype")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("serialname")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + } + + test("test local dictionary foer varchar datatype columns") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_include'='city', + | 'LONG_STRING_COLUMNS'='city') + """.stripMargin) + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + } + + test("test local dictionary describe formatted only with default configs") + { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' + """.stripMargin) + + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name,city")) + } + } + override protected def afterAll(): Unit = { sql("DROP TABLE IF EXISTS LOCAL1") } http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d24eac/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala index 44d3cca..6a73140 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala @@ -24,6 +24,7 @@ import java.text.SimpleDateFormat import java.util.Date import scala.collection.mutable +import scala.util.Try import com.univocity.parsers.common.TextParsingException import org.apache.spark.SparkException @@ -36,7 +37,7 @@ import org.apache.spark.util.CarbonReflectionUtils import org.apache.carbondata.common.exceptions.MetadataProcessException import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException -import org.apache.carbondata.common.logging.LogService +import org.apache.carbondata.common.logging.{LogService, LogServiceFactory} import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType} import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier} import org.apache.carbondata.core.constants.CarbonCommonConstants @@ -55,6 +56,8 @@ import org.apache.carbondata.streaming.parser.FieldConverter object CarbonScalaUtil { + val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) + // TODO: move this to spark module def convertSparkToCarbonDataType(dataType: DataType): CarbonDataType = { dataType match { @@ -615,4 +618,114 @@ object CarbonScalaUtil { sparkSession, schema)._1.asInstanceOf[Object] } + + /** + * this method validates the local dictionary columns configurations + * + * @param tableProperties + * @param localDictColumns + */ + def validateLocalDictionaryColumns(tableProperties: mutable.Map[String, String], + localDictColumns: Seq[String]): Unit = { + var dictIncludeColumns: Seq[String] = Seq[String]() + + // check if the duplicate columns are specified in table schema + if (localDictColumns.distinct.lengthCompare(localDictColumns.size) != 0) { + val duplicateColumns = localDictColumns + .diff(localDictColumns.distinct).distinct + val errMsg = + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE contains Duplicate Columns: " + + duplicateColumns.mkString(",") + + ". Please check create table statement." + throw new MalformedCarbonCommandException(errMsg) + } + + // check if the same column is present in both dictionary include and local dictionary columns + // configuration + if (tableProperties.get(CarbonCommonConstants.DICTIONARY_INCLUDE).isDefined) { + dictIncludeColumns = + tableProperties(CarbonCommonConstants.DICTIONARY_INCLUDE).split(",").map(_.trim) + localDictColumns.foreach { distCol => + if (dictIncludeColumns.exists(x => x.equalsIgnoreCase(distCol.trim))) { + val commonColumn = (dictIncludeColumns ++ localDictColumns) + .diff((dictIncludeColumns ++ localDictColumns).distinct).distinct + val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + + commonColumn.mkString(",") + + " specified in Dictionary include. Local Dictionary will not be " + + "generated for Dictionary include columns. Please check create table " + + "statement." + throw new MalformedCarbonCommandException(errormsg) + } + } + } + } + + /** + * this method validates the local dictionary enable property + * + * @param localDictionaryEnable + * @return + */ + def validateLocalDictionaryEnable(localDictionaryEnable: String): Boolean = { + Try(localDictionaryEnable.toBoolean) match { + case scala.util.Success(value) => + true + case scala.util.Failure(ex) => + false + } + } + + /** + * this method validates the local dictionary threshold property + * + * @param localDictionaryThreshold + * @return + */ + def validateLocalDictionaryThreshold(localDictionaryThreshold: String): Boolean = { + // if any invalid value is configured for LOCAL_DICTIONARY_THRESHOLD, then default value + // will be + // considered which is 1000 + Try(localDictionaryThreshold.toInt) match { + case scala.util.Success(value) => + if (value < 1000 || value > 100000) { + false + } else { + true + } + case scala.util.Failure(ex) => + false + } + } + + /** + * This method validate if both local dictionary include and exclude contains same column + * + * @param tableProperties + */ + def validateDuplicateLocalDictIncludeExcludeColmns(tableProperties: mutable.Map[String, + String]): Unit = { + val isLocalDictIncludeDefined = tableProperties + .get(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE) + .isDefined + val isLocalDictExcludeDefined = tableProperties + .get(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE) + .isDefined + if (isLocalDictIncludeDefined && isLocalDictExcludeDefined) { + val localDictIncludeCols = tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE) + .split(",").map(_.trim) + val localDictExcludeCols = tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE) + .split(",").map(_.trim) + localDictIncludeCols.foreach { distCol => + if (localDictExcludeCols.exists(x => x.equalsIgnoreCase(distCol.trim))) { + val duplicateColumns = (localDictIncludeCols ++ localDictExcludeCols) + .diff((localDictIncludeCols ++ localDictExcludeCols).distinct).distinct + val errMsg = "Column ambiguity as duplicate column(s):" + + duplicateColumns.mkString(",") + + " is present in LOCAL_DICTIONARY_INCLUDE " + + "and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed." + throw new MalformedCarbonCommandException(errMsg) + } + } + } + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d24eac/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala index f60a413..6bca7a4 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala @@ -275,6 +275,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { tableProperties: Map[String, String], bucketFields: Option[BucketFields], isAlterFlow: Boolean = false, + isPreAggFlow: Boolean = false, tableComment: Option[String] = None): TableModel = { // do not allow below key words as column name @@ -298,13 +299,13 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { // validate the local dictionary property if defined if (tableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE).isDefined) { - Try(tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE).toBoolean) match { - case scala.util.Success(value) => - case scala.util.Failure(ex) => - tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE, - CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT) + if (!CarbonScalaUtil + .validateLocalDictionaryEnable(tableProperties(CarbonCommonConstants + .LOCAL_DICTIONARY_ENABLE))) { + tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE, + CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT) } - } else { + } else if (!isAlterFlow) { // if LOCAL_DICTIONARY_ENABLE is not defined, consider the default value which is true tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE, CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT) @@ -312,30 +313,24 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { // validate the local dictionary threshold property if defined if (tableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD).isDefined) { - // if any invalid value is configured for LOCAL_DICTIONARY_THRESHOLD, then default value - // will be - // considered which is 1000 - Try(tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD).toInt) match { - case scala.util.Success(value) => - if (value <= 0) { - tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD, - CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT) - } - case scala.util.Failure(ex) => - LOGGER - .debug( - "invalid value is configured for local_dictionary_threshold, considering the " + - "default value") - tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD, - CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT) + if (!CarbonScalaUtil + .validateLocalDictionaryThreshold(tableProperties(CarbonCommonConstants + .LOCAL_DICTIONARY_THRESHOLD))) { + LOGGER.debug( + "invalid value is configured for local_dictionary_threshold, considering the " + + "default value") + tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD, + CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT) } } // validate the local dictionary columns defined, this we will validated if the local dictionary // is enabled, else it is not validated + // if it is preaggregate flow no need to validate anything, as all the properties will be + // inherited from parent table if (!(tableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE).isDefined && tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE).trim - .equalsIgnoreCase("false"))) { + .equalsIgnoreCase("false")) && !isPreAggFlow || isAlterFlow) { var localDictIncludeColumns: Seq[String] = Seq[String]() var localDictExcludeColumns: Seq[String] = Seq[String]() val isLocalDictIncludeDefined = tableProperties @@ -358,23 +353,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { } // validate if both local dictionary include and exclude contains same column - if (isLocalDictIncludeDefined && isLocalDictExcludeDefined) { - val localDictIncludeCols = tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE) - .split(",").map(_.trim) - val localDictExcludeCols = tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE) - .split(",").map(_.trim) - localDictIncludeCols.foreach { distCol => - if (localDictExcludeCols.exists(x => x.equalsIgnoreCase(distCol.trim))) { - val duplicateColumns = (localDictIncludeCols ++ localDictExcludeColumns) - .diff((localDictIncludeCols ++ localDictExcludeColumns).distinct).distinct - val errMsg = "Column ambiguity as duplicate column(s):" + - duplicateColumns.mkString(",") + - " is present in LOCAL_DICTIONARY_INCLUDE " + - "and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed." - throw new MalformedCarbonCommandException(errMsg) - } - } - } + CarbonScalaUtil.validateDuplicateLocalDictIncludeExcludeColmns(tableProperties) } // get no inverted index columns from table properties. @@ -467,17 +446,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { tableProperties: Map[String, String], localDictColumns: Seq[String]): Unit = { var dictIncludeColumns: Seq[String] = Seq[String]() - // check if the duplicate columns are specified in table schema - if (localDictColumns.distinct.lengthCompare(localDictColumns.size) != 0) { - val duplicateColumns = (dictIncludeColumns ++ localDictColumns) - .diff((dictIncludeColumns ++ localDictColumns).distinct).distinct - val errMsg = - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE contains Duplicate Columns: " + - duplicateColumns.mkString(",") + - ". Please check create table statement." - throw new MalformedCarbonCommandException(errMsg) - } - + // validate the local dict columns + CarbonScalaUtil.validateLocalDictionaryColumns(tableProperties, localDictColumns) // check if the column specified exists in table schema localDictColumns.foreach { distCol => if (!fields.exists(x => x.column.equalsIgnoreCase(distCol.trim))) { @@ -514,25 +484,6 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { throw new MalformedCarbonCommandException(errMsg) } } - - // check if the same column is present in both dictionary include and local dictionary columns - // configuration - if (tableProperties.get(CarbonCommonConstants.DICTIONARY_INCLUDE).isDefined) { - dictIncludeColumns = - tableProperties(CarbonCommonConstants.DICTIONARY_INCLUDE).split(",").map(_.trim) - localDictColumns.foreach { distCol => - if (dictIncludeColumns.exists(x => x.equalsIgnoreCase(distCol.trim))) { - val duplicateColumns = (dictIncludeColumns ++ localDictColumns) - .diff((dictIncludeColumns ++ localDictColumns).distinct).distinct - val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + - duplicateColumns.mkString(",") + - " specified in Dictionary include. Local Dictionary will not be " + - "generated for Dictionary include columns. Please check create table " + - "statement." - throw new MalformedCarbonCommandException(errormsg) - } - } - } } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d24eac/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala index c77d0df..8c7de56 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala @@ -299,6 +299,13 @@ class AlterTableColumnSchemaGenerator( val columnValidator = CarbonSparkFactory.getCarbonColumnValidator columnValidator.validateColumns(allColumns) + if (alterTableModel.tableProperties != null) { + CarbonUtil + .setLocalDictColumnsToWrapperSchema(newCols.asJava, + alterTableModel.tableProperties.asJava, + tableSchema.getTableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE)) + } + // populate table properties map val tablePropertiesMap = tableSchema.getTableProperties alterTableModel.tableProperties.foreach { @@ -584,7 +591,9 @@ class TableNewProcessor(cm: TableModel) { // check whether the column is a local dictionary column and set in column schema if (null != cm.tableProperties) { CarbonUtil - .setLocalDictColumnsToWrapperSchema(allColumns.asJava, cm.tableProperties.asJava) + .setLocalDictColumnsToWrapperSchema(allColumns.asJava, + cm.tableProperties.asJava, + cm.tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE)) } cm.msrCols.foreach { field => // if aggregate function is defined in case of preaggregate and agg function is sum or avg http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d24eac/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala index 857cd81..7d57be5 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala @@ -106,6 +106,24 @@ case class PreAggregateTableHelper( tableProperties.put(CarbonCommonConstants.FLAT_FOLDER, parentTable.getTableInfo.getFactTable.getTableProperties.asScala.getOrElse( CarbonCommonConstants.FLAT_FOLDER, CarbonCommonConstants.DEFAULT_FLAT_FOLDER)) + // inherit the local dictionary properties of main parent table + tableProperties + .put(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE, + parentTable.getTableInfo.getFactTable.getTableProperties.asScala + .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE, "false")) + tableProperties + .put(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD, + parentTable.getTableInfo.getFactTable.getTableProperties.asScala + .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD, + CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT)) + tableProperties + .put(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE, + parentTable.getTableInfo.getFactTable.getTableProperties.asScala + .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE, "")) + tableProperties + .put(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE, + parentTable.getTableInfo.getFactTable.getTableProperties.asScala + .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE, "")) val tableIdentifier = TableIdentifier(parentTable.getTableName + "_" + dataMapName, Some(parentTable.getDatabaseName)) @@ -119,6 +137,7 @@ case class PreAggregateTableHelper( tableProperties, None, isAlterFlow = false, + true, None) // updating the relation identifier, this will be stored in child table http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d24eac/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala index 7b19325..6b11f51 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala @@ -30,6 +30,7 @@ import org.codehaus.jackson.map.ObjectMapper import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.metadata.encoder.Encoding +import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema import org.apache.carbondata.core.util.CarbonUtil private[sql] case class CarbonDescribeFormattedCommand( @@ -113,21 +114,34 @@ private[sql] case class CarbonDescribeFormattedCommand( CarbonCommonConstants.CACHE_LEVEL_DEFAULT_VALUE), "")) val isStreaming = tblProps.asScala.getOrElse("streaming", "false") results ++= Seq(("Streaming", isStreaming, "")) - val isLocalDictEnabled = tblProps.asScala + var isLocalDictEnabled = tblProps.asScala .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE, - CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT) - results ++= Seq(("Local Dictionary Enabled", isLocalDictEnabled, "")) + CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT) + val localDictEnabled = isLocalDictEnabled.split(",") { 0 } + results ++= Seq(("Local Dictionary Enabled", localDictEnabled, "")) // if local dictionary is enabled, then only show other properties of local dictionary - if (isLocalDictEnabled.toBoolean) { - val localDictThreshold = tblProps.asScala + if (localDictEnabled.toBoolean) { + var localDictThreshold = tblProps.asScala .getOrElse(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD, CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT) + val localDictionaryThreshold = localDictThreshold.split(",") + localDictThreshold = localDictionaryThreshold { 0 } results ++= Seq(("Local Dictionary Threshold", localDictThreshold, "")) + val columns = carbonTable.getTableInfo.getFactTable.getListOfColumns.asScala if (tblProps.asScala .get(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE).isDefined) { val allLocalDictColumns = tblProps.asScala(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE) .split(",") results ++= Seq(("Local Dictionary Include", getDictColumnString(allLocalDictColumns), "")) + } else { + val builder = new StringBuilder + columns.foreach { column => + if (column.isLocalDictColumn) { + builder.append(column.getColumnName).append(",") + } + } + results ++= + Seq(("Local Dictionary Include", getDictColumnString(builder.toString().split(",")), "")) } if (tblProps.asScala .get(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE).isDefined) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d24eac/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala index f00fcf8..222d18d 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala @@ -542,7 +542,9 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser { val tblProps = tblProp.get .map(f => if (CarbonCommonConstants.TABLE_BLOCKSIZE.equalsIgnoreCase(f._1) || CarbonCommonConstants.COLUMN_GROUPS.equalsIgnoreCase(f._1) || - CarbonCommonConstants.SORT_COLUMNS.equalsIgnoreCase(f._1)) { + CarbonCommonConstants.SORT_COLUMNS.equalsIgnoreCase(f._1) || + CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE.equalsIgnoreCase(f._1) || + CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD.equalsIgnoreCase(f._1)) { throw new MalformedCarbonCommandException( s"Unsupported Table property in add column: ${ f._1 }") } else if (f._1.toLowerCase.startsWith("default.value.")) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d24eac/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala index 066819e..9b2f89c 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala @@ -305,6 +305,7 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf, tableProperties, bucketFields, isAlterFlow = false, + false, tableComment) TableNewProcessor(tableModel) } http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d24eac/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala index a6a730b..76cea55 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala @@ -24,7 +24,7 @@ import scala.collection.mutable import scala.collection.mutable.ListBuffer import org.apache.spark.SparkConf -import org.apache.spark.sql.{CarbonEnv, CarbonSession, SparkSession} +import org.apache.spark.sql.{CarbonEnv, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.hive.{CarbonRelation, CarbonSessionCatalog} import org.apache.spark.sql.hive.HiveExternalCatalog._ @@ -37,12 +37,14 @@ import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage} import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier} import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl +import org.apache.carbondata.core.metadata.datatype.DataTypes import org.apache.carbondata.core.metadata.schema.table.CarbonTable -import org.apache.carbondata.core.metadata.schema.table.column.{CarbonDimension, ColumnSchema} -import org.apache.carbondata.core.util.CarbonUtil +import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema import org.apache.carbondata.core.util.path.CarbonTablePath +import org.apache.carbondata.core.util.CarbonUtil import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo} -import org.apache.carbondata.spark.util.CommonUtil +import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil} + object AlterTableUtil { @@ -349,6 +351,10 @@ object AlterTableUtil { wrapperTableInfo, dbName, tableName) val tblPropertiesMap: mutable.Map[String, String] = thriftTable.fact_table.getTableProperties.asScala + + // validate the local dictionary properties + validateLocalDictionaryProperties(lowerCasePropertiesMap, tblPropertiesMap, carbonTable) + // below map will be used for cache invalidation. As tblProperties map is getting modified // in the next few steps the original map need to be retained for any decision making val existingTablePropertiesMap = mutable.Map(tblPropertiesMap.toSeq: _*) @@ -356,12 +362,16 @@ object AlterTableUtil { // This overrides old newProperties and update the comment parameter of thriftTable // with the newly added/modified comment since thriftTable also holds comment as its // direct property. - lowerCasePropertiesMap.foreach { property => if (validateTableProperties(property._1)) { - tblPropertiesMap.put(property._1, property._2) - } else { - val errorMessage = "Error: Invalid option(s): " + property._1.toString() - throw new MalformedCarbonCommandException(errorMessage) - }} + lowerCasePropertiesMap.foreach { property => + if (validateTableProperties(property._1)) { + tblPropertiesMap.put(property._1, property._2) + } else { + val errorMessage = "Error: Invalid option(s): " + property._1.toString() + throw new MalformedCarbonCommandException(errorMessage) + } + } + // check if duplicate columns are present in both local dictionary include and exclude + CarbonScalaUtil.validateDuplicateLocalDictIncludeExcludeColmns(tblPropertiesMap) } else { // This removes the comment parameter from thriftTable // since thriftTable also holds comment as its property. @@ -373,6 +383,8 @@ object AlterTableUtil { throw new MalformedCarbonCommandException(errorMessage) } } + // check if duplicate columns are present in both local dictionary include and exclude + CarbonScalaUtil.validateDuplicateLocalDictIncludeExcludeColmns(tblPropertiesMap) } val (tableIdentifier, schemParts, cols) = updateSchemaInfo(carbonTable, schemaConverter.fromWrapperToExternalSchemaEvolutionEntry(schemaEvolutionEntry), @@ -398,11 +410,55 @@ object AlterTableUtil { } private def validateTableProperties(propKey: String): Boolean = { - val supportedOptions = Seq("STREAMING", "COMMENT", "COLUMN_META_CACHE", "CACHE_LEVEL") + val supportedOptions = Seq("STREAMING", + "COMMENT", + "COLUMN_META_CACHE", + "CACHE_LEVEL", + "LOCAL_DICTIONARY_ENABLE", + "LOCAL_DICTIONARY_THRESHOLD", + "LOCAL_DICTIONARY_INCLUDE", + "LOCAL_DICTIONARY_EXCLUDE") supportedOptions.contains(propKey.toUpperCase) } /** + * this method validates the local dictioanry properties for alter set + * + * @param lowerCasePropertiesMap + * @param tblPropertiesMap + * @param carbonTable + */ + private def validateLocalDictionaryProperties(lowerCasePropertiesMap: mutable.Map[String, String], + tblPropertiesMap: mutable.Map[String, String], + carbonTable: CarbonTable): Unit = { + lowerCasePropertiesMap.foreach { property => + if (property._1.equalsIgnoreCase(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE)) { + if (!CarbonScalaUtil.validateLocalDictionaryEnable(property._2)) { + lowerCasePropertiesMap + .put(property._1.toLowerCase, CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT) + } else { + lowerCasePropertiesMap.put(property._1, property._2) + } + } + if (property._1.equalsIgnoreCase(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE) || + property._1.equalsIgnoreCase(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE)) { + ValidateSetTablePropertiesForLocalDict(tblPropertiesMap, carbonTable, property) + } + + if (property._1 + .equalsIgnoreCase(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD)) { + if (!CarbonScalaUtil.validateLocalDictionaryThreshold(property._2)) { + lowerCasePropertiesMap + .put(property._1, + CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT) + } else { + lowerCasePropertiesMap.put(property._1, property._2) + } + } + } + } + + /** * validate column meta cache and cache level properties if configured by the user * * @param carbonTable @@ -607,4 +663,90 @@ object AlterTableUtil { allColumnsMatch } + /** + * Validate LOCAL_DICT_COLUMNS property specified in Alter command + * @param tblPropertiesMap + * @param carbonTable + * @param property + */ + def ValidateSetTablePropertiesForLocalDict(tblPropertiesMap: mutable.Map[String, String], + carbonTable: CarbonTable, + property: (String, String)): Unit = { + var localDictColumns: Seq[String] = Seq[String]() + var dictIncludeColumns: Seq[String] = Seq[String]() + + val allColumns = carbonTable.getTableInfo.getFactTable.getListOfColumns.asScala + localDictColumns = property._2.toString.split(",").map(_.trim) + + CarbonScalaUtil.validateLocalDictionaryColumns(tblPropertiesMap, localDictColumns) + + // check if the column specified exists in table schema + localDictColumns.foreach { distCol => + if (!allColumns.exists(x => x.getColumnName.equalsIgnoreCase(distCol.trim))) { + val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + distCol.trim + + " does not exist in table. Please check create table statement." + throw new MalformedCarbonCommandException(errormsg) + } + } + + /** + * Verify if specified column is of no-dictionary string dataType + */ + localDictColumns.foreach { dictCol => + if (allColumns.exists(col => col.getColumnName.equalsIgnoreCase(dictCol) && + !col.getDataType.toString + .equalsIgnoreCase("STRING") && + !col.getDataType.toString + .equalsIgnoreCase("STRUCT") && + !col.getDataType.toString + .equalsIgnoreCase("ARRAY"))) { + val errMsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + dictCol.trim + + " is not a String/complex datatype column. LOCAL_DICTIONARY_INCLUDE" + + "/LOCAL_DICTIONARY_EXCLUDE should be no " + + "dictionary string/complex datatype column." + throw new MalformedCarbonCommandException(errMsg) + } + } + + // Validate whether any of the child columns of complex dataType column is a string column + localDictColumns.foreach { dictColm => + if (allColumns + .exists(x => x.getColumnName.equalsIgnoreCase(dictColm) && x.getNumberOfChild > 0 && + !validateChildColumns(allColumns, dictColm))) { + val errMsg = "None of the child columns specified in the complex dataType column(s) in " + + "local_dictionary_include are not of string dataType." + throw new MalformedCarbonCommandException(errMsg) + } + } + + /** + * check whether any child column present in comples type column is string type + * + * @param schemas + * @return + */ + def validateChildColumns(schemas: mutable.Buffer[ColumnSchema], + complexColumn: String): Boolean = { + var childColumnCount = 0 + var numberOfPrimitiveColumns = 0 + schemas.foreach { column => + if (childColumnCount > 0) { + if (column.getDataType.equals(DataTypes.STRING)) { + numberOfPrimitiveColumns += 1 + childColumnCount -= 1 + } else { + childColumnCount -= 1 + } + } + if (localDictColumns.exists(x => x.equalsIgnoreCase(column.getColumnName)) && + column.getNumberOfChild > 0) { + childColumnCount = column.getNumberOfChild + } + } + if (numberOfPrimitiveColumns > 0) { + return true + } + false + } + } }
