Repository: carbondata Updated Branches: refs/heads/master 7924d5e37 -> f0c77e5f5
[CARBONDATA-2689] Added validations for complex columns in alter set statements Issue: Alter set statements were not validating complex dataType columns correctly. Fix: Added a recursive method to validate string and varchar child columns of complex dataType columns. This closes #2450 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f0c77e5f Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f0c77e5f Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f0c77e5f Branch: refs/heads/master Commit: f0c77e5f54706476bafd7c802ca90cb09d277533 Parents: 7924d5e Author: praveenmeenakshi56 <[email protected]> Authored: Fri Jul 6 15:07:19 2018 +0530 Committer: kunal642 <[email protected]> Committed: Mon Jul 9 12:05:44 2018 +0530 ---------------------------------------------------------------------- .../apache/carbondata/core/util/CarbonUtil.java | 10 +- ...CreateTableWithLocalDictionaryTestCase.scala | 48 +- .../LocalDictionarySupportAlterTableTest.scala | 56 +- .../LocalDictionarySupportCreateTableTest.scala | 553 +++++++++---------- .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 18 +- .../command/carbonTableSchemaCommon.scala | 3 + .../org/apache/spark/util/AlterTableUtil.scala | 68 +-- 7 files changed, 379 insertions(+), 377 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 9423e28..eaa2b74 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -3141,8 +3141,9 @@ public final class CarbonUtil { } else { continue; } - if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column - .getDataType().equals(DataTypes.VARCHAR)) && !column.hasEncoding(Encoding.DICTIONARY) + if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || + column.getDataType().equals(DataTypes.VARCHAR)) && + !column.hasEncoding(Encoding.DICTIONARY) && localDictIncludeColumns.toLowerCase() .contains(column.getColumnName().toLowerCase())) { for (String dictColumn : listOfDictionaryIncludeColumns) { @@ -3172,8 +3173,9 @@ public final class CarbonUtil { dimensionOrdinal++; setLocalDictForComplexColumns(allColumns, dimensionOrdinal, column.getNumberOfChild()); } else { - if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column - .getDataType().equals(DataTypes.VARCHAR)) && !column.hasEncoding(Encoding.DICTIONARY)) { + if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || + column.getDataType().equals(DataTypes.VARCHAR)) && + !column.hasEncoding(Encoding.DICTIONARY)) { column.setLocalDictColumn(true); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala index b787d70..b6ecef9 100644 --- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala @@ -93,8 +93,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please " + - "check the DDL.")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " + + "the DDL.")) } test("test local dictionary custom configurations for local dict columns _004") { @@ -110,8 +110,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception1.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + - "Please check the DDL.")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " + + "the DDL.")) } test("test local dictionary custom configurations for local dict columns _005") { @@ -127,9 +127,9 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " + + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. " + "LOCAL_DICTIONARY_COLUMN should " + - "be no dictionary string/complex datatype column")) + "be no dictionary string/complex/varchar datatype column")) } test("test local dictionary custom configurations for local dict columns _006") { @@ -483,8 +483,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. " + - "Please check the DDL.")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " + + "the DDL.")) } @@ -501,8 +501,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception1.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + - "Please check the DDL.")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " + + "the DDL.")) } test("test local dictionary custom configurations when enabled for local dict columns _005") { @@ -518,9 +518,9 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " + + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. " + "LOCAL_DICTIONARY_COLUMN should " + - "be no dictionary string/complex datatype column")) + "be no dictionary string/complex/varchar datatype column")) } test("test local dictionary custom configurations when enabled for local dict columns _006") { @@ -600,8 +600,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception1.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + - "Please check the DDL.")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " + + "the DDL.")) } test("test local dictionary custom configurations when local_dictionary_exclude is configured _005") { @@ -617,9 +617,9 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " + + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. " + "LOCAL_DICTIONARY_COLUMN should " + - "be no dictionary string/complex datatype column")) + "be no dictionary string/complex/varchar datatype column")) } test("test local dictionary custom configurations when local_dictionary_exclude is configured _006") { @@ -1674,8 +1674,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception1.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + - "Please check the DDL.")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " + + "the DDL.")) } test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _005") { @@ -1697,9 +1697,9 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " + + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. " + "LOCAL_DICTIONARY_COLUMN should " + - "be no dictionary string/complex datatype column")) + "be no dictionary string/complex/varchar datatype column")) } test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _006") { @@ -1806,8 +1806,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception1.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + - "Please check the DDL.")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " + + "the DDL.")) } test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _005") { @@ -1829,9 +1829,9 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " + + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. " + "LOCAL_DICTIONARY_COLUMN should " + - "be no dictionary string/complex datatype column")) + "be no dictionary string/complex/varchar datatype column")) } test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _006") { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala index 341a28a..c7dae39 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala @@ -150,8 +150,8 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc is not a String/complex " + - "datatype column. LOCAL_DICTIONARY_COLUMN should be no dictionary string/complex datatype" + + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc is not a string/complex/varchar " + + "datatype column. LOCAL_DICTIONARY_COLUMN should be no dictionary string/complex/varchar datatype" + " column.Please check the DDL.")) } @@ -537,7 +537,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter val exception1 = intercept[Exception] { sql("alter table local1 set tblproperties('local_dictionary_exclude'='id')") } - assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE should be no dictionary string/complex datatype column.")) + assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE should be no dictionary string/complex/varchar datatype column.")) } test("test alter set for local dictionary _010") { @@ -790,9 +790,8 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter | 'local_dictionary_enable'='true') """. stripMargin) - intercept[Exception] { - sql("alter table local1 set tblproperties('local_dictionary_exclude'='dcity')") - } + sql("alter table local1 set tblproperties('local_dictionary_exclude'='dcity')") + intercept[Exception] { sql("alter table local1 set tblproperties('local_dictionary_include'='st')") } @@ -832,7 +831,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql("alter table local1 unset tblproperties('local_dictionary_include')") val descLoc2 = sql("describe formatted local1").collect descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match { - case Some(row) => assert(row.get(1).toString.contains("name,city")) + case Some(row) => assert(row.get(1).toString.contains("name,city,add")) } } @@ -1119,9 +1118,50 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter } } + test("test alter for local dictionary with custom configurations when varchar columns are specified _001") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string,sd:int>>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('long_string_columns'='name','local_dictionary_enable'='true') + """.stripMargin) + sql("alter table local1 set tblproperties('local_dictionary_exclude'='st,name')") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("st.val.si,name")) + } + } + + test("test alter for local dictionary with custom configurations when varchar columns are specified _002") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string,sd:int>>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('long_string_columns'='name','local_dictionary_enable'='true', + | 'local_dictionary_include'='name') + """.stripMargin) + sql("alter table local1 set tblproperties('local_dictionary_include'='st,name')") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("st.val.si,name")) + } + } override protected def afterAll(): Unit = { sql("DROP TABLE IF EXISTS LOCAL1") } - } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala index 5398613..32fab2c 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala @@ -65,9 +65,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations for local dict columns _002") - { + test("test local dictionary custom configurations for local dict columns _002") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { @@ -127,10 +125,10 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex " + + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar " + "datatype column. " + "LOCAL_DICTIONARY_COLUMN should " + - "be no dictionary string/complex datatype column")) + "be no dictionary string/complex/varchar datatype column")) } test("test local dictionary custom configurations for local dict columns _006") { @@ -164,8 +162,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations for local dict threshold _002") - { + test("test local dictionary custom configurations for local dict threshold _002") { sql("drop table if exists local1") sql( """ @@ -180,8 +177,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations for local dict threshold _003") - { + test("test local dictionary custom configurations for local dict threshold _003") { sql("drop table if exists local1") sql( """ @@ -196,8 +192,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations for local dict threshold _004") - { + test("test local dictionary custom configurations for local dict threshold _004") { sql("drop table if exists local1") sql( """ @@ -212,8 +207,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations for local dict threshold _005") - { + test("test local dictionary custom configurations for local dict threshold _005") { sql("drop table if exists local1") sql( """ @@ -229,8 +223,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } test("test local dictionary custom configurations with both columns and threshold configured " + - "_001") - { + "_001") { sql("drop table if exists local1") sql( """ @@ -252,8 +245,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } test("test local dictionary custom configurations with both columns and threshold configured " + - "_002") - { + "_002") { sql("drop table if exists local1") sql( """ @@ -275,8 +267,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } test("test local dictionary custom configurations with both columns and threshold configured " + - "_003") - { + "_003") { sql("drop table if exists local1") sql( """ @@ -298,8 +289,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } test("test local dictionary custom configurations with both columns and threshold configured " + - "_004") - { + "_004") { sql("drop table if exists local1") sql( """ @@ -335,8 +325,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } test("test local dictionary custom configurations with both columns and threshold configured " + - "_006") - { + "_006") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -349,8 +338,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } test("test local dictionary custom configurations with both columns and threshold configured " + - "_007") - { + "_007") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -363,8 +351,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } test("test local dictionary custom configurations with both columns and threshold configured " + - "_008") - { + "_008") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -378,8 +365,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } test("test local dictionary custom configurations with both columns and threshold configured " + - "_009") - { + "_009") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -392,8 +378,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } test("test local dictionary custom configurations with both columns and threshold configured " + - "_010") - { + "_010") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -406,8 +391,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } test("test local dictionary custom configurations with both columns and threshold configured " + - "_011") - { + "_011") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -457,8 +441,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations when enabled for local dict columns _002") - { + test("test local dictionary custom configurations when enabled for local dict columns _002") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { @@ -519,10 +502,10 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex " + + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar " + "datatype column. " + "LOCAL_DICTIONARY_COLUMN should " + - "be no dictionary string/complex datatype column")) + "be no dictionary string/complex/varchar datatype column")) } test("test local dictionary custom configurations when enabled for local dict columns _006") { @@ -539,9 +522,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when local_dictionary_exclude is configured _001") - { + test("test local dictionary custom configurations when local_dictionary_exclude is " + + "configured _001") { sql("drop table if exists local1") sql( """ @@ -559,9 +541,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when local_dictionary_exclude is configured _002") - { + test("test local dictionary custom configurations when local_dictionary_exclude is " + + "configured _002") { sql("drop table if exists local1") val exception = intercept[MalformedCarbonCommandException] { sql( @@ -577,9 +558,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte "Please check the DDL.")) } - test( - "test local dictionary custom configurations when local_dictionary_exclude is configured _003") - { + test("test local dictionary custom configurations when local_dictionary_exclude is " + + "configured _003") { sql("drop table if exists local1") val exception = intercept[MalformedCarbonCommandException] { sql( @@ -597,9 +577,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } - test( - "test local dictionary custom configurations when local_dictionary_exclude is configured _004") - { + test("test local dictionary custom configurations when local_dictionary_exclude is " + + "configured _004") { sql("drop table if exists local1") val exception1 = intercept[MalformedCarbonCommandException] { sql( @@ -616,9 +595,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte "Please check the DDL.")) } - test( - "test local dictionary custom configurations when local_dictionary_exclude is configured _005") - { + test("test local dictionary custom configurations when local_dictionary_exclude is " + + "configured _005") { sql("drop table if exists local1") val exception = intercept[MalformedCarbonCommandException] { sql( @@ -631,15 +609,14 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex " + + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar " + "datatype column. " + "LOCAL_DICTIONARY_COLUMN should " + - "be no dictionary string/complex datatype column")) + "be no dictionary string/complex/varchar datatype column")) } - test( - "test local dictionary custom configurations when local_dictionary_exclude is configured _006") - { + test("test local dictionary custom configurations when local_dictionary_exclude is " + + "configured _006") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -653,10 +630,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when local_dictionary_include and " + - "local_dictionary_exclude " + - "is configured _001") { + test("test local dictionary custom configurations when local_dictionary_include and " + + "local_dictionary_exclude is configured _001") { sql("drop table if exists local1") sql( """ @@ -680,10 +655,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when local_dictionary_include and " + - "local_dictionary_exclude " + - "is configured _002") { + test("test local dictionary custom configurations when local_dictionary_include and " + + "local_dictionary_exclude is configured _002") { sql("drop table if exists local1") sql( """ @@ -708,10 +681,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when local_dictionary_include and " + - "local_dictionary_exclude " + - "is configured _003") { + test("test local dictionary custom configurations when local_dictionary_include and " + + "local_dictionary_exclude is configured _003") { sql("drop table if exists local1") sql( """ @@ -730,10 +701,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte checkExistence(sql("describe formatted local1"), false, "Local Dictionary Exclude") } - test( - "test local dictionary custom configurations when local_dictionary_include and " + - "local_dictionary_exclude " + - "is configured _004") { + test("test local dictionary custom configurations when local_dictionary_include and " + + "local_dictionary_exclude is configured _004") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -747,10 +716,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when local_dictionary_include and " + - "local_dictionary_exclude " + - "is configured _005") { + test("test local dictionary custom configurations when local_dictionary_include and " + + "local_dictionary_exclude is configured _005") { sql("drop table if exists local1") val exception = intercept[MalformedCarbonCommandException] { sql( @@ -768,10 +735,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte "LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed.")) } - test( - "test local dictionary custom configurations when local_dictionary_include and " + - "local_dictionary_exclude " + - "is configured _006") { + test("test local dictionary custom configurations when local_dictionary_include and " + + "local_dictionary_exclude is configured _006") { sql("drop table if exists local1") sql( """ @@ -791,10 +756,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when local_dictionary_include and " + - "local_dictionary_exclude " + - "is configured _007") { + test("test local dictionary custom configurations when local_dictionary_include and " + + "local_dictionary_exclude is configured _007") { sql("drop table if exists local1") sql( """ @@ -832,8 +795,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations when enabled for local dict threshold _002") - { + test("test local dictionary custom configurations when enabled for local dict threshold _002") { sql("drop table if exists local1") sql( """ @@ -848,8 +810,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations when enabled for local dict threshold _003") - { + test("test local dictionary custom configurations when enabled for local dict threshold _003") { sql("drop table if exists local1") sql( """ @@ -864,8 +825,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations when enabled for local dict threshold _004") - { + test("test local dictionary custom configurations when enabled for local dict threshold _004") { sql("drop table if exists local1") sql( """ @@ -880,8 +840,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations when enabled for local dict threshold _005") - { + test("test local dictionary custom configurations when enabled for local dict threshold _005") { sql("drop table if exists local1") sql( """ @@ -896,10 +855,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when enabled with both columns and threshold " + - "configured _001") - { + test("test local dictionary custom configurations when enabled with both columns and threshold " + + "configured _001") { sql("drop table if exists local1") sql( """ @@ -921,10 +878,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when enabled with both columns and threshold " + - "configured _002") - { + test("test local dictionary custom configurations when enabled with both columns and threshold " + + "configured _002") { sql("drop table if exists local1") sql( """ @@ -946,10 +901,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when enabled with both columns and threshold " + - "configured _003") - { + test("test local dictionary custom configurations when enabled with both columns and threshold " + + "configured _003") { sql("drop table if exists local1") sql( """ @@ -971,10 +924,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when enabled with both columns and threshold " + - "configured _004") - { + test("test local dictionary custom configurations when enabled with both columns and threshold " + + "configured _004") { sql("drop table if exists local1") sql( """ @@ -996,10 +947,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when enabled with both columns and threshold " + - "configured _005") - { + test("test local dictionary custom configurations when enabled with both columns and threshold " + + "configured _005") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -1012,10 +961,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when enabled with both columns and threshold " + - "configured _006") - { + test("test local dictionary custom configurations when enabled with both columns and threshold " + + "configured _006") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -1028,10 +975,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when enabled with both columns and threshold " + - "configured _007") - { + test("test local dictionary custom configurations when enabled with both columns and threshold " + + "configured _007") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -1044,10 +989,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when enabled with both columns and threshold " + - "configured _008") - { + test("test local dictionary custom configurations when enabled with both columns and threshold " + + "configured _008") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -1060,10 +1003,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when enabled with both columns and threshold " + - "configured _009") - { + test("test local dictionary custom configurations when enabled with both columns and threshold " + + "configured _009") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -1076,10 +1017,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when enabled with both columns and threshold " + - "configured _010") - { + test("test local dictionary custom configurations when enabled with both columns and threshold " + + "configured _010") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -1092,10 +1031,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when enabled with both columns and threshold " + - "configured _011") - { + test("test local dictionary custom configurations when enabled with both columns and threshold " + + "configured _011") { sql("drop table if exists local1") intercept[MalformedCarbonCommandException] { sql( @@ -1140,8 +1077,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations when disabled for local dict columns _002") - { + test("test local dictionary custom configurations when disabled for local dict columns _002") { sql("drop table if exists local1") sql( """ @@ -1231,8 +1167,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations when disabled for local dict threshold _002") - { + test("test local dictionary custom configurations when disabled for local dict threshold _002") { sql("drop table if exists local1") sql( """ @@ -1263,8 +1198,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations when disabled for local dict threshold _004") - { + test("test local dictionary custom configurations when disabled for local dict threshold _004") { sql("drop table if exists local1") sql( """ @@ -1279,8 +1213,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test("test local dictionary custom configurations when disabled for local dict threshold _005") - { + test("test local dictionary custom configurations when disabled for local dict threshold _005") { sql("drop table if exists local1") sql( """ @@ -1297,8 +1230,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte test( "test local dictionary custom configurations when disabled with both columns and threshold " + - "configured _001") - { + "configured _001") { sql("drop table if exists local1") sql( """ @@ -1314,10 +1246,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when disabled with both columns and threshold " + - "configured _002") - { + test("test local dictionary custom configurations when disabled with both columns and threshold" + + " configured _002") { sql("drop table if exists local1") sql( """ @@ -1333,10 +1263,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when disabled with both columns and threshold " + - "configured _003") - { + test("test local dictionary custom configurations when disabled with both columns and threshold" + + " configured _003") { sql("drop table if exists local1") sql( """ @@ -1352,10 +1280,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when disabled with both columns and threshold " + - "configured _004") - { + test("test local dictionary custom configurations when disabled with both columns and threshold" + + " configured _004") { sql("drop table if exists local1") sql( """ @@ -1371,10 +1297,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when disabled with both columns and threshold " + - "configured _005") - { + test("test local dictionary custom configurations when disabled with both columns and threshold" + + " configured _005") { sql("drop table if exists local1") sql( @@ -1390,10 +1314,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when disabled with both columns and threshold " + - "configured _006") - { + test("test local dictionary custom configurations when disabled with both columns and threshold" + + " configured _006") { sql("drop table if exists local1") sql( """ @@ -1409,10 +1331,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when disabled with both columns and threshold " + - "configured _007") - { + test("test local dictionary custom configurations when disabled with both columns and threshold" + + " configured _007") { sql("drop table if exists local1") sql( """ @@ -1428,10 +1348,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when disabled with both columns and threshold " + - "configured _008") - { + test("test local dictionary custom configurations when disabled with both columns and threshold" + + " configured _008") { sql("drop table if exists local1") sql( """ @@ -1447,10 +1365,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when disabled with both columns and threshold " + - "configured _009") - { + test("test local dictionary custom configurations when disabled with both columns and threshold" + + " configured _009") { sql("drop table if exists local1") sql( """ @@ -1466,10 +1382,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when disabled with both columns and threshold " + - "configured _010") - { + test("test local dictionary custom configurations when disabled with both columns and threshold" + + " configured _010") { sql("drop table if exists local1") sql( """ @@ -1485,10 +1399,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when disabled with both columns and threshold " + - "configured _011") - { + test("test local dictionary custom configurations when disabled with both columns and threshold" + + " configured _011") { sql("drop table if exists local1") sql( """ @@ -1610,10 +1522,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when enabled for local dict " + - "columns _001") - { + test("test CTAS statements for local dictionary custom configurations when enabled for " + + "local dict columns _001") { sql("drop table if exists local") sql("drop table if exists local1") sql( @@ -1637,10 +1547,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when enabled for local dict " + - "columns _002") - { + test("test CTAS statements for local dictionary custom configurations when enabled for " + + "local dict columns _002") { sql("drop table if exists local") sql("drop table if exists local1") @@ -1659,10 +1567,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when enabled for local dict " + - "columns _003") - { + test("test CTAS statements for local dictionary custom configurations when enabled for " + + "local dict columns _003") { sql("drop table if exists local") sql("drop table if exists local1") @@ -1687,9 +1593,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } - test( - "test CTAS statements for local dictionary custom configurations when enabled for local dict " + - "columns _004") + test("test CTAS statements for local dictionary custom configurations when enabled for " + + "local dict columns _004") { sql("drop table if exists local") sql("drop table if exists local1") @@ -1713,10 +1618,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte "Please check the DDL.")) } - test( - "test CTAS statements for local dictionary custom configurations when enabled for local dict " + - "columns _005") - { + test("test CTAS statements for local dictionary custom configurations when enabled for " + + "local dict columns _005") { sql("drop table if exists local") sql("drop table if exists local1") sql( @@ -1735,16 +1638,14 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex " + + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar " + "datatype column. " + "LOCAL_DICTIONARY_COLUMN should " + - "be no dictionary string/complex datatype column")) + "be no dictionary string/complex/varchar datatype column")) } - test( - "test CTAS statements for local dictionary custom configurations when enabled for local dict " + - "columns _006") - { + test("test CTAS statements for local dictionary custom configurations when enabled for local dict " + + "columns _006") { sql("drop table if exists local") sql("drop table if exists local1") sql( @@ -1763,10 +1664,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when " + - "local_dictionary_exclude is configured _001") - { + test("test CTAS statements for local dictionary custom configurations when " + + "local_dictionary_exclude is configured _001") { sql("drop table if exists local") sql("drop table if exists local1") sql( @@ -1790,10 +1689,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when " + - "local_dictionary_exclude is configured _002") - { + test("test CTAS statements for local dictionary custom configurations when " + + "local_dictionary_exclude is configured _002") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -1811,10 +1708,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when " + - "local_dictionary_exclude is configured _003") - { + test("test CTAS statements for local dictionary custom configurations when " + + "local_dictionary_exclude is configured _003") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -1837,10 +1732,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte "Please check the DDL.")) } - test( - "test CTAS statements for local dictionary custom configurations when " + - "local_dictionary_exclude is configured _004") - { + test("test CTAS statements for local dictionary custom configurations when " + + "local_dictionary_exclude is configured _004") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -1863,10 +1756,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte "Please check the DDL.")) } - test( - "test CTAS statements for local dictionary custom configurations when " + - "local_dictionary_exclude is configured _005") - { + test("test CTAS statements for local dictionary custom configurations when " + + "local_dictionary_exclude is configured _005") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -1885,16 +1776,14 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex " + + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar " + "datatype column. " + "LOCAL_DICTIONARY_COLUMN should " + - "be no dictionary string/complex datatype column")) + "be no dictionary string/complex/varchar datatype column")) } - test( - "test CTAS statements for local dictionary custom configurations when " + - "local_dictionary_exclude is configured _006") - { + test("test CTAS statements for local dictionary custom configurations when " + + "local_dictionary_exclude is configured _006") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -1913,10 +1802,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when " + - "local_dictionary_include and local_dictionary_exclude " + - "is configured _001") { + test("test CTAS statements for local dictionary custom configurations when " + + "local_dictionary_include and local_dictionary_exclude is configured _001") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -1945,10 +1832,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when " + - "local_dictionary_include and local_dictionary_exclude " + - "is configured _002") { + test("test CTAS statements for local dictionary custom configurations when " + + "local_dictionary_include and local_dictionary_exclude is configured _002") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -1972,10 +1857,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte checkExistence(sql("describe formatted local1"), false, "Local Dictionary Exclude") } - test( - "test CTAS statements for local dictionary custom configurations when " + - "local_dictionary_include and local_dictionary_exclude " + - "is configured _003") { + test("test CTAS statements for local dictionary custom configurations when " + + "local_dictionary_include and local_dictionary_exclude is configured _003") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -1995,10 +1878,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when " + - "local_dictionary_include and local_dictionary_exclude " + - "is configured _004") { + test("test CTAS statements for local dictionary custom configurations when " + + "local_dictionary_include and local_dictionary_exclude is configured _004") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -2017,10 +1898,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when " + - "local_dictionary_include and local_dictionary_exclude " + - "is configured _005") { + test("test CTAS statements for local dictionary custom configurations when " + + "local_dictionary_include and local_dictionary_exclude is configured _005") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -2042,10 +1921,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when enabled for local dict " + - "threshold _001") - { + test("test CTAS statements for local dictionary custom configurations when enabled for local dict " + + "threshold _001") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -2069,10 +1946,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when enabled for local dict " + - "threshold _002") - { + test("test CTAS statements for local dictionary custom configurations when enabled for local dict " + + "threshold _002") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -2093,10 +1968,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when enabled for local dict " + - "threshold _003") - { + test("test CTAS statements for local dictionary custom configurations when enabled for local dict " + + "threshold _003") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -2118,10 +1991,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test CTAS statements for local dictionary custom configurations when first table is hive " + - "table") - { + test("test CTAS statements for local dictionary custom configurations when first table is hive " + + "table") { sql("drop table if exists local1") sql("drop table if exists local") sql( @@ -2172,9 +2043,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when complex dataType columns are given in " + - "local_dictionary_include _001") { + test("test local dictionary custom configurations when complex dataType columns are given in " + + "local_dictionary_include _001") { sql("drop table if exists local1") val exception = intercept[MalformedCarbonCommandException] { sql( @@ -2190,9 +2060,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte "local_dictionary_include are not of string dataType.")) } - test( - "test local dictionary custom configurations when complex dataType columns are given in " + - "local_dictionary_include _002") { + test("test local dictionary custom configurations when complex dataType columns are given in " + + "local_dictionary_include _002") { sql("drop table if exists local1") val exception = intercept[MalformedCarbonCommandException] { sql( @@ -2209,10 +2078,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte "local_dictionary_include are not of string dataType.")) } - test( - "test local dictionary custom configurations when complex dataType columns are given in " + - "local_dictionary_include _003") - { + test("test local dictionary custom configurations when complex dataType columns are given in " + + "local_dictionary_include _003") { sql("drop table if exists local1") sql( """ @@ -2232,10 +2099,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when complex dataType columns are given in " + - "local_dictionary_include _004") - { + test("test local dictionary custom configurations when complex dataType columns are given in " + + "local_dictionary_include _004") { sql("drop table if exists local1") sql( """ @@ -2255,10 +2120,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when complex dataType columns are given in " + - "local_dictionary_include _005") - { + test("test local dictionary custom configurations when complex dataType columns are given in " + + "local_dictionary_include _005") { sql("drop table if exists local1") sql( """ @@ -2278,9 +2141,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when complex dataType columns are given in " + - "local_dictionary_include _006") { + test("test local dictionary custom configurations when complex dataType columns are given in " + + "local_dictionary_include _006") { sql("drop table if exists local1") sql( """ @@ -2301,10 +2163,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when complex dataType columns are given in " + - "local_dictionary_include _007") - { + test("test local dictionary custom configurations when complex dataType columns are given in " + + "local_dictionary_include _007") { sql("drop table if exists local1") sql( """ @@ -2324,9 +2184,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } } - test( - "test local dictionary custom configurations when complex dataType columns are given in " + - "local_dictionary_include _008") { + test("test local dictionary custom configurations when complex dataType columns are given in " + + "local_dictionary_include _008") { sql("drop table if exists local1") val exception = intercept[MalformedCarbonCommandException] { sql( @@ -2343,6 +2202,96 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte "local_dictionary_include are not of string dataType.")) } + test("test local dictionary custom configurations when varchar columns are specified _001") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string>>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('LONG_STRING_COLUMNS'='name','local_dictionary_enable'='true','local_dictionary_include'='name') + """.stripMargin) + sql("desc formatted local1").show(30,false) + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name") && !row.get(1).toString.contains("city")) + } + } + + test("test local dictionary custom configurations when varchar columns are specified _002") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string,sd:int>>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('LONG_STRING_COLUMNS'=' Name','local_dictionary_enable'='true','local_dictionary_include'='name,st') + """.stripMargin) + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("st.val.si,name") && !row.get(1).toString.contains("city")) + } + } + + test("test local dictionary custom configurations when varchar columns are specified _003") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string,sd:int>>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('LONG_STRING_COLUMNS'='name','local_dictionary_enable'='true','local_dictionary_exclude'='name,st') + """.stripMargin) + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("st.val.si,name") && + !row.get(1).toString.contains("city")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + } + + test("test local dictionary custom configurations when varchar columns are specified _004") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string,sd:int>>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('LONG_STRING_COLUMNS'=' Name','local_dictionary_enable'='true', + | 'local_dictionary_include'='name,st','local_dictionary_exclude'='city') + """.stripMargin) + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("st.val.si,name") && + !row.get(1).toString.contains("city")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + } + override protected def afterAll(): Unit = { sql("DROP TABLE IF EXISTS LOCAL1") } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala index 84fd25f..d705fc9 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala @@ -457,17 +457,19 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { } } - // check if column is other than string datatype + // check if column is other than STRING or VARCHAR datatype localDictColumns.foreach { dictColm => if (fields .exists(x => x.column.equalsIgnoreCase(dictColm) && !x.dataType.get.equalsIgnoreCase("STRING") && + !x.dataType.get.equalsIgnoreCase("VARCHAR") && !x.dataType.get.equalsIgnoreCase("STRUCT") && !x.dataType.get.equalsIgnoreCase("ARRAY"))) { val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + dictColm.trim + - " is not a String/complex datatype column. LOCAL_DICTIONARY_COLUMN should " + - "be no dictionary string/complex datatype column.Please check the DDL." + " is not a string/complex/varchar datatype column. LOCAL_DICTIONARY_COLUMN" + + " should be no dictionary string/complex/varchar datatype column." + + "Please check the DDL." throw new MalformedCarbonCommandException(errormsg) } } @@ -685,12 +687,12 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { noInvertedIdxColsProps = tableProperties.get(CarbonCommonConstants.NO_INVERTED_INDEX).get.split(',').map(_.trim) noInvertedIdxColsProps.foreach { noInvertedIdxColProp => - if (!fields.exists(x => x.column.equalsIgnoreCase(noInvertedIdxColProp))) { - val errormsg = "NO_INVERTED_INDEX column: " + noInvertedIdxColProp + - " does not exist in table. Please check create table statement." - throw new MalformedCarbonCommandException(errormsg) - } + if (!fields.exists(x => x.column.equalsIgnoreCase(noInvertedIdxColProp))) { + val errormsg = "NO_INVERTED_INDEX column: " + noInvertedIdxColProp + + " does not exist in table. Please check create table statement." + throw new MalformedCarbonCommandException(errormsg) } + } } // check duplicate columns and only 1 col left val distinctCols = noInvertedIdxColsProps.toSet http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala index 8a014ce..6fee8c7 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala @@ -328,6 +328,7 @@ class AlterTableColumnSchemaGenerator( includeColumns.append(column) } else { if (column.getDataType.equals(DataTypes.STRING) || + column.getDataType.equals(DataTypes.VARCHAR) || column.getDataType.toString.equals("ARRAY") || column.getDataType.toString.equals("STRUCT")) { excludeColumns.append(column) @@ -348,6 +349,7 @@ class AlterTableColumnSchemaGenerator( if (null == localDictExcludeColumns) { columns.foreach { column => if (column.getDataType.equals(DataTypes.STRING) || + column.getDataType.equals(DataTypes.VARCHAR) || column.getDataType.toString.equals("ARRAY") || column.getDataType.toString.equals("STRUCT")) { includeColumns.append(column) @@ -357,6 +359,7 @@ class AlterTableColumnSchemaGenerator( columns.foreach { column => if (!localDictExcludeColumns.contains(column.getColumnName) && (column.getDataType.equals(DataTypes.STRING) || + column.getDataType.equals(DataTypes.VARCHAR) || column.getDataType.toString.equals("ARRAY") || column.getDataType.toString.equals("STRUCT"))) { includeColumns.append(column) http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala index 4659c39..b5450b3 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala @@ -42,7 +42,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema import org.apache.carbondata.core.util.path.CarbonTablePath import org.apache.carbondata.core.util.CarbonUtil -import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo} +import org.apache.carbondata.format.{Encoding, SchemaEvolutionEntry, TableInfo} import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil} @@ -691,62 +691,68 @@ object AlterTableUtil { } /** - * Verify if specified column is of no-dictionary string dataType + * Verify if specified column is of no-dictionary string or varchar dataType */ localDictColumns.foreach { dictCol => if (allColumns.exists(col => col.getColumnName.equalsIgnoreCase(dictCol) && !col.getDataType.toString .equalsIgnoreCase("STRING") && !col.getDataType.toString + .equalsIgnoreCase("VARCHAR") && + !col.getDataType.toString .equalsIgnoreCase("STRUCT") && !col.getDataType.toString .equalsIgnoreCase("ARRAY"))) { val errMsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + dictCol.trim + - " is not a String/complex datatype column. LOCAL_DICTIONARY_INCLUDE" + + " is not a string/complex/varchar datatype column. LOCAL_DICTIONARY_INCLUDE" + "/LOCAL_DICTIONARY_EXCLUDE should be no " + - "dictionary string/complex datatype column." + "dictionary string/complex/varchar datatype column." throw new MalformedCarbonCommandException(errMsg) } } - - // Validate whether any of the child columns of complex dataType column is a string column - localDictColumns.foreach { dictColm => - if (allColumns - .exists(x => x.getColumnName.equalsIgnoreCase(dictColm) && x.getNumberOfChild > 0 && - !validateChildColumns(allColumns, dictColm))) { - val errMsg = "None of the child columns specified in the complex dataType column(s) in " + - "local_dictionary_include are not of string dataType." - throw new MalformedCarbonCommandException(errMsg) + var countOfDictCols = 0 + var trav = 0 + // Validate whether any of the child columns of complex dataType column is a string or + // varchar dataType column + if (property._1.equalsIgnoreCase(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE)) { + // Validate whether any of the child columns of complex dataType column is a string column + localDictColumns.foreach { dictColm => + for (elem <- allColumns.indices) { + var column = allColumns(elem) + if (column.getColumnName.equalsIgnoreCase(dictColm) && column.getNumberOfChild > 0 && + !validateChildColumns(allColumns, column.getNumberOfChild, elem. +(1))) { + val errMsg = + "None of the child columns specified in the complex dataType column(s) in " + + "local_dictionary_include are not of string dataType." + throw new MalformedCarbonCommandException(errMsg) + } + } } } /** - * check whether any child column present in complex type column is string type + * check whether any child column present in complex type column is string or varchar type * * @param schemas * @return */ def validateChildColumns(schemas: mutable.Buffer[ColumnSchema], - complexColumn: String): Boolean = { - var childColumnCount = 0 - var numberOfPrimitiveColumns = 0 - schemas.foreach { column => - if (childColumnCount > 0) { - if (column.getDataType.equals(DataTypes.STRING)) { - primitiveComplexChildColumns.add(column.getColumnName) - numberOfPrimitiveColumns += 1 - childColumnCount -= 1 - } else { - childColumnCount -= 1 + colCount: Int, traverse: Int): Boolean = { + trav = traverse + var column: ColumnSchema = null + for (i <- 0 until colCount) { + column = schemas(trav) + if (column.getNumberOfChild > 0) { + validateChildColumns(schemas, column.getNumberOfChild, trav. +(1)) + } else { + if (column.isDimensionColumn && (column.getDataType.equals(DataTypes.STRING) || + column.getDataType.equals(DataTypes.VARCHAR))) { + countOfDictCols += 1 } - } - if ((localDictColumns.exists(x => x.equalsIgnoreCase(column.getColumnName)) || - primitiveComplexChildColumns.contains(column.getColumnName)) && - column.getNumberOfChild > 0) { - childColumnCount = column.getNumberOfChild + trav = trav + 1 } } - if (numberOfPrimitiveColumns > 0) { + if (countOfDictCols > 0) { return true } false
