[CARBONDATA-2585][CARBONDATA-2586][Local Dictionary]Added test cases for local dictionary support for alter table, set, unset and preaggregate
Added test cases for local dictionary support for alter table, set, unset and pre-aggregate All the validations related to above features are taken care in this PR This closes #2422 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/806e9b5a Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/806e9b5a Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/806e9b5a Branch: refs/heads/master Commit: 806e9b5a83ba03098a0c4bf42f335016adca5c68 Parents: a7c4b48 Author: brijoobopanna <[email protected]> Authored: Wed Jun 27 14:52:46 2018 +0530 Committer: kunal642 <[email protected]> Committed: Fri Jul 6 14:46:28 2018 +0530 ---------------------------------------------------------------------- .../ColumnLocalDictionaryGenerator.java | 7 +- .../apache/carbondata/core/util/CarbonUtil.java | 145 ++- ...CreateTableWithLocalDictionaryTestCase.scala | 64 +- .../LocalDictionarySupportAlterTableTest.scala | 1127 ++++++++++++++++++ .../LocalDictionarySupportCreateTableTest.scala | 359 +----- .../carbondata/spark/util/CarbonScalaUtil.scala | 5 +- .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 5 +- .../command/carbonTableSchemaCommon.scala | 162 ++- .../CarbonAlterTableDropColumnCommand.scala | 10 + .../table/CarbonDescribeFormattedCommand.scala | 28 +- .../org/apache/spark/util/AlterTableUtil.scala | 11 +- .../vectorreader/DropColumnTestCases.scala | 9 + 12 files changed, 1485 insertions(+), 447 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/806e9b5a/core/src/main/java/org/apache/carbondata/core/localdictionary/generator/ColumnLocalDictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/localdictionary/generator/ColumnLocalDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/localdictionary/generator/ColumnLocalDictionaryGenerator.java index 4c2665f..8aa2e19 100644 --- a/core/src/main/java/org/apache/carbondata/core/localdictionary/generator/ColumnLocalDictionaryGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/localdictionary/generator/ColumnLocalDictionaryGenerator.java @@ -39,7 +39,12 @@ public class ColumnLocalDictionaryGenerator implements LocalDictionaryGenerator this.dictionaryHolder = new MapBasedDictionaryStore(newThreshold); ByteBuffer byteBuffer = ByteBuffer.allocate( lvLength + CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length); - byteBuffer.putShort((short)CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length); + + if (lvLength == CarbonCommonConstants.SHORT_SIZE_IN_BYTE) { + byteBuffer.putShort((short) CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length); + } else { + byteBuffer.putInt(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length); + } byteBuffer.put(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY); // for handling null values try { http://git-wip-us.apache.org/repos/asf/carbondata/blob/806e9b5a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index e7bef93..9423e28 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -3022,16 +3022,16 @@ public final class CarbonUtil { * count and then sets the primitive child column as local dictionary column if it is string * datatype column or varchar datatype column * Handling for both localDictionary Include and exclude columns: - * There will be basically four scenarios which are + * There will basically be four scenarios which are * ------------------------------------------------------- * | Local_Dictionary_include | Local_Dictionary_Exclude | * ------------------------------------------------------- * | Not Defined | Not Defined | - * | Not Defined | Defined | + * | Not Defined | Defined | * | Defined | Not Defined | - * | Defined | Defined | + * | Defined | Defined | * ------------------------------------------------------- - * 1. when the both local dictionary include and exclude is not defined, then set all the no + * 1. when both local dictionary include and exclude are not defined, then set all the no * dictionary string datatype columns as local dictionary generate columns * 2. set all the no dictionary string and varchar datatype columns as local dictionary columns * except the columns present in local dictionary exclude @@ -3058,34 +3058,27 @@ public final class CarbonUtil { } if (null != isLocalDictEnabledForMainTable && Boolean .parseBoolean(isLocalDictEnabledForMainTable)) { - int childColumnCount = 0; - int excludeChildColumnCount = 0; - for (ColumnSchema column : columns) { - // for complex type columns, user gives the parent column as local dictionary column and - // only the string primitive type child column will be set as local dictionary column in the - // schema - if (childColumnCount > 0) { - if (column.getDataType().equals(DataTypes.STRING) || column.getDataType() - .equals(DataTypes.VARCHAR)) { - column.setLocalDictColumn(true); - childColumnCount -= 1; - } else { - childColumnCount -= 1; - } - } - // if complex column is defined in local dictionary include column, then get the child - // columns and set the string datatype child type as local dictionary column - if (column.getNumberOfChild() > 0 && null != localDictIncludeColumns) { - for (String dictColumn : listOfDictionaryIncludeColumns) { - if (dictColumn.trim().equalsIgnoreCase(column.getColumnName())) { - childColumnCount = column.getNumberOfChild(); - } - } - } + int ordinal = 0; + for (int i = 0; i < columns.size(); i++) { + ColumnSchema column = columns.get(i); if (null == localDictIncludeColumns) { // if local dictionary exclude columns is not defined, then set all the no dictionary - // string datatype column and varchar datatype column + // string datatype column and varchar datatype columns if (null == localDictExcludeColumns) { + // if column is complex type, call the setLocalDictForComplexColumns to set local + // dictionary for all string and varchar child columns + if (column.getDataType().isComplexType()) { + ordinal = i + 1; + ordinal = setLocalDictForComplexColumns(columns, ordinal, column.getNumberOfChild()); + i = ordinal - 1; + } else { + ordinal = i; + } + if (ordinal < columns.size()) { + column = columns.get(ordinal); + } else { + continue; + } // column should be no dictionary string datatype column or varchar datatype column if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column.getDataType().equals(DataTypes.VARCHAR)) && !column @@ -3095,16 +3088,32 @@ public final class CarbonUtil { // if local dictionary exclude columns is defined, then set for all no dictionary string // datatype columns and varchar datatype columns except excluded columns } else { - // if complex column is present in exclude column, no need to check for child column, - // just continue - if (excludeChildColumnCount > 0) { - excludeChildColumnCount -= 1; - continue; + if (!Arrays.asList(listOfDictionaryExcludeColumns).contains(column.getColumnName()) + && column.getDataType().isComplexType()) { + ordinal = i + 1; + ordinal = setLocalDictForComplexColumns(columns, ordinal, column.getNumberOfChild()); + i = ordinal - 1; + } else if ( + // if complex column is defined in Local Dictionary Exclude, then + // unsetLocalDictForComplexColumns is mainly used to increment the ordinal value + // required for traversing + Arrays.asList(listOfDictionaryExcludeColumns).contains(column.getColumnName()) + && column.getDataType().isComplexType()) { + ordinal = i + 1; + ordinal = + unsetLocalDictForComplexColumns(columns, ordinal, column.getNumberOfChild()); + i = ordinal - 1; + } else { + ordinal = i; + } - if (Arrays.asList(listOfDictionaryExcludeColumns).contains(column.getColumnName()) - && column.getNumberOfChild() > 0) { - excludeChildColumnCount = column.getNumberOfChild(); + if (ordinal < columns.size()) { + column = columns.get(ordinal); + } else { + continue; } + //if column is primitive string or varchar and no dictionary column,then set local + // dictionary if not specified as local dictionary exclude if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column.getDataType().equals(DataTypes.VARCHAR)) && !column .hasEncoding(Encoding.DICTIONARY)) { @@ -3114,8 +3123,24 @@ public final class CarbonUtil { } } } else { + // if column is complex type, call the setLocalDictForComplexColumns to set local + // dictionary for all string and varchar child columns which are defined in + // local dictionary include + if (localDictIncludeColumns.contains(column.getColumnName()) && column.getDataType() + .isComplexType()) { + ordinal = i + 1; + ordinal = setLocalDictForComplexColumns(columns, ordinal, column.getNumberOfChild()); + i = ordinal - 1; + } else { + ordinal = i; + } // if local dict columns are configured, set for all no dictionary string datatype or // varchar type column + if (ordinal < columns.size()) { + column = columns.get(ordinal); + } else { + continue; + } if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column .getDataType().equals(DataTypes.VARCHAR)) && !column.hasEncoding(Encoding.DICTIONARY) && localDictIncludeColumns.toLowerCase() @@ -3132,6 +3157,52 @@ public final class CarbonUtil { } /** + * traverse through the columns of complex column specified in local dictionary include, + * and set local dictionary for all the string and varchar child columns + * @param allColumns + * @param dimensionOrdinal + * @param childColumnCount + * @return + */ + private static int setLocalDictForComplexColumns(List<ColumnSchema> allColumns, + int dimensionOrdinal, int childColumnCount) { + for (int i = 0; i < childColumnCount; i++) { + ColumnSchema column = allColumns.get(dimensionOrdinal); + if (column.getNumberOfChild() > 0) { + dimensionOrdinal++; + setLocalDictForComplexColumns(allColumns, dimensionOrdinal, column.getNumberOfChild()); + } else { + if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column + .getDataType().equals(DataTypes.VARCHAR)) && !column.hasEncoding(Encoding.DICTIONARY)) { + column.setLocalDictColumn(true); + } + } + dimensionOrdinal++; + } + return dimensionOrdinal; + } + + /** + * traverse through the columns of complex column specified in local dictionary exclude + * @param allColumns + * @param dimensionOrdinal + * @param childColumnCount + * @return + */ + private static int unsetLocalDictForComplexColumns(List<ColumnSchema> allColumns, + int dimensionOrdinal, int childColumnCount) { + for (int i = 0; i < childColumnCount; i++) { + ColumnSchema column = allColumns.get(dimensionOrdinal); + if (column.getNumberOfChild() > 0) { + dimensionOrdinal++; + unsetLocalDictForComplexColumns(allColumns, dimensionOrdinal, column.getNumberOfChild()); + } + dimensionOrdinal++; + } + return dimensionOrdinal++; + } + + /** * This method prepares a map which will have column and local dictionary generator mapping for * all the local dictionary columns. * http://git-wip-us.apache.org/repos/asf/carbondata/blob/806e9b5a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala index 6489ac8..b787d70 100644 --- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.carbondata.cluster.sdv.generated import org.apache.spark.sql.test.util.QueryTest @@ -76,8 +93,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " + - "create table statement")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please " + + "check the DDL.")) } test("test local dictionary custom configurations for local dict columns _004") { @@ -93,9 +110,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception1.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " + - "create table " + - "statement")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + + "Please check the DDL.")) } test("test local dictionary custom configurations for local dict columns _005") { @@ -226,7 +242,6 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) } - sql("desc formatted local1").show(truncate = false) descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { case Some(row) => assert(row.get(1).toString.contains("20000")) } @@ -250,7 +265,6 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) } - sql("desc formatted local1").show(truncate = false) descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { case Some(row) => assert(row.get(1).toString.contains("10000")) } @@ -274,7 +288,6 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) } - sql("desc formatted local1").show(truncate = false) descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { case Some(row) => assert(row.get(1).toString.contains("10000")) } @@ -298,7 +311,6 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) } - sql("desc formatted local1").show(truncate = false) descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { case Some(row) => assert(row.get(1).toString.contains("10000")) } @@ -471,8 +483,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " + - "create table statement")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. " + + "Please check the DDL.")) } @@ -489,9 +501,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception1.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " + - "create table " + - "statement")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + + "Please check the DDL.")) } test("test local dictionary custom configurations when enabled for local dict columns _005") { @@ -572,7 +583,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft assert(exception.getMessage .contains( "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " + - "create table statement")) + "the DDL.")) } @@ -589,9 +600,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception1.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " + - "create table " + - "statement")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + + "Please check the DDL.")) } test("test local dictionary custom configurations when local_dictionary_exclude is configured _005") { @@ -880,7 +890,6 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) } - sql("desc formatted local1").show(truncate = false) descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { case Some(row) => assert(row.get(1).toString.contains("20000")) } @@ -906,7 +915,6 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) } - sql("desc formatted local1").show(truncate = false) descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { case Some(row) => assert(row.get(1).toString.contains("10000")) } @@ -932,7 +940,6 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) } - sql("desc formatted local1").show(truncate = false) descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { case Some(row) => assert(row.get(1).toString.contains("10000")) } @@ -958,7 +965,6 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { case Some(row) => assert(row.get(1).toString.contains("true")) } - sql("desc formatted local1").show(truncate = false) descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { case Some(row) => assert(row.get(1).toString.contains("10000")) } @@ -1645,7 +1651,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft assert(exception.getMessage .contains( "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " + - "create table statement")) + "the DDL.")) } @@ -1668,9 +1674,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception1.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " + - "create table " + - "statement")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + + "Please check the DDL.")) } test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _005") { @@ -1779,7 +1784,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft assert(exception.getMessage .contains( "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " + - "create table statement")) + "the DDL.")) } test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _004") { @@ -1801,9 +1806,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } assert(exception1.getMessage .contains( - "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " + - "create table " + - "statement")) + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + + "Please check the DDL.")) } test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _005") { http://git-wip-us.apache.org/repos/asf/carbondata/blob/806e9b5a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala new file mode 100644 index 0000000..341a28a --- /dev/null +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala @@ -0,0 +1,1127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.spark.testsuite.localdictionary + +import org.apache.spark.sql.test.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException + +class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfterAll{ + + override protected def beforeAll(): Unit = { + sql("DROP TABLE IF EXISTS LOCAL1") + } + + test("test alter table add column") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name') + """.stripMargin) + sql("alter table local1 add columns (alt string) tblproperties('local_dictionary_include'='alt')") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("20000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city,alt")) + } + } + + test("test alter table add column default configs for local dictionary") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','no_inverted_index'='name') + """.stripMargin) + sql("alter table local1 add columns (alt string)") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("20000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name,city,alt")) + } + } + + test("test alter table add column where same column is in dictionary include and local dictionary include") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name') + """.stripMargin) + val exception = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string) tblproperties('local_dictionary_include'='alt','dictionary_include'='alt')") + } + assert(exception.getMessage + .contains( + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: alt specified in Dictionary " + + "include. Local Dictionary will not be generated for Dictionary include columns. " + + "Please check the DDL.")) + } + + test("test alter table add column where duplicate columns present in local dictionary include") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name') + """.stripMargin) + val exception = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string) tblproperties('local_dictionary_include'='alt,alt')") + } + assert(exception.getMessage + .contains( + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE contains Duplicate Columns: alt. " + + "Please check the DDL.")) + } + + test("test alter table add column where duplicate columns present in local dictionary include/exclude") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','local_dictionary_include'='city', + | 'no_inverted_index'='name') + """.stripMargin) + val exception1 = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string) tblproperties" + + "('local_dictionary_include'='abc')") + } + assert(exception1.getMessage + .contains( + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + + "Please check the DDL.")) + val exception2 = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string) tblproperties" + + "('local_dictionary_exclude'='abc')") + } + assert(exception2.getMessage + .contains( + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " + + "Please check the DDL.")) + } + + test("test alter table add column for datatype validation") { + sql("drop table if exists local1") + sql( + """ | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_include'='city', 'no_inverted_index'='name') + """.stripMargin) + val exception = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string,abc int) tblproperties" + + "('local_dictionary_include'='abc')") + } + assert(exception.getMessage + .contains( + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc is not a String/complex " + + "datatype column. LOCAL_DICTIONARY_COLUMN should be no dictionary string/complex datatype" + + " column.Please check the DDL.")) + } + + test("test alter table add column where duplicate columns are present in local dictionary include and exclude") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_include'='city', 'no_inverted_index'='name') + """.stripMargin) + val exception = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string,abc string) tblproperties" + + "('local_dictionary_include'='abc','local_dictionary_exclude'='alt,abc')") + } + assert(exception.getMessage + .contains( + "Column ambiguity as duplicate column(s):abc is present in LOCAL_DICTIONARY_INCLUDE " + + "and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed.")) + } + + test("test alter table add column unsupported table property") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_include'='city', 'no_inverted_index'='name') + """.stripMargin) + val exception = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string,abc string) tblproperties" + + "('local_dictionary_enable'='abc')") + } + assert(exception.getMessage + .contains( + "Unsupported Table property in add column: local_dictionary_enable")) + val exception1 = intercept[MalformedCarbonCommandException] { + sql( + "alter table local1 add columns (alt string,abc string) tblproperties" + + "('local_dictionary_threshold'='10000')") + } + assert(exception1.getMessage + .contains( + "Unsupported Table property in add column: local_dictionary_threshold")) + } + + test("test alter table add column when main table is disabled for local dictionary") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false', + | 'local_dictionary_include'='city', 'no_inverted_index'='name') + """.stripMargin) + sql( + "alter table local1 add columns (alt string,abc string) tblproperties" + + "('local_dictionary_include'='abc')") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("false")) + } + + checkExistence(sql("DESC FORMATTED local1"), false, + "Local Dictionary Include") + } + + test("test local dictionary threshold for boundary values") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_threshold'='300000') + """.stripMargin) + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_threshold'='500') + """.stripMargin) + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + } + + test("test alter table add column for local dictionary include and exclude configs") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_include'='city', 'no_inverted_index'='name') + """.stripMargin) + sql( + "alter table local1 add columns (alt string,abc string) tblproperties" + + "('local_dictionary_include'='abc','local_dictionary_exclude'='alt')") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city,abc")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("alt")) + } + } + + test("test local dictionary foer varchar datatype columns") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_include'='city', + | 'LONG_STRING_COLUMNS'='city') + """.stripMargin) + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + } + + test("test local dictionary describe formatted only with default configs") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' + """.stripMargin) + + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name,city")) + } + } + + test("test local dictionary for invalid threshold") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_threshold'='300000') + """.stripMargin) + + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + } + + test("test alter set for local dictionary enable to disable") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_threshold'='300000') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc1.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("10000")) + } + sql("alter table local1 set tblproperties('local_dictionary_enable'='false')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("false")) + } + checkExistence(sql("DESC FORMATTED local1"), false, + "Local Dictionary Threshold") + } + + test("test alter set for local dictionary _001") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + sql("alter table local1 set tblproperties('local_dictionary_threshold'='30000')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc2.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("30000")) + } + } + + + test("test alter set for local dictionary _002") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + sql("alter table local1 set tblproperties('local_dictionary_include'='name')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name") && !row.get(1).toString.contains("city")) + } + } + + test("test alter set for local dictionary _003") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + sql("alter table local1 set tblproperties('local_dictionary_exclude'='name')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("name")) + } + } + + test("test alter set for local dictionary _004") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + sql("alter table local1 set tblproperties('local_dictionary_include'='city')") + + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + } + + test("test alter set for local dictionary _005") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + sql("alter table local1 set tblproperties('local_dictionary_include'='name,city')") + + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name,city")) + } + } + + test("test alter set for local dictionary _006") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + sql("alter table local1 set tblproperties('local_dictionary_exclude'='city')") + + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name")) + } + descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + } + + test("test alter set for local dictionary _007") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + val exception1 = intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_include'='city, ')") + } + assert(exception1.getMessage + .contains( + "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. " + + "Please check the DDL.")) + } + + test("test alter set for local dictionary _008") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + val exception1 = intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_exclude'='name')") + } + assert(exception1.getMessage.contains("Column ambiguity as duplicate column(s):name is present in LOCAL_DICTIONARY_INCLUDE and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed.")) + } + + test("test alter set for local dictionary _009") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + val exception1 = intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_exclude'='id')") + } + assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE should be no dictionary string/complex datatype column.")) + } + + test("test alter set for local dictionary _010") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('dictionary_include'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + val exception1 = intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_include'='name')") + } + assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: name specified in Dictionary include. Local Dictionary will not be generated for Dictionary include columns. Please check the DDL.")) + } + + test("test alter set for local dictionary _011") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_exclude'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + sql("alter table local1 set tblproperties('local_dictionary_exclude'='city')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name")) + } + descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + } + + test("test alter set for local dictionary _012") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_exclude'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + val exception1 = intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_exclude'='city, ')") + } + assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check the DDL.")) + } + + test("test alter set for local dictionary _013") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('dictionary_include'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + val exception1 = intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_exclude'='name')") + } + assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: name specified in Dictionary include. Local Dictionary will not be generated for Dictionary include columns. Please check the DDL.")) + } + + test("test alter set for local dictionary _014") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + val exception1 = intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_exclude'='name')") + } + assert(exception1.getMessage.contains("Column ambiguity as duplicate column(s):name is present in LOCAL_DICTIONARY_INCLUDE and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed.")) + } + + test("test alter set for local dictionary _015") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + sql("alter table local1 set tblproperties('local_dictionary_enable'='false')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("false")) + } + } + + test("test alter set for local dictionary _016") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='false') + """.stripMargin) + + sql("alter table local1 set tblproperties('local_dictionary_include'='name')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("false")) + } + } + + test("test alter set for local dictionary _017") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name','local_dictionary_exclude'='city') + """.stripMargin) + + val exception1 = intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_exclude'='name')") + } + assert(exception1.getMessage.contains("Column ambiguity as duplicate column(s):name is present in LOCAL_DICTIONARY_INCLUDE and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed.")) + } + + test("test alter unset for local dictionary") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name','local_dictionary_exclude'='city') + """.stripMargin) + + sql("alter table local1 unset tblproperties('local_dictionary_exclude')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name")) + } + } + + test("test alter set for local dictionary disable to enable") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='300000') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("false")) + } + checkExistence(sql("DESC FORMATTED local1"), false, + "Local Dictionary Threshold") + sql("alter table local1 set tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='30000')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc2.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("30000")) + } + } + + test("test alter set same column for local dictionary exclude and include") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'carbondata' tblproperties('local_dictionary_exclude'='city') + """.stripMargin) + intercept[Exception] { + sql( + "alter table local1 set tblproperties('local_dictionary_include'='name'," + + "'local_dictionary_exclude'='name')") + } + } + + test("test alter set for valid and invalid complex type as include/exclude") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int,st struct<s_id:int, + | s_name:string,s_city:array<string>>, dcity array<string>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city,dcity', + | 'local_dictionary_enable'='true') + """. + stripMargin) + val descFormatted1 = sql("describe formatted local1").collect + descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert( + row.get(1).toString.contains("city,dcity") && !row.get(1).toString.contains("name")) + } + intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_exclude'='dcity')") + } + sql("alter table local1 set tblproperties('local_dictionary_exclude'='st')") + val descFormatted2 = sql("describe formatted local1").collect + descFormatted2.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("st")) + } + descFormatted2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city,dcity")) + } + sql("alter table local1 set tblproperties('local_dictionary_exclude'='st'," + + "'local_dictionary_include'='dcity')") + val descFormatted3 = sql("describe formatted local1").collect + descFormatted3.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("st")) + } + descFormatted3.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("dcity")) + } + } + + test("test alter set for invalid complex type as include/exclude") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int,st struct<s_id:int, + | s_name:int,s_city:array<int>>, dcity array<int>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('local_dictionary_exclude'='name', + | 'local_dictionary_enable'='true') + """. + stripMargin) + intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_exclude'='dcity')") + } + intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_include'='st')") + } + } + + test("test alter unset for local dictionary disable") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int,add string) + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='false') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Enable")) match { + case Some(row) => assert(row.get(1).toString.contains("false")) + } + sql("alter table local1 unset tblproperties('local_dictionary_enable')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Enable")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + } + + test("test alter unset for local dictionary enable local dict include") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int,add string) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='city') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + sql("alter table local1 unset tblproperties('local_dictionary_include')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name,city")) + } + } + + test("test alter unset for local dictionary enable local dict exclude") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int,add string) + | STORED BY 'carbondata' tblproperties('local_dictionary_include'='city', + | 'local_dictionary_exclude'='name') + """.stripMargin) + + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + sql("alter table local1 unset tblproperties('local_dictionary_exclude')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + } + + test("test alter unset for valid/invalid complex type as include/exclude") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int,st struct<s_id:int, + | s_name:string,s_city:array<int>>, dcity array<string>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('local_dictionary_exclude'='st','local_dictionary_include'='city', + | 'local_dictionary_enable'='true') + """. + stripMargin) + val descLoc1 = sql("describe formatted local1").collect + descLoc1.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city")) + } + descLoc1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert( + row.get(1).toString.contains("st.s_name") && !row.get(1).toString.contains("st.s_id")) + } + sql("alter table local1 unset tblproperties('local_dictionary_exclude')") + sql("alter table local1 unset tblproperties('local_dictionary_include')") + val descLoc2 = sql("describe formatted local1").collect + descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name,city,st.s_name,dcity.val")) + } + } + + test("test alter table drop column for local dictionary include") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name struct<n:int,m:string>, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','no_inverted_index'='name', + | 'local_dictionary_include'='name,city') + """.stripMargin) + sql("alter table local1 drop columns (city)") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("20000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name") && !row.get(1).toString.contains("city")) + } + } + + test("test alter table drop column for local dictionary exclude") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','no_inverted_index'='name', + | 'local_dictionary_exclude'='name,city') + """.stripMargin) + sql("alter table local1 drop columns (name)") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("20000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("city") && !row.get(1).toString.contains("name")) + } + } + + test("test alter set case sensitivity") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','no_inverted_index'='name', + | 'local_dictionary_include'='city') + """.stripMargin) + intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_include'='city, CiTy')") + } + intercept[Exception] { + sql("alter table local1 set tblproperties('local_dictionary_include'='naMe , NaMe')") + } + } + + test("test alter add and drop columns") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','no_inverted_index'='name') + """.stripMargin) + sql("alter table local1 add columns(add1 string,add2 string) tblproperties('local_dictionary_exclude'='add1')") + sql("alter table local1 drop columns (add1)") + sql("alter table local1 add columns(add1 string) tblproperties('local_dictionary_include'='add1')") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("20000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name,city,add2,add1")) + } + } + + test("test alter add columns and set properties") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','no_inverted_index'='name') + """.stripMargin) + sql("alter table local1 add columns(add1 string) tblproperties('local_dictionary_include'='add1')") + sql("alter table local1 set tblproperties('local_dictionary_include'='name')") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { + case Some(row) => assert(row.get(1).toString.contains("20000")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("name") && !row.get(1).toString.contains("city,add2,add1")) + } + } + + test("test alter add columns and set properties _002") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int) + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true', + | 'local_dictionary_threshold'='20000','no_inverted_index'='name') + """.stripMargin) + sql("alter table local1 add columns(add1 string) tblproperties('local_dictionary_include'='add1')") + sql("alter table local1 set tblproperties('local_dictionary_enable'='false')") + val descLoc = sql("describe formatted local1").collect + descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("false")) + } + } + + test("test alter set on complex columns __001") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int,st struct<s_id:int, + | s_name:string,s_city:array<string>>, dcity array<string>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('local_dictionary_enable'='true') + """. + stripMargin) + sql("alter table local1 unset tblproperties('local_dictionary_exclude')") + sql("alter table local1 set tblproperties('local_dictionary_include'='st')") + + val descFormatted1 = sql("describe formatted local1").collect + descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert( + row.get(1).toString.contains("st.s_name,st.s_city.val") && + !row.get(1).toString.contains("dcity")) + } + sql("alter table local1 unset tblproperties('local_dictionary_include')") + sql("alter table local1 set tblproperties('local_dictionary_exclude'='st')") + val descFormatted2 = sql("describe formatted local1").collect + descFormatted2.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("st")) + } + descFormatted2.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("city,dcity")) + } + sql("alter table local1 set tblproperties('local_dictionary_exclude'='st'," + + "'local_dictionary_include'='dcity')") + val descFormatted3 = sql("describe formatted local1").collect + descFormatted3.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert(row.get(1).toString.contains("st")) + } + descFormatted3.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert(row.get(1).toString.contains("dcity")) + } + } + + test("test alter set on complex columns __002") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int,st struct<s_id:int, + | s_name:string,s_city:array<string>>, dcity array<string>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('local_dictionary_include'='st','local_dictionary_enable'='true') + """. + stripMargin) + sql("alter table local1 set tblproperties('local_dictionary_include'='dcity')") + + val descFormatted1 = sql("describe formatted local1").collect + descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert( + row.get(1).toString.contains("dcity.val") && + !row.get(1).toString.contains("st")) + } + } + + test("test alter set on complex columns __003") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int,st struct<s_id:int, + | s_city:array<string>,s_name:string>, dcity array<string>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('local_dictionary_include'='st','local_dictionary_enable'='true') + """. + stripMargin) + sql("alter table local1 unset tblproperties('local_dictionary_include')") + sql("alter table local1 set tblproperties('local_dictionary_exclude'='st')") + val descFormatted1 = sql("describe formatted local1").collect + descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert( + row.get(1).toString.contains("name,city,dcity.val") && + !row.get(1).toString.contains("st")) + } + descFormatted1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match { + case Some(row) => assert( + row.get(1).toString.contains("st.s_city.val,st.s_name")) + } + } + + test("test alter set on complex columns __004") { + sql("drop table if exists local1") + sql( + """ + | CREATE TABLE local1(id int, name string, city string, age int,st struct<s_id:int, + | s_city:array<string>,s_name:string>, dcity array<string>) + | STORED BY 'org.apache.carbondata.format' + | tblproperties('local_dictionary_include'='st,city,name','local_dictionary_exclude'='dcity','local_dictionary_enable'='true') + """. + stripMargin) + sql("alter table local1 unset tblproperties('local_dictionary_exclude')") + sql("alter table local1 set tblproperties('local_dictionary_include'='dcity')") + val descFormatted1 = sql("describe formatted local1").collect + descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { + case Some(row) => assert(row.get(1).toString.contains("true")) + } + descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match { + case Some(row) => assert( + row.get(1).toString.contains("dcity.val")) + } + } + + + override protected def afterAll(): Unit = { + sql("DROP TABLE IF EXISTS LOCAL1") + } + +}
