Repository: carbondata Updated Branches: refs/heads/master 9a3b0b16b -> 34ca02142
[CARBONDATA-2585]disable local dictionary by default make local dictionary false by default This closes #2570 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/34ca0214 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/34ca0214 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/34ca0214 Branch: refs/heads/master Commit: 34ca021423cff8aa2dd03f98f99ea2c46a5aa9a6 Parents: 9a3b0b1 Author: akashrn5 <[email protected]> Authored: Fri Jul 27 12:31:48 2018 +0530 Committer: ravipesala <[email protected]> Committed: Tue Jul 31 21:35:26 2018 +0530 ---------------------------------------------------------------------- .../core/constants/CarbonCommonConstants.java | 2 +- docs/data-management-on-carbondata.md | 2 +- ...CreateTableWithLocalDictionaryTestCase.scala | 70 ++++++++------------ .../TestNonTransactionalCarbonTable.scala | 43 ++---------- ...ransactionalCarbonTableWithComplexType.scala | 10 +-- .../describeTable/TestDescribeTable.scala | 4 +- .../LocalDictionarySupportAlterTableTest.scala | 52 +++++++-------- .../LocalDictionarySupportCreateTableTest.scala | 58 ++++++++-------- .../LocalDictionarySupportLoadTableTest.scala | 14 ++-- .../table/CarbonDescribeFormattedCommand.scala | 3 +- 10 files changed, 98 insertions(+), 160 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index 8bf22c9..6d7215e 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -929,7 +929,7 @@ public final class CarbonCommonConstants { /** * default value for local dictionary generation */ - public static final String LOCAL_DICTIONARY_ENABLE_DEFAULT = "true"; + public static final String LOCAL_DICTIONARY_ENABLE_DEFAULT = "false"; /** * Threshold value for local dictionary http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/docs/data-management-on-carbondata.md ---------------------------------------------------------------------- diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md index 28bc7d3..6aaaaa3 100644 --- a/docs/data-management-on-carbondata.md +++ b/docs/data-management-on-carbondata.md @@ -137,7 +137,7 @@ This tutorial is going to introduce all commands and data operations on CarbonDa | Properties | Default value | Description | | ---------- | ------------- | ----------- | - | LOCAL_DICTIONARY_ENABLE | true | By default, local dictionary will be enabled for the table | + | LOCAL_DICTIONARY_ENABLE | false | By default, local dictionary will not be enabled for the table | | LOCAL_DICTIONARY_THRESHOLD | 10000 | The maximum cardinality for local dictionary generation (range- 1000 to 100000) | | LOCAL_DICTIONARY_INCLUDE | all no-dictionary string/varchar columns | Columns for which Local Dictionary is generated. | | LOCAL_DICTIONARY_EXCLUDE | none | Columns for which Local Dictionary is not generated | http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala index b6ecef9..43d5956 100644 --- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala @@ -40,10 +40,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) - } - descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) + case Some(row) => assert(row.get(1).toString.contains("false")) } } @@ -53,7 +50,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name') """. stripMargin) val descFormatted1 = sql("describe formatted local1").collect @@ -65,21 +62,6 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft } } - test( - "test local dictionary custom configurations for local dict columns _002") - { - sql("drop table if exists local1") - - intercept[MalformedCarbonCommandException] { - sql( - """ - | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_include'='name,name') - """.stripMargin) - } - } - test("test local dictionary custom configurations for local dict columns _003") { sql("drop table if exists local1") val exception = intercept[MalformedCarbonCommandException] { @@ -87,7 +69,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_include'='') + | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='') """. stripMargin) } @@ -104,7 +86,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_include'='abc') + | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='abc') """. stripMargin) } @@ -121,7 +103,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_include'='id') + | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='id') """. stripMargin) } @@ -139,7 +121,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('dictionary_include'='name','local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','dictionary_include'='name','local_dictionary_include'='name') """. stripMargin) } @@ -151,7 +133,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -170,7 +152,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='-100') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -186,7 +168,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='21474874811') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='21474874811') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -202,7 +184,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -218,7 +200,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='hello') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='hello') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -235,7 +217,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -258,7 +240,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100','local_dictionary_include'='name') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -281,7 +263,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='','local_dictionary_include'='name') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -304,7 +286,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='vdslv','local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='vdslv','local_dictionary_include'='name') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -328,7 +310,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name,name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name,name') """.stripMargin) } } @@ -342,7 +324,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'=' ') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'=' ') """.stripMargin) } } @@ -356,7 +338,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='hello') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='hello') """.stripMargin) } } @@ -370,7 +352,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name', + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name', | 'dictionary_include'='name') """.stripMargin) } @@ -385,7 +367,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name,name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='','local_dictionary_include'='name,name') """.stripMargin) } } @@ -399,7 +381,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='Hello') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100','local_dictionary_include'='Hello') """.stripMargin) } } @@ -413,7 +395,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='23213497321591234324', + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='23213497321591234324', | 'local_dictionary_include'='name','dictionary_include'='name') """.stripMargin) } @@ -1496,7 +1478,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) + case Some(row) => assert(row.get(1).toString.contains("false")) } descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match { case Some(row) => assert(row.get(1).toString.contains("global_sort")) @@ -1516,7 +1498,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) + case Some(row) => assert(row.get(1).toString.contains("false")) } descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match { case Some(row) => assert(row.get(1).toString.contains("batch_sort")) @@ -1535,7 +1517,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) + case Some(row) => assert(row.get(1).toString.contains("false")) } descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match { case Some(row) => assert(row.get(1).toString.contains("no_sort")) @@ -1554,7 +1536,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) + case Some(row) => assert(row.get(1).toString.contains("false")) } descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match { case Some(row) => assert(row.get(1).toString.contains("local_sort")) http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala index 62c3df6..8a1d465 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala @@ -2328,15 +2328,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { |'$writerPath' """.stripMargin) val descLoc = sql("describe formatted sdkTable").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) - case None => assert(false) - } - descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) - case None => assert(false) - } - descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { - case Some(row) => assert(row.get(1).toString.contains("name,surname")) + case Some(row) => assert(row.get(1).toString.contains("false")) case None => assert(false) } FileUtils.deleteDirectory(new File(writerPath)) @@ -2357,15 +2349,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { |'$writerPath' """.stripMargin) val descLoc = sql("describe formatted sdkTable").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) - case None => assert(false) - } - descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) - case None => assert(false) - } - descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { - case Some(row) => assert(row.get(1).toString.contains("name,surname")) + case Some(row) => assert(row.get(1).toString.contains("false")) case None => assert(false) } FileUtils.deleteDirectory(new File(writerPath)) @@ -2386,15 +2370,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { |'$writerPath' """.stripMargin) val descLoc = sql("describe formatted sdkTable").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) - case None => assert(false) - } - descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) - case None => assert(false) - } - descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { - case Some(row) => assert(row.get(1).toString.contains("name,surname")) + case Some(row) => assert(row.get(1).toString.contains("false")) case None => assert(false) } FileUtils.deleteDirectory(new File(writerPath)) @@ -2416,21 +2392,12 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { FileUtils.deleteDirectory(new File(writerPath)) sql("insert into sdkTable select 's1','s2',23 ") assert(FileFactory.getCarbonFile(writerPath).exists()) - assert(testUtil.checkForLocalDictionary(testUtil.getDimRawChunk(0,writerPath))) + assert(!testUtil.checkForLocalDictionary(testUtil.getDimRawChunk(0,writerPath))) val descLoc = sql("describe formatted sdkTable").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) + case Some(row) => assert(row.get(1).toString.contains("false")) case None => assert(false) } - descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) - case None => assert(false) - } - descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { - case Some(row) => assert(row.get(1).toString.contains("name,surname")) - case None => assert(false) - } - checkAnswer(sql("select count(*) from sdkTable"), Seq(Row(1))) FileUtils.deleteDirectory(new File(writerPath)) } http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala index 7593f38..62ba03e 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala @@ -218,15 +218,7 @@ class TestNonTransactionalCarbonTableWithComplexType extends QueryTest with Befo sql("describe formatted localComplex").show(30, false) val descLoc = sql("describe formatted localComplex").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) - case None => assert(false) - } - descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) - case None => assert(false) - } - descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match { - case Some(row) => assert(row.get(1).toString.contains("name,val1.val2.street,val1.val2.city,val1.val2.WindSpeed,val1.val2.year")) + case Some(row) => assert(row.get(1).toString.contains("false")) case None => assert(false) } http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala index 93f1736..d0d89aa 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala @@ -51,10 +51,10 @@ class TestDescribeTable extends QueryTest with BeforeAndAfterAll { test("test describe formatted table desc1") { val resultCol = Seq("", "", "##Detailed Column property", "##Detailed Table Information", "ADAPTIVE", "CARBON Store Path", "Comment", "Database Name", "Last Update Time", - "SORT_COLUMNS", "SORT_SCOPE", "CACHE_LEVEL", "Streaming", "Table Block Size", "Local Dictionary Enabled", "Local Dictionary Include", "Local Dictionary Threshold","Table Data Size", "Table Index Size", "Table Name", "dec2col1", "dec2col2", "dec2col3", "dec2col4") + "SORT_COLUMNS", "SORT_SCOPE", "CACHE_LEVEL", "Streaming", "Table Block Size", "Local Dictionary Enabled","Table Data Size", "Table Index Size", "Table Name", "dec2col1", "dec2col2", "dec2col3", "dec2col4") val resultRow: Seq[Row] = resultCol map(propName => Row(f"$propName%-36s")) checkAnswer(sql("desc formatted DESC1").select("col_name"), resultRow) - assert(sql("desc formatted desc1").count() == 24) + assert(sql("desc formatted desc1").count() == 22) } test("test describe formatted for partition table") { http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala index 47a58ae..373b309 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala @@ -232,7 +232,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_threshold'='300000') + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='300000') """.stripMargin) val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { @@ -243,7 +243,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_threshold'='500') + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='500') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect descLoc1.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { @@ -278,12 +278,12 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter } } - test("test local dictionary foer varchar datatype columns") { + test("test local dictionary for varchar datatype columns") { sql("drop table if exists local1") sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_include'='city', + | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='city', | 'LONG_STRING_COLUMNS'='city') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -302,7 +302,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -325,7 +325,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_threshold'='300000') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='300000') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -340,7 +340,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_threshold'='300000') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='300000') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -367,7 +367,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -393,7 +393,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -418,7 +418,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -447,7 +447,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -473,7 +473,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -499,7 +499,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -529,7 +529,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -551,7 +551,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -570,7 +570,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -589,7 +589,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('dictionary_include'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','dictionary_include'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -608,7 +608,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_exclude'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -637,7 +637,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_exclude'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -656,7 +656,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('dictionary_include'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','dictionary_include'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -675,7 +675,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -694,7 +694,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -745,7 +745,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int) - | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name','local_dictionary_exclude'='city') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name','local_dictionary_exclude'='city') """.stripMargin) sql("alter table local1 unset tblproperties('local_dictionary_exclude')") @@ -878,7 +878,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql("alter table local1 unset tblproperties('local_dictionary_enable')") val descLoc2 = sql("describe formatted local1").collect descLoc2.find(_.get(0).toString.contains("Local Dictionary Enable")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) + case Some(row) => assert(row.get(1).toString.contains("false")) case None => assert(false) } } @@ -888,7 +888,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int,add string) - | STORED BY 'carbondata' tblproperties('local_dictionary_include'='city') + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='city') """.stripMargin) val descLoc1 = sql("describe formatted local1").collect @@ -909,7 +909,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter sql( """ | CREATE TABLE local1(id int, name string, city string, age int,add string) - | STORED BY 'carbondata' tblproperties('local_dictionary_include'='city', + | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='city', | 'local_dictionary_exclude'='name') """.stripMargin) http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala index eec1582..52c18d0 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala @@ -40,11 +40,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) - case None => assert(false) - } - descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match { - case Some(row) => assert(row.get(1).toString.contains("10000")) + case Some(row) => assert(row.get(1).toString.contains("false")) case None => assert(false) } } @@ -55,7 +51,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name') """. stripMargin) val descFormatted1 = sql("describe formatted local1").collect @@ -77,7 +73,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_include'='name,name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name,name') """.stripMargin) } } @@ -89,7 +85,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_include'='') + | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='') """. stripMargin) } @@ -106,7 +102,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_include'='abc') + | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='abc') """. stripMargin) } @@ -123,7 +119,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_include'='id') + | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='id') """. stripMargin) } @@ -142,7 +138,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('dictionary_include'='name','local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','dictionary_include'='name','local_dictionary_include'='name') """. stripMargin) } @@ -154,7 +150,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -174,7 +170,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='-100') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -190,7 +186,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='21474874811') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='21474874811') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -206,7 +202,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -222,7 +218,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='hello') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='hello') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -239,7 +235,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -264,7 +260,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100','local_dictionary_include'='name') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -289,7 +285,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='','local_dictionary_include'='name') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -314,7 +310,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='vdslv','local_dictionary_include'='name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='vdslv','local_dictionary_include'='name') """.stripMargin) val descLoc = sql("describe formatted local1").collect @@ -340,7 +336,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name, + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name, | name') """.stripMargin) } @@ -354,7 +350,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'=' ') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'=' ') """.stripMargin) } } @@ -367,7 +363,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='hello') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='hello') """.stripMargin) } } @@ -380,7 +376,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name', + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name', | 'dictionary_include'='name') """.stripMargin) } @@ -394,7 +390,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name,name') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='','local_dictionary_include'='name,name') """.stripMargin) } } @@ -407,7 +403,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='Hello') + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100','local_dictionary_include'='Hello') """.stripMargin) } } @@ -420,7 +416,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('local_dictionary_threshold'='23213497321591234324', + | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='23213497321591234324', | 'local_dictionary_include'='name','dictionary_include'='name') """.stripMargin) } @@ -1504,7 +1500,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte """ | CREATE TABLE local1(id int, name string, city string, age int) | STORED BY 'org.apache.carbondata.format' - | tblproperties('dictionary_include'='city','sort_scope'='global_sort', + | tblproperties('local_dictionary_enable'='true','dictionary_include'='city','sort_scope'='global_sort', | 'sort_columns'='city,name') """.stripMargin) @@ -1533,7 +1529,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) + case Some(row) => assert(row.get(1).toString.contains("false")) case None => assert(false) } descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match { @@ -1554,7 +1550,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) + case Some(row) => assert(row.get(1).toString.contains("false")) case None => assert(false) } descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match { @@ -1575,7 +1571,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte val descLoc = sql("describe formatted local1").collect descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match { - case Some(row) => assert(row.get(1).toString.contains("true")) + case Some(row) => assert(row.get(1).toString.contains("false")) case None => assert(false) } descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match { http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala index a3ab851..59586c0 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala @@ -65,14 +65,14 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA sql("drop table if exists local2") sql( "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" + - "('local_dictionary_threshold'='9001','local_dictionary_include'='name')") + "('local_dictionary_enable'='true','local_dictionary_threshold'='9001','local_dictionary_include'='name')") sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')") assert(checkForLocalDictionary(getDimRawChunk(0))) } test("test successful local dictionary generation for default configs") { sql("drop table if exists local2") - sql("CREATE TABLE local2(name string) STORED BY 'carbondata'") + sql("CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')") sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')") assert(checkForLocalDictionary(getDimRawChunk(0))) } @@ -90,7 +90,7 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA sql("drop table if exists local2") sql( "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" + - "('dictionary_exclude'='name')") + "('local_dictionary_enable'='true','dictionary_exclude'='name')") sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')") assert(checkForLocalDictionary(getDimRawChunk(0))) } @@ -108,7 +108,7 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA sql("drop table if exists local2") sql( "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" + - "('local_dictionary_include'='name','local_dictionary_threshold'='300000')") + "('local_dictionary_enable'='true','local_dictionary_include'='name','local_dictionary_threshold'='300000')") sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')") assert(checkForLocalDictionary(getDimRawChunk(0))) } @@ -117,7 +117,7 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA sql("drop table if exists local2") sql( "CREATE TABLE local2(name string, age string) STORED BY 'carbondata' tblproperties" + - "('local_dictionary_include'='name', 'local_dictionary_exclude'='age')") + "('local_dictionary_enable'='true','local_dictionary_include'='name', 'local_dictionary_exclude'='age')") sql("insert into table local2 values('vishal', '30')") assert(checkForLocalDictionary(getDimRawChunk(0))) assert(!checkForLocalDictionary(getDimRawChunk(1))) @@ -127,7 +127,7 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA sql("drop table if exists local2") sql( "CREATE TABLE local2(name struct<i:string,s:string>) STORED BY 'carbondata' tblproperties" + - "('local_dictionary_include'='name')") + "('local_dictionary_enable'='true','local_dictionary_include'='name')") sql("load data inpath '" + file2 + "' into table local2 OPTIONS('header'='false','COMPLEX_DELIMITER_LEVEL_1'='$', " + "'COMPLEX_DELIMITER_LEVEL_2'=':')") @@ -138,7 +138,7 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA test("test to validate local dictionary values"){ sql("drop table if exists local2") - sql("CREATE TABLE local2(name string) STORED BY 'carbondata'") + sql("CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')") sql("load data inpath '" + resourcesPath + "/localdictionary.csv" + "' into table local2") val dimRawChunk = getDimRawChunk(0) val dictionaryData = Array("vishal", "kumar", "akash", "praveen", "brijoo") http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala index 41dfea5..9b9e8bd 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala @@ -160,7 +160,8 @@ private[sql] case class CarbonDescribeFormattedCommand( } } } else { - results ++= Seq(("Local Dictionary Enabled", "false", "")) + results ++= + Seq(("Local Dictionary Enabled", CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT, "")) } /**
