Repository: carbondata Updated Branches: refs/heads/master d9f1a8115 -> 4c9f08217
[CARBONDATA-3143] Fixed local dictionary in presto Problem: Currently, local dictionary columns are not working for presto as it is not handled in the integration layer. Solution: Add local dictionary support to presto integration layer. This closes #2972 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4c9f0821 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4c9f0821 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4c9f0821 Branch: refs/heads/master Commit: 4c9f08217c7b9fa7ad33e148dbf33280e0f2b33f Parents: d9f1a81 Author: ravipesala <ravi.pes...@gmail.com> Authored: Mon Dec 3 18:27:33 2018 +0530 Committer: kumarvishal09 <kumarvishal1...@gmail.com> Committed: Mon Dec 10 19:18:32 2018 +0530 ---------------------------------------------------------------------- .../presto/CarbonColumnVectorWrapper.java | 2 +- .../presto/readers/SliceStreamReader.java | 35 +++ .../PrestoAllDataTypeLocalDictTest.scala | 291 +++++++++++++++++++ .../integrationtest/PrestoAllDataTypeTest.scala | 2 +- .../carbondata/presto/server/PrestoServer.scala | 4 +- .../presto/util/CarbonDataStoreCreator.scala | 18 +- 6 files changed, 342 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java index a80751f..f001488 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java @@ -244,7 +244,7 @@ public class CarbonColumnVectorWrapper implements CarbonColumnVector { } @Override public CarbonColumnVector getDictionaryVector() { - return this.columnVector; + return this.columnVector.getDictionaryVector(); } @Override public void putFloats(int rowId, int count, float[] src, int srcIndex) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java index ab270fc..04e5bb3 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java @@ -17,14 +17,19 @@ package org.apache.carbondata.presto.readers; +import java.util.Optional; + import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.scan.result.vector.CarbonDictionary; import org.apache.carbondata.core.scan.result.vector.impl.CarbonColumnVectorImpl; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.VariableWidthBlock; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.VarcharType; +import io.airlift.slice.Slices; import static io.airlift.slice.Slices.wrappedBuffer; @@ -63,6 +68,36 @@ public class SliceStreamReader extends CarbonColumnVectorImpl implements PrestoV } } + @Override public void setDictionary(CarbonDictionary dictionary) { + super.setDictionary(dictionary); + if (dictionary == null) { + dictionaryBlock = null; + return; + } + boolean[] nulls = new boolean[dictionary.getDictionarySize()]; + nulls[0] = true; + nulls[1] = true; + int[] dictOffsets = new int[dictionary.getDictionarySize() + 1]; + int size = 0; + for (int i = 0; i < dictionary.getDictionarySize(); i++) { + if (dictionary.getDictionaryValue(i) != null) { + dictOffsets[i] = size; + size += dictionary.getDictionaryValue(i).length; + } + } + byte[] singleArrayDictValues = new byte[size]; + for (int i = 0; i < dictionary.getDictionarySize(); i++) { + if (dictionary.getDictionaryValue(i) != null) { + System.arraycopy(dictionary.getDictionaryValue(i), 0, singleArrayDictValues, dictOffsets[i], + dictionary.getDictionaryValue(i).length); + } + } + dictOffsets[dictOffsets.length - 1] = size; + dictionaryBlock = new VariableWidthBlock(dictionary.getDictionarySize(), + Slices.wrappedBuffer(singleArrayDictValues), dictOffsets, Optional.of(nulls)); + values = (int[]) ((CarbonColumnVectorImpl) getDictionaryVector()).getDataArray(); + } + @Override public void setBatchSize(int batchSize) { this.batchSize = batchSize; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala new file mode 100644 index 0000000..356c0e8 --- /dev/null +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.presto.integrationtest + +import java.io.File +import java.sql.Timestamp + +import org.apache.hadoop.fs.permission.{FsAction, FsPermission} +import org.scalatest.{BeforeAndAfterAll, FunSuiteLike} + +import org.apache.carbondata.common.logging.LogServiceFactory +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.datastore.impl.FileFactory +import org.apache.carbondata.core.datastore.impl.FileFactory.FileType +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.presto.server.PrestoServer + + +class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll { + + private val logger = LogServiceFactory + .getLogService(classOf[PrestoAllDataTypeLocalDictTest].getCanonicalName) + + private val rootPath = new File(this.getClass.getResource("/").getPath + + "../../../..").getCanonicalPath + private val storePath = s"$rootPath/integration/presto/target/store" + private val systemPath = s"$rootPath/integration/presto/target/system" + private val PrestoServer = new PrestoServer + + // Table schema: + // +-------------+----------------+-------------+------------+ + // | Column name | Data type | Column type | Dictionary | + // +-------------+----------------+--------------+-----------+ + // | id | string | dimension | yes | + // +-------------+----------------+-------------+------------+ + // | date | date | dimension | yes | + // +-------------+----------------+-------------+------------+ + // | country | string | dimension | yes | + // +-------------+----------------+-------------+------------- + // | name | string | dimension | yes | + // +-------------+----------------+-------------+------------- + // | phonetype | string | dimension | yes | + // +-------------+----------------+-------------+------------- + // | serialname | string | dimension | true | + // +-------------+----------------+-------------+------------- + // | bonus |short decimal | measure | false | + // +-------------+----------------+-------------+------------- + // | monthlyBonus| longdecimal | measure | false | + // +-------------+----------------+-------------+------------- + // | dob | timestamp | dimension | true | + // +-------------+----------------+-------------+------------+ + // | shortField | shortfield | measure | true | + // +-------------+----------------+-------------+------------- + // |isCurrentEmp | boolean | measure | true | + // +-------------+----------------+-------------+------------+ + + override def beforeAll: Unit = { + import org.apache.carbondata.presto.util.CarbonDataStoreCreator + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, + systemPath) + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, + "Presto") + CarbonDataStoreCreator + .createCarbonStore(storePath, + s"$rootPath/integration/presto/src/test/resources/alldatatype.csv", true) + logger.info(s"\nCarbon store is created at location: $storePath") + cleanUp + PrestoServer.startServer(storePath) + } + + override def afterAll(): Unit = { + PrestoServer.stopServer() + } + + test("select string type with order by clause") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery("SELECT NAME FROM TESTDB.TESTTABLE ORDER BY NAME") + val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "akash"), + Map("NAME" -> "anubhav"), + Map("NAME" -> "bhavya"), + Map("NAME" -> "geetika"), + Map("NAME" -> "jatin"), + Map("NAME" -> "jitesh"), + Map("NAME" -> "liang"), + Map("NAME" -> "prince"), + Map("NAME" -> "ravindra"), + Map("NAME" -> "sahil"), + Map("NAME" -> null) + ) + assert(actualResult.equals(expectedResult)) + } + + test("test and filter clause with greater than expression") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery( + "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " + + "WHERE BONUS>1234 AND ID>2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," + + "BONUS ORDER BY ID") + val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 4, + "NAME" -> "prince", + "BONUS" -> java.math.BigDecimal.valueOf(9999.9990).setScale(4), + "DATE" -> "2015-07-26", + "SALARY" -> 15003.0, + "SERIALNAME" -> "ASD66902", + "COUNTRY" -> "china", + "PHONETYPE" -> "phone2435"), + Map("ID" -> 5, + "NAME" -> "bhavya", + "BONUS" -> java.math.BigDecimal.valueOf(5000.999).setScale(4), + "DATE" -> "2015-07-27", + "SALARY" -> 15004.0, + "SERIALNAME" -> "ASD90633", + "COUNTRY" -> "china", + "PHONETYPE" -> "phone2441")) + assert(actualResult.toString() equals expectedResult.toString()) + + + } + + test("test and filter clause with greater than equal to expression") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery( + "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " + + "WHERE BONUS>=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," + + "BONUS ORDER BY ID") + val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1, + "NAME" -> "anubhav", + "BONUS" -> java.math.BigDecimal.valueOf(1234.4440).setScale(4), + "DATE" -> "2015-07-23", + "SALARY" -> "5000000.0", + "SERIALNAME" -> "ASD69643", + "COUNTRY" -> "china", + "PHONETYPE" -> "phone197"), + Map("ID" -> 2, + "NAME" -> "jatin", + "BONUS" -> java.math.BigDecimal.valueOf(1234.5555).setScale(4) + , + "DATE" -> "2015-07-24", + "SALARY" -> java.math.BigDecimal.valueOf(150010.9990).setScale(3), + "SERIALNAME" -> "ASD42892", + "COUNTRY" -> "china", + "PHONETYPE" -> "phone756"), + Map("ID" -> 4, + "NAME" -> "prince", + "BONUS" -> java.math.BigDecimal.valueOf(9999.9990).setScale(4), + "DATE" -> "2015-07-26", + "SALARY" -> java.math.BigDecimal.valueOf(15003.0).setScale(1), + "SERIALNAME" -> "ASD66902", + "COUNTRY" -> "china", + "PHONETYPE" -> "phone2435"), + Map("ID" -> 5, + "NAME" -> "bhavya", + "BONUS" -> java.math.BigDecimal.valueOf(5000.9990).setScale(4), + "DATE" -> "2015-07-27", + "SALARY" -> java.math.BigDecimal.valueOf(15004.0).setScale(1), + "SERIALNAME" -> "ASD90633", + "COUNTRY" -> "china", + "PHONETYPE" -> "phone2441")) + assert(actualResult.toString() equals expectedResult.toString()) + } + test("test and filter clause with less than equal to expression") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery( + "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " + + "WHERE BONUS<=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," + + "BONUS ORDER BY ID LIMIT 2") + + val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1, + "NAME" -> "anubhav", + "BONUS" -> java.math.BigDecimal.valueOf(1234.4440).setScale(4), + "DATE" -> "2015-07-23", + "SALARY" -> "5000000.0", + "SERIALNAME" -> "ASD69643", + "COUNTRY" -> "china", + "PHONETYPE" -> "phone197"), + Map("ID" -> 3, + "NAME" -> "liang", + "BONUS" -> java.math.BigDecimal.valueOf(600.7770).setScale(4), + "DATE" -> "2015-07-25", + "SALARY" -> java.math.BigDecimal.valueOf(15002.11).setScale(2), + "SERIALNAME" -> "ASD37014", + "COUNTRY" -> "china", + "PHONETYPE" -> "phone1904")) + assert(actualResult.toString() equals expectedResult.toString()) + } + test("test equal to expression on decimal value") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery( + "SELECT ID FROM TESTDB.TESTTABLE WHERE BONUS=1234.444") + + val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1)) + + assert(actualResult equals expectedResult) + } + test("test less than expression with and operator") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery( + "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " + + "WHERE BONUS>1234 AND ID<2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," + + "BONUS ORDER BY ID") + val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1, + "NAME" -> "anubhav", + "BONUS" -> java.math.BigDecimal.valueOf(1234.4440).setScale(4), + "DATE" -> "2015-07-23", + "SALARY" -> 5000000.0, + "SERIALNAME" -> "ASD69643", + "COUNTRY" -> "china", + "PHONETYPE" -> "phone197")) + assert(actualResult.toString().equals(expectedResult.toString())) + } + test("test the result for in clause") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery("SELECT NAME from testdb.testtable WHERE PHONETYPE IN('phone1848','phone706')") + val expectedResult: List[Map[String, Any]] = List( + Map("NAME" -> "geetika"), + Map("NAME" -> "ravindra"), + Map("NAME" -> "jitesh")) + + assert(actualResult.equals(expectedResult)) + } + test("test the result for not in clause") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery( + "SELECT NAME from testdb.testtable WHERE PHONETYPE NOT IN('phone1848','phone706')") + val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "anubhav"), + Map("NAME" -> "jatin"), + Map("NAME" -> "liang"), + Map("NAME" -> "prince"), + Map("NAME" -> "bhavya"), + Map("NAME" -> "akash"), + Map("NAME" -> "sahil")) + + assert(actualResult.equals(expectedResult)) + } + test("test for null operator on date data type") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery("SELECT ID FROM TESTDB.TESTTABLE WHERE DATE IS NULL") + val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 9),Map("ID" -> null)) + assert(actualResult.equals(expectedResult)) + + } + test("test for not null operator on date data type") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DATE IS NOT NULL AND ID=9") + val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra")) + assert(actualResult.equals(expectedResult)) + + } + test("test for not null operator on timestamp type") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DOB IS NOT NULL AND ID=9") + val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"), + Map("NAME" -> "jitesh")) + assert(actualResult.equals(expectedResult)) + + } + + test("test timestamp datatype using cast operator") { + val actualResult: List[Map[String, Any]] = PrestoServer + .executeQuery("SELECT NAME AS RESULT FROM TESTDB.TESTTABLE WHERE DOB = CAST('2016-04-14 15:00:09' AS TIMESTAMP)") + val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> "jatin")) + assert(actualResult.equals(expectedResult)) + } + + private def cleanUp(): Unit = { + FileFactory.deleteFile(s"$storePath/Fact", FileType.LOCAL) + FileFactory + .createDirectoryAndSetPermission(s"$storePath/_system", + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) + FileFactory + .createDirectoryAndSetPermission(s"$storePath/.DS_Store", + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)) + FileFactory.createNewFile(s"$storePath/testdb/.DS_STORE",FileType.LOCAL) + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala index d6e322b..97539c0 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala @@ -40,7 +40,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll { + "../../../..").getCanonicalPath private val storePath = s"$rootPath/integration/presto/target/store" private val systemPath = s"$rootPath/integration/presto/target/system" - + private val PrestoServer = new PrestoServer // Table schema: // +-------------+----------------+-------------+------------+ http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala index 2f3b8f4..1b1accf 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala @@ -35,7 +35,7 @@ import org.slf4j.{Logger, LoggerFactory} import org.apache.carbondata.core.util.ThreadLocalSessionInfo import org.apache.carbondata.presto.CarbondataPlugin -object PrestoServer { +class PrestoServer { val CARBONDATA_CATALOG = "carbondata" val CARBONDATA_CONNECTOR = "carbondata" @@ -45,7 +45,7 @@ object PrestoServer { val prestoProperties: util.Map[String, String] = Map(("http-server.http.port", "8086")).asJava createSession - val queryRunner = new DistributedQueryRunner(createSession, 4, prestoProperties) + lazy val queryRunner = new DistributedQueryRunner(createSession, 4, prestoProperties) /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/4c9f0821/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala index 4b973a1..f687855 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala @@ -67,7 +67,8 @@ object CarbonDataStoreCreator { /** * Create store without any restructure */ - def createCarbonStore(storePath: String, dataFilePath: String): Unit = { + def createCarbonStore(storePath: String, dataFilePath: String, + useLocalDict: Boolean = false): Unit = { try { logger.info("Creating The Carbon Store") val dbName: String = "testdb" @@ -80,7 +81,7 @@ object CarbonDataStoreCreator { // val factFilePath: String = new File(dataFilePath).getCanonicalPath val storeDir: File = new File(absoluteTableIdentifier.getTablePath) CarbonUtil.deleteFoldersAndFiles(storeDir) - val table: CarbonTable = createTable(absoluteTableIdentifier) + val table: CarbonTable = createTable(absoluteTableIdentifier, useLocalDict) writeDictionary(dataFilePath, table, absoluteTableIdentifier) val schema: CarbonDataLoadSchema = new CarbonDataLoadSchema(table) val loadModel: CarbonLoadModel = new CarbonLoadModel() @@ -141,7 +142,8 @@ object CarbonDataStoreCreator { } } - private def createTable(absoluteTableIdentifier: AbsoluteTableIdentifier): CarbonTable = { + private def createTable(absoluteTableIdentifier: AbsoluteTableIdentifier, + useLocalDict: Boolean): CarbonTable = { val tableInfo: TableInfo = new TableInfo() tableInfo.setTablePath(absoluteTableIdentifier.getTablePath) tableInfo.setDatabaseName( @@ -151,7 +153,9 @@ object CarbonDataStoreCreator { absoluteTableIdentifier.getCarbonTableIdentifier.getTableName) val columnSchemas = new ArrayList[ColumnSchema]() val dictionaryEncoding: ArrayList[Encoding] = new ArrayList[Encoding]() - dictionaryEncoding.add(Encoding.DICTIONARY) + if (!useLocalDict) { + dictionaryEncoding.add(Encoding.DICTIONARY) + } val invertedIndexEncoding: ArrayList[Encoding] = new ArrayList[Encoding]() invertedIndexEncoding.add(Encoding.INVERTED_INDEX) @@ -225,9 +229,9 @@ object CarbonDataStoreCreator { val salary: ColumnSchema = new ColumnSchema() salary.setColumnName("salary") salary.setDataType(DataTypes.DOUBLE) - salary.setEncodingList(dictionaryEncoding) + salary.setEncodingList(new util.ArrayList[Encoding]()) salary.setColumnUniqueId(UUID.randomUUID().toString) - salary.setDimensionColumn(true) + salary.setDimensionColumn(false) salary.setSchemaOrdinal(6) salary.setColumnReferenceId(salary.getColumnUniqueId) columnSchemas.add(salary) @@ -291,6 +295,8 @@ object CarbonDataStoreCreator { new util.ArrayList[SchemaEvolutionEntry]()) tableSchema.setSchemaEvolution(schemaEvol) tableSchema.setTableId(UUID.randomUUID().toString) + tableSchema.getTableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE, + String.valueOf(useLocalDict)) tableInfo.setTableUniqueName( absoluteTableIdentifier.getCarbonTableIdentifier.getTableUniqueName )