Repository: carbondata Updated Branches: refs/heads/master 589fe1883 -> 00d6ce49d
[CARBONDATA-2635][BloomDataMap] Support different index datamaps on same column User can create different provider based index datamaps on one column, for example user can create bloomfilter datamap and lucene datamap on one column, but not able to create two bloomfilter datamap on one column. This closes #2405 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/00d6ce49 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/00d6ce49 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/00d6ce49 Branch: refs/heads/master Commit: 00d6ce49d924bf58d4480e61cd0b64d3c5e54181 Parents: 589fe18 Author: xuchuanyin <[email protected]> Authored: Sun Jun 24 12:47:21 2018 +0800 Committer: Jacky Li <[email protected]> Committed: Sat Jun 30 14:25:07 2018 +0800 ---------------------------------------------------------------------- .../core/datamap/DataMapProvider.java | 2 +- .../lucene/LuceneFineGrainDataMapSuite.scala | 26 ---------- .../testsuite/datamap/TestDataMapCommand.scala | 51 ++++++++++++++++++++ .../datamap/CarbonCreateDataMapCommand.scala | 20 ++++---- 4 files changed, 63 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/00d6ce49/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java index 05ba7cf..168159b 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java @@ -67,7 +67,7 @@ public abstract class DataMapProvider { return mainTable; } - protected final DataMapSchema getDataMapSchema() { + public final DataMapSchema getDataMapSchema() { return dataMapSchema; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/00d6ce49/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala index b3c1a1a..dc0cf52 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala @@ -225,32 +225,6 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll { sql("DROP TABLE IF EXISTS datamap_test3") } - test("test lucene fine grain data map for create datamap with Duplicate Columns") { - sql("DROP TABLE IF EXISTS datamap_test_table") - sql( - """ - | CREATE TABLE datamap_test_table(id INT, name STRING, city STRING, age INT) - | STORED BY 'carbondata' - | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT') - """.stripMargin) - val exception_duplicate_column: Exception = intercept[MalformedDataMapCommandException] { - sql( - s""" - | CREATE DATAMAP dm ON TABLE datamap_test_table - | USING 'lucene' - | DMProperties('INDEX_COLUMNS'='name') - """.stripMargin) - sql( - s""" - | CREATE DATAMAP dm1 ON TABLE datamap_test_table - | USING 'lucene' - | DMProperties('INDEX_COLUMNS'='name') - """.stripMargin) - } - assertResult("column 'name' already has datamap created")(exception_duplicate_column.getMessage) - sql("drop datamap if exists dm on table datamap_test_table") - } - test("test lucene fine grain data map with wildcard matching ") { sql("DROP TABLE IF EXISTS datamap_test_table") sql( http://git-wip-us.apache.org/repos/asf/carbondata/blob/00d6ce49/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala index afca3b2..7fc4e26 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala @@ -350,6 +350,57 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll { sql("drop table main") } + test("test create datamap: unable to create same index datamap for one column") { + sql("DROP TABLE IF EXISTS datamap_test_table") + sql( + """ + | CREATE TABLE datamap_test_table(id INT, name STRING, city STRING, age INT) + | STORED BY 'carbondata' + | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT') + """.stripMargin) + val exception_duplicate_column: Exception = intercept[MalformedDataMapCommandException] { + sql( + s""" + | CREATE DATAMAP dm ON TABLE datamap_test_table + | USING 'lucene' + | DMProperties('INDEX_COLUMNS'='name') + """.stripMargin) + sql( + s""" + | CREATE DATAMAP dm1 ON TABLE datamap_test_table + | USING 'lucene' + | DMProperties('INDEX_COLUMNS'='name') + """.stripMargin) + } + assertResult("column 'name' already has lucene index datamap created")(exception_duplicate_column.getMessage) + sql("drop table if exists datamap_test_table") + } + + test("test create datamap: able to create different index datamap for one column") { + sql("DROP TABLE IF EXISTS datamap_test_table") + sql( + """ + | CREATE TABLE datamap_test_table(id INT, name STRING, city STRING, age INT) + | STORED BY 'carbondata' + | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT') + """.stripMargin) + sql( + s""" + | CREATE DATAMAP dm ON TABLE datamap_test_table + | USING 'lucene' + | DMProperties('INDEX_COLUMNS'='name') + """.stripMargin) + sql( + s""" + | CREATE DATAMAP dm1 ON TABLE datamap_test_table + | USING 'bloomfilter' + | DMProperties('INDEX_COLUMNS'='name') + """.stripMargin) + sql("show datamap on table datamap_test_table").show(false) + checkExistence(sql("show datamap on table datamap_test_table"), true, "dm", "dm1", "lucene", "bloomfilter") + sql("drop table if exists datamap_test_table") + } + override def afterAll { sql("DROP TABLE IF EXISTS maintable") sql("drop table if exists uniqdata") http://git-wip-us.apache.org/repos/asf/carbondata/blob/00d6ce49/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala index 27e1720..a1a86a4 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala @@ -16,10 +16,7 @@ */ package org.apache.spark.sql.execution.command.datamap -import java.util - import scala.collection.JavaConverters._ -import scala.collection.mutable import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier @@ -99,17 +96,22 @@ case class CarbonCreateDataMapCommand( dataMapProvider match { case provider: IndexDataMapProvider => val datamaps = DataMapStoreManager.getInstance.getAllDataMap(mainTable).asScala - val existingIndexColumn = mutable.Set[String]() - datamaps.foreach { datamap => - datamap.getDataMapSchema.getIndexColumns.foreach(existingIndexColumn.add) - } + val thisDmProviderName = + dataMapProvider.asInstanceOf[IndexDataMapProvider].getDataMapSchema.getProviderName + val existingIndexColumn4ThisProvider = datamaps.filter { datamap => + thisDmProviderName.equalsIgnoreCase(datamap.getDataMapSchema.getProviderName) + }.flatMap { datamap => + datamap.getDataMapSchema.getIndexColumns + }.distinct provider.getIndexedColumns.asScala.foreach { column => - if (existingIndexColumn.contains(column.getColName)) { + if (existingIndexColumn4ThisProvider.contains(column.getColName)) { throw new MalformedDataMapCommandException(String.format( - "column '%s' already has datamap created", column.getColName)) + "column '%s' already has %s index datamap created", + column.getColName, thisDmProviderName)) } } + val operationContext: OperationContext = new OperationContext() val systemFolderLocation: String = CarbonProperties.getInstance().getSystemFolderLocation val createDataMapPreExecutionEvent: CreateDataMapPreExecutionEvent =
