Repository: carbondata Updated Branches: refs/heads/master 8e7895715 -> 1c4358e89
[CARBONDATA-2698][CARBONDATA-2700][CARBONDATA-2732][BloomDataMap] block some operations of bloomfilter datamap 1.Block create bloomfilter datamap index on column which its datatype is complex type; 2.Block change datatype for bloomfilter index datamap; 3.Block dropping index columns for bloomfilter index datamap This closes #2505 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1c4358e8 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1c4358e8 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1c4358e8 Branch: refs/heads/master Commit: 1c4358e89f5cba1132e9512107d3a0cb22087b7b Parents: 8e78957 Author: Sssan520 <liangap2...@aliyun.com> Authored: Mon Jul 16 10:59:43 2018 +0800 Committer: xuchuanyin <xuchuan...@hust.edu.cn> Committed: Tue Jul 17 16:34:14 2018 +0800 ---------------------------------------------------------------------- .../core/datamap/dev/DataMapFactory.java | 13 +++ .../core/metadata/schema/table/CarbonTable.java | 13 ++- .../bloom/BloomCoarseGrainDataMapFactory.java | 37 +++++++- .../datamap/CarbonCreateDataMapCommand.scala | 10 ++ .../CarbonAlterTableDataTypeChangeCommand.scala | 3 +- .../CarbonAlterTableDropColumnCommand.scala | 3 +- .../bloom/BloomCoarseGrainDataMapSuite.scala | 99 ++++++++++++++++++++ 7 files changed, 171 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java index 0889f8b..ab0f8ea 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java @@ -144,4 +144,17 @@ public abstract class DataMapFactory<T extends DataMap> { } } + /** + * whether to block operation on corresponding table or column. + * For example, bloomfilter datamap will block changing datatype for bloomindex column. + * By default it will not block any operation. + * + * @param operation table operation + * @param targets objects which the operation impact on + * @return true the operation will be blocked;false the operation will not be blocked + */ + public boolean isOperationBlocked(TableOperation operation, Object... targets) { + return false; + } + } http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java index 71256d4..995f943 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java @@ -1054,11 +1054,12 @@ public class CarbonTable implements Serializable { /** * methods returns true if operation is allowed for the corresponding datamap or not * if this operation makes datamap stale it is not allowed - * @param carbonTable - * @param operation - * @return + * @param carbonTable carbontable to be operated + * @param operation which operation on the table,such as drop column,change datatype. + * @param targets objects which the operation impact on,such as column + * @return true allow;false not allow */ - public boolean canAllow(CarbonTable carbonTable, TableOperation operation) { + public boolean canAllow(CarbonTable carbonTable, TableOperation operation, Object... targets) { try { List<TableDataMap> datamaps = DataMapStoreManager.getInstance().getAllDataMap(carbonTable); if (!datamaps.isEmpty()) { @@ -1069,6 +1070,10 @@ public class CarbonTable implements Serializable { if (factoryClass.willBecomeStale(operation)) { return false; } + // check whether the operation is blocked for datamap + if (factoryClass.isOperationBlocked(operation, targets)) { + return false; + } } } } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java ---------------------------------------------------------------------- diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java index 0fe0175..d9646d5 100644 --- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java +++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java @@ -377,7 +377,8 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa } } - @Override public boolean willBecomeStale(TableOperation operation) { + @Override + public boolean willBecomeStale(TableOperation operation) { switch (operation) { case ALTER_RENAME: return false; @@ -401,6 +402,40 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa } @Override + public boolean isOperationBlocked(TableOperation operation, Object... targets) { + switch (operation) { + case ALTER_DROP: { + // alter table drop columns + // will be blocked if the columns in bloomfilter datamap + List<String> columnsToDrop = (List<String>) targets[0]; + List<String> indexedColumnNames = dataMapMeta.getIndexedColumnNames(); + for (String indexedcolumn : indexedColumnNames) { + for (String column : columnsToDrop) { + if (column.equalsIgnoreCase(indexedcolumn)) { + return true; + } + } + } + return false; + } + case ALTER_CHANGE_DATATYPE: { + // alter table change one column datatype + // will be blocked if the column in bloomfilter datamap + String columnToChangeDatatype = (String) targets[0]; + List<String> indexedColumnNames = dataMapMeta.getIndexedColumnNames(); + for (String indexedcolumn : indexedColumnNames) { + if (indexedcolumn.equalsIgnoreCase(columnToChangeDatatype)) { + return true; + } + } + return false; + } + default: + return false; + } + } + + @Override public DataMapMeta getMeta() { return this.dataMapMeta; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala index a1a86a4..33dba28 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala @@ -95,6 +95,8 @@ case class CarbonCreateDataMapCommand( // If it is index datamap, check whether the column has datamap created already dataMapProvider match { case provider: IndexDataMapProvider => + val isBloomFilter = DataMapClassProvider.BLOOMFILTER.getShortName + .equalsIgnoreCase(dmProviderName) val datamaps = DataMapStoreManager.getInstance.getAllDataMap(mainTable).asScala val thisDmProviderName = dataMapProvider.asInstanceOf[IndexDataMapProvider].getDataMapSchema.getProviderName @@ -109,6 +111,14 @@ case class CarbonCreateDataMapCommand( throw new MalformedDataMapCommandException(String.format( "column '%s' already has %s index datamap created", column.getColName, thisDmProviderName)) + } else if (isBloomFilter) { + // if datamap provider is bloomfilter,the index column datatype cannot be complex type + if (column.isComplex) { + throw new MalformedDataMapCommandException( + s"BloomFilter datamap does not support complex datatype column: ${ + column.getColName + }") + } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala index fddbb7b..9ce79e9 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala @@ -55,7 +55,8 @@ private[sql] case class CarbonAlterTableDataTypeChangeCommand( .validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession) val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession) - if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_CHANGE_DATATYPE)) { + if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_CHANGE_DATATYPE, + alterTableDataTypeChangeModel.columnName)) { throw new MalformedCarbonCommandException( "alter table change datatype is not supported for index datamap") } http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala index 7022e98..23dbf9e 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala @@ -58,7 +58,8 @@ private[sql] case class CarbonAlterTableDropColumnCommand( .validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession) val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession) - if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_DROP)) { + if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_DROP, + alterTableDropColumnModel.columns.asJava)) { throw new MalformedCarbonCommandException( "alter table drop column is not supported for index datamap") } http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala index 4dc1837..df5ee18 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala @@ -26,6 +26,8 @@ import org.apache.spark.sql.{CarbonSession, DataFrame} import org.apache.spark.sql.test.util.QueryTest import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException} +import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.datamap.status.DataMapStatusManager import org.apache.carbondata.core.util.CarbonProperties @@ -413,6 +415,103 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with checkQuery("fakeDm", shouldHit = false) } + test("test block change datatype for bloomfilter index datamap") { + sql( + s""" + | CREATE TABLE $normalTable(id INT, name STRING, city STRING, age INT, + | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8 STRING) + | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128') + | """.stripMargin) + + sql( + s""" + | CREATE DATAMAP $dataMapName ON TABLE $normalTable + | USING 'bloomfilter' WITH DEFERRED REBUILD + | DMProperties( 'INDEX_COLUMNS'='city,id', 'BLOOM_SIZE'='640000') + """.stripMargin) + val exception: MalformedCarbonCommandException = intercept[MalformedCarbonCommandException] { + sql(s"ALTER TABLE $normalTable CHANGE id id bigint") + } + assert(exception.getMessage.contains( + "alter table change datatype is not supported for index datamap")) + } + + test("test drop index columns for bloomfilter datamap") { + sql( + s""" + | CREATE TABLE $normalTable(id INT, name STRING, city STRING, age INT, + | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8 STRING) + | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128') + | """.stripMargin) + sql( + s""" + | CREATE DATAMAP $dataMapName ON TABLE $normalTable + | USING 'bloomfilter' + | WITH DEFERRED REBUILD + | DMProperties('INDEX_COLUMNS'='city,id', 'BLOOM_SIZE'='640000') + """.stripMargin) + val exception: MalformedCarbonCommandException = intercept[MalformedCarbonCommandException] { + sql(s"alter table $normalTable drop columns(name, id)") + } + assert(exception.getMessage.contains( + "alter table drop column is not supported for index datamap")) + } + + test("test bloom datamap: bloom index column is local dictionary") { + sql( + s""" + | CREATE TABLE $normalTable(c1 string, c2 int, c3 string) + | STORED BY 'carbondata' + | """.stripMargin) + // c1 is local dictionary and will create bloom index on it + sql( + s""" + | CREATE TABLE $bloomDMSampleTable(c1 string, c2 int, c3 string) + | STORED BY 'carbondata' + | TBLPROPERTIES('local_dictionary_include'='c1', 'local_dictionary_threshold'='1000') + | """.stripMargin) + sql( + s""" + | CREATE DATAMAP $dataMapName on table $bloomDMSampleTable + | using 'bloomfilter' + | DMPROPERTIES('index_columns'='c1, c2') + | """.stripMargin) + sql( + s""" + | INSERT INTO $bloomDMSampleTable + | values ('c1v11', 11, 'c3v11'), ('c1v12', 12, 'c3v12') + | """.stripMargin) + sql( + s""" + | INSERT INTO $normalTable values ('c1v11', 11, 'c3v11'), + | ('c1v12', 12, 'c3v12') + | """.stripMargin) + checkAnswer(sql(s"select * from $bloomDMSampleTable"), + sql(s"select * from $normalTable")) + checkAnswer(sql(s"select * from $bloomDMSampleTable where c1 = 'c1v12'"), + sql(s"select * from $normalTable where c1 = 'c1v12'")) + } + + test("test create bloomfilter datamap which index column datatype is complex ") { + sql( + s""" + | CREATE TABLE $normalTable(id INT, name STRING, city Array<INT>, age INT, + | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8 STRING) + | STORED BY 'carbondata' + | """.stripMargin) + val exception: MalformedDataMapCommandException = intercept[MalformedDataMapCommandException] { + sql( + s""" + | CREATE DATAMAP $dataMapName ON TABLE $normalTable + | USING 'bloomfilter' + | WITH DEFERRED REBUILD + | DMProperties('INDEX_COLUMNS'='city,id', 'BLOOM_SIZE'='640000') + | """.stripMargin) + } + assert(exception.getMessage.contains( + "BloomFilter datamap does not support complex datatype column")) + } + override protected def afterAll(): Unit = { deleteFile(bigFile) deleteFile(smallFile)
