This is an automated email from the ASF dual-hosted git repository. ajantha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new 27a689a [CARBONDATA-3351] Block bloom datamap creation on binary datatype column 27a689a is described below commit 27a689a12a0cfce7fb151ec3348e48b4793656e5 Author: akkio-97 <akshay.nuth...@gmail.com> AuthorDate: Thu Dec 5 20:22:39 2019 +0530 [CARBONDATA-3351] Block bloom datamap creation on binary datatype column Why this PR? As per Binary Datatype design Bloom data-map creation on Binary datatype column is not supported. Handled the same. This closes #3500 --- .../testsuite/datamap/TestDataMapCommand.scala | 61 +--------------------- .../datamap/CarbonCreateDataMapCommand.scala | 7 +++ .../bloom/BloomCoarseGrainDataMapSuite.scala | 19 +++++++ 3 files changed, 27 insertions(+), 60 deletions(-) diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala index d9461b2..0da9428 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala @@ -266,66 +266,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll { sql(s"drop table if exists $tableName") } - test("test support bloomFilter on binary data type") { - val tableName = "datamapshowtest" - val datamapName = "bloomdatamap" - val datamapName2 = "bloomdatamap2" - sql(s"drop table if exists $tableName") - // for index datamap - sql(s"create table $tableName (a string, b string, c string, d binary, e binary) stored by 'carbondata'") - - sql(s"insert into $tableName values('a1','b1','c1','d1','e1')") - sql(s"insert into $tableName values('a1','b2','c2','d1','e2')") - sql(s"insert into $tableName values('a3','b3','c1','d2','e2')") - sql( - s""" - | create datamap $datamapName on table $tableName using 'bloomfilter' - | DMPROPERTIES ('index_columns'='d', 'bloom_size'='32000', 'bloom_fpp'='0.001') - """.stripMargin) - sql( - s""" - | create datamap $datamapName2 on table $tableName using 'bloomfilter' - | DMPROPERTIES ('index_columns'='e') - """.stripMargin) - var bloom1 = sql(s"select * from $tableName where d=cast('d1' as binary)") - assert(2 == bloom1.collect().length) - bloom1.collect().foreach { each => - assert(5 == each.length) - assert("a1".equals(each.get(0))) - assert("d1".equals(new String(each.getAs[Array[Byte]](3)))) - if ("b1".equals(each.get(1))) { - assert("c1".equals(each.get(2))) - assert("e1".equals(new String(each.getAs[Array[Byte]](4)))) - } else if ("b2".equals(each.get(1))) { - assert("c2".equals(each.get(2))) - assert("e2".equals(new String(each.getAs[Array[Byte]](4)))) - } else { - assert(false) - } - } - - bloom1 = sql(s"select * from $tableName where d=cast('d1' as binary) and e=cast('e1' as binary)") - assert(1 == bloom1.collect().length) - bloom1.collect().foreach { each => - assert(5 == each.length) - assert("a1".equals(each.get(0))) - assert("d1".equals(new String(each.getAs[Array[Byte]](3)))) - if ("b1".equals(each.get(1))) { - assert("c1".equals(each.get(2))) - assert("e1".equals(new String(each.getAs[Array[Byte]](4)))) - } else { - assert(false) - } - } - - val result = sql(s"show datamap on table $tableName").cache() - - checkAnswer(sql(s"show datamap on table $tableName"), - Seq(Row(datamapName, "bloomfilter", s"default.$tableName", "'bloom_fpp'='0.001', 'bloom_size'='32000', 'index_columns'='d'", "ENABLED", "NA"), - Row(datamapName2, "bloomfilter", s"default.$tableName", "'index_columns'='e'", "ENABLED", "NA"))) - result.unpersist() - sql(s"drop table if exists $tableName") - } + test("test don't support timeseries on binary data type") { val tableName = "datamapshowtest" diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala index bf4106b..d4524d0 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala @@ -27,6 +27,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.datamap.{DataMapProvider, DataMapStoreManager} import org.apache.carbondata.core.datamap.status.DataMapStatusManager import org.apache.carbondata.core.metadata.ColumnarFormatVersion +import org.apache.carbondata.core.metadata.datatype.DataTypes import org.apache.carbondata.core.metadata.schema.datamap.{DataMapClassProvider, DataMapProperty} import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema} import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil} @@ -145,6 +146,12 @@ case class CarbonCreateDataMapCommand( "column '%s' already has %s index datamap created", column.getColName, thisDmProviderName)) } else if (isBloomFilter) { + if (column.getDataType == DataTypes.BINARY) { + throw new MalformedDataMapCommandException( + s"BloomFilter datamap does not support Binary datatype column: ${ + column.getColName + }") + } // if datamap provider is bloomfilter,the index column datatype cannot be complex type if (column.isComplex) { throw new MalformedDataMapCommandException( diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala index 06e81bd..bedcc4a 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala @@ -590,6 +590,25 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with "BloomFilter datamap does not support complex datatype column")) } + test("test create bloomfilter datamap which index column datatype is Binary ") { + sql("drop table if exists binaryTable") + sql( + "CREATE TABLE binaryTable (CUST_ID binary,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB " + + "timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 " + + "decimal(30,10), DECIMAL_COLUMN2 decimal(36,36),Double_COLUMN1 double, Double_COLUMN2 " + + "double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'") + val exception: MalformedDataMapCommandException = intercept[MalformedDataMapCommandException] { + sql( + s""" + | CREATE DATAMAP binaryBloom ON TABLE binaryTable + | USING 'bloomfilter' + | DMProperties('INDEX_COLUMNS'='cust_id', 'BLOOM_SIZE'='640000') + | """.stripMargin) + } + assert(exception.getMessage.equalsIgnoreCase( + "BloomFilter datamap does not support binary datatype column: cust_id" )) + } + test("test create bloom datamap on newly added column") { val datamap1 = "datamap1" val datamap2 = "datamap2"