Repository: carbondata Updated Branches: refs/heads/master 73a5885a4 -> 7c827c0a9
[CARBONDATA-2811][BloomDataMap] Add query test case using search mode on table with bloom filter This closes #2598 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7c827c0a Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7c827c0a Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7c827c0a Branch: refs/heads/master Commit: 7c827c0a96c4f9dbbcbd46f2d5c85833e203b2ca Parents: 73a5885 Author: Manhua <kevin...@qq.com> Authored: Thu Aug 2 10:21:16 2018 +0800 Committer: QiangCai <qiang...@qq.com> Committed: Tue Sep 11 17:47:41 2018 +0800 ---------------------------------------------------------------------- .../BloomCoarseGrainDataMapFunctionSuite.scala | 2 + .../bloom/BloomCoarseGrainDataMapSuite.scala | 60 ++++++++++++++++++++ 2 files changed, 62 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/7c827c0a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala index 7ea89a9..dd2ecb4 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala @@ -830,6 +830,8 @@ class BloomCoarseGrainDataMapFunctionSuite extends QueryTest with BeforeAndAfte sql("drop table if exists test_rcd").collect() CarbonProperties.getInstance().addProperty(CarbonCommonConstants.BLOCKLET_SIZE, CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL) + CarbonProperties.getInstance().addProperty(CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB, + CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE) } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/7c827c0a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala index cca1b67..7cb841f 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala @@ -32,6 +32,7 @@ import org.apache.carbondata.core.datamap.status.DataMapStatusManager import org.apache.carbondata.core.util.CarbonProperties class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach { + val carbonSession = sqlContext.sparkSession.asInstanceOf[CarbonSession] val bigFile = s"$resourcesPath/bloom_datamap_input_big.csv" val smallFile = s"$resourcesPath/bloom_datamap_input_small.csv" val normalTable = "carbon_normal" @@ -251,6 +252,61 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with sql(s"DROP TABLE IF EXISTS $bloomDMSampleTable") } + test("test using search mode to query tabel with bloom datamap") { + sql( + s""" + | CREATE TABLE $normalTable(id INT, name STRING, city STRING, age INT, + | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8 STRING) + | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128') + | """.stripMargin) + sql( + s""" + | CREATE TABLE $bloomDMSampleTable(id INT, name STRING, city STRING, age INT, + | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8 STRING) + | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128') + | """.stripMargin) + + // load two segments + (1 to 2).foreach { i => + sql( + s""" + | LOAD DATA LOCAL INPATH '$bigFile' INTO TABLE $normalTable + | OPTIONS('header'='false') + """.stripMargin) + sql( + s""" + | LOAD DATA LOCAL INPATH '$bigFile' INTO TABLE $bloomDMSampleTable + | OPTIONS('header'='false') + """.stripMargin) + } + + sql( + s""" + | CREATE DATAMAP $dataMapName ON TABLE $bloomDMSampleTable + | USING 'bloomfilter' + | DMProperties('INDEX_COLUMNS'='city,id', 'BLOOM_SIZE'='640000') + """.stripMargin) + checkExistence(sql(s"SHOW DATAMAP ON TABLE $bloomDMSampleTable"), true, dataMapName) + + // get answer before search mode is enable + val expectedAnswer1 = sql(s"select * from $normalTable where id = 1").collect() + val expectedAnswer2 = sql(s"select * from $normalTable where city in ('city_999')").collect() + + carbonSession.startSearchMode() + assert(carbonSession.isSearchModeEnabled) + + checkAnswer( + sql(s"select * from $bloomDMSampleTable where id = 1"), expectedAnswer1) + checkAnswer( + sql(s"select * from $bloomDMSampleTable where city in ('city_999')"), expectedAnswer2) + + carbonSession.stopSearchMode() + assert(!carbonSession.isSearchModeEnabled) + + sql(s"DROP TABLE IF EXISTS $normalTable") + sql(s"DROP TABLE IF EXISTS $bloomDMSampleTable") + } + ignore("test create bloom datamap with DEFERRED REBUILD, query hit datamap") { sql( s""" @@ -848,6 +904,10 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with } override protected def afterAll(): Unit = { + // in case of search mode test case failed, stop search mode again + if (carbonSession.isSearchModeEnabled) { + carbonSession.stopSearchMode() + } deleteFile(bigFile) deleteFile(smallFile) sql(s"DROP TABLE IF EXISTS $normalTable")