Repository: carbondata Updated Branches: refs/heads/master aeb2ec4cd -> c0de9f160
[CARBONDATA-2654][Datamap] Optimize output for explaining querying with datamap Currently if we have multiple datamaps and the query hits all the datamaps, carbondata explain command will only show the first datamap and all the datamaps are not shown. In this commit, we show all the datamaps that are hitted in this query. This closes #2411 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c0de9f16 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c0de9f16 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c0de9f16 Branch: refs/heads/master Commit: c0de9f16010e0cb83db1dbd89e94e8bd3681dda7 Parents: aeb2ec4 Author: xuchuanyin <[email protected]> Authored: Thu Jun 28 20:01:03 2018 +0800 Committer: Jacky Li <[email protected]> Committed: Thu Jul 5 14:19:19 2018 +0800 ---------------------------------------------------------------------- .../datamap/dev/expr/AndDataMapExprWrapper.java | 10 ++- .../datamap/dev/expr/DataMapExprWrapper.java | 9 +- .../dev/expr/DataMapExprWrapperImpl.java | 11 ++- .../dev/expr/DataMapWrapperSimpleInfo.java | 90 ++++++++++++++++++++ .../datamap/dev/expr/OrDataMapExprWrapper.java | 11 ++- .../core/profiler/ExplainCollector.java | 17 ++-- .../core/profiler/TablePruningInfo.java | 31 ++++--- .../hadoop/api/CarbonInputFormat.java | 20 +++-- .../testsuite/datamap/CGDataMapTestCase.scala | 5 +- .../testsuite/datamap/FGDataMapTestCase.scala | 12 +-- .../bloom/BloomCoarseGrainDataMapSuite.scala | 79 ++++++++++++++++- 11 files changed, 242 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java index ec674de..c5099db 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java @@ -25,7 +25,6 @@ import org.apache.carbondata.core.datamap.DataMapLevel; import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.indexstore.ExtendedBlocklet; import org.apache.carbondata.core.indexstore.PartitionSpec; -import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; /** @@ -116,8 +115,13 @@ public class AndDataMapExprWrapper implements DataMapExprWrapper { return left.getDataMapLevel(); } - @Override public DataMapSchema getDataMapSchema() { - return left.getDataMapSchema(); + @Override + public DataMapExprWrapper getLeftDataMapWrapper() { + return left; } + @Override + public DataMapExprWrapper getRightDataMapWrapprt() { + return right; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java index 901cfc7..ec26107 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java @@ -25,7 +25,6 @@ import org.apache.carbondata.core.datamap.DataMapLevel; import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.indexstore.ExtendedBlocklet; import org.apache.carbondata.core.indexstore.PartitionSpec; -import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; /** @@ -90,8 +89,12 @@ public interface DataMapExprWrapper extends Serializable { DataMapLevel getDataMapLevel(); /** - * Get the datamap schema + * get the left datamap wrapper */ - DataMapSchema getDataMapSchema(); + DataMapExprWrapper getLeftDataMapWrapper(); + /** + * get the right datamap wrapper + */ + DataMapExprWrapper getRightDataMapWrapprt(); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java index 6537976..4643b47 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java @@ -97,8 +97,17 @@ public class DataMapExprWrapperImpl implements DataMapExprWrapper { return dataMap.getDataMapFactory().getDataMapLevel(); } - @Override public DataMapSchema getDataMapSchema() { + public DataMapSchema getDataMapSchema() { return dataMap.getDataMapSchema(); } + @Override + public DataMapExprWrapper getLeftDataMapWrapper() { + return null; + } + + @Override + public DataMapExprWrapper getRightDataMapWrapprt() { + return null; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapWrapperSimpleInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapWrapperSimpleInfo.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapWrapperSimpleInfo.java new file mode 100644 index 0000000..331e77e --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapWrapperSimpleInfo.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datamap.dev.expr; + +import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; + +/** + * schema for datamap wrapper. + * Currently a DataMapWrapper contains more than one datamap, this class is used to describe its + * schema. For example a AndDataMapExprWrapper contains BloomFilter in its left and Lucene in + * its right, then its schema would be AND(BloomFilter, Lucene) + */ +public class DataMapWrapperSimpleInfo { + enum WrapperType { + PRIMITIVE, + AND, + OR + } + + private WrapperType wrapperType; + private DataMapWrapperSimpleInfo left; + private DataMapWrapperSimpleInfo right; + private DataMapSchema schema; + + private DataMapWrapperSimpleInfo(WrapperType wrapperType, DataMapWrapperSimpleInfo left, + DataMapWrapperSimpleInfo right) { + this.wrapperType = wrapperType; + this.left = left; + this.right = right; + } + + private DataMapWrapperSimpleInfo(DataMapSchema schema) { + this.wrapperType = WrapperType.PRIMITIVE; + this.schema = schema; + } + + public static DataMapWrapperSimpleInfo fromDataMapWrapper(DataMapExprWrapper dataMapExprWrapper) { + if (dataMapExprWrapper instanceof DataMapExprWrapperImpl) { + return new DataMapWrapperSimpleInfo( + ((DataMapExprWrapperImpl) dataMapExprWrapper).getDataMapSchema()); + } else if (dataMapExprWrapper instanceof AndDataMapExprWrapper) { + return new DataMapWrapperSimpleInfo(WrapperType.AND, + fromDataMapWrapper(dataMapExprWrapper.getLeftDataMapWrapper()), + fromDataMapWrapper(dataMapExprWrapper.getRightDataMapWrapprt())); + } else { + return new DataMapWrapperSimpleInfo(WrapperType.OR, + fromDataMapWrapper(dataMapExprWrapper.getLeftDataMapWrapper()), + fromDataMapWrapper(dataMapExprWrapper.getRightDataMapWrapprt())); + } + } + + public String getDataMapWrapperName() { + if (WrapperType.PRIMITIVE == wrapperType) { + return schema.getDataMapName(); + } else { + return String.format("%s(%s, %s)", + wrapperType, left.getDataMapWrapperName(), right.getDataMapWrapperName()); + } + } + + public String getDataMapWrapperProvider() { + if (WrapperType.PRIMITIVE == wrapperType) { + return schema.getProviderName(); + } else { + return String.format("%s(%s, %s)", + wrapperType, left.getDataMapWrapperProvider(), right.getDataMapWrapperProvider()); + } + } + + @Override + public String toString() { + return "DatamapWrapperSchema: Name->" + getDataMapWrapperName() + + ", Provider->" + getDataMapWrapperProvider(); + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java index bb98535..7812969 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java @@ -27,7 +27,6 @@ import org.apache.carbondata.core.datamap.DataMapLevel; import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.indexstore.ExtendedBlocklet; import org.apache.carbondata.core.indexstore.PartitionSpec; -import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; /** @@ -109,7 +108,13 @@ public class OrDataMapExprWrapper implements DataMapExprWrapper { return left.getDataMapLevel(); } - @Override public DataMapSchema getDataMapSchema() { - return left.getDataMapSchema(); + @Override + public DataMapExprWrapper getLeftDataMapWrapper() { + return left; + } + + @Override + public DataMapExprWrapper getRightDataMapWrapprt() { + return right; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java index 5d69bb9..02e45f6 100644 --- a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java @@ -26,7 +26,7 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import org.apache.carbondata.common.annotations.InterfaceAudience; -import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; +import org.apache.carbondata.core.datamap.dev.expr.DataMapWrapperSimpleInfo; /** * An information collector used for EXPLAIN command, to print out @@ -94,24 +94,27 @@ public class ExplainCollector { } } - public static void recordDefaultDataMapPruning(DataMapSchema dataMapSchema, int numBlocklets) { + public static void recordDefaultDataMapPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo, + int numBlocklets) { if (enabled()) { TablePruningInfo scan = getCurrentTablePruningInfo(); - scan.setNumBlockletsAfterDefaultPruning(dataMapSchema, numBlocklets); + scan.setNumBlockletsAfterDefaultPruning(dataMapWrapperSimpleInfo, numBlocklets); } } - public static void recordCGDataMapPruning(DataMapSchema dataMapSchema, int numBlocklets) { + public static void recordCGDataMapPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo, + int numBlocklets) { if (enabled()) { TablePruningInfo scan = getCurrentTablePruningInfo(); - scan.setNumBlockletsAfterCGPruning(dataMapSchema, numBlocklets); + scan.setNumBlockletsAfterCGPruning(dataMapWrapperSimpleInfo, numBlocklets); } } - public static void recordFGDataMapPruning(DataMapSchema dataMapSchema, int numBlocklets) { + public static void recordFGDataMapPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo, + int numBlocklets) { if (enabled()) { TablePruningInfo scan = getCurrentTablePruningInfo(); - scan.setNumBlockletsAfterFGPruning(dataMapSchema, numBlocklets); + scan.setNumBlockletsAfterFGPruning(dataMapWrapperSimpleInfo, numBlocklets); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/profiler/TablePruningInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/profiler/TablePruningInfo.java b/core/src/main/java/org/apache/carbondata/core/profiler/TablePruningInfo.java index cec8827..651af9d 100644 --- a/core/src/main/java/org/apache/carbondata/core/profiler/TablePruningInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/profiler/TablePruningInfo.java @@ -18,7 +18,7 @@ package org.apache.carbondata.core.profiler; import org.apache.carbondata.common.annotations.InterfaceAudience; -import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; +import org.apache.carbondata.core.datamap.dev.expr.DataMapWrapperSimpleInfo; /** * Used for EXPLAIN command @@ -29,13 +29,13 @@ public class TablePruningInfo { private int totalBlocklets; private String filterStatement; - private DataMapSchema defaultDataMap; + private DataMapWrapperSimpleInfo defaultDataMap; private int numBlockletsAfterDefaultPruning; - private DataMapSchema cgDataMap; + private DataMapWrapperSimpleInfo cgDataMap; private int numBlockletsAfterCGPruning; - private DataMapSchema fgDataMap; + private DataMapWrapperSimpleInfo fgDataMap; private int numBlockletsAfterFGPruning; void addTotalBlocklets(int numBlocklets) { @@ -46,18 +46,21 @@ public class TablePruningInfo { this.filterStatement = filterStatement; } - void setNumBlockletsAfterDefaultPruning(DataMapSchema dataMapSchema, int numBlocklets) { - this.defaultDataMap = dataMapSchema; + void setNumBlockletsAfterDefaultPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo, + int numBlocklets) { + this.defaultDataMap = dataMapWrapperSimpleInfo; this.numBlockletsAfterDefaultPruning = numBlocklets; } - void setNumBlockletsAfterCGPruning(DataMapSchema dataMapSchema, int numBlocklets) { - this.cgDataMap = dataMapSchema; + void setNumBlockletsAfterCGPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo, + int numBlocklets) { + this.cgDataMap = dataMapWrapperSimpleInfo; this.numBlockletsAfterCGPruning = numBlocklets; } - void setNumBlockletsAfterFGPruning(DataMapSchema dataMapSchema, int numBlocklets) { - this.fgDataMap = dataMapSchema; + void setNumBlockletsAfterFGPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo, + int numBlocklets) { + this.fgDataMap = dataMapWrapperSimpleInfo; this.numBlockletsAfterFGPruning = numBlocklets; } @@ -77,8 +80,8 @@ public class TablePruningInfo { int skipBlocklets = numBlockletsAfterDefaultPruning - numBlockletsAfterCGPruning; builder .append(" - pruned by CG DataMap").append("\n") - .append(" - name: ").append(cgDataMap.getDataMapName()).append("\n") - .append(" - provider: ").append(cgDataMap.getProviderName()).append("\n") + .append(" - name: ").append(cgDataMap.getDataMapWrapperName()).append("\n") + .append(" - provider: ").append(cgDataMap.getDataMapWrapperProvider()).append("\n") .append(" - skipped blocklets: ").append(skipBlocklets).append("\n"); } if (fgDataMap != null) { @@ -90,8 +93,8 @@ public class TablePruningInfo { } builder .append(" - pruned by FG DataMap").append("\n") - .append(" - name: ").append(fgDataMap.getDataMapName()).append("\n") - .append(" - provider: ").append(fgDataMap.getProviderName()).append("\n") + .append(" - name: ").append(fgDataMap.getDataMapWrapperName()).append("\n") + .append(" - provider: ").append(fgDataMap.getDataMapWrapperProvider()).append("\n") .append(" - skipped blocklets: ").append(skipBlocklets).append("\n"); } return builder.toString(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java index 4b30ddf..356dd5a 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java @@ -33,6 +33,7 @@ import org.apache.carbondata.core.datamap.DataMapJob; import org.apache.carbondata.core.datamap.DataMapUtil; import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapper; +import org.apache.carbondata.core.datamap.dev.expr.DataMapWrapperSimpleInfo; import org.apache.carbondata.core.exception.InvalidConfigurationException; import org.apache.carbondata.core.indexstore.ExtendedBlocklet; import org.apache.carbondata.core.indexstore.PartitionSpec; @@ -433,12 +434,13 @@ m filterExpression DataMapJob dataMapJob = DataMapUtil.getDataMapJob(job.getConfiguration()); List<PartitionSpec> partitionsToPrune = getPartitionsToPrune(job.getConfiguration()); // First prune using default datamap on driver side. - DataMapExprWrapper dataMapExprWrapper = DataMapChooser.getDefaultDataMap( - getOrCreateCarbonTable(job.getConfiguration()), resolver); - List<ExtendedBlocklet> prunedBlocklets = dataMapExprWrapper.prune(segmentIds, - partitionsToPrune); - ExplainCollector.recordDefaultDataMapPruning(dataMapExprWrapper.getDataMapSchema(), - prunedBlocklets.size()); + DataMapExprWrapper dataMapExprWrapper = DataMapChooser + .getDefaultDataMap(getOrCreateCarbonTable(job.getConfiguration()), resolver); + List<ExtendedBlocklet> prunedBlocklets = + dataMapExprWrapper.prune(segmentIds, partitionsToPrune); + + ExplainCollector.recordDefaultDataMapPruning( + DataMapWrapperSimpleInfo.fromDataMapWrapper(dataMapExprWrapper), prunedBlocklets.size()); if (prunedBlocklets.size() == 0) { return prunedBlocklets; } @@ -463,7 +465,8 @@ m filterExpression prunedBlocklets = (List) CollectionUtils.intersection( cgPrunedBlocklets, prunedBlocklets); ExplainCollector.recordCGDataMapPruning( - cgDataMapExprWrapper.getDataMapSchema(), prunedBlocklets.size()); + DataMapWrapperSimpleInfo.fromDataMapWrapper(cgDataMapExprWrapper), + prunedBlocklets.size()); } if (prunedBlocklets.size() == 0) { @@ -481,7 +484,8 @@ m filterExpression // 'prunedBlocklets', so the intersection should keep the elements in 'fgPrunedBlocklets' prunedBlocklets = (List) CollectionUtils.intersection(fgPrunedBlocklets, prunedBlocklets); - ExplainCollector.recordFGDataMapPruning(fgDataMapExprWrapper.getDataMapSchema(), + ExplainCollector.recordFGDataMapPruning( + DataMapWrapperSimpleInfo.fromDataMapWrapper(fgDataMapExprWrapper), prunedBlocklets.size()); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala index e141a09..bc8b28b 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala @@ -425,10 +425,7 @@ class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll { val df1 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect() assert(df1(0).getString(0).contains("CG DataMap")) assert(df1(0).getString(0).contains(dataMapName1)) - val e11 = intercept[Exception] { - assert(df1(0).getString(0).contains(dataMapName2)) - } - assert(e11.getMessage.contains("did not contain \"" + dataMapName2)) + assert(df1(0).getString(0).contains(dataMapName2)) // make datamap1 invisible sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = false") http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala index 20a76f8..6f8310c 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala @@ -516,10 +516,7 @@ class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll { val df1 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect() assert(df1(0).getString(0).contains("FG DataMap")) assert(df1(0).getString(0).contains(dataMapName1)) - val e11 = intercept[Exception] { - assert(df1(0).getString(0).contains(dataMapName2)) - } - assert(e11.getMessage.contains("did not contain \"" + dataMapName2)) + assert(df1(0).getString(0).contains(dataMapName2)) // make datamap1 invisible sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = false") @@ -548,15 +545,12 @@ class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll { // make datamap1,datamap2 visible sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = true") - sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = true") + sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName2 = true") checkAnswer(sql(s"SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'"), sql("SELECT * FROM normal_test WHERE name='n502670' AND city='c2670'")) val df4 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect() assert(df4(0).getString(0).contains(dataMapName1)) - val e41 = intercept[Exception] { - assert(df3(0).getString(0).contains(dataMapName2)) - } - assert(e41.getMessage.contains("did not contain \"" + dataMapName2)) + assert(df4(0).getString(0).contains(dataMapName2)) } override protected def afterAll(): Unit = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala index a29f0f2..836c8e7 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala @@ -38,7 +38,7 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with override protected def beforeAll(): Unit = { new File(CarbonProperties.getInstance().getSystemFolderLocation).delete() - createFile(bigFile, line = 500000) + createFile(bigFile, line = 50000) createFile(smallFile) sql(s"DROP TABLE IF EXISTS $normalTable") sql(s"DROP TABLE IF EXISTS $bloomDMSampleTable") @@ -300,6 +300,83 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with sql(s"DROP TABLE IF EXISTS $bloomDMSampleTable") } + test("test bloom datamap: multiple datamaps with each on one column vs one datamap on multiple columns") { + val iterations = 1 + // 500000 lines will result to 3 blocklets and bloomfilter datamap will prune 2 blocklets. + val datamap11 = "datamap11" + val datamap12 = "datamap12" + val datamap13 = "datamap13" + val datamap2 = "datamap2" + + sql(s"DROP TABLE IF EXISTS $bloomDMSampleTable") + // create a table and 3 bloom datamaps on it, each datamap contains one index column + sql( + s""" + | CREATE TABLE $bloomDMSampleTable(id INT, name STRING, city STRING, age INT, + | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8 STRING) + | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128', 'SORT_COLUMNS'='s1') + | """.stripMargin) + sql( + s""" + | CREATE DATAMAP $datamap11 ON TABLE $bloomDMSampleTable + | USING 'bloomfilter' + | DMProperties('INDEX_COLUMNS'='id', 'BLOOM_SIZE'='64000', 'BLOOM_FPP'='0.00001') + """.stripMargin) + sql( + s""" + | CREATE DATAMAP $datamap12 ON TABLE $bloomDMSampleTable + | USING 'bloomfilter' + | DMProperties('INDEX_COLUMNS'='name', 'BLOOM_SIZE'='64000', 'BLOOM_FPP'='0.00001') + """.stripMargin) + sql( + s""" + | CREATE DATAMAP $datamap13 ON TABLE $bloomDMSampleTable + | USING 'bloomfilter' + | DMProperties('INDEX_COLUMNS'='city', 'BLOOM_SIZE'='64000', 'BLOOM_FPP'='0.00001') + """.stripMargin) + + // create a table and 1 bloom datamap on it, this datamap contains 3 index columns + sql( + s""" + | CREATE TABLE $normalTable(id INT, name STRING, city STRING, age INT, + | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8 STRING) + | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128', 'SORT_COLUMNS'='s1') + | """.stripMargin) + sql( + s""" + | CREATE DATAMAP $datamap2 ON TABLE $normalTable + | USING 'bloomfilter' + | DMProperties('INDEX_COLUMNS'='id, name, city', 'BLOOM_SIZE'='64000', 'BLOOM_FPP'='0.00001') + """.stripMargin) + + (0 until iterations).foreach { p => + sql( + s""" + | LOAD DATA LOCAL INPATH '${bigFile}' INTO TABLE $bloomDMSampleTable + | OPTIONS('header'='false') + """.stripMargin) + sql( + s""" + | LOAD DATA LOCAL INPATH '${bigFile}' INTO TABLE $normalTable + | OPTIONS('header'='false') + """.stripMargin) + } + + var res = sql(s"explain select * from $bloomDMSampleTable where id = 1 and city = 'city_1' and name='n1'") + checkExistence(res, true, datamap11, datamap12, datamap13) + res = sql(s"explain select * from $normalTable where id = 1 and city = 'city_1' and name='n1'") + checkExistence(res, true, datamap2) + // in the following cases, default blocklet datamap will prune all the blocklets + // and bloomfilter datamap will not take effects + res = sql(s"explain select * from $bloomDMSampleTable where id < 0") + checkExistence(res, false, datamap11, datamap12, datamap13) + res = sql(s"explain select * from $normalTable where id < 0") + checkExistence(res, false, datamap2) + + // we do not care about the datamap name here, only to validate the query results are them same + checkQuery("fakeDm", shouldHit = false) + } + override protected def afterAll(): Unit = { deleteFile(bigFile) deleteFile(smallFile)
