[CARBONDATA-2649] Add code for caching min/max only for specified columns 1. Supported configuring column for caching min/max in driver 2. Added test cases for query verification for COLUMN_META_CACHE and CACHE_LEVEL properties 3. Handled comments for PR #2454
This closes #2467 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dc293191 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dc293191 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dc293191 Branch: refs/heads/master Commit: dc2931917d04e14df892d22ba148c4f9f1736d3f Parents: 57b4571 Author: manishgupta88 <[email protected]> Authored: Sun Jul 8 14:24:21 2018 +0530 Committer: kunal642 <[email protected]> Committed: Wed Jul 11 13:44:32 2018 +0530 ---------------------------------------------------------------------- .../block/SegmentPropertiesAndSchemaHolder.java | 41 +-- .../indexstore/BlockletDataMapIndexStore.java | 13 +- .../core/indexstore/BlockletDetailInfo.java | 2 +- .../indexstore/blockletindex/BlockDataMap.java | 194 +++++++------- .../blockletindex/BlockletDataMap.java | 100 ++++--- .../blockletindex/BlockletDataMapModel.java | 9 + .../core/indexstore/schema/SchemaGenerator.java | 77 +++++- .../core/metadata/schema/table/CarbonTable.java | 75 ++++-- .../carbondata/core/scan/filter/FilterUtil.java | 180 ++++++++++++- .../executer/IncludeFilterExecuterImpl.java | 9 +- .../executer/RangeValueFilterExecuterImpl.java | 2 +- .../executer/RowLevelFilterExecuterImpl.java | 34 ++- .../RowLevelRangeGrtThanFiterExecuterImpl.java | 10 +- ...elRangeGrtrThanEquaToFilterExecuterImpl.java | 10 +- ...velRangeLessThanEqualFilterExecuterImpl.java | 10 +- ...RowLevelRangeLessThanFilterExecuterImpl.java | 10 +- .../resolverinfo/ColumnResolvedFilterInfo.java | 40 ++- .../DimColumnResolvedFilterInfo.java | 11 +- .../MeasureColumnResolvedFilterInfo.java | 8 +- .../core/util/BlockletDataMapUtil.java | 41 +++ ...ithColumnMetCacheAndCacheLevelProperty.scala | 260 +++++++++++++++++++ ...ithColumnMetCacheAndCacheLevelProperty.scala | 20 +- .../table/CarbonDescribeFormattedCommand.scala | 2 +- .../org/apache/spark/util/AlterTableUtil.scala | 19 +- 24 files changed, 908 insertions(+), 269 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesAndSchemaHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesAndSchemaHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesAndSchemaHolder.java index da76bc6..e094076 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesAndSchemaHolder.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesAndSchemaHolder.java @@ -30,7 +30,10 @@ import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.indexstore.schema.CarbonRowSchema; import org.apache.carbondata.core.indexstore.schema.SchemaGenerator; +import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; /** @@ -87,20 +90,20 @@ public class SegmentPropertiesAndSchemaHolder { * Method to add the segment properties and avoid construction of new segment properties until * the schema is not modified * - * @param tableIdentifier + * @param carbonTable * @param columnsInTable * @param columnCardinality * @param segmentId */ - public int addSegmentProperties(AbsoluteTableIdentifier tableIdentifier, + public int addSegmentProperties(CarbonTable carbonTable, List<ColumnSchema> columnsInTable, int[] columnCardinality, String segmentId) { SegmentPropertiesAndSchemaHolder.SegmentPropertiesWrapper segmentPropertiesWrapper = - new SegmentPropertiesAndSchemaHolder.SegmentPropertiesWrapper(tableIdentifier, + new SegmentPropertiesAndSchemaHolder.SegmentPropertiesWrapper(carbonTable, columnsInTable, columnCardinality); SegmentIdAndSegmentPropertiesIndexWrapper segmentIdSetAndIndexWrapper = this.segmentPropWrapperToSegmentSetMap.get(segmentPropertiesWrapper); if (null == segmentIdSetAndIndexWrapper) { - synchronized (getOrCreateTableLock(tableIdentifier)) { + synchronized (getOrCreateTableLock(carbonTable.getAbsoluteTableIdentifier())) { segmentIdSetAndIndexWrapper = this.segmentPropWrapperToSegmentSetMap.get(segmentPropertiesWrapper); if (null == segmentIdSetAndIndexWrapper) { @@ -109,7 +112,7 @@ public class SegmentPropertiesAndSchemaHolder { int segmentPropertiesIndex = segmentPropertiesIndexCounter.incrementAndGet(); indexToSegmentPropertiesWrapperMapping .put(segmentPropertiesIndex, segmentPropertiesWrapper); - LOGGER.info("Constructing new SegmentProperties for table: " + tableIdentifier + LOGGER.info("Constructing new SegmentProperties for table: " + carbonTable .getCarbonTableIdentifier().getTableUniqueName() + ". Current size of segment properties" + " holder list is: " + indexToSegmentPropertiesWrapperMapping.size()); @@ -123,7 +126,7 @@ public class SegmentPropertiesAndSchemaHolder { } } } else { - synchronized (getOrCreateTableLock(tableIdentifier)) { + synchronized (getOrCreateTableLock(carbonTable.getAbsoluteTableIdentifier())) { segmentIdSetAndIndexWrapper.addSegmentId(segmentId); } } @@ -244,13 +247,14 @@ public class SegmentPropertiesAndSchemaHolder { private List<ColumnSchema> columnsInTable; private int[] columnCardinality; private SegmentProperties segmentProperties; - private CarbonRowSchema[] taskSummarySchema; + private List<CarbonColumn> minMaxCacheColumns; - public SegmentPropertiesWrapper(AbsoluteTableIdentifier tableIdentifier, + public SegmentPropertiesWrapper(CarbonTable carbonTable, List<ColumnSchema> columnsInTable, int[] columnCardinality) { - this.tableIdentifier = tableIdentifier; + this.tableIdentifier = carbonTable.getAbsoluteTableIdentifier(); this.columnsInTable = columnsInTable; this.columnCardinality = columnCardinality; + this.minMaxCacheColumns = carbonTable.getMinMaxCacheColumns(); } public void initSegmentProperties() { @@ -289,20 +293,23 @@ public class SegmentPropertiesAndSchemaHolder { return columnCardinality; } - public CarbonRowSchema[] getBlockSchema() { - return SchemaGenerator.createBlockSchema(segmentProperties); + public CarbonRowSchema[] getTaskSummarySchema(boolean storeBlockletCount, + boolean filePathToBeStored) throws MemoryException { + return SchemaGenerator + .createTaskSummarySchema(segmentProperties, minMaxCacheColumns, storeBlockletCount, + filePathToBeStored); } - public CarbonRowSchema[] getBlocketSchema() { - return SchemaGenerator.createBlockletSchema(segmentProperties); + public CarbonRowSchema[] getBlockFileFooterEntrySchema() { + return SchemaGenerator.createBlockSchema(segmentProperties, minMaxCacheColumns); } - public CarbonRowSchema[] getTaskSummarySchema() { - return taskSummarySchema; + public CarbonRowSchema[] getBlockletFileFooterEntrySchema() { + return SchemaGenerator.createBlockletSchema(segmentProperties, minMaxCacheColumns); } - public void setTaskSummarySchema(CarbonRowSchema[] taskSummarySchema) { - this.taskSummarySchema = taskSummarySchema; + public List<CarbonColumn> getMinMaxCacheColumns() { + return minMaxCacheColumns; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java index 8b5eb06..33e624d 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java @@ -190,16 +190,13 @@ public class BlockletDataMapIndexStore if (null != blockletDataMapIndexWrapper) { // clear the segmentProperties cache List<BlockDataMap> dataMaps = blockletDataMapIndexWrapper.getDataMaps(); - if (null != dataMaps) { + if (null != dataMaps && !dataMaps.isEmpty()) { String segmentId = tableSegmentUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier().getSegmentId(); - for (BlockDataMap dataMap : dataMaps) { - // as segmentId will be same for all the dataMaps and segmentProperties cache is - // maintained at segment level so it need to be called only once for clearing - SegmentPropertiesAndSchemaHolder.getInstance() - .invalidate(segmentId, dataMap.getSegmentPropertiesIndex()); - break; - } + // as segmentId will be same for all the dataMaps and segmentProperties cache is + // maintained at segment level so it need to be called only once for clearing + SegmentPropertiesAndSchemaHolder.getInstance() + .invalidate(segmentId, dataMaps.get(0).getSegmentPropertiesIndex()); } } lruCache.remove(tableSegmentUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier() http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java index 3cd86b0..2425bda 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java @@ -223,7 +223,7 @@ public class BlockletDetailInfo implements Serializable, Writable { * @throws IOException */ public void readColumnSchema(byte[] schemaArray) throws IOException { - if (null != columnSchemaBinary) { + if (null != schemaArray) { columnSchemas = BlockletDataMapUtil.readColumnSchema(schemaArray); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java index 4fe3daa..f8126cc 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java @@ -16,9 +16,7 @@ */ package org.apache.carbondata.core.indexstore.blockletindex; -import java.io.IOException; -import java.io.Serializable; -import java.io.UnsupportedEncodingException; +import java.io.*; import java.util.ArrayList; import java.util.BitSet; import java.util.List; @@ -42,11 +40,11 @@ import org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore; import org.apache.carbondata.core.indexstore.row.DataMapRow; import org.apache.carbondata.core.indexstore.row.DataMapRowImpl; import org.apache.carbondata.core.indexstore.schema.CarbonRowSchema; -import org.apache.carbondata.core.indexstore.schema.SchemaGenerator; import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.metadata.blocklet.DataFileFooter; import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex; import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; import org.apache.carbondata.core.profiler.ExplainCollector; import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor; @@ -126,11 +124,13 @@ public class BlockDataMap extends CoarseGrainDataMap isLegacyStore = fileFooter.getBlockletList() == null; // init segment properties and create schema SegmentProperties segmentProperties = initSegmentProperties(blockletDataMapInfo, fileFooter); - createMemoryDMStore(blockletDataMapInfo.isAddToUnsafe()); - createSummarySchema(segmentProperties, blockletDataMapInfo.isAddToUnsafe()); + createMemorySchema(blockletDataMapInfo); + createSummaryDMStore(blockletDataMapInfo); + CarbonRowSchema[] taskSummarySchema = getTaskSummarySchema(); // check for legacy store and load the metadata - DataMapRowImpl summaryRow = loadMetadata(segmentProperties, blockletDataMapInfo, indexInfo); - finishWriting(filePath, fileName, segmentId, summaryRow); + DataMapRowImpl summaryRow = + loadMetadata(taskSummarySchema, segmentProperties, blockletDataMapInfo, indexInfo); + finishWriting(taskSummarySchema, filePath, fileName, segmentId, summaryRow); } if (LOGGER.isDebugEnabled()) { LOGGER.debug( @@ -139,13 +139,14 @@ public class BlockDataMap extends CoarseGrainDataMap } } - private void finishWriting(byte[] filePath, byte[] fileName, byte[] segmentId, - DataMapRowImpl summaryRow) throws MemoryException { + private void finishWriting(CarbonRowSchema[] taskSummarySchema, byte[] filePath, byte[] fileName, + byte[] segmentId, DataMapRowImpl summaryRow) throws MemoryException { if (memoryDMStore != null) { memoryDMStore.finishWriting(); } if (null != taskSummaryDMStore) { - addTaskSummaryRowToUnsafeMemoryStore(summaryRow, filePath, fileName, segmentId); + addTaskSummaryRowToUnsafeMemoryStore(taskSummarySchema, summaryRow, filePath, fileName, + segmentId); taskSummaryDMStore.finishWriting(); } } @@ -159,13 +160,15 @@ public class BlockDataMap extends CoarseGrainDataMap * @throws IOException * @throws MemoryException */ - protected DataMapRowImpl loadMetadata(SegmentProperties segmentProperties, - BlockletDataMapModel blockletDataMapInfo, List<DataFileFooter> indexInfo) - throws IOException, MemoryException { + protected DataMapRowImpl loadMetadata(CarbonRowSchema[] taskSummarySchema, + SegmentProperties segmentProperties, BlockletDataMapModel blockletDataMapInfo, + List<DataFileFooter> indexInfo) throws IOException, MemoryException { if (isLegacyStore) { - return loadBlockInfoForOldStore(segmentProperties, blockletDataMapInfo, indexInfo); + return loadBlockInfoForOldStore(taskSummarySchema, segmentProperties, blockletDataMapInfo, + indexInfo); } else { - return loadBlockMetaInfo(segmentProperties, blockletDataMapInfo, indexInfo); + return loadBlockMetaInfo(taskSummarySchema, segmentProperties, blockletDataMapInfo, + indexInfo); } } @@ -180,7 +183,7 @@ public class BlockDataMap extends CoarseGrainDataMap List<ColumnSchema> columnInTable = fileFooter.getColumnInTable(); int[] columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality(); segmentPropertiesIndex = SegmentPropertiesAndSchemaHolder.getInstance() - .addSegmentProperties(blockletDataMapInfo.getCarbonTable().getAbsoluteTableIdentifier(), + .addSegmentProperties(blockletDataMapInfo.getCarbonTable(), columnInTable, columnCardinality, blockletDataMapInfo.getSegmentId()); return getSegmentProperties(); } @@ -194,10 +197,11 @@ public class BlockDataMap extends CoarseGrainDataMap * @throws IOException * @throws MemoryException */ - protected DataMapRowImpl loadBlockInfoForOldStore(SegmentProperties segmentProperties, - BlockletDataMapModel blockletDataMapInfo, List<DataFileFooter> indexInfo) - throws IOException, MemoryException { + protected DataMapRowImpl loadBlockInfoForOldStore(CarbonRowSchema[] taskSummarySchema, + SegmentProperties segmentProperties, BlockletDataMapModel blockletDataMapInfo, + List<DataFileFooter> indexInfo) throws IOException, MemoryException { DataMapRowImpl summaryRow = null; + CarbonRowSchema[] schema = getFileFooterEntrySchema(); for (DataFileFooter fileFooter : indexInfo) { TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo(); BlockMetaInfo blockMetaInfo = @@ -219,9 +223,9 @@ public class BlockDataMap extends CoarseGrainDataMap CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, true); byte[][] updatedMaxValues = CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, false); - summaryRow = - loadToUnsafeBlock(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow, - blockMetaInfo, updatedMinValues, updatedMaxValues); + summaryRow = loadToUnsafeBlock(schema, taskSummarySchema, fileFooter, segmentProperties, + blockletDataMapInfo.getMinMaxCacheColumns(), blockInfo.getFilePath(), summaryRow, + blockMetaInfo, updatedMinValues, updatedMaxValues); } } return summaryRow; @@ -235,9 +239,9 @@ public class BlockDataMap extends CoarseGrainDataMap * @throws IOException * @throws MemoryException */ - private DataMapRowImpl loadBlockMetaInfo(SegmentProperties segmentProperties, - BlockletDataMapModel blockletDataMapInfo, List<DataFileFooter> indexInfo) - throws IOException, MemoryException { + private DataMapRowImpl loadBlockMetaInfo(CarbonRowSchema[] taskSummarySchema, + SegmentProperties segmentProperties, BlockletDataMapModel blockletDataMapInfo, + List<DataFileFooter> indexInfo) throws IOException, MemoryException { String tempFilePath = null; DataFileFooter previousDataFileFooter = null; int footerCounter = 0; @@ -247,6 +251,7 @@ public class BlockDataMap extends CoarseGrainDataMap List<Byte> blockletCountInEachBlock = new ArrayList<>(indexInfo.size()); byte totalBlockletsInOneBlock = 0; boolean isLastFileFooterEntryNeedToBeAdded = false; + CarbonRowSchema[] schema = getFileFooterEntrySchema(); for (DataFileFooter fileFooter : indexInfo) { TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo(); BlockMetaInfo blockMetaInfo = @@ -267,10 +272,11 @@ public class BlockDataMap extends CoarseGrainDataMap // min and max at block level. So compare min and max values and update if required BlockletMinMaxIndex currentFooterMinMaxIndex = fileFooter.getBlockletIndex().getMinMaxIndex(); - blockMinValues = compareAndUpdateMinMax(segmentProperties.getColumnsValueSize(), - currentFooterMinMaxIndex.getMinValues(), blockMinValues, true); - blockMaxValues = compareAndUpdateMinMax(segmentProperties.getColumnsValueSize(), - currentFooterMinMaxIndex.getMaxValues(), blockMaxValues, false); + blockMinValues = + compareAndUpdateMinMax(currentFooterMinMaxIndex.getMinValues(), blockMinValues, true); + blockMaxValues = + compareAndUpdateMinMax(currentFooterMinMaxIndex.getMaxValues(), blockMaxValues, + false); totalBlockletsInOneBlock++; } // as one task contains entries for all the blocklets we need iterate and load only the @@ -279,7 +285,8 @@ public class BlockDataMap extends CoarseGrainDataMap if (!blockInfo.getFilePath().equals(tempFilePath) || footerCounter == indexInfo.size()) { TableBlockInfo previousBlockInfo = previousDataFileFooter.getBlockInfo().getTableBlockInfo(); - summaryRow = loadToUnsafeBlock(previousDataFileFooter, segmentProperties, + summaryRow = loadToUnsafeBlock(schema, taskSummarySchema, previousDataFileFooter, + segmentProperties, blockletDataMapInfo.getMinMaxCacheColumns(), previousBlockInfo.getFilePath(), summaryRow, blockletDataMapInfo.getBlockMetaInfoMap().get(previousBlockInfo.getFilePath()), blockMinValues, blockMaxValues); @@ -302,17 +309,19 @@ public class BlockDataMap extends CoarseGrainDataMap } // add the last file footer entry if (isLastFileFooterEntryNeedToBeAdded) { - summaryRow = loadToUnsafeBlock(previousDataFileFooter, segmentProperties, - previousDataFileFooter.getBlockInfo().getTableBlockInfo().getFilePath(), summaryRow, - blockletDataMapInfo.getBlockMetaInfoMap() - .get(previousDataFileFooter.getBlockInfo().getTableBlockInfo().getFilePath()), - blockMinValues, blockMaxValues); + summaryRow = + loadToUnsafeBlock(schema, taskSummarySchema, previousDataFileFooter, segmentProperties, + blockletDataMapInfo.getMinMaxCacheColumns(), + previousDataFileFooter.getBlockInfo().getTableBlockInfo().getFilePath(), summaryRow, + blockletDataMapInfo.getBlockMetaInfoMap() + .get(previousDataFileFooter.getBlockInfo().getTableBlockInfo().getFilePath()), + blockMinValues, blockMaxValues); blockletCountInEachBlock.add(totalBlockletsInOneBlock); } byte[] blockletCount = ArrayUtils .toPrimitive(blockletCountInEachBlock.toArray(new Byte[blockletCountInEachBlock.size()])); // blocklet count index is the last index - summaryRow.setByteArray(blockletCount, getTaskSummarySchema().length - 1); + summaryRow.setByteArray(blockletCount, taskSummarySchema.length - 1); return summaryRow; } @@ -328,12 +337,11 @@ public class BlockDataMap extends CoarseGrainDataMap * where blocklet information is not available in index file. So load only block information * and read blocklet information in executor. */ - protected DataMapRowImpl loadToUnsafeBlock(DataFileFooter fileFooter, - SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow, - BlockMetaInfo blockMetaInfo, byte[][] minValues, byte[][] maxValues) { - int[] minMaxLen = segmentProperties.getColumnsValueSize(); - CarbonRowSchema[] schema = getSchema(); - CarbonRowSchema[] taskSummarySchema = getTaskSummarySchema(); + protected DataMapRowImpl loadToUnsafeBlock(CarbonRowSchema[] schema, + CarbonRowSchema[] taskSummarySchema, DataFileFooter fileFooter, + SegmentProperties segmentProperties, List<CarbonColumn> minMaxCacheColumns, String filePath, + DataMapRowImpl summaryRow, BlockMetaInfo blockMetaInfo, byte[][] minValues, + byte[][] maxValues) { // Add one row to maintain task level min max for segment pruning if (summaryRow == null) { summaryRow = new DataMapRowImpl(taskSummarySchema); @@ -341,16 +349,21 @@ public class BlockDataMap extends CoarseGrainDataMap DataMapRow row = new DataMapRowImpl(schema); int ordinal = 0; int taskMinMaxOrdinal = 0; - row.setRow(addMinMax(minMaxLen, schema[ordinal], minValues), ordinal); + // get min max values for columns to be cached + byte[][] minValuesForColumnsToBeCached = BlockletDataMapUtil + .getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, minValues); + byte[][] maxValuesForColumnsToBeCached = BlockletDataMapUtil + .getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, maxValues); + row.setRow(addMinMax(schema[ordinal], minValuesForColumnsToBeCached), ordinal); // compute and set task level min values - addTaskMinMaxValues(summaryRow, minMaxLen, taskSummarySchema, taskMinMaxOrdinal, - minValues, TASK_MIN_VALUES_INDEX, true); + addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal, + minValuesForColumnsToBeCached, TASK_MIN_VALUES_INDEX, true); ordinal++; taskMinMaxOrdinal++; - row.setRow(addMinMax(minMaxLen, schema[ordinal], maxValues), ordinal); + row.setRow(addMinMax(schema[ordinal], maxValuesForColumnsToBeCached), ordinal); // compute and set task level max values - addTaskMinMaxValues(summaryRow, minMaxLen, taskSummarySchema, taskMinMaxOrdinal, - maxValues, TASK_MAX_VALUES_INDEX, false); + addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal, + maxValuesForColumnsToBeCached, TASK_MAX_VALUES_INDEX, false); ordinal++; // add total rows in one carbondata file row.setInt((int) fileFooter.getNumberOfRows(), ordinal++); @@ -397,8 +410,8 @@ public class BlockDataMap extends CoarseGrainDataMap return fileName; } - private void addTaskSummaryRowToUnsafeMemoryStore(DataMapRow summaryRow, byte[] filePath, - byte[] fileName, byte[] segmentId) { + private void addTaskSummaryRowToUnsafeMemoryStore(CarbonRowSchema[] taskSummarySchema, + DataMapRow summaryRow, byte[] filePath, byte[] fileName, byte[] segmentId) { // write the task summary info to unsafe memory store if (null != summaryRow) { summaryRow.setByteArray(fileName, SUMMARY_INDEX_FILE_NAME); @@ -407,21 +420,21 @@ public class BlockDataMap extends CoarseGrainDataMap summaryRow.setByteArray(filePath, SUMMARY_INDEX_PATH); } try { - taskSummaryDMStore.addIndexRow(getTaskSummarySchema(), summaryRow); + taskSummaryDMStore.addIndexRow(taskSummarySchema, summaryRow); } catch (Exception e) { throw new RuntimeException(e); } } } - protected DataMapRow addMinMax(int[] minMaxLen, CarbonRowSchema carbonRowSchema, + protected DataMapRow addMinMax(CarbonRowSchema carbonRowSchema, byte[][] minValues) { CarbonRowSchema[] minSchemas = ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas(); DataMapRow minRow = new DataMapRowImpl(minSchemas); int minOrdinal = 0; // min value adding - for (int i = 0; i < minMaxLen.length; i++) { + for (int i = 0; i < minValues.length; i++) { minRow.setByteArray(minValues[i], minOrdinal++); } return minRow; @@ -431,16 +444,14 @@ public class BlockDataMap extends CoarseGrainDataMap * This method will compute min/max values at task level * * @param taskMinMaxRow - * @param minMaxLen * @param carbonRowSchema * @param taskMinMaxOrdinal * @param minMaxValue * @param ordinal * @param isMinValueComparison */ - protected void addTaskMinMaxValues(DataMapRow taskMinMaxRow, int[] minMaxLen, - CarbonRowSchema[] carbonRowSchema, int taskMinMaxOrdinal, byte[][] minMaxValue, int ordinal, - boolean isMinValueComparison) { + protected void addTaskMinMaxValues(DataMapRow taskMinMaxRow, CarbonRowSchema[] carbonRowSchema, + int taskMinMaxOrdinal, byte[][] minMaxValue, int ordinal, boolean isMinValueComparison) { DataMapRow row = taskMinMaxRow.getRow(ordinal); byte[][] updatedMinMaxValues = null; if (null == row) { @@ -451,12 +462,12 @@ public class BlockDataMap extends CoarseGrainDataMap updatedMinMaxValues = minMaxValue; } else { byte[][] existingMinMaxValues = getMinMaxValue(taskMinMaxRow, ordinal); - updatedMinMaxValues = compareAndUpdateMinMax(minMaxLen, minMaxValue, existingMinMaxValues, - isMinValueComparison); + updatedMinMaxValues = + compareAndUpdateMinMax(minMaxValue, existingMinMaxValues, isMinValueComparison); } int minMaxOrdinal = 0; // min/max value adding - for (int i = 0; i < minMaxLen.length; i++) { + for (int i = 0; i < updatedMinMaxValues.length; i++) { row.setByteArray(updatedMinMaxValues[i], minMaxOrdinal++); } taskMinMaxRow.setRow(row, ordinal); @@ -465,17 +476,16 @@ public class BlockDataMap extends CoarseGrainDataMap /** * This method will do min/max comparison of values and update if required * - * @param minMaxLen * @param minMaxValueCompare1 * @param minMaxValueCompare2 * @param isMinValueComparison */ - private byte[][] compareAndUpdateMinMax(int[] minMaxLen, byte[][] minMaxValueCompare1, + private byte[][] compareAndUpdateMinMax(byte[][] minMaxValueCompare1, byte[][] minMaxValueCompare2, boolean isMinValueComparison) { // Compare and update min max values - byte[][] updatedMinMaxValues = new byte[minMaxLen.length][]; + byte[][] updatedMinMaxValues = new byte[minMaxValueCompare1.length][]; System.arraycopy(minMaxValueCompare1, 0, updatedMinMaxValues, 0, minMaxValueCompare1.length); - for (int i = 0; i < minMaxLen.length; i++) { + for (int i = 0; i < minMaxValueCompare1.length; i++) { int compare = ByteUtil.UnsafeComparer.INSTANCE .compareTo(minMaxValueCompare2[i], minMaxValueCompare1[i]); if (isMinValueComparison) { @@ -489,8 +499,9 @@ public class BlockDataMap extends CoarseGrainDataMap return updatedMinMaxValues; } - protected void createMemoryDMStore(boolean addToUnsafe) throws MemoryException { - memoryDMStore = getMemoryDMStore(addToUnsafe); + protected void createMemorySchema(BlockletDataMapModel blockletDataMapModel) + throws MemoryException { + memoryDMStore = getMemoryDMStore(blockletDataMapModel.isAddToUnsafe()); } /** @@ -498,26 +509,16 @@ public class BlockDataMap extends CoarseGrainDataMap * once per datamap. It stores datamap level max/min of each column and partition information of * datamap * - * @param segmentProperties * @throws MemoryException */ - protected void createSummarySchema(SegmentProperties segmentProperties, boolean addToUnsafe) + protected void createSummaryDMStore(BlockletDataMapModel blockletDataMapModel) throws MemoryException { - // flag to check whether it is required to store blocklet count of each carbondata file as - // binary in summary schema. This will be true when it is not a legacy store (>1.1 version) - // and CACHE_LEVEL=BLOCK - boolean storeBlockletCount = !isLegacyStore; - CarbonRowSchema[] taskSummarySchema = SchemaGenerator - .createTaskSummarySchema(segmentProperties, storeBlockletCount, isFilePathStored); - SegmentPropertiesAndSchemaHolder.getInstance() - .getSegmentPropertiesWrapper(segmentPropertiesIndex) - .setTaskSummarySchema(taskSummarySchema); - taskSummaryDMStore = getMemoryDMStore(addToUnsafe); + taskSummaryDMStore = getMemoryDMStore(blockletDataMapModel.isAddToUnsafe()); } @Override public boolean isScanRequired(FilterResolverIntf filterExp) { - FilterExecuter filterExecuter = - FilterUtil.getFilterExecuterTree(filterExp, getSegmentProperties(), null); + FilterExecuter filterExecuter = FilterUtil + .getFilterExecuterTree(filterExp, getSegmentProperties(), null, getMinMaxCacheColumns()); DataMapRow unsafeRow = taskSummaryDMStore .getDataMapRow(getTaskSummarySchema(), taskSummaryDMStore.getRowCount() - 1); boolean isScanRequired = FilterExpressionProcessor @@ -529,12 +530,17 @@ public class BlockDataMap extends CoarseGrainDataMap return false; } + private List<CarbonColumn> getMinMaxCacheColumns() { + return SegmentPropertiesAndSchemaHolder.getInstance() + .getSegmentPropertiesWrapper(segmentPropertiesIndex).getMinMaxCacheColumns(); + } + private List<Blocklet> prune(FilterResolverIntf filterExp) { if (memoryDMStore.getRowCount() == 0) { return new ArrayList<>(); } List<Blocklet> blocklets = new ArrayList<>(); - CarbonRowSchema[] schema = getSchema(); + CarbonRowSchema[] schema = getFileFooterEntrySchema(); String filePath = getFilePath(); int numBlocklets = 0; if (filterExp == null) { @@ -549,8 +555,8 @@ public class BlockDataMap extends CoarseGrainDataMap // correct for old store scenarios int startIndex = 0; numBlocklets = memoryDMStore.getRowCount(); - FilterExecuter filterExecuter = - FilterUtil.getFilterExecuterTree(filterExp, getSegmentProperties(), null); + FilterExecuter filterExecuter = FilterUtil + .getFilterExecuterTree(filterExp, getSegmentProperties(), null, getMinMaxCacheColumns()); while (startIndex < numBlocklets) { DataMapRow safeRow = memoryDMStore.getDataMapRow(schema, startIndex).convertToSafeRow(); String fileName = getFileNameWithFilePath(safeRow, filePath); @@ -695,7 +701,8 @@ public class BlockDataMap extends CoarseGrainDataMap rowIndex++; } } - DataMapRow safeRow = memoryDMStore.getDataMapRow(getSchema(), rowIndex).convertToSafeRow(); + DataMapRow safeRow = + memoryDMStore.getDataMapRow(getFileFooterEntrySchema(), rowIndex).convertToSafeRow(); String filePath = getFilePath(); return createBlocklet(safeRow, getFileNameWithFilePath(safeRow, filePath), relativeBlockletId); } @@ -830,14 +837,20 @@ public class BlockDataMap extends CoarseGrainDataMap return memoryDMStore; } - protected CarbonRowSchema[] getSchema() { + protected CarbonRowSchema[] getFileFooterEntrySchema() { return SegmentPropertiesAndSchemaHolder.getInstance() - .getSegmentPropertiesWrapper(segmentPropertiesIndex).getBlockSchema(); + .getSegmentPropertiesWrapper(segmentPropertiesIndex).getBlockFileFooterEntrySchema(); } protected CarbonRowSchema[] getTaskSummarySchema() { - return SegmentPropertiesAndSchemaHolder.getInstance() - .getSegmentPropertiesWrapper(segmentPropertiesIndex).getTaskSummarySchema(); + SegmentPropertiesAndSchemaHolder.SegmentPropertiesWrapper segmentPropertiesWrapper = + SegmentPropertiesAndSchemaHolder.getInstance() + .getSegmentPropertiesWrapper(segmentPropertiesIndex); + try { + return segmentPropertiesWrapper.getTaskSummarySchema(!isLegacyStore, isFilePathStored); + } catch (MemoryException e) { + throw new RuntimeException(e); + } } /** @@ -847,7 +860,8 @@ public class BlockDataMap extends CoarseGrainDataMap */ public void convertToUnsafeDMStore() throws MemoryException { if (memoryDMStore instanceof SafeMemoryDMStore) { - UnsafeMemoryDMStore unsafeMemoryDMStore = memoryDMStore.convertToUnsafeDMStore(getSchema()); + UnsafeMemoryDMStore unsafeMemoryDMStore = memoryDMStore.convertToUnsafeDMStore( + getFileFooterEntrySchema()); memoryDMStore.freeMemory(); memoryDMStore = unsafeMemoryDMStore; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java index 5274b0e..d53e936 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java @@ -34,11 +34,12 @@ import org.apache.carbondata.core.indexstore.ExtendedBlocklet; import org.apache.carbondata.core.indexstore.row.DataMapRow; import org.apache.carbondata.core.indexstore.row.DataMapRowImpl; import org.apache.carbondata.core.indexstore.schema.CarbonRowSchema; -import org.apache.carbondata.core.indexstore.schema.SchemaGenerator; import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; import org.apache.carbondata.core.metadata.blocklet.DataFileFooter; import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; +import org.apache.carbondata.core.util.BlockletDataMapUtil; /** * Datamap implementation for blocklet. @@ -59,42 +60,32 @@ public class BlockletDataMap extends BlockDataMap implements Serializable { * @throws IOException * @throws MemoryException */ - protected DataMapRowImpl loadMetadata(SegmentProperties segmentProperties, - BlockletDataMapModel blockletDataMapInfo, List<DataFileFooter> indexInfo) - throws IOException, MemoryException { + protected DataMapRowImpl loadMetadata(CarbonRowSchema[] taskSummarySchema, + SegmentProperties segmentProperties, BlockletDataMapModel blockletDataMapInfo, + List<DataFileFooter> indexInfo) throws IOException, MemoryException { if (isLegacyStore) { - return loadBlockInfoForOldStore(segmentProperties, blockletDataMapInfo, indexInfo); + return loadBlockInfoForOldStore(taskSummarySchema, segmentProperties, blockletDataMapInfo, + indexInfo); } else { - return loadBlockletMetaInfo(segmentProperties, blockletDataMapInfo, indexInfo); + return loadBlockletMetaInfo(taskSummarySchema, segmentProperties, blockletDataMapInfo, + indexInfo); } } - /** - * Method to create blocklet schema - * - * @param addToUnsafe - * @throws MemoryException - */ - protected void createMemoryDMStore(boolean addToUnsafe) throws MemoryException { - memoryDMStore = getMemoryDMStore(addToUnsafe); + protected CarbonRowSchema[] getTaskSummarySchema() { + SegmentPropertiesAndSchemaHolder.SegmentPropertiesWrapper segmentPropertiesWrapper = + SegmentPropertiesAndSchemaHolder.getInstance() + .getSegmentPropertiesWrapper(segmentPropertiesIndex); + try { + return segmentPropertiesWrapper.getTaskSummarySchema(false, isFilePathStored); + } catch (MemoryException e) { + throw new RuntimeException(e); + } } - /** - * Creates the schema to store summary information or the information which can be stored only - * once per datamap. It stores datamap level max/min of each column and partition information of - * datamap - * - * @param segmentProperties - * @throws MemoryException - */ - protected void createSummarySchema(SegmentProperties segmentProperties, boolean addToUnsafe) - throws MemoryException { - CarbonRowSchema[] taskSummarySchema = - SchemaGenerator.createTaskSummarySchema(segmentProperties, false, isFilePathStored); - SegmentPropertiesAndSchemaHolder.getInstance() - .getSegmentPropertiesWrapper(segmentPropertiesIndex) - .setTaskSummarySchema(taskSummarySchema); - taskSummaryDMStore = getMemoryDMStore(addToUnsafe); + protected CarbonRowSchema[] getFileFooterEntrySchema() { + return SegmentPropertiesAndSchemaHolder.getInstance() + .getSegmentPropertiesWrapper(segmentPropertiesIndex).getBlockletFileFooterEntrySchema(); } /** @@ -105,11 +96,12 @@ public class BlockletDataMap extends BlockDataMap implements Serializable { * @throws IOException * @throws MemoryException */ - private DataMapRowImpl loadBlockletMetaInfo(SegmentProperties segmentProperties, - BlockletDataMapModel blockletDataMapInfo, List<DataFileFooter> indexInfo) - throws IOException, MemoryException { + private DataMapRowImpl loadBlockletMetaInfo(CarbonRowSchema[] taskSummarySchema, + SegmentProperties segmentProperties, BlockletDataMapModel blockletDataMapInfo, + List<DataFileFooter> indexInfo) throws IOException, MemoryException { String tempFilePath = null; DataMapRowImpl summaryRow = null; + CarbonRowSchema[] schema = getFileFooterEntrySchema(); // Relative blocklet ID is the id assigned to a blocklet within a part file int relativeBlockletId = 0; for (DataFileFooter fileFooter : indexInfo) { @@ -127,9 +119,9 @@ public class BlockletDataMap extends BlockDataMap implements Serializable { tempFilePath = blockInfo.getFilePath(); relativeBlockletId = 0; } - summaryRow = - loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow, - blockMetaInfo, relativeBlockletId); + summaryRow = loadToUnsafe(schema, taskSummarySchema, fileFooter, segmentProperties, + blockletDataMapInfo.getMinMaxCacheColumns(), blockInfo.getFilePath(), summaryRow, + blockMetaInfo, relativeBlockletId); // this is done because relative blocklet id need to be incremented based on the // total number of blocklets relativeBlockletId += fileFooter.getBlockletList().size(); @@ -138,13 +130,11 @@ public class BlockletDataMap extends BlockDataMap implements Serializable { return summaryRow; } - private DataMapRowImpl loadToUnsafe(DataFileFooter fileFooter, - SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow, + private DataMapRowImpl loadToUnsafe(CarbonRowSchema[] schema, CarbonRowSchema[] taskSummarySchema, + DataFileFooter fileFooter, SegmentProperties segmentProperties, + List<CarbonColumn> minMaxCacheColumns, String filePath, DataMapRowImpl summaryRow, BlockMetaInfo blockMetaInfo, int relativeBlockletId) { - int[] minMaxLen = segmentProperties.getColumnsValueSize(); List<BlockletInfo> blockletList = fileFooter.getBlockletList(); - CarbonRowSchema[] schema = getSchema(); - CarbonRowSchema[] taskSummarySchema = getTaskSummarySchema(); // Add one row to maintain task level min max for segment pruning if (!blockletList.isEmpty() && summaryRow == null) { summaryRow = new DataMapRowImpl(taskSummarySchema); @@ -155,16 +145,23 @@ public class BlockletDataMap extends BlockDataMap implements Serializable { int taskMinMaxOrdinal = 0; BlockletInfo blockletInfo = blockletList.get(index); BlockletMinMaxIndex minMaxIndex = blockletInfo.getBlockletIndex().getMinMaxIndex(); - row.setRow(addMinMax(minMaxLen, schema[ordinal], minMaxIndex.getMinValues()), ordinal); + // get min max values for columns to be cached + byte[][] minValuesForColumnsToBeCached = BlockletDataMapUtil + .getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, + minMaxIndex.getMinValues()); + byte[][] maxValuesForColumnsToBeCached = BlockletDataMapUtil + .getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, + minMaxIndex.getMaxValues()); + row.setRow(addMinMax(schema[ordinal], minValuesForColumnsToBeCached), ordinal); // compute and set task level min values - addTaskMinMaxValues(summaryRow, minMaxLen, taskSummarySchema, taskMinMaxOrdinal, - minMaxIndex.getMinValues(), TASK_MIN_VALUES_INDEX, true); + addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal, + minValuesForColumnsToBeCached, TASK_MIN_VALUES_INDEX, true); ordinal++; taskMinMaxOrdinal++; - row.setRow(addMinMax(minMaxLen, schema[ordinal], minMaxIndex.getMaxValues()), ordinal); + row.setRow(addMinMax(schema[ordinal], maxValuesForColumnsToBeCached), ordinal); // compute and set task level max values - addTaskMinMaxValues(summaryRow, minMaxLen, taskSummarySchema, taskMinMaxOrdinal, - minMaxIndex.getMaxValues(), TASK_MAX_VALUES_INDEX, false); + addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal, + maxValuesForColumnsToBeCached, TASK_MAX_VALUES_INDEX, false); ordinal++; row.setInt(blockletInfo.getNumberOfRows(), ordinal++); // add file name @@ -205,8 +202,8 @@ public class BlockletDataMap extends BlockDataMap implements Serializable { super.getDetailedBlocklet(blockletId); } int absoluteBlockletId = Integer.parseInt(blockletId); - DataMapRow safeRow = - memoryDMStore.getDataMapRow(getSchema(), absoluteBlockletId).convertToSafeRow(); + DataMapRow safeRow = memoryDMStore.getDataMapRow(getFileFooterEntrySchema(), absoluteBlockletId) + .convertToSafeRow(); short relativeBlockletId = safeRow.getShort(BLOCKLET_ID_INDEX); String filePath = getFilePath(); return createBlocklet(safeRow, getFileNameWithFilePath(safeRow, filePath), relativeBlockletId); @@ -216,11 +213,6 @@ public class BlockletDataMap extends BlockDataMap implements Serializable { return dataMapRow.getShort(BLOCKLET_ID_INDEX); } - protected CarbonRowSchema[] getSchema() { - return SegmentPropertiesAndSchemaHolder.getInstance() - .getSegmentPropertiesWrapper(segmentPropertiesIndex).getBlocketSchema(); - } - protected ExtendedBlocklet createBlocklet(DataMapRow row, String fileName, short blockletId) { ExtendedBlocklet blocklet = new ExtendedBlocklet(fileName, blockletId + ""); BlockletDetailInfo detailInfo = getBlockletDetailInfo(row, blockletId, blocklet); http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java index 180c812..7516204 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java @@ -16,11 +16,13 @@ */ package org.apache.carbondata.core.indexstore.blockletindex; +import java.util.List; import java.util.Map; import org.apache.carbondata.core.datamap.dev.DataMapModel; import org.apache.carbondata.core.indexstore.BlockMetaInfo; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; /** * It is the model object to keep the information to build or initialize BlockletDataMap. @@ -33,6 +35,8 @@ public class BlockletDataMapModel extends DataMapModel { private CarbonTable carbonTable; + private List<CarbonColumn> minMaxCacheColumns; + private String segmentId; private boolean addToUnsafe = true; @@ -44,6 +48,7 @@ public class BlockletDataMapModel extends DataMapModel { this.blockMetaInfoMap = blockMetaInfoMap; this.segmentId = segmentId; this.carbonTable = carbonTable; + this.minMaxCacheColumns = carbonTable.getMinMaxCacheColumns(); } public BlockletDataMapModel(CarbonTable carbonTable, String filePath, @@ -72,4 +77,8 @@ public class BlockletDataMapModel extends DataMapModel { public CarbonTable getCarbonTable() { return carbonTable; } + + public List<CarbonColumn> getMinMaxCacheColumns() { + return minMaxCacheColumns; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/indexstore/schema/SchemaGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/SchemaGenerator.java b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/SchemaGenerator.java index e256fd6..f5df4bb 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/SchemaGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/SchemaGenerator.java @@ -23,6 +23,8 @@ import java.util.List; import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; +import org.apache.carbondata.core.util.BlockletDataMapUtil; /** * class for creating schema for a given DataMap @@ -35,10 +37,11 @@ public class SchemaGenerator { * @param segmentProperties * @return */ - public static CarbonRowSchema[] createBlockSchema(SegmentProperties segmentProperties) { + public static CarbonRowSchema[] createBlockSchema(SegmentProperties segmentProperties, + List<CarbonColumn> minMaxCacheColumns) { List<CarbonRowSchema> indexSchemas = new ArrayList<>(); // get MinMax Schema - getMinMaxSchema(segmentProperties, indexSchemas); + getMinMaxSchema(segmentProperties, indexSchemas, minMaxCacheColumns); // for number of rows. indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.INT)); // for table block path @@ -63,10 +66,11 @@ public class SchemaGenerator { * @param segmentProperties * @return */ - public static CarbonRowSchema[] createBlockletSchema(SegmentProperties segmentProperties) { + public static CarbonRowSchema[] createBlockletSchema(SegmentProperties segmentProperties, + List<CarbonColumn> minMaxCacheColumns) { List<CarbonRowSchema> indexSchemas = new ArrayList<>(); // get MinMax Schema - getMinMaxSchema(segmentProperties, indexSchemas); + getMinMaxSchema(segmentProperties, indexSchemas, minMaxCacheColumns); // for number of rows. indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.INT)); // for table block path @@ -100,10 +104,11 @@ public class SchemaGenerator { * @throws MemoryException */ public static CarbonRowSchema[] createTaskSummarySchema(SegmentProperties segmentProperties, + List<CarbonColumn> minMaxCacheColumns, boolean storeBlockletCount, boolean filePathToBeStored) throws MemoryException { List<CarbonRowSchema> taskMinMaxSchemas = new ArrayList<>(); // get MinMax Schema - getMinMaxSchema(segmentProperties, taskMinMaxSchemas); + getMinMaxSchema(segmentProperties, taskMinMaxSchemas, minMaxCacheColumns); // for storing file name taskMinMaxSchemas .add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY)); @@ -134,17 +139,19 @@ public class SchemaGenerator { * @param minMaxSchemas */ private static void getMinMaxSchema(SegmentProperties segmentProperties, - List<CarbonRowSchema> minMaxSchemas) { + List<CarbonRowSchema> minMaxSchemas, List<CarbonColumn> minMaxCacheColumns) { // Index key - int[] minMaxLen = segmentProperties.getColumnsValueSize(); + int[] minMaxLen = getMinMaxLength(segmentProperties, minMaxCacheColumns); + int[] columnOrdinals = getColumnOrdinalsToAccess(segmentProperties, minMaxCacheColumns); // do it 2 times, one for min and one for max. for (int k = 0; k < 2; k++) { CarbonRowSchema[] mapSchemas = new CarbonRowSchema[minMaxLen.length]; for (int i = 0; i < minMaxLen.length; i++) { if (minMaxLen[i] <= 0) { boolean isVarchar = false; - if (i < segmentProperties.getDimensions().size() - && segmentProperties.getDimensions().get(i).getDataType() == DataTypes.VARCHAR) { + if (columnOrdinals[i] < segmentProperties.getDimensions().size() + && segmentProperties.getDimensions().get(columnOrdinals[i]).getDataType() + == DataTypes.VARCHAR) { isVarchar = true; } mapSchemas[i] = @@ -160,4 +167,56 @@ public class SchemaGenerator { minMaxSchemas.add(mapSchema); } } + + /** + * Method to get the min max length of each column. It will return the length of only column + * which will be cached + * + * @param segmentProperties + * @param minMaxCacheColumns + * @return + */ + private static int[] getMinMaxLength(SegmentProperties segmentProperties, + List<CarbonColumn> minMaxCacheColumns) { + int[] minMaxLen = null; + if (null != minMaxCacheColumns) { + minMaxLen = new int[minMaxCacheColumns.size()]; + int counter = 0; + for (CarbonColumn column : minMaxCacheColumns) { + minMaxLen[counter++] = segmentProperties.getColumnsValueSize()[BlockletDataMapUtil + .getColumnOrdinal(segmentProperties, column)]; + } + } else { + minMaxLen = segmentProperties.getColumnsValueSize(); + } + return minMaxLen; + } + + /** + * Method to fill the column ordinals to access based on the columns to be cached + * + * @param segmentProperties + * @param minMaxCacheColumns + * @return + */ + private static int[] getColumnOrdinalsToAccess(SegmentProperties segmentProperties, + List<CarbonColumn> minMaxCacheColumns) { + int[] columnOrdinalsTOAccess = null; + if (null != minMaxCacheColumns) { + columnOrdinalsTOAccess = new int[minMaxCacheColumns.size()]; + int counter = 0; + for (CarbonColumn column : minMaxCacheColumns) { + columnOrdinalsTOAccess[counter++] = + BlockletDataMapUtil.getColumnOrdinal(segmentProperties, column); + } + } else { + // when columns to cache is not specified then column access order will be same as the array + // index of min max length + columnOrdinalsTOAccess = new int[segmentProperties.getColumnsValueSize().length]; + for (int i = 0; i < columnOrdinalsTOAccess.length; i++) { + columnOrdinalsTOAccess[i] = i; + } + } + return columnOrdinalsTOAccess; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java index 4cb42e6..086cedd 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java @@ -1173,40 +1173,71 @@ public class CarbonTable implements Serializable { } /** - * Method to get the list of cached columns of the table + * Method to get the list of cached columns of the table. + * This method need to be used for Describe formatted like scenario where columns need to be + * displayed in the column create order * - * @param tableName * @return */ - public List<String> getCachedColumns(String tableName) { - List<String> cachedColsList = new ArrayList<>(tableDimensionsMap.size()); + public List<String> getMinMaxCachedColumnsInCreateOrder() { + List<String> cachedColsList = new ArrayList<>(); + String tableName = tableInfo.getFactTable().getTableName(); String cacheColumns = tableInfo.getFactTable().getTableProperties().get(CarbonCommonConstants.COLUMN_META_CACHE); if (null != cacheColumns && !cacheColumns.isEmpty()) { - List<CarbonDimension> carbonDimensions = tableDimensionsMap.get(tableName); - List<CarbonMeasure> carbonMeasures = tableMeasuresMap.get(tableName); String[] cachedCols = cacheColumns.split(","); for (String column : cachedCols) { - boolean found = false; - // this will avoid adding the columns which have been dropped from the table - for (CarbonDimension dimension : carbonDimensions) { - if (dimension.getColName().equals(column)) { - cachedColsList.add(column); - found = true; - break; - } + CarbonColumn carbonColumn = getColumnByName(tableName, column); + if (null != carbonColumn && !carbonColumn.isInvisible()) { + cachedColsList.add(carbonColumn.getColName()); } - // if column is not a dimension then check in measures - if (!found) { - for (CarbonMeasure measure : carbonMeasures) { - if (measure.getColName().equals(column)) { - cachedColsList.add(column); - break; - } + } + } + return cachedColsList; + } + + /** + * Method to find get carbon columns for columns to be cached. It will fill dimension first and + * then measures + * + * @return + */ + public List<CarbonColumn> getMinMaxCacheColumns() { + List<CarbonColumn> minMaxCachedColsList = null; + String tableName = tableInfo.getFactTable().getTableName(); + String cacheColumns = + tableInfo.getFactTable().getTableProperties().get(CarbonCommonConstants.COLUMN_META_CACHE); + if (null != cacheColumns) { + minMaxCachedColsList = new ArrayList<>(); + String[] cachedCols = cacheColumns.split(","); + List<String> measureColumns = new ArrayList<>(cachedCols.length); + List<CarbonDimension> complexDimensions = new ArrayList<>(cacheColumns.length()); + // add the columns in storage order: first normal dimensions, then complex dimensions + // and then measures + for (String column : cachedCols) { + CarbonDimension dimension = getDimensionByName(tableName, column); + // if found in dimension then add to dimension else add to measures + if (null != dimension) { + // first add normal dimensions and then complex dimensions + if (dimension.isComplex()) { + complexDimensions.add(dimension); + continue; } + minMaxCachedColsList.add(dimension); + } else { + measureColumns.add(column); + } + } + // add complex dimensions + minMaxCachedColsList.addAll(complexDimensions); + // search for measures columns and fill measures + for (String measureColumn : measureColumns) { + CarbonMeasure measure = getMeasureByName(tableName, measureColumn); + if (null != measure) { + minMaxCachedColsList.add(measure); } } } - return cachedColsList; + return minMaxCachedColsList; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index c3e3b78..a99bdee 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -100,6 +100,8 @@ import org.apache.carbondata.core.scan.filter.intf.RowIntf; import org.apache.carbondata.core.scan.filter.resolver.ConditionalFilterResolverImpl; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; import org.apache.carbondata.core.scan.filter.resolver.RowLevelFilterResolverImpl; +import org.apache.carbondata.core.scan.filter.resolver.RowLevelRangeFilterResolverImpl; +import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.ColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.result.vector.CarbonDictionary; @@ -135,7 +137,8 @@ public final class FilterUtil { */ private static FilterExecuter createFilterExecuterTree( FilterResolverIntf filterExpressionResolverTree, SegmentProperties segmentProperties, - Map<Integer, GenericQueryType> complexDimensionInfoMap) { + Map<Integer, GenericQueryType> complexDimensionInfoMap, + List<CarbonColumn> minMaxCacheColumns) { FilterExecuterType filterExecuterType = filterExpressionResolverTree.getFilterExecuterType(); if (null != filterExecuterType) { switch (filterExecuterType) { @@ -148,6 +151,11 @@ public final class FilterUtil { filterExpressionResolverTree.getDimColResolvedFilterInfo(), filterExpressionResolverTree.getMsrColResolvedFilterInfo(), segmentProperties); } + // return true filter expression if filter column min/max is not cached in driver + if (checkIfCurrentNodeToBeReplacedWithTrueFilterExpression(filterExpressionResolverTree, + segmentProperties, minMaxCacheColumns)) { + return new TrueFilterExecutor(); + } return getIncludeFilterExecuter( filterExpressionResolverTree.getDimColResolvedFilterInfo(), filterExpressionResolverTree.getMsrColResolvedFilterInfo(), segmentProperties); @@ -158,23 +166,37 @@ public final class FilterUtil { case OR: return new OrFilterExecuterImpl( createFilterExecuterTree(filterExpressionResolverTree.getLeft(), segmentProperties, - complexDimensionInfoMap), + complexDimensionInfoMap, minMaxCacheColumns), createFilterExecuterTree(filterExpressionResolverTree.getRight(), segmentProperties, - complexDimensionInfoMap)); + complexDimensionInfoMap, minMaxCacheColumns)); case AND: return new AndFilterExecuterImpl( createFilterExecuterTree(filterExpressionResolverTree.getLeft(), segmentProperties, - complexDimensionInfoMap), + complexDimensionInfoMap, minMaxCacheColumns), createFilterExecuterTree(filterExpressionResolverTree.getRight(), segmentProperties, - complexDimensionInfoMap)); + complexDimensionInfoMap, minMaxCacheColumns)); case ROWLEVEL_LESSTHAN: case ROWLEVEL_LESSTHAN_EQUALTO: case ROWLEVEL_GREATERTHAN_EQUALTO: case ROWLEVEL_GREATERTHAN: + // return true filter expression if filter column min/max is not cached in driver + RowLevelRangeFilterResolverImpl rowLevelRangeFilterResolver = + (RowLevelRangeFilterResolverImpl) filterExpressionResolverTree; + if (checkIfCurrentNodeToBeReplacedWithTrueFilterExpression( + rowLevelRangeFilterResolver.getDimColEvaluatorInfoList(), + rowLevelRangeFilterResolver.getMsrColEvalutorInfoList(), segmentProperties, + minMaxCacheColumns)) { + return new TrueFilterExecutor(); + } return RowLevelRangeTypeExecuterFactory .getRowLevelRangeTypeExecuter(filterExecuterType, filterExpressionResolverTree, segmentProperties); case RANGE: + // return true filter expression if filter column min/max is not cached in driver + if (checkIfCurrentNodeToBeReplacedWithTrueFilterExpression(filterExpressionResolverTree, + segmentProperties, minMaxCacheColumns)) { + return new TrueFilterExecutor(); + } return new RangeValueFilterExecuterImpl( filterExpressionResolverTree.getDimColResolvedFilterInfo(), filterExpressionResolverTree.getFilterExpression(), @@ -256,6 +278,137 @@ public final class FilterUtil { } /** + * check if current need to be replaced with TrueFilter expression. This will happen in case + * filter column min/max is not cached in the driver + * + * @param dimColEvaluatorInfoList + * @param msrColEvaluatorInfoList + * @param segmentProperties + * @param minMaxCacheColumns + * @return + */ + private static boolean checkIfCurrentNodeToBeReplacedWithTrueFilterExpression( + List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, + List<MeasureColumnResolvedFilterInfo> msrColEvaluatorInfoList, + SegmentProperties segmentProperties, List<CarbonColumn> minMaxCacheColumns) { + boolean replaceCurrentNodeWithTrueFilter = false; + ColumnResolvedFilterInfo columnResolvedFilterInfo = null; + if (!msrColEvaluatorInfoList.isEmpty()) { + columnResolvedFilterInfo = msrColEvaluatorInfoList.get(0); + replaceCurrentNodeWithTrueFilter = + checkIfFilterColumnIsCachedInDriver(columnResolvedFilterInfo, segmentProperties, + minMaxCacheColumns, true); + } else { + columnResolvedFilterInfo = dimColEvaluatorInfoList.get(0); + if (!columnResolvedFilterInfo.getDimension().hasEncoding(Encoding.IMPLICIT)) { + replaceCurrentNodeWithTrueFilter = + checkIfFilterColumnIsCachedInDriver(columnResolvedFilterInfo, segmentProperties, + minMaxCacheColumns, false); + } + } + return replaceCurrentNodeWithTrueFilter; + } + + /** + * check if current need to be replaced with TrueFilter expression. This will happen in case + * filter column min/max is not cached in the driver + * + * @param filterExpressionResolverTree + * @param segmentProperties + * @param minMaxCacheColumns + * @return + */ + private static boolean checkIfCurrentNodeToBeReplacedWithTrueFilterExpression( + FilterResolverIntf filterExpressionResolverTree, SegmentProperties segmentProperties, + List<CarbonColumn> minMaxCacheColumns) { + boolean replaceCurrentNodeWithTrueFilter = false; + ColumnResolvedFilterInfo columnResolvedFilterInfo = null; + if (null != filterExpressionResolverTree.getMsrColResolvedFilterInfo()) { + columnResolvedFilterInfo = filterExpressionResolverTree.getMsrColResolvedFilterInfo(); + replaceCurrentNodeWithTrueFilter = + checkIfFilterColumnIsCachedInDriver(columnResolvedFilterInfo, segmentProperties, + minMaxCacheColumns, true); + } else { + columnResolvedFilterInfo = filterExpressionResolverTree.getDimColResolvedFilterInfo(); + if (!columnResolvedFilterInfo.getDimension().hasEncoding(Encoding.IMPLICIT)) { + replaceCurrentNodeWithTrueFilter = + checkIfFilterColumnIsCachedInDriver(columnResolvedFilterInfo, segmentProperties, + minMaxCacheColumns, false); + } + } + return replaceCurrentNodeWithTrueFilter; + } + + /** + * Method to check whether current node needs to be replaced with true filter to avoid pruning + * for case when filter column is not cached in the min/max cached dimension + * + * @param columnResolvedFilterInfo + * @param segmentProperties + * @param minMaxCacheColumns + * @param isMeasure + * @return + */ + private static boolean checkIfFilterColumnIsCachedInDriver( + ColumnResolvedFilterInfo columnResolvedFilterInfo, SegmentProperties segmentProperties, + List<CarbonColumn> minMaxCacheColumns, boolean isMeasure) { + boolean replaceCurrentNodeWithTrueFilter = false; + CarbonColumn columnFromCurrentBlock = null; + if (isMeasure) { + columnFromCurrentBlock = segmentProperties + .getMeasureFromCurrentBlock(columnResolvedFilterInfo.getMeasure().getColumnId()); + } else { + columnFromCurrentBlock = + segmentProperties.getDimensionFromCurrentBlock(columnResolvedFilterInfo.getDimension()); + } + if (null != columnFromCurrentBlock) { + // check for filter dimension in the cached column list + if (null != minMaxCacheColumns) { + int columnIndexInMinMaxByteArray = + getFilterColumnIndexInCachedColumns(minMaxCacheColumns, columnFromCurrentBlock); + if (columnIndexInMinMaxByteArray != -1) { + columnResolvedFilterInfo.setColumnIndexInMinMaxByteArray(columnIndexInMinMaxByteArray); + } else { + // will be true only if column caching is enabled and current filter column is not cached + replaceCurrentNodeWithTrueFilter = true; + } + } else { + // if columns to be cached are not specified then in that case all columns will be cached + // and then the ordinal of column will be its index in the min/max byte array + if (isMeasure) { + columnResolvedFilterInfo.setColumnIndexInMinMaxByteArray( + segmentProperties.getLastDimensionColOrdinal() + columnFromCurrentBlock.getOrdinal()); + } else { + columnResolvedFilterInfo + .setColumnIndexInMinMaxByteArray(columnFromCurrentBlock.getOrdinal()); + } + } + } + return replaceCurrentNodeWithTrueFilter; + } + + /** + * Method to check whether the filter dimension exists in the cached dimensions for a table + * + * @param carbonDimensionsToBeCached + * @param filterColumn + * @return + */ + private static int getFilterColumnIndexInCachedColumns( + List<CarbonColumn> carbonDimensionsToBeCached, CarbonColumn filterColumn) { + int columnIndexInMinMaxByteArray = -1; + int columnCounter = 0; + for (CarbonColumn cachedColumn : carbonDimensionsToBeCached) { + if (cachedColumn.getColumnId().equals(filterColumn.getColumnId())) { + columnIndexInMinMaxByteArray = columnCounter; + break; + } + columnCounter++; + } + return columnIndexInMinMaxByteArray; + } + + /** * It gives filter executer based on columnar or column group * * @param dimColResolvedFilterInfo @@ -1282,8 +1435,23 @@ public final class FilterUtil { public static FilterExecuter getFilterExecuterTree( FilterResolverIntf filterExpressionResolverTree, SegmentProperties segmentProperties, Map<Integer, GenericQueryType> complexDimensionInfoMap) { + return getFilterExecuterTree(filterExpressionResolverTree, segmentProperties, + complexDimensionInfoMap, null); + } + + /** + * API will create an filter executer tree based on the filter resolver and minMaxColumns + * + * @param filterExpressionResolverTree + * @param segmentProperties + * @return + */ + public static FilterExecuter getFilterExecuterTree( + FilterResolverIntf filterExpressionResolverTree, SegmentProperties segmentProperties, + Map<Integer, GenericQueryType> complexDimensionInfoMap, + List<CarbonColumn> minMaxCacheColumns) { return createFilterExecuterTree(filterExpressionResolverTree, segmentProperties, - complexDimensionInfoMap); + complexDimensionInfoMap, minMaxCacheColumns); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java index 6af969e..91f46be 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java @@ -449,20 +449,15 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) { BitSet bitSet = new BitSet(1); byte[][] filterValues; - int columnIndex = 0; int chunkIndex = 0; boolean isScanRequired = false; if (isDimensionPresentInCurrentBlock) { filterValues = dimColumnExecuterInfo.getFilterKeys(); - columnIndex = dimColumnEvaluatorInfo.getColumnIndex(); - chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(columnIndex); + chunkIndex = dimColumnEvaluatorInfo.getColumnIndexInMinMaxByteArray(); isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex], filterValues); } else if (isMeasurePresentInCurrentBlock) { - columnIndex = msrColumnEvaluatorInfo.getColumnIndex(); - chunkIndex = - segmentProperties.getMeasuresOrdinalToChunkMapping().get(columnIndex) + - segmentProperties.getLastDimensionColOrdinal(); + chunkIndex = msrColumnEvaluatorInfo.getColumnIndexInMinMaxByteArray(); isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex], msrColumnExecutorInfo.getFilterKeys(), msrColumnEvaluatorInfo.getType()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java index 7919e35..24461a5 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java @@ -317,7 +317,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { BitSet bitSet = new BitSet(1); byte[][] filterValues = this.filterRangesValues; - int columnIndex = this.dimColEvaluatorInfo.getColumnIndex(); + int columnIndex = this.dimColEvaluatorInfo.getColumnIndexInMinMaxByteArray(); boolean isScanRequired = columnIndex >= blockMinValue.length || isScanRequired(blockMinValue[columnIndex], blockMaxValue[columnIndex], filterValues); if (isScanRequired) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java index 057a244..1fce7a3 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java @@ -80,7 +80,11 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { int[] dimensionChunkIndex; /** - * it has index at which given measure is stored in file + * it has index at which given measure is stored in file. + * Note: Its value can be used only for isScanRequired method as the value is incremented by + * last dimension ordinal in segmentProperties because of data writing order + * (first dimensions are written and then measures). Therefore avoid its usage in in methods other + * than isScanRequired */ int[] measureChunkIndex; @@ -164,8 +168,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { .getDimensionFromCurrentBlock(dimColEvaluatorInfoList.get(i).getDimension()); if (null != dimensionFromCurrentBlock) { dimColEvaluatorInfoList.get(i).setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); - this.dimensionChunkIndex[i] = segmentProperties.getDimensionOrdinalToChunkMapping() - .get(dimensionFromCurrentBlock.getOrdinal()); + this.dimensionChunkIndex[i] = + dimColEvaluatorInfoList.get(i).getColumnIndexInMinMaxByteArray(); isDimensionPresentInCurrentBlock[i] = true; } } @@ -182,8 +186,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { msrColEvalutorInfoList.get(i).getCarbonColumn().getColumnId()); if (null != measureFromCurrentBlock) { msrColEvalutorInfoList.get(i).setColumnIndex(measureFromCurrentBlock.getOrdinal()); - this.measureChunkIndex[i] = segmentProperties.getMeasuresOrdinalToChunkMapping() - .get(measureFromCurrentBlock.getOrdinal()); + this.measureChunkIndex[i] = msrColEvalutorInfoList.get(i).getColumnIndexInMinMaxByteArray(); isMeasurePresentInCurrentBlock[i] = true; } } @@ -224,10 +227,12 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { } if (msrColEvalutorInfoList.size() > 0) { if (isMeasurePresentInCurrentBlock[0]) { - pageNumbers = rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]] - .getPagesCount(); - numberOfRows = rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]] - .getRowCount(); + pageNumbers = + rawBlockletColumnChunks.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0) + .getColumnIndex()].getPagesCount(); + numberOfRows = + rawBlockletColumnChunks.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0) + .getColumnIndex()].getRowCount(); } else { // specific for restructure case where default values need to be filled pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages(); @@ -483,8 +488,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { Object msrValue; ColumnPage columnPage = - blockChunkHolder.getMeasureRawColumnChunks()[measureChunkIndex[0]] - .decodeColumnPage(pageIndex); + blockChunkHolder.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0) + .getColumnIndex()].decodeColumnPage(pageIndex); if (msrType == DataTypes.BOOLEAN) { msrValue = columnPage.getBoolean(index); } else if (msrType == DataTypes.SHORT) { @@ -648,10 +653,11 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { } for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) { - if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]]) { - rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]] = + int chunkIndex = msrColEvalutorInfoList.get(0).getColumnIndex(); + if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock() - .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), measureChunkIndex[0]); + .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java index 03e41eb..e561f96 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java @@ -56,7 +56,6 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute * flag to check whether default values is present in the filter value list */ private boolean isDefaultValuePresentInFilter; - private int lastDimensionColOrdinal = 0; RowLevelRangeGrtThanFiterExecuterImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, @@ -66,7 +65,6 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute null); this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; - lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal(); if (!this.msrColEvalutorInfoList.isEmpty()) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); @@ -123,7 +121,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute byte[] maxValue = null; if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) { if (isMeasurePresentInCurrentBlock[0]) { - maxValue = blockMaxValue[measureChunkIndex[0] + lastDimensionColOrdinal]; + maxValue = blockMaxValue[measureChunkIndex[0]]; isScanRequired = isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType()); } else { @@ -242,8 +240,8 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute } return bitSetGroup; } else if (isMeasurePresentInCurrentBlock[0]) { - int chunkIndex = - segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]); + int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping() + .get(msrColEvalutorInfoList.get(0).getColumnIndex()); if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk( @@ -479,7 +477,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute rawBlockletColumnChunks.getFileReader(), chunkIndex); } } else if (isMeasurePresentInCurrentBlock[0]) { - int chunkIndex = measureChunkIndex[0]; + int chunkIndex = msrColEvalutorInfoList.get(0).getColumnIndex(); if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk( http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java index 95cd356..53c2d76 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java @@ -55,7 +55,6 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte * flag to check whether default values is present in the filter value list */ private boolean isDefaultValuePresentInFilter; - private int lastDimensionColOrdinal = 0; RowLevelRangeGrtrThanEquaToFilterExecuterImpl( List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, @@ -66,7 +65,6 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte null); this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; - lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal(); if (!msrColEvalutorInfoList.isEmpty()) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); @@ -120,7 +118,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte byte[] maxValue = null; if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) { if (isMeasurePresentInCurrentBlock[0]) { - maxValue = blockMaxValue[measureChunkIndex[0] + lastDimensionColOrdinal]; + maxValue = blockMaxValue[measureChunkIndex[0]]; isScanRequired = isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType()); } else { @@ -239,8 +237,8 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte } return bitSetGroup; } else { - int chunkIndex = - segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]); + int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping() + .get(msrColEvalutorInfoList.get(0).getColumnIndex()); if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk( @@ -462,7 +460,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte rawBlockletColumnChunks.getFileReader(), chunkIndex); } } else if (isMeasurePresentInCurrentBlock[0]) { - int chunkIndex = measureChunkIndex[0]; + int chunkIndex = msrColEvalutorInfoList.get(0).getColumnIndex(); if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk( http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc293191/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java index dc5ed37..b8f3cb1 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java @@ -58,7 +58,6 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter * flag to check whether default values is present in the filter value list */ private boolean isDefaultValuePresentInFilter; - private int lastDimensionColOrdinal = 0; public RowLevelRangeLessThanEqualFilterExecuterImpl( List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, @@ -66,7 +65,6 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); - lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal(); this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; if (!msrColEvalutorInfoList.isEmpty()) { @@ -123,7 +121,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter boolean isScanRequired = false; if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) { if (isMeasurePresentInCurrentBlock[0]) { - minValue = blockMinValue[measureChunkIndex[0] + lastDimensionColOrdinal]; + minValue = blockMinValue[measureChunkIndex[0]]; isScanRequired = isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType()); } else { @@ -233,8 +231,8 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter } return bitSetGroup; } else if (isMeasurePresentInCurrentBlock[0]) { - int chunkIndex = - segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]); + int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping() + .get(msrColEvalutorInfoList.get(0).getColumnIndex()); if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk( @@ -502,7 +500,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter rawBlockletColumnChunks.getFileReader(), chunkIndex); } } else if (isMeasurePresentInCurrentBlock[0]) { - int chunkIndex = measureChunkIndex[0]; + int chunkIndex = msrColEvalutorInfoList.get(0).getColumnIndex(); if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
