akashrn5 commented on a change in pull request #3584: [CARBONDATA-3718] Support
SegmentLevel MinMax for better Pruning and less driver memory usage for cache
URL: https://github.com/apache/carbondata/pull/3584#discussion_r394315724
##########
File path:
core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
##########
@@ -186,6 +217,106 @@ private void
getTableBlockUniqueIdentifierWrappers(List<PartitionSpec> partition
}
}
+ /**
+ * Using blockLevel minmax values, identify if segment has to be added for
further pruning and to
+ * load segment index info to cache
+ * @param segment to be identified if needed for loading block datamaps
+ * @param segmentMetaDataInfo list of block level min max values
+ * @param filter filter expression
+ * @param identifiers tableBlockIndexUniqueIdentifiers
+ * @param tableBlockIndexUniqueIdentifierWrappers to add
tableBlockIndexUniqueIdentifiers
+ */
+ private void getTableBlockIndexUniqueIdentifierUsingSegmentMinMax(Segment
segment,
+ SegmentMetaDataInfo segmentMetaDataInfo, DataMapFilter filter,
+ Set<TableBlockIndexUniqueIdentifier> identifiers,
+ List<TableBlockIndexUniqueIdentifierWrapper>
tableBlockIndexUniqueIdentifierWrappers) {
+ boolean isScanRequired = false;
+ Map<String, SegmentColumnMetaDataInfo> segmentColumnMetaDataInfoMap =
+ segmentMetaDataInfo.getSegmentColumnMetaDataInfoMap();
+ int length = segmentColumnMetaDataInfoMap.size();
+ // Add columnSchemas based on the columns present in segment
+ List<ColumnSchema> columnSchemas = new ArrayList<>();
+ byte[][] min = new byte[length][];
+ byte[][] max = new byte[length][];
+ boolean[] minMaxFlag = new boolean[length];
+ int i = 0;
+
+ // get current columnSchema list for the table
+ Map<String, ColumnSchema> tableColumnSchemas =
+
this.getCarbonTable().getTableInfo().getFactTable().getListOfColumns().stream()
+ .collect(Collectors.toMap(ColumnSchema::getColumnUniqueId,
ColumnSchema::clone));
+
+ // fill min,max and columnSchema values
+ for (Map.Entry<String, SegmentColumnMetaDataInfo> columnMetaData :
+ segmentColumnMetaDataInfoMap.entrySet()) {
+ ColumnSchema columnSchema =
tableColumnSchemas.get(columnMetaData.getKey());
+ if (null != columnSchema) {
+ // get segment sort column and column drift info
+ boolean isSortColumnInBlock = columnMetaData.getValue().isSortColumn();
Review comment:
please rename
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
