Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2437#discussion_r200155521
--- Diff:
core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java
---
@@ -0,0 +1,879 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.indexstore.AbstractMemoryDMStore;
+import org.apache.carbondata.core.indexstore.BlockMetaInfo;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
+import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.indexstore.SafeMemoryDMStore;
+import org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore;
+import org.apache.carbondata.core.indexstore.row.DataMapRow;
+import org.apache.carbondata.core.indexstore.row.DataMapRowImpl;
+import org.apache.carbondata.core.indexstore.schema.CarbonRowSchema;
+import org.apache.carbondata.core.indexstore.schema.SchemaGenerator;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
+import
org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
+import
org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.profiler.ExplainCollector;
+import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import
org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.util.BlockletDataMapUtil;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataFileFooterConverter;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.Path;
+import org.xerial.snappy.Snappy;
+
+/**
+ * Datamap implementation for blocklet.
+ */
+public class BlockDataMap extends CoarseGrainDataMap implements
Serializable {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(BlockDataMap.class.getName());
+
+ protected static final long serialVersionUID = -2170289352240810993L;
+ /**
+ * for CACHE_LEVEL=BLOCK and legacy store default blocklet id will be -1
+ */
+ private static final short BLOCK_DEFAULT_BLOCKLET_ID = -1;
+
+ protected AbstractMemoryDMStore memoryDMStore;
+
+ protected AbstractMemoryDMStore taskSummaryDMStore;
+
+ // As it is a heavy object it is not recommended to serialize this object
+ protected transient SegmentProperties segmentProperties;
+
+ protected int[] columnCardinality;
+
+ protected long blockletSchemaTime;
+ /**
+ * flag to check for store from 1.1 or any prior version
+ */
+ protected boolean isLegacyStore;
+
+ @Override public void init(DataMapModel dataMapModel) throws
IOException, MemoryException {
+ long startTime = System.currentTimeMillis();
+ assert (dataMapModel instanceof BlockletDataMapModel);
+ BlockletDataMapModel blockletDataMapInfo = (BlockletDataMapModel)
dataMapModel;
+ DataFileFooterConverter fileFooterConverter = new
DataFileFooterConverter();
+ List<DataFileFooter> indexInfo = fileFooterConverter
+ .getIndexInfo(blockletDataMapInfo.getFilePath(),
blockletDataMapInfo.getFileData());
+ Path path = new Path(blockletDataMapInfo.getFilePath());
+ byte[] filePath =
path.getParent().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
+ byte[] fileName =
path.getName().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
+ byte[] segmentId =
+
blockletDataMapInfo.getSegmentId().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
+ byte[] schemaBinary = null;
+ if (!indexInfo.isEmpty()) {
+ DataFileFooter fileFooter = indexInfo.get(0);
+ // store for 1.1 or any prior version will not have any blocklet
information in file footer
+ isLegacyStore = fileFooter.getBlockletList() == null;
+ // init segment properties and create schema
+ initSegmentProperties(fileFooter);
+ schemaBinary = convertSchemaToBinary(fileFooter.getColumnInTable());
+ createSchema(segmentProperties, blockletDataMapInfo.isAddToUnsafe(),
true);
+ createSummarySchema(segmentProperties, schemaBinary, filePath,
fileName, segmentId,
+ blockletDataMapInfo.isAddToUnsafe(), true);
+ }
+ // check for legacy store and load the metadata
+ DataMapRowImpl summaryRow = loadBlockMetadata(blockletDataMapInfo,
indexInfo);
--- End diff --
Create protected methods for `createSchema`, `createSummarySchema` and
`loadMetadata` and override the same in BloclkletDataMap
---
