http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java index 0cb2918..099fffd 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java @@ -29,31 +29,12 @@ public class SegmentInfo implements Serializable { private static final long serialVersionUID = -1749874611112709431L; /** - * number of column in the segment - */ - private int numberOfColumns; - - /** * cardinality of each columns * column which is not participating in the multidimensional key cardinality will be -1; */ private int[] columnCardinality; /** - * @return the numberOfColumns - */ - public int getNumberOfColumns() { - return numberOfColumns; - } - - /** - * @param numberOfColumns the numberOfColumns to set - */ - public void setNumberOfColumns(int numberOfColumns) { - this.numberOfColumns = numberOfColumns; - } - - /** * @return the columnCardinality */ public int[] getColumnCardinality() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java index 6036569..d17d865 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java @@ -19,7 +19,13 @@ package org.apache.carbondata.core.metadata.schema.table; import java.io.IOException; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; @@ -33,7 +39,10 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; +import org.apache.carbondata.core.scan.model.QueryModel; +import org.apache.carbondata.core.scan.model.QueryProjection; import org.apache.carbondata.core.util.CarbonUtil; +import org.apache.carbondata.core.util.DataTypeConverter; import org.apache.carbondata.core.util.DataTypeUtil; import org.apache.carbondata.core.util.path.CarbonTablePath; @@ -136,10 +145,7 @@ public class CarbonTable implements Serializable { /** * During creation of TableInfo from hivemetastore the DataMapSchemas and the columns * DataTypes are not converted to the appropriate child classes. - * * This method will cast the same to the appropriate classes - * - * @param tableInfo */ public static void updateTableInfo(TableInfo tableInfo) { List<DataMapSchema> dataMapSchemas = new ArrayList<>(); @@ -153,8 +159,9 @@ public class CarbonTable implements Serializable { } tableInfo.setDataMapSchemaList(dataMapSchemas); for (ColumnSchema columnSchema : tableInfo.getFactTable().getListOfColumns()) { - columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(), - columnSchema.getPrecision(), columnSchema.getScale())); + columnSchema.setDataType( + DataTypeUtil.valueOf( + columnSchema.getDataType(), columnSchema.getPrecision(), columnSchema.getScale())); } List<DataMapSchema> childSchema = tableInfo.getDataMapSchemaList(); for (DataMapSchema dataMapSchema : childSchema) { @@ -168,10 +175,11 @@ public class CarbonTable implements Serializable { } } if (tableInfo.getFactTable().getBucketingInfo() != null) { - for (ColumnSchema columnSchema : tableInfo.getFactTable() - .getBucketingInfo().getListOfColumns()) { - columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(), - columnSchema.getPrecision(), columnSchema.getScale())); + for (ColumnSchema columnSchema : + tableInfo.getFactTable().getBucketingInfo().getListOfColumns()) { + columnSchema.setDataType( + DataTypeUtil.valueOf( + columnSchema.getDataType(), columnSchema.getPrecision(), columnSchema.getScale())); } } if (tableInfo.getFactTable().getPartitionInfo() != null) { @@ -211,6 +219,7 @@ public class CarbonTable implements Serializable { /** * fill columns as per user provided order + * * @param tableName */ private void fillCreateOrderColumn(String tableName) { @@ -229,7 +238,6 @@ public class CarbonTable implements Serializable { this.createOrderColumn.put(tableName, columns); } - /** * Fill allDimensions and allMeasures for carbon table * @@ -254,8 +262,8 @@ public class CarbonTable implements Serializable { if (columnSchema.isDimensionColumn()) { if (columnSchema.getNumberOfChild() > 0) { CarbonDimension complexDimension = - new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), -1, -1, ++complexTypeOrdinal); + new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), + -1, -1, ++complexTypeOrdinal); complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild()); allDimensions.add(complexDimension); dimensionOrdinal = @@ -268,9 +276,8 @@ public class CarbonTable implements Serializable { this.numberOfSortColumns++; } if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) { - CarbonDimension dimension = - new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), -1, -1, -1); + CarbonDimension dimension = new CarbonDimension( + columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), -1, -1, -1); if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) { this.numberOfNoDictSortColumns++; } @@ -278,25 +285,25 @@ public class CarbonTable implements Serializable { primitiveDimensions.add(dimension); } else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY) && columnSchema.getColumnGroupId() == -1) { - CarbonDimension dimension = - new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), keyOrdinal++, -1, -1); + CarbonDimension dimension = new CarbonDimension( + columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), keyOrdinal++, + -1, -1); allDimensions.add(dimension); primitiveDimensions.add(dimension); } else { columnGroupOrdinal = previousColumnGroupId == columnSchema.getColumnGroupId() ? ++columnGroupOrdinal : 0; previousColumnGroupId = columnSchema.getColumnGroupId(); - CarbonDimension dimension = new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), keyOrdinal++, + CarbonDimension dimension = new CarbonDimension( + columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), keyOrdinal++, columnGroupOrdinal, -1); allDimensions.add(dimension); primitiveDimensions.add(dimension); } } } else { - allMeasures.add(new CarbonMeasure(columnSchema, measureOrdinal++, - columnSchema.getSchemaOrdinal())); + allMeasures.add( + new CarbonMeasure(columnSchema, measureOrdinal++, columnSchema.getSchemaOrdinal())); } } fillVisibleDimensions(tableSchema.getTableName()); @@ -347,8 +354,8 @@ public class CarbonTable implements Serializable { if (columnSchema.isDimensionColumn()) { if (columnSchema.getNumberOfChild() > 0) { CarbonDimension complexDimension = - new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), -1, -1, -1); + new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), + -1, -1, -1); complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild()); parentDimension.getListOfChildDimensions().add(complexDimension); dimensionOrdinal = @@ -356,8 +363,8 @@ public class CarbonTable implements Serializable { listOfColumns, complexDimension, primitiveDimensions); } else { CarbonDimension carbonDimension = - new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), -1, -1, -1); + new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), + -1, -1, -1); parentDimension.getListOfChildDimensions().add(carbonDimension); primitiveDimensions.add(carbonDimension); } @@ -569,6 +576,7 @@ public class CarbonTable implements Serializable { } return null; } + /** * gets all children dimension for complex type * @@ -771,18 +779,13 @@ public class CarbonTable implements Serializable { return dimensionOrdinalMax; } - public void setDimensionOrdinalMax(int dimensionOrdinalMax) { - this.dimensionOrdinalMax = dimensionOrdinalMax; - } - - public boolean hasDataMapSchema() { return hasDataMapSchema; } public boolean isChildDataMap() { - return null != tableInfo.getParentRelationIdentifiers() - && !tableInfo.getParentRelationIdentifiers().isEmpty(); + return null != tableInfo.getParentRelationIdentifiers() && + !tableInfo.getParentRelationIdentifiers().isEmpty(); } /** @@ -806,4 +809,63 @@ public class CarbonTable implements Serializable { } return dataSize + indexSize; } + + /** + * Create a new QueryModel with projection all columns in the table. + */ + public QueryModel createQueryModelWithProjectAllColumns(DataTypeConverter converter) { + QueryProjection projection = new QueryProjection(); + + List<CarbonDimension> dimensions = getDimensionByTableName(getTableName()); + for (int i = 0; i < dimensions.size(); i++) { + projection.addDimension(dimensions.get(i), i); + } + List<CarbonMeasure> measures = getMeasureByTableName(getTableName()); + for (int i = 0; i < measures.size(); i++) { + projection.addMeasure(measures.get(i), i); + } + QueryModel model = QueryModel.newInstance(this); + model.setProjection(projection); + model.setConverter(converter); + return model; + } + + /** + * Create a new QueryModel with specified projection + */ + public QueryModel createQueryWithProjection(String[] projectionColumnNames, + DataTypeConverter converter) { + QueryProjection projection = createProjection(projectionColumnNames); + QueryModel queryModel = QueryModel.newInstance(this); + queryModel.setProjection(projection); + queryModel.setConverter(converter); + return queryModel; + } + + private QueryProjection createProjection(String[] projectionColumnNames) { + String factTableName = getTableName(); + QueryProjection projection = new QueryProjection(); + // fill dimensions + // If columns are null, set all dimensions and measures + int i = 0; + if (projectionColumnNames != null) { + for (String projectionColumnName : projectionColumnNames) { + CarbonDimension dimension = getDimensionByName(factTableName, projectionColumnName); + if (dimension != null) { + projection.addDimension(dimension, i); + i++; + } else { + CarbonMeasure measure = getMeasureByName(factTableName, projectionColumnName); + if (measure == null) { + throw new RuntimeException(projectionColumnName + + " column not found in the table " + factTableName); + } + projection.addMeasure(measure, i); + i++; + } + } + } + + return projection; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java index 20e539b..9a1dad1 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java @@ -26,18 +26,6 @@ import java.io.Serializable; */ public class RelationIdentifier implements Serializable, Writable { - public void setDatabaseName(String databaseName) { - this.databaseName = databaseName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public void setTableId(String tableId) { - this.tableId = tableId; - } - private String databaseName; private String tableName; @@ -50,10 +38,6 @@ public class RelationIdentifier implements Serializable, Writable { this.tableId = tableId; } - public RelationIdentifier() { - this(null, null, null); - } - public String getDatabaseName() { return databaseName; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java index 4deafd4..0d796c7 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java @@ -24,7 +24,11 @@ import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java index 13e7d49..c888418 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java @@ -45,7 +45,7 @@ public class CarbonColumn implements Serializable { /** * order in which user has created table */ - protected int schemaOrdinal; + private int schemaOrdinal; /** * Column identifier http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java index b4c052f..d15cde7 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java @@ -122,18 +122,6 @@ public class CarbonDimension extends CarbonColumn { return getEncoder().contains(Encoding.DICTIONARY); } - public int getNumDimensionsExpanded() { - if (listOfChildDimensions == null) { - // there is no child, return 1 column - return 1; - } - int columnCount = 1; - for (CarbonDimension dimension: listOfChildDimensions) { - columnCount += dimension.getNumDimensionsExpanded(); - } - return columnCount; - } - /** * @return is column participated in sorting or not */ http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java index 18eae11..7289c98 100644 --- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java @@ -128,8 +128,6 @@ public class CarbonUpdateUtil { lockStatus = updateLock.lockWithRetries(); if (lockStatus) { - AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier(); - // read the existing file if present and update the same. SegmentUpdateDetails[] oldDetails = segmentUpdateStatusManager .getUpdateStatusDetails(); @@ -389,16 +387,6 @@ public class CarbonUpdateUtil { .substring(0, completeBlockName.lastIndexOf(CarbonCommonConstants.HYPHEN)); } - /** - * returns segment id from segment name - * - * @param segmentName - * @return - */ - public static String getSegmentId(String segmentName) { - return segmentName.split(CarbonCommonConstants.UNDERSCORE)[1]; - } - public static long getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) { String segmentDirPath = tablePath.getCarbonDataDirectoryPath(segmentId); @@ -523,7 +511,7 @@ public class CarbonUpdateUtil { // aborted scenario. invalidDeleteDeltaFiles = updateStatusManager - .getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, false, + .getDeleteDeltaInvalidFilesList(block, false, allSegmentFiles, isAbortedFile); for (CarbonFile invalidFile : invalidDeleteDeltaFiles) { boolean doForceDelete = true; @@ -533,7 +521,7 @@ public class CarbonUpdateUtil { // case 1 if (CarbonUpdateUtil.isBlockInvalid(block.getSegmentStatus())) { completeListOfDeleteDeltaFiles = updateStatusManager - .getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, true, + .getDeleteDeltaInvalidFilesList(block, true, allSegmentFiles, isInvalidFile); for (CarbonFile invalidFile : completeListOfDeleteDeltaFiles) { @@ -541,7 +529,7 @@ public class CarbonUpdateUtil { } CarbonFile[] blockRelatedFiles = updateStatusManager - .getAllBlockRelatedFiles(block.getBlockName(), allSegmentFiles, + .getAllBlockRelatedFiles(allSegmentFiles, block.getActualBlockName()); // now for each invalid index file need to check the query execution time out @@ -555,7 +543,7 @@ public class CarbonUpdateUtil { } else { invalidDeleteDeltaFiles = updateStatusManager - .getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, false, + .getDeleteDeltaInvalidFilesList(block, false, allSegmentFiles, isInvalidFile); for (CarbonFile invalidFile : invalidDeleteDeltaFiles) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java index 7affe12..20db937 100644 --- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java +++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java @@ -48,14 +48,6 @@ public class DeleteDeltaBlockDetails implements Serializable { blockletDetailsMap = new TreeMap<>(); } - public String getBlockName() { - return blockName; - } - - public void setBlockName(String blockName) { - this.blockName = blockName; - } - @Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null || !(obj instanceof DeleteDeltaBlockDetails)) return false; http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java index 22785fa..6185134 100644 --- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java +++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java @@ -46,14 +46,6 @@ public class DeleteDeltaBlockletDetails implements Serializable { return deletedRows.add(row); } - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - public Integer getPageId() { return pageId; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java deleted file mode 100644 index 3c0dd4f..0000000 --- a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.core.mutate.data; - -import java.util.Map; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache; -import org.apache.carbondata.core.datastore.DataRefNode; -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; -import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager; - -/** - * This class is responsible for loading delete delta file cache based on - * blocklet id of a particular block - */ -public class BlockletDeleteDeltaCacheLoader implements DeleteDeltaCacheLoaderIntf { - private String blockletID; - private DataRefNode blockletNode; - private AbsoluteTableIdentifier absoluteIdentifier; - private static final LogService LOGGER = - LogServiceFactory.getLogService(BlockletDeleteDeltaCacheLoader.class.getName()); - - public BlockletDeleteDeltaCacheLoader(String blockletID, DataRefNode blockletNode, - AbsoluteTableIdentifier absoluteIdentifier) { - this.blockletID = blockletID; - this.blockletNode = blockletNode; - this.absoluteIdentifier = absoluteIdentifier; - } - - /** - * This method will load the delete delta cache based on blocklet id of particular block with - * the help of SegmentUpdateStatusManager. - */ - public void loadDeleteDeltaFileDataToCache() { - SegmentUpdateStatusManager segmentUpdateStatusManager = - new SegmentUpdateStatusManager(absoluteIdentifier); - Map<Integer, Integer[]> deleteDeltaFileData = null; - BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache = null; - if (null == blockletNode.getDeleteDeltaDataCache()) { - try { - deleteDeltaFileData = - segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID); - deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData, - segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, null)); - } catch (Exception e) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Unable to retrieve delete delta files"); - } - } - } else { - deleteDeltaDataCache = blockletNode.getDeleteDeltaDataCache(); - // if already cache is present then validate the cache using timestamp - String cacheTimeStamp = segmentUpdateStatusManager - .getTimestampForRefreshCache(blockletID, deleteDeltaDataCache.getCacheTimeStamp()); - if (null != cacheTimeStamp) { - try { - deleteDeltaFileData = - segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID); - deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData, - segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, cacheTimeStamp)); - } catch (Exception e) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Unable to retrieve delete delta files"); - } - } - } - } - blockletNode.setDeleteDeltaDataCache(deleteDeltaDataCache); - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java index b14660c..88b3065 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java @@ -18,7 +18,7 @@ package org.apache.carbondata.core.scan.collector; import java.util.List; -import org.apache.carbondata.core.scan.result.AbstractScannedResult; +import org.apache.carbondata.core.scan.result.BlockletScannedResult; import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch; /** @@ -27,17 +27,13 @@ import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch; public interface ScannedResultCollector { /** - * Below method will be used to aggregate the scanned result - * - * @param scannedResult scanned result - * @return how many records was aggregated + * Return the aggregated scanned result in list of rows */ - List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize); + List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize); /** - * Collects data in columnar format. - * @param scannedResult - * @param columnarBatch + * Collects result and set it in the specified columnar batch */ - void collectVectorBatch(AbstractScannedResult scannedResult, CarbonColumnarBatch columnarBatch); + void collectResultInColumnarBatch(BlockletScannedResult scannedResult, + CarbonColumnarBatch columnarBatch); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java index dc78ac6..694271e 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java @@ -19,8 +19,6 @@ package org.apache.carbondata.core.scan.collector.impl; import java.math.BigDecimal; import java.math.RoundingMode; -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.datastore.page.ColumnPage; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; @@ -29,8 +27,8 @@ import org.apache.carbondata.core.scan.collector.ScannedResultCollector; import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; import org.apache.carbondata.core.scan.executor.infos.DimensionInfo; import org.apache.carbondata.core.scan.executor.infos.MeasureInfo; -import org.apache.carbondata.core.scan.model.QueryMeasure; -import org.apache.carbondata.core.scan.result.AbstractScannedResult; +import org.apache.carbondata.core.scan.model.ProjectionMeasure; +import org.apache.carbondata.core.scan.result.BlockletScannedResult; import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch; import org.apache.carbondata.core.util.DataTypeUtil; @@ -39,38 +37,35 @@ import org.apache.carbondata.core.util.DataTypeUtil; */ public abstract class AbstractScannedResultCollector implements ScannedResultCollector { - private static final LogService LOGGER = - LogServiceFactory.getLogService(AbstractScannedResultCollector.class.getName()); - /** * table block execution infos */ - protected BlockExecutionInfo tableBlockExecutionInfos; + BlockExecutionInfo executionInfo; /** * maintains the measure information like datatype, ordinal, measure existence */ - protected MeasureInfo measureInfo; + MeasureInfo measureInfo; /** * maintains the dimension information like datatype, ordinal, measure existence */ - protected DimensionInfo dimensionInfo; + DimensionInfo dimensionInfo; - public AbstractScannedResultCollector(BlockExecutionInfo blockExecutionInfos) { - this.tableBlockExecutionInfos = blockExecutionInfos; + AbstractScannedResultCollector(BlockExecutionInfo blockExecutionInfos) { + this.executionInfo = blockExecutionInfos; measureInfo = blockExecutionInfos.getMeasureInfo(); dimensionInfo = blockExecutionInfos.getDimensionInfo(); } protected void fillMeasureData(Object[] msrValues, int offset, - AbstractScannedResult scannedResult) { + BlockletScannedResult scannedResult) { int measureExistIndex = 0; for (short i = 0; i < measureInfo.getMeasureDataTypes().length; i++) { // if measure exists is block then pass measure column // data chunk to the collector if (measureInfo.getMeasureExists()[i]) { - QueryMeasure queryMeasure = tableBlockExecutionInfos.getQueryMeasures()[measureExistIndex]; + ProjectionMeasure queryMeasure = executionInfo.getProjectionMeasures()[measureExistIndex]; msrValues[i + offset] = getMeasureData( scannedResult.getMeasureChunk(measureInfo.getMeasureOrdinals()[measureExistIndex]), scannedResult.getCurrentRowId(), queryMeasure.getMeasure()); @@ -87,8 +82,7 @@ public abstract class AbstractScannedResultCollector implements ScannedResultCol } } - protected Object getMeasureData(ColumnPage dataChunk, int index, - CarbonMeasure carbonMeasure) { + Object getMeasureData(ColumnPage dataChunk, int index, CarbonMeasure carbonMeasure) { if (!dataChunk.getNullBits().get(index)) { DataType dataType = carbonMeasure.getDataType(); if (dataType == DataTypes.BOOLEAN) { @@ -114,7 +108,8 @@ public abstract class AbstractScannedResultCollector implements ScannedResultCol return null; } - @Override public void collectVectorBatch(AbstractScannedResult scannedResult, + @Override + public void collectResultInColumnarBatch(BlockletScannedResult scannedResult, CarbonColumnarBatch columnarBatch) { throw new UnsupportedOperationException("Works only for batch collectors"); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java index cf6e6d6..4322034 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java @@ -29,9 +29,9 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; import org.apache.carbondata.core.scan.filter.GenericQueryType; -import org.apache.carbondata.core.scan.model.QueryDimension; -import org.apache.carbondata.core.scan.model.QueryMeasure; -import org.apache.carbondata.core.scan.result.AbstractScannedResult; +import org.apache.carbondata.core.scan.model.ProjectionDimension; +import org.apache.carbondata.core.scan.model.ProjectionMeasure; +import org.apache.carbondata.core.scan.result.BlockletScannedResult; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.core.util.DataTypeUtil; @@ -42,49 +42,50 @@ import org.apache.commons.lang3.ArrayUtils; */ public class DictionaryBasedResultCollector extends AbstractScannedResultCollector { - protected QueryDimension[] queryDimensions; + protected ProjectionDimension[] queryDimensions; - protected QueryMeasure[] queryMeasures; + protected ProjectionMeasure[] queryMeasures; - protected DirectDictionaryGenerator[] directDictionaryGenerators; + private DirectDictionaryGenerator[] directDictionaryGenerators; /** * query order */ protected int[] order; - protected int[] actualIndexInSurrogateKey; + private int[] actualIndexInSurrogateKey; - protected boolean[] dictionaryEncodingArray; + boolean[] dictionaryEncodingArray; - protected boolean[] directDictionaryEncodingArray; + boolean[] directDictionaryEncodingArray; - protected boolean[] implictColumnArray; + private boolean[] implictColumnArray; - protected boolean[] complexDataTypeArray; + private boolean[] complexDataTypeArray; - protected int dictionaryColumnIndex; - protected int noDictionaryColumnIndex; - protected int complexTypeColumnIndex; + int dictionaryColumnIndex; + int noDictionaryColumnIndex; + int complexTypeColumnIndex; - protected boolean isDimensionExists; + boolean isDimensionExists; - protected Map<Integer, GenericQueryType> comlexDimensionInfoMap; + private Map<Integer, GenericQueryType> comlexDimensionInfoMap; public DictionaryBasedResultCollector(BlockExecutionInfo blockExecutionInfos) { super(blockExecutionInfos); - queryDimensions = tableBlockExecutionInfos.getQueryDimensions(); - queryMeasures = tableBlockExecutionInfos.getQueryMeasures(); + queryDimensions = executionInfo.getProjectionDimensions(); + queryMeasures = executionInfo.getProjectionMeasures(); initDimensionAndMeasureIndexesForFillingData(); isDimensionExists = queryDimensions.length > 0; - this.comlexDimensionInfoMap = tableBlockExecutionInfos.getComlexDimensionInfoMap(); + this.comlexDimensionInfoMap = executionInfo.getComlexDimensionInfoMap(); } /** * This method will add a record both key and value to list object * it will keep track of how many record is processed, to handle limit scenario */ - @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) { + @Override + public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) { // scan the record and add to list List<Object[]> listBasedResult = new ArrayList<>(batchSize); @@ -118,13 +119,13 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect return listBasedResult; } - protected void fillDimensionData(AbstractScannedResult scannedResult, int[] surrogateResult, + void fillDimensionData(BlockletScannedResult scannedResult, int[] surrogateResult, byte[][] noDictionaryKeys, byte[][] complexTypeKeyArray, Map<Integer, GenericQueryType> comlexDimensionInfoMap, Object[] row, int i) { if (!dictionaryEncodingArray[i]) { if (implictColumnArray[i]) { if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID - .equals(queryDimensions[i].getDimension().getColName())) { + .equals(queryDimensions[i].getColumnName())) { row[order[i]] = DataTypeUtil.getDataBasedOnDataType( scannedResult.getBlockletId() + CarbonCommonConstants.FILE_SEPARATOR + scannedResult .getCurrentPageCounter() + CarbonCommonConstants.FILE_SEPARATOR + scannedResult @@ -153,7 +154,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect } } - protected void fillMeasureData(AbstractScannedResult scannedResult, Object[] row) { + void fillMeasureData(BlockletScannedResult scannedResult, Object[] row) { if (measureInfo.getMeasureDataTypes().length > 0) { Object[] msrValues = new Object[measureInfo.getMeasureDataTypes().length]; fillMeasureData(msrValues, 0, scannedResult); @@ -163,7 +164,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect } } - protected void initDimensionAndMeasureIndexesForFillingData() { + void initDimensionAndMeasureIndexesForFillingData() { List<Integer> dictionaryIndexes = new ArrayList<Integer>(); for (int i = 0; i < queryDimensions.length; i++) { if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY) || queryDimensions[i] @@ -190,10 +191,10 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect complexDataTypeArray = CarbonUtil.getComplexDataTypeArray(queryDimensions); order = new int[queryDimensions.length + queryMeasures.length]; for (int i = 0; i < queryDimensions.length; i++) { - order[i] = queryDimensions[i].getQueryOrder(); + order[i] = queryDimensions[i].getOrdinal(); } for (int i = 0; i < queryMeasures.length; i++) { - order[i + queryDimensions.length] = queryMeasures[i].getQueryOrder(); + order[i + queryDimensions.length] = queryMeasures[i].getOrdinal(); } directDictionaryGenerators = new DirectDictionaryGenerator[queryDimensions.length]; for (int i = 0; i < queryDimensions.length; i++) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java index e9d6740..4947621 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java @@ -23,9 +23,9 @@ import java.util.List; import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; -import org.apache.carbondata.core.scan.model.QueryDimension; -import org.apache.carbondata.core.scan.model.QueryMeasure; -import org.apache.carbondata.core.scan.result.AbstractScannedResult; +import org.apache.carbondata.core.scan.model.ProjectionDimension; +import org.apache.carbondata.core.scan.model.ProjectionMeasure; +import org.apache.carbondata.core.scan.result.BlockletScannedResult; import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch; import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; import org.apache.carbondata.core.scan.result.vector.MeasureDataVectorProcessor; @@ -35,35 +35,35 @@ import org.apache.carbondata.core.scan.result.vector.MeasureDataVectorProcessor; */ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultCollector { - protected QueryDimension[] queryDimensions; + protected ProjectionDimension[] queryDimensions; - protected QueryMeasure[] queryMeasures; + protected ProjectionMeasure[] queryMeasures; - protected ColumnVectorInfo[] dictionaryInfo; + private ColumnVectorInfo[] dictionaryInfo; - protected ColumnVectorInfo[] noDictionaryInfo; + private ColumnVectorInfo[] noDictionaryInfo; - protected ColumnVectorInfo[] complexInfo; + private ColumnVectorInfo[] complexInfo; - protected ColumnVectorInfo[] measureColumnInfo; + private ColumnVectorInfo[] measureColumnInfo; - protected ColumnVectorInfo[] allColumnInfo; + ColumnVectorInfo[] allColumnInfo; - protected ColumnVectorInfo[] implictColumnInfo; + private ColumnVectorInfo[] implictColumnInfo; public DictionaryBasedVectorResultCollector(BlockExecutionInfo blockExecutionInfos) { super(blockExecutionInfos); // initialize only if the current block is not a restructured block else the initialization // will be taken care by RestructureBasedVectorResultCollector if (!blockExecutionInfos.isRestructuredBlock()) { - queryDimensions = tableBlockExecutionInfos.getQueryDimensions(); - queryMeasures = tableBlockExecutionInfos.getQueryMeasures(); + queryDimensions = executionInfo.getProjectionDimensions(); + queryMeasures = executionInfo.getProjectionMeasures(); allColumnInfo = new ColumnVectorInfo[queryDimensions.length + queryMeasures.length]; prepareDimensionAndMeasureColumnVectors(); } } - protected void prepareDimensionAndMeasureColumnVectors() { + void prepareDimensionAndMeasureColumnVectors() { measureColumnInfo = new ColumnVectorInfo[queryMeasures.length]; List<ColumnVectorInfo> dictInfoList = new ArrayList<>(); List<ColumnVectorInfo> noDictInfoList = new ArrayList<>(); @@ -78,13 +78,13 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC implictColumnList.add(columnVectorInfo); columnVectorInfo.dimension = queryDimensions[i]; columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal(); - allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo; + allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } else if (!queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) { ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); noDictInfoList.add(columnVectorInfo); columnVectorInfo.dimension = queryDimensions[i]; columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal(); - allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo; + allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } else if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); dictInfoList.add(columnVectorInfo); @@ -92,21 +92,21 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC columnVectorInfo.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType()); columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal(); - allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo; + allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } else if (queryDimensions[i].getDimension().isComplex()) { ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); complexList.add(columnVectorInfo); columnVectorInfo.dimension = queryDimensions[i]; columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal(); columnVectorInfo.genericQueryType = - tableBlockExecutionInfos.getComlexDimensionInfoMap().get(columnVectorInfo.ordinal); - allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo; + executionInfo.getComlexDimensionInfoMap().get(columnVectorInfo.ordinal); + allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } else { ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); dictInfoList.add(columnVectorInfo); columnVectorInfo.dimension = queryDimensions[i]; columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal(); - allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo; + allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } } for (int i = 0; i < queryMeasures.length; i++) { @@ -116,7 +116,7 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC columnVectorInfo.ordinal = queryMeasures[i].getMeasure().getOrdinal(); columnVectorInfo.measure = queryMeasures[i]; this.measureColumnInfo[i] = columnVectorInfo; - allColumnInfo[queryMeasures[i].getQueryOrder()] = columnVectorInfo; + allColumnInfo[queryMeasures[i].getOrdinal()] = columnVectorInfo; } dictionaryInfo = dictInfoList.toArray(new ColumnVectorInfo[dictInfoList.size()]); noDictionaryInfo = noDictInfoList.toArray(new ColumnVectorInfo[noDictInfoList.size()]); @@ -126,11 +126,13 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC Arrays.sort(complexInfo); } - @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) { - throw new UnsupportedOperationException("collectData is not supported here"); + @Override + public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) { + throw new UnsupportedOperationException("collectResultInRow is not supported here"); } - @Override public void collectVectorBatch(AbstractScannedResult scannedResult, + @Override + public void collectResultInColumnarBatch(BlockletScannedResult scannedResult, CarbonColumnarBatch columnarBatch) { int numberOfPages = scannedResult.numberOfpages(); int filteredRows = 0; @@ -150,14 +152,15 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC return; } fillColumnVectorDetails(columnarBatch, rowCounter, requiredRows); - filteredRows = scannedResult - .markFilteredRows(columnarBatch, rowCounter, requiredRows, columnarBatch.getRowCounter()); - scanAndFillResult(scannedResult, columnarBatch, rowCounter, availableRows, requiredRows); + filteredRows = scannedResult.markFilteredRows( + columnarBatch, rowCounter, requiredRows, columnarBatch.getRowCounter()); + fillResultToColumnarBatch( + scannedResult, columnarBatch, rowCounter, availableRows, requiredRows); columnarBatch.setActualSize(columnarBatch.getActualSize() + requiredRows - filteredRows); } } - protected void scanAndFillResult(AbstractScannedResult scannedResult, + void fillResultToColumnarBatch(BlockletScannedResult scannedResult, CarbonColumnarBatch columnarBatch, int rowCounter, int availableRows, int requiredRows) { scannedResult.fillColumnarDictionaryBatch(dictionaryInfo); scannedResult.fillColumnarNoDictionaryBatch(noDictionaryInfo); @@ -174,8 +177,8 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC columnarBatch.setRowCounter(columnarBatch.getRowCounter() + requiredRows); } - protected void fillColumnVectorDetails(CarbonColumnarBatch columnarBatch, int rowCounter, - int requiredRows) { + void fillColumnVectorDetails(CarbonColumnarBatch columnarBatch, int rowCounter, int requiredRows) + { for (int i = 0; i < allColumnInfo.length; i++) { allColumnInfo[i].size = requiredRows; allColumnInfo[i].offset = rowCounter; http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java index a09676a..0780675 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java @@ -16,14 +16,12 @@ */ package org.apache.carbondata.core.scan.collector.impl; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; -import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; -import org.apache.carbondata.core.scan.model.QueryMeasure; -import org.apache.carbondata.core.scan.result.AbstractScannedResult; +import org.apache.carbondata.core.scan.model.ProjectionMeasure; +import org.apache.carbondata.core.scan.result.BlockletScannedResult; import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper; /** @@ -31,15 +29,11 @@ import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper; */ public class RawBasedResultCollector extends AbstractScannedResultCollector { - protected ByteArrayWrapper wrapper; + byte[] dictionaryKeyArray; - protected byte[] dictionaryKeyArray; + byte[][] noDictionaryKeyArray; - protected byte[][] noDictionaryKeyArray; - - protected byte[][] complexTypeKeyArray; - - protected byte[] implicitColumnByteArray; + private byte[][] complexTypeKeyArray; public RawBasedResultCollector(BlockExecutionInfo blockExecutionInfos) { super(blockExecutionInfos); @@ -49,9 +43,10 @@ public class RawBasedResultCollector extends AbstractScannedResultCollector { * This method will add a record both key and value to list object * it will keep track of how many record is processed, to handle limit scenario */ - @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) { + @Override + public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) { List<Object[]> listBasedResult = new ArrayList<>(batchSize); - QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures(); + ProjectionMeasure[] queryMeasures = executionInfo.getProjectionMeasures(); // scan the record and add to list int rowCounter = 0; while (scannedResult.hasNext() && rowCounter < batchSize) { @@ -65,24 +60,21 @@ public class RawBasedResultCollector extends AbstractScannedResultCollector { return listBasedResult; } - protected void prepareRow(AbstractScannedResult scannedResult, List<Object[]> listBasedResult, - QueryMeasure[] queryMeasures) { + void prepareRow(BlockletScannedResult scannedResult, List<Object[]> listBasedResult, + ProjectionMeasure[] queryMeasures) { Object[] row = new Object[1 + queryMeasures.length]; - wrapper = new ByteArrayWrapper(); + ByteArrayWrapper wrapper = new ByteArrayWrapper(); wrapper.setDictionaryKey(dictionaryKeyArray); wrapper.setNoDictionaryKeys(noDictionaryKeyArray); wrapper.setComplexTypesKeys(complexTypeKeyArray); - wrapper.setImplicitColumnByteArray(implicitColumnByteArray); row[0] = wrapper; fillMeasureData(row, 1, scannedResult); listBasedResult.add(row); } - protected void scanResultAndGetData(AbstractScannedResult scannedResult) { + void scanResultAndGetData(BlockletScannedResult scannedResult) { dictionaryKeyArray = scannedResult.getDictionaryKeyArray(); noDictionaryKeyArray = scannedResult.getNoDictionaryKeyArray(); complexTypeKeyArray = scannedResult.getComplexTypeKeyArray(); - implicitColumnByteArray = scannedResult.getBlockletId() - .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java index 296d9a6..8b42a4a 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java @@ -23,8 +23,8 @@ import java.util.Map; import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; import org.apache.carbondata.core.scan.executor.util.RestructureUtil; import org.apache.carbondata.core.scan.filter.GenericQueryType; -import org.apache.carbondata.core.scan.model.QueryMeasure; -import org.apache.carbondata.core.scan.result.AbstractScannedResult; +import org.apache.carbondata.core.scan.model.ProjectionMeasure; +import org.apache.carbondata.core.scan.result.BlockletScannedResult; /** * class for handling restructure scenarios for filling result @@ -35,8 +35,8 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe public RestructureBasedDictionaryResultCollector(BlockExecutionInfo blockExecutionInfos) { super(blockExecutionInfos); - queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions(); - queryMeasures = tableBlockExecutionInfos.getActualQueryMeasures(); + queryDimensions = executionInfo.getActualQueryDimensions(); + queryMeasures = executionInfo.getActualQueryMeasures(); measureDefaultValues = new Object[queryMeasures.length]; fillMeasureDefaultValues(); initDimensionAndMeasureIndexesForFillingData(); @@ -61,7 +61,8 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe * This method will add a record both key and value to list object * it will keep track of how many record is processed, to handle limit scenario */ - @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) { + @Override + public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) { // scan the record and add to list List<Object[]> listBasedResult = new ArrayList<>(batchSize); int rowCounter = 0; @@ -69,7 +70,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe byte[][] noDictionaryKeys; byte[][] complexTypeKeyArray; Map<Integer, GenericQueryType> comlexDimensionInfoMap = - tableBlockExecutionInfos.getComlexDimensionInfoMap(); + executionInfo.getComlexDimensionInfoMap(); while (scannedResult.hasNext() && rowCounter < batchSize) { Object[] row = new Object[queryDimensions.length + queryMeasures.length]; if (isDimensionExists) { @@ -107,13 +108,13 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe } protected void fillMeasureData(Object[] msrValues, int offset, - AbstractScannedResult scannedResult) { + BlockletScannedResult scannedResult) { int measureExistIndex = 0; for (short i = 0; i < measureInfo.getMeasureDataTypes().length; i++) { // if measure exists is block then pass measure column // data chunk to the collector if (measureInfo.getMeasureExists()[i]) { - QueryMeasure queryMeasure = tableBlockExecutionInfos.getQueryMeasures()[measureExistIndex]; + ProjectionMeasure queryMeasure = executionInfo.getProjectionMeasures()[measureExistIndex]; msrValues[i + offset] = getMeasureData( scannedResult.getMeasureChunk(measureInfo.getMeasureOrdinals()[measureExistIndex]), scannedResult.getCurrentRowId(), queryMeasure.getMeasure()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java index b08a1d4..6544a75 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java @@ -30,9 +30,9 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; -import org.apache.carbondata.core.scan.model.QueryDimension; -import org.apache.carbondata.core.scan.model.QueryMeasure; -import org.apache.carbondata.core.scan.result.AbstractScannedResult; +import org.apache.carbondata.core.scan.model.ProjectionDimension; +import org.apache.carbondata.core.scan.model.ProjectionMeasure; +import org.apache.carbondata.core.scan.result.BlockletScannedResult; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.commons.lang3.ArrayUtils; @@ -70,15 +70,15 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector */ private void initRestructuredKeyGenerator() { SegmentProperties segmentProperties = - tableBlockExecutionInfos.getDataBlock().getSegmentProperties(); - QueryDimension[] queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions(); + executionInfo.getDataBlock().getSegmentProperties(); + ProjectionDimension[] queryDimensions = executionInfo.getActualQueryDimensions(); List<Integer> updatedColumnCardinality = new ArrayList<>(queryDimensions.length); List<Integer> updatedDimensionPartitioner = new ArrayList<>(queryDimensions.length); - int[] dictionaryColumnBlockIndex = tableBlockExecutionInfos.getDictionaryColumnBlockIndex(); + int[] dictionaryColumnBlockIndex = executionInfo.getDictionaryColumnChunkIndex(); int dimCounterInCurrentBlock = 0; for (int i = 0; i < queryDimensions.length; i++) { if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) { - if (tableBlockExecutionInfos.getDimensionInfo().getDimensionExists()[i]) { + if (executionInfo.getDimensionInfo().getDimensionExists()[i]) { // get the dictionary key ordinal as column cardinality in segment properties // will only be for dictionary encoded columns CarbonDimension currentBlockDimension = segmentProperties.getDimensions() @@ -124,8 +124,8 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector */ private void initCurrentBlockKeyGenerator() { SegmentProperties segmentProperties = - tableBlockExecutionInfos.getDataBlock().getSegmentProperties(); - int[] dictionaryColumnBlockIndex = tableBlockExecutionInfos.getDictionaryColumnBlockIndex(); + executionInfo.getDataBlock().getSegmentProperties(); + int[] dictionaryColumnBlockIndex = executionInfo.getDictionaryColumnChunkIndex(); int[] updatedColumnCardinality = new int[dictionaryColumnBlockIndex.length]; int[] updatedDimensionPartitioner = new int[dictionaryColumnBlockIndex.length]; for (int i = 0; i < dictionaryColumnBlockIndex.length; i++) { @@ -149,9 +149,10 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector * This method will add a record both key and value to list object * it will keep track of how many record is processed, to handle limit scenario */ - @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) { + @Override + public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) { List<Object[]> listBasedResult = new ArrayList<>(batchSize); - QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getActualQueryMeasures(); + ProjectionMeasure[] queryMeasures = executionInfo.getActualQueryMeasures(); // scan the record and add to list int rowCounter = 0; while (scannedResult.hasNext() && rowCounter < batchSize) { @@ -179,7 +180,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector * @return */ private byte[] fillDictionaryKeyArrayWithLatestSchema(byte[] dictionaryKeyArray) { - QueryDimension[] actualQueryDimensions = tableBlockExecutionInfos.getActualQueryDimensions(); + ProjectionDimension[] actualQueryDimensions = executionInfo.getActualQueryDimensions(); int newKeyArrayLength = dimensionInfo.getNewDictionaryColumnCount(); long[] keyArray = null; if (null != updatedCurrentBlockKeyGenerator) { @@ -222,7 +223,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector * @return */ private byte[][] fillNoDictionaryKeyArrayWithLatestSchema(byte[][] noDictionaryKeyArray) { - QueryDimension[] actualQueryDimensions = tableBlockExecutionInfos.getActualQueryDimensions(); + ProjectionDimension[] actualQueryDimensions = executionInfo.getActualQueryDimensions(); byte[][] noDictionaryKeyArrayWithNewlyAddedColumns = new byte[noDictionaryKeyArray.length + dimensionInfo.getNewNoDictionaryColumnCount()][]; int existingColumnValueIndex = 0; http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java index 0f1b9f9..61a2992 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java @@ -26,7 +26,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; import org.apache.carbondata.core.scan.executor.util.RestructureUtil; -import org.apache.carbondata.core.scan.result.AbstractScannedResult; +import org.apache.carbondata.core.scan.result.BlockletScannedResult; import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch; import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; @@ -43,8 +43,8 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector public RestructureBasedVectorResultCollector(BlockExecutionInfo blockExecutionInfos) { super(blockExecutionInfos); - queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions(); - queryMeasures = tableBlockExecutionInfos.getActualQueryMeasures(); + queryDimensions = executionInfo.getActualQueryDimensions(); + queryMeasures = executionInfo.getActualQueryMeasures(); measureDefaultValues = new Object[queryMeasures.length]; allColumnInfo = new ColumnVectorInfo[queryDimensions.length + queryMeasures.length]; createVectorForNewlyAddedDimensions(); @@ -66,7 +66,7 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector columnVectorInfo.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType()); } - allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo; + allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } } } @@ -79,7 +79,7 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector if (!measureInfo.getMeasureExists()[i]) { // add a dummy column vector result collector object ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); - allColumnInfo[queryMeasures[i].getQueryOrder()] = columnVectorInfo; + allColumnInfo[queryMeasures[i].getOrdinal()] = columnVectorInfo; columnVectorInfo.measure = queryMeasures[i]; measureDefaultValues[i] = getMeasureDefaultValue(queryMeasures[i].getMeasure()); } @@ -97,11 +97,13 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector carbonMeasure.getDefaultValue()); } - @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) { - throw new UnsupportedOperationException("collectData is not supported here"); + @Override + public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) { + throw new UnsupportedOperationException("collectResultInRow is not supported here"); } - @Override public void collectVectorBatch(AbstractScannedResult scannedResult, + @Override + public void collectResultInColumnarBatch(BlockletScannedResult scannedResult, CarbonColumnarBatch columnarBatch) { int numberOfPages = scannedResult.numberOfpages(); while (scannedResult.getCurrentPageCounter() < numberOfPages) { @@ -124,7 +126,8 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector fillDataForNonExistingDimensions(); fillDataForNonExistingMeasures(); // fill existing dimensions and measures data - scanAndFillResult(scannedResult, columnarBatch, rowCounter, availableRows, requiredRows); + fillResultToColumnarBatch( + scannedResult, columnarBatch, rowCounter, availableRows, requiredRows); columnarBatch.setActualSize(columnarBatch.getActualSize() + requiredRows - filteredRows); } } @@ -133,11 +136,11 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector * This method will fill the default values of non existing dimensions in the current block */ private void fillDataForNonExistingDimensions() { - for (int i = 0; i < tableBlockExecutionInfos.getActualQueryDimensions().length; i++) { + for (int i = 0; i < executionInfo.getActualQueryDimensions().length; i++) { if (!dimensionInfo.getDimensionExists()[i]) { - int queryOrder = tableBlockExecutionInfos.getActualQueryDimensions()[i].getQueryOrder(); + int queryOrder = executionInfo.getActualQueryDimensions()[i].getOrdinal(); CarbonDimension dimension = - tableBlockExecutionInfos.getActualQueryDimensions()[i].getDimension(); + executionInfo.getActualQueryDimensions()[i].getDimension(); if (dimension.hasEncoding(Encoding.DIRECT_DICTIONARY)) { // fill direct dictionary column data fillDirectDictionaryData(allColumnInfo[queryOrder].vector, allColumnInfo[queryOrder], @@ -215,10 +218,10 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector * This method will fill the default values of non existing measures in the current block */ private void fillDataForNonExistingMeasures() { - for (int i = 0; i < tableBlockExecutionInfos.getActualQueryMeasures().length; i++) { + for (int i = 0; i < executionInfo.getActualQueryMeasures().length; i++) { if (!measureInfo.getMeasureExists()[i]) { - int queryOrder = tableBlockExecutionInfos.getActualQueryMeasures()[i].getQueryOrder(); - CarbonMeasure measure = tableBlockExecutionInfos.getActualQueryMeasures()[i].getMeasure(); + int queryOrder = executionInfo.getActualQueryMeasures()[i].getOrdinal(); + CarbonMeasure measure = executionInfo.getActualQueryMeasures()[i].getMeasure(); ColumnVectorInfo columnVectorInfo = allColumnInfo[queryOrder]; CarbonColumnVector vector = columnVectorInfo.vector; Object defaultValue = measureDefaultValues[i]; http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java index 1fcccca..30dd1dd 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java @@ -23,7 +23,7 @@ import java.nio.ByteBuffer; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.scan.filter.GenericQueryType; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.spark.sql.catalyst.util.GenericArrayData; import org.apache.spark.sql.types.ArrayType; @@ -86,7 +86,7 @@ public class ArrayQueryType extends ComplexQueryType implements GenericQueryType return new ArrayType(null, true); } - @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) + @Override public void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder) throws IOException { readBlockDataChunk(blockChunkHolder); children.fillRequiredBlockData(blockChunkHolder); http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java index ee43a10..98f0715 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java @@ -20,7 +20,7 @@ package org.apache.carbondata.core.scan.complextypes; import java.io.IOException; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; public class ComplexQueryType { protected String name; @@ -42,7 +42,7 @@ public class ComplexQueryType { protected byte[] copyBlockDataChunk(DimensionRawColumnChunk[] rawColumnChunks, int rowNumber, int pageNumber) { byte[] data = - rawColumnChunks[blockIndex].convertToDimColDataChunk(pageNumber).getChunkData(rowNumber); + rawColumnChunks[blockIndex].decodeColumnPage(pageNumber).getChunkData(rowNumber); byte[] output = new byte[data.length]; System.arraycopy(data, 0, output, 0, output.length); return output; @@ -51,10 +51,10 @@ public class ComplexQueryType { /* * This method will read the block data chunk from the respective block */ - protected void readBlockDataChunk(BlocksChunkHolder blockChunkHolder) throws IOException { - if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { - blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + protected void readBlockDataChunk(RawBlockletColumnChunks blockChunkHolder) throws IOException { + if (null == blockChunkHolder.getDimensionRawColumnChunks()[blockIndex]) { + blockChunkHolder.getDimensionRawColumnChunks()[blockIndex] = blockChunkHolder.getDataBlock() + .readDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java index 978a652..b8aa912 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java @@ -27,7 +27,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory; import org.apache.carbondata.core.keygenerator.mdkey.Bits; import org.apache.carbondata.core.scan.filter.GenericQueryType; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.DataTypeUtil; import org.apache.spark.sql.types.BooleanType$; @@ -113,7 +113,7 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery } } - @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) + @Override public void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder) throws IOException { readBlockDataChunk(blockChunkHolder); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/636eb799/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java index 23a9f81..1d4f141 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java @@ -25,7 +25,7 @@ import java.util.List; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.scan.filter.GenericQueryType; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; import org.apache.spark.sql.types.DataType; @@ -106,7 +106,7 @@ public class StructQueryType extends ComplexQueryType implements GenericQueryTyp return new StructType(fields); } - @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) + @Override public void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder) throws IOException { readBlockDataChunk(blockChunkHolder);
