This is an automated email from the ASF dual-hosted git repository. jackylk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new 7ed7cf1 [CARBONDATA-3764] Reduce Reusable buffer creation when few projections selected out of many columns 7ed7cf1 is described below commit 7ed7cf1f69b9b852c8f3a5bf41f6d2a9b5f175e7 Author: ajantha-bhat <ajanthab...@gmail.com> AuthorDate: Fri Apr 3 12:31:34 2020 +0530 [CARBONDATA-3764] Reduce Reusable buffer creation when few projections selected out of many columns Why is this PR needed? If few projections selected out of many columns, reusable buffer is based on columns count instead of projections count. Hence many unused objects were created. What changes were proposed in this PR? Create reusable buffers only as per the projection count. Fix spelling error from Resusable to Reusable Does this PR introduce any user interface change? No Is any new testcase added? No. Internal change This closes #3693 --- .../scan/executor/impl/AbstractQueryExecutor.java | 25 ++++++++++------------ .../scan/executor/infos/BlockExecutionInfo.java | 20 ++++++++--------- .../core/scan/result/BlockletScannedResult.java | 4 ++-- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java index 05d3876..dcb2c0f 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java @@ -382,16 +382,16 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> { dataRefNode.getBlockInfos().get(0).getDeletedDeltaFilePath(), dataRefNode.getBlockInfos().get(0).getSegment()); if (null == dimensionReusableDataBuffers || null == measureReusableDataBuffers) { - dimensionReusableDataBuffers = blockExecutionInfoForBlock.getDimensionResusableDataBuffer(); - measureReusableDataBuffers = blockExecutionInfoForBlock.getMeasureResusableDataBuffer(); + dimensionReusableDataBuffers = blockExecutionInfoForBlock.getDimensionReusableDataBuffer(); + measureReusableDataBuffers = blockExecutionInfoForBlock.getMeasureReusableDataBuffer(); } else { if (dimensionReusableDataBuffers.length == blockExecutionInfoForBlock - .getDimensionResusableDataBuffer().length) { - blockExecutionInfoForBlock.setDimensionResusableDataBuffer(dimensionReusableDataBuffers); + .getDimensionReusableDataBuffer().length) { + blockExecutionInfoForBlock.setDimensionReusableDataBuffer(dimensionReusableDataBuffers); } if (measureReusableDataBuffers.length == blockExecutionInfoForBlock - .getMeasureResusableDataBuffer().length) { - blockExecutionInfoForBlock.setMeasureResusableDataBuffer(measureReusableDataBuffers); + .getMeasureReusableDataBuffer().length) { + blockExecutionInfoForBlock.setMeasureReusableDataBuffer(measureReusableDataBuffers); } } blockExecutionInfoList.add(blockExecutionInfoForBlock); @@ -510,13 +510,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> { int[] dimensionChunkIndexes = QueryUtil.getDimensionChunkIndexes(projectDimensions, segmentProperties.getDimensionOrdinalToChunkMapping(), currentBlockFilterDimensions, allProjectionListDimensionIdexes); - int reusableBufferSize = Math.max(segmentProperties.getDimensionOrdinalToChunkMapping().size(), - projectDimensions.size()); - ReusableDataBuffer[] dimensionBuffer = new ReusableDataBuffer[reusableBufferSize]; + ReusableDataBuffer[] dimensionBuffer = new ReusableDataBuffer[projectDimensions.size()]; for (int i = 0; i < dimensionBuffer.length; i++) { dimensionBuffer[i] = new ReusableDataBuffer(); } - blockExecutionInfo.setDimensionResusableDataBuffer(dimensionBuffer); + blockExecutionInfo.setDimensionReusableDataBuffer(dimensionBuffer); int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance() .getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO, CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE)); @@ -541,13 +539,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> { projectionMeasures, expressionMeasures, segmentProperties.getMeasuresOrdinalToChunkMapping(), filterMeasures, allProjectionListMeasureIndexes); - reusableBufferSize = Math.max(segmentProperties.getMeasuresOrdinalToChunkMapping().size(), - allProjectionListMeasureIndexes.size()); - ReusableDataBuffer[] measureBuffer = new ReusableDataBuffer[reusableBufferSize]; + ReusableDataBuffer[] measureBuffer = + new ReusableDataBuffer[allProjectionListMeasureIndexes.size()]; for (int i = 0; i < measureBuffer.length; i++) { measureBuffer[i] = new ReusableDataBuffer(); } - blockExecutionInfo.setMeasureResusableDataBuffer(measureBuffer); + blockExecutionInfo.setMeasureReusableDataBuffer(measureBuffer); if (measureChunkIndexes.length > 0) { numberOfElementToConsider = measureChunkIndexes[measureChunkIndexes.length - 1] diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java index eb19852..e3b4cd9 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java @@ -203,9 +203,9 @@ public class BlockExecutionInfo { */ private boolean isDirectVectorFill; - private ReusableDataBuffer[] dimensionResusableDataBuffer; + private ReusableDataBuffer[] dimensionReusableDataBuffer; - private ReusableDataBuffer[] measureResusableDataBuffer; + private ReusableDataBuffer[] measureReusableDataBuffer; /** * It is used to read only the deleted data of a particular version. It will be used to get the @@ -588,20 +588,20 @@ public class BlockExecutionInfo { isDirectVectorFill = directVectorFill; } - public ReusableDataBuffer[] getDimensionResusableDataBuffer() { - return dimensionResusableDataBuffer; + public ReusableDataBuffer[] getDimensionReusableDataBuffer() { + return dimensionReusableDataBuffer; } - public void setDimensionResusableDataBuffer(ReusableDataBuffer[] dimensionResusableDataBuffer) { - this.dimensionResusableDataBuffer = dimensionResusableDataBuffer; + public void setDimensionReusableDataBuffer(ReusableDataBuffer[] dimensionReusableDataBuffer) { + this.dimensionReusableDataBuffer = dimensionReusableDataBuffer; } - public ReusableDataBuffer[] getMeasureResusableDataBuffer() { - return measureResusableDataBuffer; + public ReusableDataBuffer[] getMeasureReusableDataBuffer() { + return measureReusableDataBuffer; } - public void setMeasureResusableDataBuffer(ReusableDataBuffer[] measureResusableDataBuffer) { - this.measureResusableDataBuffer = measureResusableDataBuffer; + public void setMeasureReusableDataBuffer(ReusableDataBuffer[] measureReusableDataBuffer) { + this.measureReusableDataBuffer = measureReusableDataBuffer; } public boolean isReadOnlyDelta() { diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java index 6172e09..5f333ae 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java @@ -157,8 +157,8 @@ public abstract class BlockletScannedResult { public BlockletScannedResult(BlockExecutionInfo blockExecutionInfo, QueryStatisticsModel queryStatisticsModel) { - this.dimensionReusableBuffer = blockExecutionInfo.getDimensionResusableDataBuffer(); - this.measureReusableBuffer = blockExecutionInfo.getMeasureResusableDataBuffer(); + this.dimensionReusableBuffer = blockExecutionInfo.getDimensionReusableDataBuffer(); + this.measureReusableBuffer = blockExecutionInfo.getMeasureReusableDataBuffer(); this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize(); this.noDictionaryColumnChunkIndexes = blockExecutionInfo.getNoDictionaryColumnChunkIndexes(); this.dictionaryColumnChunkIndexes = blockExecutionInfo.getDictionaryColumnChunkIndex();