This is an automated email from the ASF dual-hosted git repository. jackylk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new f2fa4e5 [CARBONDATA-3593] Add BLOCKLET_SCANNED_NUM and fix TOTAL_BLOCKLET_NUM not right when blocklet filtered f2fa4e5 is described below commit f2fa4e5c57a1c4004f82a40995b29351b41a5950 Author: 沈洪 <yuling...@alipay.com> AuthorDate: Fri Nov 22 21:00:16 2019 +0800 [CARBONDATA-3593] Add BLOCKLET_SCANNED_NUM and fix TOTAL_BLOCKLET_NUM not right when blocklet filtered Add BLOCKLET_SCANNED_NUM and fix TOTAL_BLOCKLET_NUM not right when blocklet filtered. This closes #3476 --- .../AbstractDetailQueryResultIterator.java | 5 ++++ .../scan/scanner/impl/BlockletFilterScanner.java | 34 +++++++++++++++++----- .../core/stats/QueryStatisticsConstants.java | 2 ++ .../carbondata/core/stats/TaskStatistics.java | 1 + 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java index fa97d7a..25ba9a0 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java @@ -245,6 +245,11 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato .put(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, queryStatisticValidScanBlocklet); queryStatisticsModel.getRecorder().recordStatistics(queryStatisticValidScanBlocklet); + QueryStatistic scannedBlocklets = new QueryStatistic(); + queryStatisticsModel.getStatisticsTypeAndObjMap() + .put(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM, scannedBlocklets); + queryStatisticsModel.getRecorder().recordStatistics(scannedBlocklets); + QueryStatistic totalNumberOfPages = new QueryStatistic(); queryStatisticsModel.getStatisticsTypeAndObjMap() .put(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalNumberOfPages); diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java index ca1dc15..347b9ce 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java @@ -113,6 +113,10 @@ public class BlockletFilterScanner extends BlockletFullScanner { .get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED); totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalPagesScanned.getCount() + dataBlock.numberOfPages()); + QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap() + .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM); + totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, + totalBlockletStatistic.getCount() + 1); // apply min max if (isMinMaxEnabled) { if (null == dataBlock.getColumnsMaxValue() @@ -168,10 +172,6 @@ public class BlockletFilterScanner extends BlockletFullScanner { private BlockletScannedResult executeFilter(RawBlockletColumnChunks rawBlockletColumnChunks) throws FilterUnsupportedException, IOException { long startTime = System.currentTimeMillis(); - QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap() - .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM); - totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, - totalBlockletStatistic.getCount() + 1); // set the indexed data if it has any during fgdatamap pruning. BitSetGroup fgBitSetGroup = rawBlockletColumnChunks.getDataBlock().getIndexedData(); rawBlockletColumnChunks.setBitSetGroup(fgBitSetGroup); @@ -188,6 +188,11 @@ public class BlockletFilterScanner extends BlockletFullScanner { scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, scanTime.getCount() + (System.currentTimeMillis() - startTime)); + QueryStatistic scannedBlocklets = queryStatisticsModel.getStatisticsTypeAndObjMap() + .get(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM); + scannedBlocklets.addCountStatistic(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM, + scannedBlocklets.getCount() + 1); + QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.PAGE_SCANNED); scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED, @@ -211,6 +216,12 @@ public class BlockletFilterScanner extends BlockletFullScanner { .get(QueryStatisticsConstants.VALID_PAGE_SCANNED); validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED, validPages.getCount() + bitSetGroup.getValidPages()); + + QueryStatistic scannedBlocklets = queryStatisticsModel.getStatisticsTypeAndObjMap() + .get(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM); + scannedBlocklets.addCountStatistic(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM, + scannedBlocklets.getCount() + 1); + QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.PAGE_SCANNED); scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED, @@ -351,10 +362,6 @@ public class BlockletFilterScanner extends BlockletFullScanner { RawBlockletColumnChunks rawBlockletColumnChunks) throws FilterUnsupportedException, IOException { long startTime = System.currentTimeMillis(); - QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap() - .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM); - totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, - totalBlockletStatistic.getCount() + 1); // apply filter on actual data, for each page BitSet pages = this.filterExecuter.prunePages(rawBlockletColumnChunks); // if filter result is empty then return with empty result @@ -367,6 +374,11 @@ public class BlockletFilterScanner extends BlockletFullScanner { scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, scanTime.getCount() + (System.currentTimeMillis() - startTime)); + QueryStatistic scannedBlocklets = queryStatisticsModel.getStatisticsTypeAndObjMap() + .get(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM); + scannedBlocklets.addCountStatistic(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM, + scannedBlocklets.getCount() + 1); + QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.PAGE_SCANNED); scannedPages @@ -388,6 +400,12 @@ public class BlockletFilterScanner extends BlockletFullScanner { .get(QueryStatisticsConstants.VALID_PAGE_SCANNED); validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED, validPages.getCount() + pages.cardinality()); + + QueryStatistic scannedBlocklets = queryStatisticsModel.getStatisticsTypeAndObjMap() + .get(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM); + scannedBlocklets.addCountStatistic(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM, + scannedBlocklets.getCount() + 1); + QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.PAGE_SCANNED); scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED, diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java index 8a52294..31d9f93 100644 --- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java @@ -52,6 +52,8 @@ public interface QueryStatisticsConstants { String VALID_SCAN_BLOCKLET_NUM = "The num of valid scanned blocklet"; + String BLOCKLET_SCANNED_NUM = "The num of blocklet scanned"; + String VALID_PAGE_SCANNED = "The number of valid page scanned"; String TOTAL_PAGE_SCANNED = "The number of total page scanned"; diff --git a/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java b/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java index e63da18..2681f23 100644 --- a/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java +++ b/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java @@ -41,6 +41,7 @@ public class TaskStatistics implements Serializable { new Column("carbon_IO_time", QueryStatisticsConstants.READ_BLOCKlET_TIME), new Column("scan_blocks_num", QueryStatisticsConstants.SCAN_BLOCKS_NUM), new Column("total_blocklets", QueryStatisticsConstants.TOTAL_BLOCKLET_NUM), + new Column("scanned_blocklets", QueryStatisticsConstants.BLOCKLET_SCANNED_NUM), new Column("valid_blocklets", QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM), new Column("total_pages", QueryStatisticsConstants.TOTAL_PAGE_SCANNED), new Column("scanned_pages", QueryStatisticsConstants.PAGE_SCANNED),