Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2290#discussion_r191298660
--- Diff:
store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
---
@@ -158,22 +163,36 @@ private QueryModel prune(int queryId, CarbonTable
table, QueryModel queryModel,
CarbonMultiBlockSplit mbSplit, DataMapExprWrapper datamap) throws
IOException {
Objects.requireNonNull(datamap);
List<Segment> segments = new LinkedList<>();
+ HashMap<String, Integer> uniqueSegments = new HashMap<>();
for (CarbonInputSplit split : mbSplit.getAllSplits()) {
- segments.add(
- Segment.toSegment(split.getSegmentId(),
- new LatestFilesReadCommittedScope(table.getTablePath())));
+ String segmentId = split.getSegmentId();
+ if (uniqueSegments.get(segmentId) == null) {
+ segments.add(Segment.toSegment(
+ segmentId,
+ new LatestFilesReadCommittedScope(table.getTablePath(),
segmentId)));
+ uniqueSegments.put(segmentId, 1);
+ } else {
+ uniqueSegments.put(segmentId, uniqueSegments.get(segmentId) + 1);
+ }
+ }
+
+ List<DataMapDistributableWrapper> distributables =
datamap.toDistributable(segments);
+ List<ExtendedBlocklet> prunnedBlocklets = new
LinkedList<ExtendedBlocklet>();
+ for (int i = 0; i < distributables.size(); i++) {
+ DataMapDistributable dataMapDistributable =
distributables.get(i).getDistributable();
+ prunnedBlocklets.addAll(datamap.prune(dataMapDistributable, null));
}
- List<ExtendedBlocklet> prunnedBlocklets = datamap.prune(segments,
null);
- List<String> pathToRead = new LinkedList<>();
- for (ExtendedBlocklet prunnedBlocklet : prunnedBlocklets) {
- pathToRead.add(prunnedBlocklet.getPath());
+ HashMap<String, ExtendedBlocklet> pathToRead = new HashMap<>();
+ for (ExtendedBlocklet prunedBlocklet : prunnedBlocklets) {
+ pathToRead.put(prunedBlocklet.getFilePath(), prunedBlocklet);
}
List<TableBlockInfo> blocks = queryModel.getTableBlockInfos();
List<TableBlockInfo> blockToRead = new LinkedList<>();
for (TableBlockInfo block : blocks) {
- if (pathToRead.contains(block.getFilePath())) {
+ if (pathToRead.keySet().contains(block.getFilePath())) {
+
block.setDataMapWriterPath(pathToRead.get(block.getFilePath()).getDataMapWriterPath());
--- End diff --
Why need to set this?
---
