ravipesala commented on a change in pull request #3177:
[CARBONDATA-3337][CARBONDATA-3306] Distributed index server
URL: https://github.com/apache/carbondata/pull/3177#discussion_r279624078
##########
File path:
core/src/main/java/org/apache/carbondata/core/datamap/DistributableDataMapFormat.java
##########
@@ -85,33 +138,49 @@
private ExtendedBlocklet currBlocklet;
private List<DataMap> dataMaps;
- @Override public void initialize(InputSplit inputSplit,
TaskAttemptContext taskAttemptContext)
+ @Override
+ public void initialize(InputSplit inputSplit, TaskAttemptContext
taskAttemptContext)
throws IOException, InterruptedException {
- distributable = (DataMapDistributableWrapper) inputSplit;
- // clear the segmentMap and from cache in executor when there are
invalid segments
- if (invalidSegments.size() > 0) {
- DataMapStoreManager.getInstance().clearInvalidSegments(table,
invalidSegments);
- }
- TableDataMap tableDataMap = DataMapStoreManager.getInstance()
- .getDataMap(table,
distributable.getDistributable().getDataMapSchema());
+ DataMapDistributableWrapper distributable =
(DataMapDistributableWrapper) inputSplit;
+
distributable.getDistributable().getSegment().setCacheable(!isFallbackJob);
+
distributable.getDistributable().getSegment().setReadCommittedScope(readCommittedScope);
+ List<Segment> segmentsToLoad = new ArrayList<>();
+ segmentsToLoad.add(distributable.getDistributable().getSegment());
if (isJobToClearDataMaps) {
// if job is to clear datamaps just clear datamaps from cache and
return
DataMapStoreManager.getInstance()
.clearDataMaps(table.getCarbonTableIdentifier().getTableUniqueName());
// clear the segment properties cache from executor
SegmentPropertiesAndSchemaHolder.getInstance()
.invalidate(table.getAbsoluteTableIdentifier());
- blockletIterator = Collections.emptyIterator();
+ List<ExtendedBlocklet> list = new ArrayList<ExtendedBlocklet>();
+ list.add(new ExtendedBlocklet());
+ blockletIterator = list.iterator();
return;
+ } else if (invalidSegments.size() > 0) {
+ // clear the segmentMap and from cache in executor when there are
invalid segments
+ DataMapStoreManager.getInstance().clearInvalidSegments(table,
invalidSegments);
}
- dataMaps =
tableDataMap.getTableDataMaps(distributable.getDistributable());
- List<ExtendedBlocklet> blocklets = tableDataMap
- .prune(dataMaps,
- distributable.getDistributable(),
+ SegmentUpdateStatusManager updateStatusManager = new
SegmentUpdateStatusManager(table);
+ DataMapStoreManager.getInstance()
+ .refreshSegmentCacheIfRequired(table, updateStatusManager,
segmentsToLoad);
+ List<ExtendedBlocklet> blocklets = new ArrayList<>();
+ if (dataMapLevel == null) {
+ TableDataMap defaultDataMap = DataMapStoreManager.getInstance()
+ .getDataMap(table,
distributable.getDistributable().getDataMapSchema());
+ dataMaps =
defaultDataMap.getTableDataMaps(distributable.getDistributable());
+ if (table.isTransactionalTable()) {
+ blocklets = defaultDataMap.prune(dataMaps,
distributable.getDistributable(),
Review comment:
No need to pass the uniqueid for default datamap
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
