Indhumathi27 commented on a change in pull request #3584: [CARBONDATA-3718] Support SegmentLevel MinMax for better Pruning and less driver memory usage for cache URL: https://github.com/apache/carbondata/pull/3584#discussion_r394837435
########## File path: integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/InsertTaskCompletionListener.scala ########## @@ -20,21 +20,32 @@ package org.apache.carbondata.spark.rdd import org.apache.spark.TaskContext import org.apache.spark.sql.carbondata.execution.datasources.tasklisteners.CarbonLoadTaskCompletionListener import org.apache.spark.sql.execution.command.ExecutionErrors +import org.apache.spark.util.CollectionAccumulator +import org.apache.carbondata.core.segmentmeta.SegmentMetaDataInfo import org.apache.carbondata.core.util.{DataTypeUtil, ThreadLocalTaskInfo} import org.apache.carbondata.processing.loading.{DataLoadExecutor, FailureCauses} import org.apache.carbondata.spark.util.CommonUtil class InsertTaskCompletionListener(dataLoadExecutor: DataLoadExecutor, - executorErrors: ExecutionErrors) + executorErrors: ExecutionErrors, + segmentMetaDataAccumulator: CollectionAccumulator[Map[String, SegmentMetaDataInfo]], + tableName: String, + segmentId: String) extends CarbonLoadTaskCompletionListener { override def onTaskCompletion(context: TaskContext): Unit = { try { - dataLoadExecutor.close() + // fill segment level minMax to accumulator + CarbonDataRDDFactory.fillSegmentMetaDataInfoToAccumulator(tableName, Review comment: done ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services