This is an automated email from the ASF dual-hosted git repository. jackylk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new 4c05dc4 [HOTFIX] Fix ClassName for load datamaps parallel job 4c05dc4 is described below commit 4c05dc44c38ed8c730a372792432c9f7f5e869a4 Author: Indhumathi27 <indhumathi...@gmail.com> AuthorDate: Wed Mar 18 15:52:18 2020 +0530 [HOTFIX] Fix ClassName for load datamaps parallel job Why is this PR needed? Load Datamap parallel was not launching job, because the class name was not correct What changes were proposed in this PR? Change className for load datamaps parallel job Does this PR introduce any user interface change? No Is any new testcase added? No This closes #3674 --- .../main/java/org/apache/carbondata/core/datamap/DataMapUtil.java | 5 +++-- .../sql/secondaryindex/Jobs/SparkBlockletDataMapLoaderJob.scala | 5 +---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapUtil.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapUtil.java index 8e7449e..41da1c2 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapUtil.java @@ -195,9 +195,10 @@ public class DataMapUtil { if (!CarbonProperties.getInstance() .isDistributedPruningEnabled(carbonTable.getDatabaseName(), carbonTable.getTableName()) && BlockletDataMapUtil.loadDataMapsParallel(carbonTable)) { - String clsName = "org.apache.carbondata.spark.rdd.SparkBlockletDataMapLoaderJob"; + String clsName = "org.apache.spark.sql.secondaryindex.Jobs.SparkBlockletDataMapLoaderJob"; DataMapJob dataMapJob = (DataMapJob) createDataMapJob(clsName); - String className = "org.apache.carbondata.hadoop.DistributableBlockletDataMapLoader"; + String className = + "org.apache.spark.sql.secondaryindex.Jobs.DistributableBlockletDataMapLoader"; SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = getValidAndInvalidSegments(carbonTable, FileFactory.getConfiguration()); List<Segment> invalidSegments = validAndInvalidSegmentsInfo.getInvalidSegments(); diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/Jobs/SparkBlockletDataMapLoaderJob.scala b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/Jobs/SparkBlockletDataMapLoaderJob.scala index 48e7312..4d6c6b9 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/Jobs/SparkBlockletDataMapLoaderJob.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/Jobs/SparkBlockletDataMapLoaderJob.scala @@ -80,15 +80,12 @@ class SparkBlockletDataMapLoaderJob extends AbstractDataMapJob { private def addSegmentProperties(carbonTable: CarbonTable, dataMapIndexWrappers: Array[(TableBlockIndexUniqueIdentifier, BlockletDataMapDetailsWithSchema)]): Unit = { - val dataMapWrapperList = scala.collection.mutable.ArrayBuffer - .empty[(TableBlockIndexUniqueIdentifier, - BlockletDataMapDetailsWithSchema)] // use the carbon table schema only as this flow is called when schema is not modified val tableColumnSchema = CarbonUtil .getColumnSchemaList(carbonTable.getVisibleDimensions, carbonTable.getVisibleMeasures) // add segmentProperties in the segmentPropertyCache - dataMapWrapperList.foreach { entry => + dataMapIndexWrappers.foreach { entry => val segmentId = entry._1.getSegmentId val wrapper = SegmentPropertiesAndSchemaHolder.getInstance() .addSegmentProperties(carbonTable, tableColumnSchema, segmentId)