This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this
push:
new 4bfaf99 KYLIN-4646 Fix merge error
4bfaf99 is described below
commit 4bfaf99118af5e5c4f9ac6f595912e6811fe6cb9
Author: yaqian.zhang <[email protected]>
AuthorDate: Mon Jul 20 16:37:38 2020 +0800
KYLIN-4646 Fix merge error
---
.../main/java/org/apache/kylin/job/execution/AbstractExecutable.java | 2 +-
.../src/main/java/org/apache/kylin/engine/spark/SparkUHCDictionary.java | 2 +-
.../main/java/org/apache/kylin/engine/spark/job/NSparkMergingJob.java | 1 +
.../kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java | 2 +-
.../apache/kylin/engine/spark/job/ResourceDetectBeforeMergingJob.java | 2 +-
.../test/java/org/apache/kylin/engine/spark/source/CsvSourceTest.java | 2 +-
6 files changed, 6 insertions(+), 5 deletions(-)
diff --git
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 8a37a18..86ac294 100644
---
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -674,7 +674,7 @@ public abstract class AbstractExecutable implements
Executable, Idempotent {
if (cuboidsNum != null) {
return computeDriverMemory(Integer.valueOf(cuboidsNum));
}
- return 0;
+ return 1024;
}
//Default driver memory base is 1024M
diff --git
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkUHCDictionary.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkUHCDictionary.java
index 8f0767c..ad78cab 100644
---
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkUHCDictionary.java
+++
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkUHCDictionary.java
@@ -18,7 +18,7 @@
package org.apache.kylin.engine.spark;
-import com.clearspring.analytics.util.Lists;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkMergingJob.java
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkMergingJob.java
index d64b060..68d4ee2 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkMergingJob.java
+++
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkMergingJob.java
@@ -84,6 +84,7 @@ public class NSparkMergingJob extends CubingJob {
job.setParam(MetadataConstants.P_DATA_RANGE_END,
mergedSegment.getSegRange().end.toString());
job.setParam(MetadataConstants.P_OUTPUT_META_URL,
cube.getConfig().getMetadataUrl().toString());
job.setParam(MetadataConstants.P_JOB_TYPE,
String.valueOf(JobTypeEnum.INDEX_MERGE));
+ job.setParam(MetadataConstants.P_CUBOID_NUMBER,
String.valueOf(cube.getDescriptor().getAllCuboids().size()));
JobStepFactory.addStep(job, JobStepType.RESOURCE_DETECT, cube);
JobStepFactory.addStep(job, JobStepType.MERGING, cube);
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
index 0eb5cc2..f634917 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
+++
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
@@ -38,7 +38,7 @@ import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.Segments;
-import com.clearspring.analytics.util.Lists;
+import com.google.common.collect.Lists;
public class AfterMergeOrRefreshResourceMerger extends MetadataMerger {
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/ResourceDetectBeforeMergingJob.java
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/ResourceDetectBeforeMergingJob.java
index 1e6e769..6f04932 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/ResourceDetectBeforeMergingJob.java
+++
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/ResourceDetectBeforeMergingJob.java
@@ -38,7 +38,7 @@ import org.apache.spark.sql.hive.utils.ResourceDetectUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.clearspring.analytics.util.Lists;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import scala.collection.JavaConversions;
diff --git
a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/source/CsvSourceTest.java
b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/source/CsvSourceTest.java
index 44c4a4d..2094c34 100644
---
a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/source/CsvSourceTest.java
+++
b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/source/CsvSourceTest.java
@@ -18,7 +18,7 @@
package org.apache.kylin.engine.spark.source;
-import com.clearspring.analytics.util.Lists;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
