This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch pinot-spark-new in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 34885ecf152f171ac087d8fc0cf4397602c755a0 Author: Xiang Fu <[email protected]> AuthorDate: Wed Nov 6 20:24:50 2019 -0800 Use SparkContext.getOrCreate() to use shared SparkContext if possible. --- .../main/java/org/apache/pinot/spark/jobs/SegmentCreationJob.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SegmentCreationJob.java b/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SegmentCreationJob.java index 74f09ee..fdd9c10 100644 --- a/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SegmentCreationJob.java +++ b/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SegmentCreationJob.java @@ -23,11 +23,9 @@ import java.io.DataOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.Serializable; import java.net.URI; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Properties; import java.util.UUID; import javax.annotation.Nullable; @@ -57,6 +55,7 @@ import org.apache.pinot.core.segment.name.SegmentNameGenerator; import org.apache.pinot.core.segment.name.SimpleSegmentNameGenerator; import org.apache.pinot.spark.utils.JobPreparationHelper; import org.apache.pinot.spark.utils.PushLocation; +import org.apache.spark.SparkContext; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.slf4j.Logger; @@ -360,7 +359,7 @@ public class SegmentCreationJob extends BaseSegmentJob { } _properties.put(JobConfigConstants.SCHEMA, getSchema().toSingleLineJsonString()); - JavaSparkContext sparkContext = new JavaSparkContext(); + JavaSparkContext sparkContext = JavaSparkContext.fromSparkContext(SparkContext.getOrCreate()); addDepsJarToDistributedCache(sparkContext); JavaRDD<String> pathRDD = sparkContext.parallelize(dataFilePathStrs, numDataFiles); pathRDD.zipWithIndex().foreach(tuple2 -> { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
