This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new a83583e3a [#1088] feat(spark): Add dynamic allocation patch for Spark
3.0 (#1241)
a83583e3a is described below
commit a83583e3a4c623f501971cf9a7bf8d282707bc34
Author: summaryzb <[email protected]>
AuthorDate: Mon Oct 16 02:14:51 2023 -0500
[#1088] feat(spark): Add dynamic allocation patch for Spark 3.0 (#1241)
### What changes were proposed in this pull request?
Add the dynamic allocation patch for Spark 3.0
### Why are the changes needed?
https://github.com/apache/incubator-uniffle/issues/1088
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manual test spark
---
.../spark-3.0.1_dynamic_allocation_support.patch | 91 ++++++++++++++++++++++
1 file changed, 91 insertions(+)
diff --git a/patch/spark/spark-3.0.1_dynamic_allocation_support.patch
b/patch/spark/spark-3.0.1_dynamic_allocation_support.patch
new file mode 100644
index 000000000..418dc3557
--- /dev/null
+++ b/patch/spark/spark-3.0.1_dynamic_allocation_support.patch
@@ -0,0 +1,91 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+diff --git
a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+index a6d2849f4fb..d281011bbda 100644
+--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
++++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+@@ -196,7 +196,9 @@ private[spark] class ExecutorAllocationManager(
+ s"s${DYN_ALLOCATION_SUSTAINED_SCHEDULER_BACKLOG_TIMEOUT.key} must be
> 0!")
+ }
+ if (!conf.get(config.SHUFFLE_SERVICE_ENABLED)) {
+- if (conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED)) {
++ if (conf.isRssEnable) {
++ logInfo("Dynamic allocation will use remote shuffle service")
++ } else if (conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED)) {
+ logWarning("Dynamic allocation without a shuffle service is an
experimental feature.")
+ } else if (!testing) {
+ throw new SparkException("Dynamic allocation of executors requires
the external " +
+diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala
b/core/src/main/scala/org/apache/spark/SparkConf.scala
+index 40915e3904f..11d992dc992 100644
+--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
++++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
+@@ -589,6 +589,10 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable
with Logging with Seria
+ Utils.redact(this, getAll).sorted.map { case (k, v) => k + "=" + v
}.mkString("\n")
+ }
+
++ /**
++ * Return true if remote shuffle service is enabled.
++ */
++ def isRssEnable: Boolean = get("spark.shuffle.manager",
"sort").contains("RssShuffleManager")
+ }
+
+ private[spark] object SparkConf extends Logging {
+diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+index b483b526622..a5257e71e1d 100644
+--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
++++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+@@ -1851,7 +1851,8 @@ private[spark] class DAGScheduler(
+ // if the cluster manager explicitly tells us that the entire worker was
lost, then
+ // we know to unregister shuffle output. (Note that "worker"
specifically refers to the process
+ // from a Standalone cluster, where the shuffle service lives in the
Worker.)
+- val fileLost = workerLost ||
!env.blockManager.externalShuffleServiceEnabled
++ val fileLost = (workerLost ||
!env.blockManager.externalShuffleServiceEnabled) &&
++ !sc.conf.isRssEnable
+ removeExecutorAndUnregisterOutputs(
+ execId = execId,
+ fileLost = fileLost,
+diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+index 2ce11347ade..c2118416917 100644
+--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
++++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+@@ -944,7 +944,7 @@ private[spark] class TaskSetManager(
+ // The reason is the next stage wouldn't be able to fetch the data from
this dead executor
+ // so we would need to rerun these tasks on other executors.
+ if (tasks(0).isInstanceOf[ShuffleMapTask] &&
!env.blockManager.externalShuffleServiceEnabled
+- && !isZombie) {
++ && !isZombie && !conf.isRssEnable) {
+ for ((tid, info) <- taskInfos if info.executorId == execId) {
+ val index = taskInfos(tid).index
+ // We may have a running task whose partition has been marked as
successful,
+diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
+index 53ab0493f47..ee535f370b6 100644
+---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
++++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
+@@ -154,6 +154,9 @@ class ShuffledRowRDD(
+ }
+
+ override def getPreferredLocations(partition: Partition): Seq[String] = {
++ if (conf.isRssEnable) {
++ return Nil
++ }
+ val tracker =
SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
+ partition.asInstanceOf[ShuffledRowRDDPartition].spec match {
+ case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
+--
+2.39.3 (Apple Git-145)
+
