This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 3bd232dda [CELEBORN-619][CORE][SHUFFLE][FOLLOWUP] Support enable DRA
with Apache Celeborn
3bd232dda is described below
commit 3bd232dda0a8cffa4e41a76bc69afc63868f5b22
Author: Ethan Feng <[email protected]>
AuthorDate: Tue Jun 6 12:57:16 2023 +0800
[CELEBORN-619][CORE][SHUFFLE][FOLLOWUP] Support enable DRA with Apache
Celeborn
### What changes were proposed in this pull request?
Adapt Spark DRA patch for spark 3.4
### Why are the changes needed?
To support enabling DRA w/ Celeborn on Spark 3.4
### Does this PR introduce _any_ user-facing change?
Yes, this PR provides a DRA patch for Spark 3.4
### How was this patch tested?
Compiled with Spark 3.4
Closes #1546 from FMX/CELEBORN-619.
Lead-authored-by: Ethan Feng <[email protected]>
Co-authored-by: Ethan Feng <[email protected]>
Signed-off-by: Ethan Feng <[email protected]>
---
assets/spark-patch/RSS_DRA_spark3_4.patch | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/assets/spark-patch/RSS_DRA_spark3_4.patch
b/assets/spark-patch/RSS_DRA_spark3_4.patch
index 3b0bad14a..9189fd2d9 100644
--- a/assets/spark-patch/RSS_DRA_spark3_4.patch
+++ b/assets/spark-patch/RSS_DRA_spark3_4.patch
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-Subject: [PATCH] rss support RDA.
+Subject: [PATCH] [CORE][SHUFFLE] Support enabling DRA with Apache Celeborn
---
Index: core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
IDEA additional info:
@@ -22,7 +22,7 @@ Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
===================================================================
diff --git
a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
(revision efae5362cbeaf1594b18edf594e83b2cf72afce6)
-+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
(date 1685515818323)
++++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
(date 1685946134830)
@@ -209,7 +209,7 @@
} else if (decommissionEnabled &&
conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED)) {
@@ -39,7 +39,7 @@ Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
===================================================================
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
(revision efae5362cbeaf1594b18edf594e83b2cf72afce6)
-+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
(date 1685516413475)
++++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
(date 1685946134830)
@@ -2515,7 +2515,7 @@
// if the cluster manager explicitly tells us that the entire worker was
lost, then
// we know to unregister shuffle output. (Note that "worker"
specifically refers to the process
@@ -56,14 +56,13 @@ Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
===================================================================
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala
b/core/src/main/scala/org/apache/spark/util/Utils.scala
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala (revision
efae5362cbeaf1594b18edf594e83b2cf72afce6)
-+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala (date
1685515818317)
-@@ -3271,6 +3271,10 @@
++++ b/core/src/main/scala/org/apache/spark/util/Utils.scala (date
1685946145650)
+@@ -3271,6 +3271,9 @@
files.toSeq
}
+ def isRssEnabled(conf: SparkConf): Boolean =
+ conf.get("spark.shuffle.manager", "sort").contains("celeborn")
-+ }
+
/**
* Return the median number of a long array
@@ -75,7 +74,7 @@ Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
===================================================================
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
(revision efae5362cbeaf1594b18edf594e83b2cf72afce6)
-+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
(date 1685516047047)
++++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
(date 1685946134830)
@@ -1055,7 +1055,7 @@
// data from this dead executor so we would need to rerun these tasks on
other executors.
val maybeShuffleMapOutputLoss = isShuffleMapTasks &&
