RexXiong commented on code in PR #2813:
URL: https://github.com/apache/celeborn/pull/2813#discussion_r1883380879
##########
client-spark/common/src/main/java/org/apache/spark/shuffle/celeborn/SparkCommonUtils.java:
##########
@@ -19,14 +19,12 @@
import org.apache.spark.SparkConf;
import org.apache.spark.TaskContext;
-import org.apache.spark.scheduler.DAGScheduler;
public class SparkCommonUtils {
public static void validateAttemptConfig(SparkConf conf) throws
IllegalArgumentException {
+ int DEFAULT_MAX_CONSECUTIVE_STAGE_ATTEMPTS = 4;
int maxStageAttempts =
- conf.getInt(
Review Comment:
should be `Math.max(maxConsecutiveStageAttempts,
sc.getConf.get(config.STAGE_MAX_ATTEMPTS))` see `DagScheduler.maxStageAttempts`
##########
.github/workflows/maven.yml:
##########
@@ -69,6 +69,9 @@ jobs:
run: |
SPARK_BINARY_VERSION=${{ matrix.spark }}
SPARK_MAJOR_VERSION=${SPARK_BINARY_VERSION%%.*}
+ if [[ $SPARK_MAJOR_VERSION == "3" ]]; then
Review Comment:
Seems shouldn't change major version for spark 2
##########
project/CelebornBuild.scala:
##########
@@ -807,6 +809,23 @@ object Spark35 extends SparkClientProjects {
override val sparkColumnarShuffleVersion: String = "3.5"
}
+object Spark40 extends SparkClientProjects {
+
+ val sparkClientProjectPath = "client-spark/spark-3-4"
+ val sparkClientProjectName = "celeborn-client-spark-3"
Review Comment:
celeborn-client-spark-3 ->celeborn-client-spark-4
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
