This is an automated email from the ASF dual-hosted git repository.
rexxiong pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
from 45450e793 [CELEBORN-1832] MapPartitionData should create fixed thread
pool with registration of ThreadPoolSource
add ad933815b [CELEBORN-1720] Prevent stage re-run if another task attempt
is running or successful
No new revisions were added by this update.
Summary of changes:
.../task/reduce/CelebornShuffleConsumer.java | 1 +
.../shuffle/celeborn/SparkShuffleManager.java | 5 +
.../apache/spark/shuffle/celeborn/SparkUtils.java | 143 ++++++++++++++++++
.../shuffle/celeborn/CelebornShuffleReader.scala | 11 +-
...uffleFetchFailureReportTaskCleanListener.scala} | 18 +--
.../shuffle/celeborn/SparkShuffleManager.java | 4 +
.../apache/spark/shuffle/celeborn/SparkUtils.java | 143 ++++++++++++++++++
.../shuffle/celeborn/CelebornShuffleReader.scala | 3 +-
...uffleFetchFailureReportTaskCleanListener.scala} | 18 +--
.../org/apache/celeborn/client/ShuffleClient.java | 5 +-
.../apache/celeborn/client/ShuffleClientImpl.java | 5 +-
.../celeborn/client/read/CelebornInputStream.java | 7 +-
.../apache/celeborn/client/LifecycleManager.scala | 42 +++++-
.../apache/celeborn/client/DummyShuffleClient.java | 3 +-
.../celeborn/client/WithShuffleClientSuite.scala | 2 +
common/src/main/proto/TransportMessages.proto | 1 +
.../tests/spark/CelebornFetchFailureSuite.scala | 66 +--------
.../celeborn/tests/spark/SparkTestBase.scala | 62 +++++++-
.../spark/shuffle/celeborn/SparkUtilsSuite.scala | 160 +++++++++++++++++++++
.../service/deploy/cluster/ReadWriteTestBase.scala | 1 +
20 files changed, 603 insertions(+), 97 deletions(-)
copy
client-spark/{common/src/main/java/org/apache/spark/shuffle/celeborn/OpenByteArrayOutputStream.java
=>
spark-2/src/main/scala/org/apache/spark/shuffle/celeborn/ShuffleFetchFailureReportTaskCleanListener.scala}
(65%)
copy
client-spark/{common/src/main/java/org/apache/spark/shuffle/celeborn/OpenByteArrayOutputStream.java
=>
spark-3-4/src/main/scala/org/apache/spark/shuffle/celeborn/ShuffleFetchFailureReportTaskCleanListener.scala}
(65%)
create mode 100644
tests/spark-it/src/test/scala/org/apache/spark/shuffle/celeborn/SparkUtilsSuite.scala
