This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 1f982d360 [GLUTEN-6380][FOLLOWUP][CELEBORN] Add compression for
row-based shuffle (#6739)
1f982d360 is described below
commit 1f982d3601da7bb0a14d1fc1bb001155570152dd
Author: Kerwin Zhang <[email protected]>
AuthorDate: Thu Aug 8 16:46:17 2024 +0800
[GLUTEN-6380][FOLLOWUP][CELEBORN] Add compression for row-based shuffle
(#6739)
---
.../spark/shuffle/gluten/celeborn/CelebornShuffleManager.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git
a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
index a0516d177..5e14531d6 100644
---
a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
+++
b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
@@ -291,12 +291,18 @@ public class CelebornShuffleManager implements
ShuffleManager {
}
@SuppressWarnings("unchecked")
CelebornShuffleHandle<K, V, V> h = ((CelebornShuffleHandle<K, V, V>)
handle);
+
+ CelebornConf writerConf = celebornConf;
+ if (!(h.dependency() instanceof ColumnarShuffleDependency)) {
+ writerConf = rowBasedCelebornConf;
+ }
+
shuffleClient =
CelebornUtils.getShuffleClient(
h.appUniqueId(),
h.lifecycleManagerHost(),
h.lifecycleManagerPort(),
- celebornConf,
+ writerConf,
h.userIdentifier(),
false,
extension);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
