This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new f80c06892 [CELEBORN] Avoid CelebornShuffleManager#getWriter adding
shuffle id repeatedly to columnarShuffleIds (#6281)
f80c06892 is described below
commit f80c06892c4604b69f822b435bb7c6458f46ac76
Author: Nicholas Jiang <[email protected]>
AuthorDate: Mon Jul 1 08:17:52 2024 +0700
[CELEBORN] Avoid CelebornShuffleManager#getWriter adding shuffle id
repeatedly to columnarShuffleIds (#6281)
---
.../spark/shuffle/gluten/celeborn/CelebornShuffleManager.java | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git
a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
index 63fb0cc1b..e61aeb3d7 100644
---
a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
+++
b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
@@ -215,12 +215,8 @@ public class CelebornShuffleManager implements
ShuffleManager {
@Override
public boolean unregisterShuffle(int shuffleId) {
- if (columnarShuffleIds.contains(shuffleId)) {
- if (columnarShuffleManager().unregisterShuffle(shuffleId)) {
- return columnarShuffleIds.remove(shuffleId);
- } else {
- return false;
- }
+ if (columnarShuffleIds.remove(shuffleId)) {
+ return columnarShuffleManager().unregisterShuffle(shuffleId);
}
return CelebornUtils.unregisterShuffle(
lifecycleManager,
@@ -311,7 +307,6 @@ public class CelebornShuffleManager implements
ShuffleManager {
return vanillaCelebornShuffleManager().getWriter(handle, mapId,
context, metrics);
}
} else {
- columnarShuffleIds.add(handle.shuffleId());
return columnarShuffleManager().getWriter(handle, mapId, context,
metrics);
}
} catch (Exception e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
