This is an automated email from the ASF dual-hosted git repository.
marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b3b0766634 [GLUTEN-9148] Fix shuffle file permission issue when using
ColumnarShuffleManager (#9156)
b3b0766634 is described below
commit b3b0766634aebd848f64b44e8c1a0710de998e52
Author: Yuming Wang <[email protected]>
AuthorDate: Fri Mar 28 19:10:46 2025 +0800
[GLUTEN-9148] Fix shuffle file permission issue when using
ColumnarShuffleManager (#9156)
---
cpp/core/shuffle/LocalPartitionWriter.cc | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/cpp/core/shuffle/LocalPartitionWriter.cc
b/cpp/core/shuffle/LocalPartitionWriter.cc
index 6692097986..183b688f7c 100644
--- a/cpp/core/shuffle/LocalPartitionWriter.cc
+++ b/cpp/core/shuffle/LocalPartitionWriter.cc
@@ -387,7 +387,11 @@ std::string LocalPartitionWriter::nextSpilledFileDir() {
arrow::Result<std::shared_ptr<arrow::io::OutputStream>>
LocalPartitionWriter::openFile(const std::string& file) {
std::shared_ptr<arrow::io::FileOutputStream> fout;
- ARROW_ASSIGN_OR_RAISE(fout, arrow::io::FileOutputStream::Open(file));
+ auto fd = open(file.c_str(), O_WRONLY | O_CREAT | O_TRUNC);
+ // Set the shuffle file permissions to 0644 to keep it consistent with the
permissions of
+ // the built-in shuffler manager in Spark.
+ fchmod(fd, 0644);
+ ARROW_ASSIGN_OR_RAISE(fout, arrow::io::FileOutputStream::Open(fd));
if (options_.bufferedWrite) {
// The `shuffleFileBufferSize` bytes is a temporary allocation and will be
freed with file close.
// Use default memory pool and count treat the memory as executor memory
overhead to avoid unnecessary spill.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
