This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new f9f4ddc5bb [GLUTEN-6887][VL] Daily Update Velox Version (2024_11_29)
(#8086)
f9f4ddc5bb is described below
commit f9f4ddc5bba04800c9d80521fa4c50986775be16
Author: Gluten Performance Bot
<[email protected]>
AuthorDate: Fri Nov 29 16:09:37 2024 +0800
[GLUTEN-6887][VL] Daily Update Velox Version (2024_11_29) (#8086)
Upstream Velox's New Commits:
2e5dadcef by Jimmy Lu, fix: Delta update corner cases (11682)
e8fe5b967 by dependabot[bot], build(deps): Bump pypa/cibuildwheel from
2.12.1 to 2.22.0 (11643)
1ce3c7ad2 by Xiaoxuan Meng, fix: Fix the memory reclaim bytes for hash join
(11642)
4a4b4a030 by Sergey Pershin, Rename "max-coalesced-distance-bytes" and add
session property. (11671)
0d572d23a by David Reveman, build(ci): Add breeze build and test job (11637)
---
cpp/velox/compute/VeloxBackend.cc | 4 ++--
cpp/velox/config/VeloxConfig.h | 2 +-
docs/Configuration.md | 2 +-
ep/build-velox/src/get_velox.sh | 2 +-
shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala | 8 ++++----
5 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index 62cc36163d..c453b9981f 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -273,8 +273,8 @@ void VeloxBackend::initConnector() {
connectorConfMap[velox::connector::hive::HiveConfig::kMaxCoalescedBytes] =
backendConf_->get<std::string>(kMaxCoalescedBytes, "67108864"); // 64M
-
connectorConfMap[velox::connector::hive::HiveConfig::kMaxCoalescedDistanceBytes]
=
- backendConf_->get<std::string>(kMaxCoalescedDistanceBytes, "1048576");
// 1M
+ connectorConfMap[velox::connector::hive::HiveConfig::kMaxCoalescedDistance] =
+ backendConf_->get<std::string>(kMaxCoalescedDistance, "512KB"); // 512KB
connectorConfMap[velox::connector::hive::HiveConfig::kPrefetchRowGroups] =
backendConf_->get<std::string>(kPrefetchRowGroups, "1");
connectorConfMap[velox::connector::hive::HiveConfig::kLoadQuantum] =
diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h
index cb70dc6278..dc234152a3 100644
--- a/cpp/velox/config/VeloxConfig.h
+++ b/cpp/velox/config/VeloxConfig.h
@@ -118,7 +118,7 @@ const std::string kDirectorySizeGuess =
"spark.gluten.sql.columnar.backend.velox
const std::string kFilePreloadThreshold =
"spark.gluten.sql.columnar.backend.velox.filePreloadThreshold";
const std::string kPrefetchRowGroups =
"spark.gluten.sql.columnar.backend.velox.prefetchRowGroups";
const std::string kLoadQuantum =
"spark.gluten.sql.columnar.backend.velox.loadQuantum";
-const std::string kMaxCoalescedDistanceBytes =
"spark.gluten.sql.columnar.backend.velox.maxCoalescedDistanceBytes";
+const std::string kMaxCoalescedDistance =
"spark.gluten.sql.columnar.backend.velox.maxCoalescedDistance";
const std::string kMaxCoalescedBytes =
"spark.gluten.sql.columnar.backend.velox.maxCoalescedBytes";
const std::string kCachePrefetchMinPct =
"spark.gluten.sql.columnar.backend.velox.cachePrefetchMinPct";
diff --git a/docs/Configuration.md b/docs/Configuration.md
index 0c44341c36..cb8efe802e 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -89,7 +89,7 @@ The following configurations are related to Velox settings.
| spark.gluten.sql.columnar.backend.velox.filePreloadThreshold | Set
the file preload threshold for velox file scan.
|
|
| spark.gluten.sql.columnar.backend.velox.prefetchRowGroups | Set
the prefetch row groups for velox file scan.
|
|
| spark.gluten.sql.columnar.backend.velox.loadQuantum | Set
the load quantum for velox file scan.
|
|
-| spark.gluten.sql.columnar.backend.velox.maxCoalescedDistanceBytes | Set
the max coalesced distance bytes for velox file scan.
|
|
+| spark.gluten.sql.columnar.backend.velox.maxCoalescedDistance | Set
the max coalesced distance for velox file scan.
| |
| spark.gluten.sql.columnar.backend.velox.maxCoalescedBytes | Set
the max coalesced bytes for velox file scan.
|
|
| spark.gluten.sql.columnar.backend.velox.cachePrefetchMinPct | Set
prefetch cache min pct for velox file scan.
|
|
| spark.gluten.velox.awsSdkLogLevel | Log
granularity of AWS C++ SDK in velox.
| FATAL
|
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 6c7fabab88..1d3a84d858 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
set -exu
VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2024_11_28
+VELOX_BRANCH=2024_11_29
VELOX_HOME=""
OS=`uname -s`
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index b63a6bfecc..f643ad7eed 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -456,7 +456,7 @@ class GlutenConfig(conf: SQLConf) extends Logging {
conf.getConf(PREFETCH_ROW_GROUPS)
def loadQuantum: Long =
conf.getConf(LOAD_QUANTUM)
- def maxCoalescedDistanceBytes: Long =
+ def maxCoalescedDistance: String =
conf.getConf(MAX_COALESCED_DISTANCE_BYTES)
def maxCoalescedBytes: Long =
conf.getConf(MAX_COALESCED_BYTES)
@@ -2095,11 +2095,11 @@ object GlutenConfig {
.createWithDefaultString("256MB")
val MAX_COALESCED_DISTANCE_BYTES =
-
buildStaticConf("spark.gluten.sql.columnar.backend.velox.maxCoalescedDistanceBytes")
+
buildStaticConf("spark.gluten.sql.columnar.backend.velox.maxCoalescedDistance")
.internal()
.doc(" Set the max coalesced distance bytes for velox file scan")
- .bytesConf(ByteUnit.BYTE)
- .createWithDefaultString("1MB")
+ .stringConf
+ .createWithDefaultString("512KB")
val MAX_COALESCED_BYTES =
buildStaticConf("spark.gluten.sql.columnar.backend.velox.maxCoalescedBytes")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
