This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 24d949e61 [VL] Daily Update Velox Version (2024_05_09) (#5664)
24d949e61 is described below
commit 24d949e617103a98e339ecbedc1c73c35a48a513
Author: Gluten Performance Bot
<[email protected]>
AuthorDate: Fri May 10 04:20:00 2024 +0800
[VL] Daily Update Velox Version (2024_05_09) (#5664)
Upstream Velox's New Commits:
f53687563 by zjuwangg, Add more test method in Filter::AlwaysFalse class
(9718)
48cfbe3e7 by Zac Wen, Prevent duplicates in writable regions (9746)
87ea7d5fb by Daniel Munoz, Create howMuchToSkip method in UnitLoaderTools
(9706)
16e2c1375 by Daniel Munoz, UnitLoader supports initial rowsToSkip (9703)
dec2556b8 by Ubuntu, Add barrier at kernel end and scatterBits (9745)
20be1936b by PHILO-HE, Register some re-usable Presto functions for Spark
(9425)
798a12736 by Jimmy Lu, Fix crash in ArrayVectorBase::copyRangesImpl in case
target offsets or sizes is nullptr (9725)
bc6632ac9 by Karteekmurthys, Add decimal support for min_by and max_by
functions (8723)
5c4903fe2 by Zac Wen, Handle ssd cache region score overflow (9709)
49b124665 by Sergey Pershin, Improve instrumentation around Zombie Task
detection. (9663)
---
.github/workflows/velox_docker_cache.yml | 68 ++++++++++++++++++++++
.../org/apache/gluten/execution/TestOperator.scala | 2 +-
.../tests/Substrait2VeloxPlanConversionTest.cc | 6 +-
ep/build-velox/src/get_velox.sh | 2 +-
4 files changed, 73 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/velox_docker_cache.yml
b/.github/workflows/velox_docker_cache.yml
index 969053657..d79b5e717 100644
--- a/.github/workflows/velox_docker_cache.yml
+++ b/.github/workflows/velox_docker_cache.yml
@@ -51,3 +51,71 @@ jobs:
with:
path: ./cpp/build/releases/
key: cache-velox-build-${{ hashFiles('./cache-key') }}
+
+ ccache-native-lib-ubuntu-velox-ut:
+ runs-on: ubuntu-20.04
+ env:
+ CCACHE_DIR: "${{ github.workspace }}/.ccache"
+ container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx
+ steps:
+ - uses: actions/checkout@v2
+ - name: Get Ccache
+ uses: actions/cache/restore@v3
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-ubuntu-release-default
+ - name: Ensure Cache Dirs Exists
+ working-directory: ${{ github.workspace }}
+ run: |
+ mkdir -p '${{ env.CCACHE_DIR }}'
+ - name: Build Gluten velox third party
+ run: |
+ rm -rf /opt/miniconda-for-velox/
+ cd ep/build-velox/src && \
+ ./get_velox.sh
+ cd ../build/velox_ep/
+ git reset --hard
+ make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON
-DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON"
+
+ - name: CCache after
+ run: |
+ ccache -vs
+
+ - uses: actions/cache/save@v3
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-ubuntu-release-default
+ ccache-native-lib-centos-velox-ut:
+ runs-on: ubuntu-20.04
+ env:
+ CCACHE_DIR: "${{ github.workspace }}/.ccache"
+ container: ghcr.io/facebookincubator/velox-dev:circleci-avx
+ steps:
+ - uses: actions/checkout@v2
+ - name: Get Ccache
+ uses: actions/cache/restore@v3
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-centos-release-default
+ - name: Ensure Cache Dirs Exists
+ working-directory: ${{ github.workspace }}
+ run: |
+ mkdir -p '${{ env.CCACHE_DIR }}'
+ - name: Build Gluten velox third party
+ run: |
+ rm -rf /opt/miniconda-for-velox/
+ cd ep/build-velox/src && \
+ ./get_velox.sh
+ cd ../build/velox_ep/
+ source /opt/rh/gcc-toolset-9/enable
+ git reset --hard
+ make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON
-DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON"
+
+ - name: CCache after
+ run: |
+ ccache -vs
+
+ - uses: actions/cache/save@v3
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-centos-release-default
\ No newline at end of file
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
index a14a5b7e7..920b8e2bd 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
@@ -630,7 +630,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite {
}
assert(wholeStageTransformers.size == 3)
val nativePlanString = wholeStageTransformers.head.nativePlanString()
- assert(nativePlanString.contains("Aggregation[SINGLE"))
+ assert(nativePlanString.contains("Aggregation[1][SINGLE"))
assert(nativePlanString.contains("ValueStream"))
assert(wholeStageTransformers(1).nativePlanString().contains("ValueStream"))
assert(wholeStageTransformers.last.nativePlanString().contains("TableScan"))
diff --git a/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc
b/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc
index e64331610..841514261 100644
--- a/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc
+++ b/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc
@@ -258,8 +258,8 @@ TEST_F(Substrait2VeloxPlanConversionTest, ifthenTest) {
// Convert to Velox PlanNode.
auto planNode = planConverter_->toVeloxPlan(substraitPlan,
std::vector<::substrait::ReadRel_LocalFiles>{split});
ASSERT_EQ(
- "-- Project[expressions: ] -> \n "
- "-- TableScan[table: hive_table, range filters: [(hd_demo_sk,
Filter(IsNotNull, deterministic, null not allowed)),"
+ "-- Project[1][expressions: ] -> \n "
+ "-- TableScan[0][table: hive_table, range filters: [(hd_demo_sk,
Filter(IsNotNull, deterministic, null not allowed)),"
" (hd_vehicle_count, BigintRange: [1, 9223372036854775807] no nulls)],
remaining filter: "
"(and(or(equalto(\"hd_buy_potential\",\">10000\"),equalto(\"hd_buy_potential\",\"unknown\")),"
"if(greaterthan(\"hd_vehicle_count\",0),greaterthan(divide(cast
\"hd_dep_count\" as DOUBLE,"
@@ -279,7 +279,7 @@ TEST_F(Substrait2VeloxPlanConversionTest, filterUpper) {
// Convert to Velox PlanNode.
auto planNode = planConverter_->toVeloxPlan(substraitPlan,
std::vector<::substrait::ReadRel_LocalFiles>{split});
ASSERT_EQ(
- "-- Project[expressions: ] -> \n -- TableScan[table: hive_table, range
filters: "
+ "-- Project[1][expressions: ] -> \n -- TableScan[0][table: hive_table,
range filters: "
"[(key, BigintRange: [-2147483648, 2] no nulls)]] -> n0_0:INTEGER\n",
planNode->toString(true, true));
}
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 5ef78bb32..a3013dd49 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
set -exu
VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2024_05_08
+VELOX_BRANCH=2024_05_09
VELOX_HOME=""
#Set on run gluten on HDFS
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
