This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 69b3908f4 [VL][CI] Add -ntp maven option to reduce verbose download
log (#5156)
69b3908f4 is described below
commit 69b3908f490e1f7685754b11e335e58bbb34ce9a
Author: Joey <[email protected]>
AuthorDate: Wed Mar 27 21:45:05 2024 +0800
[VL][CI] Add -ntp maven option to reduce verbose download log (#5156)
---
.github/workflows/velox_docker.yml | 38 +++++++++++++++++++-------------------
1 file changed, 19 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/velox_docker.yml
b/.github/workflows/velox_docker.yml
index 1ed7794aa..2dd44ad60 100644
--- a/.github/workflows/velox_docker.yml
+++ b/.github/workflows/velox_docker.yml
@@ -106,9 +106,9 @@ jobs:
cd $GITHUB_WORKSPACE/
export JAVA_HOME=/usr/lib/jvm/${{ matrix.java }}-openjdk-amd64
echo "JAVA_HOME: $JAVA_HOME"
- mvn clean install -P${{ matrix.spark }} -P${{ matrix.java }}
-Pbackends-velox -DskipTests
+ mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }}
-Pbackends-velox -DskipTests
cd $GITHUB_WORKSPACE/tools/gluten-it
- mvn clean install -P${{ matrix.spark }} -P${{ matrix.java }} \
+ mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }} \
&& GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
--local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
&& GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
@@ -169,9 +169,9 @@ jobs:
run: |
echo "JAVA_HOME: $JAVA_HOME"
cd $GITHUB_WORKSPACE/
- mvn clean install -P${{ matrix.spark }} -P${{ matrix.java }}
-Pbackends-velox -DskipTests
+ mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }}
-Pbackends-velox -DskipTests
cd $GITHUB_WORKSPACE/tools/gluten-it
- mvn clean install -P${{ matrix.spark }} -P${{ matrix.java }}
+ mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }}
- name: Run TPC-H / TPC-DS
run: |
echo "JAVA_HOME: $JAVA_HOME"
@@ -221,13 +221,13 @@ jobs:
# cd $GITHUB_WORKSPACE/
# export MAVEN_HOME=/usr/lib/maven
# export PATH=${PATH}:${MAVEN_HOME}/bin
- # mvn clean install -P${{ matrix.spark }} -Pbackends-velox
-DskipTests
+ # mvn -ntp clean install -P${{ matrix.spark }} -Pbackends-velox
-DskipTests
# - name: TPC-DS SF30.0 Parquet local spark3.2 Q67/Q95 low memory,
memory isolation off
# run: |
# export MAVEN_HOME=/usr/lib/maven
# export PATH=${PATH}:${MAVEN_HOME}/bin
# cd tools/gluten-it && \
- # mvn clean install -Pspark-3.2 \
+ # mvn -ntp clean install -Pspark-3.2 \
# && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh parameterized \
# --local --preset=velox --benchmark-type=ds --error-on-memleak
--queries=q67,q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
# --skip-data-gen -m=OffHeapExecutionMemory \
@@ -241,7 +241,7 @@ jobs:
# export MAVEN_HOME=/usr/lib/maven
# export PATH=${PATH}:${MAVEN_HOME}/bin
# cd tools/gluten-it && \
- # mvn clean install -Pspark-3.2 \
+ # mvn -ntp clean install -Pspark-3.2 \
# && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh parameterized \
# --local --preset=velox --benchmark-type=ds --error-on-memleak
--queries=q67,q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
# --skip-data-gen -m=OffHeapExecutionMemory \
@@ -303,7 +303,7 @@ jobs:
# cd $GITHUB_WORKSPACE/
# export MAVEN_HOME=/usr/lib/maven
# export PATH=${PATH}:${MAVEN_HOME}/bin
- # mvn clean install -P${{ matrix.spark }} -Pbackends-velox
-DskipTests
+ # mvn -ntp clean install -P${{ matrix.spark }} -Pbackends-velox
-DskipTests
# - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with
Celeborn 0.4.0
# run: |
# export MAVEN_HOME=/usr/lib/maven
@@ -315,7 +315,7 @@ jobs:
# echo -e
"CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g"
> ./conf/celeborn-env.sh && \
# echo -e "celeborn.worker.commitFiles.threads
128\nceleborn.worker.sortPartition.threads 64" > ./conf/celeborn-defaults.conf \
# && bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \
- # cd $GITHUB_WORKSPACE/tools/gluten-it && mvn clean install
-Pspark-3.2,rss,celeborn-0.4 \
+ # cd $GITHUB_WORKSPACE/tools/gluten-it && mvn -ntp clean install
-Pspark-3.2,rss,celeborn-0.4 \
# && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
# --local --preset=velox-with-celeborn --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
# && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
@@ -333,7 +333,7 @@ jobs:
# echo -e
"CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g"
> ./conf/celeborn-env.sh && \
# echo -e "celeborn.worker.commitFiles.threads
128\nceleborn.worker.sortPartition.threads 64" > ./conf/celeborn-defaults.conf \
# && bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \
- # cd $GITHUB_WORKSPACE/tools/gluten-it && mvn clean install
-Pspark-3.2,rss \
+ # cd $GITHUB_WORKSPACE/tools/gluten-it && mvn -ntp clean install
-Pspark-3.2,rss \
# && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
# --local --preset=velox-with-celeborn --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
# && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
@@ -382,8 +382,8 @@ jobs:
export SPARK_SCALA_VERSION=2.12
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
- mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/"
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags
&& \
- mvn test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None
-DtagsToInclude=io.glutenproject.tags.UDFTest
+ mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/"
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags
&& \
+ mvn -ntp test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None
-DtagsToInclude=io.glutenproject.tags.UDFTest
run-spark-test-spark32-slow:
runs-on: ubuntu-20.04
@@ -419,7 +419,7 @@ jobs:
cd $GITHUB_WORKSPACE/ && \
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
- mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss
-Piceberg -Pdelta
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
+ mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss
-Piceberg -Pdelta
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
run-spark-test-spark33:
runs-on: ubuntu-20.04
@@ -462,8 +462,8 @@ jobs:
export SPARK_SCALA_VERSION=2.12 && \
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
- mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Pspark-ut
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/"
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags
&& \
- mvn test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None
-DtagsToInclude=io.glutenproject.tags.UDFTest
+ mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Prss -Pspark-ut
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/"
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags
&& \
+ mvn -ntp test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None
-DtagsToInclude=io.glutenproject.tags.UDFTest
run-spark-test-spark33-slow:
runs-on: ubuntu-20.04
@@ -499,7 +499,7 @@ jobs:
cd $GITHUB_WORKSPACE/ && \
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
- mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg
-Pdelta -Pspark-ut
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
+ mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg
-Pdelta -Pspark-ut
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
@@ -544,8 +544,8 @@ jobs:
export SPARK_SCALA_VERSION=2.12 && \
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
- mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Pspark-ut
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/"
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags
&& \
- mvn test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None
-DtagsToInclude=io.glutenproject.tags.UDFTest
+ mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Prss
-Pspark-ut
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/"
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags
&& \
+ mvn -ntp test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None
-DtagsToInclude=io.glutenproject.tags.UDFTest
run-spark-test-spark34-slow:
@@ -582,4 +582,4 @@ jobs:
cd $GITHUB_WORKSPACE/
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
- mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg
-Pdelta -Pspark-ut
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
\ No newline at end of file
+ mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg
-Pdelta -Pspark-ut
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
