PHILO-HE commented on code in PR #9955:
URL: https://github.com/apache/incubator-gluten/pull/9955#discussion_r2148987318
##########
.github/workflows/velox_weekly.yml:
##########
@@ -99,3 +100,86 @@ jobs:
sudo apt-get install -y openjdk-8-jdk
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+
+ build-on-openeuler:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ "openeuler/openeuler:24.03" ]
+ spark: [ "spark-3.5" ]
+ java: [ "java-8" ]
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ runs-on: ubuntu-24.04-arm
+ container: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: build
+ run: |
+ dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
wget git perl
+ $SETUP install_maven
+ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+ export MAVEN_HOME=/usr/lib/maven
+ export PATH=${MAVEN_HOME}/bin:${JAVA_HOME}/bin:${PATH}
+ cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+ cp package/target/thirdparty-lib/gluten-thirdparty-lib-*.jar /root/
+ - name: Set environment variables
+ run: |
+ echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
+ - name: Build gluten-it
+ run: |
+ echo "JAVA_HOME: $JAVA_HOME"
+ cd $GITHUB_WORKSPACE/
+ $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }}
-Pbackends-velox -DskipTests
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }}
+ - name: Run TPC-H
+ run: |
+ echo "JAVA_HOME: $JAVA_HOME"
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ cp /root/gluten-thirdparty-lib-*.jar package/target/lib/
+ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+ --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
+ --extra-conf=spark.gluten.loadLibFromJar=true \
+ --extra-conf=spark.gluten.loadLibOS=openEuler
--extra-conf=spark.gluten.loadLibOSVersion=24.03
+
+ build-on-openeuler-vcpkg:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ "openeuler/openeuler:24.03" ]
+ spark: [ "spark-3.5" ]
+ java: [ "java-8" ]
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ runs-on: ubuntu-24.04-arm
+ container: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: build
+ run: |
+ dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
wget git perl
+ $SETUP install_maven
+ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+ export MAVEN_HOME=/usr/lib/maven
+ export PATH=${MAVEN_HOME}/bin:${JAVA_HOME}/bin:${PATH}
+ export VCPKG_PATH=/var/cache/vcpkg
+ export VCPKG_BINARY_SOURCES="clear;files,${VCPKG_PATH},readwrite"
+ cd $GITHUB_WORKSPACE/
+ ./dev/vcpkg/setup-build-depends.sh
+ mkdir -p ${VCPKG_PATH}
+ sed -i 's/--build_arrow=OFF/--build_arrow=ON/' dev/package-vcpkg.sh
&& ./dev/package-vcpkg.sh
+ - name: Set environment variables
+ run: |
+ echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
Review Comment:
Nit: Let's move this setting to the above task for brevity. Seems we can
directly replace the above `export JAVA_HOME=` with this setting.
##########
.github/workflows/velox_weekly.yml:
##########
@@ -99,3 +100,86 @@ jobs:
sudo apt-get install -y openjdk-8-jdk
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+
+ build-on-openeuler:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ "openeuler/openeuler:24.03" ]
+ spark: [ "spark-3.5" ]
+ java: [ "java-8" ]
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ runs-on: ubuntu-24.04-arm
+ container: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: build
+ run: |
+ dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
wget git perl
+ $SETUP install_maven
+ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+ export MAVEN_HOME=/usr/lib/maven
+ export PATH=${MAVEN_HOME}/bin:${JAVA_HOME}/bin:${PATH}
+ cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+ cp package/target/thirdparty-lib/gluten-thirdparty-lib-*.jar /root/
+ - name: Set environment variables
+ run: |
+ echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
+ - name: Build gluten-it
+ run: |
+ echo "JAVA_HOME: $JAVA_HOME"
+ cd $GITHUB_WORKSPACE/
+ $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }}
-Pbackends-velox -DskipTests
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }}
+ - name: Run TPC-H
+ run: |
+ echo "JAVA_HOME: $JAVA_HOME"
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ cp /root/gluten-thirdparty-lib-*.jar package/target/lib/
+ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+ --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
+ --extra-conf=spark.gluten.loadLibFromJar=true \
+ --extra-conf=spark.gluten.loadLibOS=openEuler
--extra-conf=spark.gluten.loadLibOSVersion=24.03
+
+ build-on-openeuler-vcpkg:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ "openeuler/openeuler:24.03" ]
+ spark: [ "spark-3.5" ]
+ java: [ "java-8" ]
Review Comment:
ditto.
##########
.github/workflows/velox_weekly.yml:
##########
@@ -99,3 +100,86 @@ jobs:
sudo apt-get install -y openjdk-8-jdk
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+
+ build-on-openeuler:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ "openeuler/openeuler:24.03" ]
+ spark: [ "spark-3.5" ]
+ java: [ "java-8" ]
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ runs-on: ubuntu-24.04-arm
+ container: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: build
+ run: |
+ dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
wget git perl
+ $SETUP install_maven
+ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+ export MAVEN_HOME=/usr/lib/maven
+ export PATH=${MAVEN_HOME}/bin:${JAVA_HOME}/bin:${PATH}
+ cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+ cp package/target/thirdparty-lib/gluten-thirdparty-lib-*.jar /root/
+ - name: Set environment variables
+ run: |
+ echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
+ - name: Build gluten-it
+ run: |
+ echo "JAVA_HOME: $JAVA_HOME"
+ cd $GITHUB_WORKSPACE/
+ $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }}
-Pbackends-velox -DskipTests
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }}
+ - name: Run TPC-H
+ run: |
+ echo "JAVA_HOME: $JAVA_HOME"
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ cp /root/gluten-thirdparty-lib-*.jar package/target/lib/
+ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+ --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
+ --extra-conf=spark.gluten.loadLibFromJar=true \
+ --extra-conf=spark.gluten.loadLibOS=openEuler
--extra-conf=spark.gluten.loadLibOSVersion=24.03
+
+ build-on-openeuler-vcpkg:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ "openeuler/openeuler:24.03" ]
+ spark: [ "spark-3.5" ]
+ java: [ "java-8" ]
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ runs-on: ubuntu-24.04-arm
+ container: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: build
+ run: |
+ dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
wget git perl
+ $SETUP install_maven
+ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+ export MAVEN_HOME=/usr/lib/maven
+ export PATH=${MAVEN_HOME}/bin:${JAVA_HOME}/bin:${PATH}
Review Comment:
ditto
##########
.github/workflows/velox_weekly.yml:
##########
@@ -99,3 +100,86 @@ jobs:
sudo apt-get install -y openjdk-8-jdk
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+
+ build-on-openeuler:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ "openeuler/openeuler:24.03" ]
+ spark: [ "spark-3.5" ]
+ java: [ "java-8" ]
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ runs-on: ubuntu-24.04-arm
+ container: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: build
+ run: |
+ dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
wget git perl
+ $SETUP install_maven
+ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+ export MAVEN_HOME=/usr/lib/maven
+ export PATH=${MAVEN_HOME}/bin:${JAVA_HOME}/bin:${PATH}
Review Comment:
Remove setting MAVEN_HOME and PATH.
`$SETUP install_maven` have set it for maven, and setting PATH with
`${JAVA_HOME}/bin` is also unnecessary maybe.
##########
backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala:
##########
@@ -124,7 +126,7 @@ object SharedLibraryLoader {
} else {
throw new GlutenException(
s"Found unsupported OS($systemName, $systemVersion)! Currently,
Gluten's Velox backend" +
- " only supports Ubuntu 20.04/22.04, CentOS 7/8, Oracle 7/8" +
+ " only supports Ubuntu 20.04/22.04, CentOS 7/8, Oracle 7/8,
openEuler 24.03 " +
Review Comment:
Nit: putting a comma next to "openEuler 24.03".
##########
.github/workflows/velox_weekly.yml:
##########
@@ -99,3 +100,86 @@ jobs:
sudo apt-get install -y openjdk-8-jdk
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+
+ build-on-openeuler:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ "openeuler/openeuler:24.03" ]
+ spark: [ "spark-3.5" ]
+ java: [ "java-8" ]
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ runs-on: ubuntu-24.04-arm
+ container: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: build
+ run: |
+ dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
wget git perl
+ $SETUP install_maven
+ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+ export MAVEN_HOME=/usr/lib/maven
+ export PATH=${MAVEN_HOME}/bin:${JAVA_HOME}/bin:${PATH}
+ cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+ cp package/target/thirdparty-lib/gluten-thirdparty-lib-*.jar /root/
+ - name: Set environment variables
+ run: |
+ echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
Review Comment:
ditto.
##########
.github/workflows/velox_weekly.yml:
##########
@@ -99,3 +100,86 @@ jobs:
sudo apt-get install -y openjdk-8-jdk
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+
+ build-on-openeuler:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ "openeuler/openeuler:24.03" ]
+ spark: [ "spark-3.5" ]
+ java: [ "java-8" ]
Review Comment:
Remove unnecessary matrix setting. Directly use the values in the below.
What we added are basic tests to verify native libs. So I think we will not
extend the tests for other JDKs or Spark versions in the future.
##########
.github/workflows/velox_weekly.yml:
##########
@@ -99,3 +100,86 @@ jobs:
sudo apt-get install -y openjdk-8-jdk
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+
+ build-on-openeuler:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ "openeuler/openeuler:24.03" ]
+ spark: [ "spark-3.5" ]
+ java: [ "java-8" ]
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ runs-on: ubuntu-24.04-arm
+ container: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: build
+ run: |
+ dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
wget git perl
+ $SETUP install_maven
+ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+ export MAVEN_HOME=/usr/lib/maven
+ export PATH=${MAVEN_HOME}/bin:${JAVA_HOME}/bin:${PATH}
+ cd $GITHUB_WORKSPACE/ && ./dev/package.sh
+ cp package/target/thirdparty-lib/gluten-thirdparty-lib-*.jar /root/
+ - name: Set environment variables
+ run: |
+ echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
+ - name: Build gluten-it
+ run: |
+ echo "JAVA_HOME: $JAVA_HOME"
+ cd $GITHUB_WORKSPACE/
+ $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }}
-Pbackends-velox -DskipTests
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }}
+ - name: Run TPC-H
+ run: |
+ echo "JAVA_HOME: $JAVA_HOME"
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ cp /root/gluten-thirdparty-lib-*.jar package/target/lib/
+ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+ --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
+ --extra-conf=spark.gluten.loadLibFromJar=true \
+ --extra-conf=spark.gluten.loadLibOS=openEuler
--extra-conf=spark.gluten.loadLibOSVersion=24.03
+
+ build-on-openeuler-vcpkg:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ "openeuler/openeuler:24.03" ]
+ spark: [ "spark-3.5" ]
+ java: [ "java-8" ]
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ runs-on: ubuntu-24.04-arm
+ container: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: build
+ run: |
+ dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
wget git perl
+ $SETUP install_maven
+ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+ export MAVEN_HOME=/usr/lib/maven
+ export PATH=${MAVEN_HOME}/bin:${JAVA_HOME}/bin:${PATH}
+ export VCPKG_PATH=/var/cache/vcpkg
+ export VCPKG_BINARY_SOURCES="clear;files,${VCPKG_PATH},readwrite"
+ cd $GITHUB_WORKSPACE/
+ ./dev/vcpkg/setup-build-depends.sh
+ mkdir -p ${VCPKG_PATH}
+ sed -i 's/--build_arrow=OFF/--build_arrow=ON/' dev/package-vcpkg.sh
&& ./dev/package-vcpkg.sh
Review Comment:
Can we just use `./dev/package-vcpkg.sh --build_arrow=ON`? It may override
build arrow setting.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
