This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 1acb56f build: Support built with java 1.8 (#45)
1acb56f is described below
commit 1acb56fa9a6cb9e82bf5188166f30b4b54593f99
Author: advancedxy <xian...@apache.org>
AuthorDate: Thu Feb 22 02:15:32 2024 +0800
build: Support built with java 1.8 (#45)
Co-authored-by: Liang-Chi Hsieh <vii...@gmail.com>
---
.github/actions/java-test/action.yaml | 5 +-
.github/actions/rust-test/action.yaml | 11 +-
.github/actions/setup-builder/action.yaml | 4 +-
.github/actions/setup-macos-builder/action.yaml | 4 +-
.github/workflows/pr_build.yml | 118 ++++++++++-----------
common/pom.xml | 2 +-
.../java/org/apache/comet/parquet/FileReader.java | 4 +
pom.xml | 4 +-
.../scala/org/apache/spark/sql/GenTPCHData.scala | 2 +-
9 files changed, 81 insertions(+), 73 deletions(-)
diff --git a/.github/actions/java-test/action.yaml
b/.github/actions/java-test/action.yaml
index 6c3af79..9c7de9a 100644
--- a/.github/actions/java-test/action.yaml
+++ b/.github/actions/java-test/action.yaml
@@ -37,9 +37,10 @@ runs:
- name: Run Maven compile
shell: bash
run: |
- ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb
+ echo "JAVA_VERSION=${JAVA_VERSION}"
+ ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb
-Djava.version=${JAVA_VERSION}
- name: Run tests
shell: bash
run: |
- SPARK_HOME=`pwd` ./mvnw -B clean install
+ SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION}
diff --git a/.github/actions/rust-test/action.yaml
b/.github/actions/rust-test/action.yaml
index b66b639..a013a7b 100644
--- a/.github/actions/rust-test/action.yaml
+++ b/.github/actions/rust-test/action.yaml
@@ -51,15 +51,20 @@ runs:
shell: bash
run: |
cd common
- ../mvnw -B clean compile -DskipTests
+ ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION}
- name: Run Cargo test
shell: bash
run: |
cd core
# This is required to run some JNI related tests on the Rust side
+
JAVA_LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$JAVA_HOME/lib:$JAVA_HOME/lib/jli
+ # special handing for java 1.8 for both linux and mac distributions
+ if [ $JAVA_VERSION == "8" ]; then
+
JAVA_LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64:$JAVA_HOME/jre/lib/amd64/jli:$JAVA_HOME/jre/lib/server:$JAVA_HOME/jre/lib:$JAVA_HOME/jre/lib/jli
+ fi
RUST_BACKTRACE=1 \
-
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server:$JAVA_HOME/lib/jli
\
-
DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server:$JAVA_HOME/lib/jli
\
+ LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \
+ DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \
cargo test
diff --git a/.github/actions/setup-builder/action.yaml
b/.github/actions/setup-builder/action.yaml
index f1aeb25..6fa7164 100644
--- a/.github/actions/setup-builder/action.yaml
+++ b/.github/actions/setup-builder/action.yaml
@@ -38,7 +38,9 @@ runs:
- name: Install JDK ${{inputs.jdk-version}}
uses: actions/setup-java@v4
with:
- distribution: 'adopt'
+ # distribution is chosen to be zulu as it still offers JDK 8 with
Silicon support, which
+ # is not available in the adopt distribution
+ distribution: 'zulu'
java-version: ${{inputs.jdk-version}}
- name: Set JAVA_HOME
diff --git a/.github/actions/setup-macos-builder/action.yaml
b/.github/actions/setup-macos-builder/action.yaml
index 1f680ed..63010ea 100644
--- a/.github/actions/setup-macos-builder/action.yaml
+++ b/.github/actions/setup-macos-builder/action.yaml
@@ -59,7 +59,9 @@ runs:
- name: Install JDK ${{inputs.jdk-version}}
uses: actions/setup-java@v4
with:
- distribution: 'adopt'
+ # distribution is chosen to be zulu as it still offers JDK 8 with
Silicon support, which
+ # is not available in the adopt distribution
+ distribution: 'zulu'
java-version: ${{inputs.jdk-version}}
architecture: ${{inputs.jdk-architecture}}
diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml
index 51962cd..55117d3 100644
--- a/.github/workflows/pr_build.yml
+++ b/.github/workflows/pr_build.yml
@@ -35,105 +35,99 @@ on:
workflow_dispatch:
env:
- JAVA_VERSION: 17
RUST_VERSION: nightly
jobs:
- linux-rust-test:
- name: Rust test (amd64)
- runs-on: ubuntu-latest
+ linux-test:
+ strategy:
+ matrix:
+ os: [ubuntu-latest]
+ java_version: [8, 11, 17]
+ test-target: [rust, java]
+ fail-fast: false
+ name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{
matrix.java_version }}
+ runs-on: ${{ matrix.os }}
container:
image: amd64/rust
+ env:
+ JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}',
matrix.java_version) }}
steps:
- uses: actions/checkout@v4
- name: Setup Rust & Java toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{env.JAVA_VERSION}}
+ jdk-version: ${{ matrix.java_version }}
- uses: actions/checkout@v4
- - name: Rust test steps
+ - if: matrix.test-target == 'rust'
+ name: Rust test steps
uses: ./.github/actions/rust-test
-
- linux-java-test:
- name: Java test (amd64)
- runs-on: ubuntu-latest
- container:
- image: amd64/rust
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{env.JAVA_VERSION}}
-
- - uses: actions/checkout@v4
- - name: Java test steps
+ - if: matrix.test-target == 'java'
+ name: Java test steps
uses: ./.github/actions/java-test
-
- macos-rust-test:
- name: Rust test (macos)
- runs-on: macos-latest
+ macos-test:
+ strategy:
+ matrix:
+ os: [macos-latest]
+ java_version: [8, 11, 17]
+ test-target: [rust, java]
+ is_push_event:
+ - ${{ github.event_name == 'push' }}
+ exclude: # exclude java 11 for pull_request event
+ - java_version: 11
+ is_push_event: false
+ fail-fast: false
+ name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{
matrix.java_version }}
+ runs-on: ${{ matrix.os }}
+ env:
+ JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}',
matrix.java_version) }}
steps:
- uses: actions/checkout@v4
- name: Setup Rust & Java toolchain
uses: ./.github/actions/setup-macos-builder
with:
rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{env.JAVA_VERSION}}
+ jdk-version: ${{ matrix.java_version }}
- uses: actions/checkout@v4
- - name: Rust test steps
+ - if: matrix.test-target == 'rust'
+ name: Rust test steps
uses: ./.github/actions/rust-test
-
- macos-java-test:
- name: Java test (macos)
- runs-on: macos-latest
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-macos-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{env.JAVA_VERSION}}
-
- - uses: actions/checkout@v4
- - name: Java test steps
+ - if: matrix.test-target == 'java'
+ name: Java test steps
uses: ./.github/actions/java-test
- macos-aarch64-rust-test:
- name: Rust test (macos-aarch64)
+ macos-aarch64-test:
+ strategy:
+ matrix:
+ java_version: [8, 11, 17]
+ test-target: [rust, java]
+ is_push_event:
+ - ${{ github.event_name == 'push' }}
+ exclude: # exclude java 11 for pull_request event
+ - java_version: 11
+ is_push_event: false
+ fail-fast: false
+ name: ${{ matrix.test-target }} test on macos-aarch64 with java ${{
matrix.java_version }}
runs-on: macos-14
+ env:
+ JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}',
matrix.java_version) }}
steps:
- uses: actions/checkout@v4
- name: Setup Rust & Java toolchain
uses: ./.github/actions/setup-macos-builder
with:
rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{env.JAVA_VERSION}}
+ jdk-version: ${{ matrix.java_version }}
jdk-architecture: aarch64
protoc-architecture: aarch_64
- uses: actions/checkout@v4
- - name: Rust test steps
+ - if: matrix.test-target == 'rust'
+ name: Rust test steps
uses: ./.github/actions/rust-test
-
- macos-aarch64-java-test:
- name: Java test (macos-aarch64)
- runs-on: macos-14
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust & Java toolchain
- uses: ./.github/actions/setup-macos-builder
- with:
- rust-version: ${{env.RUST_VERSION}}
- jdk-version: ${{env.JAVA_VERSION}}
- jdk-architecture: aarch64
- protoc-architecture: aarch_64
-
- - uses: actions/checkout@v4
- - name: Java test steps
+ - if: matrix.test-target == 'java'
+ name: Java test steps
uses: ./.github/actions/java-test
diff --git a/common/pom.xml b/common/pom.xml
index f885c34..5757351 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -80,7 +80,7 @@ under the License.
<plugin>
<groupId>io.github.git-commit-id</groupId>
<artifactId>git-commit-id-maven-plugin</artifactId>
- <version>5.0.0</version>
+ <version>4.9.9</version>
<executions>
<execution>
<id>get-the-git-infos</id>
diff --git a/common/src/main/java/org/apache/comet/parquet/FileReader.java
b/common/src/main/java/org/apache/comet/parquet/FileReader.java
index eddaf3f..61cf89b 100644
--- a/common/src/main/java/org/apache/comet/parquet/FileReader.java
+++ b/common/src/main/java/org/apache/comet/parquet/FileReader.java
@@ -882,6 +882,8 @@ public class FileReader implements Closeable {
private final SQLMetric readThroughput;
/**
+ * Constructor
+ *
* @param offset where the first chunk starts
*/
ConsecutivePartList(long offset) {
@@ -1104,6 +1106,8 @@ public class FileReader implements Closeable {
}
/**
+ * End position of the last byte of these chunks
+ *
* @return the position following the last byte of these chunks
*/
public long endPos() {
diff --git a/pom.xml b/pom.xml
index 0e72f92..b3c3545 100644
--- a/pom.xml
+++ b/pom.xml
@@ -39,8 +39,8 @@ under the License.
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>11</java.version>
- <maven.compiler.source>11</maven.compiler.source>
- <maven.compiler.target>11</maven.compiler.target>
+ <maven.compiler.source>${java.version}</maven.compiler.source>
+ <maven.compiler.target>${java.version}</maven.compiler.target>
<scala.version>2.12.17</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<scala.plugin.version>4.7.2</scala.plugin.version>
diff --git a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala
b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala
index 1dcbc74..e25d4e5 100644
--- a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala
@@ -76,7 +76,7 @@ object GenTPCHData {
// Generate data
// Since dbgen may uses stdout to output the data, tables.genData needs to
run table by table
val tableNames =
- if (config.tableFilter.isBlank) tables.tables.map(_.name) else
Seq(config.tableFilter)
+ if (config.tableFilter.trim.isEmpty) tables.tables.map(_.name) else
Seq(config.tableFilter)
tableNames.foreach { tableName =>
tables.genData(
location =
s"${config.location}/tpch/sf${config.scaleFactor}_${config.format}",
