This is an automated email from the ASF dual-hosted git repository.
comphead pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new d9a1d4261f [CI] Refactor CI builders (#8826)
d9a1d4261f is described below
commit d9a1d4261f78dfd3ead235d6b850c64e0ae9bec6
Author: comphead <[email protected]>
AuthorDate: Thu Jan 11 13:16:31 2024 -0800
[CI] Refactor CI builders (#8826)
[CI] refactor CI builders
* Update .github/actions/setup-macos-builder/action.yaml
Co-authored-by: Andrew Lamb <[email protected]>
* Update .github/actions/setup-windows-builder/action.yaml
Co-authored-by: Andrew Lamb <[email protected]>
---------
Co-authored-by: Andrew Lamb <[email protected]>
---
.github/actions/setup-builder/action.yaml | 15 +----
.github/actions/setup-macos-builder/action.yaml | 47 +++++++++++++++
.github/actions/setup-rust-runtime/action.yaml | 41 +++++++++++++
.github/actions/setup-windows-builder/action.yaml | 46 +++++++++++++++
.github/workflows/rust.yml | 70 +++--------------------
5 files changed, 143 insertions(+), 76 deletions(-)
diff --git a/.github/actions/setup-builder/action.yaml
b/.github/actions/setup-builder/action.yaml
index 5a93f6f27b..5578517ec3 100644
--- a/.github/actions/setup-builder/action.yaml
+++ b/.github/actions/setup-builder/action.yaml
@@ -38,19 +38,8 @@ runs:
rustup toolchain install ${{ inputs.rust-version }}
rustup default ${{ inputs.rust-version }}
rustup component add rustfmt
- - name: Disable debuginfo generation
- # Disable full debug symbol generation to speed up CI build and keep
memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- shell: bash
- run: echo "RUSTFLAGS=-C debuginfo=1" >> $GITHUB_ENV
- - name: Disable incremental compilation
- # Disable incremental compilation to save diskspace (the CI doesn't
recompile modified files)
- # https://github.com/apache/arrow-datafusion/issues/6676
- shell: bash
- run: echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
- - name: Enable backtraces
- shell: bash
- run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
+ - name: Configure rust runtime env
+ uses: ./.github/actions/setup-rust-runtime
- name: Fixup git permissions
# https://github.com/actions/checkout/issues/766
shell: bash
diff --git a/.github/actions/setup-macos-builder/action.yaml
b/.github/actions/setup-macos-builder/action.yaml
new file mode 100644
index 0000000000..02419f6179
--- /dev/null
+++ b/.github/actions/setup-macos-builder/action.yaml
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder for MacOS
+description: 'Prepare Rust Build Environment for MacOS'
+inputs:
+ rust-version:
+ description: 'version of rust to install (e.g. stable)'
+ required: true
+ default: 'stable'
+runs:
+ using: "composite"
+ steps:
+ - name: Install protobuf compiler
+ shell: bash
+ run: |
+ mkdir -p $HOME/d/protoc
+ cd $HOME/d/protoc
+ export PROTO_ZIP="protoc-21.4-osx-x86_64.zip"
+ curl -LO
https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+ unzip $PROTO_ZIP
+ echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
+ export PATH=$PATH:$HOME/d/protoc/bin
+ protoc --version
+ - name: Setup Rust toolchain
+ shell: bash
+ run: |
+ rustup update stable
+ rustup toolchain install stable
+ rustup default stable
+ rustup component add rustfmt
+ - name: Configure rust runtime env
+ uses: ./.github/actions/setup-rust-runtime
diff --git a/.github/actions/setup-rust-runtime/action.yaml
b/.github/actions/setup-rust-runtime/action.yaml
new file mode 100644
index 0000000000..90e09a957c
--- /dev/null
+++ b/.github/actions/setup-rust-runtime/action.yaml
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Setup Rust Runtime
+description: 'Setup Rust Runtime Environment'
+runs:
+ using: "composite"
+ steps:
+ - name: Run sccache-cache
+ uses: mozilla-actions/[email protected]
+ - name: Configure runtime env
+ shell: bash
+ # do not produce debug symbols to keep memory usage down
+ # hardcoding other profile params to avoid profile override values
+ # More on Cargo profiles
https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
+ #
+ # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow
build
+ # See for more details: https://github.com/rust-lang/rust/issues/119560
+ #
+ # set RUST_MIN_STACK to avoid rust stack overflows on tpc-ds tests
+ run: |
+ echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+ echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+ echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
+ echo "RUST_MIN_STACK=3000000" >> $GITHUB_ENV
+ echo "RUST_FLAGS=-C debuginfo=line-tables-only -C incremental=false"
>> $GITHUB_ENV
+
diff --git a/.github/actions/setup-windows-builder/action.yaml
b/.github/actions/setup-windows-builder/action.yaml
new file mode 100644
index 0000000000..9ab5c4a8b1
--- /dev/null
+++ b/.github/actions/setup-windows-builder/action.yaml
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder for Windows
+description: 'Prepare Rust Build Environment for Windows'
+inputs:
+ rust-version:
+ description: 'version of rust to install (e.g. stable)'
+ required: true
+ default: 'stable'
+runs:
+ using: "composite"
+ steps:
+ - name: Install protobuf compiler
+ shell: bash
+ run: |
+ mkdir -p $HOME/d/protoc
+ cd $HOME/d/protoc
+ export PROTO_ZIP="protoc-21.4-win64.zip"
+ curl -LO
https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+ unzip $PROTO_ZIP
+ export PATH=$PATH:$HOME/d/protoc/bin
+ protoc.exe --version
+ - name: Setup Rust toolchain
+ shell: bash
+ run: |
+ rustup update stable
+ rustup toolchain install stable
+ rustup default stable
+ rustup component add rustfmt
+ - name: Configure rust runtime env
+ uses: ./.github/actions/setup-rust-runtime
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index ae6c1ee561..62992e7acf 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -96,17 +96,9 @@ jobs:
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
- rust-version: stable
+ rust-version: stable
- name: Run tests (excluding doctests)
run: cargo test --lib --tests --bins --features avro,json,backtrace
- env:
- # do not produce debug symbols to keep memory usage down
- # hardcoding other profile params to avoid profile override values
- # More on Cargo profiles
https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
- RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C
codegen-units=256"
- RUST_BACKTRACE: "1"
- # avoid rust stack overflows on tpc-ds tests
- RUST_MIN_STACK: "3000000"
- name: Verify Working Directory Clean
run: git diff --exit-code
@@ -284,24 +276,8 @@ jobs:
- uses: actions/checkout@v4
with:
submodules: true
- - name: Install protobuf compiler
- shell: bash
- run: |
- mkdir -p $HOME/d/protoc
- cd $HOME/d/protoc
- export PROTO_ZIP="protoc-21.4-win64.zip"
- curl -LO
https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
- unzip $PROTO_ZIP
- export PATH=$PATH:$HOME/d/protoc/bin
- protoc.exe --version
- # TODO: this won't cache anything, which is expensive. Setup this action
- # with a OS-dependent path.
- name: Setup Rust toolchain
- run: |
- rustup update stable
- rustup toolchain install stable
- rustup default stable
- rustup component add rustfmt
+ uses: ./.github/actions/setup-windows-builder
- name: Run tests (excluding doctests)
shell: bash
run: |
@@ -309,54 +285,22 @@ jobs:
cargo test --lib --tests --bins --features avro,json,backtrace
cd datafusion-cli
cargo test --lib --tests --bins --all-features
- env:
- # Minimize producing debug symbols to keep memory usage down
- # Set debuginfo=line-tables-only as debuginfo=0 causes immensely
slow build
- # See for more details:
https://github.com/rust-lang/rust/issues/119560
- RUSTFLAGS: "-C debuginfo=line-tables-only"
- RUST_BACKTRACE: "1"
- # avoid rust stack overflows on tpc-ds tests
- RUST_MIN_STACK: "3000000"
+
macos:
- name: cargo test (mac)
+ name: cargo test (macos)
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
with:
- submodules: true
- - name: Install protobuf compiler
- shell: bash
- run: |
- mkdir -p $HOME/d/protoc
- cd $HOME/d/protoc
- export PROTO_ZIP="protoc-21.4-osx-x86_64.zip"
- curl -LO
https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
- unzip $PROTO_ZIP
- echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
- export PATH=$PATH:$HOME/d/protoc/bin
- protoc --version
- # TODO: this won't cache anything, which is expensive. Setup this action
- # with a OS-dependent path.
+ submodules: true
- name: Setup Rust toolchain
- run: |
- rustup update stable
- rustup toolchain install stable
- rustup default stable
- rustup component add rustfmt
+ uses: ./.github/actions/setup-macos-builder
- name: Run tests (excluding doctests)
shell: bash
run: |
cargo test --lib --tests --bins --features avro,json,backtrace
cd datafusion-cli
- cargo test --lib --tests --bins --all-features
- env:
- # do not produce debug symbols to keep memory usage down
- # hardcoding other profile params to avoid profile override values
- # More on Cargo profiles
https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
- RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C
codegen-units=256"
- RUST_BACKTRACE: "1"
- # avoid rust stack overflows on tpc-ds tests
- RUST_MIN_STACK: "3000000"
+ cargo test --lib --tests --bins --all-features
test-datafusion-pyarrow:
name: cargo test pyarrow (amd64)
