This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new d2bc07d Fix CI (#10)
d2bc07d is described below
commit d2bc07d35b5f83a29b0b75da72c0eea99a83154a
Author: Andy Grove <[email protected]>
AuthorDate: Sun Apr 18 20:04:40 2021 -0600
Fix CI (#10)
* Fixed CI.
* More cleanup.
* More cleanups.
* More cleanups.
* Bumped arrow to latest.
* Always test.
* trigger CI
* Remove arrow-testing submodule
* Add arrow-testing submodule
* fix env var for AMD64 build
* delete travis ci yaml
Co-authored-by: Jorge C. Leitao <[email protected]>
---
.github/workflows/archery.yml | 64 ------
.github/workflows/cancel.yml | 77 -------
.github/workflows/cpp.yml | 395 --------------------------------
.github/workflows/cpp_cron.yml | 149 ------------
.github/workflows/csharp.yml | 121 ----------
.github/workflows/dev.yml | 63 +----
.github/workflows/dev_pr/labeler.yml | 52 -----
.github/workflows/go.yml | 125 ----------
.github/workflows/integration.yml | 83 -------
.github/workflows/java.yml | 112 ---------
.github/workflows/java_jni.yml | 83 -------
.github/workflows/js.yml | 122 ----------
.github/workflows/julia.yml | 53 -----
.github/workflows/python.yml | 154 -------------
.github/workflows/python_cron.yml | 141 ------------
.github/workflows/r.yml | 255 ---------------------
.github/workflows/ruby.yml | 290 -----------------------
.github/workflows/rust.yml | 228 ++----------------
.gitignore | 6 +
.gitmodules | 6 +-
.travis.yml | 165 -------------
ballista/rust/benchmarks/tpch/README.md | 6 +-
dev/.gitignore | 1 +
dev/archery/archery/cli.py | 5 +-
dev/archery/archery/utils/source.py | 8 +-
dev/release/rat_exclude_files.txt | 111 +--------
arrow-testing => testing | 0
27 files changed, 48 insertions(+), 2827 deletions(-)
diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml
deleted file mode 100644
index 761e045..0000000
--- a/.github/workflows/archery.yml
+++ /dev/null
@@ -1,64 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Archery & Crossbow
-
-on:
- push:
- paths:
- - '.github/workflows/archery.yml'
- - 'dev/archery/**'
- - 'dev/tasks/**'
- - 'docker-compose.yml'
- pull_request:
- paths:
- - '.github/workflows/archery.yml'
- - 'dev/archery/**'
- - 'dev/tasks/**'
- - 'docker-compose.yml'
-
-jobs:
-
- test:
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- name: Archery Unittests and Crossbow Check Config
- runs-on: ubuntu-latest
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Git Fixup
- if: ${{ github.event_name == 'pull_request' }}
- shell: bash
- run: git branch master origin/master
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: '3.6'
- - name: Install Archery, Crossbow- and Test Dependencies
- run: pip install pytest responses -e dev/archery[all]
- - name: Archery Unittests
- working-directory: dev/archery
- run: pytest -v archery
- - name: Archery Docker Validation
- run: archery docker
- - name: Crossbow Check Config
- working-directory: dev/tasks
- run: archery crossbow check-config
diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml
index de980eb..e1c6ed9 100644
--- a/.github/workflows/cancel.yml
+++ b/.github/workflows/cancel.yml
@@ -31,20 +31,6 @@ jobs:
# Unfortunately, we need to define a separate cancellation step for
# each workflow where we want to cancel stale runs.
- uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale C++ runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: cpp.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale C# runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: csharp.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
name: "Cancel stale Dev runs"
with:
cancelMode: allDuplicates
@@ -52,69 +38,6 @@ jobs:
workflowFileName: dev.yml
skipEventTypes: '["push", "schedule"]'
- uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Go runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: go.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Integration runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: integration.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Java JNI runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: java_jni.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Java runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: java.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale JS runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: js.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Julia runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: julia.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Python runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: python.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale R runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: r.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Ruby runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: ruby.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
name: "Cancel stale Rust runs"
with:
cancelMode: allDuplicates
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
deleted file mode 100644
index 0bcf346..0000000
--- a/.github/workflows/cpp.yml
+++ /dev/null
@@ -1,395 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: C++
-
-on:
- push:
- paths:
- - '.github/workflows/cpp.yml'
- - 'ci/docker/**'
- - 'ci/scripts/cpp_*'
- - 'ci/scripts/msys2_*'
- - 'ci/scripts/util_*'
- - 'cpp/**'
- - 'format/Flight.proto'
- pull_request:
- paths:
- - '.github/workflows/cpp.yml'
- - 'ci/docker/**'
- - 'ci/scripts/cpp_*'
- - 'ci/scripts/msys2_*'
- - 'ci/scripts/util_*'
- - 'cpp/**'
- - 'format/Flight.proto'
-
-env:
- ARROW_ENABLE_TIMING_TESTS: OFF
- DOCKER_VOLUME_PREFIX: ".docker/"
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- docker:
- name: ${{ matrix.title }}
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- image:
- - conda-cpp
- - ubuntu-cpp-sanitizer
- include:
- - image: conda-cpp
- title: AMD64 Conda C++
- - image: ubuntu-cpp-sanitizer
- title: AMD64 Ubuntu 20.04 C++ ASAN UBSAN
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v2
- with:
- path: .docker
- key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }}
- restore-keys: ${{ matrix.image }}-
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: |
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
- ulimit -c unlimited
- archery docker run ${{ matrix.image }}
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push ${{ matrix.image }}
-
- docker-arm:
- # NOTE: this job is specific for self-hosted runners
- # CACHING: don't use the cache plugin because of various permission
- # issues and keep the cached docker volumes permanently on the
- # host
- # PYTHON: no distributions are built for arm machines by the github
- # actions team, so python>3.6 must be preinstalled on the self
- # hosted machines
- name: ${{ matrix.title }}
- runs-on: ${{ matrix.runner }}
- # TODO(kszucs): re-enable once the self-hosted workers are properly
- # registered to github
- if: false && github.event_name == 'push'
- defaults:
- # To use certain environment variables set by .bashrc, an interactive
- # bash shell must be used
- run:
- shell: bash -i {0}
- strategy:
- fail-fast: false
- matrix:
- name:
- - arm32v7-debian-10-cpp
- - arm64v8-ubuntu-20.04-cpp
- include:
- - name: arm32v7-debian-10-cpp
- debian: 10
- title: ARM32v7 Debian 10 C++
- image: |
- -e CPP_MAKE_PARALLELISM=2 \
- -e CXXFLAGS=-Wno-psabi \
- -e ARROW_PARQUET=OFF \
- -e ARROW_FLIGHT=OFF \
- -e ARROW_GANDIVA=OFF \
- -e ARROW_ORC=OFF \
- -e CMAKE_ARGS=-DARROW_CPU_FLAG=armv7 \
- debian-cpp
- arch: 'arm32v7'
- runner: [self-hosted, linux, ARM]
- - name: arm64v8-ubuntu-20.04-cpp
- ubuntu: 20.04
- title: ARM64v8 Ubuntu 20.04 C++
- image: |
- -e CPP_MAKE_PARALLELISM=1 \
- -e ARROW_PARQUET=OFF \
- ubuntu-cpp
- arch: 'arm64v8'
- runner: [self-hosted, linux, ARM64]
- env:
- # the defaults here should correspond to the values in .env
- ARCH: ${{ matrix.arch || 'arm64v8' }}
- DEBIAN: ${{ matrix.debian || 10 }}
- FEDORA: ${{ matrix.fedora || 32 }}
- UBUNTU: ${{ matrix.ubuntu || 18.04 }}
- LLVM: 8
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Setup Archery
- run: pip install -U -e dev/archery[docker]
- - name: Execute Docker Build
- # parallelism is reduced because the ARM builders are low on memory
- run: |
- ulimit -c unlimited
- archery docker run ${{ matrix.image }}
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push ${{ matrix.image }}
-
- build-example:
- name: C++ Minimal Build Example
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Run
- run: |
- cd cpp/examples/minimal_build
- docker-compose run --rm minimal
-
- macos:
- name: AMD64 MacOS 10.15 C++
- runs-on: macos-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- env:
- ARROW_BUILD_TESTS: ON
- ARROW_DATASET: ON
- ARROW_FLIGHT: ON
- ARROW_GANDIVA: ON
- ARROW_HDFS: ON
- ARROW_HOME: /usr/local
- ARROW_JEMALLOC: ON
- # TODO(kszucs): link error in the tests
- ARROW_ORC: OFF
- ARROW_PARQUET: ON
- ARROW_PLASMA: ON
- ARROW_S3: ON
- ARROW_WITH_BROTLI: ON
- ARROW_WITH_BZ2: ON
- ARROW_WITH_LZ4: ON
- ARROW_WITH_SNAPPY: ON
- ARROW_WITH_ZLIB: ON
- ARROW_WITH_ZSTD: ON
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Install Dependencies
- shell: bash
- run: |
- rm -f /usr/local/bin/2to3
- brew update --preinstall
- brew unlink gcc@8 gcc@9
- brew bundle --file=cpp/Brewfile
- - name: Build
- shell: bash
- run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
- - name: Test
- shell: bash
- run: |
- sudo sysctl -w kern.coredump=1
- sudo sysctl -w kern.corefile=core.%N.%P
- ulimit -c unlimited # must enable within the same shell
- ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
-
- windows:
- name: AMD64 ${{ matrix.name }} C++
- runs-on: ${{ matrix.os }}
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- os:
- - windows-latest
- include:
- - os: windows-latest
- name: Windows 2019
- generator: Visual Studio 16 2019
- env:
- ARROW_BOOST_USE_SHARED: OFF
- ARROW_BUILD_BENCHMARKS: ON
- ARROW_BUILD_SHARED: ON
- ARROW_BUILD_STATIC: OFF
- ARROW_BUILD_TESTS: ON
- ARROW_DATASET: ON
- ARROW_FLIGHT: OFF
- ARROW_HDFS: ON
- ARROW_HOME: /usr
- ARROW_JEMALLOC: OFF
- ARROW_MIMALLOC: ON
- ARROW_PARQUET: ON
- ARROW_USE_GLOG: OFF
- ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
- ARROW_WITH_BROTLI: OFF
- ARROW_WITH_BZ2: OFF
- ARROW_WITH_LZ4: OFF
- ARROW_WITH_SNAPPY: ON
- ARROW_WITH_ZLIB: ON
- ARROW_WITH_ZSTD: ON
- BOOST_SOURCE: BUNDLED
- CMAKE_ARGS: '-A x64 -DOPENSSL_ROOT_DIR=C:\Program Files\OpenSSL-Win64'
- CMAKE_GENERATOR: ${{ matrix.generator }}
- CMAKE_INSTALL_LIBDIR: bin
- CMAKE_INSTALL_PREFIX: /usr
- CMAKE_UNITY_BUILD: ON
- NPROC: 2
- steps:
- - name: Disable Crash Dialogs
- run: |
- reg add `
- "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
- /v DontShowUI `
- /t REG_DWORD `
- /d 1 `
- /f
- - name: Installed Packages
- run: choco list -l
- - name: Install Dependencies
- run: choco install -y --no-progress openssl
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Build
- shell: bash
- run: |
- ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
- - name: Test
- shell: bash
- run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
-
- windows-mingw:
- name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} C++
- runs-on: windows-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- mingw-n-bits:
- - 32
- - 64
- env:
- ARROW_BUILD_SHARED: ON
- ARROW_BUILD_STATIC: OFF
- ARROW_BUILD_TESTS: ON
- ARROW_BUILD_TYPE: release
- ARROW_DATASET: ON
- ARROW_FLIGHT: ON
- ARROW_GANDIVA: ON
- ARROW_HDFS: OFF
- ARROW_HOME: /mingw${{ matrix.mingw-n-bits }}
- ARROW_JEMALLOC: OFF
- ARROW_PARQUET: ON
- ARROW_PYTHON: ON
- ARROW_S3: ON
- ARROW_USE_GLOG: OFF
- ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
- ARROW_WITH_BROTLI: ON
- ARROW_WITH_BZ2: ON
- ARROW_WITH_LZ4: ON
- ARROW_WITH_SNAPPY: ON
- ARROW_WITH_ZLIB: ON
- ARROW_WITH_ZSTD: ON
- # Don't use preinstalled Boost by empty BOOST_ROOT and
- # -DBoost_NO_BOOST_CMAKE=ON
- BOOST_ROOT: ""
- CMAKE_ARGS: >-
- -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
- -DBoost_NO_BOOST_CMAKE=ON
- CMAKE_UNITY_BUILD: ON
- steps:
- - name: Disable Crash Dialogs
- run: |
- reg add `
- "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
- /v DontShowUI `
- /t REG_DWORD `
- /d 1 `
- /f
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - uses: msys2/setup-msys2@v2
- with:
- msystem: MINGW${{ matrix.mingw-n-bits }}
- update: true
- - name: Setup MSYS2
- shell: msys2 {0}
- run: |
- ci/scripts/msys2_setup.sh cpp
- - name: Cache ccache
- uses: actions/cache@v2
- with:
- path: ccache
- key: cpp-ccache-mingw${{ matrix.mingw-n-bits }}-${{
hashFiles('cpp/**') }}
- restore-keys: cpp-ccache-mingw${{ matrix.mingw-n-bits }}-
- - name: Build
- shell: msys2 {0}
- run: |
- export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS
- ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
- - name: Download MinIO
- shell: msys2 {0}
- run: |
- mkdir -p /usr/local/bin
- wget \
- --output-document /usr/local/bin/minio.exe \
- https://dl.min.io/server/minio/release/windows-amd64/minio.exe
- chmod +x /usr/local/bin/minio.exe
- - name: Test
- shell: msys2 {0}
- run: |
- python_version=$(python -c "import sys; print('.'.join(map(str,
sys.version_info[0:2])))")
- export PYTHONHOME="$(cygpath --windows
${MINGW_PREFIX})\lib\python${python_version}"
- PYTHONPATH="${PYTHONHOME}"
- PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\lib-dynload"
- PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\site-packages"
- export PYTHONPATH
- ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build"
diff --git a/.github/workflows/cpp_cron.yml b/.github/workflows/cpp_cron.yml
deleted file mode 100644
index 9e4f3cf..0000000
--- a/.github/workflows/cpp_cron.yml
+++ /dev/null
@@ -1,149 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: C++ Cron
-
-on:
- push:
- paths:
- - '.github/workflows/cpp_cron.yml'
- pull_request:
- paths:
- - '.github/workflows/cpp_cron.yml'
- schedule:
- - cron: |
- 0 */12 * * *
-
-env:
- ARROW_ENABLE_TIMING_TESTS: OFF
- DOCKER_VOLUME_PREFIX: ".docker/"
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
- docker:
- name: ${{ matrix.title }}
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') &&
github.repository == 'apache/arrow' }}
- strategy:
- fail-fast: false
- matrix:
- name:
- - amd64-debian-10-cpp
- - amd64-fedora-33-cpp
- - amd64-ubuntu-18.04-cpp
- - amd64-ubuntu-20.04-cpp
- include:
- - name: amd64-debian-10-cpp
- image: debian-cpp
- title: AMD64 Debian 10 C++
- debian: 10
- - name: amd64-fedora-33-cpp
- image: fedora-cpp
- title: AMD64 Fedora 33 C++
- fedora: 33
- - name: amd64-ubuntu-18.04-cpp
- image: ubuntu-cpp
- title: AMD64 Ubuntu 18.04 C++
- ubuntu: 18.04
- - name: amd64-ubuntu-20.04-cpp
- image: ubuntu-cpp
- title: AMD64 Ubuntu 20.04 C++
- ubuntu: 20.04
- env:
- # the defaults here should correspond to the values in .env
- ARCH: 'amd64'
- DEBIAN: ${{ matrix.debian || 10 }}
- FEDORA: ${{ matrix.fedora || 33 }}
- UBUNTU: ${{ matrix.ubuntu || 18.04 }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v2
- with:
- path: .docker
- key: ${{ matrix.name }}-${{ hashFiles('cpp/**') }}
- restore-keys: ${{ matrix.name }}-
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: |
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
- ulimit -c unlimited
- archery docker run ${{ matrix.image }}
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push ${{ matrix.image }}
-
- oss-fuzz:
- name: OSS-Fuzz build check
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') &&
github.repository == 'apache/arrow' }}
- strategy:
- fail-fast: false
- matrix:
- ubuntu: [18.04]
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- shell: bash
- run: ci/scripts/util_cleanup.sh
- - name: Checkout OSS-Fuzz
- uses: actions/checkout@v1
- with:
- path: oss-fuzz
- repository: google/oss-fuzz
- ref: master
- - name: Install dependencies
- working-directory: ../oss-fuzz
- run: |
- python3 -m pip install setuptools
- python3 -m pip install -r infra/ci/requirements.txt
- - name: Build image
- shell: bash
- working-directory: ../oss-fuzz
- run: |
- python3 infra/helper.py build_image --pull arrow
- - name: Build fuzzers
- shell: bash
- working-directory: ../oss-fuzz
- run: |
- python3 infra/helper.py build_fuzzers arrow `pwd`/../arrow
- - name: Check build
- shell: bash
- working-directory: ../oss-fuzz
- run: |
- python3 infra/helper.py check_build arrow
diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml
deleted file mode 100644
index 03a297b..0000000
--- a/.github/workflows/csharp.yml
+++ /dev/null
@@ -1,121 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: C#
-
-on:
- push:
- paths:
- - '.github/workflows/csharp.yml'
- - 'ci/scripts/csharp_*'
- - 'csharp/**'
- pull_request:
- paths:
- - '.github/workflows/csharp.yml'
- - 'ci/scripts/csharp_*'
- - 'csharp/**'
-
-jobs:
-
- ubuntu:
- name: AMD64 Ubuntu 18.04 C# ${{ matrix.dotnet }}
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- dotnet: ['3.1.x']
- steps:
- - name: Install C#
- uses: actions/setup-dotnet@v1
- with:
- dotnet-version: ${{ matrix.dotnet }}
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Install Source Link
- shell: bash
- run: dotnet tool install --global sourcelink
- - name: Build
- shell: bash
- run: ci/scripts/csharp_build.sh $(pwd)
- - name: Test
- shell: bash
- run: ci/scripts/csharp_test.sh $(pwd)
-
- windows:
- name: AMD64 Windows 2019 18.04 C# ${{ matrix.dotnet }}
- runs-on: windows-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- dotnet: ['3.1.x']
- steps:
- - name: Install C#
- uses: actions/setup-dotnet@v1
- with:
- dotnet-version: ${{ matrix.dotnet }}
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Install Source Link
- run: dotnet tool install --global sourcelink
- - name: Build
- shell: bash
- run: ci/scripts/csharp_build.sh $(pwd)
- - name: Test
- shell: bash
- run: ci/scripts/csharp_test.sh $(pwd)
-
- macos:
- name: AMD64 MacOS 10.15 C# ${{ matrix.dotnet }}
- runs-on: macos-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- dotnet: ['3.1.x']
- steps:
- - name: Install C#
- uses: actions/setup-dotnet@v1
- with:
- dotnet-version: ${{ matrix.dotnet }}
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Install Source Link
- shell: bash
- run: dotnet tool install --global sourcelink
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Build
- shell: bash
- run: ci/scripts/csharp_build.sh $(pwd)
- - name: Test
- shell: bash
- run: ci/scripts/csharp_test.sh $(pwd)
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index d1b0184..6c6dd83 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -22,75 +22,18 @@ on:
push:
pull_request:
-env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
jobs:
lint:
name: Lint C++, Python, R, Rust, Docker, RAT
runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
+ - uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Setup Archery
run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: |
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
- ulimit -c unlimited
- archery docker run ubuntu-lint
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push ubuntu-lint
-
- release:
- name: Source Release and Merge Script
- runs-on: ubuntu-20.04
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- env:
- GIT_AUTHOR_NAME: Github Actions
- GIT_AUTHOR_EMAIL: github@actions
- GIT_COMMITTER_NAME: Github Actions
- GIT_COMMITTER_EMAIL: github@actions
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Install Python
- uses: actions/setup-python@v1
- with:
- python-version: '3.6'
- - name: Install Ruby
- uses: ruby/setup-ruby@v1
- with:
- ruby-version: '2.6'
- - name: Install Dependencies
- shell: bash
- run: |
- pip install cython setuptools pytest jira
- - name: Run Release Test
- shell: bash
- run: |
- ci/scripts/release_test.sh $(pwd)
- - name: Run Merge Script Test
- shell: bash
- run: |
- pytest -v dev/test_merge_arrow_pr.py
+ - name: Lint
+ run: archery lint --rat
diff --git a/.github/workflows/dev_pr/labeler.yml
b/.github/workflows/dev_pr/labeler.yml
index 098e1ba..5eb722d 100644
--- a/.github/workflows/dev_pr/labeler.yml
+++ b/.github/workflows/dev_pr/labeler.yml
@@ -15,60 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-"lang-c++":
- - cpp/**/*
-
-lang-c-glib:
- - c_glib/**/*
-
-lang-csharp:
- - csharp/**/*
-
-lang-go:
- - go/**/*
-
-lang-java:
- - java/**/*
-
-lang-js:
- - js/**/*
-
-lang-julia:
- - julia/**/*
-
-lang-python:
- - python/**/*
-
-lang-R:
- - r/**/*
-
-lang-ruby:
- - ruby/**/*
-
-lang-rust:
- - rust/**/*
-
datafusion:
- rust/datafusion/**/*
ballista:
- rust/ballista/**/*
-
-flight:
- - cpp/src/arrow/flight/**/*
- - r/R/flight.*
- - rust/arrow-flight/**/*
- - python/pyarrow/*flight.*
-
-gandiva:
- - c_glib/gandiva-glib/**/*
- - cpp/src/gandiva/**/*
- - ruby/red-gandiva/**/*
- - python/pyarrow/gandiva.*
-
-parquet:
- - c_glib/parquet-glib/**/*
- - cpp/src/parquet/**/*
- - r/R/parquet.*
- - ruby/red-parquet/**/*
- - rust/parquet*/**/*
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
deleted file mode 100644
index 574795f..0000000
--- a/.github/workflows/go.yml
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Go
-
-on:
- push:
- paths:
- - '.github/workflows/go.yml'
- - 'ci/docker/*_go.dockerfile'
- - 'ci/scripts/go_*'
- - 'go/**'
- pull_request:
- paths:
- - '.github/workflows/go.yml'
- - 'ci/docker/*_go.dockerfile'
- - 'ci/docker/**'
- - 'ci/scripts/go_*'
- - 'go/**'
-
-env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- docker:
- name: AMD64 Debian 10 Go ${{ matrix.go }}
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- go: [1.15]
- env:
- GO: ${{ matrix.go }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: archery docker run debian-go
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push debian-go
-
- windows:
- name: AMD64 Windows 2019 Go ${{ matrix.go }}
- runs-on: windows-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- go: [1.15]
- steps:
- - name: Install go
- uses: actions/setup-go@v1
- with:
- go-version: ${{ matrix.go }}
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Build
- shell: bash
- run: ci/scripts/go_build.sh .
- - name: Test
- shell: bash
- run: ci/scripts/go_test.sh .
-
- macos:
- name: AMD64 MacOS 10.15 Go ${{ matrix.go }}
- runs-on: macos-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- go: [1.15]
- steps:
- - name: Install go
- uses: actions/setup-go@v1
- with:
- go-version: ${{ matrix.go }}
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Build
- shell: bash
- run: ci/scripts/go_build.sh .
- - name: Test
- shell: bash
- run: ci/scripts/go_test.sh .
diff --git a/.github/workflows/integration.yml
b/.github/workflows/integration.yml
deleted file mode 100644
index 2011255..0000000
--- a/.github/workflows/integration.yml
+++ /dev/null
@@ -1,83 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Integration
-
-on:
- push:
- paths:
- - '.github/workflows/integration.yml'
- - 'ci/**'
- - 'dev/archery/**'
- - 'go/**'
- - 'integration/**'
- - 'js/**'
- - 'cpp/**'
- - 'java/**'
- - 'format/**'
- - 'rust/**'
- pull_request:
- paths:
- - '.github/workflows/integration.yml'
- - 'ci/**'
- - 'dev/archery/**'
- - 'go/**'
- - 'integration/**'
- - 'js/**'
- - 'cpp/**'
- - 'java/**'
- - 'format/**'
- - 'rust/**'
-
-env:
- DOCKER_VOLUME_PREFIX: ".docker/"
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- docker:
- name: AMD64 Conda Integration Test
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v2
- with:
- path: .docker
- key: conda-${{ hashFiles('cpp/**') }}
- restore-keys: conda-
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: archery docker run conda-integration
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push conda-integration
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
deleted file mode 100644
index 7f6f29f..0000000
--- a/.github/workflows/java.yml
+++ /dev/null
@@ -1,112 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Java
-
-on:
- push:
- paths:
- - '.github/workflows/java.yml'
- - 'ci/docker/*java*'
- - 'ci/scripts/java*.sh'
- - 'ci/scripts/util_*.sh'
- - 'format/Flight.proto'
- - 'java/**'
- pull_request:
- paths:
- - '.github/workflows/java.yml'
- - 'ci/docker/*java*'
- - 'ci/scripts/java*.sh'
- - 'ci/scripts/util_*.sh'
- - 'format/Flight.proto'
- - 'java/**'
-
-env:
- DOCKER_VOLUME_PREFIX: ".docker/"
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- debian:
- name: AMD64 Debian 9 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }}
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- jdk: [11]
- maven: [3.6.2]
- env:
- JDK: ${{ matrix.jdk }}
- MAVEN: ${{ matrix.maven }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- shell: bash
- run: ci/scripts/util_cleanup.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v2
- with:
- path: .docker
- key: maven-${{ hashFiles('java/**') }}
- restore-keys: maven-
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: archery docker run debian-java
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push debian-java
-
- macos:
- name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }}
- runs-on: macos-latest
- if: github.event_name == 'push'
- strategy:
- fail-fast: false
- matrix:
- jdk: [11]
- steps:
- - name: Set up Java
- uses: actions/setup-java@v1
- with:
- java-version: ${{ matrix.jdk }}
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Build
- shell: bash
- run: ci/scripts/java_build.sh $(pwd) $(pwd)/build
- - name: Test
- shell: bash
- run: ci/scripts/java_test.sh $(pwd) $(pwd)/build
diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml
deleted file mode 100644
index 5f25e8c..0000000
--- a/.github/workflows/java_jni.yml
+++ /dev/null
@@ -1,83 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Java JNI
-
-on:
- push:
- paths:
- - '.github/workflows/java_jni.yml'
- - 'ci/docker/**'
- - 'ci/scripts/cpp_build.sh'
- - 'ci/scripts/java_*'
- - 'cpp/**'
- - 'java/**'
- pull_request:
- paths:
- - '.github/workflows/java_jni.yml'
- - 'ci/docker/**'
- - 'ci/scripts/cpp_build.sh'
- - 'ci/scripts/java_*'
- - 'cpp/**'
- - 'java/**'
-
-env:
- DOCKER_VOLUME_PREFIX: ".docker/"
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- docker:
- name: AMD64 Debian 9 Java JNI (Gandiva, Plasma, ORC, Dataset)
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- jdk: [8]
- maven: [3.5.2]
- env:
- JDK: ${{ matrix.jdk }}
- MAVEN: ${{ matrix.maven }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v2
- with:
- path: .docker
- key: maven-${{ hashFiles('java/**') }}
- restore-keys: maven-
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: archery docker run debian-java-jni
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push debian-java-jni
diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml
deleted file mode 100644
index 354c45c..0000000
--- a/.github/workflows/js.yml
+++ /dev/null
@@ -1,122 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: NodeJS
-
-on:
- push:
- paths:
- - '.github/workflows/js.yml'
- - 'ci/docker/*js.dockerfile'
- - 'ci/scripts/js_*'
- - 'js/**'
- pull_request:
- paths:
- - '.github/workflows/js.yml'
- - 'ci/docker/*js.dockerfile'
- - 'ci/scripts/js_*'
- - 'js/**'
-
-env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- docker:
- name: AMD64 Debian 10 NodeJS 14
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: |
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
- ulimit -c unlimited
- archery docker run debian-js
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push debian-js
-
- macos:
- name: AMD64 MacOS 10.15 NodeJS ${{ matrix.node }}
- runs-on: macos-latest
- if: github.event_name == 'push'
- strategy:
- fail-fast: false
- matrix:
- node: [14]
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Install NodeJS
- uses: actions/setup-node@v1
- with:
- node-version: ${{ matrix.node }}
- - name: Build
- shell: bash
- run: ci/scripts/js_build.sh $(pwd)
- - name: Test
- shell: bash
- run: ci/scripts/js_test.sh $(pwd)
-
- # TODO(kszucs): the windows build fails with platform specific npm error
- # windows:
- # name: AMD64 Windows 2019 NodeJS ${{ matrix.node }}
- # runs-on: windows-latest
- # if: github.event_name == 'push'
- # strategy:
- # fail-fast: false
- # matrix:
- # node: [14]
- # steps:
- # - name: Checkout Arrow
- # uses: actions/checkout@v1
- # with:
- # submodules: true
- # - name: Install NodeJS
- # uses: actions/setup-node@v1
- # with:
- # node-version: ${{ matrix.node }}
- # - name: Install Platform Dependencies
- # shell: bash
- # run: yarn add -g cross-env
- # - name: Build
- # shell: bash
- # run: ci/scripts/js_build.sh $(pwd)
- # - name: Test
- # shell: bash
- # run: ci/scripts/js_test.sh $(pwd)
diff --git a/.github/workflows/julia.yml b/.github/workflows/julia.yml
deleted file mode 100644
index 64ea6c9..0000000
--- a/.github/workflows/julia.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Julia
-on:
- push:
- paths:
- - '.github/workflows/julia.yml'
- - 'julia/**'
- pull_request:
- paths:
- - '.github/workflows/julia.yml'
- - 'julia/**'
-
-jobs:
- test:
- name: AMD64 ${{ matrix.os }} Julia ${{ matrix.version }}
- env:
- JULIA_NUM_THREADS: 2
- runs-on: ${{ matrix.os }}
- strategy:
- fail-fast: false
- matrix:
- version:
- - '1.3'
- - '1' # automatically expands to the latest stable 1.x release of
Julia
- - 'nightly'
- os:
- - ubuntu-latest
- - windows-latest
- steps:
- - uses: actions/checkout@v2
- - uses: julia-actions/setup-julia@v1
- with:
- version: ${{ matrix.version }}
- arch: x64
- - uses: julia-actions/julia-runtest@v1
- with:
- project: julia/Arrow
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
deleted file mode 100644
index 9062e93..0000000
--- a/.github/workflows/python.yml
+++ /dev/null
@@ -1,154 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Python
-
-on:
- push:
- paths:
- - '.github/workflows/python.yml'
- - 'ci/**'
- - 'cpp/**'
- - 'python/**'
- pull_request:
- paths:
- - '.github/workflows/python.yml'
- - 'ci/**'
- - 'cpp/**'
- - 'python/**'
-
-env:
- DOCKER_VOLUME_PREFIX: ".docker/"
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- docker:
- name: ${{ matrix.title }}
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- name:
- - conda-python-3.8-nopandas
- - conda-python-3.6-pandas-0.23
- - conda-python-3.7-pandas-latest
- include:
- - name: conda-python-3.8-nopandas
- cache: conda-python-3.8
- image: conda-python
- title: AMD64 Conda Python 3.8 Without Pandas
- python: 3.8
- - name: conda-python-3.6-pandas-0.23
- cache: conda-python-3.6
- image: conda-python-pandas
- title: AMD64 Conda Python 3.6 Pandas 0.23
- python: 3.6
- pandas: 0.23
- - name: conda-python-3.7-pandas-latest
- cache: conda-python-3.7
- image: conda-python-pandas
- title: AMD64 Conda Python 3.7 Pandas latest
- python: 3.7
- pandas: latest
- env:
- PYTHON: ${{ matrix.python || 3.7 }}
- UBUNTU: ${{ matrix.ubuntu || 18.04 }}
- PANDAS: ${{ matrix.pandas || 'latest' }}
- NUMPY: ${{ matrix.numpy || 'latest' }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v2
- with:
- path: .docker
- key: ${{ matrix.cache }}-${{ hashFiles('cpp/**') }}
- restore-keys: ${{ matrix.cache }}-
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: |
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
- ulimit -c unlimited
- archery docker run ${{ matrix.image }}
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push ${{ matrix.image }}
-
- macos:
- name: AMD64 MacOS 10.15 Python 3
- runs-on: macos-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- env:
- ARROW_HOME: /usr/local
- ARROW_DATASET: ON
- ARROW_JEMALLOC: ON
- ARROW_ORC: ON
- ARROW_PYTHON: ON
- ARROW_PLASMA: ON
- ARROW_GANDIVA: ON
- ARROW_PARQUET: ON
- ARROW_WITH_ZLIB: ON
- ARROW_WITH_LZ4: ON
- ARROW_WITH_BZ2: ON
- ARROW_WITH_ZSTD: ON
- ARROW_WITH_SNAPPY: ON
- ARROW_WITH_BROTLI: ON
- ARROW_BUILD_TESTS: OFF
- CMAKE_ARGS: "-DPython3_EXECUTABLE=/usr/local/bin/python3"
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Install Dependencies
- shell: bash
- run: |
- rm -f /usr/local/bin/2to3
- brew update --preinstall
- brew unlink gcc@8 gcc@9
- brew bundle --file=cpp/Brewfile
- brew install coreutils
- python3 -mpip install \
- -r python/requirements-build.txt \
- -r python/requirements-test.txt
- - name: Build
- shell: bash
- run: |
- export PYTHON=python3
- ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
- ci/scripts/python_build.sh $(pwd) $(pwd)/build
- - name: Test
- shell: bash
- run: ci/scripts/python_test.sh $(pwd) $(pwd)/build
diff --git a/.github/workflows/python_cron.yml
b/.github/workflows/python_cron.yml
deleted file mode 100644
index 7a4401a..0000000
--- a/.github/workflows/python_cron.yml
+++ /dev/null
@@ -1,141 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Python Cron
-
-on:
- push:
- paths:
- - '.github/workflows/python_cron.yml'
- pull_request:
- paths:
- - '.github/workflows/python_cron.yml'
- schedule:
- - cron: |
- 0 */12 * * *
-
-env:
- DOCKER_VOLUME_PREFIX: ".docker/"
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- docker:
- name: ${{ matrix.title }}
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') &&
github.repository == 'apache/arrow' }}
- strategy:
- fail-fast: false
- matrix:
- name:
- - debian-10-python-3
- - fedora-33-python-3
- - ubuntu-18.04-python-3
- - conda-python-3.7-dask-latest
- - conda-python-3.7-turbodbc-latest
- - conda-python-3.7-kartothek-latest
- - conda-python-3.7-pandas-0.24
- - conda-python-3.7-pandas-master
- - conda-python-3.7-hdfs-2.9.2
- include:
- - name: debian-10-python-3
- cache: debian-10-python-3
- image: debian-python
- title: AMD64 Debian 10 Python 3
- debian: 10
- - name: fedora-33-python-3
- cache: fedora-33-python-3
- image: fedora-python
- title: AMD64 Fedora 33 Python 3
- fedora: 33
- - name: ubuntu-18.04-python-3
- cache: ubuntu-18.04-python-3
- image: ubuntu-python
- title: AMD64 Ubuntu 18.04 Python 3
- ubuntu: 18.04
- - name: conda-python-3.7-dask-latest
- cache: conda-python-3.7
- image: conda-python-dask
- title: AMD64 Conda Python 3.7 Dask latest
- dask: latest
- - name: conda-python-3.7-turbodbc-latest
- cache: conda-python-3.7
- image: conda-python-turbodbc
- title: AMD64 Conda Python 3.7 Turbodbc latest
- turbodbc: latest
- - name: conda-python-3.7-kartothek-latest
- cache: conda-python-3.7
- image: conda-python-kartothek
- title: AMD64 Conda Python 3.7 Kartothek latest
- kartothek: latest
- - name: conda-python-3.7-pandas-0.24
- cache: conda-python-3.7
- image: conda-python-pandas
- title: AMD64 Conda Python 3.7 Pandas 0.24
- pandas: 0.24
- - name: conda-python-3.7-pandas-master
- cache: conda-python-3.7
- image: --no-leaf-cache conda-python-pandas
- title: AMD64 Conda Python 3.7 Pandas master
- pandas: master
- - name: conda-python-3.7-hdfs-2.9.2
- cache: conda-python-3.7
- image: conda-python-hdfs
- title: AMD64 Conda Python 3.7 HDFS 2.9.2
- hdfs: 2.9.2
- env:
- # the defaults here should correspond to the values in .env
- DEBIAN: ${{ matrix.debian || 10 }}
- FEDORA: ${{ matrix.fedora || 33 }}
- UBUNTU: ${{ matrix.ubuntu || 18.04 }}
- PYTHON: ${{ matrix.python || 3.7 }}
- HDFS: ${{ matrix.hdfs || '2.9.2' }}
- DASK: ${{ matrix.dask || 'latest' }}
- TURBODBC: ${{ matrix.turbodbc || 'latest' }}
- PANDAS: ${{ matrix.pandas || 'latest' }}
- KARTOTHEK: ${{ matrix.kartothek || 'latest' }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v2
- with:
- path: .docker
- key: ${{ matrix.cache }}-${{ hashFiles('cpp/**') }}
- restore-keys: ${{ matrix.cache }}-
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: |
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
- ulimit -c unlimited
- archery docker run ${{ matrix.image }}
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push ${{ matrix.image }}
diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
deleted file mode 100644
index 7851b6b..0000000
--- a/.github/workflows/r.yml
+++ /dev/null
@@ -1,255 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: R
-
-on:
- push:
- paths:
- - ".github/workflows/r.yml"
- - "ci/scripts/r_*.sh"
- - "ci/scripts/cpp_*.sh"
- - "ci/scripts/PKGBUILD"
- - "ci/etc/rprofile"
- - "ci/docker/**"
- - "cpp/**"
- - "r/**"
- pull_request:
- paths:
- - ".github/workflows/r.yml"
- - "ci/scripts/r_*.sh"
- - "ci/scripts/cpp_*.sh"
- - "ci/scripts/PKGBUILD"
- - "ci/etc/rprofile"
- - "ci/docker/**"
- - "cpp/**"
- - "r/**"
-
-env:
- DOCKER_VOLUME_PREFIX: ".docker/"
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
- ubuntu:
- name: AMD64 Ubuntu ${{ matrix.ubuntu }} R ${{ matrix.r }}
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- r: ["3.6"]
- ubuntu: [18.04]
- env:
- R: ${{ matrix.r }}
- UBUNTU: ${{ matrix.ubuntu }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v2
- with:
- path: .docker
- key: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{
hashFiles('cpp/**') }}
- restore-keys: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-
- - name: Check pkgdown reference sections
- run: ci/scripts/r_pkgdown_check.sh
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: |
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
- ulimit -c unlimited
- archery docker run ubuntu-r
- - name: Dump install logs
- run: cat r/check/arrow.Rcheck/00install.out
- if: always()
- - name: Dump test logs
- run: cat r/check/arrow.Rcheck/tests/testthat.Rout*
- if: always()
- - name: Save the test output
- if: always()
- uses: actions/upload-artifact@v2
- with:
- name: test-output
- path: r/check/arrow.Rcheck/tests/testthat.Rout*
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push ubuntu-r
-
- bundled:
- name: "${{ matrix.config.org }}/${{ matrix.config.image }}:${{
matrix.config.tag }}"
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- config:
- - { org: "rstudio", image: "r-base", tag: "4.0-centos7" }
- - { org: "rhub", image: "debian-gcc-devel", tag: "latest" }
- env:
- R_ORG: ${{ matrix.config.org }}
- R_IMAGE: ${{ matrix.config.image }}
- R_TAG: ${{ matrix.config.tag }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- run: ci/scripts/util_cleanup.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v2
- with:
- path: .docker
- key: ${{ matrix.config.image }}-r-${{ hashFiles('cpp/**') }}
- restore-keys: ${{ matrix.config.image }}-r-
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: |
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
- ulimit -c unlimited
- archery docker run r
- - name: Dump install logs
- run: cat r/check/arrow.Rcheck/00install.out
- if: always()
- - name: Dump test logs
- run: cat r/check/arrow.Rcheck/tests/testthat.Rout*
- if: always()
- - name: Save the test output
- if: always()
- uses: actions/upload-artifact@v2
- with:
- name: test-output
- path: r/check/arrow.Rcheck/tests/testthat.Rout*
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- run: archery docker push r
-
- windows:
- name: AMD64 Windows RTools ${{ matrix.rtools }}
- runs-on: windows-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- rtools: [35, 40]
- env:
- TEST_R_WITH_ARROW: "TRUE"
- ARROW_R_CXXFLAGS: "-Werror"
- _R_CHECK_TESTS_NLINES_: 0
- steps:
- - run: git config --global core.autocrlf false
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Make R tests verbose
- # If you get a segfault/mysterious test Execution halted,
- # make this `true` to see where it dies.
- if: false
- shell: cmd
- run: |
- cd r/tests
- sed -i.bak -E -e 's/"arrow"/"arrow", reporter = "location"/'
testthat.R
- rm -f testthat.R.bak
- - name: Setup ccache
- shell: bash
- run: |
- ci/scripts/ccache_setup.sh
- echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >>
$GITHUB_ENV
- # We must enable actions/cache before r-lib/actions/setup-r to ensure
- # using system tar instead of tar provided by Rtools.
- # We can use tar provided by Rtools when we drop support for Rtools 3.5.
- # Because Rtools 4.0 or later has zstd. actions/cache requires zstd
- # when tar is GNU tar.
- - name: Cache ccache
- uses: actions/cache@v2
- with:
- path: ccache
- key: r-${{ matrix.rtools }}-ccache-mingw-${{ hashFiles('cpp/**') }}
- restore-keys: r-${{ matrix.rtools }}-ccache-mingw-
- # We use the makepkg-mingw setup that is included in rtools40 even when
- # we use the rtools35 compilers, so we always install R 4.0/Rtools40
- - uses: r-lib/actions/setup-r@master
- with:
- rtools-version: 40
- r-version: "4.0"
- Ncpus: 2
- - uses: r-lib/actions/setup-r@master
- if: ${{ matrix.rtools == 35 }}
- with:
- rtools-version: 35
- r-version: "3.6"
- Ncpus: 2
- - name: Build Arrow C++
- shell: bash
- env:
- RTOOLS_VERSION: ${{ matrix.rtools }}
- run: ci/scripts/r_windows_build.sh
- - uses: actions/upload-artifact@v1
- with:
- name: Rtools ${{ matrix.rtools }} Arrow C++
- path: libarrow.zip
- - name: Install R package dependencies
- shell: Rscript {0}
- run: |
- options(pkgType="win.binary")
- install.packages(c("remotes", "rcmdcheck"))
- remotes::install_deps("r", dependencies = TRUE)
- - name: Check
- shell: Rscript {0}
- run: |
- Sys.setenv(
- RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"),
"libarrow.zip"),
- MAKEFLAGS = paste0("-j", parallel::detectCores())
- )
- rcmdcheck::rcmdcheck("r",
- build_args = '--no-build-vignettes',
- args = c('--no-manual', '--as-cran', '--ignore-vignettes',
'--run-donttest'),
- error_on = 'warning',
- check_dir = 'check',
- timeout = 3600
- )
- - name: Dump install logs
- shell: cmd
- run: cat check/arrow.Rcheck/00install.out
- if: always()
- # We can remove this when we drop support for Rtools 3.5.
- - name: Ensure using system tar in actions/cache
- run: |
- Write-Output "${Env:windir}\System32" | `
- Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
deleted file mode 100644
index 2b99cdd..0000000
--- a/.github/workflows/ruby.yml
+++ /dev/null
@@ -1,290 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: C GLib & Ruby
-
-on:
- push:
- paths:
- - '.github/workflows/ruby.yml'
- - 'ci/docker/**'
- - 'ci/scripts/c_glib_*'
- - 'ci/scripts/cpp_*'
- - 'ci/scripts/msys2_*'
- - 'ci/scripts/ruby_*'
- - 'ci/scripts/util_*'
- - 'c_glib/**'
- - 'cpp/**'
- - 'ruby/**'
- pull_request:
- paths:
- - '.github/workflows/ruby.yml'
- - 'ci/docker/**'
- - 'ci/scripts/c_glib_*'
- - 'ci/scripts/cpp_*'
- - 'ci/scripts/msys2_*'
- - 'ci/scripts/ruby_*'
- - 'ci/scripts/util_*'
- - 'c_glib/**'
- - 'cpp/**'
- - 'ruby/**'
-
-env:
- DOCKER_VOLUME_PREFIX: ".docker/"
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- ubuntu:
- name: AMD64 Ubuntu ${{ matrix.ubuntu }} GLib & Ruby
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- ubuntu:
- - 18.04
- - 20.04
- env:
- UBUNTU: ${{ matrix.ubuntu }}
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Free Up Disk Space
- shell: bash
- run: ci/scripts/util_cleanup.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v2
- with:
- path: .docker
- key: ubuntu-${{ matrix.ubuntu }}-ruby-${{ hashFiles('cpp/**') }}
- restore-keys: ubuntu-${{ matrix.ubuntu }}-ruby-
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: |
- sudo sysctl -w kernel.core_pattern="core.%e.%p"
- ulimit -c unlimited
- archery docker run ubuntu-ruby
- - name: Docker Push
- if: success() && github.event_name == 'push' && github.repository ==
'apache/arrow'
- continue-on-error: true
- shell: bash
- run: archery docker push ubuntu-ruby
-
- macos:
- name: AMD64 MacOS 10.15 GLib & Ruby
- runs-on: macos-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- env:
- ARROW_BUILD_TESTS: OFF
- ARROW_GANDIVA: ON
- ARROW_GLIB_DEVELOPMENT_MODE: true
- ARROW_GLIB_GTK_DOC: true
- ARROW_HOME: /usr/local
- ARROW_JEMALLOC: OFF
- ARROW_ORC: OFF
- ARROW_PARQUET: ON
- ARROW_WITH_BROTLI: ON
- ARROW_WITH_LZ4: ON
- ARROW_WITH_SNAPPY: ON
- ARROW_WITH_ZLIB: ON
- ARROW_WITH_ZSTD: ON
- XML_CATALOG_FILES: /usr/local/etc/xml/catalog
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Install Homebrew Dependencies
- shell: bash
- run: |
- rm -f /usr/local/bin/2to3
- brew update --preinstall
- brew unlink gcc@8 gcc@9
- brew bundle --file=cpp/Brewfile
- brew bundle --file=c_glib/Brewfile
- - name: Install Ruby Dependencies
- run: |
- export MAKEFLAGS="-j$(sysctl -n hw.ncpu)"
- bundle install --gemfile c_glib/Gemfile
- bundle install --gemfile ruby/Gemfile
- for ruby_package_gemfile in ruby/*/Gemfile; do \
- bundle install --gemfile ${ruby_package_gemfile}
- done
- - name: Setup ccache
- run: |
- ci/scripts/ccache_setup.sh
- - name: ccache info
- id: ccache-info
- run: |
- echo "::set-output name=cache-dir::$(ccache --get-config cache_dir)"
- - name: Cache ccache
- uses: actions/cache@v2
- with:
- path: ${{ steps.ccache-info.outputs.cache-dir }}
- key: ruby-ccache-macos-${{ hashFiles('cpp/**') }}
- restore-keys: ruby-ccache-macos-
- - name: Build C++
- run: |
- ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
- - name: Build GLib
- run: |
- ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build
- - name: Test GLib
- shell: bash
- run: ci/scripts/c_glib_test.sh $(pwd) $(pwd)/build
- - name: Test Ruby
- shell: bash
- run: ci/scripts/ruby_test.sh $(pwd) $(pwd)/build
-
- windows:
- name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} GLib & Ruby
- runs-on: windows-latest
- if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
- strategy:
- fail-fast: false
- matrix:
- mingw-n-bits:
- - 64
- ruby-version:
- - 2.6
- env:
- ARROW_BUILD_SHARED: ON
- ARROW_BUILD_STATIC: OFF
- ARROW_BUILD_TESTS: OFF
- ARROW_BUILD_TYPE: release
- ARROW_DATASET: ON
- ARROW_FLIGHT: ON
- ARROW_GANDIVA: ON
- ARROW_HDFS: OFF
- ARROW_HOME: /mingw${{ matrix.mingw-n-bits }}
- ARROW_JEMALLOC: OFF
- ARROW_PARQUET: ON
- ARROW_PYTHON: OFF
- ARROW_S3: ON
- ARROW_USE_GLOG: OFF
- ARROW_WITH_BROTLI: ON
- ARROW_WITH_BZ2: ON
- ARROW_WITH_LZ4: ON
- ARROW_WITH_SNAPPY: ON
- ARROW_WITH_ZLIB: ON
- ARROW_WITH_ZSTD: ON
- # Don't use preinstalled Boost by empty BOOST_ROOT and
- # -DBoost_NO_BOOST_CMAKE=ON
- BOOST_ROOT: ""
- CMAKE_ARGS: >-
- -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
- -DBoost_NO_BOOST_CMAKE=ON
- CMAKE_UNITY_BUILD: ON
- steps:
- - name: Disable Crash Dialogs
- run: |
- reg add `
- "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
- /v DontShowUI `
- /t REG_DWORD `
- /d 1 `
- /f
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- fetch-depth: 0
- - name: Fetch Submodules and Tags
- shell: bash
- run: ci/scripts/util_checkout.sh
- - name: Setup Ruby
- uses: ruby/setup-ruby@v1
- with:
- ruby-version: ${{ matrix.ruby-version }}
- - name: Upgrade MSYS2
- run: |
- ridk exec bash ci\scripts\msys2_system_upgrade.sh
- taskkill /F /FI "MODULES eq msys-2.0.dll"
- - name: Clean MSYS2
- run: |
- ridk exec bash ci\scripts\msys2_system_clean.sh
- - name: Setup MSYS2
- run: |
- ridk exec bash ci\scripts\msys2_setup.sh ruby
- - name: Cache ccache
- uses: actions/cache@v2
- with:
- path: ccache
- key: ruby-ccache-mingw${{ matrix.mingw-n-bits }}-${{
hashFiles('cpp/**') }}
- restore-keys: ruby-ccache-mingw${{ matrix.mingw-n-bits }}-
- - name: Build C++
- run: |
- $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS
- $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")"
- $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")"
- $ErrorActionPreference = "Continue"
- ridk exec bash ci\scripts\cpp_build.sh "${source_dir}" "${build_dir}"
- - name: Build GLib
- run: |
- $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS
- $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")"
- $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")"
- $ErrorActionPreference = "Continue"
- ridk exec bash ci\scripts\c_glib_build.sh "${source_dir}"
"${build_dir}"
- - name: RubyGems info
- id: rubygems-info
- run: |
- Write-Output "::set-output name=gem-dir::$(ridk exec gem env gemdir)"
- - name: Cache RubyGems
- uses: actions/cache@v2
- with:
- path: ${{ steps.rubygems-info.outputs.gem-dir }}
- key: ruby-rubygems-mingw${{ matrix.mingw-n-bits }}-${{
hashFiles('**/Gemfile', 'ruby/*/*.gemspec') }}
- restore-keys: ruby-rubygems-mingw${{ matrix.mingw-n-bits }}-
- - name: Install test dependencies
- run: |
- bundle install --gemfile c_glib\Gemfile
- bundle install --gemfile ruby\Gemfile
- Get-ChildItem ruby\*\Gemfile | `
- ForEach-Object {bundle install --gemfile $_}
- - name: Test GLib
- run: |
- $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")"
- $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")"
- $ErrorActionPreference = "Continue"
- ridk exec bash ci\scripts\c_glib_test.sh "${source_dir}"
"${build_dir}"
- - name: Test Ruby
- run: |
- $Env:PKG_CONFIG_PATH = `
- "$(ridk exec cygpath --absolute --windows
"${Env:ARROW_HOME}/lib/pkgconfig")"
- $Env:GI_TYPELIB_PATH = `
- "$(ridk exec cygpath --absolute --windows
"${Env:ARROW_HOME}/lib/girepository-1.0")"
- $Env:RUBYOPTS = "-rdevkit"
- $Env:MAKE = "ridk exec make"
- $ErrorActionPreference = "Continue"
- rake -f ruby\Rakefile
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 4bb17a2..ac20036 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -18,16 +18,9 @@
name: Rust
on:
+ # always trigger
push:
- paths:
- - '.github/workflows/rust.yml'
- - 'rust/**'
- - 'format/Flight.proto'
pull_request:
- paths:
- - '.github/workflows/rust.yml'
- - 'rust/**'
- - 'format/Flight.proto'
jobs:
@@ -70,14 +63,13 @@ jobs:
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
- cd rust
cargo build
# Ballista is currently not part of the main workspace so requires a
separate build step
- name: Build Ballista
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
- cd rust/ballista/rust
+ cd ballista/rust
# snmalloc requires cmake so build without default features
cargo build --no-default-features
@@ -96,8 +88,6 @@ jobs:
# Disable full debug symbol generation to speed up CI build and keep
memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
- ARROW_TEST_DATA: /__w/arrow/arrow/testing/data
- PARQUET_TEST_DATA: /__w/arrow/arrow/cpp/submodules/parquet-testing/data
steps:
- uses: actions/checkout@v2
with:
@@ -123,7 +113,8 @@ jobs:
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
- cd rust
+ export ARROW_TEST_DATA=$(pwd)/testing/data
+ export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
# run tests on all workspace members with default feature list
cargo test
# test datafusion examples
@@ -131,66 +122,17 @@ jobs:
cargo test --no-default-features
cargo run --example csv_sql
cargo run --example parquet_sql
- cd ..
- cd arrow
- # re-run tests on arrow workspace with additional features
- cargo test --features=prettyprint
- cargo run --example builders
- cargo run --example dynamic_types
- cargo run --example read_csv
- cargo run --example read_csv_infer_schema
# Ballista is currently not part of the main workspace so requires a
separate test step
- name: Run Ballista tests
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
- cd rust/ballista/rust
+ export ARROW_TEST_DATA=$(pwd)/testing/data
+ export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
+ cd ballista/rust
# snmalloc requires cmake so build without default features
cargo test --no-default-features
- # test the --features "simd" of the arrow crate. This requires nightly.
- linux-test-simd:
- name: Test SIMD on AMD64 Rust ${{ matrix.rust }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [nightly-2021-03-24]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep
memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- ARROW_TEST_DATA: /__w/arrow/arrow/testing/data
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /github/home/target
- # this key equals the ones on `linux-build-lib` for re-use
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{
matrix.rust }}
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt
- - name: Run tests
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cd rust/arrow
- cargo test --features "simd"
-
windows-and-macos:
name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }}
runs-on: ${{ matrix.os }}
@@ -213,12 +155,26 @@ jobs:
shell: bash
run: |
export ARROW_TEST_DATA=$(pwd)/testing/data
- export PARQUET_TEST_DATA=$(pwd)/cpp/submodules/parquet-testing/data
+ export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
# do not produce debug symbols to keep memory usage down
export RUSTFLAGS="-C debuginfo=0"
- cd rust
cargo test
+ lint:
+ name: Lint
+ runs-on: ubuntu-latest
+ container:
+ image: amd64/rust
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup toolchain
+ run: |
+ rustup toolchain install stable
+ rustup default stable
+ rustup component add rustfmt
+ - name: Run
+ run: cargo fmt --all -- --check
+
clippy:
name: Clippy
needs: [linux-build-lib]
@@ -258,7 +214,6 @@ jobs:
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
- cd rust
cargo clippy --all-targets --workspace -- -D warnings -A
clippy::redundant_field_names
miri-checks:
@@ -290,7 +245,6 @@ jobs:
RUST_LOG: 'trace'
run: |
export MIRIFLAGS="-Zmiri-disable-isolation"
- cd rust
cargo miri setup
cargo clean
# Ignore MIRI errors until we can get a clean run
@@ -325,146 +279,12 @@ jobs:
export CARGO_TARGET_DIR="/home/runner/target"
export ARROW_TEST_DATA=$(pwd)/testing/data
- export PARQUET_TEST_DATA=$(pwd)/cpp/submodules/parquet-testing/data
+ export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
# 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0
# see https://github.com/xd009642/tarpaulin/issues/618
cargo install --version 0.16.0 cargo-tarpaulin
- cd rust
cargo tarpaulin --out Xml
- name: Report coverage
continue-on-error: true
run: bash <(curl -s https://codecov.io/bash)
-
- # test FFI against the C-Data interface exposed by pyarrow
- pyarrow-integration-test:
- name: Test Pyarrow C Data Interface
- runs-on: ubuntu-latest
- strategy:
- matrix:
- rust: [stable]
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt clippy
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /home/runner/.cargo
- key: cargo-maturin-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /home/runner/target
- # this key is not equal because maturin uses different compilation
flags.
- key: ${{ runner.os }}-${{ matrix.arch }}-target-maturin-cache-${{
matrix.rust }}-
- - uses: actions/setup-python@v2
- with:
- python-version: '3.7'
- - name: Install Python dependencies
- run: python -m pip install --upgrade pip setuptools wheel
- - name: Run tests
- run: |
- export CARGO_HOME="/home/runner/.cargo"
- export CARGO_TARGET_DIR="/home/runner/target"
-
- cd rust/arrow-pyarrow-integration-testing
-
- python -m venv venv
- source venv/bin/activate
-
- pip install maturin==0.8.2 toml==0.10.1 pyarrow==1.0.0
- maturin develop
- python -m unittest discover tests
-
- # test the arrow crate builds against wasm32 in stable rust
- wasm32-build:
- name: Build wasm32 on AMD64 Rust ${{ matrix.rust }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [nightly-2021-03-24]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep
memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- ARROW_TEST_DATA: /__w/arrow/arrow/testing/data
- PARQUET_TEST_DATA: /__w/arrow/arrow/cpp/submodules/parquet-testing/data
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /github/home/target
- key: ${{ runner.os }}-${{ matrix.arch }}-target-wasm32-cache-${{
matrix.rust }}
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup override set ${{ matrix.rust }}
- rustup component add rustfmt
- rustup target add wasm32-unknown-unknown
- - name: Build arrow crate
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cd rust/arrow
- cargo build --target wasm32-unknown-unknown
-
- # test the projects can build without default features
- default-build:
- name: Check No Defaults on AMD64 Rust ${{ matrix.rust }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [stable]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep
memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- ARROW_TEST_DATA: /__w/arrow/arrow/testing/data
- PARQUET_TEST_DATA: /__w/arrow/arrow/cpp/submodules/parquet-testing/data
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /github/home/target
- key: ${{ runner.os }}-${{ matrix.arch }}-target-wasm32-cache-${{
matrix.rust }}
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup override set ${{ matrix.rust }}
- rustup component add rustfmt
- - name: Build arrow crate
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cd rust/arrow
- cargo check --all-targets --no-default-features
diff --git a/.gitignore b/.gitignore
index 5397fe3..31bdf49 100644
--- a/.gitignore
+++ b/.gitignore
@@ -85,3 +85,9 @@ cpp/Brewfile.lock.json
target
Cargo.lock
+rusty-tags.vi
+.history
+.flatbuffers/
+
+.vscode
+venv/*
diff --git a/.gitmodules b/.gitmodules
index 86e2f06..ec5d620 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,6 +1,6 @@
-[submodule "testing"]
- path = arrow-testing
- url = https://github.com/apache/arrow-testing
[submodule "parquet-testing"]
path = parquet-testing
url = https://github.com/apache/parquet-testing.git
+[submodule "testing"]
+ path = testing
+ url = https://github.com/apache/arrow-testing
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 2cf70cc..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,165 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-dist: bionic
-
-language: minimal
-
-cache:
- directories:
- - $TRAVIS_BUILD_DIR/.docker
-
-addons:
- apt:
- packages:
- - python3-pip
-
-services:
- - docker
-
-# Note that the global "env" setting isn't inherited automatically by
-# matrix entries with their own "env", so we have to insert it explicitly.
-env: &global_env
- ARROW_ENABLE_TIMING_TESTS: "OFF"
- COMPOSE_DOCKER_CLI_BUILD: 1
- DOCKER_BUILDKIT: 0
- DOCKER_VOLUME_PREFIX: $TRAVIS_BUILD_DIR/.docker/
-
-jobs:
- include:
- - name: "C++ on ARM"
- os: linux
- arch: arm64
- env:
- <<: *global_env
- ARCH: arm64v8
- ARROW_CI_MODULES: "CPP"
- DOCKER_IMAGE_ID: ubuntu-cpp
- # ARROW_USE_GLOG=OFF is needed to avoid build error caused by
- # glog and CMAKE_UNITY_BUILD=ON.
- #
- # Disable ARROW_S3 because it often causes "No output has
- # been received in the last 10m0s, this potentially indicates
- # a stalled build or something wrong with the build itself."
- # on Travis CI.
- #
- # Limiting CPP_MAKE_PARALLELISM is required to avoid random compiler
- # crashes.
- DOCKER_RUN_ARGS: >-
- "
- -e ARROW_BUILD_STATIC=OFF
- -e ARROW_ORC=OFF
- -e ARROW_S3=OFF
- -e ARROW_USE_GLOG=OFF
- -e CMAKE_UNITY_BUILD=ON
- -e CPP_MAKE_PARALLELISM=4
- "
- # The LLVM's APT repository provides only arm64 binaries.
- # We should use LLVM provided by Ubuntu.
- LLVM: "10"
- UBUNTU: "20.04"
-
- - name: "C++ on s390x"
- os: linux
- arch: s390x
- env:
- <<: *global_env
- ARCH: s390x
- ARROW_CI_MODULES: "CPP"
- DOCKER_IMAGE_ID: ubuntu-cpp
- # Can't enable ARROW_MIMALLOC because of failures in memory pool tests.
- # Can't enable ARROW_S3 because compiler is killed while compiling
- # aws-sdk-cpp.
- DOCKER_RUN_ARGS: >-
- "
- -e ARROW_BUILD_STATIC=OFF
- -e ARROW_FLIGHT=ON
- -e ARROW_MIMALLOC=OFF
- -e ARROW_ORC=OFF
- -e ARROW_PARQUET=OFF
- -e ARROW_S3=OFF
- -e CMAKE_UNITY_BUILD=ON
- -e CPP_MAKE_PARALLELISM=4
- -e PARQUET_BUILD_EXAMPLES=OFF
- -e PARQUET_BUILD_EXECUTABLES=OFF
- -e Protobuf_SOURCE=BUNDLED
- -e cares_SOURCE=BUNDLED
- -e gRPC_SOURCE=BUNDLED
- "
- # The LLVM's APT repository provides only arm64 binaries.
- # We should use LLVM provided by Ubuntu.
- LLVM: "10"
- UBUNTU: "20.04"
-
- - name: "Go on s390x"
- os: linux
- arch: s390x
- env:
- <<: *global_env
- ARCH: s390x
- ARROW_CI_MODULES: "GO"
- DOCKER_IMAGE_ID: debian-go
-
- - name: "Java on s390x"
- os: linux
- arch: s390x
- env:
- <<: *global_env
- ARCH: s390x
- ARROW_CI_MODULES: "JAVA"
- DOCKER_IMAGE_ID: debian-java
- JDK: 11
-
- allow_failures:
- - name: "Go on s390x"
- - name: "Java on s390x"
-
-before_install:
- - eval "$(python ci/detect-changes.py)"
- - |
- arrow_ci_affected=no
- for arrow_ci_module in ${ARROW_CI_MODULES}; do
- arrow_ci_affected_variable=ARROW_CI_${arrow_ci_module}_AFFECTED
- if [ "$(eval "echo \$${arrow_ci_affected_variable}")" = "1" ]; then
- arrow_ci_affected=yes
- fi
- done
- if [ "${arrow_ci_affected}" = "no" ]; then
- travis_terminate 0
- fi
-
-install:
- - pip3 install -e dev/archery[docker]
-
-script:
- - sudo sysctl -w kernel.core_pattern="core.%e.%p"
- # This isn't allowed on Travis CI:
- # /home/travis/.travis/functions: line 109: ulimit: core file size: cannot
modify limit: Operation not permitted
- - |
- ulimit -c unlimited || :
- - |
- archery docker run \
- ${DOCKER_RUN_ARGS} \
- --volume ${PWD}/build:/build \
- ${DOCKER_IMAGE_ID}
-
-after_success:
- - |
- if [ "${TRAVIS_EVENT_TYPE}" = "push" -a \
- "${TRAVIS_REPO_SLUG}" = "apache/arrow" ]; then
- archery docker push ${DOCKER_IMAGE_ID} || :
- fi
diff --git a/ballista/rust/benchmarks/tpch/README.md
b/ballista/rust/benchmarks/tpch/README.md
index 6d77694..20c4fc7 100644
--- a/ballista/rust/benchmarks/tpch/README.md
+++ b/ballista/rust/benchmarks/tpch/README.md
@@ -41,7 +41,7 @@ To run the benchmarks it is necessary to have at least one
Ballista scheduler an
To run the scheduler from source:
```bash
-cd $ARROW_HOME/rust/ballista/rust/scheduler
+cd $ARROW_HOME/ballista/rust/scheduler
RUST_LOG=info cargo run --release
```
@@ -50,7 +50,7 @@ By default the scheduler will bind to `0.0.0.0` and listen on
port 50050.
To run the executor from source:
```bash
-cd $ARROW_HOME/rust/ballista/rust/executor
+cd $ARROW_HOME/ballista/rust/executor
RUST_LOG=info cargo run --release
```
@@ -65,7 +65,7 @@ RUST_LOG=info RUSTFLAGS='-C target-cpu=native -C lto -C
codegen-units=1 -C embed
To run the benchmarks:
```bash
-cd $ARROW_HOME/rust/ballista/rust/benchmarks/tpch
+cd $ARROW_HOME/ballista/rust/benchmarks/tpch
cargo run --release benchmark --host localhost --port 50050 --query 1 --path
$(pwd)/data --format tbl
```
diff --git a/dev/.gitignore b/dev/.gitignore
index b079293..399c309 100644
--- a/dev/.gitignore
+++ b/dev/.gitignore
@@ -18,3 +18,4 @@
# Python virtual environments for dev tools
.venv*/
+__pycache__
diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py
index bcaddf1..4bbde75 100644
--- a/dev/archery/archery/cli.py
+++ b/dev/archery/archery/cli.py
@@ -289,8 +289,7 @@ def decorate_lint_command(cmd):
@archery.command(short_help="Check Arrow source tree for errors")
[email protected]("--src", metavar="<arrow_src>", default=None,
- callback=validate_arrow_sources,
[email protected]("--src", metavar="<arrow_src>", default=".",
help="Specify Arrow source directory")
@click.option("--fix", is_flag=True, type=BOOL, default=False,
help="Toggle fixing the lint errors if the linter supports it.")
@@ -301,6 +300,8 @@ def decorate_lint_command(cmd):
@decorate_lint_command
@click.pass_context
def lint(ctx, src, fix, iwyu_all, **checks):
+ src = ArrowSources(src)
+
if checks.pop('all'):
# "--all" is given => enable all non-selected checks
for k, v in checks.items():
diff --git a/dev/archery/archery/utils/source.py
b/dev/archery/archery/utils/source.py
index d30b4f1..1ae0fe0 100644
--- a/dev/archery/archery/utils/source.py
+++ b/dev/archery/archery/utils/source.py
@@ -45,13 +45,7 @@ class ArrowSources:
----------
path : src
"""
- path = Path(path)
- # validate by checking a specific path in the arrow source tree
- if not (path / 'cpp' / 'CMakeLists.txt').exists():
- raise InvalidArrowSource(
- "No Arrow C++ sources found in {}.".format(path)
- )
- self.path = path
+ self.path = Path(path)
@property
def archery(self):
diff --git a/dev/release/rat_exclude_files.txt
b/dev/release/rat_exclude_files.txt
index 68f5668..ead9c8d 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -12,37 +12,6 @@ ci/etc/rprofile
ci/etc/*.patch
ci/vcpkg/*.patch
CHANGELOG.md
-cpp/CHANGELOG_PARQUET.md
-cpp/src/arrow/io/mman.h
-cpp/src/arrow/util/random.h
-cpp/src/arrow/status.cc
-cpp/src/arrow/status.h
-cpp/src/arrow/vendored/*
-cpp/build-support/asan_symbolize.py
-cpp/build-support/cpplint.py
-cpp/build-support/lint_exclusions.txt
-cpp/build-support/iwyu/*
-cpp/cmake_modules/FindPythonLibsNew.cmake
-cpp/cmake_modules/SnappyCMakeLists.txt
-cpp/cmake_modules/SnappyConfig.h
-cpp/examples/parquet/parquet-arrow/cmake_modules/FindArrow.cmake
-cpp/src/parquet/.parquetcppversion
-cpp/src/generated/parquet_constants.cpp
-cpp/src/generated/parquet_constants.h
-cpp/src/generated/parquet_types.cpp
-cpp/src/generated/parquet_types.h
-cpp/src/plasma/thirdparty/ae/ae.c
-cpp/src/plasma/thirdparty/ae/ae.h
-cpp/src/plasma/thirdparty/ae/ae_epoll.c
-cpp/src/plasma/thirdparty/ae/ae_evport.c
-cpp/src/plasma/thirdparty/ae/ae_kqueue.c
-cpp/src/plasma/thirdparty/ae/ae_select.c
-cpp/src/plasma/thirdparty/ae/config.h
-cpp/src/plasma/thirdparty/ae/zmalloc.h
-cpp/src/plasma/thirdparty/dlmalloc.c
-cpp/thirdparty/flatbuffers/include/flatbuffers/base.h
-cpp/thirdparty/flatbuffers/include/flatbuffers/flatbuffers.h
-cpp/thirdparty/flatbuffers/include/flatbuffers/stl_emulation.h
dev/requirements*.txt
dev/archery/MANIFEST.in
dev/archery/requirements*.txt
@@ -115,56 +84,11 @@ dev/tasks/linux-packages/apache-arrow/debian/source/format
dev/tasks/linux-packages/apache-arrow/debian/watch
dev/tasks/requirements*.txt
dev/tasks/conda-recipes/*
-docs/requirements.txt
-go/arrow/flight/Flight_grpc.pb.go
-go/arrow/go.sum
-go/arrow/Gopkg.lock
-go/arrow/flight/Flight.pb.go
-go/arrow/flight/Flight_grpc.pb.go
-go/arrow/internal/cpu/*
-go/arrow/type_string.go
-go/*.tmpldata
-go/*.s
-go/parquet/go.sum
-go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go
-go/parquet/internal/gen-go/parquet/parquet-consts.go
-go/parquet/internal/gen-go/parquet/parquet.go
-js/.npmignore
-js/closure-compiler-scripts/*
-js/src/fb/*.ts
-js/yarn.lock
-js/.eslintignore
-python/cmake_modules
-python/cmake_modules/FindPythonLibsNew.cmake
-python/cmake_modules/SnappyCMakeLists.txt
-python/cmake_modules/SnappyConfig.h
-python/MANIFEST.in
-python/manylinux1/.dockerignore
-python/pyarrow/includes/__init__.pxd
-python/pyarrow/tests/__init__.py
-python/pyarrow/vendored/*
-python/requirements*.txt
pax_global_header
MANIFEST.in
__init__.pxd
__init__.py
requirements.txt
-csharp/.gitattributes
-csharp/dummy.git/*
-csharp/src/Apache.Arrow/Flatbuf/*
-csharp/Apache.Arrow.sln
-csharp/examples/FluentBuilderExample/FluentBuilderExample.csproj
-csharp/examples/Examples.sln
-csharp/src/Apache.Arrow/Apache.Arrow.csproj
-csharp/src/Apache.Arrow/Properties/Resources.Designer.cs
-csharp/src/Apache.Arrow/Properties/Resources.resx
-csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj
-csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj
-csharp/test/Apache.Arrow.Tests/app.config
-csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj
-csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj
-csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj
-csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj
*.html
*.sgml
*.css
@@ -173,35 +97,8 @@
csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj
*.svg
*.devhelp2
*.scss
-r/R/arrowExports.R
-r/src/arrowExports.cpp
-r/DESCRIPTION
-r/LICENSE.md
-r/NAMESPACE
-r/.Rbuildignore
-r/arrow.Rproj
-r/README.md
-r/README.Rmd
-r/man/*.Rd
-r/cran-comments.md
-r/vignettes/*.Rmd
-r/tests/testthat/test-*.txt
-r/inst/include/cpp11.hpp
-r/inst/include/cpp11/*.hpp
.gitattributes
-ruby/red-arrow/.yardopts
-rust/arrow/test/data/*.csv
-rust/rust-toolchain
-rust/arrow-flight/src/arrow.flight.protocol.rs
-julia/Arrow/Project.toml
-julia/Arrow/README.md
-julia/Arrow/docs/Manifest.toml
-julia/Arrow/docs/Project.toml
-julia/Arrow/docs/make.jl
-julia/Arrow/docs/mkdocs.yml
-julia/Arrow/docs/src/index.md
-julia/Arrow/docs/src/manual.md
-julia/Arrow/docs/src/reference.md
-rust/ballista/rust/benchmarks/tpch/queries/q*.sql
-rust/ballista/rust/scheduler/testdata/*
-rust/ballista/ui/scheduler/yarn.lock
+rust-toolchain
+ballista/rust/benchmarks/tpch/queries/q*.sql
+ballista/rust/scheduler/testdata/*
+ballista/ui/scheduler/yarn.lock
diff --git a/arrow-testing b/testing
similarity index 100%
rename from arrow-testing
rename to testing
