This is an automated email from the ASF dual-hosted git repository.
raulcd pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 251758d549 GH-45159: [CI][Integration] Remove substrait
consumer-testing integration job (#45463)
251758d549 is described below
commit 251758d5491afcde23bdad153b206a0706f19686
Author: Raúl Cumplido <[email protected]>
AuthorDate: Thu Feb 13 11:19:29 2025 +0100
GH-45159: [CI][Integration] Remove substrait consumer-testing integration
job (#45463)
### Rationale for this change
The job has been failing for the last two months due to upstream
refactoring. I opened an issue upstream but we didn't got any response:
- https://github.com/substrait-io/consumer-testing/issues/196
Based on the following substrait consumer-testing PR there was a big
refactor which removed the files that we were invoking on our tests:
- https://github.com/substrait-io/consumer-testing/pull/162
I asked on the corresponding issue ~3 weeks ago for feedback on the removal
and as there's no opposition I think we should remove this unmaintained job.
### What changes are included in this PR?
Remove extremely coupled `substrait-io/consumer-testing` job.
### Are these changes tested?
No
### Are there any user-facing changes?
No
* GitHub Issue: #45159
Authored-by: Raúl Cumplido <[email protected]>
Signed-off-by: Raúl Cumplido <[email protected]>
---
ci/docker/conda-python-substrait.dockerfile | 59 -----------------------------
ci/scripts/install_substrait_consumer.sh | 36 ------------------
ci/scripts/integration_substrait.sh | 31 ---------------
dev/tasks/tasks.yml | 8 ----
docker-compose.yml | 28 --------------
5 files changed, 162 deletions(-)
diff --git a/ci/docker/conda-python-substrait.dockerfile
b/ci/docker/conda-python-substrait.dockerfile
deleted file mode 100644
index 36dd64e51e..0000000000
--- a/ci/docker/conda-python-substrait.dockerfile
+++ /dev/null
@@ -1,59 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-ARG repo
-ARG arch
-ARG python=3.9
-
-FROM ${repo}:${arch}-conda-python-${python}
-
-COPY ci/conda_env_python.txt \
- ci/conda_env_sphinx.txt \
- /arrow/ci/
-
-# Note: openjdk is pinned to 17 because the
-# substrait repo currently pins to jdk 17.
-# Newer jdk versions are currently failing
-# due to the recent upgrade to Gradle 8 via
-# install_substrait_consumer.sh.
-# https://github.com/substrait-io/substrait-java/issues/274
-RUN mamba install -q -y \
- --file arrow/ci/conda_env_python.txt \
- --file arrow/ci/conda_env_sphinx.txt \
- $([ "$python" == "3.9" ] && echo "pickle5") \
- python=${python} \
- openjdk=17 \
- nomkl && \
- mamba clean --all
-
-
-ARG substrait=latest
-COPY ci/scripts/install_substrait_consumer.sh /arrow/ci/scripts/
-
-RUN /arrow/ci/scripts/install_substrait_consumer.sh
-
-ENV ARROW_ACERO=ON \
- ARROW_COMPUTE=ON \
- ARROW_CSV=ON \
- ARROW_DATASET=ON \
- ARROW_FILESYSTEM=ON \
- ARROW_FLIGHT=OFF \
- ARROW_FLIGHT_SQL=OFF \
- ARROW_GANDIVA=OFF \
- ARROW_JSON=ON \
- ARROW_SUBSTRAIT=ON \
- ARROW_TESTING=OFF
diff --git a/ci/scripts/install_substrait_consumer.sh
b/ci/scripts/install_substrait_consumer.sh
deleted file mode 100755
index 2e6d299f68..0000000000
--- a/ci/scripts/install_substrait_consumer.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-
-echo "Install Substrait Consumer Test Suite";
-
-git clone https://github.com/substrait-io/consumer-testing.git
-cd consumer-testing
-# avoid installing pyarrow
-grep -v 'pyarrow\|arrow-nightlies' requirements.txt > requirements-no-arrow.txt
-pip install -r requirements-no-arrow.txt
-
-pip install -r requirements-build.txt
-# setup substrait-java
-git submodule init
-git submodule update --init
-./build-and-copy-isthmus-shadow-jar.sh
-# install substrait_consumer library
-python setup.py install
diff --git a/ci/scripts/integration_substrait.sh
b/ci/scripts/integration_substrait.sh
deleted file mode 100755
index 152a8d9440..0000000000
--- a/ci/scripts/integration_substrait.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-
-# check that optional pyarrow modules are available
-# because pytest would just skip the substrait tests
-echo "Substrait Integration Tests"
-echo "Validating imports"
-python -c "import pyarrow.substrait"
-python -c "from substrait_consumer.consumers.acero_consumer import
AceroConsumer"
-
-echo "Executing pytest"
-cd consumer-testing
-pytest -r s
substrait_consumer/tests/functional/extension_functions/test_boolean_functions.py
--producer isthmus --consumer acero
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index ba0cdbabff..2f12faa1c5 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -1079,14 +1079,6 @@ tasks:
PYTEST_ARGS: "-m hypothesis -W error"
image: conda-python-pandas
- test-conda-python-3.10-substrait:
- ci: github
- template: docker-tests/github.linux.yml
- params:
- env:
- PYTHON: "3.10"
- image: conda-python-substrait
-
test-ubuntu-22.04-python-313-freethreading:
ci: github
template: docker-tests/github.linux.yml
diff --git a/docker-compose.yml b/docker-compose.yml
index 52e8558e05..9a969bedbe 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -128,7 +128,6 @@ x-hierarchy:
- conda-python-hdfs
- conda-python-no-numpy
- conda-python-spark
- - conda-python-substrait
- conda-verify-rc
- conan
- debian-cpp:
@@ -1475,33 +1474,6 @@ services:
/arrow/ci/scripts/python_build.sh /arrow /build &&
/arrow/ci/scripts/integration_dask.sh"]
- conda-python-substrait:
- # Usage:
- # docker compose build conda
- # docker compose build conda-cpp
- # docker compose build conda-python
- # docker compose build conda-python-substrait
- # docker compose run --rm conda-python-substrait
- image: ${REPO}:${ARCH}-conda-python-${PYTHON}
- build:
- context: .
- dockerfile: ci/docker/conda-python-substrait.dockerfile
- cache_from:
- - ${REPO}:${ARCH}-conda-python-${PYTHON}
- args:
- repo: ${REPO}
- arch: ${ARCH}
- python: ${PYTHON}
- shm_size: *shm-size
- environment:
- <<: [*common, *ccache]
- ARROW_SUBSTRAIT: "ON"
- volumes: *conda-volumes
- command:
- ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
- /arrow/ci/scripts/python_build.sh /arrow /build &&
- /arrow/ci/scripts/integration_substrait.sh"]
-
conda-python-cpython-debug:
# Usage:
# docker compose build conda
