This is an automated email from the ASF dual-hosted git repository.
apitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 51d50d7500 GH-43559: [Python][CI] Add a Crossbow job with a debug
CPython interpreter (#43565)
51d50d7500 is described below
commit 51d50d750012d3ca04127f6723c4f1e69ff4f5dc
Author: Antoine Pitrou <[email protected]>
AuthorDate: Tue Aug 6 09:41:19 2024 +0200
GH-43559: [Python][CI] Add a Crossbow job with a debug CPython interpreter
(#43565)
### Rationale for this change
Debug builds of CPython help catch low-level errors when using the Python C
API. This is illustrated in GH-43487: a debug build of CPython detected that we
were incref'ing a Python object without holding the GIL (which is a race
condition otherwise).
### What changes are included in this PR?
1. Add a Docker build with a conda-installed debug interpreter.
2. Add a Crossbow job to run said Docker build with Python 3.12.
### Are these changes tested?
Yes, by the adding Crossbow job. The job now fails with a crash in
`test_udf.py`, because of GH-43487.
### Are there any user-facing changes?
No.
* GitHub Issue: #43559
Authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
ci/docker/conda-python-cpython-debug.dockerfile | 28 +++++++++++++++++++++++++
dev/tasks/tasks.yml | 8 +++++++
docker-compose.yml | 25 ++++++++++++++++++++++
3 files changed, 61 insertions(+)
diff --git a/ci/docker/conda-python-cpython-debug.dockerfile
b/ci/docker/conda-python-cpython-debug.dockerfile
new file mode 100644
index 0000000000..87bdcafe40
--- /dev/null
+++ b/ci/docker/conda-python-cpython-debug.dockerfile
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG repo
+ARG arch
+ARG python=3.8
+FROM ${repo}:${arch}-conda-python-${python}
+
+# (Docker oddity: ARG needs to be repeated after FROM)
+ARG python=3.8
+RUN mamba install -y
"conda-forge/label/python_debug::python=${python}[build=*_cpython]" && \
+ mamba clean --all
+# Quick check that we do have a debug mode CPython
+RUN python -c "import sys; sys.gettotalrefcount()"
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 3b00bc0040..07a4d638f1 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -1205,6 +1205,14 @@ tasks:
image: conda-python
{% endfor %}
+ test-conda-python-3.12-cpython-debug:
+ ci: github
+ template: docker-tests/github.linux.yml
+ params:
+ env:
+ PYTHON: 3.12
+ image: conda-python-cpython-debug
+
test-conda-python-emscripten:
ci: github
template: docker-tests/github.linux.yml
diff --git a/docker-compose.yml b/docker-compose.yml
index cf22324f7c..daa5c74bcb 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -119,6 +119,7 @@ x-hierarchy:
- conda-python:
- conda-python-pandas:
- conda-python-docs
+ - conda-python-cpython-debug
- conda-python-cython2
- conda-python-dask
- conda-python-emscripten
@@ -1440,6 +1441,30 @@ services:
volumes: *conda-volumes
command: *python-conda-command
+ conda-python-cpython-debug:
+ # Usage:
+ # docker-compose build conda
+ # docker-compose build conda-cpp
+ # docker-compose build conda-python
+ # docker-compose build conda-python-cpython-debug
+ # docker-compose run --rm conda-python-cpython-debug
+ image: ${REPO}:${ARCH}-conda-python-${PYTHON}-cpython-debug
+ build:
+ context: .
+ dockerfile: ci/docker/conda-python-cpython-debug.dockerfile
+ cache_from:
+ - ${REPO}:${ARCH}-conda-python-${PYTHON}-cpython-debug
+ args:
+ repo: ${REPO}
+ arch: ${ARCH}
+ python: ${PYTHON}
+ shm_size: *shm-size
+ environment:
+ <<: [*common, *ccache]
+ PYTEST_ARGS: # inherit
+ volumes: *conda-volumes
+ command: *python-conda-command
+
################################## R ########################################
ubuntu-r:
