This is an automated email from the ASF dual-hosted git repository.
jonkeane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 6d03215543 GH-41450: [R][CI] rhub/container follow ons (#41451)
6d03215543 is described below
commit 6d0321554374523ae0633d6bfe42cdeeb3b5d145
Author: Jonathan Keane <[email protected]>
AuthorDate: Sun May 12 11:00:26 2024 -0400
GH-41450: [R][CI] rhub/container follow ons (#41451)
More CI changes:
* GitHub Issue: #41450 (specifically use the rhub containers approach for
clang sanitizer, remove some of our work arounds)
* Remove CentOS 7 CI support for R
Authored-by: Jonathan Keane <[email protected]>
Signed-off-by: Jonathan Keane <[email protected]>
---
.env | 3 ---
.github/workflows/r.yml | 3 +--
ci/docker/linux-r.dockerfile | 3 ---
ci/scripts/java_jni_manylinux_build.sh | 3 ---
ci/scripts/r_docker_configure.sh | 20 --------------------
ci/scripts/r_sanitize.sh | 2 ++
ci/scripts/r_test.sh | 3 ---
dev/tasks/r/azure.linux.yml | 1 -
dev/tasks/r/github.packages.yml | 7 +++----
dev/tasks/tasks.yml | 13 ++-----------
docker-compose.yml | 16 ++++++----------
r/tools/test-nixlibs.R | 4 ----
r/tools/ubsan.supp | 1 +
r/vignettes/install.Rmd | 33 ---------------------------------
14 files changed, 15 insertions(+), 97 deletions(-)
diff --git a/.env b/.env
index ab2e4b4fbe..27474b2c73 100644
--- a/.env
+++ b/.env
@@ -86,9 +86,6 @@ ARROW_R_DEV=TRUE
R_PRUNE_DEPS=FALSE
TZ=UTC
-# Any non-empty string will install devtoolset-${DEVTOOLSET_VERSION}
-DEVTOOLSET_VERSION=
-
# Used through docker-compose.yml and serves as the default version for the
# ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
# docker tags more readable.
diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
index 8228aaad7c..aba7734765 100644
--- a/.github/workflows/r.yml
+++ b/.github/workflows/r.yml
@@ -192,12 +192,11 @@ jobs:
fail-fast: false
matrix:
config:
- - { org: "rhub", image: "ubuntu-gcc12", tag: "latest", devtoolset:
"" }
+ - { org: "rhub", image: "ubuntu-gcc12", tag: "latest" }
env:
R_ORG: ${{ matrix.config.org }}
R_IMAGE: ${{ matrix.config.image }}
R_TAG: ${{ matrix.config.tag }}
- DEVTOOLSET_VERSION: ${{ matrix.config.devtoolset }}
steps:
- name: Checkout Arrow
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #
v4.0.0
diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile
index d368a6629c..7b7e989adc 100644
--- a/ci/docker/linux-r.dockerfile
+++ b/ci/docker/linux-r.dockerfile
@@ -27,9 +27,6 @@ ENV R_BIN=${r_bin}
ARG r_dev=FALSE
ENV ARROW_R_DEV=${r_dev}
-ARG devtoolset_version=
-ENV DEVTOOLSET_VERSION=${devtoolset_version}
-
ARG r_prune_deps=FALSE
ENV R_PRUNE_DEPS=${r_prune_deps}
diff --git a/ci/scripts/java_jni_manylinux_build.sh
b/ci/scripts/java_jni_manylinux_build.sh
index da4987d307..4921ce170b 100755
--- a/ci/scripts/java_jni_manylinux_build.sh
+++ b/ci/scripts/java_jni_manylinux_build.sh
@@ -35,9 +35,6 @@ echo "=== Clear output directories and leftovers ==="
rm -rf ${build_dir}
echo "=== Building Arrow C++ libraries ==="
-devtoolset_version=$(rpm -qa "devtoolset-*-gcc" --queryformat %{VERSION} | \
- grep -o "^[0-9]*")
-devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}"
: ${ARROW_ACERO:=ON}
export ARROW_ACERO
: ${ARROW_BUILD_TESTS:=ON}
diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh
index 52db2e6df6..8a962fe576 100755
--- a/ci/scripts/r_docker_configure.sh
+++ b/ci/scripts/r_docker_configure.sh
@@ -67,26 +67,6 @@ sloppiness = include_file_ctime
hash_dir = false" >> ~/.ccache/ccache.conf
fi
-# Special hacking to try to reproduce quirks on centos using non-default build
-# tooling.
-if [[ -n "$DEVTOOLSET_VERSION" ]]; then
- $PACKAGE_MANAGER install -y centos-release-scl
- $PACKAGE_MANAGER install -y "devtoolset-$DEVTOOLSET_VERSION"
-
- # Enable devtoolset here so that `which gcc` finds the right compiler below
- source /opt/rh/devtoolset-${DEVTOOLSET_VERSION}/enable
-
- # Build images which require the devtoolset don't have CXX17 variables
- # set as the system compiler doesn't support C++17
- if [ ! "`{R_BIN} CMD config CXX17`" ]; then
- mkdir -p ~/.R
- echo "CC = $(which gcc) -fPIC" >> ~/.R/Makevars
- echo "CXX17 = $(which g++) -fPIC" >> ~/.R/Makevars
- echo "CXX17STD = -std=c++17" >> ~/.R/Makevars
- echo "CXX17FLAGS = ${CXX11FLAGS}" >> ~/.R/Makevars
- fi
-fi
-
if [ -f "${ARROW_SOURCE_HOME}/ci/scripts/r_install_system_dependencies.sh" ];
then
"${ARROW_SOURCE_HOME}/ci/scripts/r_install_system_dependencies.sh"
fi
diff --git a/ci/scripts/r_sanitize.sh b/ci/scripts/r_sanitize.sh
index f7ed07f0c8..fb3e9a5836 100755
--- a/ci/scripts/r_sanitize.sh
+++ b/ci/scripts/r_sanitize.sh
@@ -46,6 +46,8 @@ unset ARROW_R_DEV
export ARROW_R_VERBOSE_TEST=TRUE
export
UBSAN_OPTIONS="print_stacktrace=1,suppressions=/arrow/r/tools/ubsan.supp"
+# From the old rhub image
https://github.com/r-hub/rhub-linux-builders/blob/master/fedora-clang-devel-san/Dockerfile
+export
ASAN_OPTIONS="alloc_dealloc_mismatch=0:detect_leaks=0:detect_odr_violation=0"
# run tests
pushd tests
diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh
index 95a49ee83a..e13da45e2d 100755
--- a/ci/scripts/r_test.sh
+++ b/ci/scripts/r_test.sh
@@ -48,9 +48,6 @@ if [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then
fi
export _R_CHECK_COMPILATION_FLAGS_KNOWN_="${_R_CHECK_COMPILATION_FLAGS_KNOWN_}
${ARROW_R_CXXFLAGS}"
-# These should generally be picked up, but are slightly wrong in rhub's
containers it appears
-# https://github.com/r-hub/containers/pull/63
-export _R_CHECK_COMPILATION_FLAGS_KNOWN_="${_R_CHECK_COMPILATION_FLAGS_KNOWN_}
-Wno-parentheses -Werror=format-security -Wp,-D_FORTIFY_SOURCE=3"
if [ "$ARROW_R_DEV" = "TRUE" ]; then
# These are sometimes used in the Arrow C++ build and are not a problem
diff --git a/dev/tasks/r/azure.linux.yml b/dev/tasks/r/azure.linux.yml
index e26a59629f..28893a8172 100644
--- a/dev/tasks/r/azure.linux.yml
+++ b/dev/tasks/r/azure.linux.yml
@@ -38,7 +38,6 @@ jobs:
export R_ORG={{ r_org }}
export R_IMAGE={{ r_image }}
export R_TAG={{ r_tag }}
- export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }}
export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }}
docker-compose pull --ignore-pull-failures r
docker-compose build r
diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml
index 9ca7e59a95..41d8b230f8 100644
--- a/dev/tasks/r/github.packages.yml
+++ b/dev/tasks/r/github.packages.yml
@@ -299,14 +299,14 @@ jobs:
# choosing a binary on this OS. If libarrow_binary is TRUE, we're on
# an OS that is not in the allowlist, so we have to opt-in to use the
# binary. Other env vars used in r_docker_configure.sh can be added
- # here (like devtoolset) and wired up in the later steps.
+ # here and wired up in the later steps.
- {image: "rhub/ubuntu-clang", libarrow_binary: "TRUE"}
# fedora-clang-devel cannot use binaries bc of libc++ (uncomment to
see the error)
# - {image: "rhub/fedora-clang-devel", libarrow_binary: "TRUE"}
- {image: "rhub/ubuntu-release"} # currently ubuntu-22.04
- {image: "rocker/r-ver:4.0.0"} # ubuntu-20.04
- - {image: "rstudio/r-base:4.1-focal"} # ubuntu-20.04
- - {image: "rstudio/r-base:4.2-centos7", devtoolset: "8"}
+ - {image: "rstudio/r-base:4.1-focal"}
+ - {image: "rstudio/r-base:4.2-jammy"}
- {image: "rstudio/r-base:4.3-noble"}
steps:
# Get the arrow checkout just for the docker config scripts
@@ -317,7 +317,6 @@ jobs:
- name: Install system requirements
env:
ARROW_R_DEV: "TRUE" # To install curl/openssl in
r_docker_configure.sh
- DEVTOOLSET_VERSION: {{ '${{ matrix.config.devtoolset }}' }}
shell: bash
run: |
# Make sure R is on the path for the R-hub devel versions (where
RPREFIX is set in its dockerfile)
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 9d68e57c75..5bf5037652 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -1410,15 +1410,6 @@ tasks:
GCC_VERSION: 12
image: ubuntu-r-only-r
- test-r-rstudio-r-base-4.2-centos7-devtoolset-8:
- ci: azure
- template: r/azure.linux.yml
- params:
- r_org: rstudio
- r_image: r-base
- r_tag: 4.2-centos7
- devtoolset_version: 8
-
test-r-minimal-build:
ci: azure
template: r/azure.linux.yml
@@ -1436,13 +1427,13 @@ tasks:
R_PRUNE_DEPS: TRUE
image: ubuntu-r-sanitizer
- test-fedora-r-clang-sanitizer:
+ test-r-clang-sanitizer:
ci: github
template: docker-tests/github.linux.yml
params:
env:
R_PRUNE_DEPS: TRUE
- image: fedora-r-clang-sanitizer
+ image: r-clang-sanitizer
{% for go_version, staticcheck in [("1.21", "v0.4.7"), ("1.22", "latest")] %}
test-debian-12-go-{{ go_version }}:
diff --git a/docker-compose.yml b/docker-compose.yml
index 7a4d455dfe..a1d8f60a26 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -162,7 +162,7 @@ x-hierarchy:
- ubuntu-r-valgrind
- ubuntu-swift
- ubuntu-verify-rc
- - fedora-r-clang-sanitizer
+ - r-clang-sanitizer
- r
- r-revdepcheck
# helper services
@@ -1472,7 +1472,6 @@ services:
args:
base: ${R_ORG}/${R_IMAGE}:${R_TAG}
r_dev: ${ARROW_R_DEV}
- devtoolset_version: ${DEVTOOLSET_VERSION}
tz: ${TZ}
r_prune_deps: ${R_PRUNE_DEPS}
r_custom_ccache: ${R_CUSTOM_CCACHE}
@@ -1484,7 +1483,6 @@ services:
ARROW_R_DEV: ${ARROW_R_DEV}
# To test for CRAN release, delete ^^ these two env vars so we download
the Apache release
ARROW_USE_PKG_CONFIG: "false"
- devtoolset_version: ${DEVTOOLSET_VERSION}
volumes:
- .:/arrow:delegated
command: >
@@ -1517,19 +1515,17 @@ services:
/bin/bash -c "
/arrow/ci/scripts/r_sanitize.sh /arrow"
- fedora-r-clang-sanitizer:
- image: ${REPO}:r-rhub-fedora-clang-devel-latest
+ r-clang-sanitizer:
+ image: ${REPO}:r-rhub-clang-devel-latest
build:
context: .
dockerfile: ci/docker/linux-r.dockerfile
cache_from:
- - ${REPO}:r-rhub-fedora-clang-devel-latest
+ - ${REPO}:r-rhub-clang-devel-latest
args:
- # TODO: change this to rhub/clang-asan
- base: rhub/fedora-clang-devel-san
+ base: rhub/clang-asan
r_dev: ${ARROW_R_DEV}
- devtoolset_version: ${DEVTOOLSET_VERSION}
- r_bin: RDsan
+ r_bin: R
tz: ${TZ}
r_prune_deps: ${R_PRUNE_DEPS}
shm_size: *shm-size
diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R
index 6996f234ce..02e822c342 100644
--- a/r/tools/test-nixlibs.R
+++ b/r/tools/test-nixlibs.R
@@ -158,10 +158,6 @@ test_that("check_allowlist", {
})
test_that("find_latest_nightly()", {
- skip_if(
- getRversion() > "4.4.0",
- "long last version components (>8) fail to max on r-devel"
- )
tf <- tempfile()
tf_uri <- paste0("file://", tf)
on.exit(unlink(tf))
diff --git a/r/tools/ubsan.supp b/r/tools/ubsan.supp
index ff88cf9841..34854e79bc 100644
--- a/r/tools/ubsan.supp
+++ b/r/tools/ubsan.supp
@@ -16,3 +16,4 @@
# under the License.
vptr:include/c++/8/bits/shared_ptr_base.h
+function:cleancall.c
\ No newline at end of file
diff --git a/r/vignettes/install.Rmd b/r/vignettes/install.Rmd
index cc90c5ff08..c7b8251ccc 100644
--- a/r/vignettes/install.Rmd
+++ b/r/vignettes/install.Rmd
@@ -28,35 +28,6 @@ For `gcc`, this generally means version 7 or newer. Most
contemporary Linux
distributions have a new enough compiler; however, CentOS 7 is a notable
exception, as it ships with gcc 4.8.
-If you are on CentOS 7, to build arrow you will need to install a newer
`devtoolset`, and you'll need to update R's Makevars to define the `CXX17`
variables. This script installs `devtoolset-8` and configures R to be able to
use C++17:
-
-```
-#!/usr/bin/env bash
-
-yum install -y centos-release-scl
-yum install -y devtoolset-8
-# Optional: also install cloud storage dependencies, as described below
-yum install -y libcurl-devel openssl-devel
-
-source /opt/rh/devtoolset-8/enable
-
-if [ ! `R CMD config CXX17` ]; then
- mkdir -p ~/.R
- echo "CC = $(which gcc) -fPIC" >> ~/.R/Makevars
- echo "CXX17 = $(which g++) -fPIC" >> ~/.R/Makevars
- echo "CXX17STD = -std=c++17" >> ~/.R/Makevars
- echo "CXX17FLAGS = ${CXX11FLAGS}" >> ~/.R/Makevars
-fi
-```
-
-Note that the C++17 compiler is only required at *build* time. You don't need
-to enable the devtoolset every time you load the package. What's more, if you
-install a binary package from RStudio Package Manager (see method 1a below),
you
-do not need to set up any of this. Likewise, if you `R CMD INSTALL --build`
-arrow on a CentOS machine with the newer compilers, you can take the binary
-package it produces and install it on any other CentOS machine without those
-compilers.
-
### Libraries
Optional support for reading from cloud storage--AWS S3 and
@@ -517,10 +488,6 @@ The install script should work everywhere, so if libarrow
fails to compile,
please [report an issue](https://issues.apache.org/jira/projects/ARROW/issues)
so that we can improve the script.
-### Known installation issues
-
-* On CentOS, building the package requires a more modern `devtoolset` than the
default system compilers. See "System dependencies" above.
-
## Contributing
We are constantly working to make the installation process as painless as
