jonkeane commented on code in PR #13149:
URL: https://github.com/apache/arrow/pull/13149#discussion_r880819817
##########
dev/tasks/macros.jinja:
##########
@@ -221,3 +222,79 @@ on:
cp ${formula} $(brew --repository homebrew/core)/Formula/
done
{% endmacro %}
+
+{%- macro github_change_r_pkg_version(is_fork, version = '\\2.\'\"$(date
+%Y%m%d)\"\'' ) -%}
+ - name: Modify version
+ shell: bash
+ run: |
+ cd arrow/r
+ sed -i.bak -E -e \
+ 's/(^Version: )([0-9]+\.[0-9]+\.[0-9]+).*$/\1{{ version }}/' \
+ DESCRIPTION
+ head DESCRIPTION
+ rm -f DESCRIPTION.bak
+ cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
tools/apache-arrow.rb
+
+ # Pin the git commit in the formula to match
+ cd tools
+ if [ "{{ is_fork }}" == "true" ]; then
+ sed -i.bak -E -e 's/apache\/arrow.git"$/{{
arrow.github_repo.split("/") | join("\/") }}.git", :revision => "'"{{
arrow.head }}"'"/' apache-arrow.rb
+ else
+ sed -i.bak -E -e 's/arrow.git"$/arrow.git", :revision => "'"{{
arrow.head }}"'"/' apache-arrow.rb
+ fi
+ rm -f apache-arrow.rb.bak
+{% endmacro %}
+
+{%- macro github_test_r_src_pkg() -%}
+
source("https://raw.githubusercontent.com/apache/arrow/master/ci/etc/rprofile";)
+
+ install.packages(
+ "arrow",
+ repos = c(getOption("arrow.dev_repo"), "https://cloud.r-project.org";),
+ verbose = TRUE
+ )
+
+ library(arrow)
+ read_parquet(system.file("v0.7.1.parquet", package = "arrow"))
+
+ # Our Version should always be > CRAN so we would detect a CRAN version here.
+ stopifnot(packageVersion("arrow") == {{
'"${{needs.source.outputs.version}}"' }})
+{% endmacro %}
+
+{%- macro github_setup_local_r_repo(is_fork, os, get_bin = 'true') -%}
+ - name: Setup local repo
+ if: {{ is_fork }}
+ shell: bash
+ run: mkdir repo
+ - name: Get windows binary
+ if: {{ '${{' }} {{ is_fork }} && ({{ os }} == 'windows-latest') && {{
get_bin }} {{ '}}' }}
+ uses: actions/download-artifact@v3
+ with:
+ name: r-windows-libarrow
+ path: repo/libarrow/bin/windows
+ - name: Get ubuntu binary
+ if: {{ '${{' }} {{ is_fork }} && ({{ os }} != 'windows-latest') && {{
get_bin }} {{ '}}' }}
+ uses: actions/download-artifact@v3
+ with:
+ name: r-ubuntu-libarrow
+ path: repo/libarrow/bin/ubuntu-18.04
+ - name: Get centos binary
+ if: {{ '${{' }} {{ is_fork }} && ({{ os }} != 'windows-latest') && {{
get_bin }} {{ '}}' }}
+ uses: actions/download-artifact@v3
+ with:
+ name: r-centos-libarrow
+ path: repo/libarrow/bin/centos-7
Review Comment:
Do these both run when the `os` is not windows-latest? Should they be run
conditionally or collapsed into one?
##########
dev/tasks/r/github.nightly.yml:
##########
@@ -0,0 +1,467 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% import 'macros.jinja' as macros with context %}
+# root of the R nightly repo where the first dir is the mount point used by
davfs2 ("nightly")
+# TODO update with nightlies.apache.org/arrow/r
+{% set r_root = "nightly/r" %}
+{% set webdav_domain = "https://nightly.wujciak.de:8080"; %}
+{% set repo_domain = "https://nightly.wujciak.de/r"; %}
+# This allows us to set a custom version via param:
+# crossbow submit --param custom_version=8.5.3 r-nightly-packages
+# if the param is unset defaults to the usual Ymd naming scheme
+{% set version = custom_version|default('\\2.\'\"$(date +%Y%m%d)\"\'') %}
+# We need this as boolean and string
+{% set is_upstream_b = arrow.github_repo == 'apache/arrow' %}
+# use filter to cast to string and convert to lowercase to match yaml boolean
+{% set is_fork = (not is_upstream_b)|lower %}
+{% set is_upstream = is_upstream_b|lower %}
+{% set upload_pkg = "env.UPLOAD_PKG == 'true' && success() && " + is_upstream
%}
+{% set upload_libarrow = "env.UPLOAD_LIBARROW == 'true' && success() && " +
is_upstream %}
+
+{{ macros.github_header() }}
+
+env:
+ # prevent upload of R packages or libarrow binaries via param (e.g. for
release)
+ # crossbow submit --param upload_pkgs=false --param upload_libs=false
r-nightly-packages
+ UPLOAD_PKG: {{ upload_pkgs|default("true") }}
+ UPLOAD_LIBARROW: {{ upload_libs|default("true")}}
+
+jobs:
+ source:
+ # This job will change the version to either the custom_version param or
YMD format.
+ # The output allows other steps to use the exact version to prevent issues
(e.g. date changes during run)
+ name: Source Package
+ runs-on: ubuntu-latest
+ outputs:
+ version: {{ '${{ steps.save-version.outputs.version }}' }}
+ steps:
+ {{ macros.github_checkout_arrow()|indent }}
+ {{ macros.github_change_r_pkg_version(is_fork, version)|indent }}
+ - name: Save Version
+ id: save-version
+ shell: bash
+ run: |
+ echo "::set-output name=version::$(grep ^Version arrow/r/DESCRIPTION
| sed s/Version:\ //)"
+
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ install-r: false
+
+ - name: Build R source package
+ shell: bash
+ run: |
+ cd arrow/r
+ # Copy in the Arrow C++ source
+ make sync-cpp
+ R CMD build --no-build-vignettes .
+
+ - name: Install davfs2 & Mount Repository
+ if: {{ '${{ ' + upload_pkg + ' }}' }}
+ run: |
+ sudo apt update && sudo apt install davfs2
+
+ # Create mount point
+ mkdir nightly
+
+ # make credentials available
+ sudo bash -c 'echo "{{ webdav_domain }} {{ '${{
secrets.CROSSBOW_NIGHTLIES_USER }} ${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }}"
>> /etc/davfs2/secrets'
+
+ # mount with read&write access
+ sudo mount -t davfs {{ webdav_domain }} nightly -o rw
+
+ - name: Upload Source Package
+ if: {{ '${{ ' + upload_pkg + ' }}' }}
+ shell: bash
+ run: |
+ # ensure repo structure is set up, this job is the fastest and
should fix any issue before the other jobs try
+ # to push into non existent folder with curl, which would fail
silently.
+ # Add any new architectures or R versions here
+ sudo mkdir -p {{ r_root }}/src/contrib
+ {% for os in ["ubuntu-18.04",
+ "centos-7",
+ "windows"] %}
+ sudo mkdir -p {{ r_root }}/libarrow/bin/{{ os }}
+ {% endfor %}
+ {% for os in ["windows", "macosx", "macosx/big-sur-arm64"] %}
+ {% for r_version in ["4.1", "4.2"] %}
+ sudo mkdir -p {{ r_root }}/bin/{{ os }}/contrib/{{ r_version }}
+ {% endfor %}
+ {% endfor %}
+ sudo cp arrow/r/arrow_*.tar.gz {{ r_root }}/src/contrib
+ - name: Update Repository Index
+ if: {{ '${{ ' + upload_pkg + ' }}' }}
+ shell: sudo Rscript {0}
+ run: |
+ if(file.exists("{{ r_root }}/src/contrib/PACKAGES")) {
+ tools::update_PACKAGES("{{ r_root }}/src/contrib" ,
+ type = "source",
+ latestOnly = FALSE,
+ verbose = TRUE)
+ } else {
+ tools::write_PACKAGES("{{ r_root }}/src/contrib" ,
+ type = "source",
+ latestOnly = FALSE,
+ verbose = TRUE)
+ }
+
+ - name: Unmount Repository
+ # ensures all changes are written
+ if: {{ '${{ ' + upload_pkg + ' }}' }}
+ shell: bash
+ run: sudo umount nightly
+
+ - name: Upload package artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: r-src-pkg
+ path: arrow/r/arrow_*.tar.gz
+
+ linux-cpp:
+ name: C++ Binary {{ '${{ matrix.config.os }}-${{ matrix.config.version }}'
}}
+ runs-on: ubuntu-latest
+ needs: source
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ - { os: ubuntu, version: "18.04" }
+ - { os: centos, version: "7" }
+ env:
+ UBUNTU: {{ '${{ matrix.config.version }}' }}
+ steps:
+ {{ macros.github_checkout_arrow()|indent }}
+ {{ macros.github_change_r_pkg_version(is_fork, '${{
needs.source.outputs.version }}')|indent }}
+ {{ macros.github_install_archery()|indent }}
+ - name: Build libarrow
+ shell: bash
+ run: |
+ sudo sysctl -w kernel.core_pattern="core.%e.%p"
+ ulimit -c unlimited
+ archery docker run {{ '${{ matrix.config.os}}' }}-cpp-static
+ - name: Bundle libarrow
+ shell: bash
+ env:
+ PKG_FILE: arrow-{{ '${{ needs.source.outputs.version }}' }}.zip
+ VERSION: {{ '${{ needs.source.outputs.version }}' }}
+ run: |
+ cd arrow/r/libarrow/dist
+ # These files were created by the docker user so we have to sudo to
get them
+ sudo -E zip -r $PKG_FILE lib/ include/
+
+ - name: Upload libarrow
+ if: {{ '${{ ' + upload_libarrow + ' }}' }}
+ shell: bash
+ env:
+ USER: {{ '${{ secrets.CROSSBOW_NIGHTLIES_USER }}' }}
+ PW: {{ '${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }}
+ PKG_FILE: arrow-{{ '${{ needs.source.outputs.version }}' }}.zip
+ run: |
+ cd arrow/r/libarrow/dist
+ export REPO_PATH={{ 'r/libarrow/bin/${{ matrix.config.os }}-${{
matrix.config.version }}' }}
+ curl -s --fail --show-error -u $USER:$PW -T $PKG_FILE {{
webdav_domain }}/$REPO_PATH/
+
+ - name: Upload binary artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: r-{{ '${{ matrix.config.os}}' }}-libarrow
+ path: arrow/r/libarrow/dist/arrow-*.zip
+
+ windows-cpp:
+ name: C++ Binary Windows RTools (40 only)
+ needs: source
+ runs-on: windows-latest
+ steps:
+ - run: git config --global core.autocrlf false
+ {{ macros.github_checkout_arrow()|indent }}
+ {{ macros.github_change_r_pkg_version(is_fork, '${{
needs.source.outputs.version }}')|indent }}
+
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ rtools-version: 40
+ r-version: "4.0"
+ Ncpus: 2
+
+ - name: Build Arrow C++ with rtools40
+ shell: bash
+ env:
+ ARROW_HOME: "arrow"
+ run: arrow/ci/scripts/r_windows_build.sh
+
+ - name: Upload Binary
+ if: {{ '${{ ' + upload_libarrow + ' }}' }}
+ shell: bash
+ env:
+ USER: {{ '${{ secrets.CROSSBOW_NIGHTLIES_USER }}' }}
+ PW: {{ '${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }}
+ run: |
+ VERSION=$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //)
+
+ cd build
+ curl -s --fail --show-error -u $USER:$PW -T arrow-$VERSION.zip {{
webdav_domain }}/r/libarrow/bin/windows/
+
+ - name: Upload binary artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: r-windows-libarrow
+ path: build/arrow-*.zip
+
+ r-packages:
+ if: true && !cancelled()
+ needs: [source, windows-cpp]
+ name: {{ '${{ matrix.platform }} ${{ matrix.r_version.r }}' }}
+ runs-on: {{ '${{ matrix.platform }}' }}
+ strategy:
+ fail-fast: false
+ matrix:
+ platform:
+ - windows-latest
+ # This is newer than what CRAN builds on, but Travis is no longer an
option for us, so...
+ - macos-10.15
+ # - devops-managed # No M1 until the runner application runs native
+ r_version:
+ - { rtools: 40, r: "4.1" }
+ - { rtools: 42, r: "4.2" }
+ steps:
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ r-version: {{ '${{ matrix.r_version.r }}' }}
+ rtools-version: {{ '${{ matrix.r_version.rtools }}' }}
+ Ncpus: 2
+ {{ macros.github_setup_local_r_repo(is_fork, 'matrix.platform')|indent }}
+ - name: Build Binary
+ shell: Rscript {0}
+ run: |
+ on_windows <- tolower(Sys.info()[["sysname"]]) == "windows"
+
+ # Install dependencies by installing (yesterday's) binary, then
removing it
+ install.packages(c("arrow", "cpp11"),
+ type = "binary",
+ repos = c("{{ repo_domain }}", "https://cloud.r-project.org";)
+ )
+ remove.packages("arrow")
+
+ dev_repo <- "{{ repo_domain }}"
+
+ # Setup local repo
+ if ({{is_fork|upper}}) {
+ dev_repo <- paste0(
+ ifelse(on_windows, "file:", "file://"),
+ getwd(),
+ "/repo")
+ }
+ options(arrow.dev_repo = dev_repo)
+
+ # Build
+ Sys.setenv(MAKEFLAGS = paste0("-j", parallel::detectCores()))
+ INSTALL_opts <- "--build"
+ if (!on_windows) {
+ # Windows doesn't support the --strip arg
+ INSTALL_opts <- c(INSTALL_opts, "--strip")
+ }
+
+
+ install.packages(
+ "arrow",
+ type = "source",
+ repos = dev_repo,
+ INSTALL_opts = INSTALL_opts
+ )
+
+ # Test
+ library(arrow)
+ read_parquet(system.file("v0.7.1.parquet", package = "arrow"))
+ - name: Upload package
+ if: {{ '${{ ' + upload_pkg + ' }}' }}
+ shell: bash
+ env:
+ USER: {{ '${{ secrets.CROSSBOW_NIGHTLIES_USER }}' }}
+ PW: {{ '${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }}
+ VERSION: {{ '${{ needs.source.outputs.version }}' }}
+ run: |
+ REPO_PATH=r$(Rscript -e "cat(contrib.url('', type = 'binary'))")
+ EXT=$(if [[ {{ '${{ matrix.platform }}' }} == windows* ]]; then echo
zip; else echo tgz; fi)
+
+ curl -s --fail --show-error -u $USER:$PW -T arrow_$VERSION.$EXT {{
webdav_domain }}/$REPO_PATH/
Review Comment:
How much extra work would it be to move this step to after the
`test-*-binary` step?
It would be nice if we had the whole test run and succeed and only after
that upload to the webdav server. This also has the added benefit where the two
failure modes of failing to upload (or uploading, but the server not being
ready to serve the file when the next step is kicked off) are either super
clear or side-stepped entirely.
This would involve adding a bit like Neal original suggested that would
imitate the repo on local host during the test stage, but that shouldn't be _so
hard_ yeah?
##########
dev/tasks/r/github.nightly.yml:
##########
@@ -0,0 +1,467 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% import 'macros.jinja' as macros with context %}
+# root of the R nightly repo where the first dir is the mount point used by
davfs2 ("nightly")
+# TODO update with nightlies.apache.org/arrow/r
+{% set r_root = "nightly/r" %}
+{% set webdav_domain = "https://nightly.wujciak.de:8080"; %}
+{% set repo_domain = "https://nightly.wujciak.de/r"; %}
+# This allows us to set a custom version via param:
+# crossbow submit --param custom_version=8.5.3 r-nightly-packages
+# if the param is unset defaults to the usual Ymd naming scheme
+{% set version = custom_version|default('\\2.\'\"$(date +%Y%m%d)\"\'') %}
+# We need this as boolean and string
+{% set is_upstream_b = arrow.github_repo == 'apache/arrow' %}
+# use filter to cast to string and convert to lowercase to match yaml boolean
+{% set is_fork = (not is_upstream_b)|lower %}
+{% set is_upstream = is_upstream_b|lower %}
+{% set upload_pkg = "env.UPLOAD_PKG == 'true' && success() && " + is_upstream
%}
+{% set upload_libarrow = "env.UPLOAD_LIBARROW == 'true' && success() && " +
is_upstream %}
+
+{{ macros.github_header() }}
+
+env:
+ # prevent upload of R packages or libarrow binaries via param (e.g. for
release)
+ # crossbow submit --param upload_pkgs=false --param upload_libs=false
r-nightly-packages
+ UPLOAD_PKG: {{ upload_pkgs|default("true") }}
+ UPLOAD_LIBARROW: {{ upload_libs|default("true")}}
+
+jobs:
+ source:
+ # This job will change the version to either the custom_version param or
YMD format.
+ # The output allows other steps to use the exact version to prevent issues
(e.g. date changes during run)
+ name: Source Package
+ runs-on: ubuntu-latest
+ outputs:
+ version: {{ '${{ steps.save-version.outputs.version }}' }}
+ steps:
+ {{ macros.github_checkout_arrow()|indent }}
+ {{ macros.github_change_r_pkg_version(is_fork, version)|indent }}
+ - name: Save Version
+ id: save-version
+ shell: bash
+ run: |
+ echo "::set-output name=version::$(grep ^Version arrow/r/DESCRIPTION
| sed s/Version:\ //)"
+
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ install-r: false
+
+ - name: Build R source package
+ shell: bash
+ run: |
+ cd arrow/r
+ # Copy in the Arrow C++ source
+ make sync-cpp
+ R CMD build --no-build-vignettes .
+
+ - name: Install davfs2 & Mount Repository
+ if: {{ '${{ ' + upload_pkg + ' }}' }}
+ run: |
+ sudo apt update && sudo apt install davfs2
+
+ # Create mount point
+ mkdir nightly
+
+ # make credentials available
+ sudo bash -c 'echo "{{ webdav_domain }} {{ '${{
secrets.CROSSBOW_NIGHTLIES_USER }} ${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }}"
>> /etc/davfs2/secrets'
+
+ # mount with read&write access
+ sudo mount -t davfs {{ webdav_domain }} nightly -o rw
+
+ - name: Upload Source Package
+ if: {{ '${{ ' + upload_pkg + ' }}' }}
+ shell: bash
+ run: |
+ # ensure repo structure is set up, this job is the fastest and
should fix any issue before the other jobs try
+ # to push into non existent folder with curl, which would fail
silently.
+ # Add any new architectures or R versions here
+ sudo mkdir -p {{ r_root }}/src/contrib
+ {% for os in ["ubuntu-18.04",
+ "centos-7",
+ "windows"] %}
+ sudo mkdir -p {{ r_root }}/libarrow/bin/{{ os }}
+ {% endfor %}
+ {% for os in ["windows", "macosx", "macosx/big-sur-arm64"] %}
+ {% for r_version in ["4.1", "4.2"] %}
+ sudo mkdir -p {{ r_root }}/bin/{{ os }}/contrib/{{ r_version }}
+ {% endfor %}
+ {% endfor %}
+ sudo cp arrow/r/arrow_*.tar.gz {{ r_root }}/src/contrib
+ - name: Update Repository Index
+ if: {{ '${{ ' + upload_pkg + ' }}' }}
+ shell: sudo Rscript {0}
+ run: |
+ if(file.exists("{{ r_root }}/src/contrib/PACKAGES")) {
+ tools::update_PACKAGES("{{ r_root }}/src/contrib" ,
+ type = "source",
+ latestOnly = FALSE,
+ verbose = TRUE)
+ } else {
+ tools::write_PACKAGES("{{ r_root }}/src/contrib" ,
+ type = "source",
+ latestOnly = FALSE,
+ verbose = TRUE)
+ }
+
+ - name: Unmount Repository
+ # ensures all changes are written
+ if: {{ '${{ ' + upload_pkg + ' }}' }}
+ shell: bash
+ run: sudo umount nightly
+
+ - name: Upload package artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: r-src-pkg
+ path: arrow/r/arrow_*.tar.gz
+
+ linux-cpp:
+ name: C++ Binary {{ '${{ matrix.config.os }}-${{ matrix.config.version }}'
}}
+ runs-on: ubuntu-latest
+ needs: source
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ - { os: ubuntu, version: "18.04" }
+ - { os: centos, version: "7" }
+ env:
+ UBUNTU: {{ '${{ matrix.config.version }}' }}
+ steps:
+ {{ macros.github_checkout_arrow()|indent }}
+ {{ macros.github_change_r_pkg_version(is_fork, '${{
needs.source.outputs.version }}')|indent }}
+ {{ macros.github_install_archery()|indent }}
+ - name: Build libarrow
+ shell: bash
+ run: |
+ sudo sysctl -w kernel.core_pattern="core.%e.%p"
+ ulimit -c unlimited
+ archery docker run {{ '${{ matrix.config.os}}' }}-cpp-static
+ - name: Bundle libarrow
+ shell: bash
+ env:
+ PKG_FILE: arrow-{{ '${{ needs.source.outputs.version }}' }}.zip
+ VERSION: {{ '${{ needs.source.outputs.version }}' }}
+ run: |
+ cd arrow/r/libarrow/dist
+ # These files were created by the docker user so we have to sudo to
get them
+ sudo -E zip -r $PKG_FILE lib/ include/
+
+ - name: Upload libarrow
+ if: {{ '${{ ' + upload_libarrow + ' }}' }}
+ shell: bash
+ env:
+ USER: {{ '${{ secrets.CROSSBOW_NIGHTLIES_USER }}' }}
+ PW: {{ '${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }}
+ PKG_FILE: arrow-{{ '${{ needs.source.outputs.version }}' }}.zip
+ run: |
+ cd arrow/r/libarrow/dist
+ export REPO_PATH={{ 'r/libarrow/bin/${{ matrix.config.os }}-${{
matrix.config.version }}' }}
+ curl -s --fail --show-error -u $USER:$PW -T $PKG_FILE {{
webdav_domain }}/$REPO_PATH/
+
+ - name: Upload binary artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: r-{{ '${{ matrix.config.os}}' }}-libarrow
+ path: arrow/r/libarrow/dist/arrow-*.zip
+
+ windows-cpp:
+ name: C++ Binary Windows RTools (40 only)
+ needs: source
+ runs-on: windows-latest
+ steps:
+ - run: git config --global core.autocrlf false
+ {{ macros.github_checkout_arrow()|indent }}
+ {{ macros.github_change_r_pkg_version(is_fork, '${{
needs.source.outputs.version }}')|indent }}
+
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ rtools-version: 40
+ r-version: "4.0"
+ Ncpus: 2
+
+ - name: Build Arrow C++ with rtools40
+ shell: bash
+ env:
+ ARROW_HOME: "arrow"
+ run: arrow/ci/scripts/r_windows_build.sh
+
+ - name: Upload Binary
+ if: {{ '${{ ' + upload_libarrow + ' }}' }}
+ shell: bash
+ env:
+ USER: {{ '${{ secrets.CROSSBOW_NIGHTLIES_USER }}' }}
+ PW: {{ '${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }}
+ run: |
+ VERSION=$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //)
+
+ cd build
+ curl -s --fail --show-error -u $USER:$PW -T arrow-$VERSION.zip {{
webdav_domain }}/r/libarrow/bin/windows/
+
+ - name: Upload binary artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: r-windows-libarrow
+ path: build/arrow-*.zip
+
+ r-packages:
+ if: true && !cancelled()
+ needs: [source, windows-cpp]
+ name: {{ '${{ matrix.platform }} ${{ matrix.r_version.r }}' }}
+ runs-on: {{ '${{ matrix.platform }}' }}
+ strategy:
+ fail-fast: false
+ matrix:
+ platform:
+ - windows-latest
+ # This is newer than what CRAN builds on, but Travis is no longer an
option for us, so...
+ - macos-10.15
+ # - devops-managed # No M1 until the runner application runs native
+ r_version:
+ - { rtools: 40, r: "4.1" }
+ - { rtools: 42, r: "4.2" }
+ steps:
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ r-version: {{ '${{ matrix.r_version.r }}' }}
+ rtools-version: {{ '${{ matrix.r_version.rtools }}' }}
+ Ncpus: 2
+ {{ macros.github_setup_local_r_repo(is_fork, 'matrix.platform')|indent }}
+ - name: Build Binary
+ shell: Rscript {0}
+ run: |
+ on_windows <- tolower(Sys.info()[["sysname"]]) == "windows"
+
+ # Install dependencies by installing (yesterday's) binary, then
removing it
+ install.packages(c("arrow", "cpp11"),
+ type = "binary",
+ repos = c("{{ repo_domain }}", "https://cloud.r-project.org";)
+ )
+ remove.packages("arrow")
+
+ dev_repo <- "{{ repo_domain }}"
+
+ # Setup local repo
+ if ({{is_fork|upper}}) {
+ dev_repo <- paste0(
+ ifelse(on_windows, "file:", "file://"),
+ getwd(),
+ "/repo")
+ }
+ options(arrow.dev_repo = dev_repo)
+
+ # Build
+ Sys.setenv(MAKEFLAGS = paste0("-j", parallel::detectCores()))
+ INSTALL_opts <- "--build"
+ if (!on_windows) {
+ # Windows doesn't support the --strip arg
+ INSTALL_opts <- c(INSTALL_opts, "--strip")
+ }
+
+
+ install.packages(
+ "arrow",
+ type = "source",
+ repos = dev_repo,
+ INSTALL_opts = INSTALL_opts
+ )
+
+ # Test
+ library(arrow)
+ read_parquet(system.file("v0.7.1.parquet", package = "arrow"))
+ - name: Upload package
+ if: {{ '${{ ' + upload_pkg + ' }}' }}
+ shell: bash
+ env:
+ USER: {{ '${{ secrets.CROSSBOW_NIGHTLIES_USER }}' }}
+ PW: {{ '${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }}
+ VERSION: {{ '${{ needs.source.outputs.version }}' }}
+ run: |
+ REPO_PATH=r$(Rscript -e "cat(contrib.url('', type = 'binary'))")
+ EXT=$(if [[ {{ '${{ matrix.platform }}' }} == windows* ]]; then echo
zip; else echo tgz; fi)
+
+ curl -s --fail --show-error -u $USER:$PW -T arrow_$VERSION.$EXT {{
webdav_domain }}/$REPO_PATH/
Review Comment:
Oh actually I hadn't gotten there yet, it looks like you _do_ have the step
`update-repo` which does do the updating after the tests run too. So in this
case we're actually updating the repo twice? Once her and then once again below?
Or maybe I'm super turned around?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
