assignUser commented on code in PR #13149: URL: https://github.com/apache/arrow/pull/13149#discussion_r881527772
########## dev/tasks/r/github.nightly.yml: ########## @@ -0,0 +1,467 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} +# root of the R nightly repo where the first dir is the mount point used by davfs2 ("nightly") +# TODO update with nightlies.apache.org/arrow/r +{% set r_root = "nightly/r" %} +{% set webdav_domain = "https://nightly.wujciak.de:8080"; %} +{% set repo_domain = "https://nightly.wujciak.de/r"; %} +# This allows us to set a custom version via param: +# crossbow submit --param custom_version=8.5.3 r-nightly-packages +# if the param is unset defaults to the usual Ymd naming scheme +{% set version = custom_version|default('\\2.\'\"$(date +%Y%m%d)\"\'') %} +# We need this as boolean and string +{% set is_upstream_b = arrow.github_repo == 'apache/arrow' %} +# use filter to cast to string and convert to lowercase to match yaml boolean +{% set is_fork = (not is_upstream_b)|lower %} +{% set is_upstream = is_upstream_b|lower %} +{% set upload_pkg = "env.UPLOAD_PKG == 'true' && success() && " + is_upstream %} +{% set upload_libarrow = "env.UPLOAD_LIBARROW == 'true' && success() && " + is_upstream %} + +{{ macros.github_header() }} + +env: + # prevent upload of R packages or libarrow binaries via param (e.g. for release) + # crossbow submit --param upload_pkgs=false --param upload_libs=false r-nightly-packages + UPLOAD_PKG: {{ upload_pkgs|default("true") }} + UPLOAD_LIBARROW: {{ upload_libs|default("true")}} + +jobs: + source: + # This job will change the version to either the custom_version param or YMD format. + # The output allows other steps to use the exact version to prevent issues (e.g. date changes during run) + name: Source Package + runs-on: ubuntu-latest + outputs: + version: {{ '${{ steps.save-version.outputs.version }}' }} + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_change_r_pkg_version(is_fork, version)|indent }} + - name: Save Version + id: save-version + shell: bash + run: | + echo "::set-output name=version::$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //)" + + - uses: r-lib/actions/setup-r@v2 + with: + install-r: false + + - name: Build R source package + shell: bash + run: | + cd arrow/r + # Copy in the Arrow C++ source + make sync-cpp + R CMD build --no-build-vignettes . + + - name: Install davfs2 & Mount Repository + if: {{ '${{ ' + upload_pkg + ' }}' }} + run: | + sudo apt update && sudo apt install davfs2 + + # Create mount point + mkdir nightly + + # make credentials available + sudo bash -c 'echo "{{ webdav_domain }} {{ '${{ secrets.CROSSBOW_NIGHTLIES_USER }} ${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }}" >> /etc/davfs2/secrets' + + # mount with read&write access + sudo mount -t davfs {{ webdav_domain }} nightly -o rw + + - name: Upload Source Package + if: {{ '${{ ' + upload_pkg + ' }}' }} + shell: bash + run: | + # ensure repo structure is set up, this job is the fastest and should fix any issue before the other jobs try + # to push into non existent folder with curl, which would fail silently. + # Add any new architectures or R versions here + sudo mkdir -p {{ r_root }}/src/contrib + {% for os in ["ubuntu-18.04", + "centos-7", + "windows"] %} + sudo mkdir -p {{ r_root }}/libarrow/bin/{{ os }} + {% endfor %} + {% for os in ["windows", "macosx", "macosx/big-sur-arm64"] %} + {% for r_version in ["4.1", "4.2"] %} + sudo mkdir -p {{ r_root }}/bin/{{ os }}/contrib/{{ r_version }} + {% endfor %} + {% endfor %} + sudo cp arrow/r/arrow_*.tar.gz {{ r_root }}/src/contrib + - name: Update Repository Index + if: {{ '${{ ' + upload_pkg + ' }}' }} + shell: sudo Rscript {0} + run: | + if(file.exists("{{ r_root }}/src/contrib/PACKAGES")) { + tools::update_PACKAGES("{{ r_root }}/src/contrib" , + type = "source", + latestOnly = FALSE, + verbose = TRUE) + } else { + tools::write_PACKAGES("{{ r_root }}/src/contrib" , + type = "source", + latestOnly = FALSE, + verbose = TRUE) + } + + - name: Unmount Repository + # ensures all changes are written + if: {{ '${{ ' + upload_pkg + ' }}' }} + shell: bash + run: sudo umount nightly + + - name: Upload package artifact + uses: actions/upload-artifact@v3 + with: + name: r-src-pkg + path: arrow/r/arrow_*.tar.gz + + linux-cpp: + name: C++ Binary {{ '${{ matrix.config.os }}-${{ matrix.config.version }}' }} + runs-on: ubuntu-latest + needs: source + strategy: + fail-fast: false + matrix: + config: + - { os: ubuntu, version: "18.04" } + - { os: centos, version: "7" } + env: + UBUNTU: {{ '${{ matrix.config.version }}' }} + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.version }}')|indent }} + {{ macros.github_install_archery()|indent }} + - name: Build libarrow + shell: bash + run: | + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited + archery docker run {{ '${{ matrix.config.os}}' }}-cpp-static + - name: Bundle libarrow + shell: bash + env: + PKG_FILE: arrow-{{ '${{ needs.source.outputs.version }}' }}.zip + VERSION: {{ '${{ needs.source.outputs.version }}' }} + run: | + cd arrow/r/libarrow/dist + # These files were created by the docker user so we have to sudo to get them + sudo -E zip -r $PKG_FILE lib/ include/ + + - name: Upload libarrow + if: {{ '${{ ' + upload_libarrow + ' }}' }} + shell: bash + env: + USER: {{ '${{ secrets.CROSSBOW_NIGHTLIES_USER }}' }} + PW: {{ '${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }} + PKG_FILE: arrow-{{ '${{ needs.source.outputs.version }}' }}.zip + run: | + cd arrow/r/libarrow/dist + export REPO_PATH={{ 'r/libarrow/bin/${{ matrix.config.os }}-${{ matrix.config.version }}' }} + curl -s --fail --show-error -u $USER:$PW -T $PKG_FILE {{ webdav_domain }}/$REPO_PATH/ + + - name: Upload binary artifact + uses: actions/upload-artifact@v3 + with: + name: r-{{ '${{ matrix.config.os}}' }}-libarrow + path: arrow/r/libarrow/dist/arrow-*.zip + + windows-cpp: + name: C++ Binary Windows RTools (40 only) + needs: source + runs-on: windows-latest + steps: + - run: git config --global core.autocrlf false + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.version }}')|indent }} + + - uses: r-lib/actions/setup-r@v2 + with: + rtools-version: 40 + r-version: "4.0" + Ncpus: 2 + + - name: Build Arrow C++ with rtools40 + shell: bash + env: + ARROW_HOME: "arrow" + run: arrow/ci/scripts/r_windows_build.sh + + - name: Upload Binary + if: {{ '${{ ' + upload_libarrow + ' }}' }} + shell: bash + env: + USER: {{ '${{ secrets.CROSSBOW_NIGHTLIES_USER }}' }} + PW: {{ '${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }} + run: | + VERSION=$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //) + + cd build + curl -s --fail --show-error -u $USER:$PW -T arrow-$VERSION.zip {{ webdav_domain }}/r/libarrow/bin/windows/ + + - name: Upload binary artifact + uses: actions/upload-artifact@v3 + with: + name: r-windows-libarrow + path: build/arrow-*.zip + + r-packages: + if: true && !cancelled() + needs: [source, windows-cpp] + name: {{ '${{ matrix.platform }} ${{ matrix.r_version.r }}' }} + runs-on: {{ '${{ matrix.platform }}' }} + strategy: + fail-fast: false + matrix: + platform: + - windows-latest + # This is newer than what CRAN builds on, but Travis is no longer an option for us, so... + - macos-10.15 + # - devops-managed # No M1 until the runner application runs native + r_version: + - { rtools: 40, r: "4.1" } + - { rtools: 42, r: "4.2" } + steps: + - uses: r-lib/actions/setup-r@v2 + with: + r-version: {{ '${{ matrix.r_version.r }}' }} + rtools-version: {{ '${{ matrix.r_version.rtools }}' }} + Ncpus: 2 + {{ macros.github_setup_local_r_repo(is_fork, 'matrix.platform')|indent }} + - name: Build Binary + shell: Rscript {0} + run: | + on_windows <- tolower(Sys.info()[["sysname"]]) == "windows" + + # Install dependencies by installing (yesterday's) binary, then removing it + install.packages(c("arrow", "cpp11"), + type = "binary", + repos = c("{{ repo_domain }}", "https://cloud.r-project.org";) + ) + remove.packages("arrow") + + dev_repo <- "{{ repo_domain }}" + + # Setup local repo + if ({{is_fork|upper}}) { + dev_repo <- paste0( + ifelse(on_windows, "file:", "file://"), + getwd(), + "/repo") + } + options(arrow.dev_repo = dev_repo) + + # Build + Sys.setenv(MAKEFLAGS = paste0("-j", parallel::detectCores())) + INSTALL_opts <- "--build" + if (!on_windows) { + # Windows doesn't support the --strip arg + INSTALL_opts <- c(INSTALL_opts, "--strip") + } + + + install.packages( + "arrow", + type = "source", + repos = dev_repo, + INSTALL_opts = INSTALL_opts + ) + + # Test + library(arrow) + read_parquet(system.file("v0.7.1.parquet", package = "arrow")) + - name: Upload package + if: {{ '${{ ' + upload_pkg + ' }}' }} + shell: bash + env: + USER: {{ '${{ secrets.CROSSBOW_NIGHTLIES_USER }}' }} + PW: {{ '${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }} + VERSION: {{ '${{ needs.source.outputs.version }}' }} + run: | + REPO_PATH=r$(Rscript -e "cat(contrib.url('', type = 'binary'))") + EXT=$(if [[ {{ '${{ matrix.platform }}' }} == windows* ]]; then echo zip; else echo tgz; fi) + + curl -s --fail --show-error -u $USER:$PW -T arrow_$VERSION.$EXT {{ webdav_domain }}/$REPO_PATH/ Review Comment: Regarding the "only upload once everything finishes" I could do that but I modeled this workflow 1:1 to arrow-r-nightly and it would also mean that we don't upload even if just one of the test jobs (e.g. fedora devel) fails and we don't know if the failure is even related to the build. As these are nightlies and are not expected to be 100% stable I think the way it works now is preferable, especially now that we have multiple versions available in the cran repo and if there is an issue you can just use an older one. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
