This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new ed3e55c4f4 ARROW-17001: [Release][R] Use apache artifactory for
libarrow binaries. (#13622)
ed3e55c4f4 is described below
commit ed3e55c4f4ad71b21f298acd250dac2145d093c3
Author: Jacob Wujciak-Jens <[email protected]>
AuthorDate: Thu Jul 21 07:02:53 2022 +0200
ARROW-17001: [Release][R] Use apache artifactory for libarrow binaries.
(#13622)
@nealrichardson This should be it, though it is hard to test end2end.
@kou Thanks for your suggestions, please look over my other changes to the
process.
Lead-authored-by: Jacob Wujciak-Jens <[email protected]>
Co-authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
dev/release/05-binary-upload.sh | 4 +++
dev/release/binary-task.rb | 64 +++++++++++++++++++++++++++++++++++++++++
dev/release/post-02-binary.sh | 4 +++
r/tools/nixlibs.R | 15 +++++++++-
r/tools/winlibs.R | 33 ++++++++++-----------
5 files changed, 103 insertions(+), 17 deletions(-)
diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh
index efa3640b6c..d6163e7059 100755
--- a/dev/release/05-binary-upload.sh
+++ b/dev/release/05-binary-upload.sh
@@ -74,6 +74,7 @@ fi
: ${UPLOAD_DOCS:=${UPLOAD_DEFAULT}}
: ${UPLOAD_NUGET:=${UPLOAD_DEFAULT}}
: ${UPLOAD_PYTHON:=${UPLOAD_DEFAULT}}
+: ${UPLOAD_R:=${UPLOAD_DEFAULT}}
: ${UPLOAD_UBUNTU:=${UPLOAD_DEFAULT}}
rake_tasks=()
@@ -104,6 +105,9 @@ fi
if [ ${UPLOAD_PYTHON} -gt 0 ]; then
rake_tasks+=(python:rc)
fi
+if [ ${UPLOAD_R} -gt 0 ]; then
+ rake_tasks+=(r:rc)
+fi
if [ ${UPLOAD_UBUNTU} -gt 0 ]; then
rake_tasks+=(apt:rc)
apt_targets+=(ubuntu)
diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb
index 2cedd9c734..857517d108 100644
--- a/dev/release/binary-task.rb
+++ b/dev/release/binary-task.rb
@@ -803,6 +803,7 @@ class BinaryTask
define_docs_tasks
define_nuget_tasks
define_python_tasks
+ define_r_tasks
define_summary_tasks
end
@@ -1899,6 +1900,67 @@ APT::FTPArchive::Release::Description
"#{apt_repository_description}";
"{python-sdist,wheel-*}/**/*")
end
+ def define_r_rc_tasks(label, id, rc_dir)
+ directory rc_dir
+
+ namespace id do
+ namespace :rc do
+ desc "Prepare #{label} packages"
+ task :prepare => rc_dir do
+ progress_label = "Preparing #{label}"
+ progress_reporter = ProgressReporter.new(progress_label)
+
+ pattern = "r-binary-packages/r-lib*.{zip,tgz}"
+ Pathname(artifacts_dir).glob(pattern) do |path|
+ destination_path = [
+ rc_dir,
+ # r-lib__libarrow__bin__centos-7__arrow-8.0.0.zip
+ # --> libarrow/bin/centos-7/arrow-8.0.0.zip
+ path.basename.to_s.gsub(/\Ar-lib__/, "").gsub(/__/, "/"),
+ ].join("/")
+ copy_artifact(path, destination_path, progress_reporter)
+ end
+
+ progress_reporter.finish
+ end
+
+ desc "Sign #{label} packages"
+ task :sign => rc_dir do
+ sign_dir(label, rc_dir)
+ end
+
+ desc "Upload #{label} packages"
+ task :upload do
+ uploader =
+ ArtifactoryUploader.new(api_key: artifactory_api_key,
+ destination_prefix: full_version,
+ distribution: id.to_s,
+ rc: rc,
+ source: rc_dir,
+ staging: staging?)
+ uploader.upload
+ end
+ end
+
+ desc "Release RC #{label} packages"
+ rc_tasks = [
+ "#{id}:rc:prepare",
+ "#{id}:rc:sign",
+ "#{id}:rc:upload",
+ ]
+ task :rc => rc_tasks
+ end
+ end
+
+ def define_r_tasks
+ label = "R"
+ id = :r
+ r_rc_dir = "#{rc_dir}/r/#{full_version}"
+ r_release_dir = "#{release_dir}/r/#{full_version}"
+ define_r_rc_tasks(label, id, r_rc_dir)
+ define_generic_data_release_tasks(label, id, r_release_dir)
+ end
+
def define_summary_tasks
namespace :summary do
desc "Show RC summary"
@@ -1914,6 +1976,7 @@ Success! The release candidate binaries are available
here:
https://apache.jfrog.io/artifactory/arrow/docs#{suffix}-rc/
https://apache.jfrog.io/artifactory/arrow/nuget#{suffix}-rc/#{full_version}
https://apache.jfrog.io/artifactory/arrow/python#{suffix}-rc/#{full_version}
+ https://apache.jfrog.io/artifactory/arrow/r#{suffix}-rc/#{full_version}
https://apache.jfrog.io/artifactory/arrow/ubuntu#{suffix}-rc/
SUMMARY
end
@@ -1931,6 +1994,7 @@ Success! The release binaries are available here:
https://apache.jfrog.io/artifactory/arrow/docs#{suffix}/
https://apache.jfrog.io/artifactory/arrow/nuget#{suffix}/#{version}
https://apache.jfrog.io/artifactory/arrow/python#{suffix}/#{version}
+ https://apache.jfrog.io/artifactory/arrow/r#{suffix}/#{version}
https://apache.jfrog.io/artifactory/arrow/ubuntu#{suffix}/
SUMMARY
end
diff --git a/dev/release/post-02-binary.sh b/dev/release/post-02-binary.sh
index a9cdc6020c..f46bd73bf9 100755
--- a/dev/release/post-02-binary.sh
+++ b/dev/release/post-02-binary.sh
@@ -52,6 +52,7 @@ fi
: ${DEPLOY_DOCS:=${DEPLOY_DEFAULT}}
: ${DEPLOY_NUGET:=${DEPLOY_DEFAULT}}
: ${DEPLOY_PYTHON:=${DEPLOY_DEFAULT}}
+: ${DEPLOY_R:=${DEPLOY_DEFAULT}}
: ${DEPLOY_UBUNTU:=${DEPLOY_DEFAULT}}
rake_tasks=()
@@ -82,6 +83,9 @@ fi
if [ ${DEPLOY_PYTHON} -gt 0 ]; then
rake_tasks+=(python:release)
fi
+if [ ${DEPLOY_R} -gt 0 ]; then
+ rake_tasks+=(r:release)
+fi
if [ ${DEPLOY_UBUNTU} -gt 0 ]; then
rake_tasks+=(apt:release)
apt_targets+=(ubuntu)
diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R
index 019d6eca86..b76af09ec5 100644
--- a/r/tools/nixlibs.R
+++ b/r/tools/nixlibs.R
@@ -22,7 +22,20 @@ dst_dir <- paste0("libarrow/arrow-", VERSION)
# TESTING is set in test-nixlibs.R; it won't be set when called from configure
test_mode <- exists("TESTING")
-arrow_repo <- paste0(getOption("arrow.dev_repo",
"https://nightlies.apache.org/arrow/r";), "/libarrow/")
+# Prevent error with binary selection during testing.
+if (test_mode && is.na(VERSION)) {
+ VERSION <- "8.0.0.9000"
+}
+
+dev_version <- package_version(VERSION)[1, 4]
+
+# Small dev versions are added for R-only changes during CRAN submission.
+if (is.na(dev_version) || dev_version < 100) {
+ VERSION <- package_version(VERSION)[1, 1:3]
+ arrow_repo <-
sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s/libarrow/";, VERSION)
+} else {
+ arrow_repo <- paste0(getOption("arrow.dev_repo",
"https://nightlies.apache.org/arrow/r";), "/libarrow/")
+}
options(.arrow.cleanup = character()) # To collect dirs to rm on exit
on.exit(unlink(getOption(".arrow.cleanup")))
diff --git a/r/tools/winlibs.R b/r/tools/winlibs.R
index e42d02b180..5aeea2e417 100644
--- a/r/tools/winlibs.R
+++ b/r/tools/winlibs.R
@@ -27,7 +27,7 @@ if
(!file.exists(sprintf("windows/arrow-%s/include/arrow/api.h", VERSION))) {
}
file.copy(localfile, "lib.zip")
} else {
- # Download static arrow from rwinlib
+ # Download static arrow from the apache artifactory
quietly <- !identical(tolower(Sys.getenv("ARROW_R_DEV")), "true")
get_file <- function(template, version) {
try(
@@ -37,28 +37,29 @@ if
(!file.exists(sprintf("windows/arrow-%s/include/arrow/api.h", VERSION))) {
silent = quietly
)
}
+
# URL templates
nightly <- paste0(
getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r";),
"/libarrow/bin/windows/arrow-%s.zip"
)
+ # %1$s uses the first variable for both substitutions
+ artifactory <-
"https://apache.jfrog.io/artifactory/arrow/r/%1$s/libarrow/bin/windows/arrow-%1$s.zip";
rwinlib <- "https://github.com/rwinlib/arrow/archive/v%s.zip";
- # First look for a nightly
- get_file(nightly, VERSION)
- # If not found, then check rwinlib
- if (!file.exists("lib.zip")) {
- get_file(rwinlib, VERSION)
- }
- if (!file.exists("lib.zip")) {
- # Try a different version
- # First, try pruning off a dev number, i.e. go from 0.14.1.1 to 0.14.1
- VERSION <- sub("^([0-9]+\\.[0-9]+\\.[0-9]+).*$", "\\1", VERSION)
- get_file(rwinlib, VERSION)
- }
- if (!file.exists("lib.zip")) {
- # Next, try without a patch release, i.e. go from 0.14.1 to 0.14.0
- VERSION <- sub("^([0-9]+\\.[0-9]+\\.).*$", "\\10", VERSION)
+
+ dev_version <- package_version(VERSION)[1, 4]
+
+ # Small dev versions are added for R-only changes during CRAN submission.
+ if (is.na(dev_version) || dev_version < 100) {
+ VERSION <- package_version(VERSION)[1, 1:3]
get_file(rwinlib, VERSION)
+
+ # If not found, fall back to apache artifactory
+ if (!file.exists("lib.zip")) {
+ get_file(artifactory, VERSION)
+ }
+ } else {
+ get_file(nightly, VERSION)
}
}
dir.create("windows", showWarnings = FALSE)
