This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 3c0a7a6353 ARROW-17353: [Release][R] Validate binaries version (#14396)
3c0a7a6353 is described below
commit 3c0a7a63533ced67762c1c04ee81e5516325453a
Author: Sutou Kouhei <[email protected]>
AuthorDate: Wed Oct 19 13:36:24 2022 +0900
ARROW-17353: [Release][R] Validate binaries version (#14396)
We generate version in Crossbow not CI job.
We can use file name validation feature by using version generated by
Crossbow.
We use "X.Y.Z" version for RC version such as "10.0.0" for "10.0.0-rc1".
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
dev/archery/archery/crossbow/core.py | 43 ++++++++++++++++++++--
.../tests/fixtures/crossbow-job-no-failure.yaml | 4 +-
.../crossbow/tests/fixtures/crossbow-job.yaml | 4 +-
dev/tasks/macros.jinja | 12 ++++--
dev/tasks/r/github.packages.yml | 8 +---
dev/tasks/tasks.yml | 22 +++++------
r/tools/nixlibs.R | 2 +-
r/tools/winlibs.R | 5 ++-
8 files changed, 72 insertions(+), 28 deletions(-)
diff --git a/dev/archery/archery/crossbow/core.py
b/dev/archery/archery/crossbow/core.py
index c8ea8a13a4..a83c190d12 100644
--- a/dev/archery/archery/crossbow/core.py
+++ b/dev/archery/archery/crossbow/core.py
@@ -738,14 +738,16 @@ class Target(Serializable):
(currently only an email address where the notification should be sent).
"""
- def __init__(self, head, branch, remote, version, email=None):
+ def __init__(self, head, branch, remote, version, r_version, email=None):
self.head = head
self.email = email
self.branch = branch
self.remote = remote
self.github_repo = "/".join(_parse_github_user_repo(remote))
self.version = version
+ self.r_version = r_version
self.no_rc_version = re.sub(r'-rc\d+\Z', '', version)
+ self.no_rc_r_version = re.sub(r'-rc\d+\Z', '', r_version)
# TODO(ARROW-17552): Remove "master" from default_branch after
# migration to "main".
self.default_branch = ['main', 'master']
@@ -791,8 +793,39 @@ class Target(Serializable):
if email is None:
email = repo.user_email
+ version_dev_match = re.match(r".*\.dev(\d+)$", version)
+ if version_dev_match:
+ with open(f"{repo.path}/r/DESCRIPTION") as description_file:
+ description = description_file.read()
+ r_version_pattern = re.compile(r"^Version:\s*(.*)$",
+ re.MULTILINE)
+ r_version = re.findall(r_version_pattern, description)[0]
+ if r_version:
+ version_dev = int(version_dev_match[1])
+ # "1_0000_00_00 +" is for generating a greater version
+ # than YYYYMMDD. For example, 1_0000_00_01
+ # (version_dev == 1 case) is greater than 2022_10_16.
+ #
+ # Why do we need a greater version than YYYYMMDD? It's
+ # for keeping backward compatibility. We used
+ # MAJOR.MINOR.PATCH.YYYYMMDD as our nightly package
+ # version. (See also ARROW-16403). If we use "9000 +
+ # version_dev" here, a developer that used
+ # 9.0.0.20221016 can't upgrade to the later nightly
+ # package unless we release 10.0.0. Because 9.0.0.9234
+ # or something is less than 9.0.0.20221016.
+ r_version_dev = 1_0000_00_00 + version_dev
+ # version: 10.0.0.dev234
+ # r_version: 9.0.0.9000
+ # -> 9.0.0.100000234
+ r_version = re.sub(r"\.9000\Z", f".{r_version_dev}", r_version)
+ else:
+ r_version = version
+ else:
+ r_version = version
+
return cls(head=head, email=email, branch=branch, remote=remote,
- version=version)
+ version=version, r_version=r_version)
def is_default_branch(self):
# TODO(ARROW-17552): Switch the condition to "is" instead of "in"
@@ -1105,7 +1138,10 @@ class Job(Serializable):
'version': target.version,
'no_rc_version': target.no_rc_version,
'no_rc_semver_version': target.no_rc_semver_version,
- 'no_rc_snapshot_version': target.no_rc_snapshot_version}
+ 'no_rc_snapshot_version': target.no_rc_snapshot_version,
+ 'r_version': target.r_version,
+ 'no_rc_r_version': target.no_rc_r_version,
+ }
for task_name, task in task_definitions.items():
task = task.copy()
artifacts = task.pop('artifacts', None) or [] # because of yaml
@@ -1260,6 +1296,7 @@ class Config(dict):
branch='master',
remote='https://github.com/apache/arrow',
version='1.0.0dev123',
+ r_version='0.13.0.100000123',
email='[email protected]'
)
job = Job.from_config(config=self,
diff --git
a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml
b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml
index 15e8ca3ff5..eb03bbee0b 100644
--- a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml
+++ b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml
@@ -5,7 +5,9 @@ target: !Target
branch: refs/pull/4435/merge
remote: https://github.com/apache/arrow
version: 0.13.0.dev306
+ r_version: 0.12.0.100000306
no_rc_version: 0.13.0.dev306
+ no_rc_r_version: 0.12.0.100000306
tasks:
docker-cpp-cmake32: !Task
ci: circle
@@ -64,4 +66,4 @@ branch: ursabot-1
_queue: !Queue
path: the_path
github_token: xxxxxxxxx
- _remote_url: https://github.com/apache/crossbow
\ No newline at end of file
+ _remote_url: https://github.com/apache/crossbow
diff --git a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml
b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml
index 90eab70498..f6de07dd45 100644
--- a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml
+++ b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml
@@ -5,7 +5,9 @@ target: !Target
branch: refs/pull/4435/merge
remote: https://github.com/apache/arrow
version: 0.13.0.dev306
+ r_version: 0.12.0.100000306
no_rc_version: 0.13.0.dev306
+ no_rc_r_version: 0.12.0.100000306
tasks:
docker-cpp-cmake32: !Task
ci: circle
@@ -64,4 +66,4 @@ branch: ursabot-1
_queue: !Queue
path: the_path
github_token: xxxxxxxxx
- _remote_url: https://github.com/apache/crossbow
\ No newline at end of file
+ _remote_url: https://github.com/apache/crossbow
diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja
index 3bec472bcf..bd3358e073 100644
--- a/dev/tasks/macros.jinja
+++ b/dev/tasks/macros.jinja
@@ -269,7 +269,7 @@ on:
rm -f apache-arrow*.rb.bak
{% endmacro %}
-{%- macro github_change_r_pkg_version(is_fork, version = '\\2.\'\"$(date
+%Y%m%d)\"\'' ) -%}
+{%- macro github_change_r_pkg_version(is_fork, version) -%}
- name: Modify version
shell: bash
run: |
@@ -339,12 +339,16 @@ on:
# tree not available in git-bash on windows
run: |
ls -R repo
- - name: Add dev repo to .Rprofile
+ - name: Add repos to .Rprofile
shell: Rscript {0}
run: |
- str <- paste0("options(arrow.dev_repo ='file://", getwd(), "/repo' )")
- print(str)
profile_path <- file.path(getwd(), ".Rprofile")
+ repo <- paste0("file://", getwd(), "/repo")
+ str <- paste0("options(arrow.repo = '", repo, "' )")
+ print(str)
+ write(str, file = profile_path, append = TRUE)
+ str <- paste0("options(arrow.dev_repo = '", repo, "' )")
+ print(str)
write(str, file = profile_path, append = TRUE)
# Set envvar for later steps by appending to $GITHUB_ENV
write(paste0("R_PROFILE_USER=", profile_path), file =
Sys.getenv("GITHUB_ENV"), append = TRUE)
diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml
index efe071a766..e53579f741 100644
--- a/dev/tasks/r/github.packages.yml
+++ b/dev/tasks/r/github.packages.yml
@@ -17,10 +17,6 @@
{% import 'macros.jinja' as macros with context %}
-# This allows us to set a custom version via param:
-# crossbow submit --param custom_version=8.5.3 r-binary-packages
-# if the param is unset defaults to the usual Ymd naming scheme
-{% set package_version = custom_version|replace("Unset", "\\2.\'\"$(date
+%Y%m%d)\"\'") %}
{% set is_fork = macros.is_fork %}
{{ macros.github_header() }}
@@ -35,7 +31,7 @@ jobs:
pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }}
steps:
{{ macros.github_checkout_arrow()|indent }}
- {{ macros.github_change_r_pkg_version(is_fork, package_version)|indent }}
+ {{ macros.github_change_r_pkg_version(is_fork,
arrow.no_rc_r_version)|indent }}
- name: Save Version
id: save-version
shell: bash
@@ -281,7 +277,7 @@ jobs:
{{ macros.github_test_r_src_pkg()|indent(8) }}
'
- name: Upload binary artifact
- if: matrix.config.devtoolset
+ if: matrix.config.devtoolset
uses: actions/upload-artifact@v3
with:
name: r-pkg_centos7
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 81a2c24bba..bdf53ff1da 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -957,17 +957,17 @@ tasks:
params:
custom_version: Unset
artifacts:
- - r-lib__libarrow__bin__windows__arrow-[0-9\.]+\.zip
- - r-lib__libarrow__bin__centos-7__arrow-[0-9\.]+\.zip
- - r-lib__libarrow__bin__ubuntu-18.04__arrow-[0-9\.]+\.zip
- - r-lib__libarrow__bin__ubuntu-22.04__arrow-[0-9\.]+\.zip
- - r-pkg__bin__windows__contrib__4.1__arrow_[0-9\.]+\.zip
- - r-pkg__bin__windows__contrib__4.2__arrow_[0-9\.]+\.zip
- - r-pkg__bin__macosx__contrib__4.1__arrow_[0-9\.]+\.tgz
- - r-pkg__bin__macosx__contrib__4.2__arrow_[0-9\.]+\.tgz
- - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_[0-9\.]+\.tgz
- - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_[0-9\.]+\.tgz
- - r-pkg__src__contrib__arrow_[0-9\.]+\.tar\.gz
+ - r-lib__libarrow__bin__windows__arrow-{no_rc_r_version}\.zip
+ - r-lib__libarrow__bin__centos-7__arrow-{no_rc_r_version}\.zip
+ - r-lib__libarrow__bin__ubuntu-18.04__arrow-{no_rc_r_version}\.zip
+ - r-lib__libarrow__bin__ubuntu-22.04__arrow-{no_rc_r_version}\.zip
+ - r-pkg__bin__windows__contrib__4.1__arrow_{no_rc_r_version}\.zip
+ - r-pkg__bin__windows__contrib__4.2__arrow_{no_rc_r_version}\.zip
+ - r-pkg__bin__macosx__contrib__4.1__arrow_{no_rc_r_version}\.tgz
+ - r-pkg__bin__macosx__contrib__4.2__arrow_{no_rc_r_version}\.tgz
+ -
r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_{no_rc_r_version}\.tgz
+ -
r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_{no_rc_r_version}\.tgz
+ - r-pkg__src__contrib__arrow_{no_rc_r_version}\.tar\.gz
########################### Release verification ############################
diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R
index f066e4494d..817563b85c 100644
--- a/r/tools/nixlibs.R
+++ b/r/tools/nixlibs.R
@@ -32,7 +32,7 @@ dev_version <- package_version(VERSION)[1, 4]
# Small dev versions are added for R-only changes during CRAN submission.
if (is.na(dev_version) || dev_version < 100) {
VERSION <- package_version(VERSION)[1, 1:3]
- arrow_repo <-
sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s/libarrow/";, VERSION)
+ arrow_repo <- paste0(getOption("arrow.repo",
sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s";, VERSION)),
"/libarrow/")
} else {
arrow_repo <- paste0(getOption("arrow.dev_repo",
"https://nightlies.apache.org/arrow/r";), "/libarrow/")
}
diff --git a/r/tools/winlibs.R b/r/tools/winlibs.R
index 165c98da5e..d941da4baa 100644
--- a/r/tools/winlibs.R
+++ b/r/tools/winlibs.R
@@ -44,7 +44,10 @@ if
(!file.exists(sprintf("windows/arrow-%s/include/arrow/api.h", VERSION))) {
"/libarrow/bin/windows/arrow-%s.zip"
)
# %1$s uses the first variable for both substitutions
- artifactory <-
"https://apache.jfrog.io/artifactory/arrow/r/%1$s/libarrow/bin/windows/arrow-%1$s.zip";
+ artifactory <- paste0(
+ getOption("arrow.repo",
"https://apache.jfrog.io/artifactory/arrow/r/%1$s";),
+ "/libarrow/bin/windows/arrow-%1$s.zip"
+ )
rwinlib <- "https://github.com/rwinlib/arrow/archive/v%s.zip";
dev_version <- package_version(VERSION)[1, 4]
