This is an automated email from the ASF dual-hosted git repository. assignuser pushed a commit to branch release-20.0.0-rc1 in repository https://gitbox.apache.org/repos/asf/arrow.git
commit 71a2a293cd659766996500f29e059a57749f3942 Author: Sutou Kouhei <[email protected]> AuthorDate: Tue Apr 15 05:02:13 2025 +0900 GH-46083: [Release][Packages] Use Artifactory for APT/Yum repositories again (#46108) ### Rationale for this change It seems that ASF's Maven repository (Nexus 2) isn't suitable for APT/Yum repositories because it can't update existing files. We need to update existing files for APT/Yum repositories' metadata. Let's keep using Artifactory for 20.0.0. We may revisit this after 20.0.0 release. ### What changes are included in this PR? * Use Artifatory again. * Use packages.apache.org not apache.jfrog.io. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. (Keep the existing URL.) * GitHub Issue: #46083 Lead-authored-by: Sutou Kouhei <[email protected]> Co-authored-by: Sutou Kouhei <[email protected]> Co-authored-by: Jacob Wujciak-Jens <[email protected]> Signed-off-by: Jacob Wujciak-Jens <[email protected]> --- dev/release/.env.example | 12 - dev/release/02-source-test.rb | 12 +- dev/release/02-source.sh | 12 +- dev/release/05-binary-upload.sh | 12 +- dev/release/binary-task.rb | 459 ++++----------------- dev/release/download_rc_binaries.py | 12 +- dev/release/post-03-binary.sh | 10 +- dev/release/verify-apt.sh | 11 +- dev/release/verify-yum.sh | 35 +- .../apache-arrow-apt-source/debian/rules | 2 +- .../apache-arrow-release/yum/Apache-Arrow.repo | 12 +- ruby/red-arrow/red-arrow.gemspec | 2 +- 12 files changed, 135 insertions(+), 456 deletions(-) diff --git a/dev/release/.env.example b/dev/release/.env.example index e751ce4dbc..c1e012aacb 100644 --- a/dev/release/.env.example +++ b/dev/release/.env.example @@ -31,18 +31,6 @@ # You must set this. #ARTIFACTORY_API_KEY=secret -# The Apache Sofotware Foundation ID to upload artifacts to -# repository.apache.org. -# -# You must set this. -#ASF_USER=kou - -# The Apache Sofotware Foundation password to upload artifacts to -# repository.apache.org. -# -# You must set this. -#ASF_PASSWORD=secret - # The GitHub token to upload artifacts to GitHub Release. # # You must set this. diff --git a/dev/release/02-source-test.rb b/dev/release/02-source-test.rb index 85b58ab206..3a9398531d 100644 --- a/dev/release/02-source-test.rb +++ b/dev/release/02-source-test.rb @@ -160,12 +160,12 @@ The vote will be open for at least 72 hours. [1]: https://github.com/apache/arrow/issues?q=is%3Aissue+milestone%3A#{@release_version}+is%3Aclosed [2]: https://github.com/apache/arrow/tree/#{@current_commit} [3]: https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-#{@release_version}-rc0 -[4]: https://apache.jfrog.io/artifactory/arrow/almalinux-rc/ -[5]: https://apache.jfrog.io/artifactory/arrow/amazon-linux-rc/ -[6]: https://apache.jfrog.io/artifactory/arrow/centos-rc/ -[7]: https://apache.jfrog.io/artifactory/arrow/debian-rc/ -[8]: https://apache.jfrog.io/artifactory/arrow/python-rc/#{@release_version}-rc0 -[9]: https://apache.jfrog.io/artifactory/arrow/ubuntu-rc/ +[4]: https://packages.apache.org/artifactory/arrow/almalinux-rc/ +[5]: https://packages.apache.org/artifactory/arrow/amazon-linux-rc/ +[6]: https://packages.apache.org/artifactory/arrow/centos-rc/ +[7]: https://packages.apache.org/artifactory/arrow/debian-rc/ +[8]: https://packages.apache.org/artifactory/arrow/ubuntu-rc/ +[9]: https://github.com/apache/arrow/releases/tag/apache-arrow-#{@release_version}-rc0 [10]: https://github.com/apache/arrow/blob/#{@current_commit}/CHANGELOG.md [11]: https://arrow.apache.org/docs/developers/release_verification.html [12]: #{verify_pr_url || "null"} diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh index b50aa94936..414a2462dd 100755 --- a/dev/release/02-source.sh +++ b/dev/release/02-source.sh @@ -189,12 +189,12 @@ The vote will be open for at least 72 hours. [1]: https://github.com/apache/arrow/issues?q=is%3Aissue+milestone%3A${version}+is%3Aclosed [2]: https://github.com/apache/arrow/tree/${release_hash} [3]: ${rc_url} -[4]: https://apache.jfrog.io/artifactory/arrow/almalinux-rc/ -[5]: https://apache.jfrog.io/artifactory/arrow/amazon-linux-rc/ -[6]: https://apache.jfrog.io/artifactory/arrow/centos-rc/ -[7]: https://apache.jfrog.io/artifactory/arrow/debian-rc/ -[8]: https://apache.jfrog.io/artifactory/arrow/python-rc/${version}-rc${rc} -[9]: https://apache.jfrog.io/artifactory/arrow/ubuntu-rc/ +[4]: https://packages.apache.org/artifactory/arrow/almalinux-rc/ +[5]: https://packages.apache.org/artifactory/arrow/amazon-linux-rc/ +[6]: https://packages.apache.org/artifactory/arrow/centos-rc/ +[7]: https://packages.apache.org/artifactory/arrow/debian-rc/ +[8]: https://packages.apache.org/artifactory/arrow/ubuntu-rc/ +[9]: https://github.com/apache/arrow/releases/tag/apache-arrow-${version}-rc${rc} [10]: https://github.com/apache/arrow/blob/${release_hash}/CHANGELOG.md [11]: https://arrow.apache.org/docs/developers/release_verification.html [12]: ${verify_pr_url} diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh index ae3ff936a7..327517e801 100755 --- a/dev/release/05-binary-upload.sh +++ b/dev/release/05-binary-upload.sh @@ -121,19 +121,19 @@ rake_tasks=() apt_targets=() yum_targets=() if [ "${UPLOAD_ALMALINUX}" -gt 0 ]; then - rake_tasks+=(yum:rc:artifactory yum:rc) + rake_tasks+=(yum:rc) yum_targets+=(almalinux) fi if [ "${UPLOAD_AMAZON_LINUX}" -gt 0 ]; then - rake_tasks+=(yum:rc:artifactory yum:rc) + rake_tasks+=(yum:rc) yum_targets+=(amazon-linux) fi if [ "${UPLOAD_CENTOS}" -gt 0 ]; then - rake_tasks+=(yum:rc:artifactory yum:rc) + rake_tasks+=(yum:rc) yum_targets+=(centos) fi if [ "${UPLOAD_DEBIAN}" -gt 0 ]; then - rake_tasks+=(apt:rc:artifactory apt:rc) + rake_tasks+=(apt:rc) apt_targets+=(debian) fi if [ "${UPLOAD_DOCS}" -gt 0 ]; then @@ -143,7 +143,7 @@ if [ "${UPLOAD_R}" -gt 0 ]; then rake_tasks+=(r:rc) fi if [ "${UPLOAD_UBUNTU}" -gt 0 ]; then - rake_tasks+=(apt:rc:artifactory apt:rc) + rake_tasks+=(apt:rc) apt_targets+=(ubuntu) fi rake_tasks+=(summary:rc) @@ -162,8 +162,6 @@ docker_run \ )" \ ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \ ARTIFACTS_DIR="${tmp_dir}/artifacts" \ - ASF_PASSWORD="${ASF_PASSWORD}" \ - ASF_USER="${ASF_USER}" \ DEB_PACKAGE_NAME="${DEB_PACKAGE_NAME:-}" \ DRY_RUN="${DRY_RUN:-no}" \ GPG_KEY_ID="${GPG_KEY_ID}" \ diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb index a5ffcfbcaa..30805cd339 100644 --- a/dev/release/binary-task.rb +++ b/dev/release/binary-task.rb @@ -693,7 +693,7 @@ class BinaryTask end def build_api_url(path, parameters) - uri_string = "https://apache.jfrog.io/artifactory/api/#{path}"; + uri_string = "https://packages.apache.org/artifactory/api/#{path}"; unless parameters.empty? uri_string << "?" escaped_parameters = parameters.collect do |key, value| @@ -705,7 +705,7 @@ class BinaryTask end def build_deployed_url(path) - uri_string = "https://apache.jfrog.io/artifactory/arrow"; + uri_string = "https://packages.apache.org/artifactory/arrow"; uri_string << "/#{@prefix}" unless @prefix.nil? uri_string << "/#{path}" URI(uri_string) @@ -1623,139 +1623,6 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; merged_dir = "#{apt_rc_repositories_dir}/merged" upload_dir = "#{apt_rc_repositories_dir}/upload" - namespace :artifactory do - desc "Copy .deb packages" - task :copy do - apt_targets.each do |distribution, code_name, component| - progress_label = "Copying: #{distribution} #{code_name}" - progress_reporter = ProgressReporter.new(progress_label) - - distribution_dir = "#{incoming_dir}/#{distribution}" - pool_dir = "#{distribution_dir}/pool/#{code_name}" - rm_rf(pool_dir, verbose: verbose?) - mkdir_p(pool_dir, verbose: verbose?) - source_dir_prefix = "#{artifacts_dir}/#{distribution}-#{code_name}" - Dir.glob("#{source_dir_prefix}-*/apache-arrow-apt-source*") do |path| - base_name = File.basename(path) - package_name = "apache-arrow-apt-source" - destination_path = [ - pool_dir, - component, - package_name[0], - package_name, - base_name, - ].join("/") - copy_artifact(path, - destination_path, - progress_reporter) - if base_name.end_with?(".deb") - latest_apt_source_package_path = [ - distribution_dir, - "#{package_name}-latest-#{code_name}.deb" - ].join("/") - copy_artifact(path, - latest_apt_source_package_path, - progress_reporter) - end - end - progress_reporter.finish - end - end - - desc "Download dists/ for RC APT repositories" - task :download do - apt_distributions.each do |distribution| - not_checksum_pattern = /.+(?<!\.asc|\.sha512)\z/ - base_distribution_dir = "#{base_dir}/#{distribution}" - pattern = not_checksum_pattern - download_distribution(:artifactory, - distribution, - base_distribution_dir, - :base, - pattern: pattern, - prefix: "dists") - end - end - - desc "Sign .deb packages" - task :sign do - apt_distributions.each do |distribution| - distribution_dir = "#{incoming_dir}/#{distribution}" - Dir.glob("#{distribution_dir}/**/*.dsc") do |path| - begin - sh({"LANG" => "C"}, - "gpg", - "--verify", - path, - out: IO::NULL, - err: IO::NULL, - verbose: false) - rescue - sh("debsign", - "--no-re-sign", - "-k#{gpg_key_id}", - path, - out: default_output, - verbose: verbose?) - end - end - sign_dir(distribution, distribution_dir) - end - end - - desc "Update RC APT repositories" - task :update do - apt_update(base_dir, incoming_dir, merged_dir) - apt_targets.each do |distribution, code_name, component| - dists_dir = "#{merged_dir}/#{distribution}/dists/#{code_name}" - next unless File.exist?(dists_dir) - sign_dir("#{distribution} #{code_name}", - dists_dir) - end - end - - desc "Upload .deb packages and RC APT repositories" - task :upload do - apt_distributions.each do |distribution| - upload_distribution_dir = "#{upload_dir}/#{distribution}" - incoming_distribution_dir = "#{incoming_dir}/#{distribution}" - merged_dists_dir = "#{merged_dir}/#{distribution}/dists" - - rm_rf(upload_distribution_dir, verbose: verbose?) - mkdir_p(upload_distribution_dir, verbose: verbose?) - Dir.glob("#{incoming_distribution_dir}/*") do |path| - next if File.basename(path) == "dists" - cp_r(path, - upload_distribution_dir, - preserve: true, - verbose: verbose?) - end - cp_r(merged_dists_dir, - upload_distribution_dir, - preserve: true, - verbose: verbose?) - write_uploaded_files(upload_distribution_dir) - uploader = ArtifactoryUploader.new(api_key: artifactory_api_key, - distribution: distribution, - rc: rc, - source: upload_distribution_dir, - staging: staging?) - uploader.upload - end - end - end - - desc "Release RC APT repositories to Artifactory" - apt_rc_artifactory_tasks = [ - "apt:rc:artifactory:copy", - "apt:rc:artifactory:download", - "apt:rc:artifactory:sign", - "apt:rc:artifactory:update", - "apt:rc:artifactory:upload", - ] - apt_rc_artifactory_tasks.unshift("apt:staging:prepare") if staging? - task :artifactory => apt_rc_artifactory_tasks - desc "Copy .deb packages" task :copy do apt_targets.each do |distribution, code_name, component| @@ -1767,8 +1634,7 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; rm_rf(pool_dir, verbose: verbose?) mkdir_p(pool_dir, verbose: verbose?) source_dir_prefix = "#{artifacts_dir}/#{distribution}-#{code_name}" - Dir.glob("#{source_dir_prefix}*/**/*") do |path| - next if File.directory?(path) + Dir.glob("#{source_dir_prefix}-*/*") do |path| base_name = File.basename(path) package_name = ENV["DEB_PACKAGE_NAME"] if package_name.nil? or package_name.empty? @@ -1809,7 +1675,7 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; not_checksum_pattern = /.+(?<!\.asc|\.sha512)\z/ base_distribution_dir = "#{base_dir}/#{distribution}" pattern = not_checksum_pattern - download_distribution(:maven_repository, + download_distribution(:artifactory, distribution, base_distribution_dir, :base, @@ -1875,13 +1741,12 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; upload_distribution_dir, preserve: true, verbose: verbose?) - uploader = - MavenRepositoryUploader.new(asf_user: asf_user, - asf_password: asf_password, - distribution: distribution, - rc: rc, - source: upload_distribution_dir, - staging: staging?) + write_uploaded_files(upload_distribution_dir) + uploader = ArtifactoryUploader.new(api_key: artifactory_api_key, + distribution: distribution, + rc: rc, + source: upload_distribution_dir, + staging: staging?) uploader.upload end end @@ -1895,6 +1760,7 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; "apt:rc:update", "apt:rc:upload", ] + apt_rc_tasks.unshift("apt:staging:prepare") if staging? task :rc => apt_rc_tasks end end @@ -1903,13 +1769,11 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; directory apt_release_repositories_dir namespace :apt do - namespace :artifactory do - desc "Release APT repository on Artifactory" - task :release do - apt_distributions.each do |distribution| - release_distribution(distribution, - list: uploaded_files_name) - end + desc "Release APT repository" + task :release do + apt_distributions.each do |distribution| + release_distribution(distribution, + list: uploaded_files_name) end end end @@ -2071,20 +1935,18 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; def define_yum_staging_tasks namespace :yum do - namespace :artifactory do - namespace :staging do - desc "Prepare staging environment for Yum repositories on Artifactory" - task :prepare do - yum_distributions.each do |distribution| - prepare_staging(distribution) - end + namespace :staging do + desc "Prepare staging environment for Yum repositories" + task :prepare do + yum_distributions.each do |distribution| + prepare_staging(distribution) end + end - desc "Delete staging environment for Yum repositories on Artifactory" - task :delete do - yum_distributions.each do |distribution| - delete_staging(distribution) - end + desc "Delete staging environment for Yum repositories" + task :delete do + yum_distributions.each do |distribution| + delete_staging(distribution) end end end @@ -2098,159 +1960,6 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; incoming_dir = "#{yum_rc_repositories_dir}/incoming" upload_dir = "#{yum_rc_repositories_dir}/upload" - namespace :artifactory do - desc "Copy RPM packages" - task :copy do - yum_targets.each do |distribution, distribution_version| - progress_label = "Copying: #{distribution} #{distribution_version}" - progress_reporter = ProgressReporter.new(progress_label) - - destination_prefix = [ - incoming_dir, - distribution, - distribution_version, - ].join("/") - rm_rf(destination_prefix, verbose: verbose?) - source_dir_prefix = - "#{artifacts_dir}/#{distribution}-#{distribution_version}" - Dir.glob("#{source_dir_prefix}*/apache-arrow-release-*") do |path| - base_name = File.basename(path) - type = base_name.split(".")[-2] - destination_paths = [] - case type - when "src" - destination_paths << [ - destination_prefix, - "Source", - "SPackages", - base_name, - ].join("/") - when "noarch" - yum_architectures.each do |architecture| - destination_paths << [ - destination_prefix, - architecture, - "Packages", - base_name, - ].join("/") - end - else - destination_paths << [ - destination_prefix, - type, - "Packages", - base_name, - ].join("/") - end - destination_paths.each do |destination_path| - copy_artifact(path, - destination_path, - progress_reporter) - end - case base_name - when /\A(apache-arrow-release)-.*\.noarch\.rpm\z/ - package_name = $1 - latest_release_package_path = [ - destination_prefix, - "#{package_name}-latest.rpm" - ].join("/") - copy_artifact(path, - latest_release_package_path, - progress_reporter) - end - end - - progress_reporter.finish - end - end - - desc "Download repodata for RC Yum repositories" - task :download do - yum_distributions.each do |distribution| - distribution_dir = "#{base_dir}/#{distribution}" - download_distribution(:artifactory, - distribution, - distribution_dir, - :base, - pattern: /\/repodata\//) - end - end - - desc "Sign RPM packages" - task :sign do - rpm_sign(incoming_dir) - yum_targets.each do |distribution, distribution_version| - source_dir = [ - incoming_dir, - distribution, - distribution_version, - ].join("/") - sign_dir("#{distribution}-#{distribution_version}", - source_dir) - end - end - - desc "Update RC Yum repositories" - task :update do - yum_update(base_dir, incoming_dir) - yum_targets.each do |distribution, distribution_version| - target_dir = [ - incoming_dir, - distribution, - distribution_version, - ].join("/") - target_dir = Pathname(target_dir) - next unless target_dir.directory? - target_dir.glob("*") do |arch_dir| - next unless arch_dir.directory? - sign_label = - "#{distribution}-#{distribution_version} #{arch_dir.basename}" - sign_dir(sign_label, - arch_dir.to_s) - end - end - end - - desc "Upload RC Yum repositories on Artifactory" - task :upload => yum_rc_repositories_dir do - yum_distributions.each do |distribution| - incoming_target_dir = "#{incoming_dir}/#{distribution}" - upload_target_dir = "#{upload_dir}/#{distribution}" - - rm_rf(upload_target_dir, verbose: verbose?) - mkdir_p(upload_target_dir, verbose: verbose?) - cp_r(Dir.glob("#{incoming_target_dir}/*"), - upload_target_dir.to_s, - preserve: true, - verbose: verbose?) - write_uploaded_files(upload_target_dir) - - uploader = ArtifactoryUploader.new(api_key: artifactory_api_key, - distribution: distribution, - rc: rc, - source: upload_target_dir, - staging: staging?, - # Don't remove old repodata - # because our implementation - # doesn't support it. - sync: false, - sync_pattern: /\/repodata\//) - uploader.upload - end - end - end - - desc "Release RC Yum packages on Artifactory" - yum_rc_artifactory_tasks = [ - "yum:rc:artifactory:copy", - "yum:rc:artifactory:download", - "yum:rc:artifactory:sign", - "yum:rc:artifactory:update", - "yum:rc:artifactory:upload", - ] - yum_rc_artifactory_tasks.unshift("yum:staging:prepare") if staging? - task :artifactory => yum_rc_artifactory_tasks - desc "Copy RPM packages" task :copy do yum_targets.each do |distribution, distribution_version| @@ -2265,8 +1974,7 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; rm_rf(destination_prefix, verbose: verbose?) source_dir_prefix = "#{artifacts_dir}/#{distribution}-#{distribution_version}" - Dir.glob("#{source_dir_prefix}*/**/*") do |path| - next if File.directory?(path) + Dir.glob("#{source_dir_prefix}*/*.rpm") do |path| base_name = File.basename(path) type = base_name.split(".")[-2] destination_paths = [] @@ -2321,7 +2029,7 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; task :download do yum_distributions.each do |distribution| distribution_dir = "#{base_dir}/#{distribution}" - download_distribution(:maven_repository, + download_distribution(:artifactory, distribution, distribution_dir, :base, @@ -2376,19 +2084,18 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; upload_target_dir.to_s, preserve: true, verbose: verbose?) + write_uploaded_files(upload_target_dir) - uploader = MavenRepositoryUploader.new(asf_user: asf_user, - asf_password: asf_password, - distribution: distribution, - rc: rc, - source: upload_target_dir, - # Don't remove old - # repodata. Because - # removing files - # aren't supported - # on Maven repository. - sync: false, - sync_pattern: /\/repodata\//) + uploader = ArtifactoryUploader.new(api_key: artifactory_api_key, + distribution: distribution, + rc: rc, + source: upload_target_dir, + staging: staging?, + # Don't remove old repodata + # because our implementation + # doesn't support it. + sync: false, + sync_pattern: /\/repodata\//) uploader.upload end end @@ -2402,6 +2109,7 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; "yum:rc:update", "yum:rc:upload", ] + yum_rc_tasks.unshift("yum:staging:prepare") if staging? task :rc => yum_rc_tasks end end @@ -2410,33 +2118,31 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; directory yum_release_repositories_dir namespace :yum do - namespace :artifactory do - desc "Release Yum packages on Artifactory" - task :release => yum_release_repositories_dir do - yum_distributions.each do |distribution| - release_distribution(distribution, - list: uploaded_files_name) - - distribution_dir = "#{yum_release_repositories_dir}/#{distribution}" - download_distribution(distribution, - distribution_dir, - :rc, - pattern: /\/repodata\//) - uploader = ArtifactoryUploader.new(api_key: artifactory_api_key, - distribution: distribution, - source: distribution_dir, - staging: staging?, - # Don't remove old repodata for - # unsupported distribution version - # such as Amazon Linux 2. - # This keeps garbage in repodata/ - # for currently available - # distribution versions but we - # accept it for easy to implement. - sync: false, - sync_pattern: /\/repodata\//) - uploader.upload - end + desc "Release Yum packages" + task :release => yum_release_repositories_dir do + yum_distributions.each do |distribution| + release_distribution(distribution, + list: uploaded_files_name) + + distribution_dir = "#{yum_release_repositories_dir}/#{distribution}" + download_distribution(distribution, + distribution_dir, + :rc, + pattern: /\/repodata\//) + uploader = ArtifactoryUploader.new(api_key: artifactory_api_key, + distribution: distribution, + source: distribution_dir, + staging: staging?, + # Don't remove old repodata for + # unsupported distribution version + # such as Amazon Linux 2. + # This keeps garbage in repodata/ + # for currently available + # distribution versions but we + # accept it for easy to implement. + sync: false, + sync_pattern: /\/repodata\//) + uploader.upload end end end @@ -2592,17 +2298,11 @@ APT::FTPArchive::Release::Description "#{apt_repository_description}"; suffix << "-staging" if staging? puts(<<-SUMMARY) Success! The release candidate binaries are available here: - https://repository.apache.org/content/repositories/staging/org/apache/arrow/almalinux/ - https://repository.apache.org/content/repositories/staging/org/apache/arrow/amazon-linux/ - https://repository.apache.org/content/repositories/staging/org/apache/arrow/centos/ - https://repository.apache.org/content/repositories/staging/org/apache/arrow/debian/ - https://repository.apache.org/content/repositories/staging/org/apache/arrow/ubuntu/ - - https://apache.jfrog.io/artifactory/arrow/almalinux#{suffix}-rc/ - https://apache.jfrog.io/artifactory/arrow/amazon-linux#{suffix}-rc/ - https://apache.jfrog.io/artifactory/arrow/centos#{suffix}-rc/ - https://apache.jfrog.io/artifactory/arrow/debian#{suffix}-rc/ - https://apache.jfrog.io/artifactory/arrow/ubuntu#{suffix}-rc/ + https://packages.apache.org/artifactory/arrow/almalinux#{suffix}-rc/ + https://packages.apache.org/artifactory/arrow/amazon-linux#{suffix}-rc/ + https://packages.apache.org/artifactory/arrow/centos#{suffix}-rc/ + https://packages.apache.org/artifactory/arrow/debian#{suffix}-rc/ + https://packages.apache.org/artifactory/arrow/ubuntu#{suffix}-rc/ SUMMARY end @@ -2611,21 +2311,12 @@ Success! The release candidate binaries are available here: suffix = "" suffix << "-staging" if staging? puts(<<-SUMMARY) -Click the "release" button manually at -https://repository.apache.org/#stagingRepositories . - Success! The release binaries are available here: - https://repo1.maven.org/maven2/org/apache/arrow/almalinux/ - https://repo1.maven.org/maven2/org/apache/arrow/amazon-linux/ - https://repo1.maven.org/maven2/org/apache/arrow/centos/ - https://repo1.maven.org/maven2/org/apache/arrow/debian/ - https://repo1.maven.org/maven2/org/apache/arrow/ubuntu/ - - https://apache.jfrog.io/artifactory/arrow/almalinux#{suffix}/ - https://apache.jfrog.io/artifactory/arrow/amazon-linux#{suffix}/ - https://apache.jfrog.io/artifactory/arrow/centos#{suffix}/ - https://apache.jfrog.io/artifactory/arrow/debian#{suffix}/ - https://apache.jfrog.io/artifactory/arrow/ubuntu#{suffix}/ + https://packages.apache.org/artifactory/arrow/almalinux#{suffix}/ + https://packages.apache.org/artifactory/arrow/amazon-linux#{suffix}/ + https://packages.apache.org/artifactory/arrow/centos#{suffix}/ + https://packages.apache.org/artifactory/arrow/debian#{suffix}/ + https://packages.apache.org/artifactory/arrow/ubuntu#{suffix}/ SUMMARY end end diff --git a/dev/release/download_rc_binaries.py b/dev/release/download_rc_binaries.py index e9defccc63..3bc0012116 100755 --- a/dev/release/download_rc_binaries.py +++ b/dev/release/download_rc_binaries.py @@ -152,6 +152,10 @@ class Downloader: return (int(match.group(1)), int(match.group(2)), int(match.group(3))) +class Artifactory(Downloader): + URL_ROOT = "https://packages.apache.org/artifactory/arrow"; + + class Maven(Downloader): URL_ROOT = "https://repository.apache.org"; + \ "/content/repositories/staging/org/apache/arrow" @@ -278,11 +282,11 @@ def download_rc_binaries(version, rc_number, re_match=None, dest=None, prefix = '' filter = None elif package_type in ARROW_REPOSITORY_PACKAGE_TYPES: - downloader = Maven() - prefix = package_type + downloader = Artifactory() + prefix = f'{package_type}-rc' else: - downloader = Maven() - prefix = f'{package_type}/{version_string}' + downloader = Artifactory() + prefix = f'{package_type}-rc/{version_string}' filter = None files = downloader.get_file_list(prefix, filter=filter) downloader.download_files(files, re_match=re_match, dest=dest, diff --git a/dev/release/post-03-binary.sh b/dev/release/post-03-binary.sh index 1ebc97aac3..d0487585dd 100755 --- a/dev/release/post-03-binary.sh +++ b/dev/release/post-03-binary.sh @@ -57,26 +57,26 @@ rake_tasks=() apt_targets=() yum_targets=() if [ "${DEPLOY_ALMALINUX}" -gt 0 ]; then - rake_tasks+=(yum:artifactory:release) + rake_tasks+=(yum:release) yum_targets+=(almalinux) fi if [ "${DEPLOY_AMAZON_LINUX}" -gt 0 ]; then - rake_tasks+=(yum:artifactory:release) + rake_tasks+=(yum:release) yum_targets+=(amazon-linux) fi if [ "${DEPLOY_CENTOS}" -gt 0 ]; then - rake_tasks+=(yum:artifactory:release) + rake_tasks+=(yum:release) yum_targets+=(centos) fi if [ "${DEPLOY_DEBIAN}" -gt 0 ]; then - rake_tasks+=(apt:artifactory:release) + rake_tasks+=(apt:release) apt_targets+=(debian) fi if [ "${DEPLOY_R}" -gt 0 ]; then rake_tasks+=(r:release) fi if [ "${DEPLOY_UBUNTU}" -gt 0 ]; then - rake_tasks+=(apt:artifactory:release) + rake_tasks+=(apt:release) apt_targets+=(ubuntu) fi rake_tasks+=(summary:release) diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh index 255b414c27..02296c2079 100755 --- a/dev/release/verify-apt.sh +++ b/dev/release/verify-apt.sh @@ -66,11 +66,10 @@ ${APT_INSTALL} \ code_name="$(lsb_release --codename --short)" distribution="$(lsb_release --id --short | tr 'A-Z' 'a-z')" -production_repository_base_url="https://repo1.maven.org/maven2/org/apache/arrow/${distribution}"; -staging_repository_base_url="https://repository.apache.org/content/repositories/staging/org/apache/arrow/${distribution}"; -repository_base_url="${production_repository_base_url}" +artifactory_base_url="https://packages.apache.org/artifactory/arrow/${distribution}"; if [ "${TYPE}" = "rc" ]; then - repository_base_url="${staging_repository_base_url}" + suffix=${TYPE%-release} + artifactory_base_url+="-${suffix}" fi workaround_missing_packages=() @@ -106,7 +105,7 @@ else apt_source_base_name="apache-arrow-apt-source-latest-${code_name}.deb" curl \ --output "${apt_source_base_name}" \ - "${repository_base_url}/${apt_source_base_name}" + "${artifactory_base_url}/${apt_source_base_name}" ${APT_INSTALL} "./${apt_source_base_name}" fi @@ -132,7 +131,7 @@ else rc) sed \ -i"" \ - -e "s,^URIs: ${production_repository_base_url},URIs: ${staging_repository_base_url},g" \ + -e "s,^URIs: \\(.*\\)/,URIs: \\1-${suffix}/,g" \ /etc/apt/sources.list.d/apache-arrow.sources ;; esac diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh index 7bc8f4583c..8691e538ec 100755 --- a/dev/release/verify-yum.sh +++ b/dev/release/verify-yum.sh @@ -36,12 +36,7 @@ SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TOP_SOURCE_DIR="${SOURCE_DIR}/../.." local_prefix="${TOP_SOURCE_DIR}/dev/tasks/linux-packages" -production_repository_base_url="https://repo1.maven.org/maven2/org/apache/arrow"; -staging_repository_base_url="https://repository.apache.org/content/repositories/staging/org/apache/arrow"; -repository_base_url="${production_repository_base_url}" -if [ "${TYPE}" = "rc" ]; then - repository_base_url="${staging_repository_base_url}" -fi +artifactory_base_url="https://packages.apache.org/artifactory/arrow"; distribution=$(. /etc/os-release && echo "${ID}") distribution_version=$(. /etc/os-release && echo "${VERSION_ID}" | grep -o "^[0-9]*") @@ -162,29 +157,33 @@ if [ "${TYPE}" = "local" ]; then ${install_command} "${release_path}" else package_version="${VERSION}" + if [ "${TYPE}" = "rc" ]; then + suffix=${TYPE%-release} + distribution_prefix+="-${suffix}" + fi ${install_command} \ - ${repository_base_url}/${distribution_prefix}/${repository_version}/apache-arrow-release-latest.rpm + ${artifactory_base_url}/${distribution_prefix}/${repository_version}/apache-arrow-release-latest.rpm fi if [ "${TYPE}" = "local" ]; then sed \ -i"" \ - -e "s,baseurl=${production_repository_base_url}/,baseurl=file://${local_prefix}/yum/repositories/,g" \ + -e "s,baseurl=https://packages\.apache\.org/artifactory/arrow/,baseurl=file://${local_prefix}/yum/repositories/,g"; \ /etc/yum.repos.d/Apache-Arrow.repo keys="${local_prefix}/KEYS" if [ -f "${keys}" ]; then cp "${keys}" /etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow fi else - case "${TYPE}" in - rc) - suffix=${TYPE%-release} - sed \ - -i"" \ - -e "s,baseurl=${production_repository_base_url},baseurl=${staging_repository_base_url},g" \ - /etc/yum.repos.d/Apache-Arrow.repo - ;; - esac + if [ "${TYPE}" = "rc" ]; then + suffix=${TYPE%-release} + sed \ + -i"" \ + -e "s,/almalinux/,/almalinux-${suffix}/,g" \ + -e "s,/centos/,/centos-${suffix}/,g" \ + -e "s,/amazon-linux/,/amazon-linux-${suffix}/,g" \ + /etc/yum.repos.d/Apache-Arrow.repo + fi fi echo "::endgroup::" @@ -299,7 +298,7 @@ fi echo "::group::Test coexistence with old library" ${uninstall_command} apache-arrow-release if ${install_command} \ - ${repository_base_url}/${distribution_prefix}/${repository_version}/apache-arrow-release-latest.rpm; then + ${artifactory_base_url}/${distribution_prefix}/${repository_version}/apache-arrow-release-latest.rpm; then ${clean_command} all if [ "${have_arrow_libs}" = "yes" ]; then ${install_command} ${enablerepo_epel} arrow-libs diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules index a7d0637bd1..1e3be48c31 100755 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules +++ b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules @@ -24,7 +24,7 @@ override_dh_auto_build: distribution=$$(lsb_release --id --short | tr 'A-Z' 'a-z'); \ code_name=$$(lsb_release --codename --short); \ echo "Types: deb deb-src"; \ - echo "URIs: https://repo1.maven.org/maven2/org/apache/arrow/$${distribution}/";; \ + echo "URIs: https://packages.apache.org/artifactory/arrow/$${distribution}/";; \ echo "Suites: $${code_name}"; \ echo "Components: main"; \ echo "Signed-By: /usr/share/keyrings/apache-arrow-apt-source.asc"; \ diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/Apache-Arrow.repo b/dev/tasks/linux-packages/apache-arrow-release/yum/Apache-Arrow.repo index 7eb7670626..6a27acf9a2 100644 --- a/dev/tasks/linux-packages/apache-arrow-release/yum/Apache-Arrow.repo +++ b/dev/tasks/linux-packages/apache-arrow-release/yum/Apache-Arrow.repo @@ -17,42 +17,42 @@ [apache-arrow-almalinux] name=Apache Arrow for AlmaLinux $releasever - $basearch -baseurl=https://repo1.maven.org/maven2/org/apache/arrow/almalinux/$releasever/$basearch/ +baseurl=https://packages.apache.org/artifactory/arrow/almalinux/$releasever/$basearch/ gpgcheck=1 enabled=0 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow [apache-arrow-amazon-linux-2023] name=Apache Arrow for Amazon Linux 2023 - $basearch -baseurl=https://repo1.maven.org/maven2/org/apache/arrow/amazon-linux/2023/$basearch/ +baseurl=https://packages.apache.org/artifactory/arrow/amazon-linux/2023/$basearch/ gpgcheck=1 enabled=0 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow [apache-arrow-centos-stream] name=Apache Arrow for CentOS Stream $releasever - $basearch -baseurl=https://repo1.maven.org/maven2/org/apache/arrow/centos/$stream/$basearch/ +baseurl=https://packages.apache.org/artifactory/arrow/centos/$stream/$basearch/ gpgcheck=1 enabled=0 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow [apache-arrow-centos] name=Apache Arrow for CentOS $releasever - $basearch -baseurl=https://repo1.maven.org/maven2/org/apache/arrow/centos/$releasever/$basearch/ +baseurl=https://packages.apache.org/artifactory/arrow/centos/$releasever/$basearch/ gpgcheck=1 enabled=0 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow [apache-arrow-rhel] name=Apache Arrow for RHEL $releasever - $basearch -baseurl=https://repo1.maven.org/maven2/org/apache/arrow/almalinux/$releasever/$basearch/ +baseurl=https://packages.apache.org/artifactory/arrow/almalinux/$releasever/$basearch/ gpgcheck=1 enabled=0 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow [apache-arrow-rhel7] name=Apache Arrow for RHEL 7 - $basearch -baseurl=https://repo1.maven.org/maven2/org/apache/arrow/centos/7/$basearch/ +baseurl=https://packages.apache.org/artifactory/arrow/centos/7/$basearch/ gpgcheck=1 enabled=0 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow diff --git a/ruby/red-arrow/red-arrow.gemspec b/ruby/red-arrow/red-arrow.gemspec index d23962d096..53e8d8bc08 100644 --- a/ruby/red-arrow/red-arrow.gemspec +++ b/ruby/red-arrow/red-arrow.gemspec @@ -62,7 +62,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency("gio2", ">= 4.2.3") spec.add_runtime_dependency("pkg-config") - repository_url_prefix = "https://repo1.maven.org/maven2/org/apache/arrow"; + repository_url_prefix = "https://packages.apache.org/artifactory/arrow"; [ # Try without additional repository ["amazon_linux", "arrow-glib-devel"],
