This is an automated email from the ASF dual-hosted git repository.
jlprat pushed a commit to branch 3.8
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/3.8 by this push:
new 39668b16eb6 KAFKA-17735: release.py must not use home.apache.org
(#17421)
39668b16eb6 is described below
commit 39668b16eb6026b4b4004f0945d609cb663d8431
Author: Colin Patrick McCabe <[email protected]>
AuthorDate: Tue Oct 8 15:40:27 2024 -0700
KAFKA-17735: release.py must not use home.apache.org (#17421)
Previously, Apache Kafka was uploading release candidate (RC) artifacts
to users' home directories on home.apache.org. However, since this
resource has been decommissioned, we need to follow the standard
approach of putting release candidate artifacts into the appropriate
subversion directory, at https://dist.apache.org/repos/dist/dev/kafka/.
Reviewers: Justine Olshan <[email protected]>
---
release/README.md | 1 -
release/release.py | 12 +++++++----
release/runtime.py | 4 ++--
release/sftp.py | 59 ---------------------------------------------------
release/svn.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++
release/templates.py | 24 ++++++++++-----------
6 files changed, 82 insertions(+), 78 deletions(-)
diff --git a/release/README.md b/release/README.md
index 0222164cd3d..1f56f679011 100644
--- a/release/README.md
+++ b/release/README.md
@@ -8,7 +8,6 @@ This directory contains the tools used to publish a release.
* python 3.12
* git
* gpg 2.4
-* sftp
The full instructions for producing a release are available in
https://cwiki.apache.org/confluence/display/KAFKA/Release+Process.
diff --git a/release/release.py b/release/release.py
index 7710789bba8..77d23e66ee0 100644
--- a/release/release.py
+++ b/release/release.py
@@ -70,10 +70,12 @@ import git
import gpg
import notes
import preferences
-import sftp
+import svn
import templates
import textfiles
+from svn import SVN_DEV_URL
+
def get_jdk(version):
"""
@@ -243,7 +245,6 @@ def verify_prerequeisites():
except Exception as e:
fail(f"Pre-requisite not met: {name}. Error: {e}")
prereq('Apache Maven CLI (mvn) in PATH', lambda: "Apache Maven" in
execute("mvn -v"))
- prereq('Apache sftp connection', lambda: sftp.test(apache_id))
prereq("svn CLI in PATH", lambda: "svn" in execute("svn --version"))
prereq("Verifying that you have no unstaged git changes", lambda:
git.has_unstaged_changes())
prereq("Verifying that you have no staged git changes", lambda:
git.has_staged_changes())
@@ -298,7 +299,7 @@ git.commit(f"Bump version to {release_version}")
git.create_tag(rc_tag)
git.switch_branch(starting_branch)
-# Note that we don't use tempfile here because mkdtemp causes problems with
sftp and being able to determine the absolute path to a file.
+# Note that we don't use tempfile here because mkdtemp causes problems with
being able to determine the absolute path to a file.
# Instead we rely on a fixed path
work_dir = os.path.join(repo_dir, ".release_work_dir")
clean_up_work_dir = lambda: cmd("Cleaning up work directory", f"rm -rf
{work_dir}")
@@ -315,6 +316,7 @@ git.clone(repo_dir, 'kafka', cwd=work_dir)
git.create_branch(release_version, rc_tag, cwd=kafka_dir)
current_year = datetime.datetime.now().year
cmd("Verifying the correct year in NOTICE", f"grep {current_year} NOTICE",
cwd=kafka_dir)
+svn.checkout_svn_dev(work_dir)
print("Generating release notes")
try:
@@ -348,7 +350,9 @@ for filename in os.listdir(artifacts_dir):
cmd("Listing artifacts to be uploaded:", f"ls -R {artifacts_dir}")
cmd("Zipping artifacts", f"tar -czf {artifact_name}.tar.gz {artifact_name}",
cwd=work_dir)
-sftp.upload_artifacts(apache_id, artifacts_dir)
+
+confirm_or_fail(f"Going to check in artifacts to svn under
{SVN_DEV_URL}/{rc_tag}. OK?")
+svn.commit_artifacts(rc_tag, artifacts_dir, work_dir)
confirm_or_fail("Going to build and upload mvn artifacts based on these
settings:\n" + textfiles.read(global_gradle_props) + '\nOK?')
cmd("Building and uploading archives", "./gradlewAll publish", cwd=kafka_dir,
env=jdk8_env, shell=True)
diff --git a/release/runtime.py b/release/runtime.py
index a4cfd11e66b..c0d5d67eafb 100644
--- a/release/runtime.py
+++ b/release/runtime.py
@@ -133,12 +133,12 @@ def cmd(action, cmd_arg, *args, **kwargs):
try:
output = execute(cmd_arg, *args, stderr=subprocess.STDOUT,
**kwargs)
print(_prefix("> ", output.strip()))
- return
+ return True
except subprocess.CalledProcessError as e:
print(e.output.decode("utf-8"))
if allow_failure:
- return
+ return False
retry = confirm("Retry?")
diff --git a/release/sftp.py b/release/sftp.py
deleted file mode 100644
index f9ac042cf1e..00000000000
--- a/release/sftp.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-Auxiliary functions to interact with sftp(1).
-"""
-
-import subprocess
-
-from runtime import (
- cmd,
- confirm_or_fail,
- execute,
- fail,
-)
-
-REMOTE_DIR = "public_html"
-
-
-def mkdirp(apache_id, dir):
- cmd_desc = f"Creating '{dir}' in your Apache home directory"
- cmd_str = f"sftp -b - {apache_id}@home.apache.org"
- stdin_str = f"mkdir {dir}\n"
- cmd(cmd_desc, cmd_str, stdin=stdin_str, allow_failure=True)
-
-
-def upload(apache_id, destination, dir):
- cmd_desc = f"Uploading '{dir}' under {REMOTE_DIR} in your Apache home
directory, this may take a while."
- cmd_str = f"sftp -b - {apache_id}@home.apache.org"
- stdin_str = f"cd {destination}\nput -r {dir}\n"
- cmd(cmd_desc, cmd_str, stdin=stdin_str)
-
-
-def upload_artifacts(apache_id, dir):
- mkdirp(apache_id, REMOTE_DIR)
- upload(apache_id, REMOTE_DIR, dir)
- confirm_or_fail(f"Are the artifacts present in your Apache home:
https://home.apache.org/~{apache_id}/ ?")
-
-
-def test(apache_id):
- """
- Test the ability to estalish an sftp session.
- """
- execute(f"sftp {apache_id}@home.apache.org", input="bye")
-
diff --git a/release/svn.py b/release/svn.py
new file mode 100644
index 00000000000..4869f79ddf1
--- /dev/null
+++ b/release/svn.py
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Auxiliary functions to interact with svn(1).
+"""
+
+import os
+import shutil
+import subprocess
+
+from runtime import cmd
+
+SVN_DEV_URL="https://dist.apache.org/repos/dist/dev/kafka";
+
+def delete_old_rc_directory_if_needed(rc_tag, src, work_dir):
+ svn_dev = os.path.join(work_dir, "svn_dev")
+ cmd_desc = f"Check if {rc_tag} exists in the subversion repository."
+ cmd_str = f"svn info --show-item revision {SVN_DEV_URL}/{rc_tag}"
+ if not cmd(cmd_desc, cmd_str, cwd = svn_dev, allow_failure = True):
+ print(f"Nothing under {SVN_DEV_URL}/{rc_tag}. Continuing.")
+ return
+ cmd_desc = f"Committing the deletion of {SVN_DEV_URL}/{rc_tag} from the
svn repo."
+ cmd_str = f"svn delete -m Remove_{rc_tag} {SVN_DEV_URL}/{rc_tag}"
+ cmd(cmd_desc, cmd_str, cwd = svn_dev)
+
+def commit_artifacts(rc_tag, src, work_dir):
+ delete_old_rc_directory_if_needed(rc_tag, src, work_dir)
+ svn_dev = os.path.join(work_dir, "svn_dev")
+ dst = os.path.join(svn_dev, rc_tag)
+ print(f"Copying {src} to {dst}")
+ shutil.copytree(src, dst)
+ cmd_desc = f"Adding {SVN_DEV_URL}/{rc_tag} to the svn repo."
+ cmd_str = f"svn add ./{rc_tag}"
+ cmd(cmd_desc, cmd_str, cwd = svn_dev)
+ cmd_desc = f"Committing the addition of {SVN_DEV_URL}/{rc_tag} to the svn
repo. Please wait, this may take a while."
+ cmd_str = f"svn commit -m Add_{rc_tag}"
+ cmd(cmd_desc, cmd_str, cwd = svn_dev)
+
+def checkout_svn_dev(work_dir):
+ svn_dev = os.path.join(work_dir, "svn_dev")
+ if os.path.exists(svn_dev):
+ shutil.rmtree(svn_dev)
+ cmd_desc = f"Checking out {SVN_DEV_URL} at {svn_dev}"
+ cmd_str = f"svn checkout --depth empty {SVN_DEV_URL}/ {svn_dev}"
+ cmd(cmd_desc, cmd_str)
diff --git a/release/templates.py b/release/templates.py
index 5eb3fc74f93..89826e1097c 100644
--- a/release/templates.py
+++ b/release/templates.py
@@ -167,23 +167,23 @@ Now you should sanity check it before proceeding. All
subsequent steps start mak
Some suggested steps:
- * Grab the source archive and make sure it compiles:
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz
- * Grab one of the binary distros and run the quickstarts against them:
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka_2.13-{release_version}.tgz
- * Extract and verify one of the site docs jars:
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka_2.13-{release_version}-site-docs.tgz
+ * Grab the source archive and make sure it compiles:
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz
+ * Grab one of the binary distros and run the quickstarts against them:
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}.tgz
+ * Extract and verify one of the site docs jars:
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}-site-docs.tgz
* Build a sample against jars in the staging repo: (TODO: Can we get a
temporary URL before "closing" the staged artifacts?)
* Validate GPG signatures on at least one file:
- wget
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz
&&
- wget
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.asc
&&
- wget
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.md5
&&
- wget
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.sha1
&&
- wget
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.sha512
&&
+ wget
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz
&&
+ wget
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.asc
&&
+ wget
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.md5
&&
+ wget
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.sha1
&&
+ wget
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.sha512
&&
gpg --verify kafka-{release_version}-src.tgz.asc
kafka-{release_version}-src.tgz &&
gpg --print-md md5 kafka-{release_version}-src.tgz | diff -
kafka-{release_version}-src.tgz.md5 &&
gpg --print-md sha1 kafka-{release_version}-src.tgz | diff -
kafka-{release_version}-src.tgz.sha1 &&
gpg --print-md sha512 kafka-{release_version}-src.tgz | diff -
kafka-{release_version}-src.tgz.sha512 &&
rm kafka-{release_version}-src.tgz* &&
echo "OK" || echo "Failed"
- * Validate the javadocs look ok. They are at
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/javadoc/
+ * Validate the javadocs look ok. They are at
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/javadoc/
*******************************************************************************************************************************************************
"""
@@ -201,7 +201,7 @@ This is the first candidate for release of Apache Kafka
{release_version}.
<DESCRIPTION OF MAJOR CHANGES, INCLUDE INDICATION OF MAJOR/MINOR RELEASE>
Release notes for the {release_version} release:
-https://home.apache.org/~{apache_id}/kafka-{rc_tag}/RELEASE_NOTES.html
+https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/RELEASE_NOTES.html
*** Please download, test and vote by <VOTING DEADLINE, e.g. Monday, March 28,
9am PT>
<THE RELEASE POLICY
(https://www.apache.org/legal/release-policy.html#release-approval) REQUIRES
VOTES TO BE OPEN FOR MINIMUM OF 3 DAYS THEREFORE VOTING DEADLINE SHOULD BE AT
LEAST 72 HOURS FROM THE TIME THIS EMAIL IS SENT.>
@@ -210,7 +210,7 @@ Kafka's KEYS file containing PGP keys we use to sign the
release:
https://kafka.apache.org/KEYS
* Release artifacts to be voted upon (source and binary):
-https://home.apache.org/~{apache_id}/kafka-{rc_tag}/
+https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/
<USE docker/README.md FOR STEPS TO CREATE RELEASE CANDIDATE DOCKER IMAGE>
* Docker release artifacts to be voted upon:
@@ -221,7 +221,7 @@ apache/kafka-native:{rc_tag}
https://repository.apache.org/content/groups/staging/org/apache/kafka/
* Javadoc:
-https://home.apache.org/~{apache_id}/kafka-{rc_tag}/javadoc/
+https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/javadoc/
* Tag to be voted upon (off {dev_branch} branch) is the {release_version} tag:
https://github.com/apache/kafka/releases/tag/{rc_tag}
