This is an automated email from the ASF dual-hosted git repository.
ccondit pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-release.git
The following commit(s) were added to refs/heads/master by this push:
new 75c3958 [YUNIKORN-2424] Expose checksums for reproducible builds
(#171)
75c3958 is described below
commit 75c3958bb322f73fd0e8d629669365ff46f4330d
Author: Craig Condit <[email protected]>
AuthorDate: Tue Feb 20 10:15:34 2024 -0600
[YUNIKORN-2424] Expose checksums for reproducible builds (#171)
As part of generating the release source archive, speculatively
generate build artifacts and embed the resultant checksums in the
README.md for the release.
Closes: #171
---
release-top-level-artifacts/README.md | 22 +++++-
tools/build-release.py | 130 +++++++++++++++++++++++++++++++++-
2 files changed, 149 insertions(+), 3 deletions(-)
diff --git a/release-top-level-artifacts/README.md
b/release-top-level-artifacts/README.md
index b25fa27..dfbc145 100644
--- a/release-top-level-artifacts/README.md
+++ b/release-top-level-artifacts/README.md
@@ -102,11 +102,28 @@ Open `helm-charts/yunikorn/values.yaml` and replace the
docker image information
For more instructions, please refer to [User
Guide](https://yunikorn.apache.org/docs/).
## Deploying a convenience build
-Apache YuniKorn provides a convenience release with pre-build docker images
and helm charts.
+Apache YuniKorn provides a convenience release with pre-built Docker images
and Helm charts.
These can be accessed via the [downloads
page](https://yunikorn.apache.org/community/download) and instructions are
located in the [User Guide](https://yunikorn.apache.org/docs/).
-The convenience build images are multi architecture images. Supported
architectures are `amd64` and `arm64v8`.
+The convenience build images are multi-architecture images. Supported
architectures are `amd64` and `arm64`.
+
+## Reproducible builds
+Building YuniKorn from source generates reproducible build artifacts which
+depend only on the version of YuniKorn built and the go compiler version used.
+
+This release was compiled by the official release manager using Go version
`@GO_VERSION@`
+and generated binary artifacts with the following SHA-512 checksums:
+
+### linux/amd64
+```
+@AMD64_BINARIES@
+```
+
+### linux/arm64
+```
+@ARM64_BINARIES@
+```
## Testing the build
Running the unit tests is supported via the make command.
@@ -133,3 +150,4 @@ Running unit tests adds the following requirements:
* json-server
Please check the [documentation](https://yunikorn.apache.org/docs/) for
further details.
+
diff --git a/tools/build-release.py b/tools/build-release.py
index 8331855..cd92b4a 100755
--- a/tools/build-release.py
+++ b/tools/build-release.py
@@ -96,7 +96,38 @@ def build_release(email_address):
# build the helm package
call_helm(staging_dir, release_base, version, email_address)
- # generate source code tarball
+ # Ensure release dirs are clean (and generate build.date files)
+ clean_release(release_base)
+
+ # generate staging source code tarball
+ tarball_name = release_package_name + "-staging.tar.gz"
+ tarball_path = os.path.join(staging_dir, tarball_name)
+ print("creating tarball %s" % tarball_path)
+ with tarfile.open(tarball_path, "w:gz") as tar:
+ tar.add(os.path.join(release_base, "LICENSE"),
arcname=release_package_name + "/LICENSE")
+ tar.add(os.path.join(release_base, "NOTICE"),
arcname=release_package_name + "/NOTICE")
+ tar.add(release_base, arcname=release_package_name,
filter=exclude_files)
+
+ # generate yunikorn-web reproducible binaries
+ web_hashes_amd64 = build_web_and_generate_hashes(staging_dir,
release_package_name, "x86_64")
+ web_hashes_arm64 = build_web_and_generate_hashes(staging_dir,
release_package_name, "aarch64")
+
+ # generate yunikorn-k8shim reproducible binaries
+ shim_hashes_amd64 = build_shim_and_generate_hashes(staging_dir,
release_package_name, "x86_64")
+ shim_hashes_arm64 = build_shim_and_generate_hashes(staging_dir,
release_package_name, "aarch64")
+
+ # merge hashes
+ hashes_amd64 = "\n".join([shim_hashes_amd64, web_hashes_amd64])
+ hashes_arm64 = "\n".join([shim_hashes_arm64, web_hashes_arm64])
+
+ # remove staging tarball
+ os.remove(tarball_path)
+
+ # update reproducible build information in README
+ go_version = get_go_version()
+ update_reproducible_build_info(release_base, go_version, hashes_amd64,
hashes_arm64)
+
+ # generate final source code tarball
tarball_name = release_package_name + ".tar.gz"
tarball_path = os.path.join(staging_dir, tarball_name)
print("creating tarball %s" % tarball_path)
@@ -203,6 +234,74 @@ def download_sourcecode(base_path, repo_meta):
update_dep_ref(name, os.path.join(base_path, alias))
return sha
+# Run distclean on the source code path
+def clean_release(local_repo_path):
+ print("ensuring local source repo is clean")
+ path = os.getcwd()
+ os.chdir(local_repo_path)
+ retcode = subprocess.call(['make', 'distclean'])
+ if retcode:
+ fail("failed to clean staging repo")
+ os.chdir(path)
+
+# Unpack tarball into tmp dir
+def unpack_staging_tarball(staging_dir, dest_dir, release_name):
+ path = os.getcwd()
+ os.chdir(staging_dir)
+ retcode = subprocess.call(['rm', '-rf', dest_dir])
+ if retcode:
+ fail("failed to clean dest dir")
+ retcode = subprocess.call(['mkdir', dest_dir])
+ if retcode:
+ fail("failed to create dest dir")
+ os.chdir(dest_dir)
+ retcode = subprocess.call(['tar', 'xf', os.path.join(staging_dir,
"%s-staging.tar.gz" % release_name)])
+ if retcode:
+ fail("failed to unpack tarball")
+ os.chdir(path)
+
+# Generate binaries for yunikorn-web and compute checksums
+def build_web_and_generate_hashes(staging_dir, release_name, arch):
+ print("generating reproducible build artifacts for yunikorn-web (%s)" %
arch)
+ path = os.getcwd()
+ tmp_dir = os.path.join(staging_dir, "tmp")
+ release_dir = os.path.join(tmp_dir, release_name)
+ unpack_staging_tarball(staging_dir, tmp_dir, release_name)
+ web_dir = os.path.join(release_dir, "web")
+ os.chdir(web_dir)
+ retcode = subprocess.call(['make', 'HOST_ARCH=' + arch,
'build_server_prod'])
+ if retcode:
+ fail("failed to build yunikorn-web (%s)" % arch)
+ hash = get_checksum("build/prod/yunikorn-web", "yunikorn-web")
+ os.chdir(staging_dir)
+ retcode = subprocess.call(['rm', '-rf', 'tmp'])
+ if retcode:
+ fail("failed to clean temp dir")
+ os.chdir(path)
+ return hash
+
+# Generate binaries for yunikorn-k8shim and compute checksums
+def build_shim_and_generate_hashes(staging_dir, release_name, arch):
+ print("generating reproducible build artifacts for yunikorn-k8shim (%s)" %
arch)
+ path = os.getcwd()
+ tmp_dir = os.path.join(staging_dir, "tmp")
+ release_dir = os.path.join(tmp_dir, release_name)
+ unpack_staging_tarball(staging_dir, tmp_dir, release_name)
+ shim_dir = os.path.join(release_dir, "k8shim")
+ os.chdir(shim_dir)
+ retcode = subprocess.call(['make', 'HOST_ARCH=' + arch, 'scheduler',
'plugin', 'admission'])
+ if retcode:
+ fail("failed to build yunikorn-k8shim (%s)" % arch)
+ adm_hash = get_checksum("build/bin/yunikorn-admission-controller",
"yunikorn-admission-controller")
+ scheduler_hash = get_checksum("build/bin/yunikorn-scheduler",
"yunikorn-scheduler")
+ plugin_hash = get_checksum("build/bin/yunikorn-scheduler-plugin",
"yunikorn-scheduler-plugin")
+ hash = "\n".join([adm_hash, scheduler_hash, plugin_hash])
+ os.chdir(staging_dir)
+ retcode = subprocess.call(['rm', '-rf', 'tmp'])
+ if retcode:
+ fail("failed to clean temp dir")
+ os.chdir(path)
+ return hash
# K8shim depends on yunikorn-core and scheduler-interface
def update_dep_ref_k8shim(local_repo_path):
@@ -301,6 +400,14 @@ def update_required_go_version(base_path, local_repo_path):
print(f" - go version: {go_version}")
replace(os.path.join(base_path, "README.md"), 'Go 1.16', 'Go ' +
go_version)
+# update reproducible build information in README
+def update_reproducible_build_info(base_path, go_version, hashes_amd64,
hashes_arm64):
+ print("recording go compiler used for reproducible builds")
+ replace(os.path.join(base_path, "README.md"), '@GO_VERSION@', go_version)
+ print("recording build artifact hashes (amd64)")
+ replace(os.path.join(base_path, "README.md"), '@AMD64_BINARIES@',
hashes_amd64)
+ print("recording build artifact hashes (arm64)")
+ replace(os.path.join(base_path, "README.md"), '@ARM64_BINARIES@',
hashes_arm64)
# update required Node.js and angular versions in the README.md
def update_required_node_and_angular_versions(base_path, local_repo_path):
@@ -364,6 +471,19 @@ def write_checksum(tarball_file, tarball_name):
sha_file.close()
print("sha512 checksum: %s" % sha)
+# Generate a checksum for a file
+def get_checksum(file_path, file_name):
+ print("generating sha512 checksum for %s" % file_name)
+ h = hashlib.sha512()
+ # read the file and generate the sha
+ with open(file_path, 'rb') as file:
+ while True:
+ data = file.read(65536)
+ if not data:
+ break
+ h.update(data)
+ sha = h.hexdigest()
+ return "%s %s" % (sha, file_name)
# Sign the source archive if an email is provided
def call_gpg(tarball_file, email_address):
@@ -379,6 +499,14 @@ def call_gpg(tarball_file, email_address):
if retcode:
fail("failed to create gpg signature")
+# Determine the specific go compiler in use
+def get_go_version():
+ command = ['go', 'env', 'GOVERSION']
+ result = subprocess.run(command, capture_output=True, text=True)
+ if result.returncode:
+ fail("failed to get go version")
+ output = re.sub(r'^go', r'', result.stdout.strip())
+ return output
# Package the helm chart and sign if an email is provided
def call_helm(staging_dir, base_path, version, email_address):
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
