This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 5549045eb1 Improve CI: add caching for Bookkeeper old release
downloads and improve maven caching and error logs (#4722)
5549045eb1 is described below
commit 5549045eb16d0817c02fa301af91dbd9dda41f14
Author: Lari Hotari <[email protected]>
AuthorDate: Wed Mar 11 01:55:08 2026 +0200
Improve CI: add caching for Bookkeeper old release downloads and improve
maven caching and error logs (#4722)
---
.github/workflows/bk-ci.yml | 36 ++++++-
.github/workflows/bk-streamstorage-python.yml | 2 +
.../workflows/ci-all-released-versions-update.yaml | 68 ++++++++++++
.github/workflows/java21-daily-build.yml | 14 ++-
.github/workflows/windows-daily-build.yml | 15 ++-
.gitignore | 5 +-
metadata-drivers/etcd/pom.xml | 24 -----
tests/README.md | 6 +-
.../all-released-versions-image/Dockerfile | 43 ++++----
.../all-released-versions-image/pom.xml | 33 ++++++
.../scripts/download-released-versions.sh | 116 +++++++++++++++++++++
.../scripts/install-all-tarballs.sh | 14 ++-
.../scripts/install-tarball.sh | 25 +++--
tests/docker-images/all-versions-image/Dockerfile | 2 +-
14 files changed, 336 insertions(+), 67 deletions(-)
diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml
index 625f87e844..64116a3fe6 100644
--- a/.github/workflows/bk-ci.yml
+++ b/.github/workflows/bk-ci.yml
@@ -72,6 +72,8 @@ jobs:
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-bookkeeper-all-
- name: Set up JDK 17
if: steps.check_changes.outputs.docs_only != 'true'
@@ -118,7 +120,7 @@ jobs:
- step_name: Replication Tests
module: bookkeeper-server
flag: replication
- test_args: "-Dtest='org.apache.bookkeeper.replication.**'"
+ test_args: "-DreuseForks=false
-Dtest='org.apache.bookkeeper.replication.**'"
- step_name: Remaining Tests
module: bookkeeper-server
flag: remaining
@@ -150,6 +152,8 @@ jobs:
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-bookkeeper-all-
- name: Set up JDK 17
uses: actions/setup-java@v4
@@ -193,7 +197,7 @@ jobs:
- name: Upload Surefire reports
uses: actions/upload-artifact@v4
- if: failure()
+ if: ${{ !success() }}
continue-on-error: true
with:
name: unit-${{ matrix.step_name }}-reports
@@ -231,6 +235,8 @@ jobs:
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-bookkeeper-all-
- name: Set up JDK 17
uses: actions/setup-java@v4
@@ -251,7 +257,7 @@ jobs:
$GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror
- name: Build with Maven
- run: mvn -B -nsu clean install -Pdocker -DskipTests
+ run: mvn -B -nsu clean install -Pdocker,currentVersionOnly -DskipTests
- name: Run metadata driver tests
# Exclude jetcd-core-shaded from integration tests, as it’s a POM-only
project used internally,
@@ -290,7 +296,7 @@ jobs:
- name: Upload Surefire reports
uses: actions/upload-artifact@v4
- if: failure()
+ if: ${{ !success() }}
continue-on-error: true
with:
name: integration-tests-reports
@@ -326,6 +332,8 @@ jobs:
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-bookkeeper-all-
- name: Set up JDK 17
uses: actions/setup-java@v4
@@ -333,6 +341,14 @@ jobs:
distribution: 'temurin'
java-version: 17
+ - name: Restore released versions cache
+ uses: actions/cache/restore@v4
+ with:
+ path:
tests/docker-images/all-released-versions-image/.released-versions
+ key: released-versions-${{
hashFiles('tests/docker-images/all-released-versions-image/scripts/download-released-versions.sh')
}}
+ restore-keys: |
+ released-versions-
+
- name: Pick ubuntu mirror for the docker image build
run: |
# pick the closest ubuntu mirror and set it to UBUNTU_MIRROR
environment variable
@@ -351,6 +367,10 @@ jobs:
run: |
mvn -B -nsu -DbackwardCompatTests -DfailIfNoTests -pl
:bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct
test
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: ./dev/ci-tool print_thread_dumps
+
- name: Upload container logs on failure
uses: actions/upload-artifact@v4
if: ${{ !success() }}
@@ -375,7 +395,7 @@ jobs:
- name: Upload Surefire reports
uses: actions/upload-artifact@v4
- if: failure()
+ if: ${{ !success() }}
continue-on-error: true
with:
name: backward-compatibility-tests-reports
@@ -408,6 +428,8 @@ jobs:
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-bookkeeper-all-
- name: Set up JDK 17
uses: actions/setup-java@v4
@@ -440,6 +462,8 @@ jobs:
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-bookkeeper-all-
- name: Set up JDK 17
uses: actions/setup-java@v4
@@ -481,6 +505,8 @@ jobs:
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-bookkeeper-all-
- name: Set up JDK ${{ matrix.jdk_version }}
uses: actions/setup-java@v4
diff --git a/.github/workflows/bk-streamstorage-python.yml
b/.github/workflows/bk-streamstorage-python.yml
index cb8f17c060..097b18a9b4 100644
--- a/.github/workflows/bk-streamstorage-python.yml
+++ b/.github/workflows/bk-streamstorage-python.yml
@@ -66,6 +66,8 @@ jobs:
!~/.m2/repository/org/apache/bookkeeper
!~/.m2/repository/org/apache/distributedlog
key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-bookkeeper-all-
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
diff --git a/.github/workflows/ci-all-released-versions-update.yaml
b/.github/workflows/ci-all-released-versions-update.yaml
new file mode 100644
index 0000000000..43c8755107
--- /dev/null
+++ b/.github/workflows/ci-all-released-versions-update.yaml
@@ -0,0 +1,68 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Keeps the GitHub Actions cache of released BookKeeper version tarballs
+# up-to-date so that backward-compatibility test containers don't need to
+# re-download everything on every run.
+#
+# The cache key is derived from the hash of the download script. When a new
+# version is added to the script the hash changes, the previous cache is
+# restored as a starting point (via restore-keys), and only the newly added
+# files are downloaded. The schedule trigger ensures the cache is accessed
+# regularly so that it never expires due to the GitHub Actions 7-day inactivity
+# eviction policy.
+#
+
+name: CI - Released Versions Cache Update
+on:
+ push:
+ branches:
+ - master
+ - branch-*
+ paths:
+ -
'tests/docker-images/all-released-versions-image/scripts/download-released-versions.sh'
+ - '.github/workflows/ci-all-released-versions-update.yaml'
+ schedule:
+ - cron: '35 */12 * * *'
+ workflow_dispatch:
+
+jobs:
+ update-released-versions-cache:
+ name: Update released versions cache
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+
+ steps:
+ - name: checkout
+ uses: actions/checkout@v4
+
+ - name: Cache released versions
+ id: cache
+ uses: actions/cache@v4
+ timeout-minutes: 10
+ with:
+ path:
tests/docker-images/all-released-versions-image/.released-versions
+ key: released-versions-${{
hashFiles('tests/docker-images/all-released-versions-image/scripts/download-released-versions.sh')
}}
+ restore-keys: |
+ released-versions-
+
+ - name: Download released versions
+ if: steps.cache.outputs.cache-hit != 'true'
+ run: |
+ bash
tests/docker-images/all-released-versions-image/scripts/download-released-versions.sh
\
+ tests/docker-images/all-released-versions-image/.released-versions
diff --git a/.github/workflows/java21-daily-build.yml
b/.github/workflows/java21-daily-build.yml
index fa599a32a3..331ad9b61c 100644
--- a/.github/workflows/java21-daily-build.yml
+++ b/.github/workflows/java21-daily-build.yml
@@ -27,6 +27,16 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
+ - name: Cache local Maven repository
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/bookkeeper
+ !~/.m2/repository/org/apache/distributedlog
+ key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-bookkeeper-all-
- name: Set up JDK 21
uses: actions/setup-java@v4
with:
@@ -35,12 +45,12 @@ jobs:
- name: Build with Maven
run: mvn -B clean install
- name: Aggregates all test reports to ./test-reports and
./surefire-reports directories If failure
- if: failure()
+ if: ${{ !success() }}
continue-on-error: true
uses: ./.github/actions/copy-test-reports
- name: Upload Surefire reports
uses: actions/upload-artifact@v4
- if: failure()
+ if: ${{ !success() }}
continue-on-error: true
with:
name: jdk21-tests-reports
diff --git a/.github/workflows/windows-daily-build.yml
b/.github/workflows/windows-daily-build.yml
index c367065e3e..513c28241b 100644
--- a/.github/workflows/windows-daily-build.yml
+++ b/.github/workflows/windows-daily-build.yml
@@ -27,6 +27,17 @@ jobs:
runs-on: windows-latest
steps:
- uses: actions/checkout@v4
+ - name: Cache local Maven repository
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/bookkeeper
+ !~/.m2/repository/org/apache/distributedlog
+ key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ Linux-bookkeeper-all-
+ ${{ runner.os }}-bookkeeper-all-
- name: Set up JDK 21
uses: actions/setup-java@v4
with:
@@ -35,12 +46,12 @@ jobs:
- name: Build with Maven
run: mvn -B clean install
- name: Aggregates all test reports to ./test-reports and
./surefire-reports directories If failure
- if: failure()
+ if: ${{ !success() }}
continue-on-error: true
uses: ./.github/actions/copy-test-reports
- name: Upload Surefire reports
uses: actions/upload-artifact@v4
- if: failure()
+ if: ${{ !success() }}
continue-on-error: true
with:
name: windows-tests-reports
diff --git a/.gitignore b/.gitignore
index 27b4fba873..42309e4d62 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,4 +47,7 @@ package-lock.json
build/
.gradle/
*.log
-*.dat
\ No newline at end of file
+*.dat
+
+# Downloaded released BookKeeper versions (cached by CI, not committed)
+.released-versions/
\ No newline at end of file
diff --git a/metadata-drivers/etcd/pom.xml b/metadata-drivers/etcd/pom.xml
index bb03d5b853..0179eaad96 100644
--- a/metadata-drivers/etcd/pom.xml
+++ b/metadata-drivers/etcd/pom.xml
@@ -57,30 +57,6 @@
<groupId>io.grpc</groupId>
<artifactId>grpc-netty-shaded</artifactId>
</dependency>
- <dependency>
- <groupId>org.arquillian.cube</groupId>
- <artifactId>arquillian-cube-docker</artifactId>
- <version>${arquillian-cube.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.github.docker-java</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.jboss.arquillian.junit</groupId>
- <artifactId>arquillian-junit-standalone</artifactId>
- <version>${arquillian-junit.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.github.docker-java</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
diff --git a/tests/README.md b/tests/README.md
index 53f5410464..13cbc3cacc 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -77,7 +77,8 @@ Remember to start the unix socket proxy for docker as
described in the previous
# remove possible remaining python client versions from previous builds
git clean -fdx -- stream/clients/python
# build the project and docker images
-mvn -B -nsu clean install -Pdocker -DskipTests
+# currentVersionOnly profile avoids downloading old Bookkeeper releases.
+mvn -B -nsu clean install -Pdocker,currentVersionOnly -DskipTests
docker images | grep apachebookkeeper
```
@@ -93,6 +94,9 @@ mvn -f tests/pom.xml test -DintegrationTests
-DredirectTestOutputToFile=false -D
### Running backward compatibility tests
+Building docker images with `-Pdocker` is required to build the docker images
for backward compatibility tests.
+The `currentVersionOnly` profile shouldn't be active.
+
```bash
# Test current server with old clients
mvn -DbackwardCompatTests -pl :backward-compat-current-server-old-clients test
-DredirectTestOutputToFile=false -DtestRetryCount=0
diff --git a/tests/docker-images/all-released-versions-image/Dockerfile
b/tests/docker-images/all-released-versions-image/Dockerfile
index 80721ac0b6..a7ae5392d9 100644
--- a/tests/docker-images/all-released-versions-image/Dockerfile
+++ b/tests/docker-images/all-released-versions-image/Dockerfile
@@ -17,6 +17,28 @@
# under the License.
#
+# Stage 1: download and install all released BookKeeper versions.
+# Pre-downloaded tarballs from .released-versions (populated and cached by CI)
+# are copied into the build; the download script fills any gaps, then all
+# versions are extracted and installed into /opt/bookkeeper/.
+FROM alpine:3 AS installer
+
+ARG CURRENT_VERSION_ONLY=false
+
+RUN apk add --no-cache bash curl gnupg ca-certificates coreutils
+
+RUN mkdir -p /opt/bookkeeper /etc/supervisord/conf.d
+
+# Copy pre-downloaded tarballs and key files from the .released-versions cache.
+# The download script will fetch anything that is still missing.
+COPY .released-versions/ /released-versions/
+ADD scripts/download-released-versions.sh /download-released-versions.sh
+ADD scripts/install-all-tarballs.sh /install-all-tarballs.sh
+ADD scripts/install-tarball.sh /install-tarball.sh
+
+RUN if [ "$CURRENT_VERSION_ONLY" != "true" ]; then bash
/download-released-versions.sh && bash /install-all-tarballs.sh; fi
+
+# Stage 2: runtime image.
FROM eclipse-temurin:17-jdk-jammy
MAINTAINER Apache BookKeeper <[email protected]>
@@ -39,29 +61,12 @@ RUN sed -i -e
"s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://arc
&& echo networkaddress.cache.ttl=1 >>
$JAVA_HOME/conf/security/java.security \
&& echo networkaddress.cache.negative.ttl=1 >>
$JAVA_HOME/conf/security/java.security
-RUN mkdir /tarballs
-WORKDIR /tarballs
-RUN wget -nv
https://archive.apache.org/dist/zookeeper/bookkeeper/bookkeeper-4.1.0/bookkeeper-server-4.1.0-bin.tar.gz{,.sha1,.md5,.asc}
-RUN wget -nv
https://archive.apache.org/dist/zookeeper/bookkeeper/bookkeeper-4.2.0/bookkeeper-server-4.2.0-bin.tar.gz{,.sha1,.md5,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.8.2/bookkeeper-server-4.8.2-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.9.2/bookkeeper-server-4.9.2-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.10.0/bookkeeper-server-4.10.0-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.11.1/bookkeeper-server-4.11.1-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.12.1/bookkeeper-server-4.12.1-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.13.0/bookkeeper-server-4.13.0-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.8/bookkeeper-server-4.14.8-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.15.5/bookkeeper-server-4.15.5-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.16.7/bookkeeper-server-4.16.7-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv
https://archive.apache.org/dist/bookkeeper/bookkeeper-4.17.2/bookkeeper-server-4.17.2-bin.tar.gz{,.sha512,.asc}
-
-RUN wget -nv https://dist.apache.org/repos/dist/release/bookkeeper/KEYS
-RUN wget -nv
http://svn.apache.org/repos/asf/zookeeper/bookkeeper/dist/KEYS?p=1620552 -O
KEYS.old
+COPY --from=installer /opt/bookkeeper /opt/bookkeeper
RUN mkdir -p /etc/supervisord/conf.d && mkdir -p /var/run/supervisor && mkdir
-p /var/log/bookkeeper
ADD conf/supervisord.conf /etc/supervisord.conf
-ADD scripts/install-all-tarballs.sh /install-all-tarballs.sh
+COPY --from=installer /etc/supervisord/conf.d /etc/supervisord/conf.d
ADD scripts/install-tarball.sh /install-tarball.sh
-RUN bash /install-all-tarballs.sh && rm -rf /tarballs
WORKDIR /
ADD scripts/update-conf-and-boot.sh /update-conf-and-boot.sh
diff --git a/tests/docker-images/all-released-versions-image/pom.xml
b/tests/docker-images/all-released-versions-image/pom.xml
index f90e07adc2..8ea35172b2 100644
--- a/tests/docker-images/all-released-versions-image/pom.xml
+++ b/tests/docker-images/all-released-versions-image/pom.xml
@@ -25,7 +25,16 @@
<artifactId>all-released-versions-image</artifactId>
<name>Apache BookKeeper :: Tests :: Docker Images :: All Released
Versions</name>
<packaging>pom</packaging>
+ <properties>
+ <current.version.only>false</current.version.only>
+ </properties>
<profiles>
+ <profile>
+ <id>currentVersionOnly</id>
+ <properties>
+ <current.version.only>true</current.version.only>
+ </properties>
+ </profile>
<profile>
<id>docker</id>
<activation>
@@ -35,6 +44,29 @@
</activation>
<build>
<plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>download-released-versions</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>bash</executable>
+ <environmentVariables>
+
<CURRENT_VERSION_ONLY>${current.version.only}</CURRENT_VERSION_ONLY>
+ </environmentVariables>
+ <arguments>
+
<argument>${project.basedir}/scripts/download-released-versions.sh</argument>
+ <argument>${project.basedir}/.released-versions</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>com.spotify</groupId>
<artifactId>dockerfile-maven-plugin</artifactId>
@@ -64,6 +96,7 @@
<buildArgs>
<UBUNTU_MIRROR>${UBUNTU_MIRROR}</UBUNTU_MIRROR>
<UBUNTU_SECURITY_MIRROR>${UBUNTU_SECURITY_MIRROR}</UBUNTU_SECURITY_MIRROR>
+
<CURRENT_VERSION_ONLY>${current.version.only}</CURRENT_VERSION_ONLY>
</buildArgs>
</configuration>
</plugin>
diff --git
a/tests/docker-images/all-released-versions-image/scripts/download-released-versions.sh
b/tests/docker-images/all-released-versions-image/scripts/download-released-versions.sh
new file mode 100755
index 0000000000..10de1f7483
--- /dev/null
+++
b/tests/docker-images/all-released-versions-image/scripts/download-released-versions.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Downloads all released BookKeeper versions and GPG key files to DEST_DIR.
+# Skips files that already exist (idempotent).
+#
+# Usage: download-released-versions.sh [DEST_DIR]
+# DEST_DIR defaults to /released-versions
+
+set -e
+
+DEST_DIR="${1:-/released-versions}"
+mkdir -p "$DEST_DIR"
+
+if [ "${CURRENT_VERSION_ONLY}" = "true" ]; then
+ echo "Skipping downloading of old BookKeeper versions..."
+ exit 0
+fi
+
+if command -v curl &>/dev/null; then
+ # --retry 5 : retry up to 5 times on transient failures
+ # --retry-delay 5 : wait 5 s between retries
+ # --connect-timeout 30 : fail if the connection cannot be
established within 30 s
+ # --speed-limit 1 --speed-time 15 : abort (and retry) if the transfer is
idle for 15 s
+ _fetch() { curl -fsSL --retry 5 --retry-delay 5 --connect-timeout 30
--speed-limit 1 --speed-time 15 "$1" -o "$2"; }
+else
+ _fetch() { wget -nv "$1" -O "$2"; }
+fi
+
+download_if_missing() {
+ local url="$1"
+ local filename="$2"
+ local dest="$DEST_DIR/$filename"
+ if [ ! -f "$dest" ]; then
+ echo "Downloading $filename ..."
+ _fetch "$url" "$dest"
+ else
+ echo "Skipping $filename (already exists)"
+ fi
+}
+
+download_tarball() {
+ local base_url="$1"
+ local tarball="$2"
+ shift 2
+ # remaining args are checksum/signature extensions
+ download_if_missing "$base_url/$tarball" "$tarball"
+ for ext in "$@"; do
+ download_if_missing "$base_url/$tarball.$ext" "$tarball.$ext"
+ done
+}
+
+# 4.1.0 – 4.2.0 (old path under zookeeper/)
+download_tarball \
+ "https://archive.apache.org/dist/zookeeper/bookkeeper/bookkeeper-4.1.0"; \
+ "bookkeeper-server-4.1.0-bin.tar.gz" sha1 md5 asc
+download_tarball \
+ "https://archive.apache.org/dist/zookeeper/bookkeeper/bookkeeper-4.2.0"; \
+ "bookkeeper-server-4.2.0-bin.tar.gz" sha1 md5 asc
+
+# 4.8.2 and later (canonical bookkeeper/ path)
+download_tarball \
+ "https://archive.apache.org/dist/bookkeeper/bookkeeper-4.8.2"; \
+ "bookkeeper-server-4.8.2-bin.tar.gz" sha512 asc
+download_tarball \
+ "https://archive.apache.org/dist/bookkeeper/bookkeeper-4.9.2"; \
+ "bookkeeper-server-4.9.2-bin.tar.gz" sha512 asc
+download_tarball \
+ "https://archive.apache.org/dist/bookkeeper/bookkeeper-4.10.0"; \
+ "bookkeeper-server-4.10.0-bin.tar.gz" sha512 asc
+download_tarball \
+ "https://archive.apache.org/dist/bookkeeper/bookkeeper-4.11.1"; \
+ "bookkeeper-server-4.11.1-bin.tar.gz" sha512 asc
+download_tarball \
+ "https://archive.apache.org/dist/bookkeeper/bookkeeper-4.12.1"; \
+ "bookkeeper-server-4.12.1-bin.tar.gz" sha512 asc
+download_tarball \
+ "https://archive.apache.org/dist/bookkeeper/bookkeeper-4.13.0"; \
+ "bookkeeper-server-4.13.0-bin.tar.gz" sha512 asc
+download_tarball \
+ "https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.8"; \
+ "bookkeeper-server-4.14.8-bin.tar.gz" sha512 asc
+download_tarball \
+ "https://archive.apache.org/dist/bookkeeper/bookkeeper-4.15.5"; \
+ "bookkeeper-server-4.15.5-bin.tar.gz" sha512 asc
+download_tarball \
+ "https://archive.apache.org/dist/bookkeeper/bookkeeper-4.16.7"; \
+ "bookkeeper-server-4.16.7-bin.tar.gz" sha512 asc
+download_tarball \
+ "https://archive.apache.org/dist/bookkeeper/bookkeeper-4.17.2"; \
+ "bookkeeper-server-4.17.2-bin.tar.gz" sha512 asc
+
+# GPG key files
+download_if_missing \
+ "https://dist.apache.org/repos/dist/release/bookkeeper/KEYS"; \
+ "KEYS"
+download_if_missing \
+ "http://svn.apache.org/repos/asf/zookeeper/bookkeeper/dist/KEYS?p=1620552";
\
+ "KEYS.old"
diff --git
a/tests/docker-images/all-released-versions-image/scripts/install-all-tarballs.sh
b/tests/docker-images/all-released-versions-image/scripts/install-all-tarballs.sh
index fca7bbc67c..4272b1b7bc 100755
---
a/tests/docker-images/all-released-versions-image/scripts/install-all-tarballs.sh
+++
b/tests/docker-images/all-released-versions-image/scripts/install-all-tarballs.sh
@@ -20,12 +20,16 @@
set -e
-gpg --import KEYS
-gpg --import KEYS.old
+gpg --allow-weak-key-signatures --import /released-versions/KEYS
+gpg --allow-weak-key-signatures --import /released-versions/KEYS.old
+
+# Keys not present in either KEYS file, fetched from keyserver.
+# 8C75C738C33372AE198FD10CC238A8CAAC055FD2 — used to sign
bookkeeper-server-4.9.2
+gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys \
+ 8C75C738C33372AE198FD10CC238A8CAAC055FD2
mkdir -p /opt/bookkeeper
-for T in bookkeeper-server-*-bin.tar.gz; do
- /install-tarball.sh $T
+for T in /released-versions/bookkeeper-server-*-bin.tar.gz; do
+ /install-tarball.sh "$T"
done
-
diff --git
a/tests/docker-images/all-released-versions-image/scripts/install-tarball.sh
b/tests/docker-images/all-released-versions-image/scripts/install-tarball.sh
index c9d28eb494..65b5985914 100755
--- a/tests/docker-images/all-released-versions-image/scripts/install-tarball.sh
+++ b/tests/docker-images/all-released-versions-image/scripts/install-tarball.sh
@@ -20,30 +20,41 @@
set -e
-TARBALL=$1
+TARBALL_PATH=$1
+TARBALL=$(basename "$TARBALL_PATH")
+TARBALL_DIR=$(dirname "$TARBALL_PATH")
-if [ ! -f "$TARBALL" ]; then
- echo "tar file '$TARBALL' doesn't exist. exiting."
+if [ ! -f "$TARBALL_PATH" ]; then
+ echo "tar file '$TARBALL_PATH' doesn't exist. exiting."
exit 0
fi
+# Change to the directory containing the tarball so that checksum files
+# (which reference the tarball by bare filename) resolve correctly.
+cd "$TARBALL_DIR"
+
if [ -f $TARBALL.sha1 ]; then
sha1sum --check $TARBALL.sha1 > /dev/null
fi
if [ -f $TARBALL.sha512 ]; then
sha512sum --check $TARBALL.sha512 > /dev/null
fi
-if [ -f $T.md5 ]; then
+if [ -f $TARBALL.md5 ]; then
md5sum --check $TARBALL.md5 > /dev/null
fi
-if [ -f $T.asc ]; then
+if [ -f $TARBALL.asc ]; then
gpg --verify $TARBALL.asc
fi
VERSION=$(echo $TARBALL | sed -nE
's!^bookkeeper-(dist-)?server-([^-]*(-SNAPSHOT)?)-bin.tar.gz$!\2!p')
-tar -zxf $TARBALL
+# Extract into a temporary directory to avoid polluting the /released-versions
volume.
+EXTRACT_DIR=$(mktemp -d)
+cd "$EXTRACT_DIR"
+tar -zxf "$TARBALL_PATH"
mv bookkeeper-server-$VERSION /opt/bookkeeper/$VERSION
+cd /
+rm -rf "$EXTRACT_DIR"
VERSION_BASE=$(echo $VERSION | sed 's/-SNAPSHOT//')
# if version isn't 4.18 or higher, use Java 8
@@ -61,4 +72,4 @@ stdout_logfile=/var/log/bookkeeper/stdout-$VERSION.log
directory=/opt/bookkeeper/$VERSION
command=/opt/bookkeeper/$VERSION/bin/bookkeeper bookie
$JAVA_ENV
-EOF
\ No newline at end of file
+EOF
diff --git a/tests/docker-images/all-versions-image/Dockerfile
b/tests/docker-images/all-versions-image/Dockerfile
index cff955245c..3d51c0c7ae 100644
--- a/tests/docker-images/all-versions-image/Dockerfile
+++ b/tests/docker-images/all-versions-image/Dockerfile
@@ -34,7 +34,7 @@ COPY --from=openjdk-8 /opt/java/openjdk /opt/java/openjdk-8
RUN echo networkaddress.cache.ttl=1 >>
/opt/java/openjdk-8/jre/lib/security/java.security \
&& echo networkaddress.cache.negative.ttl=1 >>
/opt/java/openjdk-8/jre/lib/security/java.security
-RUN /install-tarball.sh *.tar.gz && rm -rf /tarballs
+RUN /install-tarball.sh /tarballs/*.tar.gz && rm -rf /tarballs
WORKDIR /
CMD ["/update-conf-and-boot.sh"]
\ No newline at end of file
