This is an automated email from the ASF dual-hosted git repository.
frankvicky pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new b2c1a0fb9ff KAFKA-18841: Enable to test docker image locally (#19028)
b2c1a0fb9ff is described below
commit b2c1a0fb9ffbf19713362b7a5fba3d1a24e24338
Author: PoAn Yang <pay...@apache.org>
AuthorDate: Tue Aug 26 10:30:14 2025 +0800
KAFKA-18841: Enable to test docker image locally (#19028)
### Case 1: no --kafka-url and --kafka-archive
Should fail. One of argument (--kafka-url/--kafka-archive) is required.
```
> python docker_build_test.py apache/kafka --image-tag KAFKA-18841
--image-type jvm --build
usage: docker_build_test.py [-h] [--image-tag TAG] [--image-type
{jvm,native}] [--build] [--test] (--kafka-url KAFKA_URL |
--kafka-archive KAFKA_ARCHIVE) image
docker_build_test.py: error: one of the arguments --kafka-url/-u
--kafka-archive/-a is required
```
### Case 2: --kafka-url with native
```
> python docker_build_test.py apache/kafka --image-tag KAFKA-18841
--image-type native --kafka-url
https://dist.apache.org/repos/dist/dev/kafka/4.0.0-rc0/kafka_2.13-4.0.0.tgz
--build
```
### Case 3: --karka-url with jvm
```
> python docker_build_test.py apache/kafka --image-tag KAFKA-18841
--image-type jvm --kafka-url
https://dist.apache.org/repos/dist/dev/kafka/4.0.0-rc0/kafka_2.13-4.0.0.tgz
--build
```
### Case 4: --kafka-archive with native
```
> ./gradlew clean releaseTarGz
> cd docker
> python docker_build_test.py apache/kafka --image-tag KAFKA-18841
--image-type native --kafka-archive
</absolute/path/to/core/build/distributions/kafka_2.13-4.1.0-SNAPSHOT.tgz>
--build
```
### Case 5: --kafka-archive with jvm
```
> ./gradlew clean releaseTarGz
> cd docker
> python docker_build_test.py apache/kafka --image-tag KAFKA-18841
--image-type jvm --kafka-archive
</absolute/path/to/core/build/distributions/kafka_2.13-4.1.0-SNAPSHOT.tgz>
--build
```
Reviewers: Vedarth Sharma <vesha...@confluent.io>, Chia-Ping Tsai
<chia7...@gmail.com>, TengYao Chi <frankvi...@apache.org>
---------
Signed-off-by: PoAn Yang <pay...@apache.org>
---
docker/README.md | 4 ++++
docker/common.py | 4 +++-
docker/docker_build_test.py | 18 +++++++++-------
docker/jvm/Dockerfile | 52 +++++++++++++++++++++++++++++----------------
docker/native/Dockerfile | 19 ++++++++++-------
5 files changed, 62 insertions(+), 35 deletions(-)
diff --git a/docker/README.md b/docker/README.md
index 9c2916d9293..c4b9d49d0ea 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -130,6 +130,10 @@ python docker_build_test.py kafka/test --image-tag=3.6.0
--image-type=jvm --kafk
```
python docker_build_test.py kafka/test --image-tag=3.8.0 --image-type=native
--kafka-url=https://archive.apache.org/dist/kafka/3.8.0/kafka_2.13-3.8.0.tgz
```
+- Example(local build archive with jvm or native image type) :- To build and
test an image named test with local build archive
+```
+python docker_build_test.py kafka/test --image-tag=local-build
--image-type=<jvm/native>
--kafka-archive=</absolute/path/to/core/build/distributions/kafka_2.13-4.1.0-SNAPSHOT.tgz>
+```
Creating a Release Candidate
----------------------------
diff --git a/docker/common.py b/docker/common.py
index 9c0f901823f..f04a484a187 100644
--- a/docker/common.py
+++ b/docker/common.py
@@ -33,12 +33,14 @@ def get_input(message):
raise ValueError("This field cannot be empty")
return value
-def build_docker_image_runner(command, image_type):
+def build_docker_image_runner(command, image_type, kafka_archive=None):
temp_dir_path = tempfile.mkdtemp()
current_dir = os.path.dirname(os.path.realpath(__file__))
copy_tree(f"{current_dir}/{image_type}", f"{temp_dir_path}/{image_type}")
copy_tree(f"{current_dir}/resources",
f"{temp_dir_path}/{image_type}/resources")
copy_file(f"{current_dir}/server.properties",
f"{temp_dir_path}/{image_type}")
+ if kafka_archive:
+ copy_file(kafka_archive, f"{temp_dir_path}/{image_type}/kafka.tgz")
command = command.replace("$DOCKER_FILE",
f"{temp_dir_path}/{image_type}/Dockerfile")
command = command.replace("$DOCKER_DIR", f"{temp_dir_path}/{image_type}")
try:
diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py
index 793148573f3..fab6e65263d 100755
--- a/docker/docker_build_test.py
+++ b/docker/docker_build_test.py
@@ -25,9 +25,11 @@ Usage:
Example command:-
docker_build_test.py <image_name> --image-tag <image_tag> --image-type
<image_type> --kafka-url <kafka_url>
+ docker_build_test.py <image_name> --image-tag <image_tag> --image-type
<image_type> --kafka-archive <kafka_archive>
This command will build an image with <image_name> as image name,
<image_tag> as image_tag (it will be latest by default),
<image_type> as image type (jvm by default), <kafka_url> for the kafka
inside the image and run tests on the image.
+ <kafka_archive> can be passed as an alternative to <kafka_url> to use
a local kafka archive. The path of kafka_archive should be absolute.
-b can be passed as additional argument if you just want to build the
image.
-t can be passed if you just want to run tests on the image.
"""
@@ -41,10 +43,6 @@ from common import execute, build_docker_image_runner
import tempfile
import os
-def build_docker_image(image, tag, kafka_url, image_type):
- image = f'{image}:{tag}'
- build_docker_image_runner(f"docker build -f $DOCKER_FILE -t {image}
--build-arg kafka_url={kafka_url} --build-arg build_date={date.today()}
$DOCKER_DIR", image_type)
-
def run_docker_tests(image, tag, kafka_url, image_type):
temp_dir_path = tempfile.mkdtemp()
try:
@@ -69,16 +67,20 @@ if __name__ == '__main__':
parser.add_argument("image", help="Image name that you want to keep for
the Docker image")
parser.add_argument("--image-tag", "-tag", default="latest", dest="tag",
help="Image tag that you want to add to the image")
parser.add_argument("--image-type", "-type", choices=["jvm", "native"],
default="jvm", dest="image_type", help="Image type you want to build")
- parser.add_argument("--kafka-url", "-u", dest="kafka_url", help="Kafka url
to be used to download kafka binary tarball in the docker image")
parser.add_argument("--build", "-b", action="store_true",
dest="build_only", default=False, help="Only build the image, don't run tests")
parser.add_argument("--test", "-t", action="store_true", dest="test_only",
default=False, help="Only run the tests, don't build the image")
+
+ archive_group = parser.add_mutually_exclusive_group(required=True)
+ archive_group.add_argument("--kafka-url", "-u", dest="kafka_url",
help="Kafka url to be used to download kafka binary tarball in the docker
image")
+ archive_group.add_argument("--kafka-archive", "-a", dest="kafka_archive",
help="Kafka archive to be used to extract kafka binary tarball in the docker
image")
+
args = parser.parse_args()
if args.build_only or not (args.build_only or args.test_only):
if args.kafka_url:
- build_docker_image(args.image, args.tag, args.kafka_url,
args.image_type)
- else:
- raise ValueError("--kafka-url is a required argument for docker
image")
+ build_docker_image_runner(f"docker build -f $DOCKER_FILE -t
{args.image}:{args.tag} --build-arg kafka_url={args.kafka_url} --build-arg
build_date={date.today()} --no-cache --progress=plain $DOCKER_DIR",
args.image_type)
+ elif args.kafka_archive:
+ build_docker_image_runner(f"docker build -f $DOCKER_FILE -t
{args.image}:{args.tag} --build-arg build_date={date.today()} --no-cache
--progress=plain $DOCKER_DIR", args.image_type, args.kafka_archive)
if args.test_only or not (args.build_only or args.test_only):
run_docker_tests(args.image, args.tag, args.kafka_url, args.image_type)
diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile
index f98f50a2e03..3d2f06820d4 100644
--- a/docker/jvm/Dockerfile
+++ b/docker/jvm/Dockerfile
@@ -23,20 +23,27 @@ USER root
# Get kafka from https://archive.apache.org/dist/kafka and pass the url
through build arguments
ARG kafka_url
+ENV KAFKA_URL=$kafka_url
+
COPY jsa_launch /etc/kafka/docker/jsa_launch
COPY server.properties /etc/kafka/docker/server.properties
+COPY *kafka.tgz kafka.tgz
+
RUN set -eux ; \
apk update ; \
apk upgrade ; \
- apk add --no-cache wget gcompat gpg gpg-agent procps bash; \
+ apk add --no-cache bash; \
+ if [ -n "$KAFKA_URL" ]; then \
+ apk add --no-cache wget gcompat gpg gpg-agent procps; \
+ wget -nv -O kafka.tgz "$KAFKA_URL"; \
+ wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \
+ wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
+ gpg --import KEYS; \
+ gpg --batch --verify kafka.tgz.asc kafka.tgz; \
+ fi; \
mkdir opt/kafka; \
- wget -nv -O kafka.tgz "$kafka_url"; \
- wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \
- tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
- wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
- gpg --import KEYS; \
- gpg --batch --verify kafka.tgz.asc kafka.tgz
+ tar xfz kafka.tgz -C opt/kafka --strip-components 1;
# Generate jsa files using dynamic CDS for kafka server start command and
kafka storage format command
RUN /etc/kafka/docker/jsa_launch
@@ -53,6 +60,9 @@ USER root
ARG kafka_url
ARG build_date
+ENV KAFKA_URL=$kafka_url
+
+COPY *kafka.tgz kafka.tgz
LABEL org.label-schema.name="kafka" \
org.label-schema.description="Apache Kafka" \
@@ -60,17 +70,25 @@ LABEL org.label-schema.name="kafka" \
org.label-schema.vcs-url="https://github.com/apache/kafka"; \
maintainer="Apache Kafka"
-RUN set -eux ; \
+RUN mkdir opt/kafka; \
+ set -eux ; \
apk update ; \
apk upgrade ; \
- apk add --no-cache wget gcompat gpg gpg-agent procps bash; \
- mkdir opt/kafka; \
- wget -nv -O kafka.tgz "$kafka_url"; \
- wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \
- tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
- wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
- gpg --import KEYS; \
- gpg --batch --verify kafka.tgz.asc kafka.tgz; \
+ apk add --no-cache bash; \
+ if [ -n "$KAFKA_URL" ]; then \
+ apk add --no-cache wget gcompat gpg gpg-agent procps; \
+ wget -nv -O kafka.tgz "$KAFKA_URL"; \
+ wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \
+ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
+ wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
+ gpg --import KEYS; \
+ gpg --batch --verify kafka.tgz.asc kafka.tgz; \
+ rm kafka.tgz kafka.tgz.asc KEYS; \
+ apk del wget gpg gpg-agent; \
+ else \
+ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
+ rm kafka.tgz; \
+ fi; \
mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \
mkdir -p /etc/kafka/docker /usr/logs /mnt/shared/config; \
adduser -h /home/appuser -D --shell /bin/bash appuser; \
@@ -79,8 +97,6 @@ RUN set -eux ; \
chmod -R ug+w /etc/kafka /var/lib/kafka /etc/kafka/secrets; \
cp /opt/kafka/config/log4j2.yaml /etc/kafka/docker/log4j2.yaml; \
cp /opt/kafka/config/tools-log4j2.yaml
/etc/kafka/docker/tools-log4j2.yaml; \
- rm kafka.tgz kafka.tgz.asc KEYS; \
- apk del wget gpg gpg-agent; \
apk cache clean;
COPY server.properties /etc/kafka/docker/server.properties
diff --git a/docker/native/Dockerfile b/docker/native/Dockerfile
index ca85f35562d..010edbcd51c 100644
--- a/docker/native/Dockerfile
+++ b/docker/native/Dockerfile
@@ -29,15 +29,18 @@ ENV TARGET_PATH="$KAFKA_DIR/kafka.Kafka"
COPY native-image-configs $NATIVE_CONFIGS_DIR
COPY native_command.sh native_command.sh
-RUN mkdir $KAFKA_DIR; \
- microdnf install wget; \
- wget -nv -O kafka.tgz "$KAFKA_URL"; \
- wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \
+COPY *kafka.tgz /app
+
+RUN if [ -n "$KAFKA_URL" ]; then \
+ microdnf install wget; \
+ wget -nv -O kafka.tgz "$KAFKA_URL"; \
+ wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \
+ wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
+ gpg --import KEYS; \
+ gpg --batch --verify kafka.tgz.asc kafka.tgz; \
+ fi; \
+ mkdir $KAFKA_DIR; \
tar xfz kafka.tgz -C $KAFKA_DIR --strip-components 1; \
- wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
- gpg --import KEYS; \
- gpg --batch --verify kafka.tgz.asc kafka.tgz; \
- rm kafka.tgz ; \
# Build the native-binary of the apache kafka using graalVM native-image.
/app/native_command.sh $NATIVE_IMAGE_PATH $NATIVE_CONFIGS_DIR
$KAFKA_LIBS_DIR $TARGET_PATH
