This is an automated email from the ASF dual-hosted git repository.
chia7712 pushed a commit to branch 3.9
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/3.9 by this push:
new afdde056717 KAFKA-19876 Replace the base image since openjdk image is
deprecated (#20897)
afdde056717 is described below
commit afdde056717a6b658e7e251c34688856991a6336
Author: Ming-Yen Chung <[email protected]>
AuthorDate: Fri Nov 21 23:41:21 2025 +0800
KAFKA-19876 Replace the base image since openjdk image is deprecated
(#20897)
The `openjdk` image is deprecated. I replaced it with
`eclipse-temurin:8-jdk-jammy`. Trunk(#20886) uses `sapmachine`, but
since `sapmachine` does not provide JDK 8 and older Kafka versions still
require it. I switched to `eclipse-temurin` instead. Using `jammy` also
requires upgrading the `ducktape` and `psutil` versions. So I also
cherry-picked #17480, #17542, #17547 #17740, #19940, #20178 that upgrade
dependencies and update their usage.
I've tested a simple `pluggable_test` on my local mac with image
`eclipse-temurin:8-jdk-jammy`
```
...
docker build --memory=3200m --build-arg ducker_creator= --build-arg
jdk_version=eclipse-temurin:8-jdk-jammy --build-arg UID=501 --build-arg
KAFKA_MODE=jvm -t ducker-ak-eclipse-temurin-8-jdk-jammy -f
/Users/ming/code/kafka/tests/docker/Dockerfile -- .
...
================================================================================
SESSION REPORT (ALL TESTS)
ducktape version: 0.12.0
session_id: 2025-11-20--002
run time: 26.096 seconds
tests run: 2
passed: 2
flaky: 0
failed: 0
ignored: 0
================================================================================
test_id:
kafkatest.tests.client.pluggable_test.PluggableConsumerTest.test_start_stop.metadata_quorum=ISOLATED_KRAFT
status: PASS
run time: 13.961 seconds
--------------------------------------------------------------------------------
test_id:
kafkatest.tests.client.pluggable_test.PluggableConsumerTest.test_start_stop.metadata_quorum=ZK
status: PASS
run time: 11.888 seconds
--------------------------------------------------------------------------------
```
Reviewers: PoAn Yang <[email protected]>, Chia-Ping Tsai
<[email protected]>
---
docker/common.py | 5 ++---
docker/docker_build_test.py | 3 +--
docker/docker_official_image_build_test.py | 10 +++++-----
docker/prepare_docker_official_image_source.py | 11 ++++-------
tests/README.md | 2 +-
tests/docker/Dockerfile | 15 +++++++++------
tests/docker/ducker-ak | 5 +++--
tests/{unit/setup.cfg => docker/requirements.txt} | 18 ++++++++++--------
tests/setup.cfg | 2 +-
tests/setup.py | 12 ++++--------
tests/unit/setup.cfg | 2 +-
11 files changed, 41 insertions(+), 44 deletions(-)
diff --git a/docker/common.py b/docker/common.py
index aaab1eae1ff..a7cd3bfd203 100644
--- a/docker/common.py
+++ b/docker/common.py
@@ -18,7 +18,6 @@
import subprocess
import tempfile
import os
-from distutils.dir_util import copy_tree
import shutil
def execute(command):
@@ -34,8 +33,8 @@ def get_input(message):
def build_docker_image_runner(command, image_type):
temp_dir_path = tempfile.mkdtemp()
current_dir = os.path.dirname(os.path.realpath(__file__))
- copy_tree(f"{current_dir}/{image_type}", f"{temp_dir_path}/{image_type}")
- copy_tree(f"{current_dir}/resources",
f"{temp_dir_path}/{image_type}/resources")
+ shutil.copytree(f"{current_dir}/{image_type}",
f"{temp_dir_path}/{image_type}")
+ shutil.copytree(f"{current_dir}/resources",
f"{temp_dir_path}/{image_type}/resources")
command = command.replace("$DOCKER_FILE",
f"{temp_dir_path}/{image_type}/Dockerfile")
command = command.replace("$DOCKER_DIR", f"{temp_dir_path}/{image_type}")
try:
diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py
index 793148573f3..1c3499c865c 100755
--- a/docker/docker_build_test.py
+++ b/docker/docker_build_test.py
@@ -34,7 +34,6 @@ Usage:
from datetime import date
import argparse
-from distutils.dir_util import copy_tree
import shutil
from test.docker_sanity_test import run_tests
from common import execute, build_docker_image_runner
@@ -49,7 +48,7 @@ def run_docker_tests(image, tag, kafka_url, image_type):
temp_dir_path = tempfile.mkdtemp()
try:
current_dir = os.path.dirname(os.path.realpath(__file__))
- copy_tree(f"{current_dir}/test/fixtures", f"{temp_dir_path}/fixtures")
+ shutil.copytree(f"{current_dir}/test/fixtures",
f"{temp_dir_path}/fixtures", dirs_exist_ok=True)
execute(["wget", "-nv", "-O", f"{temp_dir_path}/kafka.tgz", kafka_url])
execute(["mkdir", f"{temp_dir_path}/fixtures/kafka"])
execute(["tar", "xfz", f"{temp_dir_path}/kafka.tgz", "-C",
f"{temp_dir_path}/fixtures/kafka", "--strip-components", "1"])
diff --git a/docker/docker_official_image_build_test.py
b/docker/docker_official_image_build_test.py
index 3da68854c23..32869a1f4b2 100644
--- a/docker/docker_official_image_build_test.py
+++ b/docker/docker_official_image_build_test.py
@@ -34,7 +34,6 @@ Usage:
"""
import argparse
-from distutils.dir_util import copy_tree
import shutil
from common import execute
from docker_build_test import run_docker_tests
@@ -46,10 +45,11 @@ def build_docker_official_image(image, tag, kafka_version,
image_type):
image = f'{image}:{tag}'
current_dir = os.path.dirname(os.path.realpath(__file__))
temp_dir_path = tempfile.mkdtemp()
-
copy_tree(f"{current_dir}/docker_official_images/{kafka_version}/{image_type}",
- f"{temp_dir_path}/{image_type}")
-
copy_tree(f"{current_dir}/docker_official_images/{kafka_version}/jvm/resources",
- f"{temp_dir_path}/{image_type}/resources")
+
shutil.copytree(f"{current_dir}/docker_official_images/{kafka_version}/{image_type}",
+ f"{temp_dir_path}/{image_type}", dirs_exist_ok=True)
+
shutil.copytree(f"{current_dir}/docker_official_images/{kafka_version}/jvm/resources",
+ f"{temp_dir_path}/{image_type}/resources", dirs_exist_ok=True)
+ shutil.copy(f"{current_dir}/server.properties",
f"{temp_dir_path}/{image_type}")
command = f"docker build -f $DOCKER_FILE -t {image} $DOCKER_DIR"
command = command.replace("$DOCKER_FILE",
f"{temp_dir_path}/{image_type}/Dockerfile")
command = command.replace("$DOCKER_DIR", f"{temp_dir_path}/{image_type}")
diff --git a/docker/prepare_docker_official_image_source.py
b/docker/prepare_docker_official_image_source.py
index 25d57c53e0f..bbc539b5c4c 100644
--- a/docker/prepare_docker_official_image_source.py
+++ b/docker/prepare_docker_official_image_source.py
@@ -33,7 +33,6 @@ Usage:
from datetime import date
import argparse
-from distutils.dir_util import copy_tree
import os
import shutil
import re
@@ -61,12 +60,10 @@ if __name__ == '__main__':
args = parser.parse_args()
kafka_url =
f"https://archive.apache.org/dist/kafka/{args.kafka_version}/kafka_2.13-{args.kafka_version}.tgz";
current_dir = os.path.dirname(os.path.realpath(__file__))
- new_dir = os.path.join(
- current_dir, f'docker_official_images', args.kafka_version)
+ new_dir = os.path.join(current_dir, 'docker_official_images',
args.kafka_version)
if os.path.exists(new_dir):
shutil.rmtree(new_dir)
os.makedirs(new_dir)
- copy_tree(os.path.join(current_dir, args.image_type),
os.path.join(new_dir, args.kafka_version, args.image_type))
- copy_tree(os.path.join(current_dir, 'resources'), os.path.join(new_dir,
args.kafka_version, args.image_type, 'resources'))
- remove_args_and_hardcode_values(
- os.path.join(new_dir, args.kafka_version, args.image_type,
'Dockerfile'), args.kafka_version, kafka_url)
+ shutil.copytree(os.path.join(current_dir, args.image_type),
os.path.join(new_dir, args.image_type), dirs_exist_ok=True)
+ shutil.copytree(os.path.join(current_dir, 'resources'),
os.path.join(new_dir, args.image_type, 'resources'), dirs_exist_ok=True)
+ remove_args_and_hardcode_values(os.path.join(new_dir, args.image_type,
'Dockerfile'), args.kafka_version, kafka_url)
diff --git a/tests/README.md b/tests/README.md
index e81aaecb140..b9b310a4e54 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -157,7 +157,7 @@
https://cwiki.apache.org/confluence/display/KAFKA/tutorial+-+set+up+and+run+Kafk
$ cd kafka/tests
$ virtualenv -p python3 venv
$ . ./venv/bin/activate
- $ python3 setup.py develop
+ $ python3 -m pip install --editable .
$ cd .. # back to base kafka directory
* Run the bootstrap script to set up Vagrant for testing
diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile
index be5a2363572..498cf62b242 100644
--- a/tests/docker/Dockerfile
+++ b/tests/docker/Dockerfile
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-ARG jdk_version=openjdk:8
+ARG jdk_version
FROM $jdk_version AS build-native-image
WORKDIR /build
@@ -60,11 +60,11 @@ ARG ducker_creator=default
LABEL ducker.creator=$ducker_creator
# Update Linux and install necessary utilities.
-# we have to install git since it is included in openjdk:8 but not openjdk:11
RUN apt update && apt install -y sudo git netcat iptables rsync unzip wget
curl jq coreutils openssh-server net-tools vim python3-pip python3-dev
libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute iproute2
iputils-ping && apt-get -y clean
RUN python3 -m pip install -U pip==21.1.1;
-# NOTE: ducktape 0.11.4 requires python3.9
-RUN pip3 install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm
ipaddress enum34 debugpy && pip3 install --upgrade "ducktape==0.11.4"
+# NOTE: ducktape 0.12.0 supports py 3.9, 3.10, 3.11 and 3.12
+COPY requirements.txt requirements.txt
+RUN pip3 install --upgrade -r requirements.txt
COPY --from=build-native-image /build/kafka-binary/ /opt/kafka-binary/
# Set up ssh
@@ -147,10 +147,13 @@ RUN useradd -u $UID -ms /bin/bash ducker \
&& mkdir -p /home/ducker/ \
&& rsync -aiq /root/.ssh/ /home/ducker/.ssh \
&& chown -R ducker /home/ducker/ /mnt/ /var/log/ \
- && echo "PATH=$(runuser -l ducker -c 'echo $PATH'):$JAVA_HOME/bin" >>
/home/ducker/.ssh/environment \
- && echo 'PATH=$PATH:'"$JAVA_HOME/bin" >> /home/ducker/.profile \
&& echo 'ducker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+# Symlink all JDK binaries (java, jcmd, jps, etc.) to /usr/bin.
+# We don't add PATH env to ~/.ssh/environment because on some Linux
distributions,
+# PAM (Pluggable Authentication Modules) may re-read /etc/environment,
overwriting PATH.
+RUN cp -sn $JAVA_HOME/bin/* /usr/bin/
+
USER ducker
CMD sudo service ssh start && tail -f /dev/null
diff --git a/tests/docker/ducker-ak b/tests/docker/ducker-ak
index 21f0f642194..dcff0bda912 100755
--- a/tests/docker/ducker-ak
+++ b/tests/docker/ducker-ak
@@ -44,8 +44,9 @@ docker_run_memory_limit="2000m"
# The default number of cluster nodes to bring up if a number is not specified.
default_num_nodes=14
-# The default OpenJDK base image.
-default_jdk="openjdk:8"
+# The default JDK base image with apt-get support.
+# The openjdk image has been officially deprecated. For more information, see:
https://hub.docker.com/_/openjdk
+default_jdk="eclipse-temurin:8-jdk-jammy"
# The default ducker-ak image name.
default_image_name="ducker-ak"
diff --git a/tests/unit/setup.cfg b/tests/docker/requirements.txt
similarity index 70%
copy from tests/unit/setup.cfg
copy to tests/docker/requirements.txt
index 3470da12185..91eaae441b4 100644
--- a/tests/unit/setup.cfg
+++ b/tests/docker/requirements.txt
@@ -13,11 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# pytest configuration (can also be defined in tox.ini or pytest.ini file)
-#
-# To ease possible confusion, prefix muckrake *unit* tests with 'check'
instead of 'test', since
-# many muckrake files, classes, and methods have 'test' somewhere in the name
-[pytest]
-python_files=check_*.py
-python_classes=Check
-python_functions=check_*
+cffi
+virtualenv
+pyasn1
+boto3
+pycrypto
+pywinrm
+ipaddress
+debugpy
+psutil
+ducktape==0.12.0
\ No newline at end of file
diff --git a/tests/setup.cfg b/tests/setup.cfg
index 974d5bb9a97..61e54430368 100644
--- a/tests/setup.cfg
+++ b/tests/setup.cfg
@@ -20,7 +20,7 @@
#
# To ease possible confusion, 'check' instead of 'test' as a prefix for unit
tests, since
# many system test files, classes, and methods have 'test' somewhere in the
name
-[pytest]
+[tool:pytest]
testpaths=unit
python_files=check_*.py
python_classes=Check
diff --git a/tests/setup.py b/tests/setup.py
index fd3b9738676..17476849d52 100644
--- a/tests/setup.py
+++ b/tests/setup.py
@@ -15,27 +15,24 @@
import re
import sys
-from setuptools import find_packages, setup
-from setuptools.command.test import test as TestCommand
+from setuptools import find_packages, setup, Command
version = ''
with open('kafkatest/__init__.py', 'r') as fd:
version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(),
re.MULTILINE).group(1)
-class PyTest(TestCommand):
+class PyTest(Command):
user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")]
def initialize_options(self):
- TestCommand.initialize_options(self)
self.pytest_args = []
def finalize_options(self):
- TestCommand.finalize_options(self)
self.test_args = []
self.test_suite = True
- def run_tests(self):
+ def run(self):
# import here, cause outside the eggs aren't loaded
import pytest
print(self.pytest_args)
@@ -51,8 +48,7 @@ setup(name="kafkatest",
license="apache2.0",
packages=find_packages(),
include_package_data=True,
- install_requires=["ducktape==0.11.4", "requests==2.31.0"],
- tests_require=["pytest", "mock"],
+ install_requires=["ducktape==0.12.0", "requests==2.32.4",
"psutil==5.7.2", "pytest==8.3.3", "mock==5.1.0"],
cmdclass={'test': PyTest},
zip_safe=False
)
diff --git a/tests/unit/setup.cfg b/tests/unit/setup.cfg
index 3470da12185..8de3c3de792 100644
--- a/tests/unit/setup.cfg
+++ b/tests/unit/setup.cfg
@@ -17,7 +17,7 @@
#
# To ease possible confusion, prefix muckrake *unit* tests with 'check'
instead of 'test', since
# many muckrake files, classes, and methods have 'test' somewhere in the name
-[pytest]
+[tool:pytest]
python_files=check_*.py
python_classes=Check
python_functions=check_*
