This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2cd8ae8bb38 Fix compat-test Kafka setup: upgrade to 4.2.1 (KRaft)
after 3.9.2 removal from downloads.apache.org (#18985)
2cd8ae8bb38 is described below
commit 2cd8ae8bb386fa3596d85f56f079fab12141d00e
Author: Xiang Fu <[email protected]>
AuthorDate: Tue Jul 14 06:48:38 2026 +0200
Fix compat-test Kafka setup: upgrade to 4.2.1 (KRaft) after 3.9.2 removal
from downloads.apache.org (#18985)
Kafka 3.9.2 was dropped from the live Apache mirror (only current
releases are hosted there), and the archive.apache.org fallback is
bandwidth-throttled so every 300s-capped curl attempt times out,
failing the compatibility test jobs after ~20 minutes.
- Bump default Kafka to 4.2.1 (available on downloads.apache.org).
The live download URL is still tried first; on failure (404 once the
version rotates off) it falls back to the archive mirror, now logging
the HTTP status.
- Convert the single-node setup to KRaft (Kafka 4.x is KRaft-only):
combined broker+controller config, controller listener on 19093,
storage format before start. Note Kafka 4.x brokers require clients
>= 2.1 (KIP-896); both suite sides use the kafka30 plugin (3.9.x).
- Make the archive fallback stall-tolerant (abort below 100KB/s for
60s or after 30 min, single retry) so the worst case stays within
the job timeout, and clean up partial downloads on failure.
---
compatibility-verifier/compCheck.sh | 34 +++++++++++++++++++++++++---------
1 file changed, 25 insertions(+), 9 deletions(-)
diff --git a/compatibility-verifier/compCheck.sh
b/compatibility-verifier/compCheck.sh
index 91d92a39df3..7ab06270fb8 100755
--- a/compatibility-verifier/compCheck.sh
+++ b/compatibility-verifier/compCheck.sh
@@ -212,22 +212,36 @@ function setupKafkaBinary() {
rm -rf "${workingDir}/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}"
if [ ! -f "${KAFKA_ARCHIVE}" ]; then
echo "Downloading Kafka from ${KAFKA_DOWNLOAD_URL}"
- curl -fSL --retry 3 --retry-delay 5 --connect-timeout 30 --max-time 300
"${KAFKA_DOWNLOAD_URL}" -o "${KAFKA_ARCHIVE}"
1>${LOG_DIR}/kafka.download.${logCount}.log 2>&1
+ # downloads.apache.org only hosts current releases (older ones rotate to
archive.apache.org), so a
+ # 404 here is expected once ${KAFKA_VERSION} ages out; fall back to the
archive mirror on failure
+ httpCode=$(curl -fSL --retry 3 --retry-delay 5 --connect-timeout 30
--max-time 300 -w "%{http_code}" \
+ "${KAFKA_DOWNLOAD_URL}" -o "${KAFKA_ARCHIVE}"
2>>${LOG_DIR}/kafka.download.${logCount}.log)
if [ $? -ne 0 ]; then
- echo "Primary download failed, trying archive mirror:
${KAFKA_ARCHIVE_URL}"
+ echo "Primary download failed (HTTP ${httpCode}), trying archive
mirror: ${KAFKA_ARCHIVE_URL}"
rm -f "${KAFKA_ARCHIVE}"
- curl -fSL --retry 3 --retry-delay 5 --connect-timeout 30 --max-time
300 "${KAFKA_ARCHIVE_URL}" -o "${KAFKA_ARCHIVE}"
1>>${LOG_DIR}/kafka.download.${logCount}.log 2>&1 || exit 1
+ # archive.apache.org is bandwidth-throttled: abort if throughput stays
below 100KB/s for 60s (too
+ # slow to ever finish the ~130MB tarball) or after 30 min; a single
retry keeps the worst case
+ # around 1h, within the compat job timeout
+ curl -fSL --retry 1 --retry-delay 5 --connect-timeout 30 --speed-limit
102400 --speed-time 60 --max-time 1800 \
+ "${KAFKA_ARCHIVE_URL}" -o "${KAFKA_ARCHIVE}"
1>>${LOG_DIR}/kafka.download.${logCount}.log 2>&1 || { rm -f
"${KAFKA_ARCHIVE}"; exit 1; }
fi
fi
- tar -xzf "${KAFKA_ARCHIVE}" -C "${workingDir}"
1>>${LOG_DIR}/kafka.download.${logCount}.log 2>&1 || exit 1
+ tar -xzf "${KAFKA_ARCHIVE}" -C "${workingDir}"
1>>${LOG_DIR}/kafka.download.${logCount}.log 2>&1 || { rm -f
"${KAFKA_ARCHIVE}"; exit 1; }
mv "${workingDir}/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}"
"${KAFKA_HOME}"
fi
rm -rf "${KAFKA_DATA_DIR}"
mkdir -p "${KAFKA_DATA_DIR}"
+ # Kafka 4.x is KRaft-only: run a single node acting as both broker and
controller
cat <<EOF >"${KAFKA_CONFIG_FILE}"
-listeners=${KAFKA_LISTENER}
+process.roles=broker,controller
+node.id=1
[email protected]:${KAFKA_CONTROLLER_PORT}
+listeners=${KAFKA_LISTENER},CONTROLLER://127.0.0.1:${KAFKA_CONTROLLER_PORT}
advertised.listeners=${KAFKA_LISTENER}
+inter.broker.listener.name=PLAINTEXT
+controller.listener.names=CONTROLLER
+listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
num.io.threads=8
num.network.threads=3
socket.send.buffer.bytes=102400
@@ -239,9 +253,10 @@ num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
-zookeeper.connect=localhost:${ZK_PORT}/kafka
-zookeeper.connection.timeout.ms=6000
EOF
+ # The data dir is wiped above, so the KRaft storage must be re-formatted
before every start
+ "${KAFKA_HOME}/bin/kafka-storage.sh" format -t
"$("${KAFKA_HOME}/bin/kafka-storage.sh" random-uuid)" \
+ -c "${KAFKA_CONFIG_FILE}" 1>${LOG_DIR}/kafka.format.${logCount}.log 2>&1
|| exit 1
}
function startKafkaService() {
@@ -460,11 +475,12 @@ SERVER_NETTY_PORT=8098
SERVER_2_NETTY_PORT=9098
SERVER_GRPC_PORT=8090
SERVER_2_GRPC_PORT=9090
-KAFKA_VERSION="${KAFKA_VERSION:-3.9.2}"
+KAFKA_VERSION="${KAFKA_VERSION:-4.2.1}"
KAFKA_SCALA_VERSION="${KAFKA_SCALA_VERSION:-2.13}"
KAFKA_DOWNLOAD_URL="${KAFKA_DOWNLOAD_URL:-https://downloads.apache.org/kafka/${KAFKA_VERSION}/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz}";
KAFKA_ARCHIVE_URL="https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz";
KAFKA_PORT="${KAFKA_PORT:-19092}"
+ KAFKA_CONTROLLER_PORT="${KAFKA_CONTROLLER_PORT:-19093}"
KAFKA_LISTENER="PLAINTEXT://127.0.0.1:${KAFKA_PORT}"
KAFKA_HOME="${workingDir}/kafka"
KAFKA_ARCHIVE="${workingDir}/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz"
@@ -493,7 +509,7 @@ setupCompatTester
# check that the default ports are open
if ! checkPortAvailable ${SERVER_ADMIN_PORT} || ! checkPortAvailable
${SERVER_NETTY_PORT} || ! checkPortAvailable ${SERVER_GRPC_PORT} ||
! checkPortAvailable ${BROKER_QUERY_PORT} || ! checkPortAvailable
${CONTROLLER_PORT} || ! checkPortAvailable ${ZK_PORT} ||
- ! checkPortAvailable ${KAFKA_PORT} ||
+ ! checkPortAvailable ${KAFKA_PORT} || ! checkPortAvailable
${KAFKA_CONTROLLER_PORT} ||
{ [ -f "${SERVER_CONF_2}" ] && { ! checkPortAvailable ${SERVER_2_ADMIN_PORT}
|| ! checkPortAvailable ${SERVER_2_NETTY_PORT} || ! checkPortAvailable
${SERVER_2_GRPC_PORT}; } ; }; then
exit 1
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
