This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 369eb8adcd Harden Kafka integration tests for reliable single-JVM
nightly runs (#7533)
369eb8adcd is described below
commit 369eb8adcd7a4fd38a09a19bf1885157023dc6c4
Author: Matt Casters <[email protected]>
AuthorDate: Wed Jul 15 11:07:33 2026 +0200
Harden Kafka integration tests for reliable single-JVM nightly runs (#7533)
Empty BOOTSTRAP_SERVERS parameter defaults were wiped by activateParameters
after #7517/#7521, causing "No resolvable bootstrap urls" on Jenkins.
Set non-empty defaults, wait for a healthy broker, and pin the Kafka image.
---
.../integration-tests/integration-tests-kafka.yaml | 26 ++++++++++++++++++----
.../kafka/main-0001-kafka-test-basic.hwf | 4 ++--
.../kafka/main-0002-kafka-test-mapping.hwf | 4 ++--
.../kafka/main-HOP3504-kafka-test-error-path.hwf | 4 ++--
integration-tests/kafka/project-config.json | 4 ++++
integration-tests/scripts/run-tests.sh | 10 +++++++++
integration-tests/scripts/run-workflow.hwf | 5 +++++
7 files changed, 47 insertions(+), 10 deletions(-)
diff --git a/docker/integration-tests/integration-tests-kafka.yaml
b/docker/integration-tests/integration-tests-kafka.yaml
index 0273ec87cb..c0025f52b2 100644
--- a/docker/integration-tests/integration-tests-kafka.yaml
+++ b/docker/integration-tests/integration-tests-kafka.yaml
@@ -20,13 +20,21 @@ services:
extends:
file: integration-tests-base.yaml
service: integration_test
+ environment:
+ # Available to hop-run / nested workflows as a variable (see
run-tests.sh -p)
+ - BOOTSTRAP_SERVERS=kafka:9092
+ depends_on:
+ kafka:
+ condition: service_healthy
links:
- kafka
kafka:
- image: 'apache/kafka:latest'
- ports:
- - '9092'
+ # Pin to match lib kafka-clients.version; avoid :latest drift on nightly CI
+ image: 'apache/kafka:3.9.2'
+ hostname: kafka
+ # No host port publish: tests use the compose network (kafka:9092) only.
+ # Publishing 9092 on the host risks colliding with other services on ASF
agents.
environment:
KAFKA_NODE_ID: 1
KAFKA_PROCESS_ROLES: broker,controller
@@ -40,4 +48,14 @@ services:
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_NUM_PARTITIONS: 3
-
+ healthcheck:
+ # Wait until the broker accepts connections on the PLAINTEXT listener
+ test:
+ [
+ "CMD-SHELL",
+ "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server
localhost:9092 >/dev/null 2>&1 || exit 1",
+ ]
+ interval: 5s
+ timeout: 10s
+ retries: 20
+ start_period: 20s
diff --git a/integration-tests/kafka/main-0001-kafka-test-basic.hwf
b/integration-tests/kafka/main-0001-kafka-test-basic.hwf
index f85098a0e2..28e6cc4ab2 100644
--- a/integration-tests/kafka/main-0001-kafka-test-basic.hwf
+++ b/integration-tests/kafka/main-0001-kafka-test-basic.hwf
@@ -30,8 +30,8 @@ limitations under the License.
<parameters>
<parameter>
<name>BOOTSTRAP_SERVERS</name>
- <description/>
- <default_value/>
+ <description>Kafka bootstrap servers (docker IT network
hostname)</description>
+ <default_value>kafka:9092</default_value>
</parameter>
</parameters>
<actions>
diff --git a/integration-tests/kafka/main-0002-kafka-test-mapping.hwf
b/integration-tests/kafka/main-0002-kafka-test-mapping.hwf
index e9ba2175a6..4d9d8ba1df 100644
--- a/integration-tests/kafka/main-0002-kafka-test-mapping.hwf
+++ b/integration-tests/kafka/main-0002-kafka-test-mapping.hwf
@@ -30,8 +30,8 @@ limitations under the License.
<parameters>
<parameter>
<name>BOOTSTRAP_SERVERS</name>
- <description/>
- <default_value/>
+ <description>Kafka bootstrap servers (docker IT network
hostname)</description>
+ <default_value>kafka:9092</default_value>
</parameter>
</parameters>
<actions>
diff --git a/integration-tests/kafka/main-HOP3504-kafka-test-error-path.hwf
b/integration-tests/kafka/main-HOP3504-kafka-test-error-path.hwf
index 1bad78f56d..56375542b8 100644
--- a/integration-tests/kafka/main-HOP3504-kafka-test-error-path.hwf
+++ b/integration-tests/kafka/main-HOP3504-kafka-test-error-path.hwf
@@ -31,8 +31,8 @@ limitations under the License.
<parameters>
<parameter>
<name>BOOTSTRAP_SERVERS</name>
- <default_value/>
- <description/>
+ <default_value>kafka:9092</default_value>
+ <description>Kafka bootstrap servers (docker IT network
hostname)</description>
</parameter>
</parameters>
<actions>
diff --git a/integration-tests/kafka/project-config.json
b/integration-tests/kafka/project-config.json
index 6a91171e1c..4d0f13ac4e 100644
--- a/integration-tests/kafka/project-config.json
+++ b/integration-tests/kafka/project-config.json
@@ -8,6 +8,10 @@
"name" : "HOP_LICENSE_HEADER_FILE",
"value" : "${PROJECT_HOME}/../asf-header.txt",
"description" : "This will automatically serialize the ASF license
header into pipelines and workflows in the integration test projects"
+ }, {
+ "name" : "BOOTSTRAP_SERVERS",
+ "value" : "kafka:9092",
+ "description" : "Kafka bootstrap servers for the docker integration-test
network"
} ]
}
}
\ No newline at end of file
diff --git a/integration-tests/scripts/run-tests.sh
b/integration-tests/scripts/run-tests.sh
index ccbbb8147a..3d5fa333f6 100755
--- a/integration-tests/scripts/run-tests.sh
+++ b/integration-tests/scripts/run-tests.sh
@@ -71,6 +71,16 @@ chmod 777 "${CURRENT_DIR}"/../surefire-reports/ 2>/dev/null
|| true
if [ -z "${BOOTSTRAP_SERVERS}" ]; then
BOOTSTRAP_SERVERS=kafka:9092
fi
+# Best-effort diagnostics when running the kafka project (helps Jenkins triage)
+if [ "${PROJECT_NAME}" = "kafka" ] || [ "$(basename "${PROJECT_NAME}"
2>/dev/null)" = "kafka" ]; then
+ echo "Kafka IT: BOOTSTRAP_SERVERS=${BOOTSTRAP_SERVERS}"
+ if command -v getent >/dev/null 2>&1; then
+ echo "Kafka IT: getent hosts kafka => $(getent hosts kafka 2>&1 || true)"
+ fi
+ if command -v nc >/dev/null 2>&1; then
+ nc -z -w 2 kafka 9092 && echo "Kafka IT: kafka:9092 is reachable" || echo
"Kafka IT: kafka:9092 is NOT reachable yet"
+ fi
+fi
# Get database parameters
if [ -z "${POSTGRES_HOST}" ]; then
diff --git a/integration-tests/scripts/run-workflow.hwf
b/integration-tests/scripts/run-workflow.hwf
index 6d0cce8dc7..3f059a5c07 100644
--- a/integration-tests/scripts/run-workflow.hwf
+++ b/integration-tests/scripts/run-workflow.hwf
@@ -34,6 +34,11 @@ limitations under the License.
<default_value/>
<description>The filename of the workflow to execute</description>
</parameter>
+ <parameter>
+ <name>BOOTSTRAP_SERVERS</name>
+ <default_value>kafka:9092</default_value>
+ <description>Kafka bootstrap servers (docker IT network
hostname)</description>
+ </parameter>
</parameters>
<actions>
<action>
