damccorm commented on code in PR #26153:
URL: https://github.com/apache/beam/pull/26153#discussion_r1165686005
##########
sdks/java/io/kafka/build.gradle:
##########
@@ -125,48 +116,24 @@ kafkaVersions.each {kv ->
}
}
-kafkaVersions.each {kv ->
- task "kafkaVersion${kv.key}BatchIT"(type: Test) {
- group = "Verification"
- description = "Runs KafkaIO IT tests with Kafka clients API $kv.value"
- outputs.upToDateWhen { false }
- testClassesDirs = sourceSets.test.output.classesDirs
- classpath = configurations."kafkaVersion${kv.key}" +
sourceSets.test.runtimeClasspath
-
- def pipelineOptions = [
- '--sourceOptions={' +
- '"numRecords": "1000",' +
- '"keySizeBytes": "10",' +
- '"valueSizeBytes": "90"' +
- '}',
- "--readTimeout=120",
- "--kafkaTopic=beam",
- "--withTestcontainers=true",
- "--kafkaContainerVersion=5.5.2",
- ]
-
- systemProperty "beamTestPipelineOptions",
groovy.json.JsonOutput.toJson(pipelineOptions)
- include '**/KafkaIOIT.class'
-
- filter {
- excludeTestsMatching "*InStreaming"
- if (!(kv.key in sdfKafkaVersions)) {
- excludeTestsMatching "*DynamicPartitions" //admin client create
partitions does not exist in kafka 0.11.0.3 and kafka sdf does not appear to
work for kafka versions <2.0.1
- excludeTestsMatching "*SDFResumesCorrectly" //Kafka SDF does not work
for kafka versions <2.0.1
- excludeTestsMatching "*StopReadingFunction" //Kafka SDF does not work
for kafka versions <2.0.1
- excludeTestsMatching "*WatermarkUpdateWithSparseMessages" //Kafka SDF
does not work for kafka versions <2.0.1
- }
- }
- }
-}
-
+//Because this runs many integration jobs in parallel, each of which use a
+//container, it can fail locally due to performance limitations on a desktop.
+//To avoid this, use --max-workers=N, where N is less than half your CPUs.
+//4 is a good start for parallelism without overloading your computer.
Review Comment:
Ok - I can live without it at first. If we see these tests failing (at a
higher than normal rate) with Jenkins infra-like issues, we should consider
clawing this back though
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
