johnjcasey commented on code in PR #26153:
URL: https://github.com/apache/beam/pull/26153#discussion_r1165661339
##########
sdks/java/io/kafka/build.gradle:
##########
@@ -125,48 +116,24 @@ kafkaVersions.each {kv ->
}
}
-kafkaVersions.each {kv ->
- task "kafkaVersion${kv.key}BatchIT"(type: Test) {
- group = "Verification"
- description = "Runs KafkaIO IT tests with Kafka clients API $kv.value"
- outputs.upToDateWhen { false }
- testClassesDirs = sourceSets.test.output.classesDirs
- classpath = configurations."kafkaVersion${kv.key}" +
sourceSets.test.runtimeClasspath
-
- def pipelineOptions = [
- '--sourceOptions={' +
- '"numRecords": "1000",' +
- '"keySizeBytes": "10",' +
- '"valueSizeBytes": "90"' +
- '}',
- "--readTimeout=120",
- "--kafkaTopic=beam",
- "--withTestcontainers=true",
- "--kafkaContainerVersion=5.5.2",
- ]
-
- systemProperty "beamTestPipelineOptions",
groovy.json.JsonOutput.toJson(pipelineOptions)
- include '**/KafkaIOIT.class'
-
- filter {
- excludeTestsMatching "*InStreaming"
- if (!(kv.key in sdfKafkaVersions)) {
- excludeTestsMatching "*DynamicPartitions" //admin client create
partitions does not exist in kafka 0.11.0.3 and kafka sdf does not appear to
work for kafka versions <2.0.1
- excludeTestsMatching "*SDFResumesCorrectly" //Kafka SDF does not work
for kafka versions <2.0.1
- excludeTestsMatching "*StopReadingFunction" //Kafka SDF does not work
for kafka versions <2.0.1
- excludeTestsMatching "*WatermarkUpdateWithSparseMessages" //Kafka SDF
does not work for kafka versions <2.0.1
- }
- }
- }
-}
-
+//Because this runs many integration jobs in parallel, each of which use a
+//container, it can fail locally due to performance limitations on a desktop.
+//To avoid this, use --max-workers=N, where N is less than half your CPUs.
+//4 is a good start for parallelism without overloading your computer.
Review Comment:
I could add artificial dependencies between some of these tasks, to try and
limit parallelism, but that would be a janky fix
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
