johnjcasey commented on code in PR #26153:
URL: https://github.com/apache/beam/pull/26153#discussion_r1165579025
##########
sdks/java/io/kafka/build.gradle:
##########
@@ -125,48 +116,24 @@ kafkaVersions.each {kv ->
}
}
-kafkaVersions.each {kv ->
- task "kafkaVersion${kv.key}BatchIT"(type: Test) {
- group = "Verification"
- description = "Runs KafkaIO IT tests with Kafka clients API $kv.value"
- outputs.upToDateWhen { false }
- testClassesDirs = sourceSets.test.output.classesDirs
- classpath = configurations."kafkaVersion${kv.key}" +
sourceSets.test.runtimeClasspath
-
- def pipelineOptions = [
- '--sourceOptions={' +
- '"numRecords": "1000",' +
- '"keySizeBytes": "10",' +
- '"valueSizeBytes": "90"' +
- '}',
- "--readTimeout=120",
- "--kafkaTopic=beam",
- "--withTestcontainers=true",
- "--kafkaContainerVersion=5.5.2",
- ]
-
- systemProperty "beamTestPipelineOptions",
groovy.json.JsonOutput.toJson(pipelineOptions)
- include '**/KafkaIOIT.class'
-
- filter {
- excludeTestsMatching "*InStreaming"
- if (!(kv.key in sdfKafkaVersions)) {
- excludeTestsMatching "*DynamicPartitions" //admin client create
partitions does not exist in kafka 0.11.0.3 and kafka sdf does not appear to
work for kafka versions <2.0.1
- excludeTestsMatching "*SDFResumesCorrectly" //Kafka SDF does not work
for kafka versions <2.0.1
- excludeTestsMatching "*StopReadingFunction" //Kafka SDF does not work
for kafka versions <2.0.1
- excludeTestsMatching "*WatermarkUpdateWithSparseMessages" //Kafka SDF
does not work for kafka versions <2.0.1
- }
- }
- }
-}
-
+//Because this runs many integration jobs in parallel, each of which use a
+//container, it can fail locally due to performance limitations on a desktop.
+//To avoid this, use --max-workers=N, where N is less than half your CPUs.
+//4 is a good start for parallelism without overloading your computer.
Review Comment:
They are running fine on the Jenkins workers right now. I've tried to limit
parallelism to just one sub-project, but I can only manage to configure the
entire build to have a max number of workes
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
