kamilwu commented on a change in pull request #13388:
URL: https://github.com/apache/beam/pull/13388#discussion_r528684823
##########
File path: .test-infra/jenkins/job_LoadTests_GBK_Flink_Go.groovy
##########
@@ -30,178 +30,196 @@ String now = new Date().format('MMddHHmmss',
TimeZone.getTimeZone('UTC'))
def batchScenarios = {
[
[
- title : 'Load test: 2GB of 10B records',
+ title : 'Group By Key Go Load test: 2GB of 10B records',
test : 'group_by_key',
runner : CommonTestProperties.Runner.FLINK,
+ influx_namespace: 'flink',
pipelineOptions: [
- job_name : "load-tests-go-flink-batch-gbk-1-${now}",
- influx_measurement : 'go_batch_gbk_1',
- input_options : """
+ job_name : "load-tests-go-flink-batch-gbk-1-${now}",
+ influx_measurement : 'go_batch_gbk_1',
+ input_options : """
{
"num_records": 200000000,
"key_size": 1,
"value_size": 9
}
- """.trim().replaceAll("\\s", ""),
- iterations : 1,
- fanout : 1,
- parallelism : 5,
- endpoint : 'localhost:8099',
- environment_type : 'DOCKER',
- environment_config :
"${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
+ """.trim().replaceAll("\\s", ""),
+ iterations : 1,
+ fanout : 1,
+ parallelism : 5,
+ endpoint : 'localhost:8099',
+ environment_type : 'DOCKER',
+ environment_config : "${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
]
],
[
- title : 'Load test: 2GB of 100B records',
+ title : 'Group By Key Go Load test: 2GB of 100B records',
test : 'group_by_key',
runner : CommonTestProperties.Runner.FLINK,
+ influx_namespace: 'flink',
pipelineOptions: [
- job_name : "load-tests-go-flink-batch-gbk-1-${now}",
- influx_measurement : 'go_batch_gbk_1',
- input_options : """
+ job_name : "load-tests-go-flink-batch-gbk-1-${now}",
+ influx_measurement : 'go_batch_gbk_1',
+ input_options : """
{
"num_records": 20000000,
"key_size": 10,
"value_size": 90
}
- """.trim().replaceAll("\\s", ""),
- iterations : 1,
- fanout : 1,
- parallelism : 5,
- endpoint : 'localhost:8099',
- environment_type : 'DOCKER',
- environment_config :
"${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
+ """.trim().replaceAll("\\s", ""),
+ iterations : 1,
+ fanout : 1,
+ parallelism : 5,
+ endpoint : 'localhost:8099',
+ environment_type : 'DOCKER',
+ environment_config : "${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
]
],
[
- title : 'Load test: 2GB of 100kB records',
+ title : 'Group By Key Go Load test: 2GB of 100kB records',
test : 'group_by_key',
runner : CommonTestProperties.Runner.FLINK,
+ influx_namespace: 'flink',
pipelineOptions: [
- job_name : "load-tests-go-flink-batch-gbk-1-${now}",
- influx_measurement : 'go_batch_gbk_1',
+ job_name : "load-tests-go-flink-batch-gbk-1-${now}",
+ influx_measurement : 'go_batch_gbk_1',
iterations : 1,
fanout : 1,
parallelism : 5,
- input_options : """
+ input_options : """
{
"num_records": 20000,
"key_size": 10000,
"value_size": 90000
}
- """.trim().replaceAll("\\s", ""),
- endpoint : 'localhost:8099',
- environment_type : 'DOCKER',
- environment_config :
"${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
+ """.trim().replaceAll("\\s", ""),
+ endpoint : 'localhost:8099',
+ environment_type : 'DOCKER',
+ environment_config :
"${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
]
],
[
- title : 'Load test: fanout 4 times with 2GB 10-byte records
total',
+ title : 'Group By Key Go Load test: fanout 4 times with 2GB
10-byte records total',
test : 'group_by_key',
runner : CommonTestProperties.Runner.FLINK,
+ influx_namespace: 'flink',
pipelineOptions: [
- job_name : "load-tests-go-flink-batch-gbk-1-${now}",
- influx_measurement : 'go_batch_gbk_1',
- iterations : 1,
- fanout : 4,
- parallelism : 16,
- input_options : """
+ job_name : "load-tests-go-flink-batch-gbk-1-${now}",
+ influx_measurement : 'go_batch_gbk_1',
+ iterations : 1,
+ fanout : 4,
+ parallelism : 16,
+ input_options : """
{
"num_records": 5000000,
"key_size": 10,
"value_size": 90
}
- """.trim().replaceAll("\\s", ""),
- endpoint : 'localhost:8099',
- environment_type : 'DOCKER',
- environment_config :
"${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
+ """.trim().replaceAll("\\s", ""),
+ endpoint : 'localhost:8099',
+ environment_type : 'DOCKER',
+ environment_config : "${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
]
],
[
- title : 'Load test: fanout 8 times with 2GB 10-byte records
total',
+ title : 'Group By Key Go Load test: fanout 8 times with 2GB
10-byte records total',
test : 'group_by_key',
runner : CommonTestProperties.Runner.FLINK,
+ influx_namespace: 'flink',
pipelineOptions: [
- job_name : "load-tests-go-flink-batch-gbk-1-${now}",
- influx_measurement : 'go_batch_gbk_1',
- iterations : 1,
- fanout : 8,
- parallelism : 16,
- input_options : """
+ job_name : "load-tests-go-flink-batch-gbk-1-${now}",
+ influx_measurement : 'go_batch_gbk_1',
+ iterations : 1,
+ fanout : 8,
+ parallelism : 16,
+ input_options : """
{
"num_records": 5000000,
"key_size": 10,
"value_size": 90
}
- """.trim().replaceAll("\\s", ""),
- endpoint : 'localhost:8099',
- environment_type : 'DOCKER',
- environment_config :
"${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
+ """.trim().replaceAll("\\s", ""),
+ endpoint : 'localhost:8099',
+ environment_type : 'DOCKER',
+ environment_config : "${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
]
],
[
- title : 'Load test: reiterate 4 times 10kB values',
+ title : 'Group By Key Go Load test: reiterate 4 times 10kB
values',
test : 'group_by_key',
runner : CommonTestProperties.Runner.FLINK,
+ influx_namespace: 'flink',
pipelineOptions: [
- job_name : "load-tests-go-flink-batch-gbk-1-${now}",
- influx_measurement : 'go_batch_gbk_1',
- iterations : 4,
- fanout : 1,
- parallelism : 5,
- input_options : """
+ job_name : "load-tests-go-flink-batch-gbk-1-${now}",
+ influx_measurement : 'go_batch_gbk_1',
+ iterations : 4,
+ fanout : 1,
+ parallelism : 5,
+ input_options : """
{
"num_records": 5000000,
"key_size": 10,
"value_size": 90,
"num_hot_keys": 200,
"hot_key_fraction": 1
}
- """.trim().replaceAll("\\s", ""),
- endpoint : 'localhost:8099',
- environment_type : 'DOCKER',
- environment_config :
"${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
+ """.trim().replaceAll("\\s", ""),
+ endpoint : 'localhost:8099',
+ environment_type : 'DOCKER',
+ environment_config : "${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
]
],
[
- title : 'Load test: reiterate 4 times 2MB values',
+ title : 'Group By Key Go Load test: reiterate 4 times 2MB
values',
test : 'group_by_key',
runner : CommonTestProperties.Runner.FLINK,
+ influx_namespace: 'flink',
pipelineOptions: [
- job_name : "load-tests-go-flink-batch-gbk-1-${now}",
- influx_measurement : 'go_batch_gbk_1',
- iterations : 4,
- fanout : 1,
- parallelism : 5,
- input_options : """
+ job_name : "load-tests-go-flink-batch-gbk-1-${now}",
+ influx_measurement : 'go_batch_gbk_1',
+ iterations : 4,
+ fanout : 1,
+ parallelism : 5,
+ input_options : """
{
"num_records": 20000000,
"key_size": 10,
"value_size": 90,
"num_hot_keys": 10,
"hot_key_fraction": 1
}
- """.trim().replaceAll("\\s", ""),
- endpoint : 'localhost:8099',
- environment_type : 'DOCKER',
- environment_config :
"${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
+ """.trim().replaceAll("\\s", ""),
+ endpoint : 'localhost:8099',
+ environment_type : 'DOCKER',
+ environment_config : "${DOCKER_CONTAINER_REGISTRY}/beam_go_sdk:latest",
]
],
].each { test -> test.pipelineOptions.putAll(additionalPipelineArgs) }
}
def loadTestJob = { scope, triggeringContext, mode ->
- def numberOfWorkers = 5
+ Map<Integer, List> testScenariosByParallelism = batchScenarios().groupBy {
test ->
+ test.pipelineOptions.parallelism
+ }
+ Integer initialParallelism =
testScenariosByParallelism.keySet().iterator().next()
+ List initialScenarios = testScenariosByParallelism.remove(initialParallelism)
- Flink flink = new Flink(scope,
"beam_LoadTests_Go_GBK_Flink_${mode.capitalize()}")
+ def flink = new Flink(scope,
"beam_LoadTests_Python_Combine_Flink_${mode.capitalize()}")
Review comment:
`beam_LoadTests_Python_Combine_Flink_` -> `beam_LoadTests_Go_GBK_Flink_`
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
