This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 696a5a34666 Fix Go Load tests CoGBK and GBK (#34071)
696a5a34666 is described below
commit 696a5a34666e4ba99e81528d9b02417f08f1da21
Author: akashorabek <[email protected]>
AuthorDate: Wed Feb 26 19:48:57 2025 +0500
Fix Go Load tests CoGBK and GBK (#34071)
---
.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml | 4 +---
.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml | 6 +-----
.../go_CoGBK_Flink_Batch_MultipleKey.txt | 4 ++--
3 files changed, 4 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml
b/.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml
index 291ce23ef4f..949c3c64f5a 100644
--- a/.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml
+++ b/.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml
@@ -52,7 +52,7 @@ env:
GCS_BUCKET: gs://beam-flink-cluster
FLINK_DOWNLOAD_URL:
https://archive.apache.org/dist/flink/flink-1.17.0/flink-1.17.0-bin-scala_2.12.tgz
HADOOP_DOWNLOAD_URL:
https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
- FLINK_TASKMANAGER_SLOTS: 1
+ FLINK_TASKMANAGER_SLOTS: 5
DETACHED_MODE: true
HARNESS_IMAGES_TO_PULL:
gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
JOB_SERVER_IMAGE:
gcr.io/apache-beam-testing/beam_portability/beam_flink1.17_job_server:latest
@@ -95,8 +95,6 @@ jobs:
- name: Start Flink with parallelism 5
env:
FLINK_NUM_WORKERS: 5
- HIGH_MEM_MACHINE: n1-highmem-16
- HIGH_MEM_FLINK_PROPS:
flink:taskmanager.memory.process.size=16g,flink:taskmanager.memory.flink.size=6g,flink:taskmanager.memory.jvm-overhead.max=10g,flink:jobmanager.memory.process.size=16g,flink:jobmanager.memory.jvm-overhead.max=10g,flink:jobmanager.memory.flink.size=6g,flink:restart-strategy=fixed-delay,flink:restart-strategy.fixed-delay.attempts=3,flink:restart-strategy.fixed-delay.delay=10s
run: |
cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh
create
# The env variables are created and populated in the
test-arguments-action as
"<github.job>_test_arguments_<argument_file_paths_index>"
diff --git a/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml
b/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml
index 36f90de5c77..31ce48f3f51 100644
--- a/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml
+++ b/.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml
@@ -52,7 +52,7 @@ env:
GCS_BUCKET: gs://beam-flink-cluster
FLINK_DOWNLOAD_URL:
https://archive.apache.org/dist/flink/flink-1.17.0/flink-1.17.0-bin-scala_2.12.tgz
HADOOP_DOWNLOAD_URL:
https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
- FLINK_TASKMANAGER_SLOTS: 1
+ FLINK_TASKMANAGER_SLOTS: 5
DETACHED_MODE: true
HARNESS_IMAGES_TO_PULL:
gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
JOB_SERVER_IMAGE:
gcr.io/apache-beam-testing/beam_portability/beam_flink1.17_job_server:latest
@@ -98,8 +98,6 @@ jobs:
- name: Start Flink with parallelism 5
env:
FLINK_NUM_WORKERS: 5
- HIGH_MEM_MACHINE: n1-highmem-16
- HIGH_MEM_FLINK_PROPS:
flink:taskmanager.memory.process.size=16g,flink:taskmanager.memory.flink.size=8g,flink:taskmanager.memory.jvm-overhead.max=8g,flink:jobmanager.memory.process.size=16g,flink:jobmanager.memory.jvm-overhead.max=8g,flink:jobmanager.memory.flink.size=8g,flink:restart-strategy=fixed-delay,flink:restart-strategy.fixed-delay.attempts=3,flink:restart-strategy.fixed-delay.delay=10s
run: |
cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh
create
# The env variables are created and populated in the
test-arguments-action as
"<github.job>_test_arguments_<argument_file_paths_index>"
@@ -142,8 +140,6 @@ jobs:
- name: Restart Flink with parallelism 16
env:
FLINK_NUM_WORKERS: 16
- HIGH_MEM_MACHINE: n1-highmem-16
- HIGH_MEM_FLINK_PROPS:
flink:taskmanager.memory.process.size=16g,flink:taskmanager.memory.flink.size=12g,flink:taskmanager.memory.jvm-overhead.max=4g,flink:jobmanager.memory.process.size=6g,flink:jobmanager.memory.jvm-overhead.max=
2g,flink:jobmanager.memory.flink.size=4g
run: |
cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh
restart
- name: run GBK Flink Batch Go Load Test 4 (fanout 4)
diff --git
a/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_MultipleKey.txt
b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_MultipleKey.txt
index ec2c5c3e040..504b464eb1f 100644
---
a/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_MultipleKey.txt
+++
b/.github/workflows/load-tests-pipeline-options/go_CoGBK_Flink_Batch_MultipleKey.txt
@@ -16,8 +16,8 @@
--influx_measurement=go_batch_cogbk_1
--influx_namespace=flink
---input_options=''{\"num_records\":1000000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":5,\"hot_key_fraction\":1}''
---co_input_options=''{\"num_records\":100000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}''
+--input_options=''{\"num_records\":10000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":5,\"hot_key_fraction\":1}''
+--co_input_options=''{\"num_records\":1000,\"key_size\":10,\"value_size\":90,\"num_hot_keys\":1000,\"hot_key_fraction\":1}''
--iterations=1
--parallelism=5
--endpoint=localhost:8099
