This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 587fa177b69 Fix beam_LoadTests_Go_GBK_Dataflow_Batch job (#28960)
587fa177b69 is described below
commit 587fa177b69698db8c747bcdd739020f9e6f92ce
Author: Vitaly Terentyev <[email protected]>
AuthorDate: Tue Oct 17 23:39:36 2023 +0400
Fix beam_LoadTests_Go_GBK_Dataflow_Batch job (#28960)
* Fix prepare test arguments step
* Fix license headers
* Refactoring
---
.../beam_LoadTests_Go_GBK_Dataflow_Batch.yml | 43 ++++++++++++----------
...tch_100b.txt => go_GBK_Dataflow_Batch_100b.txt} | 9 +----
...h_100kb.txt => go_GBK_Dataflow_Batch_100kb.txt} | 9 +----
...Batch_10b.txt => go_GBK_Dataflow_Batch_10b.txt} | 9 +----
...ut_4.txt => go_GBK_Dataflow_Batch_Fanout_4.txt} | 9 +----
...ut_8.txt => go_GBK_Dataflow_Batch_Fanout_8.txt} | 9 +----
... => go_GBK_Dataflow_Batch_Reiteration_10KB.txt} | 9 +----
...t => go_GBK_Dataflow_Batch_Reiteration_2MB.txt} | 9 +----
8 files changed, 37 insertions(+), 69 deletions(-)
diff --git a/.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml
b/.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml
index 30f8ef076d7..9656c14ed99 100644
--- a/.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml
+++ b/.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml
@@ -67,19 +67,22 @@ jobs:
comment_phrase: ${{ matrix.job_phrase }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- - name: Prepare configs
- #Reads config files, excludes comments, appends current date to the
job_name parameter
- id: set_configs
- shell: bash
- run: |
- CURDATE=$(date '+%m%d%H%M%S' --utc)
- CONFIG_ARR=('config_GBK_Go_Batch_10b.txt'
'config_GBK_Go_Batch_100b.txt' 'config_GBK_Go_Batch_100b.txt'
'config_GBK_Go_Batch_Fanout_4.txt' 'config_GBK_Go_Batch_Fanout_8.txt'
'config_GBK_Go_Batch_Reiteration_10KB.txt',
'config_GBK_Go_Batch_Reiteration_2MB.txt')
- for INDEX in ${!CONFIG_ARR[@]}
- do
- CURCONFIG=$(grep -v "^#.*"
./.github/workflows/load-tests-job-configs/${CONFIG_ARR[INDEX]} | tr '\n' ' ')
- CURCONFIG=$(echo
"${CURCONFIG/load-tests-go-dataflow-batch-gbk-$((INDEX +
1))-/load-tests-go-dataflow-batch-gbk-$((INDEX + 1))-$CURDATE}")
- echo "prepared_config_$((INDEX + 1))=$CURCONFIG" >> $GITHUB_OUTPUT
- done
+ - name: Prepare test arguments
+ uses: ./.github/actions/test-arguments-action
+ with:
+ test-type: load
+ test-language: go
+ argument-file-paths: |
+ ${{ github.workspace
}}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_10b.txt
+ ${{ github.workspace
}}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100b.txt
+ ${{ github.workspace
}}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100kb.txt
+ ${{ github.workspace
}}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_4.txt
+ ${{ github.workspace
}}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_8.txt
+ ${{ github.workspace
}}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_10KB.txt
+ ${{ github.workspace
}}/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_2MB.txt
+ - name: get current time
+ run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV
+ # The env variables are created and populated in the
test-arguments-action as
"<github.job>_test_arguments_<argument_file_paths_index>"
- name: run GBK Dataflow Batch Go Load Test 1 (10 b records)
uses: ./.github/actions/gradle-command-self-hosted-action
with:
@@ -87,7 +90,7 @@ jobs:
arguments: |
-PloadTest.mainClass=group_by_key \
-Prunner=DataflowRunner \
- '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_1
}}' \
+ '-PloadTest.args=${{
env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_1 }}
--job_name=load-tests-go-dataflow-batch-gbk-1-${{env.NOW_UTC}}' \
- name: run GBK Dataflow Batch Go Load Test 2 (100 b records)
uses: ./.github/actions/gradle-command-self-hosted-action
with:
@@ -95,7 +98,7 @@ jobs:
arguments: |
-PloadTest.mainClass=group_by_key \
-Prunner=DataflowRunner \
- '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_2
}}' \
+ '-PloadTest.args=${{
env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_2 }}
--job_name=load-tests-go-dataflow-batch-gbk-2-${{env.NOW_UTC}}' \
- name: run GBK Dataflow Batch Go Load Test 3 (100 kb records)
uses: ./.github/actions/gradle-command-self-hosted-action
with:
@@ -103,7 +106,7 @@ jobs:
arguments: |
-PloadTest.mainClass=group_by_key \
-Prunner=DataflowRunner \
- '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_3
}}' \
+ '-PloadTest.args=${{
env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_3 }}
--job_name=load-tests-go-dataflow-batch-gbk-3-${{env.NOW_UTC}}' \
- name: run GBK Dataflow Batch Go Load Test 4 (fanout 4)
uses: ./.github/actions/gradle-command-self-hosted-action
with:
@@ -111,7 +114,7 @@ jobs:
arguments: |
-PloadTest.mainClass=group_by_key \
-Prunner=DataflowRunner \
- '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_4
}}' \
+ '-PloadTest.args=${{
env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_4 }}
--job_name=load-tests-go-dataflow-batch-gbk-4-${{env.NOW_UTC}}' \
- name: run GBK Dataflow Batch Go Load Test 5 (fanout 8)
uses: ./.github/actions/gradle-command-self-hosted-action
with:
@@ -119,7 +122,7 @@ jobs:
arguments: |
-PloadTest.mainClass=group_by_key \
-Prunner=DataflowRunner \
- '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_5
}}' \
+ '-PloadTest.args=${{
env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_5 }}
--job_name=load-tests-go-dataflow-batch-gbk-5-${{env.NOW_UTC}}' \
- name: run GBK Dataflow Batch Go Load Test 6 (reiterate 4 times 10 kb)
uses: ./.github/actions/gradle-command-self-hosted-action
with:
@@ -127,7 +130,7 @@ jobs:
arguments: |
-PloadTest.mainClass=group_by_key \
-Prunner=DataflowRunner \
- '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_6
}}' \
+ '-PloadTest.args=${{
env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_6 }}
--job_name=load-tests-go-dataflow-batch-gbk-6-${{env.NOW_UTC}}' \
- name: run GBK Dataflow Batch Go Load Test 7 (reiterate 4 times 2 mb)
uses: ./.github/actions/gradle-command-self-hosted-action
with:
@@ -135,4 +138,4 @@ jobs:
arguments: |
-PloadTest.mainClass=group_by_key \
-Prunner=DataflowRunner \
- '-PloadTest.args=${{ steps.set_configs.outputs.prepared_config_7
}}'
\ No newline at end of file
+ '-PloadTest.args=${{
env.beam_LoadTests_Go_GBK_Dataflow_Batch_test_arguments_7 }}
--job_name=load-tests-go-dataflow-batch-gbk-7-${{env.NOW_UTC}}'
\ No newline at end of file
diff --git
a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_100b.txt
b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100b.txt
similarity index 76%
rename from
.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_100b.txt
rename to
.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100b.txt
index f3ebed91b6a..b5266fa8304 100644
--- a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_100b.txt
+++ b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100b.txt
@@ -1,4 +1,3 @@
-###############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,10 +12,8 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
---job_name=load-tests-go-dataflow-batch-gbk-2-
---project=apache-beam-testing
+# limitations under the License.
+
--region=us-central1
--temp_location=gs://temp-storage-for-perf-tests/loadtests
--staging_location=gs://temp-storage-for-perf-tests/loadtests
@@ -29,6 +26,4 @@
--autoscaling_algorithm=NONE
--environment_type=DOCKER
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
---influx_db_name=beam_test_metrics
---influx_hostname=http://10.128.0.96:8086
--runner=DataflowRunner
\ No newline at end of file
diff --git
a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_100kb.txt
b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100kb.txt
similarity index 76%
rename from
.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_100kb.txt
rename to
.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100kb.txt
index e5007c7d5b9..072ab494515 100644
--- a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_100kb.txt
+++ b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_100kb.txt
@@ -1,4 +1,3 @@
-###############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,10 +12,8 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
---job_name=load-tests-go-dataflow-batch-gbk-3-
---project=apache-beam-testing
+# limitations under the License.
+
--region=us-central1
--temp_location=gs://temp-storage-for-perf-tests/loadtests
--staging_location=gs://temp-storage-for-perf-tests/loadtests
@@ -29,6 +26,4 @@
--autoscaling_algorithm=NONE
--environment_type=DOCKER
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
---influx_db_name=beam_test_metrics
---influx_hostname=http://10.128.0.96:8086
--runner=DataflowRunner
\ No newline at end of file
diff --git
a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_10b.txt
b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_10b.txt
similarity index 76%
rename from .github/workflows/load-tests-job-configs/config_GBK_Go_Batch_10b.txt
rename to .github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_10b.txt
index 7683eac5cb9..2e69dd0b457 100644
--- a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_10b.txt
+++ b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_10b.txt
@@ -1,4 +1,3 @@
-###############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,10 +12,8 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
---job_name=load-tests-go-dataflow-batch-gbk-1-
---project=apache-beam-testing
+# limitations under the License.
+
--region=us-central1
--temp_location=gs://temp-storage-for-perf-tests/loadtests
--staging_location=gs://temp-storage-for-perf-tests/loadtests
@@ -29,6 +26,4 @@
--autoscaling_algorithm=NONE
--environment_type=DOCKER
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
---influx_db_name=beam_test_metrics
---influx_hostname=http://10.128.0.96:8086
--runner=DataflowRunner
\ No newline at end of file
diff --git
a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Fanout_4.txt
b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_4.txt
similarity index 76%
rename from
.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Fanout_4.txt
rename to
.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_4.txt
index 5792b3bf0b9..6371123142d 100644
--- a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Fanout_4.txt
+++
b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_4.txt
@@ -1,4 +1,3 @@
-###############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,10 +12,8 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
---job_name=load-tests-go-dataflow-batch-gbk-4-
---project=apache-beam-testing
+# limitations under the License.
+
--region=us-central1
--temp_location=gs://temp-storage-for-perf-tests/loadtests
--staging_location=gs://temp-storage-for-perf-tests/loadtests
@@ -29,6 +26,4 @@
--autoscaling_algorithm=NONE
--environment_type=DOCKER
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
---influx_db_name=beam_test_metrics
---influx_hostname=http://10.128.0.96:8086
--runner=DataflowRunner
\ No newline at end of file
diff --git
a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Fanout_8.txt
b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_8.txt
similarity index 76%
rename from
.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Fanout_8.txt
rename to
.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_8.txt
index 369fb25aa0e..77d5f2e0162 100644
--- a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Fanout_8.txt
+++
b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Fanout_8.txt
@@ -1,4 +1,3 @@
-###############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,10 +12,8 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
---job_name=load-tests-go-dataflow-batch-gbk-5-
---project=apache-beam-testing
+# limitations under the License.
+
--region=us-central1
--temp_location=gs://temp-storage-for-perf-tests/loadtests
--staging_location=gs://temp-storage-for-perf-tests/loadtests
@@ -29,6 +26,4 @@
--autoscaling_algorithm=NONE
--environment_type=DOCKER
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
---influx_db_name=beam_test_metrics
---influx_hostname=http://10.128.0.96:8086
--runner=DataflowRunner
\ No newline at end of file
diff --git
a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Reiteration_10KB.txt
b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_10KB.txt
similarity index 77%
rename from
.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Reiteration_10KB.txt
rename to
.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_10KB.txt
index 9eb878d4e9f..7b27693ed8c 100644
---
a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Reiteration_10KB.txt
+++
b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_10KB.txt
@@ -1,4 +1,3 @@
-###############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,10 +12,8 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
---job_name=load-tests-go-dataflow-batch-gbk-6
---project=apache-beam-testing
+# limitations under the License.
+
--region=us-central1
--temp_location=gs://temp-storage-for-perf-tests/loadtests
--staging_location=gs://temp-storage-for-perf-tests/loadtests
@@ -29,6 +26,4 @@
--autoscaling_algorithm=NONE
--environment_type=DOCKER
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
---influx_db_name=beam_test_metrics
---influx_hostname=http://10.128.0.96:8086
--runner=DataflowRunner
\ No newline at end of file
diff --git
a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Reiteration_2MB.txt
b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_2MB.txt
similarity index 77%
rename from
.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Reiteration_2MB.txt
rename to
.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_2MB.txt
index aa26473ca43..40971e54645 100644
---
a/.github/workflows/load-tests-job-configs/config_GBK_Go_Batch_Reiteration_2MB.txt
+++
b/.github/workflows/load-tests-job-configs/go_GBK_Dataflow_Batch_Reiteration_2MB.txt
@@ -1,4 +1,3 @@
-###############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,10 +12,8 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
---job_name=load-tests-go-dataflow-batch-gbk-7
---project=apache-beam-testing
+# limitations under the License.
+
--region=us-central1
--temp_location=gs://temp-storage-for-perf-tests/loadtests
--staging_location=gs://temp-storage-for-perf-tests/loadtests
@@ -29,6 +26,4 @@
--autoscaling_algorithm=NONE
--environment_type=DOCKER
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
---influx_db_name=beam_test_metrics
---influx_hostname=http://10.128.0.96:8086
--runner=DataflowRunner
\ No newline at end of file
