This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f824adcae1f Add GitHub Workflow Replacements for Jenkins
job_PerformanceTests_JDBC (#28602)
f824adcae1f is described below
commit f824adcae1fd4495c1478631ea0c2a40568e8061
Author: Aleksandr Dudko <[email protected]>
AuthorDate: Thu Sep 28 17:58:17 2023 +0400
Add GitHub Workflow Replacements for Jenkins job_PerformanceTests_JDBC
(#28602)
* Add GitHub Workflow Replacements for Jenkins job_PerformanceTests_JDBC
* Add fix for beam_PerformanceTests_BiqQueryIO_Write_Python_Batch and
beam_PerformanceTests_BiqQueryIO_Read_Python
---
...eam_PerformanceTests_BiqQueryIO_Read_Python.yml | 35 +++++------
...formanceTests_BiqQueryIO_Write_Python_Batch.yml | 33 +++++------
...d_Python.yml => beam_PerformanceTests_JDBC.yml} | 68 +++++++++++++---------
.../performance-tests-job-configs/JDBC.txt | 29 +++++++++
..._Read_Python.txt => biqQueryIO_Read_Python.txt} | 11 +---
...ython.txt => biqQueryIO_Write_Python_Batch.txt} | 11 +---
6 files changed, 106 insertions(+), 81 deletions(-)
diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml
b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml
index 224689ee908..58958de69c8 100644
--- a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml
+++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: Performance BigQueryIO Read Python
+name: PerformanceTests BiqQueryIO Read Python
on:
issue_comment:
@@ -47,6 +47,7 @@ env:
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+ beam_PerformanceTests_BiqQueryIO_Read_Python_test_arguments_1: ''
jobs:
beam_PerformanceTests_BiqQueryIO_Read_Python:
@@ -69,24 +70,20 @@ jobs:
comment_phrase: ${{ matrix.job_phrase }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- - name: Install Python
- uses: actions/setup-python@v4
+ - name: Setup environment
+ uses: ./.github/actions/setup-environment-action
with:
- python-version: '3.8'
- # This code block prepares configuration data for a task.
- - name: Prepare config
- id: set_config
- shell: bash
- run: |
- # Capture the current date and time in a specific format
- CURDATE=$(date '+%m%d%H%M%S' --utc)
- # Reads the configurations file and exclude lines starting with #
and then remove line breaks
- CURCONFIG=$(grep -v "^#.*"
./.github/workflows/performance-tests-job-configs/config_PerformanceTests_BiqQueryIO_Read_Python.txt
| tr '\n' ' ')
- # Appends the value of CURDATE to the original configuration
- CONFIGWITHDATE=$(echo
"${CURCONFIG/bqio_read_python_/bqio_read_python_$CURDATE}")
- # Assigns $CONFIGWITHDATE to a variable named prepared_config
- echo "prepared_config=$CONFIGWITHDATE" >> $GITHUB_OUTPUT
- - name: run BigQueryIO Read Python Performance Test
+ python-version: 3.8
+ - name: Prepare test arguments
+ uses: ./.github/actions/test-arguments-action
+ with:
+ test-type: load
+ test-language: python
+ argument-file-paths: |
+ ${{ github.workspace
}}/.github/workflows/performance-tests-job-configs/biqQueryIO_Read_Python.txt
+ arguments: |
+ --job_name=performance-tests-bqio-read-python-10gb$(date
'+%m%d%H%M%S' --utc)
+ - name: Run BigQueryIO Read Performance Test Python
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
@@ -94,4 +91,4 @@ jobs:
-PloadTest.mainClass=apache_beam.io.gcp.bigquery_read_perf_test \
-PpythonVersion=3.8 \
-Prunner=DataflowRunner \
- '-PloadTest.args=${{steps.set_config.outputs.prepared_config}}'
\ No newline at end of file
+
'-PloadTest.args=${{env.beam_PerformanceTests_BiqQueryIO_Read_Python_test_arguments_1}}'
\ No newline at end of file
diff --git
a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml
b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml
index 24cd1b25ca7..f0fcd20bd3b 100644
--- a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml
+++ b/.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: Performance BigQueryIO Write Python Batch
+name: PerformanceTests BiqQueryIO Write Python Batch
on:
issue_comment:
@@ -47,6 +47,7 @@ env:
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+ beam_PerformanceTests_BiqQueryIO_Write_Python_Batch_test_arguments_1: ''
jobs:
beam_PerformanceTests_BiqQueryIO_Write_Python_Batch:
@@ -69,23 +70,19 @@ jobs:
comment_phrase: ${{ matrix.job_phrase }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- - name: Install Python
- uses: actions/setup-python@v4
+ - name: Setup environment
+ uses: ./.github/actions/setup-environment-action
with:
- python-version: '3.8'
- # This code block prepares configuration data for a task.
- - name: Prepare config
- id: set_config
- shell: bash
- run: |
- # capture the current date and time in a specific format
- CURDATE=$(date '+%m%d%H%M%S' --utc)
- # reads the configurations file and exclude lines starting with #
and then remove line breaks
- CURCONFIG=$(grep -v "^#.*"
./.github/workflows/performance-tests-job-configs/config_PerformanceTests_BiqQueryIO_Write_Python.txt
| tr '\n' ' ')
- # appends the value of CURDATE to the original configuration
- CONFIGWITHDATE=$(echo
"${CURCONFIG/bqio_write_python_/bqio_write_python_$CURDATE}")
- # assigns $CONFIGWITHDATE to a variable named prepared_config
- echo "prepared_config=$CONFIGWITHDATE" >> $GITHUB_OUTPUT
+ python-version: 3.8
+ - name: Prepare test arguments
+ uses: ./.github/actions/test-arguments-action
+ with:
+ test-type: load
+ test-language: python
+ argument-file-paths: |
+ ${{ github.workspace
}}/.github/workflows/performance-tests-job-configs/biqQueryIO_Write_Python_Batch.txt
+ arguments: |
+ --job_name=performance-tests-bqio-write-python-batch-10gb$(date
'+%m%d%H%M%S' --utc)
- name: run BigQueryIO Write Batch Python Performance Test
uses: ./.github/actions/gradle-command-self-hosted-action
with:
@@ -94,4 +91,4 @@ jobs:
-PloadTest.mainClass=apache_beam.io.gcp.bigquery_write_perf_test \
-PpythonVersion=3.8 \
-Prunner=DataflowRunner \
- '-PloadTest.args=${{steps.set_config.outputs.prepared_config}}'
\ No newline at end of file
+
'-PloadTest.args=${{env.beam_PerformanceTests_BiqQueryIO_Write_Python_Batch_test_arguments_1}}'
\ No newline at end of file
diff --git a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml
b/.github/workflows/beam_PerformanceTests_JDBC.yml
similarity index 55%
copy from .github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml
copy to .github/workflows/beam_PerformanceTests_JDBC.yml
index 224689ee908..c5995480e9d 100644
--- a/.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml
+++ b/.github/workflows/beam_PerformanceTests_JDBC.yml
@@ -13,13 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: Performance BigQueryIO Read Python
+name: PerformanceTests JDBC
on:
issue_comment:
types: [created]
schedule:
- - cron: '0 2 * * *'
+ - cron: '30 1,13 * * *'
workflow_dispatch:
#Setting explicit permissions for the action to avoid the default permissions
which are `write-all` in case of pull_request_target event
@@ -47,20 +47,21 @@ env:
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+ beam_PerformanceTests_JDBC_test_arguments_1: ''
jobs:
- beam_PerformanceTests_BiqQueryIO_Read_Python:
+ beam_PerformanceTests_JDBC:
if: |
github.event_name == 'workflow_dispatch' ||
github.event_name == 'schedule' ||
- github.event.comment.body == 'Run BigQueryIO Read Performance Test
Python'
+ github.event.comment.body == 'Run Java JdbcIO Performance Test'
runs-on: [self-hosted, ubuntu-20.04, main]
- timeout-minutes: 240
+ timeout-minutes: 100
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
strategy:
matrix:
- job_name: ["beam_PerformanceTests_BiqQueryIO_Read_Python"]
- job_phrase: ["Run BigQueryIO Read Performance Test Python"]
+ job_name: ["beam_PerformanceTests_JDBC"]
+ job_phrase: ["Run Java JdbcIO Performance Test"]
steps:
- uses: actions/checkout@v3
- name: Setup repository
@@ -69,29 +70,40 @@ jobs:
comment_phrase: ${{ matrix.job_phrase }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- - name: Install Python
- uses: actions/setup-python@v4
+ - name: Authenticate on GCP
+ id: auth
+ uses: google-github-actions/auth@v1
with:
- python-version: '3.8'
- # This code block prepares configuration data for a task.
- - name: Prepare config
- id: set_config
- shell: bash
+ credentials_json: ${{ secrets.GCP_SA_KEY }}
+ project_id: ${{ secrets.GCP_PROJECT_ID }}
+ - name: Set k8s access
+ uses: ./.github/actions/setup-k8s-access
+ with:
+ cluster_name: io-datastores
+ k8s_namespace: ${{ matrix.job_name }}-${{ github.run_id }}
+ - name: Install Postgres for dev
+ id: postgres_for_dev
run: |
- # Capture the current date and time in a specific format
- CURDATE=$(date '+%m%d%H%M%S' --utc)
- # Reads the configurations file and exclude lines starting with #
and then remove line breaks
- CURCONFIG=$(grep -v "^#.*"
./.github/workflows/performance-tests-job-configs/config_PerformanceTests_BiqQueryIO_Read_Python.txt
| tr '\n' ' ')
- # Appends the value of CURDATE to the original configuration
- CONFIGWITHDATE=$(echo
"${CURCONFIG/bqio_read_python_/bqio_read_python_$CURDATE}")
- # Assigns $CONFIGWITHDATE to a variable named prepared_config
- echo "prepared_config=$CONFIGWITHDATE" >> $GITHUB_OUTPUT
- - name: run BigQueryIO Read Python Performance Test
+ kubectl apply -f ${{ github.workspace
}}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml
+ kubectl wait svc/postgres-for-dev
--for=jsonpath='{.status.loadBalancer.ingress[0].ip}' --timeout=120s
+ loadbalancer_ip=$(kubectl get svc postgres-for-dev -o
jsonpath='{.status.loadBalancer.ingress[0].ip}')
+ echo "pfd_ip=$loadbalancer_ip" >> $GITHUB_OUTPUT
+ - name: Prepare test arguments
+ uses: ./.github/actions/test-arguments-action
+ with:
+ test-type: performance
+ test-language: java
+ argument-file-paths: |
+ ${{ github.workspace
}}/.github/workflows/performance-tests-job-configs/JDBC.txt
+ arguments: |
+ --postgresServerName=${{ steps.postgres_for_dev.outputs.pfd_ip }}
+ - name: Run Java JdbcIO Performance Test
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :sdks:python:apache_beam:testing:load_tests:run
+ gradle-command: :sdks:java:io:jdbc:integrationTest
arguments: |
- -PloadTest.mainClass=apache_beam.io.gcp.bigquery_read_perf_test \
- -PpythonVersion=3.8 \
- -Prunner=DataflowRunner \
- '-PloadTest.args=${{steps.set_config.outputs.prepared_config}}'
\ No newline at end of file
+ --tests org.apache.beam.sdk.io.jdbc.JdbcIOIT \
+ --info \
+ -Dfilesystem=hdfs \
+ -DintegrationTestRunner=dataflow \
+
-DintegrationTestPipelineOptions='[${{env.beam_PerformanceTests_JDBC_test_arguments_1}}]'
\ No newline at end of file
diff --git a/.github/workflows/performance-tests-job-configs/JDBC.txt
b/.github/workflows/performance-tests-job-configs/JDBC.txt
new file mode 100644
index 00000000000..74fd85145c5
--- /dev/null
+++ b/.github/workflows/performance-tests-job-configs/JDBC.txt
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+--tempRoot=gs://temp-storage-for-perf-tests
+--runner=DataflowRunner
+--numberOfRecords=5000000
+--bigQueryDataset=beam_performance
+--bigQueryTable=jdbcioit_results
+--influxMeasurement=jdbcioit_results
+--postgresUsername=postgres
+--postgresPassword=uuinkks
+--postgresDatabaseName=postgres
+--postgresSsl=false
+--postgresPort=5432
+--autoscalingAlgorithm=NONE
+--numWorkers=5
\ No newline at end of file
diff --git
a/.github/workflows/performance-tests-job-configs/config_PerformanceTests_BiqQueryIO_Read_Python.txt
b/.github/workflows/performance-tests-job-configs/biqQueryIO_Read_Python.txt
similarity index 75%
rename from
.github/workflows/performance-tests-job-configs/config_PerformanceTests_BiqQueryIO_Read_Python.txt
rename to
.github/workflows/performance-tests-job-configs/biqQueryIO_Read_Python.txt
index facf8b91ddb..12428da8d09 100644
---
a/.github/workflows/performance-tests-job-configs/config_PerformanceTests_BiqQueryIO_Read_Python.txt
+++ b/.github/workflows/performance-tests-job-configs/biqQueryIO_Read_Python.txt
@@ -1,4 +1,3 @@
-###############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,10 +12,8 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
---job_name=performance-tests-bqio-read-python-10gb0917155348
---project=apache-beam-testing
+# limitations under the License.
+
--region=us-central1
--temp_location=gs://temp-storage-for-perf-tests/loadtests
--input_dataset=beam_performance
@@ -25,9 +22,7 @@
--metrics_dataset=beam_performance
--metrics_table=bqio_read_10GB_results
--influx_measurement=python_bqio_read_10GB_results
---influx_db_name=beam_test_metrics
---influx_hostname=http://10.128.0.96:8086
--input_options=''{\\"num_records\\":10485760,\\"key_size\\":1,\\"value_size\\":1024,\\"algorithm\\":\\"lcg\\"}''
---num_****s=5
+--num_workers=5
--autoscaling_algorithm=NONE
--runner=DataflowRunner
\ No newline at end of file
diff --git
a/.github/workflows/performance-tests-job-configs/config_PerformanceTests_BiqQueryIO_Write_Python.txt
b/.github/workflows/performance-tests-job-configs/biqQueryIO_Write_Python_Batch.txt
similarity index 74%
rename from
.github/workflows/performance-tests-job-configs/config_PerformanceTests_BiqQueryIO_Write_Python.txt
rename to
.github/workflows/performance-tests-job-configs/biqQueryIO_Write_Python_Batch.txt
index b8c6bd9265f..9edec48d3d2 100644
---
a/.github/workflows/performance-tests-job-configs/config_PerformanceTests_BiqQueryIO_Write_Python.txt
+++
b/.github/workflows/performance-tests-job-configs/biqQueryIO_Write_Python_Batch.txt
@@ -1,4 +1,3 @@
-###############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,10 +12,8 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
---job_name=performance-tests-bqio-write-python-batch-10gb0917155348
---project=apache-beam-testing
+# limitations under the License.
+
--region=us-central1
--temp_location=gs://temp-storage-for-perf-tests/loadtests
--output_dataset=beam_performance
@@ -25,9 +22,7 @@
--metrics_dataset=beam_performance
--metrics_table=bqio_write_10GB_results
--influx_measurement=python_bqio_write_10GB_results
---influx_db_name=beam_test_metrics
---influx_hostname=http://10.128.0.96:8086
--input_options=''{\\"num_records\\":10485760,\\"key_size\\":1,\\"value_size\\":1024,\\"algorithm\\":\\"lcg\\"}''
---num_****s=5
+--num_workers=5
--autoscaling_algorithm=NONE
--runner=DataflowRunner
\ No newline at end of file
