This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 941e77d295a beam_PostCommit_Java_Tpcds (#28495)
941e77d295a is described below
commit 941e77d295a41ef55e01448bf46dcd4e35a97c4c
Author: magicgoody <[email protected]>
AuthorDate: Thu Sep 28 19:46:32 2023 +0600
beam_PostCommit_Java_Tpcds (#28495)
---
.github/workflows/README.md | 5 ++-
...yml => beam_PostCommit_Java_Tpcds_Dataflow.yml} | 49 +++++++++++++++++-----
...st.yml => beam_PostCommit_Java_Tpcds_Flink.yml} | 46 +++++++++++++++-----
...st.yml => beam_PostCommit_Java_Tpcds_Spark.yml} | 45 +++++++++++++++-----
.github/workflows/beam_PostCommit_Website_Test.yml | 2 +-
5 files changed, 112 insertions(+), 35 deletions(-)
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index 7a77c777471..f360bc15ae2 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -223,7 +223,10 @@ Please note that jobs with matrix need to have matrix
element in the comment. Ex
| [ PostCommit Java PVR Spark3 Streaming
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml)
| N/A |`Run Java Spark v3 PortableValidatesRunner Streaming`| [](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml)
|
| [ PostCommit Java PVR Spark Batch
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml)
| N/A |`Run Java Spark PortableValidatesRunner Batch`| [](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml)
|
| [ PostCommit Java Sickbay
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Sickbay.yml)
| N/A |`Run Java Sickbay`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Sickbay.yml)
|
-| [ PostCommit Java ValidatesRunner Dataflow JavaVersions
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Java.yml)
| ['11','17'] |`Run Dataflow ValidatesRunner Java (matrix_element)`|
[](https://github.com/apache/beam/act
[...]
+| [ PostCommit Java Tpcds Dataflow
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml)
| N/A |`Run Dataflow Runner Tpcds Tests`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml)
|
+| [ PostCommit Java Tpcds Flink
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml)
| N/A |`Run Flink Runner Tpcds Tests`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml)
|
+| [ PostCommit Java Tpcds Spark
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml)
| N/A |`Run Spark Runner Tpcds Tests`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml)
|
+| [ PostCommit Java ValidatesRunner Dataflow JavaVersions
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Java.yml)
| ['11','17'] |`Run Dataflow ValidatesRunner Java (matrix_element)`|
[](https://github.com/apache/beam/actions/workflows/b
[...]
| [ PostCommit Java ValidatesRunner Dataflow Streaming
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml)
| N/A |`Run Dataflow Streaming ValidatesRunner`|
[](https://github.com/apache/beam/actions/workflows/beam_Post
[...]
| [ PostCommit Java ValidatesRunner Dataflow V2 Streaming
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml)
| N/A |`Run Java Dataflow V2 ValidatesRunner Streaming`|
[](https://github.com/apache/beam/actions
[...]
| [ PostCommit Java ValidatesRunner Dataflow V2
](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml)
| N/A |`Run Java Dataflow V2 ValidatesRunner`|
[](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Da
[...]
diff --git a/.github/workflows/beam_PostCommit_Website_Test.yml
b/.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml
similarity index 60%
copy from .github/workflows/beam_PostCommit_Website_Test.yml
copy to .github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml
index 6155a45ef6a..6a458e17437 100644
--- a/.github/workflows/beam_PostCommit_Website_Test.yml
+++ b/.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: PostCommit Website Test
+name: PostCommit Java Tpcds Dataflow
on:
issue_comment:
@@ -47,30 +47,57 @@ env:
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
-
+ GRADLE_COMMAND_ARGUMENTS: |
+ --runner=DataflowRunner
+ --region=us-central1
+ --numWorkers=4
+ --maxNumWorkers=4
+ --autoscalingAlgorithm=NONE
+ --dataSize=1GB
+ --sourceType=PARQUET
+ --dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned
+ --resultsDirectory=gs://beam-tpcds/results/dataflow/
+ --tpcParallel=1
+ tpcdsBigQueryArgs: |
+ --bigQueryTable=tpcds
+ --bigQueryDataset=tpcds
+ --project=apache-beam-testing
+ --resourceNameMode=QUERY_RUNNER_AND_MODE
+ --exportSummaryToBigQuery=true
+ --tempLocation=gs://temp-storage-for-perf-tests/tpcds
+ tpcdsInfluxDBArgs: |
+ --influxDatabase=beam_test_metrics
+ --influxHost=http://10.128.0.96:8086
+ --baseInfluxMeasurement=tpcds
+ --exportSummaryToInfluxDB=true
+ --influxRetentionPolicy=forever
+ tpcdsQueriesArg: 3,7,10,25,26,29,35,38,40,42,43,52,55,69,79,83,84,87,93,96
+
jobs:
- beam_PostCommit_Website_Test:
+ beam_PostCommit_Java_Tpcds_Dataflow:
if: |
github.event_name == 'workflow_dispatch' ||
github.event_name == 'schedule' ||
- github.event.comment.body == 'Run Full Website Test'
+ github.event.comment.body == 'Run Dataflow Runner Tpcds Tests'
runs-on: [self-hosted, ubuntu-20.04, main]
- timeout-minutes: 30
+ timeout-minutes: 240
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
strategy:
matrix:
- job_name: ["beam_PostCommit_Website_Test"]
- job_phrase: ["Run Full Website Test"]
+ job_name: ["beam_PostCommit_Java_Tpcds_Dataflow"]
+ job_phrase: ["Run Dataflow Runner Tpcds Tests"]
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v3
- name: Setup repository
uses: ./.github/actions/setup-action
with:
comment_phrase: ${{ matrix.job_phrase }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- - name: run PostCommit Website Test script
+ - name: run PostCommit Java Tpcds Dataflow script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :website:testWebsite
- arguments: -PdisableExternal=false
\ No newline at end of file
+ gradle-command: :sdks:java:testing:tpcds:run
+ arguments: |
+ -Ptpcds.runner=:runners:google-cloud-dataflow-java \
+ "-Ptpcds.args=${{env.tpcdsBigQueryArgs}}
${{env.tpcdsInfluxDBArgs}} ${{ env.GRADLE_COMMAND_ARGUMENTS }}
--queries=${{env.tpcdsQueriesArg}}" \
diff --git a/.github/workflows/beam_PostCommit_Website_Test.yml
b/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml
similarity index 62%
copy from .github/workflows/beam_PostCommit_Website_Test.yml
copy to .github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml
index 6155a45ef6a..e062b108aef 100644
--- a/.github/workflows/beam_PostCommit_Website_Test.yml
+++ b/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: PostCommit Website Test
+name: PostCommit Java Tpcds Flink
on:
issue_comment:
@@ -47,30 +47,54 @@ env:
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
-
+ GRADLE_COMMAND_ARGUMENTS: |
+ --runner=FlinkRunner
+ --parallelism=4
+ --dataSize=1GB
+ --sourceType=PARQUET
+ --dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned
+ --resultsDirectory=gs://beam-tpcds/results/flink/
+ --tpcParallel=1
+ tpcdsBigQueryArgs: |
+ --bigQueryTable=tpcds
+ --bigQueryDataset=tpcds
+ --project=apache-beam-testing
+ --resourceNameMode=QUERY_RUNNER_AND_MODE
+ --exportSummaryToBigQuery=true
+ --tempLocation=gs://temp-storage-for-perf-tests/tpcds
+ tpcdsInfluxDBArgs: |
+ --influxDatabase=beam_test_metrics
+ --influxHost=http://10.128.0.96:8086
+ --baseInfluxMeasurement=tpcds
+ --exportSummaryToInfluxDB=true
+ --influxRetentionPolicy=forever
+ tpcdsQueriesArg: 3,7,10,25,26,29,35,38,40,42,43,52,55,69,79,83,84,87,93,96
+
jobs:
- beam_PostCommit_Website_Test:
+ beam_PostCommit_Java_Tpcds_Flink:
if: |
github.event_name == 'workflow_dispatch' ||
github.event_name == 'schedule' ||
- github.event.comment.body == 'Run Full Website Test'
+ github.event.comment.body == 'Run Flink Runner Tpcds Tests'
runs-on: [self-hosted, ubuntu-20.04, main]
- timeout-minutes: 30
+ timeout-minutes: 240
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
strategy:
matrix:
- job_name: ["beam_PostCommit_Website_Test"]
- job_phrase: ["Run Full Website Test"]
+ job_name: ["beam_PostCommit_Java_Tpcds_Flink"]
+ job_phrase: ["Run Flink Runner Tpcds Tests"]
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v3
- name: Setup repository
uses: ./.github/actions/setup-action
with:
comment_phrase: ${{ matrix.job_phrase }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- - name: run PostCommit Website Test script
+ - name: run PostCommit Java Tpcds Flink script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :website:testWebsite
- arguments: -PdisableExternal=false
\ No newline at end of file
+ gradle-command: :sdks:java:testing:tpcds:run
+ arguments: |
+ -Ptpcds.runner=:runners:flink:1.13 \
+ "-Ptpcds.args=${{env.tpcdsBigQueryArgs}}
${{env.tpcdsInfluxDBArgs}} ${{ env.GRADLE_COMMAND_ARGUMENTS }}
--queries=${{env.tpcdsQueriesArg}}" \
diff --git a/.github/workflows/beam_PostCommit_Website_Test.yml
b/.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml
similarity index 63%
copy from .github/workflows/beam_PostCommit_Website_Test.yml
copy to .github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml
index 6155a45ef6a..6dc6cbd5f1a 100644
--- a/.github/workflows/beam_PostCommit_Website_Test.yml
+++ b/.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: PostCommit Website Test
+name: PostCommit Java Tpcds Spark
on:
issue_comment:
@@ -47,30 +47,53 @@ env:
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
-
+ GRADLE_COMMAND_ARGUMENTS: |
+ --runner=SparkRunner
+ --dataSize=1GB
+ --sourceType=PARQUET
+ --dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned
+ --resultsDirectory=gs://beam-tpcds/results/spark3-rdd/
+ --tpcParallel=1
+ tpcdsBigQueryArgs: |
+ --bigQueryTable=tpcds
+ --bigQueryDataset=tpcds
+ --project=apache-beam-testing
+ --resourceNameMode=QUERY_RUNNER_AND_MODE
+ --exportSummaryToBigQuery=true
+ --tempLocation=gs://temp-storage-for-perf-tests/tpcds
+ tpcdsInfluxDBArgs: |
+ --influxDatabase=beam_test_metrics
+ --influxHost=http://10.128.0.96:8086
+ --baseInfluxMeasurement=tpcds
+ --exportSummaryToInfluxDB=true
+ --influxRetentionPolicy=forever
+ tpcdsQueriesArg: 3,7,10,25,26,29,35,38,40,42,43,52,55,69,79,83,84,87,93,96
+
jobs:
- beam_PostCommit_Website_Test:
+ beam_PostCommit_Java_Tpcds_Spark:
if: |
github.event_name == 'workflow_dispatch' ||
github.event_name == 'schedule' ||
- github.event.comment.body == 'Run Full Website Test'
+ github.event.comment.body == 'Run Spark Runner Tpcds Tests'
runs-on: [self-hosted, ubuntu-20.04, main]
- timeout-minutes: 30
+ timeout-minutes: 240
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
strategy:
matrix:
- job_name: ["beam_PostCommit_Website_Test"]
- job_phrase: ["Run Full Website Test"]
+ job_name: ["beam_PostCommit_Java_Tpcds_Spark"]
+ job_phrase: ["Run Spark Runner Tpcds Tests"]
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v3
- name: Setup repository
uses: ./.github/actions/setup-action
with:
comment_phrase: ${{ matrix.job_phrase }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- - name: run PostCommit Website Test script
+ - name: run PostCommit Java Tpcds Spark script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :website:testWebsite
- arguments: -PdisableExternal=false
\ No newline at end of file
+ gradle-command: :sdks:java:testing:tpcds:run
+ arguments: |
+ -Ptpcds.runner=:runners:spark:3 \
+ "-Ptpcds.args=${{env.tpcdsBigQueryArgs}}
${{env.tpcdsInfluxDBArgs}} ${{ env.GRADLE_COMMAND_ARGUMENTS }}
--queries=${{env.tpcdsQueriesArg}}" \
diff --git a/.github/workflows/beam_PostCommit_Website_Test.yml
b/.github/workflows/beam_PostCommit_Website_Test.yml
index 6155a45ef6a..b186e8ffcd8 100644
--- a/.github/workflows/beam_PostCommit_Website_Test.yml
+++ b/.github/workflows/beam_PostCommit_Website_Test.yml
@@ -55,7 +55,7 @@ jobs:
github.event_name == 'schedule' ||
github.event.comment.body == 'Run Full Website Test'
runs-on: [self-hosted, ubuntu-20.04, main]
- timeout-minutes: 30
+ timeout-minutes: 60
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
strategy:
matrix:
