This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 49dd2df027b Remove shuffle mode=appliance in tests (#24879)
49dd2df027b is described below
commit 49dd2df027b52a7484326d58e311c3c310aa9d6c
Author: Yi Hu <[email protected]>
AuthorDate: Thu Jan 5 20:34:52 2023 -0500
Remove shuffle mode=appliance in tests (#24879)
---
.test-infra/jenkins/job_LoadTests_Combine_Python.groovy | 3 +--
.test-infra/jenkins/job_LoadTests_GBK_Python.groovy | 3 +--
.test-infra/jenkins/job_LoadTests_GBK_Python_reiterate.groovy | 3 +--
.test-infra/jenkins/job_LoadTests_ParDo_Python.groovy | 3 +--
.test-infra/jenkins/job_LoadTests_SideInput_Python.groovy | 3 +--
.test-infra/jenkins/job_PerformanceTests_KafkaIO_IT.groovy | 3 +--
runners/google-cloud-dataflow-java/build.gradle | 7 +------
sdks/python/scripts/run_integration_test.sh | 2 --
8 files changed, 7 insertions(+), 20 deletions(-)
diff --git a/.test-infra/jenkins/job_LoadTests_Combine_Python.groovy
b/.test-infra/jenkins/job_LoadTests_Combine_Python.groovy
index 7c4f1ead0cf..99fd10e3bdd 100644
--- a/.test-infra/jenkins/job_LoadTests_Combine_Python.groovy
+++ b/.test-infra/jenkins/job_LoadTests_Combine_Python.groovy
@@ -100,8 +100,7 @@ def loadTestConfigurations = { datasetName, mode ->
def addStreamingOptions(test){
test.pipelineOptions << [streaming: null,
- // TODO(https://github.com/apache/beam/issues/20806) remove
shuffle_mode=appliance with runner v2 once issue is resolved.
- experiments: "use_runner_v2,shuffle_mode=appliance"
+ experiments: "use_runner_v2"
]
}
diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy
b/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy
index 9a38af439bd..1a772704ed7 100644
--- a/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy
+++ b/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy
@@ -156,8 +156,7 @@ def addStreamingOptions(test) {
// Use the new Dataflow runner, which offers improved efficiency of
Dataflow jobs.
// See
https://cloud.google.com/dataflow/docs/guides/deploying-a-pipeline#dataflow-runner-v2
// for more details.
- // TODO(https://github.com/apache/beam/issues/20806) remove
shuffle_mode=appliance with runner v2 once issue is resolved.
- experiments: 'use_runner_v2,shuffle_mode=appliance',
+ experiments: 'use_runner_v2',
]
}
diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Python_reiterate.groovy
b/.test-infra/jenkins/job_LoadTests_GBK_Python_reiterate.groovy
index 3fa262ab591..d1960abce17 100644
--- a/.test-infra/jenkins/job_LoadTests_GBK_Python_reiterate.groovy
+++ b/.test-infra/jenkins/job_LoadTests_GBK_Python_reiterate.groovy
@@ -86,8 +86,7 @@ def addStreamingOptions(test) {
// Use the new Dataflow runner, which offers improved efficiency of
Dataflow jobs.
// See
https://cloud.google.com/dataflow/docs/guides/deploying-a-pipeline#dataflow-runner-v2
// for more details.
- // TODO(https://github.com/apache/beam/issues/20806) remove
shuffle_mode=appliance with runner v2 once issue is resolved.
- experiments: 'use_runner_v2,shuffle_mode=appliance',
+ experiments: 'use_runner_v2',
]
}
diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Python.groovy
b/.test-infra/jenkins/job_LoadTests_ParDo_Python.groovy
index 44e9497dac9..090361a21a5 100644
--- a/.test-infra/jenkins/job_LoadTests_ParDo_Python.groovy
+++ b/.test-infra/jenkins/job_LoadTests_ParDo_Python.groovy
@@ -131,8 +131,7 @@ def addStreamingOptions(test) {
// Use the new Dataflow runner, which offers improved efficiency of
Dataflow jobs.
// See
https://cloud.google.com/dataflow/docs/guides/deploying-a-pipeline#dataflow-runner-v2
// for more details.
- // TODO(https://github.com/apache/beam/issues/20806) remove
shuffle_mode=appliance with runner v2 once issue is resolved.
- experiments: 'use_runner_v2,shuffle_mode=appliance',
+ experiments: 'use_runner_v2',
]
}
diff --git a/.test-infra/jenkins/job_LoadTests_SideInput_Python.groovy
b/.test-infra/jenkins/job_LoadTests_SideInput_Python.groovy
index 404d74c41ad..5ed7cc6381d 100644
--- a/.test-infra/jenkins/job_LoadTests_SideInput_Python.groovy
+++ b/.test-infra/jenkins/job_LoadTests_SideInput_Python.groovy
@@ -39,8 +39,7 @@ def fromTemplate = { mode, name, id, datasetName,
testSpecificOptions ->
influx_measurement : "python_${mode}_sideinput_${id}",
num_workers : 10,
autoscaling_algorithm: 'NONE',
- // TODO(https://github.com/apache/beam/issues/20806) remove
shuffle_mode=appliance with runner v2 once issue is resolved.
- experiments : 'use_runner_v2,shuffle_mode=appliance',
+ experiments : 'use_runner_v2',
] << testSpecificOptions
]
}
diff --git a/.test-infra/jenkins/job_PerformanceTests_KafkaIO_IT.groovy
b/.test-infra/jenkins/job_PerformanceTests_KafkaIO_IT.groovy
index 4403e8525bf..d513dd96a7e 100644
--- a/.test-infra/jenkins/job_PerformanceTests_KafkaIO_IT.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_KafkaIO_IT.groovy
@@ -106,8 +106,7 @@ job(jobName) {
readTimeout : '1500',
bigQueryTable : 'kafkaioit_results_runner_v2',
influxMeasurement : 'kafkaioit_results_runner_v2',
- // TODO(https://github.com/apache/beam/issues/20806) remove
shuffle_mode=appliance with runner v2 once issue is resolved.
- experiments :
'use_runner_v2,shuffle_mode=appliance,use_unified_worker',
+ experiments : 'use_runner_v2,use_unified_worker',
]
steps {
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index 4e1e7ebae24..4b7d11e99f5 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -429,8 +429,6 @@ createCrossLanguageValidatesRunnerTask(
"--project=${dataflowProject}",
"--region=${dataflowRegion}",
"--sdk_harness_container_image_overrides=.*java.*,${dockerJavaImageContainer}:${dockerTag}",
- // TODO(https://github.com/apache/beam/issues/20806) remove
shuffle_mode=appliance with runner v2 once issue is resolved
- "--experiments=shuffle_mode=appliance",
],
javaPipelineOptions: [
"--runner=TestDataflowRunner",
@@ -439,8 +437,6 @@ createCrossLanguageValidatesRunnerTask(
"--tempRoot=${dataflowValidatesTempRoot}",
"--sdkContainerImage=${dockerJavaImageContainer}:${dockerTag}",
"--sdkHarnessContainerImageOverrides=.*python.*,${dockerPythonImageContainer}:${dockerTag}",
- // TODO(https://github.com/apache/beam/issues/20806) remove
shuffle_mode=appliance with runner v2 once issue is resolved.
- "--experiments=shuffle_mode=appliance",
],
pytestOptions: [
"--capture=no",
@@ -463,8 +459,7 @@ task validatesRunnerV2 {
description = "Runs the ValidatesRunner tests on Dataflow Runner V2"
dependsOn(createRunnerV2ValidatesRunnerTest(
name: 'validatesRunnerV2Test',
- // TODO(https://github.com/apache/beam/issues/20806) remove
shuffle_mode=appliance with runner v2 once issue is resolved.
- pipelineOptions: runnerV2PipelineOptions +
['--experiments=shuffle_mode=appliance'],
+ pipelineOptions: runnerV2PipelineOptions,
excludedCategories: [
'org.apache.beam.sdk.testing.UsesOnWindowExpiration',
'org.apache.beam.sdk.testing.UsesStatefulParDo',
diff --git a/sdks/python/scripts/run_integration_test.sh
b/sdks/python/scripts/run_integration_test.sh
index d38f1bf3baa..508d9f50421 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -247,8 +247,6 @@ if [[ -z $PIPELINE_OPTS ]]; then
# Add --runner_v2 if provided
if [[ "$RUNNER_V2" = true ]]; then
opts+=("--experiments=use_runner_v2")
- # TODO(https://github.com/apache/beam/issues/20806) remove
shuffle_mode=appliance with runner v2 once issue is resolved.
- opts+=("--experiments=shuffle_mode=appliance")
if [[ "$STREAMING" = true ]]; then
# Dataflow Runner V2 only supports streaming engine.
opts+=("--enable_streaming_engine")
